当前位置: 首页 > 工具软件 > 增量超越 > 使用案例 >

增量实现

史骏祥
2023-12-01
###参数
source ExitCodeCheck.sh
queueName=$1;
hive_db_name_360=$2;
stat_date=$3;
end_date=$4;

#### 新理赔 新增 1
hive -v -e "use ${hive_db_name_360};
set mapred.job.queue.name=queue_gbd_ide_02;
set hive.cli.print.header=false;
set hive.map.aggr=true;
set hive.groupby.skewindata=true;


set hive.exec.parallel=true;


insert overwrite table GBD_DM_PAC_SAFE.MID_T1_T1_AUTO_CLAIM_PROPERTY_NEW
  select A.REPORT_NO, A.CASE_TIMES
    from (select A.REPORT_NO, A.CASE_TIMES
            from cx_edw_safe.claim_claimdata_clm_whole_case_base a
           where to_date(A.updated_date) < to_date('${end_date}')
             and to_date(A.updated_date) >= to_date('${stat_date}')
             and a.MIGRATE_FROM in ('N', 'n')
          union all
          select A.REPORT_NO, A.CASE_TIMES
            from cx_edw_safe.claim_claimdata_clm_investigation_task a
           where to_date(A.updated_date) < to_date('${end_date}')
             and to_date(A.updated_date) >= to_date('${stat_date}')
          union all
          select A.REPORT_NO, A.CASE_TIMES
            from cx_edw_safe.claim_claimdata_clm_car_rob_info a
           where to_date(A.updated_date) < to_date('${end_date}')
             and to_date(A.updated_date) >= to_date('${stat_date}')
          union all
          select A.REPORT_NO, A.CASE_TIMES
            from cx_edw_safe.claim_claimdata_clm_task_department a
           where to_date(A.updated_date) < to_date('${end_date}')
             and to_date(A.updated_date) >= to_date('${stat_date}')
             and task_code = 'clm_029') A
;
";
exitCodeCheck $?

echo "Run Done 主题临时表!"


#!/bin/sh
source ExitCodeCheck.sh


opts=$@


getparam(){
arg=$1
echo $opts |xargs -n1 |cut -b 2- |awk -F'=' '{if($1=="'"$arg"'") print $2}'
}

IncStart=`getparam inc_start`
IncEnd=`getparam inc_end`
oracle_connection=`getparam jdbc_str`
oracle_username=`getparam db_user`
oracle_password=`getparam db_psw`
dataName=`getparam db_sid`
queueName=`getparam hdp_queue`
hdfshostname=`getparam hdfs_host`;


IncStartYear=`echo ${IncStart:0:4}`;
IncStartMonth=`echo ${IncStart:4:2}`;
IncStartDay=`echo ${IncStart:6:2}`;
IncStartAll=${IncStartYear}"-"${IncStartMonth}"-"${IncStartDay}" 00:00:00.0";
IncEndYear=`echo ${IncEnd:0:4}`;
IncEndMonth=`echo ${IncEnd:4:2}`;
IncEndDay=`echo ${IncEnd:6:2}`;
IncEndAll=${IncEndYear}"-"${IncEndMonth}"-"${IncEndDay}" 00:00:00.0";


hadoop dfs -rmr /apps-data/hduser0101/sx_360_safe/import/MDMDATA_QUESTON_PROJECT;


sqoop import -D mapred.job.queue.name=${queueName} \
--connect ${oracle_connection}                     \
--username ${oracle_username}                      \
--password ${oracle_password}                      \
--query "SELECT /*+ parallel(4) */                 \
PHONE_NO,                                          \
AGENT_NO,                                          \
DATE_INVEST,                                       \
SEQUENCE_NO,                                       \
PRODUCT_NO,                                        \
PRODUCT_NAME,                                      \
QUESTION_DESC,                                     \
OPTION_DESC,                                       \
OPTION_DETAIL,                                     \
TRUNC(DATE_EFF,'dd') as DATE_EFF,                  \
TRUNC(DATE_INVALID,'dd') as DATE_INVALID,          \
decode(NAME,null,'19000101 00:00:00', nvl(to_char(DATE_INVEST,'yyyymmdd hh24:mi:ss'),'19000101 00:00:00')  ||  NAME) as NAME, \
decode(ID_TYPE,null,'19000101 00:00:00', nvl(to_char(DATE_INVEST,'yyyymmdd hh24:mi:ss'),'19000101 00:00:00') || decode(ID_TYPE,'港澳回乡证或台胞证','港澳台证件','港澳台通行证','港澳台证件',ID_TYPE)) as ID_DESCRIPTION, \
decode(ID_NO,null,'19000101 00:00:00', nvl(to_char(DATE_INVEST,'yyyymmdd hh24:mi:ss'),'19000101 00:00:00')  ||  ID_NO) as ID_NO, \
decode(DATE_BIRTH,null,'19000101 00:00:00', nvl(to_char(DATE_INVEST,'yyyymmdd hh24:mi:ss'),'19000101 00:00:00')  ||  DATE_BIRTH) as DATE_BIRTH, \
decode(SEX,null,'19000101 00:00:00', nvl(to_char(DATE_INVEST,'yyyymmdd hh24:mi:ss'),'19000101 00:00:00') || decode(SEX,'9',NULL,SEX)) as SEX, \
LBS_CLIENT_TYPE,                                   \
CUSTOMER_FLAG,                                     \
CHANNEL_SOURCE,                                    \
DATA_SOURCE,                                       \
CASE WHEN TRUNC(DATE_EFF,'dd') <= last_day(to_date('${IncStart}','yyyymmdd')) \
      AND TRUNC(DATE_INVALID,'dd') > last_day(to_date('${IncStart}','yyyymmdd')) \
     THEN 'Y'                                      \
     ELSE 'N'                                      \
      END AS IS_VALID_FLAG,                        \
QUESTION_NO,                                       \
OPTION_NO,                                         \
ID_TYPE_2                                          \
from TMP_2015051318174439              \
where 1=1                                          \
  and \$CONDITIONS "                               \
-m 1                                               \
--hive-table SX_360_SAFE.MDMDATA_QUESTON_PROJECT   \
--hive-drop-import-delims \
--fetch-size 5000 \
--target-dir /apps-data/hduser0101/sx_360_safe/import/mdmdata_queston_project \
--hive-overwrite \
--null-string '\\N' \
--null-non-string '\\N' \
--hive-import;
exitCodeCheck $?

 类似资料: