{"job":{"content":[{"reader":{"name":"mysqlreader","parameter":{"connection":[{"jdbcUrl":["jdbc:mysql://${source_db_host}:${source_db_port}/${source_db}?characterEncoding=utf8"],"querySql":["SELECT FROM table_name "]}],"username":"${source_db_user}","password":"${source_db_pwd}","where":""}},"writer":{"name":"gpdbwriter","parameter":{"postSql":[],"preSql":["TRUNCATE TABLE ods.table_name ","VACUUM ods.table_name"],"column":["c1","c2","column3name"....],"connection":[{"jdbcUrl":"jdbc:postgresql://${target_db_host}:${target_db_port}/${target_db}","table":["ods.table_name"]}],"username":"${target_db_user}","password":"${target_db_pwd}","num_copy_writer":2,"num_copy_processor":2,"copy_queue_size":5000,"segment_reject_limit":0}}}],"setting":{"speed":{"channel":"3"}}}
1.2 数据从kafka到数仓GP
{"job":{"content":[{"reader":{"name":"kafkareader","parameter":{"topic":"${topic}","bootstrapServers":"${bootstrap_servers}","partitions": ${partitions},"groupId":"${group_id}","parsingRules":"${parsing_rules}","timeout": ${timeout},"retries": ${retries},"offsetReset":"${offset_reset}","kafkaRecordKeys":"col1,col2,col3,col4.....","logPath":"/opt/module/datax/log/errorlog"}},"writer":{"name":"gpdbwriter","parameter":{"postSql":[],"preSql":["DELETE FROM ods.table_name WHERE TO_CHAR(TO_TIMESTAMP(TO_NUMBER(COALESCE(send_time,'0'),'9999999999')):: TIMESTAMP WITHOUT TIME ZONE,'yyyy-mm-dd') < '${del_start_date}' "],"column":["col1","col2","col3","col4"],"connection":[{"jdbcUrl":"jdbc:postgresql://${target_db_host}:${target_db_port}/${target_db}","table":["ods.table_name"]}],"username":"${target_db_user}","password":"${target_db_pwd}","segment_reject_limit":0,"copy_queue_size":5000,"num_copy_processor":4,"num_copy_writer":4}}}],"setting":{"speed":{"channel":3}}}}