datax如何在windows与linux下运行JSON脚本

DataX

DataX 是阿里巴巴集团内被广泛使用的离线数据同步工具/平台，实现包括 MySQL、Oracle、SqlServer、Postgre、HDFS、Hive、ADS、HBase、TableStore(OTS)、MaxCompute(ODPS)、DRDS 等各种异构数据源之间高效的数据同步功能。

项目地址

https://github.com/alibaba/DataX

项目需求

定时从sqlserver搬运数据到mysql，而且还是增量(数据更新)

定时任务xxljob

这个xxl-job是个很厉害的一个开源分布式定时任务架构。有兴趣的可以进入xxl社区了解更多内容。

xxl-job项目地址：

https://github.com/xuxueli/xxl-job

代码

1. 下载datax解压好
   

2. 项目配置文件引入
   windows datax的安装路径改成自己的电脑上的

3. 业务代码

	@Autowired
	private Environment env;
	
	@Value("${itf.conf.windowsCommand}")
	private String windowsCommand;//windows命令
	
	@Value("${itf.conf.linuxCommand}")
	private String linuxCommand;//linux命令
	
	@Value("${itf.conf.dataxPath}")
	private String dataxPath;//datax安装路径
    
    /**
	 * 
	 *@Title: runDatax 方法名
	 *@Description:（运行Datax） 
	 * @param newfilePath（脚本文件路径） 
	 * @throws Exception 
	 *@author：zhaoyinguang
	 *@data 2018年10月30日
	 */
	public void runDatax(String newfilePath) {
		BufferedReader in = null;
		Process pr=null;
		try {
			if(env.getProperty("datax.system.type").equals("linux")){
				String linuxcmd = linuxCommand+" "+newfilePath;
				XxlJobLogger.log("XXL-JOB," + linuxcmd );
				pr = Runtime.getRuntime().exec(linuxcmd);
			}else{
				 String windowcmd = windowsCommand+newfilePath;
				XxlJobLogger.log("XXL-JOB," + windowcmd );
				 pr = Runtime.getRuntime().exec(windowcmd,null,new File(dataxPath));
			}
			in = new BufferedReader(new InputStreamReader(pr.getInputStream()));
			String line = null;
			boolean hasData = false;
			while ((line = in.readLine()) != null){
					XxlJobLogger.log(line);
					hasData = true;
			}
			if(!hasData) {
				XxlJobLogger.log("XXL-JOB, 没有读取到脚本内容。"  );
			}
			
			in.close();
			pr.waitFor();
		} catch (Exception e) {
			WebLogs.printStackTrace(e);
			XxlJobLogger.log("datax运行有错误 请检查脚本");
			e.printStackTrace();
			XxlJobLogger.log(e);
		}
		File file=new File(newfilePath);
		 if(file.exists()&&file.isFile())
	         file.delete();
	 }

4. 脚本

{
    "job": {
        "setting": {
            "speed": {
                 "byte": 1048576
            }
        },
        "content": [
            {
                "reader": {
                    "name": "mysqlreader",
                    "parameter": {
                        "username": "%1%",
                        "password": "%2%",
						"where":"",
                        "connection": [
                            {
                                "querySql": ["select 
									ca20.ICAT20,
									ca20.ICAT01,
									ca20.CASPZT,
									ca20.CADJZT,
									ca20.CAYSFY,
									ca20.CAFHSJ,
									o.id,
									ca20.CREATETIME,
									ca20.UPTIMESTAMP
									from cat20 ca20
									left join xtm02 x02 on ca20.ixtm02 = x02.ixtm02
									left join itf_idos_mn_organization o on o.OrgCode = x02.XTJGDM
									where date_format(ca20.UPTIMESTAMP,'%Y%m%d')  in (%5%)"
								],
                                "jdbcUrl": [
									"jdbc:%3%"
                                ]
                            }
                        ]
                    }
                },
               "writer": {
                    "name": "mysqlwriter",
                    "parameter": {
                        "writeMode": "update",
                        "username": "%1%",
                        "password": "%2%",
                        "column": [
                            "ID",
							"SalesOrderID",
							"RegFeeStatus",
							"ApplicationStatus",
							"PreregFee",
							"ReviewOn2",
							"OrganizationID",
							"CreatedOn",
							"ModifiedOn"
                        ],
                        "session": [
                        	"set session sql_mode='ANSI'"
                        ],
                        "connection": [
                            {
                                "jdbcUrl": "jdbc:%4%",
                                "table": [
                                    "itf_idos_bs_license"
                                ]
                            }
                        ]
                    }
                }
            }
        ]
    }
}