datax 持续数据同步_Datax 数据同步

官方Hello入门例子

{

"job": {

"content": [

{

"reader": {

"name": "streamreader",

"parameter": {

"sliceRecordCount": 10,

"column": [

{

"type": "long",

"value": "10"

},

{

"type": "string",

"value": "hello，你好，世界-DataX"

}

]

}

},

"writer": {

"name": "streamwriter",

"parameter": {

"encoding": "UTF-8",

"print": true

}

}

}

],

"setting": {

"speed": {

"channel": 5

}

}

}

}

运行命令：

python  /data/tools/datax/bin/datax.py ./hello_datax.json

java调用datax 参考：

ublic class TestDatax {

public static void main(String[] args) {

try {

WebLogs.info("start");

String windowcmd = "cmd /c python datax.py D:\\Software\\install\\Environment\\DataX\\datax\\job\\mysql2mysql.json";

WebLogs.info(windowcmd);

//.exec("你的命令",null,new File("datax安装路径"));

Process pr = Runtime.getRuntime().exec(windowcmd,null,new File("D:\\Software\\install\\Environment\\DataX\\datax\\bin"));

BufferedReader in = new BufferedReader(new InputStreamReader(pr.getInputStream()));

String line = null;

while ((line = in.readLine()) != null) {

WebLogs.info(line);

}

in.close();

pr.waitFor();

WebLogs.info("end");

} catch (Exception e) {

e.printStackTrace();

}

}

}

网上一个mysql2mysql的例子

{

"job": {

"content": [

{

"reader": {

"name": "mysqlreader",

"parameter": {

"column": [

"id",

"name"

],

"connection": [

{

"jdbcUrl": ["jdbc:mysql://127.0.0.1:3306/dq"],

"table": ["table1"]

}

],

"password": "123456",

"username": "root"

}

},

"writer": {

"name": "mysqlwriter",

"parameter": {

"column": [

"id",

"name"

],

"connection": [

{

"jdbcUrl": "jdbc:mysql://ip地址:端口/test",

"table": ["table2"]

}

],

"password": "123456",

"username": "root"

}

}

}

],

"setting": {

"speed": {

"channel": "1"

}

}

}

}

mysql到本地：

{

"job": {

"content": [

{

"reader": {

"name": "mysqlreader",

"parameter": {

"username": "root",

"password": "root",

"column": ["ID","VISIT_IP"],

"connection": [

{

"table": [

"visit_log"

],

"jdbcUrl": [

"jdbc:mysql://192.168.0.197:3306/qyyx_test"

]

}

]

}

},

"writer": {

"name": "streamwriter",

"parameter": {

"print": false,

"encoding": "UTF-8"

}

}

}

]

}

}

本地方件到本地文件：

{

"setting": {},

"job": {

"setting": {

"speed": {

"channel": 2

}

},

"content": [

{

"reader": {

"name": "txtfilereader",

"parameter": {

"path": ["/home/haiwei.luo/case00/data"],

"encoding": "UTF-8",

"column": ["*"],

"fieldDelimiter": ","

}

},

"writer": {

"name": "txtfilewriter",

"parameter": {

"path": "/home/haiwei.luo/case00/result",

"fileName": "luohw",

"writeMode": "truncate",

"format": "yyyy-MM-dd"

}

}

}

]

}

}

HDFS打印到本地控制台

{

"job": {

"setting": {

"speed": {

"channel": 3

}

},

"content": [

{

"reader": {

"name": "hdfsreader",

"parameter": {

"path": "/user/hive/warehouse/test.db/nnn/*",

"defaultFS": "hdfs://192.168.200.45:8020",

"column": ["*"],

"fileType": "orc",

"encoding": "UTF-8",

"fieldDelimiter": ","

}

},

"writer": {

"name": "streamwriter",

"parameter": {

"print": true

}

}

}

]

}

}

oracle到本地文件

{

"job": {

"content": [

{

"reader": {

"name": "mysqlreader",

"parameter": {

"username": "root",

"password": "root",

"column": ["ID","VISIT_IP","BEF_URL","NOW_URL","CORP_CODE","PROMOTE_NO","PLATFORM","COOKIE_EANBLED","USER_AGENT","APP_VERSION","APP_NAME","COLOR","SCREEN","JAVA_ENABLED","JS_VERSION","SID","CREATE_BY","CREATE_TIME","MODIFY_BY","MODIFY_TIME"],

"connection": [

{

"table": [

"visit_log"

],

"jdbcUrl": [

"jdbc:mysql://192.168.0.197:3306/qyyx_test"

]

}

]

}

},

"writer": {

"name": "txtfilewriter",

"parameter": {

"path": "/opt/result/",

"fileName": "shun",

"writeMode": "truncate",

"format": "yyyy-MM-dd"

}

}

}

],

"setting": {

"speed": {

"channel": 10

}

}

}

}

本地文件到HDFS

orcle打印到Linux

{

"job": {

"setting": {

"speed": {

"channel": 5

}

},

"content": [

{

"reader": {

"name": "oraclereader",

"parameter": {

"username": "edw_src",

"password": "123456",

"where": "",

"connection": [

{

"querySql": [

"select * from ORDER_DETAIL_BAK"

],

"jdbcUrl": [

"jdbc:oracle:thin:@192.168.0.197:1521:orcl"

]

}

]

}

},

"writer": {

"name": "streamwriter",

"parameter": {

"visible": true,

"encoding": "UTF-8"

}

}

}

]

}

}

oracle到HDFS

oracle到本地文件

{

"job": {

"setting": {

"speed": {

"channel": 200

}

},

"content": [

{

"reader": {

"name": "oraclereader",

"parameter": {

"username": "edw_src",

"password": "123456",

"where": "",

"connection": [

{

"querySql": [

"select * from ORDER_DETAIL_BAK"

],

"jdbcUrl": [

"jdbc:oracle:thin:@192.168.0.197:1521:orcl"

]

}

]

}

},

"writer": {

"name": "txtfilewriter",

"parameter": {

"path": "/opt/result/",

"fileName": "shun",

"writeMode": "truncate",

"format": "yyyy-MM-dd"

}

}

}

]

}

}