1.mongdb连接
连接方式如下(重要!!!!,账号密码必须建立在db下面,如果默认再admin下面,导致无法切换库,连接报错)
2.mysql增量同步到mongodb的json如下,(可以参考先前文章mysql增量时间同步方式)
{
"job": {
"setting": {
"speed": {
"channel": 3,
"byte": 1048576
},
"errorLimit": {
"record": 0,
"percentage": 0.02
}
},
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "123",
"password": "123",
"connection": [
{
"jdbcUrl": [
"jdbc:mysql://192.168.1.101:3306/test"
],
"querySql": [
"SELECT 'test'as source,(CASE when gateway_id=9 then 'test' ELSE 'tnc' end) as app_code,target ,target_account_id as account_id, null as user_name, sms_content as content, send_time ,send_id as outer_id FROM qfc_mcs_send_mass_info where sms_type = '3' and status in (3,4) and send_time >= FROM_UNIXTIME(${lastTime}) and send_time < FROM_UNIXTIME(${currentTime});"
]
}
]
}
},
"writer": {
"name": "mongodbwriter",
"parameter": {
"address": [
"192.168.1.100:27017"
],
"userName": "username",
"userPassword": "password",
"dbName": "DBname",
"collectionName": "message_send_info",
"column": [
{
"name": "sourceSystem",
"type": "String"
},
{
"name": "app_code",
"type": "String"
},
{
"name": "target",
"type": "String"
},
{
"name": "account_id",
"type": "Long"
},
{
"name": "user_name",
"type": "String"
},
{
"name": "content",
"type": "String"
},
{
"name": "send_time",
"type": "Date"
},
{
"name": "outer_id",
"type": "Long"
}
],
"upsertInfo": {
"isUpsert": false,
"upsertKey": ""
}
}
}
}
]
}
}
如果需要更新isUpsert改写true upsertKey为更新的字段
3.字段参考
DataX 内部类型 | MongoDB 数据类型 |
---|---|
Long | int, Long |
Doubl | double |
String | string, array |
Date | date |
Bytes | bytes |