+、crontab
*/5 * * * * /data/datax/sh/syn_product_summary.sh > /dev/null 2>&1
+、sh 脚本 syn_product_summary.sh
source /etc/profile
cur_time=$(date +%s)
start_time=$(($cur_time - 360))
/usr/bin/python /usr/local/datax/bin/datax.py /data/datax/config/mro_summary_max_mysql_elastic_product.json -p "-Dstart_time=$start_time" >>/var/log/datax/summary_log.`date +%Y%m%d` 3>&1 &
+、datax 同步脚本 mro_summary_max_mysql_elastic_product.json 含 groovy脚本
{
"job": {
"content": [{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "mro_account",
"password": "******",
"connection": [{
"querySql": ["select product_id as uni_key, product_id,brand_name,model,category_name,color,unit, \" \" as category_attr, concat(brand_name, \" \", model, \" \", category_name, \" \", unit) as summary,bn, \"AAAAA\" as support_sku, 0.00 as support_price, attr_json from beta_product where product_state=1 and update_time > FROM_UNIXTIME($start_time)"],
"jdbcUrl": ["jdbc:mysql://rm-*.mysql.rds.aliyuncs.com:3306/mro_*"]
}]
}
},
"transformer": [
{
"name": "dx_groovy",
"parameter": {
"code": "Column column = record.getColumn(12);\n def attrJson = column.asString();\n def attrStr = new StringBuffer();\n if (column != null && !attrJson.equals(\"\")) {\n def jsonSlurper = new JsonSlurper();\n def map = jsonSlurper.parseText(attrJson);\n map.each{\n attrStr.append(\"$it.attributeValue \");\n }\n}\n record.setColumn(7, new StringColumn(attrStr.toString()));\n return record;",
"extraPackage": [
"import groovy.json.JsonSlurper;"
]
}
}
],
"writer": {
"name": "elasticsearchwriter",
"parameter": {
"endpoint": "http://172.16.166.101:9200",
"accessId": "elastic",
"accessKey": "******",
"index": "product_summary_max",
"type": "_doc",
"cleanup": false,
"settings": {
"index" :{
"number_of_shards": 2,
"number_of_replicas": 1
}
},
"discovery": false,
"batchSize": 20000,
"splitter": ",",
"column": [
{
"name": "uniKey",
"type": "id"
},
{
"name":"productId",
"type" : "long"
},
{
"name":"brandName",
"type" : "text",
"analyzer" : "my_ik"
},
{
"name":"model",
"type" : "text",
"analyzer" : "ik_max_word"
},
{
"name":"categoryName",
"type" : "text",
"analyzer" : "my_ik"
},
{
"name":"color",
"type" : "text",
"analyzer" : "ik_max_word"
},
{
"name":"unit",
"type" : "text",
"analyzer" : "ik_max_word"
},
{
"name":"categoryAttr",
"type" : "text",
"analyzer" : "ik_max_word"
},
{
"name":"summary",
"type" : "text",
"analyzer" : "my_ik"
},
{
"name":"bn",
"type" : "text"
},
{
"name":"support_sku",
"type" : "text"
},
{
"name":"support_price",
"type" : "float"
},
{
"name":"remark",
"type" : "text"
}
]
}
}
}],
"setting": {
"speed": {
"channel": 5
}
}
}
}