在datax中提前将MySQL数据进行大小写转换
修改datax的json文件
{
"job": {
"content": [{
"transformer": [{
"parameter": {
"code": "for(int i=0;i<record.getColumnNumber();i++){if(record.getColumn(i).getByteSize()!=0){Column column = record.getColumn(i); def str = column.asString(); def newStr=null; newStr=str.replaceAll(\"[\\r\\n]\",\"\"); record.setColumn(i, new StringColumn(newStr)); };};return record;",
"extraPackage": []
},
"name": "dx_groovy"
}],
"writer": {
"parameter": {
"writeMode": "append",
"fieldDelimiter": "\u0001",
"column": [{
"type": "string",
"name": "id"
}, {
"type": "string",
"name": "code"
}, {
"type": "string",
"name": "title"
}, {
"type": "int",
"name": "print_count"
}, {
"type": "string",
"name": "effective_date"
}, {
"type": "string",
"name": "expiry_date"
}, {
"type": "string",
"name": "time_slots"
}, {
"type": "string",
"name": "department_id"
}, {
"type": "string",
"name": "department_name"
}, {
"type": "int",
"name": "status"
}, {
"type": "int",
"name": "type"
}, {
"type": "string",
"name": "created_date"
}, {
"type": "string",
"name": "created_by"
}, {
"type": "string",
"name": "last_modified_date"
}, {
"type": "string",
"name": "last_modified_by"
}, {
"type": "int",
"name": "rest_day"
}],
"path": "${targetdir}",
"fileType": "text",
"defaultFS": "hdfs://mycluster:8020",
"compress": "gzip",
"fileName": "cn_work_hours"
},
"name": "hdfswriter"
},
"reader": {
"parameter": {
"username": "dw_readonly",
"connection": [{
"querySql": [
"
select lower(id),
code,
title,
print_count,
effective_date,
expiry_date,
lower(REPLACE(time_slots, '.', ',')),
lower(department_id),
upper(department_name),
status,
type,
created_date,
created_by,
last_modified_date,
last_modified_by,
rest_day
from cn_work_hours
"
],
"jdbcUrl": ["jdbc:mysql://******"]
}],
"password": "****",
"splitPk": ""
},
"name": "mysqlreader"
}
}],
"setting": {
"speed": {
"channel": 3
},
"errorLimit": {
"record": 0,
"percentage": 0.02
}
}
}
}