需求
使用datax将mysql中的数据拉取到hive的ods层
步骤
首先在mysql中确定好需要拉取的表user_extend,
然后对应在hive中创建好空表,等待拉取
这里对应创建的hive表格如下
CREATE TABLE snbap_ods.ods_user_extend (
user_id bigint ,
user_gender bigint ,
is_pregnant_woman tinyint ,
is_have_children tinyint ,
is_have_car tinyint ,
phone_brand string ,
phone_brand_level string ,
phone_cnt int ,
change_phone_cnt int ,
is_maja tinyint ,
majia_account_cnt int ,
loyal_model string ,
shopping_type_model string ,
weight int ,
height int
)
stored as textfile
location '/datax/mysql/snbap_ods/ods/ods_user_extend';
在datax的bin目录下创建json脚本 vi datax1.json
{
"job": {
"setting": {
"speed": {
"byte":1048576,
"channel":"4"
}
},
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"connection": [
{
"querySql": [
"select * from user_extend"
],
"jdbcUrl": [
"jdbc:mysql://localhost:3306/snbap_ods"
]
}
]
}
},
"writer": {
"name": "hdfswriter",
"parameter": {
"defaultFS":"hdfs://192.168.1.54:9000",
"fileType":"text",
"path":"/datax/mysql/snbap_ods/ods/ods_user_extend/",
"fileName":"user_extend",
"column": [
{
"name":"user_id","type":"bigint"}
{
"name":"user_gender","type":"bigint"}
{
"name":"is_pregnant_woman","type":"tinyint"}
{
"name":"is_have_children","type":"tinyint"}
{
"name":"is_have_car","type":"tinyint"}
{
"name":"phone_brand","type":"string"}
{
"name":"phone_brand_level","type":"string"}
{
"name":"phone_cnt","type":"int"}
{
"name":"change_phone_cnt","type":"int"}
{
"name":"is_maja","type":"tinyint"}
{
"name":"majia_account_cnt","type":"int"}
{
"name":"loyal_model","type":"string"}
{
"name":"shopping_type_model","type":"string"}
{
"name":"weight","type":"int"}
{
"name":"height","type":"int"}
],
"writeMode":"append",
"fieldDelimiter":"\u0001",
"compress":"gzip"
}
}
}
]
}
}
在datax的bin目录下运行以上json脚本
python /opt/install/datax/bin/datax.py /opt/install/datax