DataX导入数据从FTP到HDFS的Json配置文件

最新推荐文章于 2024-05-31 20:55:59 发布

王壮_

最新推荐文章于 2024-05-31 20:55:59 发布

阅读量158

点赞数 1

分类专栏：大数据文章标签： hdfs 大数据 etl

本文链接：https://blog.csdn.net/qq_50833617/article/details/137348370

版权

大数据专栏收录该内容

5 篇文章 0 订阅

订阅专栏

{
    "job": {
        "setting": {
            "speed": {
                "channel": 2
            }
        },
        "content": [
            {
                "reader": {
                    "name": "ftpreader",
                    "parameter": {
                        "protocol": "sftp",
                        "host": "hadoop13",
                        "port": 22,
                        "username": "root",
                        "password": "123456",
                        "path": [
                            "/opt/data/ftptohdfs.txt"
                        ],
                        "column": [
                            {
                                "index": 0,
                                "type": "long"
                            },
                            {
                                "index": 1,
                                "type": "string"
                            },
                            {
                                "index": 2,
                                "type": "date",
                                "format": "yyyy-MM-dd"
                            },
                            {
                                "index": 3,
                                "type": "date",
                                "format": "yyyy-MM-dd HH:mm:ss"
                            }
                        ],
                        "encoding": "UTF-8",
                        "fieldDelimiter": ","
                    }
                },
                "writer": {
                    "name": "hdfswriter",
                    "parameter": {
                        "column": [{
                                         "name": "user_id",
                                         "type": "INT"
                                     },
                                     {
                                        "name": "user_name",
                                         "type": "STRING"
                                     },
                                     {
                                        "name": "user_birthday",
                                         "type": "date"
                                     },
                                     {
                                        "name": "user_birthday",
                                         "type": "timestamp"
                                     }],
                        "compress": "",
                        "defaultFS": "hdfs://hdfs-cluster", 
                        "hadoopConfig":{
                                "dfs.nameservices": "hdfs-cluster",
                                "dfs.ha.namenodes.hdfs-cluster": "nn1,nn2",
                                "dfs.namenode.rpc-address.hdfs-cluster.nn1": "hadoop11:8020",
                                "dfs.namenode.rpc-address.hdfs-cluster.nn2": "hadoop12:8020",
                                "dfs.client.failover.proxy.provider.hdfs-cluster": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
                        },
                        "fieldDelimiter": "\t",
                        "fileName": "ftp_test",
                        "fileType": "text",
                        "path": "/test_datax/ftp2hdfs",
                        "writeMode": "append"
                    }
                }
            }
        ]
    }
}