FlinkX安装与使用

FlinkX安装

上传flinkX

解压flinkX

#先安装unzip
yum install unzip
#解压
unzip flinkx-1.10.zip -d /usr/local/soft/

配置环境变量

vim /etc/profile
#追加以下内容
#flinkx
export FLINKX_HOME=/usr/local/soft/flinkx-1.10
export PATH=$FLINKX_HOME/bin:$PATH

给bin/flinkx这个文件加上执行权限

chmod a+x flinkx

修改配置文件，设置运行端口

vim/flinkconf/flink-conf.yaml
#web服务端口，不指定会随机生成一个
rest.bind-port: 8888

FlinkX使用

mysqltohdfs

#配置文件
{
    "job": {
        "content": [
            {
                "reader": {
                    "parameter": {
                        "username": "root",
                        "password": "123456",
                        "connection": [
                            {
                                "jdbcUrl": [
                                    "jdbc:mysql://master:3306/student?characterEncoding=utf8"
                                ],
                                "table": [
                                    "student"
                                ]
                            }
                        ],
                        "column": [
                            "*"
                        ],
                        "customSql": "",
                        "where": "clazz = '理科二班'",
                        "splitPk": "",
                        "queryTimeOut": 1000,
                        "requestAccumulatorInterval": 2
                    },
                    "name": "mysqlreader"
                },
                "writer": {
                    "name": "hdfswriter",
                    "parameter": {
                        "path": "hdfs://master:9000/data/flinkx/student",
                        "defaultFS": "hdfs://master:9000",
                        "column": [
                            {
                                "name": "col1",
                                "index": 0,
                                "type": "string"
                            },
                            {
                                "name": "col2",
                                "index": 1,
                                "type": "string"
                            },
                            {
                                "name": "col3",
                                "index": 2,
                                "type": "string"
                            },
                            {
                                "name": "col4",
                                "index": 3,
                                "type": "string"
                            },
                            {
                                "name": "col5",
                                "index": 4,
                                "type": "string"
                            },
                            {
                                "name": "col6",
                                "index": 5,
                                "type": "string"
                            }
                        ],
                        "fieldDelimiter": ",",
                        "fileType": "text",
                        "writeMode": "overwrite"
                    }
                }
            }
        ],
        "setting": {
            "restore": {
                "isRestore": false,
                "isStream": false
            },
            "errorLimit": {},
            "speed": {
                "channel": 1
            }
        }
    }
}

启动任务

flinkx -mode local -job ./scripts/mysqlTohdfs.json -pluginRoot syncplugins/ -flinkconf flinkconf/

通过web界面查看任务运行情况

http://master:8888

监控日志

tail -f nohup.out