软件版本号 jdk1.8、apache-flume-1.6.0-bin、kafka_2.8.0-0.8.0、zookeeper-3.4.5
需要向apache-flume-1.6.0-bin/lib 放入两个jar包
flume-ng-sql-source-1.3.7.jar -->flume的mysql source 下载地址: https://github.com/keedio/flume-ng-sql-source/
mysql-connector-java-5.1.35.jar -->mysql 驱动包 。
*******flume的配置文件信息如下:*********
agent.sources = s1
agent.channels = c1 c2
agent.sinks = HDFS kfk
agent.sources.sql-source.type = org.keedio.flume.source.SQLSource
agent.sources.sql-source.channels = c1 c2
agent.sources.sql-source.connection.url = jdbc:mysql://192.168.2.164:3306/test
agent.sources.sql-source.user = root
agent.sources.sql-source.password = root
agent.sources.sql-source.table = /**数据库表名字*/
agent.sources.sql-source.columns.to.select = *
agent.sources.sql-source.incremental.column.name = id /**表里面的某个字段,用来判断增量*/
agent.sources.sql-source.incremental.value = 0
agent.sources.sql-source.run.query.delay=5000
agent.sources.sql-source.status.file.path = /var/lib/flume /**有就不用动,没有就创建 mkdir -p /var/lib/flume*/
agent.sources.sql-source.status.file.name = sql-source.status /** touch sql-source.status 创建文件 */
/** chmod -R 777 sql-source.status 修改文件权限 sql-source.status 文件用来记录当前记录的数量*/
#c1
agent.channels.ch1.type = memory
agent.channels.ch1.capacity = 1000
agent.channels.ch1.transactionCapacity = 100
#c2
agent.channels.ch2.type = memory
agent.channels.ch2.capacity = 1000
agent.channels.ch2.transactionCapacity = 100
#配置hdfs sink
agent.sinks.HDFS.channel = c1
agent.sinks.HDFS.type = hdfs
agent.sinks.HDFS.hdfs.path = hdfs://dbserver:9000/flume/mysql3 /**存放数据的hdfs目录*/
agent.sinks.HDFS.hdfs.fileType = DataStream
agent.sinks.HDFS.hdfs.writeFormat = Text
agent.sinks.HDFS.hdfs.rollSize = 268435456
agent.sinks.HDFS.hdfs.rollInterval = 0
agent.sinks.HDFS.hdfs.rollCount = 0
# 配置 kafka sink
agent.sinks.kfk.type = org.apache.flume.sink.kafka.KafkaSink
agent.sinks.kfk.brokerList=192.168.2.171:9092,192.168.2.172:9092,192.168.2.173:9092
agent.sinks.kfk.topic=mytopic
agent.sinks.kfk.requiredAcks = 1
agent.sinks.kfk.batchSize = 2
agent.sinks.kfk.channel = c2
****上面的方法实现没有问题,出问题很有可能是哪个细节写错了,好好检查一下英文单词、ip等别写错!!********