Hadoop大数据开发__Flume和HBase、Kafka集成

1、下载解压Flume

cd /home/hadoop/app
tar -zxvf apache-flume-1.10.0-bin.tar.gz
rm apache-flume-1.10.0-bin.tar.gz
ln -s apache-flume-1.10.0-bin kafka

2、修改Flume配置文件
监测目录下是否有新文件，如果有就解析出来，发给hbase、kafka

#定义agent名， source、channel、sink的名称
a4.sources = r1
a4.channels = c1 c2 #c1给kafka用、c2给hbase用
a4.sinks = k1 k2 #k1为kafka、k2为hbase
 
#具体定义source
a4.sources.r1.type = spooldir
a4.sources.r1.spoolDir = /home/hadoop/app/flume/flumelogs #监控此目录下是否有新文件

#具体定义channel-for kafka
a4.channels.c1.type = memory
a4.channels.c1.capacity = 1000
a4.channels.c1.transactionCapacity = 100

#具体定义channel-for hbase
a4.channels.c2.type = memory
a4.channels.c2.capacity = 1000
a4.channels.c2.transactionCapacity = 100
 
#具体定义sink-kafka
a4.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a4.sinks.k1.topic = mydemo2 #topic名称
a4.sinks.k1.brokerList = hadoop01:9092,hadoop02:9092,hadoop03:9092 #kafka的broker地址
a4.sinks.k1.requiredAcks = 1
a4.sinks.k1.batchSize = 20

#具体定义sink-hbase
a4.sinks.k2.type = org.apache.flume.sink.hbase2.HBase2Sink #这里注意HBase2以后必须这么写
a4.sinks.k2.table = sogoulogs #表名
a4.sinks.k2.columnFamily = info #列族的名字
a4.sinks.k2.zookeeperQuorum = hadoop01:2181,hadoop02:2181,hadoop03:2181 #zookeeper的地址
a4.sinks.k2.serializer = org.apache.flume.sink.hbase2.SimpleHBase2EventSerializer #这里注意HBase2以后必须这么写
a4.sinks.k2.serializer.payloadColumn = title #列族中的列名
 
#组装source、channel、sink
a4.sources.r1.channels = c1 c2
a4.sinks.k1.channel = c1
a4.sinks.k2.channel = c2

3、启动kafka的消费者

cd /home/hadoop/app/kafka/bin
#创建topic
kafka-topics.sh --bootstrap-server hadoop01:9092 --create --topic mydemo2 --replication-factor 3 --partitions 3
#启动消费者
kafka-console-consumer.sh --bootstrap-server hadoop01:9092,hadoop02:9092,hadoop03:9092 --topic mydemo2
#启动生产者
kafka-console-producer.sh --broker-list hadoop01:9092 --topic mydemo2
#如果配置正确，生产者生产的消息，在消费者控制台可以打印出来

4、启动Flume进行测试、同时发送给kafka、hbase

cd /home/hadoop/app/flume/bin
#启动flume
bin/flume-ng agent -n agent -c conf -f conf/a4.properties -Dflume.root.logger=INFO
#在别的目录做成dj.txt
fas
fdsa
fas123
#把dj.txt复制到以下目录中
/home/hadoop/app/flume/flumelogs
#在kafka的消费者控制台能够打印出文件的内容
fas
fdsa
fas123
#在hbase中也会看到内容
cd /home/hadoop/app/hbase/bin
hbase shell #启动hbase命令行
scan 'sogoulogs' #查看表的内容