资料
链接:https://pan.baidu.com/s/1RhLk4Gva3ee1FOHDEraw6A
提取码:yyds
flume
前期准备
tar -zxvf apache-flume-1.9.0-bin.tar.gz
修改配置文件
mv flume-conf.properties.template flume-conf.properties
vi log4j.properties
flume.log.dir=/home/bigdata/module/apache-flume-1.9.0-bin/log
文件到kafka
先创建一个工作目录
mkdir job
vi jobs/file_to_kafka.conf
# 1 组件定义
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# 2 配置 source
a1.sources.r1.type = TAILDIR
a1.sources.r1.filegroups = f1
#监控的文件
a1.sources.r1.filegroups.f1 = /home/bigdata/module/apache-flume-1.9.0-bin/app.*
#断点续传记录文件
a1.sources.r1.positionFile = /home/bigdata/module/apache-flume-1.9.0-bin/taildir_position.json
# 3 配置 channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# 4 配置 sink
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.bootstrap.servers = hadoop102:9092,hadoop103:9092,hadoop104:9092
a1.sinks.k1.kafka.topic = first
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.producer.acks = 1
a1.sinks.k1.kafka.producer.linger.ms = 1
# 5 拼接组件
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
先创建断点续传文件
touch /home/bigdata/module/apache-flume-1.9.0-bin/taildir_position.json
先测试下
bin/flume-ng agent -c conf/ -n a1 -f job/file_to_kafka.conf -Dflume.root.logger=DEBUG,console
启动一个kaka的消费者
./kafka-console-consumer.sh --bootstrap-server hadoop102:9092 --topic first
然后在监控的app.log追加内存
echo "hello " >> app.log
查看kafka消费者,如图消费到了追加文件的数据
启停脚本
vi flume_file_to_kafka.sh
#!/bin/bash
case $1 in
"start"){
for i in hadoop102
do
echo " --------启动 $i 采集flume-------"
ssh $i "nohup /home/bigdata/module/apache-flume-1.9.0-bin/bin/flume-ng agent --conf /home/bigdata/module/apache-flume-1.9.0-bin/conf --conf-file /home/bigdata/module/apache-flume-1.9.0-bin/job/file_to_kafka.conf --name a1 -Dflume.root.logger=INFO,console >/dev/null 2>&1 &"
done
};;
"stop"){
for i in hadoop102
do
echo " --------停止 $i 采集flume-------"
ssh $i "ps -ef | grep file_to_kafka.conf | grep -v grep |awk '{print \$2}' | xargs -n1 kill -9 "
done
};;
esac
chmod 744 flume_file_to_kafka.sh
上面是文件到kafka
kafka到控制台
现在配置kafka到控制台
vi kafka_to_file.conf
# 1 组件定义
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# 2 配置 source
a1.sources.r1.type = org.apache.flume.source.kafka.KafkaSource
a1.sources.r1.batchSize = 50
a1.sources.r1.batchDurationMillis = 200
a1.sources.r1.kafka.bootstrap.servers = hadoop102:9092
a1.sources.r1.kafka.topics = first
a1.sources.r1.kafka.consumer.group.id = custom.g.id
# 3 配置 channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# 4 配置 sink
a1.sinks.k1.type = logger
# 5 拼接组件
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
然后下面的操作是和上面执行操作一样这里就不在重复讲解
Flink
flink生产者
pom.xml
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>1.13.0</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.12</artifactId>
<version>1.13.0</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.12</artifactId>
<version>1.13.0</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka_2.12</artifactId>
<version>1.13.0</version>
</dependency>
log4j.properties
log4j.rootLogger=error, stdout,R
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-ddHH:mm:ss,SSS} %5p --- [%50t] %-80c(line:%5L) : %m%n
log4j.appender.R=org.apache.log4j.RollingFileAppender
log4j.appender.R.File=../log/agent.log
log4j.appender.R.MaxFileSize=1024KB
log4j.appender.R.MaxBackupIndex=1
log4j.appender.R.layout=org.apache.log4j.PatternLayout
log4j.appender.R.layout.ConversionPattern=%d{yyyy-MM-ddHH:mm:ss,SSS} %5p --- [%50t] %-80c(line:%6L) : %m%n
public class FlinkProducer {
public static void main(String[] args) throws Exception {
// 0 初始化 flink 环境
StreamExecutionEnvironment env =
StreamExecutionEnvironment.getExecutionEnvironment();
//和主题的分区一致
env.setParallelism(7);
// 1 读取集合中数据
ArrayList<String> wordsList = new ArrayList<>();
wordsList.add("hello");
wordsList.add("world");
DataStream<String> stream = env.fromCollection(wordsList);
// 2 kafka 生产者配置信息
Properties properties = new Properties();
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "hadoop102:9092");
// 3 创建 kafka 生产者
FlinkKafkaProducer kafkaProducer = new FlinkKafkaProducer<String>(
"first",
new SimpleStringSchema(),
properties
);
// 4 生产者和 flink 流关联
stream.addSink(kafkaProducer);
// 5 执行
env.execute();
}
}
flink消费者
public class FlinkConsumer {
public static void main(String[] args) throws Exception {
// 0 初始化 flink 环境
StreamExecutionEnvironment env =
StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(7);
// 1 kafka 消费者配置信息
Properties properties = new Properties();
properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
"hadoop102:9092");
// 2 创建 kafka 消费者
FlinkKafkaConsumer<String> kafkaConsumer = new FlinkKafkaConsumer<>(
"first",
new SimpleStringSchema(),
properties
);
// 3 消费者和 flink 流关联
env.addSource(kafkaConsumer).print();
// 4 执行
env.execute();
}
}
生产者测试
./kafka-console-producer.sh --bootstrap-server hadoop102:9092 --topic first