Kafka入门到精通-阶段二(外部系统集成)

资料

链接：https://pan.baidu.com/s/1RhLk4Gva3ee1FOHDEraw6A 
提取码：yyds 

flume

前期准备

tar -zxvf apache-flume-1.9.0-bin.tar.gz

修改配置文件 

mv flume-conf.properties.template flume-conf.properties

vi log4j.properties

flume.log.dir=/home/bigdata/module/apache-flume-1.9.0-bin/log

文件到kafka

先创建一个工作目录

mkdir job

vi jobs/file_to_kafka.conf

# 1 组件定义
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# 2 配置 source
a1.sources.r1.type = TAILDIR
a1.sources.r1.filegroups = f1
#监控的文件
a1.sources.r1.filegroups.f1 = /home/bigdata/module/apache-flume-1.9.0-bin/app.*
#断点续传记录文件
a1.sources.r1.positionFile = /home/bigdata/module/apache-flume-1.9.0-bin/taildir_position.json
# 3 配置 channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# 4 配置 sink
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.bootstrap.servers = hadoop102:9092,hadoop103:9092,hadoop104:9092
a1.sinks.k1.kafka.topic = first
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.producer.acks = 1
a1.sinks.k1.kafka.producer.linger.ms = 1
# 5 拼接组件
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

先创建断点续传文件 

touch /home/bigdata/module/apache-flume-1.9.0-bin/taildir_position.json

先测试下

bin/flume-ng agent -c conf/ -n a1 -f job/file_to_kafka.conf -Dflume.root.logger=DEBUG,console

启动一个kaka的消费者

./kafka-console-consumer.sh --bootstrap-server hadoop102:9092 --topic first

 然后在监控的app.log追加内存

echo "hello " >> app.log

查看kafka消费者，如图消费到了追加文件的数据

启停脚本

vi flume_file_to_kafka.sh

#!/bin/bash
 
case $1 in
"start"){
        for i in hadoop102
        do
                echo " --------启动 $i 采集flume-------"
                ssh $i "nohup /home/bigdata/module/apache-flume-1.9.0-bin/bin/flume-ng agent --conf /home/bigdata/module/apache-flume-1.9.0-bin/conf --conf-file /home/bigdata/module/apache-flume-1.9.0-bin/job/file_to_kafka.conf --name a1 -Dflume.root.logger=INFO,console >/dev/null 2>&1 &"
        done
};; 
"stop"){
        for i in hadoop102
        do
                echo " --------停止 $i 采集flume-------"
                ssh $i "ps -ef | grep file_to_kafka.conf | grep -v grep |awk  '{print \$2}' | xargs -n1 kill -9 "
        done
 
};;
esac

chmod 744 flume_file_to_kafka.sh

上面是文件到kafka

kafka到控制台

现在配置kafka到控制台

 vi kafka_to_file.conf

# 1 组件定义
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# 2 配置 source
a1.sources.r1.type = org.apache.flume.source.kafka.KafkaSource
a1.sources.r1.batchSize = 50
a1.sources.r1.batchDurationMillis = 200
a1.sources.r1.kafka.bootstrap.servers = hadoop102:9092
a1.sources.r1.kafka.topics = first
a1.sources.r1.kafka.consumer.group.id = custom.g.id
# 3 配置 channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# 4 配置 sink
a1.sinks.k1.type = logger
# 5 拼接组件
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

然后下面的操作是和上面执行操作一样这里就不在重复讲解

Flink

flink生产者

pom.xml

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>1.13.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.12</artifactId>
            <version>1.13.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.12</artifactId>
            <version>1.13.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_2.12</artifactId>
            <version>1.13.0</version>
        </dependency>

log4j.properties

log4j.rootLogger=error, stdout,R
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-ddHH:mm:ss,SSS} %5p --- [%50t] %-80c(line:%5L) : %m%n
log4j.appender.R=org.apache.log4j.RollingFileAppender
log4j.appender.R.File=../log/agent.log
log4j.appender.R.MaxFileSize=1024KB
log4j.appender.R.MaxBackupIndex=1
log4j.appender.R.layout=org.apache.log4j.PatternLayout
log4j.appender.R.layout.ConversionPattern=%d{yyyy-MM-ddHH:mm:ss,SSS} %5p --- [%50t] %-80c(line:%6L) : %m%n

public class FlinkProducer {
    public static void main(String[] args) throws Exception {
        // 0 初始化 flink 环境
        StreamExecutionEnvironment env =
                StreamExecutionEnvironment.getExecutionEnvironment();
        //和主题的分区一致
        env.setParallelism(7);
        // 1 读取集合中数据
        ArrayList<String> wordsList = new ArrayList<>();
        wordsList.add("hello");
        wordsList.add("world");
        DataStream<String> stream = env.fromCollection(wordsList);
        // 2 kafka 生产者配置信息
        Properties properties = new Properties();
        properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "hadoop102:9092");
        // 3 创建 kafka 生产者
        FlinkKafkaProducer kafkaProducer = new FlinkKafkaProducer<String>(
                "first",
                new SimpleStringSchema(),
                properties
        );
        // 4 生产者和 flink 流关联
        stream.addSink(kafkaProducer);
        // 5 执行
        env.execute();
    }
}

flink消费者

public class FlinkConsumer {
    public static void main(String[] args) throws Exception {

        // 0 初始化 flink 环境
        StreamExecutionEnvironment env =
                StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(7);
        // 1 kafka 消费者配置信息
        Properties properties = new Properties();
        properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
                "hadoop102:9092");
        // 2 创建 kafka 消费者
        FlinkKafkaConsumer<String> kafkaConsumer = new FlinkKafkaConsumer<>(
                "first",
                new SimpleStringSchema(),
                properties
        );
        // 3 消费者和 flink 流关联
        env.addSource(kafkaConsumer).print();
        // 4 执行
        env.execute();
    }
}

生产者测试

 ./kafka-console-producer.sh --bootstrap-server hadoop102:9092 --topic first