Flink读取Netty数据示例代码

冰玉翔龙

已于 2022-07-09 17:04:25 修改

阅读量710

点赞数

分类专栏： Java ETL 文章标签： flink 大数据 netty

于 2022-06-29 17:23:17 首次发布

本文链接：https://blog.csdn.net/sjy_2010/article/details/125525846

版权

Java 同时被 2 个专栏收录

86 篇文章 0 订阅

订阅专栏

ETL

2 篇文章 0 订阅

订阅专栏

本示例代码记录每张表数据的爬取进度

每张表实时ID
每张表实时爬取数量
记录每张表记录总数

import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;

import java.io.Serializable;

/**
 * 爬取的当前信息
 *
 * @author songjianyong
 */
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class SpiderCurrentInfo implements Serializable {
    private static final long serialVersionUID = 1L;

    /**
     * 表名
     */
    private String table;

    /**
     * 当前爬取完的最大主键
     */
    private Integer primaryKey;

    /**
     * 表记录总数或需要爬取的总数
     */
    private Integer total;

    /**
     * 已爬取完的记录总数
     */
    private Integer finishNum;

    /**
     * 是否爬取完毕
     */
    private Boolean finish;
}

一、引入依赖

            <!--flink核心包-->
            <dependency>
                <groupId>org.apache.flink</groupId>
                <artifactId>flink-java</artifactId>
                <version>1.15.0</version>
            </dependency>
            <!--flink流处理包-->
            <dependency>
                <groupId>org.apache.flink</groupId>
                <artifactId>flink-streaming-java</artifactId>
                <version>1.15.0</version>
            </dependency>
            <dependency>
                <groupId>org.apache.flink</groupId>
                <artifactId>flink-clients</artifactId>
                <version>1.15.0</version>
            </dependency>

二、Netty服务端代码

    Map<String, Channel> CHANNEL_MAP = new HashMap<>(4);    

    /**
     * 绑定端口服务
     */
    private void bindNetty() {
        Thread nettySpiderProgressRate = new Thread(() -> {
            try {
                bind();
            } catch (InterruptedException e) {
                log.error(e.getMessage(), e);
                throw new RuntimeException(e);
            }
        });
        nettySpiderProgressRate.setName("netty-spider-progress-rate");
        nettySpiderProgressRate.start();
    }

    void bind() throws InterruptedException {
        /* 配置服务端的NIO线程组 */
        EventLoopGroup bossGroup = new NioEventLoopGroup();
        EventLoopGroup workerGroup = new NioEventLoopGroup();

        try {
            ServerBootstrap b = new ServerBootstrap();

            b.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class).option(ChannelOption.SO_BACKLOG, 1024)
                    .childHandler(new ChildChannelHandler());

            ChannelFuture f = b.bind(bindPort).sync();
            f.channel().closeFuture().sync();
        } finally {
            bossGroup.shutdownGracefully();
            workerGroup.shutdownGracefully();
        }

    }

    static class ChildChannelHandler extends ChannelInitializer<SocketChannel> {

        @Override
        protected void initChannel(SocketChannel ch) {

            ChannelPipeline channelPipeline = ch.pipeline();
            if (SystemUtils.IS_OS_WINDOWS) {
                channelPipeline.addLast(new LoggingHandler(LogLevel.INFO));
            }
            channelPipeline.addLast(new ReadTimeoutHandler(Long.MAX_VALUE, TimeUnit.SECONDS));
            channelPipeline.addLast(new WriteTimeoutHandler(Long.MAX_VALUE, TimeUnit.SECONDS));
            channelPipeline.addLast(new LineBasedFrameDecoder(1024));
            channelPipeline.addLast(new StringDecoder(StandardCharsets.UTF_8));
            channelPipeline.addLast(new SpiderServerHandler());

        }
    }

    static class SpiderServerHandler extends ChannelInboundHandlerAdapter {
        @Override
        public void channelActive(ChannelHandlerContext ctx) {
            String id = ctx.channel().id().asLongText();
            CHANNEL_MAP.put(id, ctx.channel());
            log.warn("收到连接：{}", id);
        }

        @Override
        public void exceptionCaught(ChannelHandlerContext ctx, Throwable e) {
            log.error(e.getMessage(), e);
        }
    }

注意：netty发现30秒后未读取到数据会自动断开Flink的Socket连接，此处默认时间调整为：

Long.MAX_VALUE, TimeUnit.SECONDS

另一种方式就是设置.option(ChannelOption.SO_KEEPALIVE, true)

三、Flink监听代码

    void flink() {
        StreamExecutionEnvironment streamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> textStream = streamExecutionEnvironment.socketTextStream("localhost", bindPort, System.lineSeparator());

        SingleOutputStreamOperator<SpiderCurrentInfo> operator = textStream.flatMap(new SpiderFlatMap());

        SingleOutputStreamOperator<SpiderCurrentInfo> reduce = operator.keyBy((KeySelector<SpiderCurrentInfo, String>) SpiderCurrentInfo::getTable).
                reduce(new SpiderReduce());

        reduce.addSink(new SpiderSink());

        Thread progressRate = new Thread(() -> {
            try {
                streamExecutionEnvironment.execute("spider progress rate");
            } catch (Exception e) {
                log.error(e.getMessage(), e);
                throw new RuntimeException(e);
            }
        });
        progressRate.setName("fink-spider-progress-rate");
        progressRate.start();
    }

1、Flink之Reduce处理示例代码

import org.apache.flink.annotation.Public;
import org.apache.flink.api.common.functions.ReduceFunction;

import java.util.Objects;

/**
 * 爬取评分进度Reduce处理
 *
 * @author songjianyong
 */
@Public
public class SpiderReduce implements ReduceFunction<SpiderCurrentInfo> {
    private static final long serialVersionUID = 1L;

    @Override
    public SpiderCurrentInfo reduce(SpiderCurrentInfo v1, SpiderCurrentInfo v2) {
        boolean finish = Objects.equals(v1.getFinish(), true) || Objects.equals(v2.getFinish(), true);
        return SpiderCurrentInfo.builder()
                .table(v1.getTable())
                .primaryKey(v1.getPrimaryKey() >= v2.getPrimaryKey() ? v1.getPrimaryKey() : v2.getPrimaryKey())
                .total(v1.getTotal())
                .finish(finish)
                .finishNum(v1.getFinishNum() >= v2.getFinishNum() ? v1.getFinishNum() + 1 : v2.getFinishNum() + 1)
                .build();
    }
}

2、Flink自定义数据扁平化处理示例代码

import com.alibaba.fastjson.JSON;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple4;
import org.apache.flink.util.Collector;

/**
 * 数据扁平化处理
 *
 * @author songjianyong
 */
public class SpiderFlatMap implements FlatMapFunction<String, SpiderCurrentInfo> {
    private static final long serialVersionUID = 1L;

    @Override
    public void flatMap(String value, Collector<SpiderCurrentInfo> out) {
        SpiderCurrentInfo info = JSON.parseObject(value, SpiderCurrentInfo.class);
        info.setFinishNum(0);
        out.collect(info);
    }
}

3、Flink数据下沉落地示例代码

import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.apache.flink.annotation.Public;
import org.apache.flink.api.java.tuple.Tuple4;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Objects;

import static com.song.spider.imdb.constant.Constants.ALL_SPIDER_PROCESS;
import static com.song.spider.imdb.service.impl.BaseService.TMP_DIR;

/**
 * 爬取进程沉淀
 *
 * @author songjianyong
 */
@Slf4j
@Public
public class SpiderSink implements SinkFunction<SpiderCurrentInfo> {
    private static final long serialVersionUID = 1L;

    @Override
    public void invoke(SpiderCurrentInfo value, Context context) {
        ALL_SPIDER_PROCESS.put(value.getTable(), Triple.of(value.getPrimaryKey(), value.getTotal(), value.getFinishNum()));

        Path path = Paths.get(TMP_DIR, value.getTable());
        try {
            if (Objects.equals(true, value.getFinish())) {
                ALL_SPIDER_PROCESS.remove(value.getTable());
                Files.deleteIfExists(path);
                return;
            }
            Files.write(path, value.getPrimaryKey().toString().getBytes(StandardCharsets.UTF_8));
        } catch (IOException e) {
            log.error(e.getMessage(), e);
            throw new RuntimeException(e);
        }
    }
}

四、整合启动

    @Override
    public void run(String... args) {
        bindNetty();
        flink();
    }

启动netty在前，Flink监听启动在后！

netty写入数据示例代码

    /**
     * 发送数据到Flink
     *
     * @param info 发送信息
     */
    protected void sendToFlink(SpiderCurrentInfo info) {
        String v = JSON.toJSONString(info);
        for (Map.Entry<String, Channel> entry : CHANNEL_MAP.entrySet()) {
            Channel channel = entry.getValue();
            channel.writeAndFlush(Unpooled.copiedBuffer((v + System.lineSeparator()).getBytes(StandardCharsets.UTF_8)));
        }
    }

Flink简介https://www.kancloud.cn/zhangpn/flink/1717796