java Flink（四十一）Flink+avro+广播流broadcast实现流量的清洗_maven flink-avro

        <version>${flink.version}</version>
        <scope>provided</scope>
    </dependency>
    <dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpclient</artifactId>
        <version>4.5.10</version>
    </dependency>
    <dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>fastjson</artifactId>
        <version>1.2.8</version>
    </dependency>
    <dependency>
        <groupId>org.apache.avro</groupId>
        <artifactId>avro</artifactId>
        <version>1.9.2</version>
    </dependency>
    <dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpcore</artifactId>
        <version>4.4.1</version>
    </dependency>
    <dependency>
        <groupId>org.projectlombok</groupId>
        <artifactId>lombok</artifactId>
        <version>1.18.16</version>
    </dependency>
    <dependency>
        <groupId>org.projectlombok</groupId>
        <artifactId>lombok</artifactId>
        <version>1.18.16</version>
        <scope>compile</scope>
    </dependency>
</dependencies>

<build>
    <plugins>
        <plugin>
            <groupId>org.apache.avro</groupId>
            <artifactId>avro-maven-plugin</artifactId>
            <version>1.9.2</version>
            <executions>
                <execution>
                    <phase>generate-sources</phase>
                    <goals>
                        <goal>schema</goal>
                    </goals>
                    <configuration>
                        <sourceDirectory>${project.basedir}/src/main/resources/</sourceDirectory>
                        <outputDirectory>${project.basedir}/src/main/java/com/msxf</outputDirectory>
                    </configuration>
                </execution>
            </executions>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.1</version>
            <configuration>
                <source>${jdk.version}</source>
                <target>${jdk.version}</target>
                <encoding>${project.build.sourceEncoding}</encoding>
            </configuration>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>3.1.1</version>
            <executions>
                <execution>
                    <phase>package</phase>
                    <goals>
                        <goal>shade</goal>
                    </goals>
                    <configuration>
                        <finalName>${jar.name}</finalName>
                        <artifactSet>
                            <excludes>
                                <exclude>com.google.code.findbugs:jsr305</exclude>
                                <exclude>org.slf4j:*</exclude>
                                <exclude>log4j:*</exclude>
                                <exclude>org.glassfish.jersey.core:jersey-common</exclude>
                            </excludes>
                        </artifactSet>
                        <relocations>
                            <relocation>
                                <pattern>com.google.common</pattern>
                                <shadedPattern>com.shade.google.common</shadedPattern>
                            </relocation>
                            <relocation>
                                <pattern>org.apache.kafka</pattern>
                                <shadedPattern>org.shade.apache.kafka</shadedPattern>
                            </relocation>
                        </relocations>
                        <filters>
                            <filter>
                                <artifact>*</artifact>
                                <includes>
                                    <include>org/apache/htrace/**</include>
                                    <include>org/apache/avro/**</include>
                                    <include>com/msxf/**</include>
                                    <include>org/apache/flink/streaming/**</include>
                                    <include>org/apache/flink/connector/**</include>
                                    <include>org/apache/kafka/**</include>
                                    <include>org/apache/hive/**</include>
                                    <include>org/apache/hadoop/hive/**</include>
                                    <include>org/apache/curator/**</include>
                                    <include>org/apache/zookeeper/**</include>
                                    <include>org/apache/jute/**</include>
                                    <include>org/apache/thrift/**</include>
                                    <include>org/apache/http/**</include>
                                    <include>org/I0Itec/**</include>
                                    <include>jline/**</include>
                                    <include>com/yammer/**</include>
                                    <include>kafka/**</include>
                                    <include>org/apache/hadoop/hbase/**</include>
                                    <include>com/alibaba/fastjson/**</include>
                                    <include>org/elasticsearch/action/**</include>
                                    <include>io/confluent/**</include>
                                    <include>com/fasterxml/**</include>
                                    <include>org/elasticsearch/**</include>
                                    <include>hbase-default.xml</include>
                                    <include>hbase-site.xml</include>
                                </includes>
                            </filter>
                            <filter>
                                <artifact>org.apache.hadoop.hive.*:*</artifact>
                                <excludes>
                                    <exclude></exclude>
                                    <exclude></exclude>
                                    <exclude></exclude>
                                </excludes>
                            </filter>
                        </filters>
                    </configuration>
                </execution>
            </executions>
        </plugin>
    </plugins>
</build>

```

2、模拟元数据流

正常数据流应该通过其他方式（比如访问数据库、KAFKA流）获取，本次我们直接自定义source

package source;

import org.apache.flink.streaming.api.functions.source.RichSourceFunction;


public class StringSource extends RichSourceFunction<String> {

    Boolean running = true;
    @Override
    public void run(SourceContext ctx) throws Exception {
        while (running){
            String value = String.format("{\"data\":[{\"name\":\"id\",\"comment\":\"ID\"}" +
                    ",{\"name\":\"age\",\"comment\":\"年龄\"}" +
                    ",{\"name\":\"sex\",\"comment\":\"性别\"}]}");
            ctx.collect(value);
            running=false;
        }
    }

    @Override
    public void cancel() {
        running=false;
    }
}

3、模拟流量流

模拟流量数据，本身也是通过KAFKA获取实时流量数据，本文是简单Demo，所以也通过自定义Source获取

package source;

import org.apache.flink.streaming.api.functions.source.RichSourceFunction;

import java.util.Arrays;
import java.util.List;
import java.util.Random;


public class FlowSourceFunction extends RichSourceFunction<String> {

    Boolean running=true;

    private final List<String> USERS = Arrays.asList("张三","李四","牛二");

    private final List<String> BEHAVIOR = Arrays.asList("login", "out", "delete");

    private final List<String> SEX = Arrays.asList("男", "女");

    Random random = new Random();

    @Override
    public void run(SourceContext<String> ctx) throws Exception {
        while (running){
            String id = USERS.get(random.nextInt(USERS.size()));
            String age = String.valueOf(random.nextInt(100));
            String sex = SEX.get(random.nextInt(SEX.size()));
            String time = String.valueOf(System.currentTimeMillis());
            String res = String.format("{\"id\":\"%s\"," +
                    "\"age\":\"%s\"," +
                    "\"sex\":\"%s\"," +
                    "\"time\":\"%s\"}",id,age,sex,time);
            ctx.collect(res);
            Thread.sleep(1000);
        }
    }

    @Override
    public void cancel() {
        running=false;
    }
}

4、MapFunction处理元数据信息

处理元数据信息

package func;

import bean.SchemaInfo;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.apache.avro.Schema;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.avro.SchemaBuilder;
import org.apache.flink.util.StringUtils;

import java.util.HashSet;
import java.util.stream.Stream;


public class MetaDataMapFunction extends RichMapFunction<String,SchemaInfo> {

    private String db;
    private String table;

    private final String NAME="name";

    public MetaDataMapFunction(String db, String table) {
        this.db = db;
        this.table = table;
    }

    @Override
    public SchemaInfo map(String value) throws Exception {
        String[] aliases = {db.concat(".").concat(table)};
        //存储fields
        HashSet<Object> fieldsSet = new HashSet<>();
        //schema支持多种类型，这里我们选择构建常见的record类型
        //初始化结果:
        //{"type":"record","name":"flow","namespace":"com.flow","doc":"fow_event","fields":[],"aliases":["db.table"]}
        SchemaBuilder.RecordBuilder<Schema> recordBuilder = SchemaBuilder.record("flow").namespace("com.flow").aliases(aliases).doc("fow_event");
        //初始化之后要完善schema中的fields,首先获取fields
        SchemaBuilder.FieldAssembler<Schema> fields = recordBuilder.fields();
        //处理元数据流
        JSONObject obj = JSON.parseObject(value);
        //过滤掉不包含ID的数据
        Stream<JSONObject> data= obj.getJSONArray("data").stream().filter(
                o -> !StringUtils.isNullOrWhitespaceOnly(JSON.parseObject(o.toString()).getOrDefault(NAME, "").toString())
        ).map(o->JSONObject.parseObject(o.toString()));
        data.forEach(
                o->{
                    String name = o.get("name").toString();
                    String comment = o.get("comment").toString();
                    buildFields(fields,name,comment);
                    fieldsSet.add(name);
                }
        );
        Schema schema = fields.endRecord();
        return new SchemaInfo(schema.toString(),fieldsSet);
    }

    public void buildFields(SchemaBuilder.FieldAssembler<Schema> fields,String name,String comment){
        fields.name(name)//字段名称
                .doc(comment)//描述内容、注释
                .orderAscending()//排序方式无
                .type()//类型
                .optional()
                .stringType();
    }
}

定义返回对象

package bean;

import lombok.AllArgsConstructor;
import lombok.Data;

import java.util.HashSet;



@Data
@AllArgsConstructor
public class SchemaInfo {
    public String info;

    public HashSet set;

}

5、延迟处理主流（流量流）

package func;

import com.alibaba.fastjson.JSON;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.configuration.Configuration;

/**
 * 
 * 1、延时map的初始化，阻塞主流数据，等待广播流schema写入广播状态，保证主流数据可以获取到schema
 * 2、标准化数据
 */
public class DelayEtlMap extends RichMapFunction<String,String> {

    private final long delayTime;

    public DelayEtlMap(long delayTime) {
        this.delayTime = delayTime;
    }

    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
        Thread.sleep(delayTime);
    }

    @Override
    public String map(String value) throws Exception {
        return value;
    }
}

6、双流关联，处理数据

package func;

import bean.FlowData;
import bean.SchemaInfo;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.apache.flink.api.common.state.BroadcastState;
import org.apache.flink.api.common.state.MapStateDescriptor;
import org.apache.flink.api.common.state.ReadOnlyBroadcastState;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction;
import org.apache.flink.util.Collector;

import java.text.SimpleDateFormat;
import java.util.HashMap;

/**
 * 
 * processElement 处理业务流数据
 * processBroadcastElement 处理广播流数据
 */
public class FlowBrodCastFunction extends BroadcastProcessFunction<String, SchemaInfo, FlowData> {

    @Override
    public void processElement(String value, BroadcastProcessFunction<String, SchemaInfo, FlowData>.ReadOnlyContext ctx, Collector<FlowData> out) throws Exception {
        ReadOnlyBroadcastState<String, SchemaInfo> flowMetaData = ctx.getBroadcastState(new MapStateDescriptor<String, SchemaInfo>(
                "flowMetaData"
                , TypeInformation.of(String.class)
                , TypeInformation.of(SchemaInfo.class)));
        SchemaInfo schema = flowMetaData.get("schema");
        JSONObject jsonObject = JSON.parseObject(value);
        Long time = jsonObject.getLong("time");
        //创建时间格式
        SimpleDateFormat yyyyMMdd = new SimpleDateFormat("yyyyMMdd");
        String dt = yyyyMMdd.format(time);
        //获取元数据字段
        HashMap<String, String> map = new HashMap<>();
        schema.getSet().forEach(
                o->{
                    map.put(o.toString(),jsonObject.get(o).toString());
                }
        );
        out.collect(new FlowData(schema.getInfo(),map,time));
    }

    @Override
    public void processBroadcastElement(SchemaInfo value, BroadcastProcessFunction<String, SchemaInfo, FlowData>.Context ctx, Collector<FlowData> out) throws Exception {


# 总结

无论是哪家公司，都很重视高并发高可用的技术，重视基础，重视JVM。面试是一个双向选择的过程，不要抱着畏惧的心态去面试，不利于自己的发挥。同时看中的应该不止薪资，还要看你是不是真的喜欢这家公司，是不是能真的得到锻炼。其实我写了这么多，只是我自己的总结，并不一定适用于所有人，相信经过一些面试，大家都会有这些感触。

最后我整理了一些面试真题资料，技术知识点剖析教程，还有和广大同仁一起交流学习共同进步，还有一些职业经验的分享。

![面试了阿里，滴滴，网易，蚂蚁，最终有幸去了网易【面试题分享】](https://img-blog.csdnimg.cn/img_convert/c7255ce10c87bb025a10cab15954a536.webp?x-oss-process=image/format,png)

oid processBroadcastElement(SchemaInfo value, BroadcastProcessFunction<String, SchemaInfo, FlowData>.Context ctx, Collector<FlowData> out) throws Exception {


# 总结

无论是哪家公司，都很重视高并发高可用的技术，重视基础，重视JVM。面试是一个双向选择的过程，不要抱着畏惧的心态去面试，不利于自己的发挥。同时看中的应该不止薪资，还要看你是不是真的喜欢这家公司，是不是能真的得到锻炼。其实我写了这么多，只是我自己的总结，并不一定适用于所有人，相信经过一些面试，大家都会有这些感触。

最后我整理了一些面试真题资料，技术知识点剖析教程，还有和广大同仁一起交流学习共同进步，还有一些职业经验的分享。

[外链图片转存中...(img-PnQzs8Lo-1718732685118)]