Flink算子(ProcessFunction，map和Flatmap)

Flink提供三层API，每个API在简洁性和表达之间提供不同的权衡，并针对不同的用例
SQL/Table API(dynamic tables)
DataStream API(streams,windows)
ProcessFunction(event,state,time)

 

不要跟ProcessWindowFunction混为一谈
ProcessFunction是一个低阶的流处理操作，它可以访问流处理程序的基础构建模块：
事件（event）（流元素）
状态（state）（容错性，一致性，仅在keyed stream中）
定时器（times）（event time和processing time，仅在keyed stream中）

ProcessFunction可以看做是一个具有keyed state和timers访问权的FlatMapFunction
通过FuntimeContext访问keyed state

计时器允许应用程序对处理时间和事件时间中的更改做出响应。对processElement(...)函数每次
调用都获得一个Context对象，该对象可以访问元素的event time timestamp和TimerService

TimerService可用于将来的event/process time 瞬间注册回调。当达到计时器的特定时间时，将调用onTimer(...)方法。在该调用期间，所有状态都再次限定
在创建计时器时使用的键的范围内，从而允许计时器操作键控状态

简单来说：ProcessFunction可以看做是一个具有keyed state和timers访问权限的FlatMapFunction，是一个低阶的流处理操作算子；需要说明的是：ProcessFunction不同于
windowFunction和ProcessWindowFunction,后2者属于DateStream Api的范畴，大家使用的时候可以注意一下：ProcessFunction类可以重谢Open和Close方法。

ProcessFunction：

package Flink_API;

import com.alibaba.fastjson.JSON;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010;
import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
import org.apache.flink.util.Collector;

import java.io.Serializable;
import java.util.Properties;

public class TestProcessFunction {
    public static void main(String[] rags) throws Exception {
        //获取Flink的运行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.setParallelism(1);

        //读取用户浏览信息
        Properties consumerProperties=new Properties();
        consumerProperties.setProperty("bootstrap.servers","page01");
        consumerProperties.setProperty("groud.id","browsegroup");

        DataStreamSource<String> dataStreamSource=env.addSource(new FlinkKafkaConsumer010<String>("topic",new SimpleStringSchema(),consumerProperties));

        //解析数据
        DataStream<UserBrowseLog> processData=dataStreamSource.process(new ProcessFunction<String, UserBrowseLog>() {
            @Override
            public void processElement(String s, Context context, Collector<UserBrowseLog> collector) throws Exception {
                try{
                    UserBrowseLog browseLog= JSON.parseObject(s,UserBrowseLog.class);
                    if(browseLog!=null){
                        collector.collect(browseLog);
                    }

                }catch(Exception e){
                    System.out.print("解析Json_UserBrowseLog异常，异常信息是："+e.getMessage());

                }
            }
        });
        processData.print();
        env.execute("TestProcessFunction");

    }
    public static class UserBrowseLog implements Serializable {
        private String userID;
        private String eventTime;
        private String eventType;
        private String productID;
        private Integer productPrice;

        public String getUserID() {
            return userID;
        }

        public void setUserID(String userID) {
            this.userID = userID;
        }

        public String getEventTime() {
            return eventTime;
        }

        public void setEventTime(String eventTime) {
            this.eventTime = eventTime;
        }

        public String getEventType() {
            return eventType;
        }

        public void setEventType(String eventType) {
            this.eventType = eventType;
        }

        public String getProductID() {
            return productID;
        }

        public void setProductID(String productID) {
            this.productID = productID;
        }

        public Integer getProductPrice() {
            return productPrice;
        }

        public void setProductPrice(Integer productPrice) {
            this.productPrice = productPrice;
        }

        @Override
        public String toString() {
            return "UserBrowseLog{" +
                    "userID='" + userID + '\'' +
                    ", eventTime='" + eventTime + '\'' +
                    ", eventType='" + eventType + '\'' +
                    ", productID='" + productID + '\'' +
                    ", productPrice=" + productPrice +
                    '}';
        }
    }
}

数据处理的核心就是对数据进行各种转化Transformation操作，在Flink上就是通过转换将一个或多个DataStream转换成新的DataStream

 

Map算子：输入一个元素，然后返回一个元素，中间可以进行清洗转换等操作，一对一转换，即一条转换成另外一条 

 

package Flink_API;

import com.alibaba.fastjson.JSON;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010;
import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
import org.apache.flink.util.Collector;

import java.io.Serializable;
import java.util.Properties;

public class TestMap {
        public static void main(String[] rags) throws Exception {
            //获取Flink的运行环境
            StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
            env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
            env.setParallelism(1);

            //读取用户浏览信息
            Properties consumerProperties=new Properties();
            consumerProperties.setProperty("bootstrap.servers","page01");
            consumerProperties.setProperty("groud.id","browsegroup");


            DataStreamSource<String> dataStreamSource=env.addSource(new FlinkKafkaConsumer010<String>("topic",new SimpleStringSchema(),consumerProperties));

            //解析数据
            DataStream<UserBrowseLog> processData=dataStreamSource.map(new MapFunction<String, UserBrowseLog>() {

                @Override
                public UserBrowseLog map(String s) throws Exception {
                    UserBrowseLog browseLog=null;
                    try{
                        browseLog= JSON.parseObject(s, UserBrowseLog.class);
                        System.out.print(browseLog==null);

                    }catch(Exception e){
                        System.out.print("解析Json_UserBrowseLog异常，异常信息是："+e.getMessage());
                    }
                    return browseLog;
                }
            });
            processData.print();
            env.execute("TestProcessFunction");

        }
        public static class UserBrowseLog implements Serializable {
            private String userID;
            private String eventTime;
            private String eventType;
            private String productID;
            private Integer productPrice;

            public String getUserID() {
                return userID;
            }

            public void setUserID(String userID) {
                this.userID = userID;
            }

            public String getEventTime() {
                return eventTime;
            }

            public void setEventTime(String eventTime) {
                this.eventTime = eventTime;
            }

            public String getEventType() {
                return eventType;
            }

            public void setEventType(String eventType) {
                this.eventType = eventType;
            }

            public String getProductID() {
                return productID;
            }

            public void setProductID(String productID) {
                this.productID = productID;
            }

            public Integer getProductPrice() {
                return productPrice;
            }

            public void setProductPrice(Integer productPrice) {
                this.productPrice = productPrice;
            }

            @Override
            public String toString() {
                return "UserBrowseLog{" +
                        "userID='" + userID + '\'' +
                        ", eventTime='" + eventTime + '\'' +
                        ", eventType='" + eventType + '\'' +
                        ", productID='" + productID + '\'' +
                        ", productPrice=" + productPrice +
                        '}';
            }
        }

}

FlatMap：输入一个元素，可以返回0个，1个活多个元素，即编程0行或者多行

小知识汇总：

Flink提交方式俩种：测试的时候用第一种，部署的时候用第二种
第一种：先创建一个集群，然后把任务提交到集群（yarn-session.sh表示启动yarn集群，-n 2表示启动俩个taskManager，-jm 1024表示jobmanager内存大小，-tm 1024表示taskmanager内存大小）
yarn-session.sh -n 2 -jm 1024 -tm 1024 -d
flink run -c xxxx.jar
第二种：提交任务，先创建一个临时集群
flink run -m yarn-cluster -yn 2-yjm 1024 -ytm 1024 -c xxxxx.jar

 

package Flink_API;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010;
import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
import org.apache.flink.util.Collector;

import java.io.Serializable;
import java.util.Properties;

public class TestFlatMap {
        public static void main(String[] rags) throws Exception {
            //获取Flink的运行环境
            StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
            env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
            env.setParallelism(1);

            //读取用户浏览信息
            Properties consumerProperties=new Properties();
            consumerProperties.setProperty("bootstrap.servers","page01");
            consumerProperties.setProperty("groud.id","browsegroup");


            DataStreamSource<String> dataStreamSource=env.addSource(new FlinkKafkaConsumer010<String>("topic",new SimpleStringSchema(),consumerProperties));

            //解析数据
            DataStream<Tuple2<String,Integer>> processData=dataStreamSource.flatMap(new FlatMapFunction<String, Tuple2<String,Integer>>() {
                @Override
                public void flatMap(String s, Collector<Tuple2<String,Integer>> collector) throws Exception {
                    String[] split =s.split("\\w+");
                    for(String word:split){
                        collector.collect(Tuple2.of(word,1));
                    }
                }
            }).setParallelism(3);
            processData.print().setParallelism(1);
            env.execute("TestFlatMap");

        }
        public static class UserBrowseLog implements Serializable {
            private String userID;
            private String eventTime;
            private String eventType;
            private String productID;
            private Integer productPrice;

            public String getUserID() {
                return userID;
            }

            public void setUserID(String userID) {
                this.userID = userID;
            }

            public String getEventTime() {
                return eventTime;
            }

            public void setEventTime(String eventTime) {
                this.eventTime = eventTime;
            }

            public String getEventType() {
                return eventType;
            }

            public void setEventType(String eventType) {
                this.eventType = eventType;
            }

            public String getProductID() {
                return productID;
            }

            public void setProductID(String productID) {
                this.productID = productID;
            }

            public Integer getProductPrice() {
                return productPrice;
            }

            public void setProductPrice(Integer productPrice) {
                this.productPrice = productPrice;
            }

            @Override
            public String toString() {
                return "UserBrowseLog{" +
                        "userID='" + userID + '\'' +
                        ", eventTime='" + eventTime + '\'' +
                        ", eventType='" + eventType + '\'' +
                        ", productID='" + productID + '\'' +
                        ", productPrice=" + productPrice +
                        '}';
            }
        }
}