Flink学习之路（三）—— Transform

Transfrom是Flink的转换算子，可以将数据转换成不同的模样，是Flink最底层的API，以下为常用算子，其他算子可以参照官网去测试

官网：https://flink.apache.org/

Transform

Map

package com.dahuan.transform;

import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class Transform_Map {
    public static void main(String[] args) throws Exception{
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism( 1 );

        //从文件读取数据
        String path = "E:\\Project\\FlinkTutorials\\Flink-Scala\\src\\main\\resources\\sensor.txt";
        DataStreamSource<String> stringDataStreamSource = env.readTextFile( path );

        //map 返回数据的长度
        stringDataStreamSource.map( new MapFunction<String, Integer>() {
            @Override
            public Integer map(String value) throws Exception {
                return value.length();

            }
        } ).print();


        env.execute("Transform_Map");

    }
}

FlatMap

package com.dahuan.transform;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

public class Transform_FlatMap {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism( 1 );

        //从文件读取数据
        String path = "E:\\Project\\FlinkTutorials\\Flink-Scala\\src\\main\\resources\\sensor.txt";
        DataStreamSource<String> stringDataStreamSource = env.readTextFile( path );

        //按逗号分隔字段
        stringDataStreamSource.flatMap( new FlatMapFunction<String, String>() {
            @Override
            public void flatMap(String value, Collector<String> out) throws Exception {
                //每个字段的值按逗号分隔
                String[] splits = value.split( "," );
                //使用for循环将splits遍历出来变成split
                for (String split : splits) {
                    //输出
                    out.collect( split );
                }
            }
        } ).print();


        env.execute( "Transform_FlapMap" );
    }

}

KeyBy

package com.dahuan.transform;

import com.dahuan.bean.SensorReading;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class Transform_KeyBy {
    public static void main(String[] args) throws Exception{
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism( 1 );

        //从文件读取数据
        String path = "E:\\Project\\FlinkTutorials\\Flink-Scala\\src\\main\\resources\\sensor.txt";
        DataStreamSource<String> stringDataStreamSource = env.readTextFile( path );

        /**
         * DataStream → KeyedStream：逻辑地将一个流拆分成不相交的分区，每个分
         * 区包含具有相同 key 的元素，在内部以 hash 的形式实现的。
         */

        //将String类型转换成SensorReading类型
        DataStream<SensorReading> dataStream = stringDataStreamSource.map( new MapFunction<String, SensorReading>() {
            @Override
            public SensorReading map(String value) throws Exception {
                //文件中的每一组数据用逗号分割
                String[] split = value.split( "," );
                return new SensorReading(split[0],new Long(split[1]),new Double( split[2] ));
            }
        } );
        //按照id进行分组
        KeyedStream<SensorReading, Tuple> keyedStream = dataStream.keyBy( "id" );
        //取出当前最大的温度值
        SingleOutputStreamOperator<SensorReading> temperature = keyedStream.max( "temperature" );//TODO maxBy直接取出当前最大值的时间戳


        temperature.print();
        env.execute("Transform_KeyBy");
    }
}

Filter

package com.dahuan.transform;

import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class Transform_Filter {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism( 1 );

        //从文件读取数据
        String path = "E:\\Project\\FlinkTutorials\\Flink-Scala\\src\\main\\resources\\sensor.txt";
        DataStreamSource<String> stringDataStreamSource = env.readTextFile( path );

        //筛选sensor_1开头的id对应的数据
        stringDataStreamSource.filter( new FilterFunction<String>() {
            @Override
            public boolean filter(String value) throws Exception {
                //包含sensor_10
                return value.startsWith( "sensor_1" );
            }
        } ).print();

        env.execute( "Transform_Filter" );
    }
}

Reduce

package com.dahuan.transform;

import com.dahuan.bean.SensorReading;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class Transform_Reduce {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism( 1 );

        //从文件读取数据
        String path = "E:\\Project\\FlinkTutorials\\Flink-Scala\\src\\main\\resources\\sensor.txt";
        DataStream<String> inputStream = env.readTextFile( path );

        //使用Java1.8的Lambda表达式将String类型转换为SensorReading类型
        DataStream<SensorReading> mapStream = inputStream.map( data -> {
            String[] splits = data.split( "," );
            return new SensorReading( splits[0], new Long( splits[1] ), new Double( splits[2] ) );
        } );

        //按照id分组
        KeyedStream<SensorReading, Tuple> keyedStream = mapStream.keyBy( "id" );

        //reduce聚合，取最大的温度值，以及当前的时间戳
/*        SingleOutputStreamOperator<SensorReading> reduce = keyedStream.reduce( new ReduceFunction<SensorReading>() {
            @Override
            public SensorReading reduce(SensorReading value1, SensorReading value2) throws Exception {

                return new SensorReading(value1.getId(),value2.getTimestamp(),Math.max( value1.getTemperature(),value2.getTemperature() ));
            }
        } );*/


        //TODO  Lambda表达式
        SingleOutputStreamOperator<SensorReading> reduce = keyedStream.reduce( (curData, newData) -> {

            //curData对应id,newData对应时间戳,保持一个参数对应一个值,使得此值为最新值
             return new SensorReading(curData.getId(),newData.getTimestamp(),
                     //对比两个值的温度值,比较哪两个温度值最高
                     Math.max( curData.getTemperature(),newData.getTemperature() ));



        } );


        reduce.print();
        env.execute( "Transform_Reduce" );
    }
}

MultipleStreams（Split，Select）

package com.dahuan.transform;

import com.dahuan.bean.SensorReading;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SplitStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

import java.util.Collection;
import java.util.Collections;

public class Transform_MultipleStreams {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism( 1 );

        //从文件读取数据
        String path = "E:\\Project\\FlinkTutorials\\Flink-Scala\\src\\main\\resources\\sensor.txt";
        DataStreamSource<String> stringDataStreamSource = env.readTextFile( path );

        //输入类型是String 类型，返回什么类型就输出什么类型
        DataStream<SensorReading> dataStream = stringDataStreamSource.map( new MapFunction<String, SensorReading>() {
            @Override
            public SensorReading map(String value) throws Exception {
                String[] split = value.split( "," );
                return new SensorReading( split[0], new Long( split[1] ), new Double( split[2] ) );
            }
        } );

        //TODO  分流 , 按照温度值30为界分为两条流
        SplitStream<SensorReading> splitStream = dataStream.split( new OutputSelector<SensorReading>() {
            @Override
            public Iterable<String> select(SensorReading sensorReading) {
                /**
                 * 看这个值的温度值是否大于30度,如果大于30 输出 high , 如果小于 30 输出low
                 *                                             返回一个只包含指定对象的不可变列表。
                 */
                return (sensorReading.getTemperature() > 30) ? Collections.singletonList( "high" ) : Collections.singletonList( "low" );
            }
        } );

        DataStream<SensorReading> highTempStream = splitStream.select( "high" );
        DataStream<SensorReading> lowTempStream = splitStream.select( "low" );
        DataStream<SensorReading> allTempStream = splitStream.select( "high", "low" );


        highTempStream.print("high");
        lowTempStream.print("low");
        allTempStream.print("all");

        env.execute("Transform_MultipleStreams");


    }
}

Connect

package com.dahuan.transform;

import com.dahuan.bean.SensorReading;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import org.apache.flink.streaming.api.datastream.*;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.CoMapFunction;

import java.util.Collections;

public class Transform_Connect {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism( 1 );

        //从文件读取数据
        String path = "E:\\Project\\FlinkTutorials\\Flink-Scala\\src\\main\\resources\\sensor.txt";
        DataStreamSource<String> stringDataStreamSource = env.readTextFile( path );

        //输入类型是String 类型，返回什么类型就输出什么类型
        DataStream<SensorReading> dataStream = stringDataStreamSource.map( new MapFunction<String, SensorReading>() {
            @Override
            public SensorReading map(String value) throws Exception {
                String[] split = value.split( "," );
                return new SensorReading( split[0], new Long( split[1] ), new Double( split[2] ) );
            }
        } );

        //TODO  分流 , 按照温度值30为界分为两条流
        SplitStream<SensorReading> splitStream = dataStream.split( new OutputSelector<SensorReading>() {
            @Override
            public Iterable<String> select(SensorReading sensorReading) {
                /**
                 * 看这个值的温度值是否大于30度,如果大于30 输出 high , 如果小于 30 输出low
                 *                                             返回一个只包含指定对象的不可变列表。
                 */
                return (sensorReading.getTemperature() > 30) ? Collections.singletonList( "high" ) : Collections.singletonList( "low" );
            }
        } );

        DataStream<SensorReading> highTempStream = splitStream.select( "high" );
        DataStream<SensorReading> lowTempStream = splitStream.select( "low" );
        DataStream<SensorReading> allTempStream = splitStream.select( "high", "low" );

        /*
        highTempStream.print("high");
        lowTempStream.print("low");
        allTempStream.print("all");*/


        //TODO 合流，connect , 将高温流转换成二元组类型 , 与低温流连接合并之后，输出状态信息
        SingleOutputStreamOperator<Tuple2<String, Double>> waringStream = highTempStream.map( new MapFunction<SensorReading, Tuple2<String, Double>>() {
            @Override
            public Tuple2<String, Double> map(SensorReading value) throws Exception {
                //获取id 和 温度值
                return new Tuple2<>(value.getId(),value.getTemperature());
            }
        } );

        //获取出来之后的高温流 用connect 方法联合低温流 使得低温流的变量与高温流相同
        ConnectedStreams<Tuple2<String, Double>, SensorReading> connectedStreams = waringStream.connect( lowTempStream );

        //CoMapFunction接口是将两个流在map上实现
        DataStream<Object> resultStream  = connectedStreams.map( new CoMapFunction<Tuple2<String, Double>, SensorReading, Object>() {


            @Override
            public Object map1(Tuple2<String, Double> value) throws Exception {
               return new Tuple3<>(value.f0,value.f1,"high temp warning");
            }

            @Override
            public Object map2(SensorReading value) throws Exception {
                return new Tuple2<>(value.getId(),"normal");
            }
        } );

        resultStream.print();



        //union 联合多条流,可以重复
        highTempStream.union( lowTempStream,allTempStream ).print("union");



        env.execute( "Transform_Connect" );


    }
}

END