深入浅出 Flink 02 之 Transformation

1.1 map和filter

/*** 数据源：1 2 3 4 5.....源源不断过来 
* 通过map打印一下接受到数据 
* * 通过filter过滤一下数据，我们只需要偶数
 */ 
 public class MapDemo {
 
  public static void main(String[] args) throws Exception { 
  
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  
  DataStreamSource<Long> numberStream = env.addSource(new MyNoParalleSource()).setParallelism(1); 
  
  SingleOutputStreamOperator<Long> dataStream = numberStream.map(new MapFunction<Long, Long>() {
   @Override public Long map(Long value) throws Exception { 
   
   System.out.println("接受到了数据："+value); return value; } }); 
   
   SingleOutputStreamOperator<Long> filterDataStream = dataStream.filter(new FilterFunction<Long>() { 
   
   @Override public boolean filter(Long number) throws Exception {
   return number % 2 == 0; 
   }
  }
 ); 
 	filterDataStream.print().setParallelism(1); 	
 	env.execute("StreamingDemoWithMyNoPralalleSource");
 	 }
  }

1.2 flatMap，keyBy和sum

/**
* 滑动窗口实现单词计数 
* * 数据源：socket 
* * 需求：每隔1秒计算最近2秒单词出现的次数 
* ** 练习算子：
* flatMap 
* keyBy:
*  dataStream.keyBy("someKey")  // 指定对象中的 "someKey"字段作为分组key 
*  dataStream.keyBy(0) //指定Tuple中的第一个元素作为分组key 
* sum
 */
 public class WindowWordCountJava { 
 public static void main(String[] args) throws Exception {
  int port;
   try{
   ParameterTool parameterTool = ParameterTool.fromArgs(args); 
   port = parameterTool.getInt("port"); 
   }catch (Exception e){ 
   System.err.println("no port set,user default port 9988");
    port=9988; }
    //步骤一：获取flink运行环境（stream） 
    StreamExecutionEnvironment env= StreamExecutionEnvironment.getExecutionEnvironment(); 
    String hostname="10.126.88.226"; 
    String delimiter="\n";
	 //步骤二：获取数据源
  DataStreamSource<String> textStream = env.socketTextStream(hostname, port, delimiter);
   //步骤三：执行transformation操作 
   SingleOutputStreamOperator<WordCount> wordCountStream = textStream.flatMap(new FlatMapFunction<String, WordCount>() {
    public void flatMap(String line, Collector<WordCount> out) throws Exception { 
    String[] fields = line.split("\t"); 
    for (String word : fields) { 
    out.collect(new WordCount(word, 1L)); 
  		  } 
  		  } 
    }).keyBy("word")
 .timeWindow(Time.seconds(2), Time.seconds(1))//每隔1秒计算最近2秒
  .sum("count");
wordCountStream.print().setParallelism(1);//打印并设置并行度 
//步骤四：运行程序 
env.execute("socket word count");
	 }

public static class WordCount{ 
public String word; 
public long count; 
public WordCount(){ }
public WordCount(String word,long count){ 
		this.word=word; 
		this.count=count; 
}
@Override public String toString() {
	 return "WordCount{" + "word='" + word + '\'' + ", count=" + count + '}'; 
		 }
	}
}

1.3 union

/**
  * 合并多个流，新的流会包含所有流中的数据，但是union是一个限制，就是所有合并的	流类型必须是一致 的
*/
public class unionDemo { public static void main(String[] args) throws Exception { 
//获取Flink的运行环境 
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 //获取数据源
  DataStreamSource<Long> text1 = env.addSource(new MyNoParalleSource()).setParallelism(1);//注意：针对此source，并行度只能设置为1 DataStreamSource<Long> text2 = env.addSource(new MyNoParalleSource()).setParallelism(1); //把text1和text2组装到一起 
  DataStream<Long> text = text1.union(text2);
DataStream<Long> num = text.map(new MapFunction<Long, Long>() { 
@Override public Long map(Long value) throws Exception {
 		System.out.println("原始接收到数据：" + value); return value;
 	 }
   }
);
 //每2秒钟处理一次数据 
DataStream<Long> sum = num.timeWindowAll(Time.seconds(2)).sum(0);
//打印结果 
	sum.print().setParallelism(1); 
	String jobName = unionDemo.class.getSimpleName(); 
	env.execute(jobName); 
	} 
}

1.4 connect,conMap和conFlatMap

/**
  * 和union类似，但是只能连接两个流，两个流的数据类型可以不同，会对两个流中的数据应用不同的处理 方法
  */
public class ConnectionDemo { 
public static void main(String[] args) throws Exception { 
//获取Flink的运行环境 
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 
//获取数据源 
DataStreamSource<Long> text1 = env.addSource(new MyNoParalleSource()).setParallelism(1);//注意：针对此source，并行度只能设置为1 DataStreamSource<Long> text2 = env.addSource(new MyNoParalleSource()).setParallelism(1);
SingleOutputStreamOperator<String> text2_str = text2.map(new MapFunction<Long, String>() { 
		@Override public String map(Long value) throws Exception {
			 return "str_" + value;
			 }
	 }
);
ConnectedStreams<Long, String> connectStream = text1.connect(text2_str); 
SingleOutputStreamOperator<Object> result = connectStream.map(new 		CoMapFunction<Long, String, Object>() { 
		@Override public Object map1(Long value) throws Exception { 
		return value; 
}
@Override public Object map2(String value) throws Exception {
		 return value; 
		} 
	}
);
//打印结果 
result.print().setParallelism(1); 
String jobName = ConnectionDemo.class.getSimpleName(); 
env.execute(jobName);