1-flink wordCount
基础入门:正经教程
目标,对一个有界的数据集 dataSet(内容和大小固定,比如一个日志文件)或无界的数据stream(数据流)进行处理,统计每个单词出现的次数
代码实现
.pom
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.example</groupId>
<artifactId>flinkLear</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<flink-kafka.version>1.12.0</flink-kafka.version>
<flink.version>1.12.0</flink.version>
<scala.binary.version>2.12</scala.binary.version>
<scala.version>2.12.7</scala.version>
<slf4j.version>1.7.30</slf4j.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
<version>${flink-kafka.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime-web_2.12</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
<version>2.14.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.2</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.3.0</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
java版
dataSet
package cn.jiayeli.flinkDemos;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.operators.Order;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.DataSource;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;
/**
* 批处理
*/
public class DataSetWordCount {
private static String path = "src/main/resources/words.data";
public static void main(String[] args) throws Exception {
//根据系统环境获取一个dataset 执行环境
ExecutionEnvironment dataSetEnv = ExecutionEnvironment.getExecutionEnvironment();
DataSource<String> stringDataSource = dataSetEnv.readTextFile(path);
stringDataSource
.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
@Override
public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
for (String s : value.split("\\s+")) {
out.collect(new Tuple2<>(s, 1));
}
}
})
/*
// flatmap.map 和上面的flatmap效果一样,看起来上面写法效率更高
.flatMap(new FlatMapFunction<String, String>() {
@Override
public void flatMap(String value, Collector<String> out) throws Exception {
Arrays.stream(value.split("\\s+")).forEach(e -> out.collect(e));
}
})
.map(new MapFunction<String, Tuple2<String, Integer>>() {
@Override
public Tuple2<String, Integer> map(String value) throws Exception {
return new Tuple2<String, Integer>(value, 1);
}
})*/
// dataSet 独有的算子,groupBy
.groupBy(e -> e.f0)
.reduce((e, ee) -> new Tuple2<>(e.f0, e.f1 + ee.f1))
//分区内排序
.sortPartition(e -> e.f1, Order.DESCENDING)
.print();
}
}
dataStream
先启动一个nc,命令
nc -lk 10010
package cn.jiayeli.flinkDemos.wordCount;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
import scala.Tuple2;
import java.util.Arrays;
/**
*流处理
*/
public class StreamDemo {
public static void main(String[] args) throws Exception {
//获取一个流执行环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
DataStream<String> dataStream = env.socketTextStream("localhost", 10010, '\n', 10);
dataStream.print();
dataStream
//过滤 W 开头的行,读的时候按行分割
.filter((FilterFunction<String>) e -> !e.startsWith("W"))
.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
@Override
public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
Arrays.stream(value.split("\\s+")).forEach(e -> out.collect(new Tuple2<>(e, 1)));
}
})
.keyBy(e -> e._1)
.reduce((e, ee) -> new Tuple2<>(e._1, e._2 + ee._2))
.print();
env.execute("streamWordCount");
}
}
scala版
dataSet
import org.apache.flink.api.scala.{AggregateDataSet, ExecutionEnvironment, createTypeInformation}
object WorldCount {
def main(args: Array[String]): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
print(ExecutionEnvironment.getDefaultLocalParallelism)
val wc: AggregateDataSet[(String, Int)] = env.readTextFile("/home/kuro/Documents/workspace/bigdata/flinkLear/src/main/resources/words.data")
.flatMap(_.split("\\s+"))
.map((_, 1))
.groupBy(1)
.sum(1)
wc.print()
}
}
dataStream
同java版 Stream,先开启一个nc 模拟数据流
package wordCount
import org.apache.flink.api.scala.createTypeInformation
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
object StreamDemo {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI()
env.socketTextStream("localhost", 10010)
.filter(!_.startsWith("W"))
.flatMap(_.split("\\s+"))
.map((_, 1))
.keyBy(_._1)
.reduce((e, ee) => (e._1, e._2+ee._2))
.print()
env.execute("streamDemo")
}
}
对比及注意事项
scala 得利于语言上的类型推导等特点,代码写起来比较简洁明了。java也可以使用lambda来进行代码的精简,但是运行会报错,建议不要使用lamba,或者使用lambda时注意运行时类型转换。
从来面代码可以看出flink的基本编程模型为
source -> tranceform --> sink