林子雨大数据实验八Flink部分代码

1.pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>dblab</groupId>
    <artifactId>FlinkWordCount</artifactId>
    <version>1.0-SNAPSHOT</version>
    <dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>1.9.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.11</artifactId>
            <version>1.9.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.11</artifactId>
            <version>1.9.1</version>
        </dependency>
    </dependencies>

2.WordCountData.java

package cn.edu.xmu;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;

public class WordCountData {
    public static final String[] WORDS=new String[]{"To be, or not to be,--that is the question:--", "Whether \'tis nobler in the mind to suffer", "The slings and arrows of outrageous fortune", "Or to take arms against a sea of troubles,", "And by opposing end them?--To die,--to sleep,--", "No more; and by a sleep to say we end", "The heartache, and the thousand natural shocks", "That flesh is heir to,--\'tis a consummation", "Devoutly to be wish\'d. To die,--to sleep;--", "To sleep! perchance to dream:--ay, there\'s the rub;", "For in that sleep of death what dreams may come,", "When we have shuffled off this mortal coil,", "Must give us pause: there\'s the respect", "That makes  calamity  of  so  long  life;",  "For who would  bear the whips and  scorns  of time,",  "The oppressor\'s  wrong,  the  proud  man\'s  contumely,",  "The  pangs  of  despis\'d  love,  the  law\'s delay,", "The insolence of office, and the spurns", "That patient merit of the unworthy takes,", "When he himself might his quietus make", "With a bare bodkin? who would these fardels bear,", "To grunt and sweat under a weary life,", "But that the dread of something after death,--", "The undiscover\'d country, from whose bourn", "No traveller returns,--puzzles the will,", "And makes us rather bear those ills we have", "Than fly to others that we know not of?", "Thus conscience does make cowards of us all;", "And thus the native hue of resolution", "Is sicklied o\'er with the pale cast of thought;",  "And enterprises of great  pith and  moment,",  "With this  regard, their currents turn awry,", "And lose the name of action.--Soft you now!", "The fair Ophelia!--Nymph, in thy orisons", "Be all my sins remember\'d."};
    public WordCountData() {
    }
    public static DataSet<String> getDefaultTextLineDataset(ExecutionEnvironment env){                   return env.fromElements(WORDS);
    }
}

3.第二个文件WordCountTokenizer.java

package cn.edu.xmu;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;

public class WordCountTokenizer implements FlatMapFunction<String, Tuple2<String,Integer>>{         public WordCountTokenizer(){}
    public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws     Exception { 
    String[] tokens = value.toLowerCase().split("\\W+");
    int len = tokens.length;
    for(int i = 0; i<len;i++){
        String tmp = tokens[i];
        if(tmp.length()>0){
            out.collect(new Tuple2<String, Integer>(tmp,Integer.valueOf(1)));
             }
        }
    }
}

4.第三个文件WordCount.java

package cn.edu.xmu;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.AggregateOperator;
import org.apache.flink.api.java.utils.ParameterTool;

public class WordCount {
        public WordCount(){}
        public static void main(String[] args) throws Exception {
        ParameterTool params = ParameterTool.fromArgs(args);
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();         env.getConfig().setGlobalJobParameters(params);
        Object text;
        //如果没有指定输入路径， 则默认使用 WordCountData 中提供的数据         if(params.has("input")){
        text = env.readTextFile(params.get("input"));
    }else{
        System.out.println("Executing WordCount example with default input data set.");         System.out.println("Use -- input to specify file input.");
        text = WordCountData.getDefaultTextLineDataset(env);
}
        AggregateOperator                   counts                   =                   ((DataSet)text).flatMap(new
        WordCountTokenizer()).groupBy(new int[]{0}).sum(1);
//如果没有指定输出， 则默认打印到控制台
        if(params.has("output")){
        counts.writeAsCsv(params.get("output"),"\n", " ");
        env.execute();
    }else{
        System.out.println("Printing  result  to  stdout.  Use  --output  to  specify  outputpath.");
    counts.print();
    }
    }
}

5.数据流词频统计

wordcount,java

package cn.edu.xmu;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.util.Collector;

public class WordCount {
    public static void main(String[] args) throws Exception {
//定义 socket 的端口号
    int port;
    try {
        ParameterTool parameterTool = ParameterTool.fromArgs(args);
        port = parameterTool.getInt("port");
    } catch (Exception e) {
    System.err.println("指定 port 参数，默认值为 9000");
    port = 9000;
}

//获取运行环境
        StreamExecutionEnvironment   env          =
StreamExecutionEnvironment.getExecutionEnvironment();

//连接 socket 获取输入的数据
        DataStreamSource<String> text = env.socketTextStream("127.0.0.1", port, "\n");

//计算数据
        DataStream<WordWithCount>              windowCount              =              text.flatMap(new
FlatMapFunction<String, WordWithCount>() {
        public   void   flatMap(String   value,    Collector<WordWithCount>    out)   throws
Exception {
        String[] splits = value.split("\\s");
        for (String word : splits) {
            out.collect(new WordWithCount(word, 1L));
            }
        }
    })//打平操作，把每行的单词转为<word,count>类型的数据
        .keyBy("word")//针对相同的 word 数据进行分组
        .timeWindow(Time.seconds(2),  Time.seconds(1))//指定计算数据的窗口大 小和滑动窗口大小
        .sum("count");

//把数据打印到控制台
    windowCount.print()
    .setParallelism(1);//使用一个并行度
//注意： 因为 flink 是懒加载的， 所以必须调用execute 方法，上面的代码才会执行             env.execute("streaming word count");
}

/**
*  主要为了存储单词以及单词出现的次数
*/
public static class WordWithCount {
        public String word;
        public long count;

        public WordWithCount() {
}

        public WordWithCount(String word, long count) {
        this.word = word;
        this.count = count;
}

        @Override
        public String toString() {
        return "WordWithCount{" +
        "word='" + word + '\'' +
        ", count=" + count +
        '}';
    }
    }
}