wordCount就是对一大堆单词进行个数统计,然后排序。从网上找篇英文文章放到本地文档。
pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.fei</groupId>
<artifactId>word-count</artifactId>
<version>0.0.1-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<version>1.3.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.0.2</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>
WordCount.java
package com.fei;
import java.util.Arrays;
import java.util.List;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
/**
* 单词统计,并按降序排序,输出前10个单词及个数
* @author Jfei
*
*/
public class WordCount {
public static void main(String[] args) {
//1.本地模式,创建spark配置及上下文
SparkConf conf = new SparkConf().setAppName("wordCount").setMaster("local");
JavaSparkContext sc = new JavaSparkContext(conf);
//2.读取本地文件,并创建RDD
JavaRDD<String> linesRDD = sc.textFile("e:\\words.txt");
//3.每个单词由空格隔开,将每行的linesRDD拆分为每个单词的RDD
JavaRDD<String> wordsRDD = linesRDD.flatMap(s -> Arrays.asList(s.split("\\s")));
//相当于 ==>
/*JavaRDD<String> wordsRDD = linesRDD.flatMap(new FlatMapFunction<String, String>(){
private static final long serialVersionUID = 1L;
@Override
public Iterable<String> call(String line) throws Exception {
return Arrays.asList(line.split(" "));
}
});*/
//4.将每个单词转为key-value的RDD,并给每个单词计数为1
JavaPairRDD<String,Integer> wordsPairRDD = wordsRDD.mapToPair(s -> new Tuple2<String,Integer>(s, 1));
//相当于 ==>
/*JavaPairRDD<String,Integer> wordsPairRDD = wordsRDD.mapToPair(new PairFunction<String, String, Integer>() {
private static final long serialVersionUID = 1L;
@Override
public Tuple2<String, Integer> call(String word) throws Exception {
return new Tuple2<String,Integer>(word,1);
}
});*/
//5.计算每个单词出现的次数
JavaPairRDD<String,Integer> wordsCountRDD = wordsPairRDD.reduceByKey((a,b) -> a+b);
//相当于 ==>
/*JavaPairRDD<String,Integer> wordsCountRDD = wordsPairRDD.reduceByKey(new Function2<Integer, Integer, Integer>() {
@Override
public Integer call(Integer v1, Integer v2) throws Exception {
return v1 + v2;
}
});*/
//6.因为只能对key进行排序,所以需要将wordsCountRDD进行key-value倒置,返回新的RDD
JavaPairRDD<Integer,String> wordsCountRDD2 = wordsCountRDD.mapToPair(s -> new Tuple2<Integer,String>(s._2, s._1));
//相当于 ==>
/*JavaPairRDD<Integer,String> wordsCountRDD2 = wordsCountRDD.mapToPair(new PairFunction<Tuple2<String,Integer>, Integer, String>() {
private static final long serialVersionUID = 1L;
@Override
public Tuple2<Integer, String> call(Tuple2<String, Integer> t) throws Exception {
return new Tuple2<Integer,String>(t._2,t._1);
}
});*/
//7.对wordsCountRDD2进行排序,降序desc
JavaPairRDD<Integer,String> wordsCountRDD3 = wordsCountRDD2.sortByKey(false);
//8.只取前10个
List<Tuple2<Integer, String>> result = wordsCountRDD3.take(10);
//9.打印
result.forEach(t -> System.out.println(t._2 + " " + t._1));
sc.close();
}
}
如果JDK不是1.8的,那修改下pom.xml及代码中不要使用lambda表达式