大数据-Spark（四）
通过IDEA开发spark程序
构建maven工程，添加pom依赖
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.3.3</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.2</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
<configuration>
<args>
<arg>-dependencyfile</arg>
<arg>${project.build.directory}/.scala_dependencies</arg>
</args>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass></mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>利用scala语言开发spark程序实现单词统计--本地运行
代码开发
package com.kaikeba
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
//todo: 利用scala语言开发spark程序实现单词统计
object WordCount {
def main(args: Array[String]): Unit = {
//1、构建sparkConf对象 设置application名称和master地址
val sparkConf: SparkConf = new SparkConf().setAppName("WordCount").setMaster("local[2]")
//2、构建sparkContext对象,该对象非常重要，它是所有spark程序的执行入口
// 它内部会构建 DAGScheduler和 TaskScheduler 对象
val sc = new SparkContext(sparkConf)
//设置日志输出级别
sc.setLogLevel("warn")
//3、读取数据文件
val data: RDD[String] = sc.textFile("E:\\words.txt")
//4、 切分每一行，获取所有单词
val words: RDD[String] = data.flatMap(x=>x.split(" "))
//5、每个单词计为1
val wordAndOne: RDD[(String, Int)] = words.map(x => (x,1))
//6、相同单词出现的1累加
val result: RDD[(String, Int)] = wordAndOne.reduceByKey((x,y)=>x+y)
//按照单词出现的次数降序排列 第二个参数默认是true表示升序，设置为false表示降序
val sortedRDD: RDD[(String, Int)] = result.sortBy( x=> x._2,false)
//7、收集数据打印
val finalResult: Array[(String, Int)] = sortedRDD.collect()
finalResult.foreach(println)
//8、关闭sc
sc.stop()
}
}利用scala语言开发spark程序实现单词统计--集群运行
代码开发
package com.kaikeba
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
//todo: 利用scala语言开发spark程序实现单词统计
object WordCountOnSpark {
def main(args: Array[String]): Unit = {
//1、构建sparkConf对象 设置application名称
val sparkConf: SparkConf = new SparkConf().setAppName("WordCountOnSpark")
//2、构建sparkContext对象,该对象非常重要，它是所有spark程序的执行入口
// 它内部会构建 DAGScheduler和 TaskScheduler 对象
val sc = new SparkContext(sparkConf)
//设置日志输出级别
sc.setLogLevel("warn")
//3、读取数据文件
val data: RDD[String] = sc.textFile(args(0))
//4、 切分每一行，获取所有单词
val words: RDD[String] = data.flatMap(x=>x.split(" "))
//5、每个单词计为1
val wordAndOne: RDD[(String, Int)] = words.map(x => (x,1))
//6、相同单词出现的1累加
val result: RDD[(String, Int)] = wordAndOne.reduceByKey((x,y)=>x+y)
//7、把计算结果保存在hdfs上
result.saveAsTextFile(args(1))
//8、关闭sc
sc.stop()
}
}打成jar包提交到集群中运行
spark-submit \
--master spark://node01:7077,node02:7077 \
--class com.kaikeba.WordCountOnSpark \
--executor-memory 1g \
--total-executor-cores 4 \
original-spark_class04-1.0-SNAPSHOT.jar \
/words.txt /out利用java语言开发spark程序实现单词统计--本地运行
代码开发
package com.kaikeba;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
//todo: 利用java语言开发spark的单词统计程序
public class JavaWordCount {
public static void main(String[] args) {
//1、创建SparkConf对象
SparkConf sparkConf = new SparkConf().setAppName("JavaWordCount").setMaster("local[2]");
//2、构建JavaSparkContext对象
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
//3、读取数据文件
JavaRDD<String> data = jsc.textFile("E:\\words.txt");
//4、切分每一行获取所有的单词 scala: data.flatMap(x=>x.split(" "))
JavaRDD<String> wordsJavaRDD = data.flatMap(new FlatMapFunction<String, String>() {
public Iterator<String> call(String line) throws Exception {
String[] words = line.split(" ");
return Arrays.asList(words).iterator();
}
});
//5、每个单词计为1 scala: wordsJavaRDD.map(x=>(x,1))
JavaPairRDD<String, Integer> wordAndOne = wordsJavaRDD.mapToPair(new PairFunction<String, String, Integer>() {
public Tuple2<String, Integer> call(String word) throws Exception {
return new Tuple2<String, Integer>(word, 1);
}
});
//6、相同单词出现的1累加 scala: wordAndOne.reduceByKey((x,y)=>x+y)
JavaPairRDD<String, Integer> result = wordAndOne.reduceByKey(new Function2<Integer, Integer, Integer>() {
public Integer call(Integer v1, Integer v2) throws Exception {
return v1 + v2;
}
});
//按照单词出现的次数降序 (单词，次数) -->(次数,单词).sortByKey----> (单词，次数)
JavaPairRDD<Integer, String> reverseJavaRDD = result.mapToPair(new PairFunction<Tuple2<String, Integer>, Integer, String>() {
public Tuple2<Integer, String> call(Tuple2<String, Integer> t) throws Exception {
return new Tuple2<Integer, String>(t._2, t._1);
}
});
JavaPairRDD<String, Integer> sortedRDD = reverseJavaRDD.sortByKey(false).mapToPair(new PairFunction<Tuple2<Integer, String>, String, Integer>() {
public Tuple2<String, Integer> call(Tuple2<Integer, String> t) throws Exception {
return new Tuple2<String, Integer>(t._2, t._1);
}
});
//7、收集打印
List<Tuple2<String, Integer>> finalResult = sortedRDD.collect();
for (Tuple2<String, Integer> t : finalResult) {
System.out.println("单词："+t._1 +"\t次数："+t._2);
}
jsc.stop();
}
}此博文仅供学习参考，如有错误欢迎指正。
上一篇《大数据-Spark（三）》
下一篇《大数据-Spark（五）》
扫一扫
568
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
