第二天练习总结:
scala版代码:
package com
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
object WordCount {
def main(args: Array[String]): Unit = {
val conf = new SparkConf()
conf.setMaster("local").setAppName("WordCount")//设定模式+s跑的名称
val sc = new SparkContext(conf)//加载进来
val lines:RDD[String] = sc.textFile("./words")//加载文件,一行一行读数据
val words:RDD[String] = lines.flatMap(line=>{
line.split(" ")
})//切分成每一个单词
val pairword: RDD[(String, Int)] = words.map(word=>{
new Tuple2(word,1)
})
val result = pairword.reduceByKey((v1:Int,v2:Int)=>{v1+v2})
result.foreach(tuple=>{
println(tuple)
})
sc.stop()
}
}
Java版代码:
package spark;
import java.util.Arrays;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import scala.Tuple2;
public class JavaSparkWordCount {
public static void main(String[] args) {
/**
* conf
* 1.可以设置spark的运行模式
* 2.可以设置spark在weibui中显示的application的名称
* 3.可以设置当前spark application 运行的资源(内存+core)
* spark运行模式:
* 1.local --本地模式
* 2.stanalone -- Spark自带的资源调度框架,支持分布式搭建,Spark任务可以依赖standalone调度资源
* 3.yarn -- hadoop生态圈中的资源调度框架。
* 4.mesos
*/
SparkConf conf = new SparkConf();
conf.setMaster("local");
conf.setAppName("JavaSparkWordCount");
JavaSparkContext sc = new JavaSparkContext(conf);//是通往集群的唯一通道
JavaRDD<String> lines = sc.textFile("./words");
JavaRDD<String> words =lines.flatMap(new FlatMapFunction<String, String>() {
@Override
public Iterable<String> call(String line) throws Exception {
return Arrays.asList(line.split(" "));
}
});
JavaPairRDD<String, Integer> pairWords = words.mapToPair(new PairFunction<String, String , Integer>() {
@Override
public Tuple2<String, Integer> call(String word) throws Exception {
return new Tuple2<String,Integer>(word,1);
}
});
JavaPairRDD<String, Integer> result = pairWords.reduceByKey(new Function2<Integer, Integer, Integer>() {
@Override
public Integer call(Integer v1, Integer v2) throws Exception {
// TODO Auto-generated method stub
return v1+v2;
}
});
result.foreach(new VoidFunction<Tuple2<String, Integer>>(){
@Override
public void call(Tuple2<String, Integer> tuple) throws Exception {
System.out.println(tuple);
}
});
sc.stop();
}
}