scala版+java版的wordcount

第二天练习总结：
scala版代码：

package com

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD


object WordCount {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
    conf.setMaster("local").setAppName("WordCount")//设定模式+s跑的名称
    val sc = new SparkContext(conf)//加载进来
    val lines:RDD[String] = sc.textFile("./words")//加载文件,一行一行读数据
    val words:RDD[String] =  lines.flatMap(line=>{
      line.split(" ")
    })//切分成每一个单词
    val pairword: RDD[(String, Int)] = words.map(word=>{
      new Tuple2(word,1)
    })
    val result = pairword.reduceByKey((v1:Int,v2:Int)=>{v1+v2})
    result.foreach(tuple=>{
       println(tuple)
    })
    sc.stop()
    
  }
}

Java版代码：

package spark;

import java.util.Arrays;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;

import scala.Tuple2;

public class JavaSparkWordCount {
	public static void main(String[] args) {
		/**
		 * conf 
		 * 1.可以设置spark的运行模式
		 * 2.可以设置spark在weibui中显示的application的名称
		 * 3.可以设置当前spark application 运行的资源（内存+core）
 		 * spark运行模式：
		 * 1.local --本地模式
		 * 2.stanalone -- Spark自带的资源调度框架，支持分布式搭建，Spark任务可以依赖standalone调度资源
		 * 3.yarn -- hadoop生态圈中的资源调度框架。
		 * 4.mesos
		 */
		SparkConf conf = new SparkConf();
		conf.setMaster("local");
		conf.setAppName("JavaSparkWordCount");
		
		JavaSparkContext sc = new JavaSparkContext(conf);//是通往集群的唯一通道
		JavaRDD<String> lines = sc.textFile("./words");
		JavaRDD<String> words =lines.flatMap(new FlatMapFunction<String, String>() {

			@Override
			public Iterable<String> call(String line) throws Exception {
				return Arrays.asList(line.split(" "));
			}
		});
		
		JavaPairRDD<String, Integer> pairWords = words.mapToPair(new PairFunction<String, String , Integer>() {

			@Override
			public Tuple2<String, Integer> call(String word) throws Exception {
				
				return new Tuple2<String,Integer>(word,1);
			}
		});
		
		JavaPairRDD<String, Integer> result = pairWords.reduceByKey(new Function2<Integer, Integer, Integer>() {
			
			@Override
			public Integer call(Integer v1, Integer v2) throws Exception {
				// TODO Auto-generated method stub
				return v1+v2;
			}
		});
		result.foreach(new VoidFunction<Tuple2<String, Integer>>(){

			@Override
			public void call(Tuple2<String, Integer> tuple) throws Exception {
				
				System.out.println(tuple);
			}
		});
		sc.stop();	
	}
}