spark本地debug wordcount

环境

1、win
2、linux搭建spark 3.0.1
3、scala 2.12.13
4、idea

idea开启远程应用

修改spark conf

cd ./spark/conf
vim ./spark-enc.sh
export SPARK_MASTER_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=10000"

debug 开始调试

scala代码 使用local[2]模式

object WordCount {
  def main(args: Array[String]): Unit = {
    //创建SparkContext， 使用SparkContext才能向集群申请资源，创建RDD
    val conf = new SparkConf().setAppName("WordCount").setMaster("local[2]")
    val sc = new SparkContext(conf);

    //第一步创建RDD：指定从HDFS中读取数据创建RDD
    //读取数据
    val lines: RDD[String] = sc.textFile(args(0));

    //读数据新型切分 压平
    val words: RDD[String] = lines.flatMap(_.split(" "))

    //将单词和 1 组合
    val wordAndOne: RDD[(String, Int)] = words.map((_, 1))

    //聚合（优点：现在分区内进行局部聚合，然后在全局进行聚合）
    var reduced: RDD[(String, Int)] = wordAndOne.reduceByKey(_ + _)

    //排序
    val sorted: RDD[(String, Int)] = reduced.sortBy(_._2)

    //将计算好的结果保存到hdfs
    sorted.saveAsTextFile(args(1))

    //释放资源
    sc.stop();
  }


  /*
  *
  * - 生产环境打包【非本地测试】
  *  [root@master bin]# spark-submit --master spark://master:7077 --class com.hui.spark.test.WordCount /root/spark-1.0-SNAPSHOT.jar hdfs://master:9000/wc hdfs://master:9000/out1
  *  +++++ jar位置 + main 方法的参数
  * */
}

java

public class WordCountJava {

    //使用java代码，编写wordCount实例
    public static void main(String[] args) {
        // 1
        SparkConf conf = new SparkConf().setAppName("WordCountJava");
        JavaSparkContext jsc = new JavaSparkContext(conf);

        // 2
        JavaRDD<String> lines = jsc.textFile(args[0]);

        // 3
        JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
            @Override
            public Iterator<String> call(String lines) throws Exception {
                //返回Iterator类型
                return Arrays.stream(lines.split(" ")).iterator();
            }
        });

        /*// 4
        JavaRDD<Tuple2<String, Integer>> wordAndOne = words.map(new Function<String, Tuple2<String, Integer>>() {
            @Override
            public Tuple2<String, Integer> call(String word) throws Exception {
                return Tuple2.apply(word, 1);
            }
        });

        // 5 java api 无法调用reduceByKey
        wordAndOne.reduce*/

        // 4 将RDD变包装类
        JavaPairRDD<String, Integer> wordAndOne = words.mapToPair(new PairFunction<String, String, Integer>() {
            @Override
            public Tuple2<String, Integer> call(String words) throws Exception {
                return Tuple2.apply(words, 1);
            }
        });

        // 6
        JavaPairRDD<String, Integer> reduced = wordAndOne.reduceByKey(new Function2<Integer, Integer, Integer>() {
            @Override
            public Integer call(Integer v1, Integer v2) throws Exception {
                return v1 + v2;
            }
        });

        // 7 排序只能根据Key, 不能根据value
        // result.sortByKey()
        // 解决：交换次序
        JavaPairRDD<Integer, String> swapped = reduced.mapToPair(new PairFunction<Tuple2<String, Integer>, Integer, String>() {
            @Override
            public Tuple2<Integer, String> call(Tuple2<String, Integer> tp) throws Exception {
                return tp.swap(); //交换次序
            }
        });
        swapped.sortByKey(false); //降序

        // 8 再交换顺序
        JavaPairRDD<String, Integer> result = swapped.mapToPair(new PairFunction<Tuple2<Integer, String>, String, Integer>() {
            @Override
            public Tuple2<String, Integer> call(Tuple2<Integer, String> tp) throws Exception {
                return tp.swap();
            }
        });

        // 9
        result.saveAsTextFile(args[1]);

        // 10
        jsc.stop();
		
		// lambada todo 
        JavaRDD<String> lines = jsc.textFile(args[0]);

        JavaRDD<String> words = lines.flatMap(line -> Arrays.stream(line.split(" ")).iterator());

        JavaPairRDD<String, Integer> wordAndOne = words.mapToPair(w -> Tuple2.apply(w, 1));

        JavaPairRDD<String, Integer> reduced = wordAndOne.reduceByKey(((v1, v2) -> v1 + v2));

        JavaPairRDD<Integer, String> swapped = reduced.mapToPair(tp -> tp.swap());

        JavaPairRDD<Integer, String> sorted = swapped.sortByKey(false);

        JavaPairRDD<String, Integer> result = sorted.mapToPair(Tuple2::swap);
        
        result.saveAsTextFile(args[1]);

        jsc.stop();
    }
}

main 方法参数idea 设置

报错

Caused by: com.fasterxml.jackson.databind.JsonMappingException: Scala module 2.10.0 requires Jackson Databind version >= 2.10.0 and < 2.11.0

由于引入spark依赖，自带的版本冲突，重新加入jackson的依赖，修改jackson的版本

		<dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.10.5</version>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-core</artifactId>
            <version>2.10.5</version>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-annotations</artifactId>
            <version>2.10.5</version>
        </dependency>