Java实现Spark词配对Wordcount计数

需求

  使用Spark实现对一个文档中的每一行的单词进行词配对计数，要求去标点符号，将大写符号统一转化成为小写单词。

  举例说明，最初的文档为：

“a a, A b
a b c

则处理后的结果为：

(a a) 2
(a b) 2
(a c) 1
(b a) 4
(b c) 1
(c a) 1
(c b) 1

实现过程

开启hadoop和spark

进入Hadoop所在的文件夹并执行启动语句：

$ sbin/start-all.sh

进入Spark所在的文件夹并执行启动语句：

$ sbin/start-all.sh

jar包处理

将编码程序打包成jar包进行处理

HDFS文件设置

使用hdfs创建文件夹，并将input文件放在hdfs文件夹下：

$ hadoop dfs -mkdir -p /wordcount2/input
$ hadoop dfs -put /Users/liuqi/Desktop/input2.txt /wordcount2/input
$ bin/hdfs dfs -ls /wordcount2/input

spark-submit程序

运行mapreduce程序：

$ bin/spark-submit --class WordCount --num-executors 2 --executor-memory 6g --executor-cores 4 /Users/liuqi/Desktop/wordcountspark.jar /wordcount2/output

结果显示如下：

注：如果中间有错，则删除对应文件重新进行操作：

$ bin/hdfs dfs -rm -r /wordcount2/input

查询结果

结果保存在本地：

$ hadoop dfs -getmerge /wordcount2/output /Users/liuqi/Desktop/wordcount2/

附加代码

注：这里只是显示java代码，整个工程去我的CSDN博客进行下载。

WordCount.java:

import scala.Tuple2;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
/** 
 *  
 * WordCount 
 * @author 刘琦
 */ 

public final class WordCount {
  private static final Pattern SPACE = Pattern.compile(" ");
  public static void main(String[] args) throws Exception {
    if (args.length < 1) {
      System.err.println("Usage: WordCount <file>");
      System.exit(1);
    }

    SparkConf sparkConf = new SparkConf().setAppName("WordCount");
    JavaSparkContext ctx = new JavaSparkContext(sparkConf);
    JavaRDD<String> lines = ctx.textFile("hdfs://localhost:54310/wordcount2/input");

    JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() {

        public Iterator<String> call(String s) throws Exception {
            // TODO Auto-generated method stub
            //大小写转化与去符号
            String newStr = s.toLowerCase().replaceAll("[\\d\\pP\\p{Punct}]", "");;
            String[] wordResult = SPACE.split(newStr);
            List<String> wordNewResult = new ArrayList<String>();
            String[][] result = new String[wordResult.length][2];
            for (int i = 0; i < wordResult.length; i ++){
                result[i][0] = wordResult[i];
                 result[i][1] = "0";
            }

            //对每一行的单词进行处理
            for(int i = 0; i < wordResult.length ; i++){
                for(int j = 0; j < wordResult.length; j++){
                    if(i == j){
                        continue;
                    }else if (result[i][1].equals("1")){
                        //这个词之前出现过了，这里只统计它之后还有没有相同的数据,后来发现不需要这一步了，相同的只计算一次就好
                        if(i<j && result[i][0].equals(result[j][0])){
                            result[j][1] = "1";
                            //wordNewResult.add("(" + result[i][0] + "  " + result[j][0] + ")");
                            //word.set("(" + result[i][0] + "  " + result[j][0] + ")");
//                          output.collect(word, one);
                        }
                    }else{
                        //这个词之前没有出现过
                        if (!result[i][0].equals(result[j][0])){
                            //普通操作
                            wordNewResult.add("(" + result[i][0] + "  " + result[j][0] + ")");
//                          word.set("(" + result[i][0] + "  " + result[j][0] + ")");  
                        }else{
                            //说明两个单词是一样的,并且这个单词之前没有统计过
                            result[j][1] = "1";
                            wordNewResult.add("(" + result[i][0] + "  " + result[j][0] + ")");
//                          word.set("(" + result[i][0] + "  " + result[j][0] + ")");  
                        }
                    }
                }
            }

            return wordNewResult.iterator();
        }

      });

      JavaPairRDD<String, Integer> ones = words.mapToPair(new PairFunction<String, String, Integer>() {
        public Tuple2<String, Integer> call(String s) {
          return new Tuple2<String, Integer>(s, 1);
        }
      });

      JavaPairRDD<String, Integer> counts = ones.reduceByKey(new Function2<Integer, Integer, Integer>() {
        public Integer call(Integer i1, Integer i2) {
          return i1 + i2;
        }
      });

      //创建Hdfs文件，打开Hdfs输出流
      HdfsOperate.openHdfsFile("hdfs://localhost:54310/wordcount2/output");

      List<Tuple2<String, Integer>> output = counts.collect();
      for (Tuple2<?,?> tuple : output) {
        System.out.println(tuple._1() + ": " + tuple._2());
        HdfsOperate.writeString(tuple._1() + ": " + tuple._2());
      }
      ctx.stop();
      //关闭Hdfs输出流
      HdfsOperate.closeHdfsFile();
    }

  }

HdfsOperate.java：

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.web.resources.ExceptionHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.net.URI;
/**
 * 使用Hadoop的FileSystem把数据写入到HDFS
 */
public class HdfsOperate implements Serializable{

    private static Logger logger = LoggerFactory.getLogger(HdfsOperate.class);
    private static Configuration conf = new Configuration();
    private static BufferedWriter writer = null;

    //在hdfs的目标位置新建一个文件，得到一个输出流
    public static void openHdfsFile(String path) throws Exception {
        FileSystem fs = FileSystem.get(URI.create(path),conf);
        writer = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(path))));
        if(null!=writer){
            logger.info("[HdfsOperate]>> initialize writer succeed!");
        }
    }

    //往hdfs文件中写入数据
    public static void writeString(String line) {
        try {
            writer.write(line + "\n");
        }catch(Exception e){
            logger.error("[HdfsOperate]>> writer a line error:"  ,  e);
        }
    }

    //关闭hdfs输出流
    public static void closeHdfsFile() {
        try {
            if (null != writer) {
                writer.close();
                logger.info("[HdfsOperate]>> closeHdfsFile close writer succeed!");
            }
            else{
                logger.error("[HdfsOperate]>> closeHdfsFile writer is null");
            }
        }catch(Exception e){
            logger.error("[HdfsOperate]>> closeHdfsFile close hdfs error:" + e);
        }
    }

}