SparkDemo--WordCount原生

任务总体逻辑

这是一个典型的WordCount，word来自于任务配置的输入参数，可输入多个word,
计算出<word,出现的次数>，然后将结果输入到mysql的spark_demo表中。

任务配置情况

任务执行结果

主类的代码解读

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.*;
import org.apache.spark.sql.sources.In;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructType;
import scala.Tuple2;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;

public class Demo {
    public static void main(String[] args) {
        final JavaSparkContext sparkContext = new JavaSparkContext(new SparkConf().setAppName("SparkMysql"));
        SQLContext sqlContext = new SQLContext(sparkContext);
        String url="jdbc:mysql://rm-bp10u65wcf06a9y3vqo.mysql.rds.aliyuncs.com/testforuser";//刚一开始url写错了，没有加“jdbc:mysql://”
        String table="spark_demo";
        Properties connectionProperties=new Properties();
        connectionProperties.put("user","testuser");
        connectionProperties.put("password","dtforuser123");
        connectionProperties.put("driver","com.mysql.jdbc.Driver");

        //读取输入参数将其构造成RDD
        JavaRDD<String> words=sparkContext.parallelize(Arrays.asList(args));
        
        //根据上面的JavaRDD转换为<word,1>的JavaPairRDD
        JavaPairRDD<String,Integer> wordsPair=words.mapToPair(new PairFunction<String, String, Integer>() {

            public Tuple2<String, Integer> call(String s) throws Exception {
                return new Tuple2<String, Integer>(s,1);
            }
        });
        //根据上面的JavaPairRDD,以word为Key进行reduce,并累加其出现的次数
        JavaPairRDD<String,Integer> wordsRDD=wordsPair.reduceByKey(new Function2<Integer, Integer, Integer>() {
            public Integer call(Integer integer, Integer integer2) throws Exception {
                return integer+integer2;
            }
        });
        //根据上面的JavaPairRDD，转换为Row类型的RDD，以方便写入到数据库
        JavaRDD<Row> wordsRow=wordsRDD.map(new Function<Tuple2<String, Integer>, Row>() {
            public Row call(Tuple2<String, Integer> tuple) throws Exception {
                return RowFactory.create(tuple._1,tuple._2);
            }
        });
        //动态构造DataFrame的元数据
        List structFields=new ArrayList();
        structFields.add(DataTypes.createStructField("name",DataTypes.StringType,true));
        structFields.add(DataTypes.createStructField("age",DataTypes.IntegerType,true));

        //构建StructType,用于最后DataFrame元数据的描述
        StructType structType=DataTypes.createStructType(structFields);

        //基于已有的元数据和RDD<ROW>来构造DataFrame
        Dataset<Row> personsDS= sqlContext.createDataFrame(wordsRow,structType);

        //将数据写入person表中
        personsDS.write().mode("append").jdbc(url,table,connectionProperties);
        
        //任务结束停止运行
        sparkContext.stop();
    }
}

---

补充：整个项目
链接：https://pan.baidu.com/s/1JRu0Ztf8SkqXR-8hTgyu5w
提取码：awv3