任务总体逻辑
这是一个典型的WordCount,word来自于任务配置的输入参数,可输入多个word,
计算出<word,出现的次数>,然后将结果输入到mysql的spark_demo表中。
任务配置情况
任务执行结果
主类的代码解读
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.*;
import org.apache.spark.sql.sources.In;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructType;
import scala.Tuple2;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
public class Demo {
public static void main(String[] args) {
final JavaSparkContext sparkContext = new JavaSparkContext(new SparkConf().setAppName("SparkMysql"));
SQLContext sqlContext = new SQLContext(sparkContext);
String url="jdbc:mysql://rm-bp10u65wcf06a9y3vqo.mysql.rds.aliyuncs.com/testforuser";//刚一开始url写错了,没有加“jdbc:mysql://”
String table="spark_demo";
Properties connectionProperties=new Properties();
connectionProperties.put("user","testuser");
connectionProperties.put("password","dtforuser123");
connectionProperties.put("driver","com.mysql.jdbc.Driver");
//读取输入参数将其构造成RDD
JavaRDD<String> words=sparkContext.parallelize(Arrays.asList(args));
//根据上面的JavaRDD转换为<word,1>的JavaPairRDD
JavaPairRDD<String,Integer> wordsPair=words.mapToPair(new PairFunction<String, String, Integer>() {
public Tuple2<String, Integer> call(String s) throws Exception {
return new Tuple2<String, Integer>(s,1);
}
});
//根据上面的JavaPairRDD,以word为Key进行reduce,并累加其出现的次数
JavaPairRDD<String,Integer> wordsRDD=wordsPair.reduceByKey(new Function2<Integer, Integer, Integer>() {
public Integer call(Integer integer, Integer integer2) throws Exception {
return integer+integer2;
}
});
//根据上面的JavaPairRDD,转换为Row类型的RDD,以方便写入到数据库
JavaRDD<Row> wordsRow=wordsRDD.map(new Function<Tuple2<String, Integer>, Row>() {
public Row call(Tuple2<String, Integer> tuple) throws Exception {
return RowFactory.create(tuple._1,tuple._2);
}
});
//动态构造DataFrame的元数据
List structFields=new ArrayList();
structFields.add(DataTypes.createStructField("name",DataTypes.StringType,true));
structFields.add(DataTypes.createStructField("age",DataTypes.IntegerType,true));
//构建StructType,用于最后DataFrame元数据的描述
StructType structType=DataTypes.createStructType(structFields);
//基于已有的元数据和RDD<ROW>来构造DataFrame
Dataset<Row> personsDS= sqlContext.createDataFrame(wordsRow,structType);
//将数据写入person表中
personsDS.write().mode("append").jdbc(url,table,connectionProperties);
//任务结束停止运行
sparkContext.stop();
}
}
补充:整个项目
链接:https://pan.baidu.com/s/1JRu0Ztf8SkqXR-8hTgyu5w
提取码:awv3