【java-spark】mysql2mysql

环境：2.4版本的spark包、eclipse本地模式
testcc表数据

新建表testcc_test

CREATE TABLE `testcc_test` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `c` varchar(255) DEFAULT NULL,
  `mark` varchar(255) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8;

运行代码，插入数据
结果如下图

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;


public class Mysql2Mysql {

	public static void main(String[] args) {
		SparkConf conf = new SparkConf().setAppName("Mysql2Mysql");
		conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
		conf.setMaster("local");
		SparkSession sparkSession = SparkSession.builder().config(conf).getOrCreate();
		SparkContext sc = sparkSession.sparkContext();
		JavaSparkContext jsc = JavaSparkContext.fromSparkContext(sc);
		SQLContext sqlContext = new SQLContext(sparkSession);
		String url = "jdbc:mysql://localhost:3306/yb?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&zeroDateTimeBehavior=CONVERT_TO_NULL";
		// 查找mysql的表名
		String table = "testcc";
		// 设置数据库参数
		Properties connectionProperties = new Properties();
		connectionProperties.put("user", "root");
		connectionProperties.put("password", "root");
		connectionProperties.put("driver", "com.mysql.jdbc.Driver");
		// 读取表中所有数据
		Dataset<Row> jdbcDS = sqlContext.read().jdbc(url, table, connectionProperties).select("c");//* 默认所有字段
		// 显示数据
//		jdbcDS.show();

		JavaRDD<Row> rowRdd = jdbcDS.javaRDD().map(i -> {
			List<String> values = new ArrayList<>();
			Map<String, Object> result = new HashMap<String, Object>();
			String value = "null";
			value = ObjectUtils.toString(i.getAs("c"));

			return RowFactory.create(value,value+"1");
		});

		//mysql字段集合
		List<StructField> structFields = new ArrayList<StructField>();
		structFields.add(DataTypes.createStructField("c", DataTypes.StringType, true));
		structFields.add(DataTypes.createStructField("mark", DataTypes.StringType, true));

		// 构建StructType，用于最后DataFrame元数据的描述
		StructType structType = DataTypes.createStructType(structFields);

		Dataset<Row> createDataFrame = sqlContext.createDataFrame(rowRdd, structType);

		createDataFrame.write().mode("append").jdbc(url, "testcc_test", connectionProperties);

	}

}