sparksql读文件到mysql_spark读文件写mysql(java版)

packageorg.langtong.sparkdemo;importcom.fasterxml.jackson.databind.ObjectMapper;importorg.apache.spark.SparkConf;importorg.apache.spark.SparkContext;importorg.apache.spark.api.java.JavaPairRDD;importorg.apache.spark.api.java.JavaRDD;importorg.apache.spark.api.java.JavaSparkContext;importorg.apache.spark.api.java.function.FlatMapFunction;importorg.apache.spark.api.java.function.Function;importorg.apache.spark.api.java.function.Function2;importorg.apache.spark.api.java.function.PairFunction;importorg.apache.spark.broadcast.Broadcast;import org.apache.spark.sql.*;importorg.apache.spark.sql.types.DataTypes;importorg.apache.spark.sql.types.StructField;importorg.apache.spark.sql.types.StructType;importorg.apache.spark.storage.StorageLevel;importscala.Tuple2;importjava.io.IOException;importjava.io.Serializable;import java.util.*;importjava.util.Map.Entry;/*** spark读文件 入mysql

* org.langtong.sparkdemo.SparkReadFile2MysqlFull

*

*@authorAdministrator*/

public class SparkReadFile2MysqlFull implementsSerializable {private static final long serialVersionUID = 1L;private staticProperties connectionProperties;private staticJavaSparkContext jsc;private staticSparkSession lalala;private staticSQLContext sqlContext;private static String url = "jdbc:mysql://192.168.2.258:3306/diagbot?Unicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false";private static String mysqldriver = "com.mysql.jdbc.Driver";private static String user = "root";private static String password = "diagbot@db";private static String[] serials = new String[]{"13", "14", "15", "16", "17","18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32"};private static String[] tables = new String[]{"emrdata_1", "emrdata_2","emrdata_3", "emrdata_4", "emrdata_5", "emrdata_6", "emrdata_7","emrdata_8", "emrdata_9"};private static Map map_t = new HashMap();private static Map map_s = new HashMap();private static Map map_r = new HashMap();private static Map map_k = new HashMap();public static void main(String[] args) throwsIOException {for (int i = 0; i <= 57; i++) {

StringBuffer stringBuffer= newStringBuffer();

stringBuffer.append("hdfs://192.168.2.258:9000/datas/parquetFile/ord_detail_0718/part");

stringBuffer.append(i);

ordDetail_full(stringBuffer.toString(), i);

}//ordDetail_full("/opt/diagbot/datas/srrsh/ord_detail/part0");

}

@SuppressWarnings("unused")private static voidemrdataIndedx(String filePath, JavaSparkContext jsc,

SQLContext sqlContext) {

JavaRDD javaRDD =jsc.textFile(filePath);

String[] fields= {"pk_dcemr", "pk_dcpv", "empi", "code_pati","code_pvtype", "code_ref_emr", "code_emr_type","name_emr_type", "prlog_rdn", "code_dept_emr", "name_dept_emr","code_psn_edit", "data_source", "source_pk", "create_time"};

JavaRDD mapPartitions = javaRDD.mapPartitions(new FlatMapFunction, Row>() {private static final long serialVersionUID = 1L;

ObjectMapper mapper= newObjectMapper();

@SuppressWarnings("unchecked")public Iterator call(Iteratoriterator)throwsException {

ArrayList arrayList = new ArrayList();//TODO Auto-generated method stub

while(iterator.hasNext()) {try{

String next=iterator.next();

map_t= mapper.readValue(next, Map.class);for (Entryentry : map_t.entrySet()) {

map_s.put(entry.getKey(),

String.valueOf(entry.getValue()));

}

}catch(Exception e) {//TODO: handle exception

return null;

}

arrayList.add(createEmerIndexRow(map_s));

}returnarrayList.iterator();

}

});

StructType schema=createStructType(fields);

HashMap> hashMap2 = new HashMap>();

Dataset createDataFrame1 =sqlContext.createDataFrame(mapPartitions,

schema);

createDataFrame1.write().mode(SaveMode.Append)

.jdbc(url,"emrdataindex_1", connectionProperties);

}public static JavaRDD emrdata(JavaRDDjavaRDD) {

JavaRDD rowRDD = javaRDD.map(new Function() {private static final long serialVersionUID = 1L;public Row call(String line) throwsException {//TODO Auto-generated method stub

try{

ObjectMapper mapper= newObjectMapper();

map_t= mapper.readValue(line, Map.class);for (Entryentry : map_t.entrySet()) {

map_s.put(entry.getKey(),

String.valueOf(entry.getValue()));

}

}catch(Exception e) {//TODO: handle exception

map_s.put("pvcode", "99999999");

map_s.put("remark", line);

map_s.put("pcode", "3100|H02");returncreateEmrdataRow(map_s);

}returncreateEmrdataRow(map_s);

}

});returnrowRDD;

}private staticStructType createStructType(String[] fields) {

LinkedList structFieldsList = new LinkedList();for(String field : fields) {

structFieldsList.add(DataTypes.createStructField(field,

DataTypes.StringType,true));

}

StructType schema=DataTypes.createStructType(structFieldsList);returnschema;

}private staticJavaSparkContext getContext(String master) {

SparkConf conf= new SparkConf().setAppName("SparkReadkwz").setMaster(

master);

conf.set("spark.scheduler.mode", "FAIR");

JavaSparkContext jsc= newJavaSparkContext(conf);returnjsc;

}private static voidordRec(String filePath) {

String master= "spark://192.168.2.258:7077";

jsc=getContext(master);

sqlContext= newSQLContext(jsc);

connectionProperties= newProperties();

connectionProperties.put("user", user);

connectionProperties.put("password", password);

connectionProperties.put("driver", "com.mysql.jdbc.Driver");

JavaRDD javaRDD =jsc.textFile(filePath);

String[] ordrecFields= {"pk_ord_record", "pk_dcord", "pk_dcpv", "code_pvtype", "name_pvtype", "pvcode","code_ord", "empi", "code_pati", "code_sex", "name_sex", "age", "code_dept", "name_dept", "bed","pk_dcordrisreq", "code_req", "code_rep", "code_rep_type", "name_rep_type", "code_eu_type","name_eu_type", "code_eu_item", "name_eu_item", "create_time"};

StructType schema=createStructType(ordrecFields);

JavaRDD mapPartitions = ja