spark 表关联 java_java操作spark ，将两个表进行jion操作JavaPairRDD

private static final longserialVersionUID= 1L;

@OverridepublicTuple2 call(Row row) throwsException {

return newTuple2(row.getAs("job_name"),row.getAs("job_log_id"));

}

}).join(scoreinforDataSourceDs.javaRDD().mapToPair(newPairFunction() {

private static final longserialVersionUID= 1L;

@OverridepublicTuple2 call(Row row) throwsException {

return newTuple2(row.getAs("job_name"),row.getAs("job_log_id"));

}

}));

//调用RowFactory工厂方法生成记录JavaRDD reusltRowRDD = resultRDD.map(newFunction>, Row>() {

@OverridepublicRow call(Tuple2> tuple) throwsException {

returnRowFactory.create(tuple._1,tuple._2._1,tuple._2._2);

}

});

/*** 动态构造DataFrame的元数据，一般而言，有多少列以及每列的具体类型可能来自于json文件，也可能来自于数据库*/// List structFields = new ArrayList();// structFields.add(DataTypes.createStructField("job_name", DataTypes.StringType,true));// structFields.add(DataTypes.createStructField("job_log_id", DataTypes.StringType,true));// structFields.add(DataTypes.createStructField("job_message", DataTypes.StringType,true));// //构建StructType,用于最后DataFrame元数据的描述// StructType structType = DataTypes.createStructType(structFields);// //生成Dataset// Dataset personDS = sqlContext.createDataFrame(reusltRowRDD,structType);// personDS.show();/*** 1.当Dataframe要把通过SparkSQL，core、ml等复杂操作的数据写入数据库的时候首先是权限的问题，必须确保数据库授权了当前操作SparkSQL的用户；* 2.Dataframe要写数据到DB的时候，一般都不可以直接写进去，而是要转成RDD，通过RDD写数据到DB中，*/// personDS.javaRDD().foreachPartition(new VoidFunction>(){// @Override// public void call(Iterator rowIterator) throws Exception {// Connection connection = null;//数据库连接// Statement statement = null; // try{// connection = DriverManager.getConnection("jdbc:mysql://localhost:3306/spark","root","123456");// statement = connection.createStatement();// while (rowIterator.hasNext()){// String sqlText = "insert into userscoreinfor (name,age,score) values (";// Row row = rowIterator.next();// String name = row.getAs("job_name");// int age = row.getAs("job_log_id");// String score = row.getAs("job_message");// sqlText+="'"+name+"',"+"'"+age+"',"+"'"+score+"')";// statement.execute(sqlText);// } }catch (SQLException e){// e.printStackTrace();// }finally {// if (connection != null){// connection.close();// }