java spark dataset_java spark list 转为 RDD 转为 dataset 写入表中

package com.example.demo;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.sql.Dataset;

import org.apache.spark.sql.Row;

import org.apache.spark.sql.SQLContext;

import org.apache.spark.sql.SparkSession;

public class DemoApplication {

public static void main(String[] args) {

///*-----------------------线上调用方式--------------------------*/

// 读入店铺id数据

SparkSession spark = SparkSession.builder().appName("demo_spark").enableHiveSupport().getOrCreate();

Dataset vender_set = spark.sql("select pop_vender_id from app.app_sjzt_payout_apply_with_order where dt = '2019-08-05' and pop_vender_id is not null");

System.out.println( "数据读取 OK" );

JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());

//JavaSparkContext sc = new JavaSparkContext();

SQLContext sqlContext = new SQLContext(sc);

// 将数据去重，转换成 List 格式

vender_set = vender_set.distinct();

vender_set = vender_set.na().fill(0L);

JavaRDD vender= vender_set.toJavaRDD();

List vender_list = vender.collect();

// 遍历商家id，调用jsf接口,创建list 保存返回数据

List list_temp = new ArrayList();

for(Row row:vender_list) {

String id = row.getString(0);

String result = service.venderDownAmountList(id);

System.out.println( "接口调用返回值 OK" );

// 解析json串 ,按照JSONObject 和 JSONArray 一层一层解析 并过返回滤数据

JSONObject jsonOBJ = JSON.parseObject(result);

JSONArray data = jsonOBJ.getJSONArray("data");

if (data != null) {

JSONObject data_all = data.getJSONObject(0);

double amount = data_all.getDouble("jfDownAmount");

// 将商家id 和 倒挂金额存下来

list_temp.add("{\"vender_id\":"+id+",\"amount\":"+amount+"}");

}

else {

continue;

}

System.out.println( "解析 OK" );

}

// list 转为 RDD

JavaRDD venderRDD = sc.parallelize(list_temp);

// 注册成表

Dataset vender_table = sqlContext.read().json(venderRDD);

vender_table.registerTempTable("vender");

System.out.println( "注册表 OK" );

// 写入数据库

spark.sql("insert overwrite table dev.dev_jypt_vender_dropaway_amount select vender.vender_id,vender.amount from vender");

System.out.println( "写入数据表 OK" );

sc.stop();

System.out.println( "Hello World!" );

}

}