SparkRDD 转存 HIVE

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.Optional;
import org.apache.spark.sql.*;
import org.apache.spark.sql.hive.HiveContext;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import scala.Tuple2;

import java.util.ArrayList;
import java.util.List;

public class TaskResultSaveAndUpdate {
    public static void main(String[] args) {
        
        SparkConf conf = new SparkConf();
        conf.setMaster("local");
        conf.setAppName("hive");
        JavaSparkContext sc = new JavaSparkContext(conf);
        SparkSession jsc = SparkSession.builder().appName("Test").master("local").enableHiveSupport().getOrCreate();
        HiveContext hiveCtx = new HiveContext(jsc);
        SQLContext sqlCtx = new SQLContext(jsc);
       
			<逻辑略>
        JavaRDD<Row> map = stringTuple2JavaPairRDD.map(f -> {
			<逻辑略>
            return RowFactory.create(user, A1, A2, A3, A4, A5, A6);
        });
        
        List<StructField> structFields = new ArrayList<StructField>();
        structFields.add(DataTypes.createStructField( "user", DataTypes.StringType, true ));
        structFields.add(DataTypes.createStructField( "A1", DataTypes.StringType, true ));
        structFields.add(DataTypes.createStructField( "A2", DataTypes.StringType, true ));
        structFields.add(DataTypes.createStructField( "A3", DataTypes.StringType, true ));
        structFields.add(DataTypes.createStructField( "A4", DataTypes.StringType, true ));
        structFields.add(DataTypes.createStructField( "A5", DataTypes.StringType, true ));
        structFields.add(DataTypes.createStructField( "A6", DataTypes.StringType, true ));
        StructType structType = DataTypes.createStructType(structFields);
        Dataset<Row> brandDF = sqlCtx.createDataFrame(map,structType);
        brandDF.registerTempTable("tableTmp");

        hiveCtx.sql("use test");
        hiveCtx.sql("CREATE TABLE IF NOT EXISTS tableHiveTmp (user string,A1 string,A2 string,A3 string,A4 string,A5 string, A6 string)");
        hiveCtx.sql("insert into tableHiveTmp select * from tableTmp");

    }
}
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值