动态为数值型列设置默认值0

最新推荐文章于 2023-04-15 16:21:58 发布

jane3von

最新推荐文章于 2023-04-15 16:21:58 发布

阅读量711

点赞数

分类专栏： spark

本文链接：https://blog.csdn.net/jane3von/article/details/119445462

版权

spark 专栏收录该内容

6 篇文章 0 订阅

订阅专栏

public Dataset<Row> setDefaultOneZero(Dataset<Row> ds,String formName){
        StructType structType = ds.schema();
        int len = structType.fields().length;
        String keyStr = global.getMap().get("default.parameter").replaceAll("\\|",",");
        String tmpViewName = formName+"_zeroTable";
        tmpViewName = tmpViewName.replaceAll("-","_");
        boolean isExist = global.getSparkSession().catalog().tableExists(tmpViewName);
        if(isExist==true){
            Dataset<Row> defaultDs = global.getSparkSession().sql("select * from "+tmpViewName);
            Seq<String> keys = HadoopUtils.getNCellSchema(keyStr);
            defaultDs = defaultDs.join(ds,keys,"left");
            for(int i=0;i<len;i++) {
                String type = structType.fields()[i].dataType().toString().toLowerCase();
                String name = structType.fields()[i].name();
                if (type.startsWith("decimal")||type.startsWith("int")||
                        type.startsWith("double")||type.startsWith("long")||
                        type.startsWith("bigdecimal")) {

                    defaultDs = defaultDs.selectExpr("*","case when "+name+" is not NULL then "+name+" else 0 end as "+name+"_1")
                            .drop(col(name)).withColumnRenamed(name+"_1",name);
                }
            }
            return defaultDs;
        }else{
            return ds;
        }
    }

public void generateZeroDS(){
        String key = "default.parameter";
        String isOpen = Optional.ofNullable(global.getMap().get("default.zero.enable")).orElse("false");
        if("true".equalsIgnoreCase(isOpen)){
            String defaultStr[] = global.getMap().get(key).split("\\|");
            for(String formName : global.getFormNameList()){
               
                String strDs = String.format("{'%s':'%s','%s':'%s','%s':'%s'}",
                        defaultStr[0],
                        new DateUtils().getAsOfDate(global.getArgument().getCobDate()),
                        defaultStr[1],
                        global.getIdMap().get(formName),
                        defaultStr[2],
                        global.getBranchMap().get(formName)
                );
                ArrayList<String> tmpList = new ArrayList<>();
                tmpList.add(strDs);
                logger.info("generateZeroDS: strDs="+strDs);
                JavaRDD<String> tmpRdd = Global.sc.parallelize(tmpList);
                Dataset<Row> zeroDs = global.getSparkSession().read()
                        .option(Global.TIMESTAMP_FORMAT,Global.TIMESTAMPFORMATVALUE)
                        .json(tmpRdd);
                String s = formName+"_zeroTable";
                s = s.replaceAll("-","_");
                zeroDs.createOrReplaceTempView(s);
                logger.info("generateZeroDS: zeroTable generate in sparkSession !");
                zeroDs.show(false);
            }
        }
    }