自定义flink sql format

除官网支持的format 有一些场景需要自己定义format
举例格式为(key:value,key:value)
key有两种 带B_为before列
代码如下:

public class tra2oneRowDeserializationSchema implements DeserializationSchema<RowData> {
    private static final long seriaVersionUid = 1L;
    private final boolean failOnMissingField;
    private final boolean ignoreParseErrors;
    private final TypeInformation<RowData> resultTypeInfo;
    private final ObjectMapper objectMapper = new ObjectMapper();
    private final JsonToRowDataConverters.JsonToRowDataConverter runtimeConverter;
    private final TimestampFormat timeStampFormat;

    public trans2oneRowDeserializationSchema(
            RowType rowType,
            TypeInformation<RowData> resultTypeInfo,
            boolean failOnMissingField,
            boolean ignoreParseErrors,
            TimestampFormat timeStampFormat) {
        if (ignoreParseErrors && failOnMissingField) {
            throw new IllegalArgumentException("Unable ignoreParseErrors and failOnMissingField are both enable");
        }
        this.resultTypeInfo = checkNotNull(resultTypeInfo);
        this.failOnMissingField = failOnMissingField;
        this.ignoreParseErrors = ignoreParseErrors;
        this.runtimeConverter = new JsonToRowDataConverters(failOnMissingField, ignoreParseErrors, timeStampFormat).createConverter(checkNotNull(rowType));
        this.timeStampFormat = timeStampFormat;
        boolean hasDecimalType = LogicalTypeChecks.hasNested(rowType, t -> t instanceof DecimalType);
        if (hasDecimalType) {
            objectMapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS);
        }
        objectMapper.configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true);
    }

    @Override
    public RowData deserialize(byte[] message) throws IOException {
        if (message == null || message.length <= 0) {
            return null;
        }
        String s = new String(message);
        String data = JSONObject.parse(s).toString();
        String[] split = data.split("\\|\\|\\$");
        GenericRowData genericRowData = new GenericRowData(split.length);
        for (int i = 0; i < split.length; i++) {
            genericRowData.setField(i,(split[i].split("\\|\\|\\#",3))[2]);
        }
        return genericRowData;
    }

    @Override
    public boolean isEndOfStream(RowData nextElement) {
        return false;
    }

    @Override
    public TypeInformation<RowData> getProducedType() {
        return null;
    }
}

Factory

public class trans2oneRowFactory implements DeserializationFormatFactory, SerializationFormatFactory {
    public static final String IDENTIFIER = "trans";

    @Override
    public DecodingFormat<DeserializationSchema<RowData>> createDecodingFormat(DynamicTableFactory.Context context, ReadableConfig formatOptions) {
        FactoryUtil.validateFactoryOptions(this, formatOptions);
        try {
            validateFormatOptions(formatOptions);
        } catch (ValidationException e) {
            e.printStackTrace();
        }
        final boolean failOnMissingField = formatOptions.get(FAIL_ON_MISSING_FIELD);
        final boolean igoreParseErrors = formatOptions.get(IGNORE_PARSE_ERRORS);
        TimestampFormat timestampOption = OneRowFormatUtil.getTimestampFormat(formatOptions);
        return new DecodingFormat<DeserializationSchema<RowData>>() {
            @Override
            public DeserializationSchema<RowData> createRuntimeDecoder(DynamicTableSource.Context context, DataType physicalDataType) {
                final RowType rowType = (RowType) physicalDataType.getLogicalType();
                final TypeInformation<RowData> rowDataTypeInfo = context.createTypeInformation(physicalDataType);
                return new trans2oneRowDeserializationSchema(rowType,rowDataTypeInfo,failOnMissingField,igoreParseErrors,timestampOption);

            }

            @Override
            public ChangelogMode getChangelogMode() {
                return ChangelogMode.insertOnly();
            }
        };
    }

    @Override
    public EncodingFormat<SerializationSchema<RowData>> createEncodingFormat(DynamicTableFactory.Context context, ReadableConfig formatOptions) {
        return null;
    }

    @Override
    public String factoryIdentifier() {
        return null;
    }

    @Override
    public Set<ConfigOption<?>> requiredOptions() {
        return null;
    }

    @Override
    public Set<ConfigOption<?>> optionalOptions() {
        return null;
    }


    static void validateFormatOptions(ReadableConfig tableOptions) throws ValidationException {
        Boolean failOnMissingField = tableOptions.get(FAIL_ON_MISSING_FIELD);
        Boolean ignoreParseErrors = tableOptions.get(IGNORE_PARSE_ERRORS);
        if (ignoreParseErrors && failOnMissingField) {
            throw new ValidationException(FAIL_ON_MISSING_FIELD.key()
                    + " and "
                    + IGNORE_PARSE_ERRORS.key()
                    + " shouldn't both be true");
        }
        String timestampFormat = tableOptions.get(TIMESTAMP_FORMAT);
        if (!OneRowFormatUtil.TIMESTAMP_FORMAT_ENUM.contains(timestampFormat)) {
            throw new ValidationException(
                    String.format(
                            "Unsupported value '%s' for %s. Supported values are [SQL,ISO-8601].",
                            timestampFormat, TIMESTAMP_FORMAT.key()));
        }
    }

    private trans2oneRowFactory() {
    }
}

public enum TimeStampFormat {
    SQL,
    ISO_8601
}
public class OneRowFormatUtil {
    public static final Set<String> TIMESTAMP_FORMAT_ENUM =
            new HashSet<>(Arrays.asList("SQL", "ISO-8601"));


    public static TimestampFormat getTimestampFormat(ReadableConfig config){
        String timestampFormat = config.get(TIMESTAMP_FORMAT);
        switch (timestampFormat){
            case "SQL":
                return TimestampFormat.SQL;
            case "ISO-8601":
                return TimestampFormat.ISO_8601;
            default:
                throw new TableException(String.format("Unsupoorted",timestampFormat));
        }
    }

    public static <T> T checkNotNull(@Nullable T reference){
        if (reference == null){
            throw new NullPointerException();
        }
        return reference;
    }
}
  • 10
    点赞
  • 7
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值