除官网支持的format 有一些场景需要自己定义format
举例格式为(key:value,key:value)
key有两种 带B_为before列
代码如下:
public class tra2oneRowDeserializationSchema implements DeserializationSchema<RowData> {
private static final long seriaVersionUid = 1L;
private final boolean failOnMissingField;
private final boolean ignoreParseErrors;
private final TypeInformation<RowData> resultTypeInfo;
private final ObjectMapper objectMapper = new ObjectMapper();
private final JsonToRowDataConverters.JsonToRowDataConverter runtimeConverter;
private final TimestampFormat timeStampFormat;
public trans2oneRowDeserializationSchema(
RowType rowType,
TypeInformation<RowData> resultTypeInfo,
boolean failOnMissingField,
boolean ignoreParseErrors,
TimestampFormat timeStampFormat) {
if (ignoreParseErrors && failOnMissingField) {
throw new IllegalArgumentException("Unable ignoreParseErrors and failOnMissingField are both enable");
}
this.resultTypeInfo = checkNotNull(resultTypeInfo);
this.failOnMissingField = failOnMissingField;
this.ignoreParseErrors = ignoreParseErrors;
this.runtimeConverter = new JsonToRowDataConverters(failOnMissingField, ignoreParseErrors, timeStampFormat).createConverter(checkNotNull(rowType));
this.timeStampFormat = timeStampFormat;
boolean hasDecimalType = LogicalTypeChecks.hasNested(rowType, t -> t instanceof DecimalType);
if (hasDecimalType) {
objectMapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS);
}
objectMapper.configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true);
}
@Override
public RowData deserialize(byte[] message) throws IOException {
if (message == null || message.length <= 0) {
return null;
}
String s = new String(message);
String data = JSONObject.parse(s).toString();
String[] split = data.split("\\|\\|\\$");
GenericRowData genericRowData = new GenericRowData(split.length);
for (int i = 0; i < split.length; i++) {
genericRowData.setField(i,(split[i].split("\\|\\|\\#",3))[2]);
}
return genericRowData;
}
@Override
public boolean isEndOfStream(RowData nextElement) {
return false;
}
@Override
public TypeInformation<RowData> getProducedType() {
return null;
}
}
Factory
public class trans2oneRowFactory implements DeserializationFormatFactory, SerializationFormatFactory {
public static final String IDENTIFIER = "trans";
@Override
public DecodingFormat<DeserializationSchema<RowData>> createDecodingFormat(DynamicTableFactory.Context context, ReadableConfig formatOptions) {
FactoryUtil.validateFactoryOptions(this, formatOptions);
try {
validateFormatOptions(formatOptions);
} catch (ValidationException e) {
e.printStackTrace();
}
final boolean failOnMissingField = formatOptions.get(FAIL_ON_MISSING_FIELD);
final boolean igoreParseErrors = formatOptions.get(IGNORE_PARSE_ERRORS);
TimestampFormat timestampOption = OneRowFormatUtil.getTimestampFormat(formatOptions);
return new DecodingFormat<DeserializationSchema<RowData>>() {
@Override
public DeserializationSchema<RowData> createRuntimeDecoder(DynamicTableSource.Context context, DataType physicalDataType) {
final RowType rowType = (RowType) physicalDataType.getLogicalType();
final TypeInformation<RowData> rowDataTypeInfo = context.createTypeInformation(physicalDataType);
return new trans2oneRowDeserializationSchema(rowType,rowDataTypeInfo,failOnMissingField,igoreParseErrors,timestampOption);
}
@Override
public ChangelogMode getChangelogMode() {
return ChangelogMode.insertOnly();
}
};
}
@Override
public EncodingFormat<SerializationSchema<RowData>> createEncodingFormat(DynamicTableFactory.Context context, ReadableConfig formatOptions) {
return null;
}
@Override
public String factoryIdentifier() {
return null;
}
@Override
public Set<ConfigOption<?>> requiredOptions() {
return null;
}
@Override
public Set<ConfigOption<?>> optionalOptions() {
return null;
}
static void validateFormatOptions(ReadableConfig tableOptions) throws ValidationException {
Boolean failOnMissingField = tableOptions.get(FAIL_ON_MISSING_FIELD);
Boolean ignoreParseErrors = tableOptions.get(IGNORE_PARSE_ERRORS);
if (ignoreParseErrors && failOnMissingField) {
throw new ValidationException(FAIL_ON_MISSING_FIELD.key()
+ " and "
+ IGNORE_PARSE_ERRORS.key()
+ " shouldn't both be true");
}
String timestampFormat = tableOptions.get(TIMESTAMP_FORMAT);
if (!OneRowFormatUtil.TIMESTAMP_FORMAT_ENUM.contains(timestampFormat)) {
throw new ValidationException(
String.format(
"Unsupported value '%s' for %s. Supported values are [SQL,ISO-8601].",
timestampFormat, TIMESTAMP_FORMAT.key()));
}
}
private trans2oneRowFactory() {
}
}
public enum TimeStampFormat {
SQL,
ISO_8601
}
public class OneRowFormatUtil {
public static final Set<String> TIMESTAMP_FORMAT_ENUM =
new HashSet<>(Arrays.asList("SQL", "ISO-8601"));
public static TimestampFormat getTimestampFormat(ReadableConfig config){
String timestampFormat = config.get(TIMESTAMP_FORMAT);
switch (timestampFormat){
case "SQL":
return TimestampFormat.SQL;
case "ISO-8601":
return TimestampFormat.ISO_8601;
default:
throw new TableException(String.format("Unsupoorted",timestampFormat));
}
}
public static <T> T checkNotNull(@Nullable T reference){
if (reference == null){
throw new NullPointerException();
}
return reference;
}
}