java动态执行的job,java-Flink动态更新 streaming job

YamlReader reader = new YamlReader(topologyConfig);

EventTopologyConfig eventTopologyConfig = reader.read(EventTopologyConfig.class);

long checkPointInterval = eventTopologyConfig.getCheckPointInterval();

topics = eventTopologyConfig.getTopics();

List<EventConfig> eventTypesList = eventTopologyConfig.getEventsType();

CachedSchemaRegistryClient registryClient = new CachedSchemaRegistryClient(schemaRegistryUrl, 1000);

FlinkKafkaConsumer flinkKafkaConsumer = new FlinkKafkaConsumer(topics,

new KafkaGenericAvroDeserializationSchema(schemaRegistryUrl),

properties);

DataStream<GenericRecord> dataStream = streamExecutionEnvironment.addSource(flinkKafkaConsumer).name("source");

try {

for (EventConfig eventConfig : eventTypesList) {

LOG.info("creating a stream for ", eventConfig.getEvent_name());

final StreamingFileSink sink = StreamingFileSink.forBulkFormat

(path, ParquetAvroWriters.forGenericRecord(SchemaUtils.getSchema(eventConfig.getSchema_subject(), registryClient)))

.withBucketAssigner(new EventTimeBucketAssigner())

.build();

DataStream<GenericRecord> outStream = dataStream.filter((FilterFunction<GenericRecord>) genericRecord -> {

if (genericRecord != null && genericRecord.get(EVENT_NAME).toString().equals(eventConfig.getEvent_name())) {

return true;

}

return false;

});

outStream.addSink(sink).name(eventConfig.getSink_id()).setParallelism(parallelism);

}

} catch (Exception e) {

e.printStackTrace();

}