背景:
flink1.8
实现暂停消费kafka
并能接受到通知,继续消费kafka
需求描述:
实现思路
类1:
package org.apache.flink.streaming.connectors.kafka.internal;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @author hulb
* @date 2019-05-06 09:13
*/
public class ProcessUtils {
private final static String TIME_FORMAT_DAY = "yyyyMMdd";
private final static String FULL_TIME_FORMAT_DAY = "yyyy-MM-dd HH:mm:ss";
private final static Integer dayTimeLength = 10;
private final static String ENABLE_FLAG = "enable";
private final static Integer TIME_FLAG_START = 19;
private final static Integer TIME_FLAG_END= 2;
/**
* 获得今日的数据处理日期
*
* @return
*/
public static String generateProcessTime() {
DateTime dateTime = new DateTime();
return dateTime.toString(TIME_FORMAT_DAY);
}
public static String getNowTime() {
DateTime dateTime = new DateTime();
return dateTime.toString(FULL_TIME_FORMAT_DAY);
}
public static boolean timeFlag() {
return true;
// DateTime dateTime = new DateTime();
// Integer hour = dateTime.getHourOfDay();
// if(hour > TIME_FLAG_START || hour < TIME_FLAG_END){
// return true;
// }
// return false;
}
/**
* 进行程序启动的初始状态检测.如果日切 已经 开始 且 没有完成。则进行consumer的pause操作
* @return
*/
public static boolean startCheck(String processTime,Properties properties) {
//flink 程序今日是否触发过暂停逻辑
String checkpointSql = properties.getProperty("database.checkpoint.sql");
String decodeCheckSql = decode(checkpointSql);
//String flinkStateSql = "select count(1) from switch_checkpoint_status s where s.checkpoint_completed = '1' and switch_date = '";
String flinkStateSql = decodeCheckSql + processTime +"'";
Integer flinkStateFlag = JDBCHelper.getAll(JDBCHelper.getStateConn(properties),flinkStateSql);
//oracle库是否已经完成 日切。
String querySql = properties.getProperty("database.query.sql");
String decodeOracleSql = decode(querySql);
//String oracleSql = "select count(1) from BA_EOD_RESTART b where b.time = '" + processTime + "'";
String oracleSql = decodeOracleSql + processTime + "'";
Integer oracleFlag = JDBCHelper.getAll(JDBCHelper.getConn(properties),oracleSql);
if(flinkStateFlag >= 1 && oracleFlag == 0){
return true;
}else {
return false;
}
}
/**
* 如果未指定,则用当日日期
* 如果指定了,就用指定的日期
*
* @param properties
* @return
*/
public static String generateProcessTime(Properties properties) {
String processTime = properties.getProperty("init.process.time");
if (processTime == null || "".equals(processTime)) {
return generateProcessTime();
}
//格式判断 如果出问题 直接报错
return processTime;
}
public static Boolean pauseEnable(Properties properties) {
String processTime = properties.getProperty("event.pause.enable");
if (processTime != null && ENABLE_FLAG.equals(processTime)) {
return true;
}
//格式判断 如果出问题 直接报错
return false;
}
public static Boolean saveStateEnable(Properties properties) {
String processTime = properties.getProperty("save.database.enable");
if (processTime != null && ENABLE_FLAG.equals(processTime)) {
return true;
}
//格式判断 如果出问题 直接报错
return false;
}
/**
* 比较处理日期和柜面日期
* 先对数据进行格式化截取后再做比较
* 判断新的数据是否是新的账面日期的数据 则需要根据日期做 + 1 操作后 再判断是否相等
*
* @param dataTime kafka数据的dataTime 为完整时间戳格式
* @param processTime 为标准的yyyyMMdd格式
* @return
*/
public static Boolean compareProcessTimeAndDataTime(String dataTime, String processTime) {
if (dataTime != null && dataTime.length() > dayTimeLength) {
// 截取日期部分并去除 - 字符串
String dayTime = dataTime.substring(0, dayTimeLength).replace("-", "");
// 判断是否为新的一天
/// return !processTime.equals(dayTime);
if (increaseProcessTime(processTime).equals(dayTime)) {
System.out.println("比较dataTime 和 processTime -- 此时为true 需要进行暂停" + dataTime+" - "+processTime);
//如果时间时间比处理时间正好大一天,则返回true需要进行暂停
return true;
} else {
return false;
}
} else {
return false;
}
}
/**
* 将处理柜面日期新增一天
*
* @param oldTime
* @return
*/
public static synchronized String increaseProcessTime(String oldTime) {
DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern(TIME_FORMAT_DAY);
DateTime dateTime = dateTimeFormatter.parseDateTime(oldTime);
String dateTimePlus = dateTime.plusDays(1).toString(TIME_FORMAT_DAY);
//System.out.println("增加一天后的新的一天日期: " + dateTimePlus);
return dateTimePlus;
}
public static String decode(String encodeString) {
if (encodeString != null) {
return new String(Base64.getDecoder().decode(encodeString));
} else {
return "";
}
}
/**
* 开始暂停
*
* @return
*/
public static void saveState(Boolean flag,Properties kafkaProperties, String processTime,String desc) {
System.err.print("ProcessUtils状态,processTime:"+processTime+" desc: "+desc);
if(flag){
JDBCHelper.updateState(kafkaProperties, processTime, desc);
}
}
}
类2:负责恢复读取消息
package org.apache.flink.streaming.connectors.kafka.internal;
import java.util.Properties;
/**
* @author hulb
* @date 2019-10-12 16:54
*/
public class ResumeThread extends Thread {
private String processTime;
private Properties kafkaProperties;
private KafkaConsumerThread consumer;
public ResumeThread(String processTime,Properties kafkaProperties,KafkaConsumerThread consumer){
this.processTime = processTime;
this.kafkaProperties = kafkaProperties;
this.consumer= consumer;
}
@Override
public void run(){
String querySql = kafkaProperties.getProperty("database.query.sql");
String sql = ProcessUtils.decode(querySql) + processTime + "'";
while (true) {
try {
//条件1.日切结束
Integer result = JDBCHelper.getAll(JDBCHelper.getConn(kafkaProperties), sql);
//条件2. 余额数据成功同步到hbase中。如果任务没有同步成功。?
//如果有值 则break
if (result > 0) {
consumer.setResumed(true);
consumer.setResumedThreadStart(false);
break;
} else {
//查询
Thread.sleep(10000);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
类3:
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.streaming.connectors.kafka.internal;
import com.alibaba.fastjson.JSONObject;
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.io.ratelimiting.FlinkConnectorRateLimiter;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.connectors.kafka.internals.ClosableBlockingQueue;
import org.apache.flink.streaming.connectors.kafka.internals.KafkaCommitCallback;
import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartitionState;
import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartitionStateSentinel;
import org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaMetricWrapper;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.consumer.OffsetCommitCallback;
import org.apache.kafka.common.Metric;
import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.errors.WakeupException;
import org.slf4j.Logger;
import javax.annotation.Nonnull;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* The thread the runs the {@link KafkaConsumer}, connecting to the brokers and polling records.
* The thread pushes the data into a {@link Handover} to be picked up by the fetcher that will
* deserialize and emit the records.
*
* <p><b>IMPORTANT:</b> This thread must not be interrupted when attempting to shut it down.
* The Kafka consumer code was found to not always handle interrupts well, and to even
* deadlock in certain situations.
*
* <p>Implementation Note: This code is written to be reusable in later versions of the KafkaConsumer.
* Because Kafka is not maintaining binary compatibility, we use a "call bridge" as an indirection
* to the KafkaConsumer calls that change signature.
*/
@Internal
public class KafkaConsumerThread extends Thread {
/**
* Logger for this consumer.
*/
private final Logger log;
/**
* The handover of data and exceptions between the consumer thread and the task thread.
*/
private final Handover handover;
/**
* The next offsets that the main thread should commit and the commit callback.
*/
private final AtomicReference<Tuple2<Map<TopicPartition, OffsetAndMetadata>, KafkaCommitCallback>> nextOffsetsToCommit;
/**
* The configuration for the Kafka consumer.
*/
private final Properties kafkaProperties;
/**
* The queue of unassigned partitions that we need to assign to the Kafka consumer.
*/
private final ClosableBlockingQueue<KafkaTopicPartitionState<TopicPartition>> unassignedPartitionsQueue;
/**
* The indirections on KafkaConsumer methods, for cases where KafkaConsumer compatibility is broken.
*/
private final KafkaConsumerCallBridge09 consumerCallBridge;
/**
* The maximum number of milliseconds to wait for a fetch batch.
*/
private final long pollTimeout;
/**
* Flag whether to add Kafka's metrics to the Flink metrics.
*/
private final boolean useMetrics;
/**
* @deprecated We should only be publishing to the {{@link #consumerMetricGroup}}.
* This is kept to retain compatibility for metrics.
**/
@Deprecated
private final MetricGroup subtaskMetricGroup;
/**
* We get this from the outside to publish metrics.
*/
private final MetricGroup consumerMetricGroup;
/**
* Reference to the Kafka consumer, once it is created.
*/
private volatile KafkaConsumer<byte[], byte[]> consumer;
/**
* This lock is used to isolate the consumer for partition reassignment.
*/
private final Object consumerReassignmentLock;
/**
* Indication if this consumer has any assigned partition.
*/
private boolean hasAssignedPartitions;
/**
* Flag to indicate whether an external operation ({@link #setOffsetsToCommit(Map, KafkaCommitCallback)}
* or {@link #shutdown()}) had attempted to wakeup the consumer while it was isolated for partition reassignment.
*/
private volatile boolean hasBufferedWakeup;
/**
* Flag to mark the main work loop as alive.
*/
private volatile boolean running;
/**
* Flag tracking whether the latest commit request has completed.
*/
private volatile boolean commitInProgress;
/**
* Ratelimiter.
*/
private FlinkConnectorRateLimiter rateLimiter;
/**
* 任务是否暂停
*/
private volatile boolean pausing;
public boolean isResumed() {
return resumed;
}
public void setResumed(boolean resumed) {
this.resumed = resumed;
}
private volatile boolean resumed;
public void setResumedThreadStart(boolean resumedThreadStart) {
this.resumedThreadStart = resumedThreadStart;
}
private volatile boolean resumedThreadStart;
/**
* 是否 是暂停后的 第一个"checkPoint"
*/
private boolean firstCheckpointOnPause;
/**
* 今日份的处理时间 比如20190506日只处理0506日的数据 一旦遇到0507账面日期的数据就进行加1
*/
private volatile String processTime = "";
//private static volatile AtomicInteger atomicIntegerProcessTime;
private volatile Boolean pauseEnable;
private volatile Boolean saveStateEnable;
private volatile Boolean firstCheck;
private static String PAUSE_MEAAGE = "{\"table\":\"PAUSE\",\"op_type\":\"I\",\"after\":{\"PAUSE\":\"PAUSE\"}}";
public KafkaConsumerThread(
Logger log,
Handover handover,
Properties kafkaProperties,
ClosableBlockingQueue<KafkaTopicPartitionState<TopicPartition>> unassignedPartitionsQueue,
KafkaConsumerCallBridge09 consumerCallBridge,
String threadName,
long pollTimeout,
boolean useMetrics,
MetricGroup consumerMetricGroup,
MetricGroup subtaskMetricGroup,
FlinkConnectorRateLimiter rateLimiter) {
super(threadName);
setDaemon(true);
this.log = checkNotNull(log);
this.handover = checkNotNull(handover);
this.kafkaProperties = checkNotNull(kafkaProperties);
this.consumerMetricGroup = checkNotNull(consumerMetricGroup);
this.subtaskMetricGroup = checkNotNull(subtaskMetricGroup);
this.consumerCallBridge = checkNotNull(consumerCallBridge);
this.unassignedPartitionsQueue = checkNotNull(unassignedPartitionsQueue);
this.pollTimeout = pollTimeout;
this.useMetrics = useMetrics;
this.consumerReassignmentLock = new Object();
this.nextOffsetsToCommit = new AtomicReference<>();
this.running = true;
if (rateLimiter != null) {
this.rateLimiter = rateLimiter;
}
/**
* 生成初始化的数据
*/
processTime = ProcessUtils.generateProcessTime(kafkaProperties);
this.pauseEnable = ProcessUtils.pauseEnable(kafkaProperties);
this.saveStateEnable = ProcessUtils.saveStateEnable(kafkaProperties);
this.firstCheck = true;
}
// ------------------------------------------------------------------------
@Override
public void run() {
// early exit check
if (!running) {
return;
}
// this is the means to talk to FlinkKafkaConsumer's main thread
final Handover handover = this.handover;
// This method initializes the KafkaConsumer and guarantees it is torn down properly.
// This is important, because the consumer has multi-threading issues,
// including concurrent 'close()' calls.
try {
this.consumer = getConsumer(kafkaProperties);
} catch (Throwable t) {
handover.reportError(t);
return;
}
// from here on, the consumer is guaranteed to be closed properly
try {
// register Kafka's very own metrics in Flink's metric reporters
if (useMetrics) {
// register Kafka metrics to Flink
Map<MetricName, ? extends Metric> metrics = consumer.metrics();
if (metrics == null) {
// MapR's Kafka implementation returns null here.
log.info("Consumer implementation does not support metrics");
} else {
// we have Kafka metrics, register them
for (Map.Entry<MetricName, ? extends Metric> metric : metrics.entrySet()) {
consumerMetricGroup.gauge(metric.getKey().name(), new KafkaMetricWrapper(metric.getValue()));
// TODO this metric is kept for compatibility purposes; should remove in the future
subtaskMetricGroup.gauge(metric.getKey().name(), new KafkaMetricWrapper(metric.getValue()));
}
}
}
// early exit check
if (!running) {
return;
}
// the latest bulk of records. May carry across the loop if the thread is woken up
// from blocking on the handover
ConsumerRecords<byte[], byte[]> records = null;
// reused variable to hold found unassigned new partitions.
// found partitions are not carried across loops using this variable;
// they are carried across via re-adding them to the unassigned partitions queue
List<KafkaTopicPartitionState<TopicPartition>> newPartitions;
// main fetch loop
while (running) {
// check if there is something to commit
if (!commitInProgress) {
// get and reset the work-to-be committed, so we don't repeatedly commit the same
final Tuple2<Map<TopicPartition, OffsetAndMetadata>, KafkaCommitCallback> commitOffsetsAndCallback =
nextOffsetsToCommit.getAndSet(null);
if (commitOffsetsAndCallback != null) {
log.debug("Sending async offset commit request to Kafka broker");
// also record that a commit is already in progress
// the order here matters! first set the flag, then send the commit command.
commitInProgress = true;
if (pausing && firstCheckpointOnPause) {
System.out.println("进行暂停后的第一次offset提交"+ProcessUtils.getNowTime());
// 暂停
consumer.commitSync(commitOffsetsAndCallback.f0);
commitInProgress = false;
/**
* TODO 将topic 写入oracle状态表,格式: 日期、 topicName 、状态(已做过checkpoint)
* TODO 当订阅的所有topic 都在暂停后 做过 checkpoint,重启应用
*
*/
Set<TopicPartition> topicPartitions = consumer.assignment();
//commitOffsetsAndCallback.f0.get("ds").offset();
for (TopicPartition topicPartition : topicPartitions) {
System.out.println("进行暂停后的第一次offset提交,topic:"+topicPartition.topic()+ProcessUtils.getNowTime());
JDBCHelper.saveCheckpointState(kafkaProperties, processTime, "0", topicPartition.topic(), "1");
}
//置为false 这样在pause后的第二个checkpoint就不会将状态写入到oracle中了。
firstCheckpointOnPause = false;
} else {
consumer.commitAsync(commitOffsetsAndCallback.f0, new CommitCallback(commitOffsetsAndCallback.f1));
}
}
}
try {
if (hasAssignedPartitions) {
newPartitions = unassignedPartitionsQueue.pollBatch();
} else {
// if no assigned partitions block until we get at least one
// instead of hot spinning this loop. We rely on a fact that
// unassignedPartitionsQueue will be closed on a shutdown, so
// we don't block indefinitely
newPartitions = unassignedPartitionsQueue.getBatchBlocking();
}
if (newPartitions != null) {
reassignPartitions(newPartitions);
}
} catch (AbortedReassignmentException e) {
continue;
}
if (!hasAssignedPartitions) {
// Without assigned partitions KafkaConsumer.poll will throw an exception
continue;
}
if (firstCheck) {
//判断是否在日切好,如果没有日切好,直接暂停。
System.out.println("进行是否暂停判断");
if (ProcessUtils.startCheck(processTime, kafkaProperties)) {
System.out.println("暂停==============pauseTopicPartitions");
consumerCallBridge.pauseTopicPartitions(consumer, consumer.assignment());
if(!this.resumedThreadStart){
System.out.println("启动监听日切thread");
ResumeThread resumeThread = new ResumeThread(processTime, kafkaProperties, this);
resumeThread.start();
this.resumedThreadStart=true;
}
}
//只有重启的时候需要检查一次今日是否做过一次暂停,且日切没有结束
firstCheck = false;
}
SimpleStringSchema simpleStringSchema = new SimpleStringSchema();
// get the next batch of records, unless we did not manage to hand the old batch over
if (records == null) {
try {
records = getRecordsFromKafka();
} catch (WakeupException we) {
continue;
}
}
Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> pendingRecords = new HashMap<>();
Iterator<ConsumerRecord<byte[], byte[]>> recordIter = records.iterator();
ConsumerRecord<byte[], byte[]> record;
while (recordIter.hasNext()) {
record = recordIter.next();
final String value = simpleStringSchema.deserialize(record.value());
/// 临时暂停的逻辑 如果发了一条value是让程序临时暂停的 但是不能把这条消息阻塞在这里, 要继续往下走,让这条数据被消费掉。
//TODO 如果时间在晚上10点到第二天2点前,才进行判断,否则跳过。
Boolean timeFlag = ProcessUtils.timeFlag();
if (pauseEnable && timeFlag) {
if(PAUSE_MEAAGE.equals(value)){
pausing = true;
TopicPartition tp = new TopicPartition(record.topic(), record.partition());
if (!pendingRecords.containsKey(tp)) {
pendingRecords.put(tp, new LinkedList<>());
}
pendingRecords.get(tp).add(record);
}else{
try {
JSONObject jsonObject = JSONObject.parseObject(value);
String tableName = jsonObject.getString("table");
String chTableName = kafkaProperties.getProperty("table.name.ch");
// 只有chnobook才会触发判断
if (chTableName.equals(tableName)) {
String dataObject = kafkaProperties.getProperty("table.data.object");
String timeColumn = kafkaProperties.getProperty("table.time.column");
// 获得日期字段
String dataTime = jsonObject
.getJSONObject(dataObject)
.getString(timeColumn);
if (ProcessUtils.compareProcessTimeAndDataTime(dataTime, processTime)) {
pausing = true;
}
}
} catch (Exception e) {
continue;
}
}
}
// TODO 如果开始日切
if (pausing && !resumedThreadStart) {
ProcessUtils.saveState(saveStateEnable, kafkaProperties, processTime, "start pause");
consumerCallBridge.pauseTopicPartitions(consumer, consumer.assignment());
firstCheckpointOnPause = true;
//启动新的进程监听 日切状态 如果日切状态OK,将恢复标识resume 置为true 。下面逻辑即可恢复。 重新读取程序。
if(!resumedThreadStart){
System.out.println("启动监听日切thread");
ResumeThread resumeThread = new ResumeThread(processTime, kafkaProperties, this);
resumeThread.start();
ProcessUtils.saveState(saveStateEnable, kafkaProperties, processTime, "start waitCompute");
this.resumedThreadStart=true;
}
}
if(!pausing){
TopicPartition tp = new TopicPartition(record.topic(), record.partition());
if (!pendingRecords.containsKey(tp)) {
pendingRecords.put(tp, new LinkedList<>());
}
pendingRecords.get(tp).add(record);
}
}
if (resumed) {
processTime = ProcessUtils.increaseProcessTime(processTime);
System.out.println("增加一天后的新的processTime日期: " + processTime);
ProcessUtils.saveState(saveStateEnable, kafkaProperties, processTime, "start resume");
consumerCallBridge.resumeTopicPartitions(consumer, consumer.assignment());
ProcessUtils.saveState(saveStateEnable, kafkaProperties, processTime, "resumed");
//恢复标识
firstCheckpointOnPause = false;
//将暂停标识置为 false 表示已经处于非暂停状态
pausing = false;
//将需要恢复标识置为false 防止下次进入此逻辑进行恢复
resumed = false;
}
//如果是处于暂停状态(防止pendingRecords为空 而将错误的record往下传递) 或者 record不为空
if (pausing || !pendingRecords.isEmpty()) {
records = new ConsumerRecords<>(pendingRecords);
}
try {
handover.produce(records);
records = null;
} catch (Handover.WakeupException e) {
// fall through the loop
}
}
// end main fetch loop
} catch (Throwable t) {
// let the main thread know and exit
// it may be that this exception comes because the main thread closed the handover, in
// which case the below reporting is irrelevant, but does not hurt either
handover.reportError(t);
} finally {
// make sure the handover is closed if it is not already closed or has an error
handover.close();
// If a ratelimiter was created, make sure it's closed.
if (rateLimiter != null) {
rateLimiter.close();
}
// make sure the KafkaConsumer is closed
try {
consumer.close();
} catch (Throwable t) {
log.warn("Error while closing Kafka consumer", t);
}
}
}
/**
* Shuts this thread down, waking up the thread gracefully if blocked (without Thread.interrupt() calls).
*/
public void shutdown() {
running = false;
// wake up all blocking calls on the queue
unassignedPartitionsQueue.close();
// We cannot call close() on the KafkaConsumer, because it will actually throw
// an exception if a concurrent call is in progress
// this wakes up the consumer if it is blocked handing over records
handover.wakeupProducer();
// this wakes up the consumer if it is blocked in a kafka poll
synchronized (consumerReassignmentLock) {
if (consumer != null) {
consumer.wakeup();
} else {
// the consumer is currently isolated for partition reassignment;
// set this flag so that the wakeup state is restored once the reassignment is complete
hasBufferedWakeup = true;
}
}
// If a ratelimiter was created, make sure it's closed.
if (rateLimiter != null) {
rateLimiter.close();
}
}
/**
* Tells this thread to commit a set of offsets. This method does not block, the committing
* operation will happen asynchronously.
*
* <p>Only one commit operation may be pending at any time. If the committing takes longer than
* the frequency with which this method is called, then some commits may be skipped due to being
* superseded by newer ones.
*
* @param offsetsToCommit The offsets to commit
* @param commitCallback callback when Kafka commit completes
*/
void setOffsetsToCommit(
Map<TopicPartition, OffsetAndMetadata> offsetsToCommit,
@Nonnull KafkaCommitCallback commitCallback) {
// record the work to be committed by the main consumer thread and make sure the consumer notices that
if (nextOffsetsToCommit.getAndSet(Tuple2.of(offsetsToCommit, commitCallback)) != null) {
log.warn("Committing offsets to Kafka takes longer than the checkpoint interval. " +
"Skipping commit of previous offsets because newer complete checkpoint offsets are available. " +
"This does not compromise Flink's checkpoint integrity.");
}
// if the consumer is blocked in a poll() or handover operation, wake it up to commit soon
handover.wakeupProducer();
synchronized (consumerReassignmentLock) {
if (consumer != null) {
consumer.wakeup();
} else {
// the consumer is currently isolated for partition reassignment;
// set this flag so that the wakeup state is restored once the reassignment is complete
hasBufferedWakeup = true;
}
}
}
// ------------------------------------------------------------------------
/**
* Reestablishes the assigned partitions for the consumer.
* The reassigned partitions consists of the provided new partitions and whatever partitions
* was already previously assigned to the consumer.
*
* <p>The reassignment process is protected against wakeup calls, so that after
* this method returns, the consumer is either untouched or completely reassigned
* with the correct offset positions.
*
* <p>If the consumer was already woken-up prior to a reassignment resulting in an
* interruption any time during the reassignment, the consumer is guaranteed
* to roll back as if it was untouched. On the other hand, if there was an attempt
* to wakeup the consumer during the reassignment, the wakeup call is "buffered"
* until the reassignment completes.
*
* <p>This method is exposed for testing purposes.
*/
@VisibleForTesting
void reassignPartitions(List<KafkaTopicPartitionState<TopicPartition>> newPartitions) throws Exception {
if (newPartitions.size() == 0) {
return;
}
hasAssignedPartitions = true;
boolean reassignmentStarted = false;
// since the reassignment may introduce several Kafka blocking calls that cannot be interrupted,
// the consumer needs to be isolated from external wakeup calls in setOffsetsToCommit() and shutdown()
// until the reassignment is complete.
final KafkaConsumer<byte[], byte[]> consumerTmp;
synchronized (consumerReassignmentLock) {
consumerTmp = this.consumer;
this.consumer = null;
}
final Map<TopicPartition, Long> oldPartitionAssignmentsToPosition = new HashMap<>();
try {
for (TopicPartition oldPartition : consumerTmp.assignment()) {
oldPartitionAssignmentsToPosition.put(oldPartition, consumerTmp.position(oldPartition));
}
final List<TopicPartition> newPartitionAssignments =
new ArrayList<>(newPartitions.size() + oldPartitionAssignmentsToPosition.size());
newPartitionAssignments.addAll(oldPartitionAssignmentsToPosition.keySet());
newPartitionAssignments.addAll(convertKafkaPartitions(newPartitions));
// reassign with the new partitions
consumerCallBridge.assignPartitions(consumerTmp, newPartitionAssignments);
reassignmentStarted = true;
// old partitions should be seeked to their previous position
for (Map.Entry<TopicPartition, Long> oldPartitionToPosition : oldPartitionAssignmentsToPosition.entrySet()) {
consumerTmp.seek(oldPartitionToPosition.getKey(), oldPartitionToPosition.getValue());
}
// offsets in the state of new partitions may still be placeholder sentinel values if we are:
// (1) starting fresh,
// (2) checkpoint / savepoint state we were restored with had not completely
// been replaced with actual offset values yet, or
// (3) the partition was newly discovered after startup;
// replace those with actual offsets, according to what the sentinel value represent.
for (KafkaTopicPartitionState<TopicPartition> newPartitionState : newPartitions) {
if (newPartitionState.getOffset() == KafkaTopicPartitionStateSentinel.EARLIEST_OFFSET) {
consumerCallBridge.seekPartitionToBeginning(consumerTmp, newPartitionState.getKafkaPartitionHandle());
newPartitionState.setOffset(consumerTmp.position(newPartitionState.getKafkaPartitionHandle()) - 1);
} else if (newPartitionState.getOffset() == KafkaTopicPartitionStateSentinel.LATEST_OFFSET) {
consumerCallBridge.seekPartitionToEnd(consumerTmp, newPartitionState.getKafkaPartitionHandle());
newPartitionState.setOffset(consumerTmp.position(newPartitionState.getKafkaPartitionHandle()) - 1);
} else if (newPartitionState.getOffset() == KafkaTopicPartitionStateSentinel.GROUP_OFFSET) {
// the KafkaConsumer by default will automatically seek the consumer position
// to the committed group offset, so we do not need to do it.
newPartitionState.setOffset(consumerTmp.position(newPartitionState.getKafkaPartitionHandle()) - 1);
} else {
consumerTmp.seek(newPartitionState.getKafkaPartitionHandle(), newPartitionState.getOffset() + 1);
}
}
} catch (WakeupException e) {
// a WakeupException may be thrown if the consumer was invoked wakeup()
// before it was isolated for the reassignment. In this case, we abort the
// reassignment and just re-expose the original consumer.
synchronized (consumerReassignmentLock) {
this.consumer = consumerTmp;
// if reassignment had already started and affected the consumer,
// we do a full roll back so that it is as if it was left untouched
if (reassignmentStarted) {
consumerCallBridge.assignPartitions(
this.consumer, new ArrayList<>(oldPartitionAssignmentsToPosition.keySet()));
for (Map.Entry<TopicPartition, Long> oldPartitionToPosition : oldPartitionAssignmentsToPosition.entrySet()) {
this.consumer.seek(oldPartitionToPosition.getKey(), oldPartitionToPosition.getValue());
}
}
// no need to restore the wakeup state in this case,
// since only the last wakeup call is effective anyways
hasBufferedWakeup = false;
// re-add all new partitions back to the unassigned partitions queue to be picked up again
for (KafkaTopicPartitionState<TopicPartition> newPartition : newPartitions) {
unassignedPartitionsQueue.add(newPartition);
}
// this signals the main fetch loop to continue through the loop
throw new AbortedReassignmentException();
}
}
// reassignment complete; expose the reassigned consumer
synchronized (consumerReassignmentLock) {
this.consumer = consumerTmp;
// restore wakeup state for the consumer if necessary
if (hasBufferedWakeup) {
this.consumer.wakeup();
hasBufferedWakeup = false;
}
}
}
@VisibleForTesting
KafkaConsumer<byte[], byte[]> getConsumer(Properties kafkaProperties) {
return new KafkaConsumer<>(kafkaProperties);
}
@VisibleForTesting
FlinkConnectorRateLimiter getRateLimiter() {
return rateLimiter;
}
// -----------------------------------------------------------------------
// Rate limiting methods
// -----------------------------------------------------------------------
/**
* @param records List of ConsumerRecords.
* @return Total batch size in bytes, including key and value.
*/
private int getRecordBatchSize(ConsumerRecords<byte[], byte[]> records) {
int recordBatchSizeBytes = 0;
for (ConsumerRecord<byte[], byte[]> record : records) {
// Null is an allowed value for the key
if (record.key() != null) {
recordBatchSizeBytes += record.key().length;
}
recordBatchSizeBytes += record.value().length;
}
return recordBatchSizeBytes;
}
/**
* Get records from Kafka. If the rate-limiting feature is turned on, this method is called at
* a rate specified by the {@link #rateLimiter}.
*
* @return ConsumerRecords
*/
@VisibleForTesting
protected ConsumerRecords<byte[], byte[]> getRecordsFromKafka() {
ConsumerRecords<byte[], byte[]> records = consumer.poll(pollTimeout);
if (rateLimiter != null) {
int bytesRead = getRecordBatchSize(records);
rateLimiter.acquire(bytesRead);
}
return records;
}
// ------------------------------------------------------------------------
// Utilities
// ------------------------------------------------------------------------
private static List<TopicPartition> convertKafkaPartitions(List<KafkaTopicPartitionState<TopicPartition>> partitions) {
ArrayList<TopicPartition> result = new ArrayList<>(partitions.size());
for (KafkaTopicPartitionState<TopicPartition> p : partitions) {
result.add(p.getKafkaPartitionHandle());
}
return result;
}
private class CommitCallback implements OffsetCommitCallback {
private final KafkaCommitCallback internalCommitCallback;
CommitCallback(KafkaCommitCallback internalCommitCallback) {
this.internalCommitCallback = checkNotNull(internalCommitCallback);
}
@Override
public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception ex) {
commitInProgress = false;
if (ex != null) {
log.warn("Committing offsets to Kafka failed. This does not compromise Flink's checkpoints.", ex);
internalCommitCallback.onException(ex);
} else {
internalCommitCallback.onSuccess();
}
}
}
private class ResumeCallback extends Thread {
private String processTime;
private Properties kafkaProperties;
ResumeCallback() {
}
public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception ex) {
pausing = false;
}
}
/**
* Utility exception that serves as a signal for the main loop to continue through the loop
* if a reassignment attempt was aborted due to an pre-reassignment wakeup call on the consumer.
*/
private static class AbortedReassignmentException extends Exception {
private static final long serialVersionUID = 1L;
}
}
类4:jdbc工具
package org.apache.flink.streaming.connectors.kafka.internal;
import java.sql.*;
import java.util.Properties;
/**
* @author hulb
* @date 2019-05-07 14:35
*/
public class JDBCHelper {
/**
* 获取数据库连接 TODO 这里需要优化代码
*
* @param driver 驱动名称
* @param url 连接地址
* @param userName 用户名
* @param password 密码
* @return
*/
private static Connection getConn(String url, String driver, String userName, String password) {
Connection conn = null;
try {
Class.forName(driver);
conn = DriverManager.getConnection(url, userName, password);
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
}
return conn;
}
/**
* 是日切的系统库
* @param kafkaProperties
* @return
*/
public static Connection getConn(Properties kafkaProperties) {
String url = kafkaProperties.getProperty("database.url");
String driver = kafkaProperties.getProperty("database.driver");
String userName = kafkaProperties.getProperty("database.username");
String password = kafkaProperties.getProperty("database.password");
return getConn(url, driver, userName, password);
}
/**
* 是用户自定义的oracle库 保存state等信息
* @param kafkaProperties
* @return
*/
public static Connection getStateConn(Properties kafkaProperties) {
String url = kafkaProperties.getProperty("database.state.url");
String driver = kafkaProperties.getProperty("database.state.driver");
String userName = kafkaProperties.getProperty("database.state.username");
String password = kafkaProperties.getProperty("database.state.password");
return getConn(url, driver, userName, password);
}
/**
* 根据指定参数信息和查询sql 查询 oracle 数据
*
* @param querySql
* @return
*/
public static Integer getAll(Connection conn, String querySql) {
PreparedStatement pstmt;
Integer count = 0;
try {
pstmt = conn.prepareStatement(querySql);
ResultSet rs = pstmt.executeQuery();
int col = rs.getMetaData().getColumnCount();
//解析 count的结果 是 0 还是大于 0
while (rs.next()) {
for (int i = 1; i <= col; i++) {
count = rs.getInt(1);
}
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
try {
conn.close();
} catch (Exception e) {
e.printStackTrace();
}
}
return count;
}
/**
* 更新当前状态
* 1. 开始暂停等待的时候写一次数据库状态 (processTime 开始等待)
* 2. 等待完毕 继续执行任务的时候写一次数据库状态 (先查询之前有没有这个状态,有则修改,没有则新增)
*
* @return
*/
public static Integer updateState(Properties kafkaProperties,
String processTime, String status) {
String statTable = kafkaProperties.getProperty("database.state.table");
String sql = String.format("INSERT INTO %s(create_time,process_time,run_desc) VALUES('%s','%s','%s')",
statTable, ProcessUtils.getNowTime(), processTime, status);
System.out.println(sql);
return executeUpdate(kafkaProperties, sql);
}
/**
* 将checkpoint结果写到oracle库中
*
* @param kafkaProperties
* @param processTime
* @param status
* @return
*/
public static Integer saveCheckpointState(Properties kafkaProperties,
String processTime, String status, String topic, String checkpointCompleted) {
System.out.println("将topic是否进行checkpoint写入到oracle中");
String statTable = kafkaProperties.getProperty("database.checkpoint.table", "switch_checkpoint_status");
String groupId = kafkaProperties.getProperty("group.id", "group_id");
String sql = String.format("INSERT INTO %s(switch_date,topic,group_id,checkpoint_completed) VALUES('%s','%s','%s','%s')",
statTable, processTime, topic, groupId, checkpointCompleted);
System.out.println(sql);
return executeUpdate(kafkaProperties, sql);
}
/**
* 根据指定的数据库连接配置进行update 或者insert 操作
*
* @param kafkaProperties
* @param sql
* @return
*/
public static Integer executeUpdate(Properties kafkaProperties, String sql) {
int rtn = 0;
Connection conn = null;
PreparedStatement pstmt = null;
try {
conn = getStateConn(kafkaProperties);
conn.setAutoCommit(false);
pstmt = conn.prepareStatement(sql);
rtn = pstmt.executeUpdate();
conn.commit();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
conn.close();
} catch (Exception e) {
e.printStackTrace();
}
}
System.out.println("执行成功:"+sql);
return rtn;
}
}