Flink的keyby延时源码

public class RecordWriter<T extends IOReadableWritable> {

==FullBuffer
/**
* This is used to send LatencyMarks to a random target channel.
*/
public void randomEmit(T record) throws IOException, InterruptedException {
sendToTarget(record, rng.nextInt(numChannels));
}

private void sendToTarget(T record, int targetChannel) throws IOException, InterruptedException {
RecordSerializer<T> serializer = serializers[targetChannel];

SerializationResult result = serializer.addRecord(record);

while (result.isFullBuffer()) {
if (tryFinishCurrentBufferBuilder(targetChannel, serializer)) {
// If this was a full record, we are done. Not breaking
// out of the loop at this point will lead to another
// buffer request before breaking out (that would not be
// a problem per se, but it can lead to stalls in the
// pipeline).
if (result.isFullRecord()) {
break;
}
}
BufferBuilder bufferBuilder = requestNewBufferBuilder(targetChannel);

result = serializer.continueWritingWithNextBufferBuilder(bufferBuilder);
}
checkState(!serializer.hasSerializedData(), "All data should be written at once");

if (flushAlways) {
targetPartition.flush(targetChannel);
}
}



==getBufferTimeout
package org.apache.flink.streaming.runtime.tasks;
public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>>

@VisibleForTesting
public static <OUT> List<StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createStreamRecordWriters(
StreamConfig configuration,
Environment environment) {
List<StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>> streamRecordWriters = new ArrayList<>();
List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader());
Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader());

for (int i = 0; i < outEdgesInOrder.size(); i++) {
StreamEdge edge = outEdgesInOrder.get(i);
streamRecordWriters.add(
createStreamRecordWriter(
edge,
i,
environment,
environment.getTaskInfo().getTaskName(),
chainedConfigs.get(edge.getSourceId()).getBufferTimeout()));
}
return streamRecordWriters;

}

http://vinoyang.com/2016/12/30/flink-runtime-producer-result-partition/
http://vinoyang.com/2017/01/04/flink-runtime-consumer-input-gate/
http://vinoyang.com/2017/01/08/flink-runtime-netty-part-1/
http://vinoyang.com/2017/01/12/flink-runtime-netty-part-2/
http://vinoyang.com/2017/01/15/flink-runtime-netty-part-3/
http://vinoyang.com/2016/12/14/flink-runtime-NetworkEnvironment/
http://vinoyang.com/2016/12/28/flink-runtime-communicate-api/
http://vinoyang.com/archives/2016/12/
http://vinoyang.com/2016/12/20/flink-runtime-unified-data-exchange/
  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值