通过构建A->B->C三个算子,为方便测试断开算子间的链接·env.disableOperatorChaining()
,在B算子中增加Thread.sleep
降低其数据处理能力。观察现象如下:
案例代码
package com.dayue;
import com.alibaba.fastjson.JSONObject;
import lombok.SneakyThrows;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import java.util.concurrent.atomic.AtomicInteger;
/**
* 观察反压情况
* A->B->C
* 假设B点计算压力大
* 则B点显示为红色,busy 100%
* A点背压点,backpressure比较高
*
*/
public class back_pressure_demo1 {
@SneakyThrows
public static void main(String[] args) {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(2);
env.disableOperatorChaining();
DataStreamSource<String> source = env.addSource(new demo2());
SingleOutputStreamOperator<JSONObject> map = source.map(new MapFunction<String, String>() {
@Override
public String map(String value) throws Exception {
return value;
}
}).map(new MapFunction<String, String>() {
@Override
public String map(String value) throws Exception {
Thread.sleep(1000);
return value + "," + System.currentTimeMillis();
}
}).map(new RichMapFunction<String, JSONObject>() {
@Override
public JSONObject map(String value) throws Exception {
JSONObject json = new JSONObject();
String[] split = value.split(",");
json.put("province", split[0]);
json.put("id", split[1]);
json.put("ord", split[2]);
json.put("ts", split[3]);
return json;
}
});
map.print();
env.execute(back_pressure_demo1.class.getSimpleName());
}
public static class demo2 extends RichParallelSourceFunction<String> {
@Override
public void run(SourceContext<String> sourceContext) throws Exception {
AtomicInteger i = new AtomicInteger(0);
String[] province = {"河南省", "广东省", "江苏省", "北京市", "重庆市"};
while (true) {
sourceContext.collect(province[RandomUtils.nextInt(0, 5)] + "," + RandomStringUtils.randomAlphabetic(5) + "," + i.getAndIncrement());
Thread.sleep(10);
}
}
@Override
public void cancel() {
}
}
}