Prometheus监控Flink CDC任务

一、背景

• 目前项目存在Flink CDC 同步MySQL数据到列数据库，Flink同步过程中存在任务失败，导致数据同步断开，对业务查询造成了影响。现搭建一个SpringBoot 项目使用micrometer配合Prometheus监控Flink CDC的运行状况，任务失败能够通过钉钉提示。

二、项目准备

• 已经搭建好一套Prometheus的监控系统，参考Prometheus Alertmanager集成钉钉告警

三、Flink CDC监控项目开发

• 相关jar导入

		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-web</artifactId>
		</dependency>

		<!--actuator -->
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-actuator</artifactId>
		</dependency>

		<!-- prometheus -->
		<dependency>
			<groupId>io.micrometer</groupId>
			<artifactId>micrometer-registry-prometheus</artifactId>
		</dependency>

		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>
		</dependency>

		<dependency>
			<groupId>org.projectlombok</groupId>
			<artifactId>lombok</artifactId>
		</dependency>

		<!-- okhttp -->
		<dependency>
			<groupId>com.squareup.okhttp3</groupId>
			<artifactId>okhttp</artifactId>
		</dependency>

		<dependency>
			<groupId>com.alibaba</groupId>
			<artifactId>fastjson</artifactId>
			<version>1.2.60</version>
		</dependency>
		
			<!--hutool -->
		<dependency>
			<groupId>cn.hutool</groupId>
			<artifactId>hutool-all</artifactId>
			<version>5.7.17</version>
		</dependency>
		
			<dependency>
			<groupId>com.alibaba.cloud</groupId>
			<artifactId>spring-cloud-starter-alibaba-nacos-config</artifactId>
		</dependency>

		<dependency>
			<groupId>com.alibaba.cloud</groupId>
			<artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
		</dependency>

• 定义接口ICustomPrometheusService，flinkMeter方法用于获取任务信息。
• setFlinkUrl和setFlinkJobName方法使用nacos动态刷新功能，可以实时修改期望运行任务名称以及flink API地址。

public interface ICustomPrometheusService {
	
	/**
	 * flink监控信息
	 * @methodName meter
	 * @param registry void
	 * @author wys
	 * @date 2024-08-30
	 */
	public void flinkMeter(MeterRegistry registry);
	
	/**
	 * 设置Flink API地址
	 * @methodName setFlinkUrl
	 * @param flinkUrl void
	 * @author wys
	 * @date 2024-08-30
	 */
	public void setFlinkUrl(String flinkUrl);
	
	/**
	 * 设置Flink 数据同步任务名称
	 * @methodName setFlinkJobName
	 * @param flinkJobName void
	 * @author wys
	 * @date 2024-08-30
	 */
	public void setFlinkJobName(String flinkJobName);

}

• 接口实现，这里通过Flink提供的API接口进行任务信息的读取，Gauge 设置任务信息。
• flinkUrl为Flink提供的API地址，flinkNames属性是我们期望运行的Flink CDC任务名称，多个任务名使用逗号分割。
• 通过比较期望运行任务和实际运行任务，来设置哪些任务在线，哪些任务离线。离线的任务通过Prometheus发送钉钉通知。

@Slf4j
@Service
public class CustomPrometheusServiceImpl implements ICustomPrometheusService {

	@Resource
	private RestTemplateUtil restTemplateUtil;

	@Value("${flink.url}")
	private String flinkUrl;
	
	@Value("${flink.job.names:no_job}")
	private String flinkNames;
	
	private List<String> jobNames=new ArrayList<>();
	
	private Map<String,FlinkJobDto> jobMap=new HashMap<>();
	
	private Map<String,Gauge> gaugeMap=new HashMap<>();
	
	private final static int UP=1;
	
	private final static int DOWN=0;
	
	
	@PostConstruct
	public void initJobNames(){
		jobNames =Arrays.asList(flinkNames.split(","));
	}

	@Override
	public void flinkMeter(MeterRegistry registry) {
		List<FlinkJobDto> list=new ArrayList<FlinkJobDto>();
		try {
			FlinkDto flinkInfo= restTemplateUtil.getForObject(flinkUrl, param -> {},FlinkDto.class);
			list = flinkInfo.getJobs();
		} catch (Exception e) {
			log.error("调用flink API接口失败,怀疑Flink掉线");
		}
		//注册运行中的任务
		for(FlinkJobDto job:list){
			setFlinkJobInfo(job, registry, UP);
		}
		//运行中的任务
		Set<String> runJobs= list.stream().map(FlinkJobDto::getName).collect(Collectors.toSet());
		//离线的任务
		List<String> offlineJob= jobNames.stream().filter(item->!runJobs.contains(item)).collect(Collectors.toList());
		//存在离线任务则修改状态
		if(!ObjectUtils.isEmpty(offlineJob)){
			for(String name:offlineJob){
				setFlinkJobInfo(getFailJob(name), registry, DOWN);
			}
		}
		
		//移除的任务
		List<String> registryJobs=gaugeMap.keySet().stream().collect(Collectors.toList());
		List<String> removeJobs= registryJobs.stream().filter(item->!jobNames.contains(item)).collect(Collectors.toList());
		if(!ObjectUtils.isEmpty(removeJobs)){
			for(String name:removeJobs){
				Gauge gauge=gaugeMap.get(name);
				registry.remove(gauge);
				jobMap.remove(name);
				gaugeMap.remove(name);
			}
		}
	}
	

	/**
	 * 注册信息
	 * @methodName setFlinkJobInfo
	 * @param job
	 * @param registry void
	 * @author wys
	 * @date 2024-08-30
	 */
	private void setFlinkJobInfo(FlinkJobDto job,MeterRegistry registry,int up){
		//任务已经注册则修改对象信息
		if(jobMap.containsKey(job.getName())){
			FlinkJobDto flinkJob=jobMap.get(job.getName());
			if(up==1){
				flinkJob=BeanUtil.copyProperties(job, FlinkJobDto.class);
			}
			flinkJob.setUp(up);
			return ;
		}
		Date date =new Date(job.getStartTime());
		//注册信息
		Gauge gauge= Gauge.builder("flink.job.infos", job, FlinkJobDto::getUp)
		 .tag("name", job.getName())
		 .tag("jid",job.getJid())
		 .tag("startTime", DateUtil.format(date, "yyyy-MM-dd HH:mm:ss"))
		 .register(registry);
		job.setUp(up);
		//存入缓存用于后续动态更新
		jobMap.put(job.getName(), job);
		gaugeMap.put(job.getName(), gauge);
	}
	
	/**
	 * 获取失败的任务
	 * @methodName getFailJob
	 * @param name
	 * @return FlinkJobDto
	 * @author wys
	 * @date 2024-09-02
	 */
	private FlinkJobDto getFailJob(String name){
		FlinkJobDto allFailJob=new FlinkJobDto();
		allFailJob.setName(name);
		allFailJob.setJid("");
		allFailJob.setUp(0);
		allFailJob.setStartTime(new Date().getTime());
		return allFailJob;
	}

	@Override
	public void setFlinkUrl(String flinkUrl) {
		if(!ObjectUtils.isEmpty(flinkUrl)){
			this.flinkUrl=flinkUrl;
			log.info("修改Flink API地址:{}",flinkUrl);
		}
	}

	@Override
	public void setFlinkJobName(String flinkJobName) {
		if(!ObjectUtils.isEmpty(flinkJobName)){
			this.flinkNames=flinkJobName;
			jobNames =Arrays.asList(flinkNames.split(","));
			log.info("修改Flink同步任务名称:{}",flinkJobName);
		}
	}
	

}

• 监控信息配置

import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.config.MeterFilter;

@Configuration
public class MeterRegistrConfig {
	
	@Bean
    MeterRegistryCustomizer<MeterRegistry> configurer() {
        return (registry) -> registry.config()
        		.meterFilter(MeterFilter.denyNameStartsWith("tomcat"))
        		.meterFilter(MeterFilter.denyNameStartsWith("logback"))
        		.meterFilter(MeterFilter.denyNameStartsWith("system"))
        		.meterFilter(MeterFilter.denyNameStartsWith("process"))
        		.meterFilter(MeterFilter.denyNameStartsWith("http"))
        		.meterFilter(MeterFilter.denyNameStartsWith("jvm"));
    }


}

• 配置每5秒钟获取一次运行任务

import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import javax.annotation.Resource;

import org.springframework.stereotype.Component;

import com.wellsun.custom.prometheus.service.ICustomPrometheusService;

import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.binder.MeterBinder;

@Component
public class CustomMeterBinder implements MeterBinder {
	
	@Resource
	private ICustomPrometheusService customPrometheusService;
	
	private ScheduledExecutorService executors = Executors.newSingleThreadScheduledExecutor(); 
	
	@Override
	public void bindTo(MeterRegistry registry) {
		//5秒钟采集一次数据
		executors.scheduleAtFixedRate(()->customPrometheusService.flinkMeter(registry), 1, 5, TimeUnit.SECONDS);
	
	}

}

• nacos配置文件

management.endpoints.web.exposure.include=*
management.metrics.tags.application= ${spring.application.name}
server.tomcat.mbeanregistry.enabled=true

##flinkAPI地址
flink.url= http://xxx:8081/jobs/overview
##同步任务名称
flink.job.names=name,name1,name2

• nacos配置监听

import javax.annotation.Resource;

import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.alibaba.cloud.nacos.NacosConfigManager;
import com.alibaba.nacos.api.config.ConfigChangeEvent;
import com.alibaba.nacos.api.config.ConfigChangeItem;
import com.alibaba.nacos.client.config.listener.impl.AbstractConfigChangeListener;
import com.wellsun.custom.prometheus.service.ICustomPrometheusService;


@Component(value = "datacenterNacosConfigListener")
public class NacosConfigListener extends AbstractConfigChangeListener implements InitializingBean {

	@Resource
	private NacosConfigManager nacosConfigManager;
	
	@Autowired
	private ICustomPrometheusService customPrometheusService;
	

	@Override
	public void afterPropertiesSet() throws Exception {
		nacosConfigManager.getConfigService().addListener("prometheus-config", "prometheus-group", this);
	}

	@Override
	public void receiveConfigChange(ConfigChangeEvent event) {
		for (ConfigChangeItem changeItem : event.getChangeItems()) {
			if(changeItem.getKey().equals("flink.url")){
				customPrometheusService.setFlinkUrl(changeItem.getNewValue());
			}else if(changeItem.getKey().equals("flink.job.names")){
				customPrometheusService.setFlinkJobName(changeItem.getNewValue());
			}
		}
	}

}

四、Prometheus配置

• Prometheus添加该服务监控信息

  - job_name: 'flink-job'
    static_configs:
      - targets: ['xxx:16100']
        labels:
          __metrics_path__: "/actuator/prometheus" #自定义监控服务
          group: flink

• 添加监控规则

 - alert: Flink任务下线
   expr: flink_job_infos{ job="flink-job"} == 0
   for: 20s
   labels:
     severity: 严重告警
   annotations:
     summary: "Flink同步任务失败, 请尽快处理!"
     description: "{{$labels.name}}同步任务失败. "

• 自定义监控应用作为一个子服务注册到nacos。启动项目查看监控信息。
  
• 测试任务上下线，可以看到钉钉推送相应告警