使用flink-cdc技术（2.3.0）解析binlog实现实时数据大屏

最新推荐文章于 2024-07-29 12:04:40 发布

日记塔

最新推荐文章于 2024-07-29 12:04:40 发布

阅读量85

点赞数

文章标签： flink 大数据

使用flink-cdc技术（2.3.0）解析binlog实现实时数据大屏。

一、项目环境

该项目主要为一个数据大屏，采用了flink-cdc技术（2.3.0），flink版本为（1.14.6），利用他，实现自动获取并解析Mysql的binlog，来实时把对应的数据解析出来用于大屏展示。

<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-base -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-base</artifactId>
            <version>1.14.6</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.ververica/flink-connector-mysql-cdc -->
        <dependency>
            <groupId>com.ververica</groupId>
            <artifactId>flink-connector-mysql-cdc</artifactId>
            <version>2.3.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.ververica/flink-sql-connector-mysql-cdc -->
        <dependency>
            <groupId>com.ververica</groupId>
            <artifactId>flink-sql-connector-mysql-cdc</artifactId>
            <version>2.3.0</version>
        </dependency>

该文章主要参考了，我自己的 https://blog.51cto.com/lenglingx/11242672【十一分钟上手Flink CDC】

同样这篇就是“使用mysql-binlog-connector-java技术（0.29.2）解析binlog实现实时数据大屏”的Flink-CDC的技术实现。

同样使用了多线程来，不过线程池是放到了程序代码了。上篇用到mysql-binlog-connector-java技术放到单独一个叫做sdk包目录，这篇我把flink-cdc的技术放到一个dese包的目录。

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.cqsym</groupId>
    <artifactId>newbig</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>


    <dependencyManagement>
        <dependencies>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-dependencies</artifactId>
                <version>2.6.15</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>
        </dependencies>
    </dependencyManagement>

    <dependencies>
        <!--lombok -->
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-thymeleaf</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-json</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-devtools</artifactId>
            <optional>true</optional>
            <!-- optional=true,依赖不会传递，该项目依赖devtools；之后依赖myboot项目的项目如果想要使用devtools，需要重新引入 -->
            <scope>true</scope>
        </dependency>
        <!-- jackson-datatype-jsr310 -->
        <dependency>
            <groupId>com.fasterxml.jackson.datatype</groupId>
            <artifactId>jackson-datatype-jsr310</artifactId>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-jdbc</artifactId>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-data-jpa -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-jpa</artifactId>
        </dependency>
        <!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>8.0.30</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-data-redis -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-redis</artifactId>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-base -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-base</artifactId>
            <version>1.14.6</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.ververica/flink-connector-mysql-cdc -->
        <dependency>
            <groupId>com.ververica</groupId>
            <artifactId>flink-connector-mysql-cdc</artifactId>
            <version>2.3.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.ververica/flink-sql-connector-mysql-cdc -->
        <dependency>
            <groupId>com.ververica</groupId>
            <artifactId>flink-sql-connector-mysql-cdc</artifactId>
            <version>2.3.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-streaming-java -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.12</artifactId>
            <version>1.14.6</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-clients -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.12</artifactId>
            <version>1.14.6</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-runtime-web -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-runtime-web_2.12</artifactId>
            <version>1.14.6</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-table-runtime -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-runtime_2.12</artifactId>
            <version>1.14.6</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-scala-bridge_2.12</artifactId>
            <version>1.14.6</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner_2.12</artifactId>
            <version>1.14.6</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>32.1.3-jre</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.83</version>
        </dependency>
    </dependencies>

    <build>
        <finalName>newbig</finalName>
        <resources>
            <resource>
                <directory>src/main/resources</directory>
            </resource>
        </resources>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <version>2.6.15</version>
                <configuration>
                    <fork>true</fork>
                    <includeSystemScope>true</includeSystemScope>
                    <!--fork : 如果没有该项配置，肯呢个devtools不会起作用，即应用不会restart -->
                    <!--这里写上main方法所在类的路径-->
                    <mainClass>com.cqsym.newbig.Application</mainClass>
                </configuration>
                <executions>
                    <execution>
                        <goals>
                            <goal>repackage</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-surefire-plugin</artifactId>
                <version>2.22.2</version>
                <configuration>
                    <skip>true</skip>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.11.0</version>
                <configuration>
                    <source>8</source>
                    <target>8</target>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-resources-plugin</artifactId>
                <version>3.3.1</version>
                <configuration>
                    <encoding>UTF-8</encoding>
                    <!-- 过滤后缀为pem、pfx的证书文件 -->
                    <nonFilteredFileExtensions>
                        <nonFilteredFileExtension>p12</nonFilteredFileExtension>
                        <nonFilteredFileExtension>cer</nonFilteredFileExtension>
                        <nonFilteredFileExtension>pem</nonFilteredFileExtension>
                        <nonFilteredFileExtension>pfx</nonFilteredFileExtension>
                    </nonFilteredFileExtensions>
                </configuration>
            </plugin>
        </plugins>

    </build>
</project>

1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.
72.
73.
74.
75.
76.
77.
78.
79.
80.
81.
82.
83.
84.
85.
86.
87.
88.
89.
90.
91.
92.
93.
94.
95.
96.
97.
98.
99.
100.
101.
102.
103.
104.
105.
106.
107.
108.
109.
110.
111.
112.
113.
114.
115.
116.
117.
118.
119.
120.
121.
122.
123.
124.
125.
126.
127.
128.
129.
130.
131.
132.
133.
134.
135.
136.
137.
138.
139.
140.
141.
142.
143.
144.
145.
146.
147.
148.
149.
150.
151.
152.
153.
154.
155.
156.
157.
158.
159.
160.
161.
162.
163.
164.
165.
166.
167.
168.
169.
170.
171.
172.
173.
174.
175.
176.
177.
178.
179.
180.
181.
182.
183.
184.
185.
186.
187.
188.
189.
190.
191.
192.
193.
194.
195.
196.
197.
198.
199.
200.
201.
202.
203.
204.
205.
206.
207.
208.
209.
210.
211.
212.
213.
214.
215.
216.
217.
218.

application-dev.properties

#-----------------自定义信息配置---------------------
com.imddy.layuiadmin.title=BOOT
com.imddy.layuiadmin.description=学习一下BOOT是什么
#-----------------自定义信息配置---------------------
executeTask=2

#----------数据库基础配置--------------------
#spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver
#spring.datasource.url=jdbc:mysql://127.0.0.1:3306/layuiadmin2?useUnicode=true&useSSL=false&characterEncoding=UTF-8&serverTimezone=UTC
#spring.datasource.url=jdbc:mysql://127.0.0.1:3306/layuiadmin2?useUnicode=true&useSSL=false&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&zeroDateTimeBehavior=convertToNull
#spring.datasource.url=jdbc:mysql://127.0.0.1:3306/twmshelp?useUnicode=true&useSSL=false&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai
#spring.datasource.username=root
#spring.datasource.password=root__
#spring.datasource.name=HikaraPool-100
#----------数据库连接池基础配置--------------------
spring.datasource.type=com.zaxxer.hikari.HikariDataSource
# 指定连接池的名称 - 默认自动生成
spring.datasource.hikari.pool-name=HikaraPool-1
# 如果你的驱动程序支持JDBC4，强烈建议不要设置此属性。
#spring.datasource.hikari.connection-test-query=select 1
# 连接超时时间 - 默认值：30秒。
spring.datasource.hikari.connection-timeout=30000
# 连接池中允许闲置的最长时间 - 默认值：10分钟
spring.datasource.hikari.idle-timeout=600000
# 一个连接生命时长（毫秒），超时而没被使用则被释放 - 默认值：30分钟
spring.datasource.hikari.max-lifetime=1800000
# 连接池中允许的最大连接数，包括闲置和使用中的连接 - 默认值：10
spring.datasource.hikari.maximum-pool-size=100
# 连接池中允许的最小空闲连接数 - 默认值：10。
spring.datasource.hikari.minimum-idle=10
# 连接被测试活动的最长时间 - 默认值：5秒。
spring.datasource.hikari.validation-timeout=5000

spring.jpa.show-sql=true
spring.jpa.open-in-view=false

spring.datasource.primary.driver-class-name=com.mysql.cj.jdbc.Driver
spring.datasource.primary.url=jdbc:mysql://127.0.0.1:3306/newbig?useUnicode=true&useSSL=false&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai
spring.datasource.primary.username=root
spring.datasource.primary.password=root__
# 指定为HikariDataSource
spring.datasource.primary.type=com.zaxxer.hikari.HikariDataSource
# hikari连接池配置 对应 HikariConfig 配置属性类
spring.datasource.primary.hikari.pool-name=HikariCP-Primary
#最小空闲连接数
spring.datasource.primary.hikari.minimum-idle=5
# 空闲连接存活最大时间，默认10分钟
spring.datasource.primary.hikari.idle-timeout=600000
# 连接池最大连接数，默认是10
spring.datasource.primary.hikari.maximum-pool-size=10
# 此属性控制从池返回的连接的默认自动提交行为,默认值：true
spring.datasource.primary.hikari.auto-commit=true
# 此属性控制池中连接的最长生命周期，值0表示无限生命周期，默认30分钟
spring.datasource.primary.hikari.max-lifetime=1800000
# 数据库连接超时时间,默认30秒
spring.datasource.primary.hikari.connection-timeout=30000
# 连接测试query,如果你的驱动程序支持JDBC4，强烈建议不要设置此属性。
#spring.datasource.primary.hikari.connection-test-query=SELECT 1

spring.datasource.second.driver-class-name=com.mysql.cj.jdbc.Driver
spring.datasource.second.url=jdbc:mysql://192.168.203.150:3306/twms?useUnicode=true&useSSL=false&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai
spring.datasource.second.username=aliyun_root
spring.datasource.second.password=root__
spring.datasource.second.host=192.168.203.150
spring.datasource.second.port=3306
# 指定为HikariDataSource
spring.datasource.second.type=com.zaxxer.hikari.HikariDataSource
# hikari连接池配置 对应 HikariConfig 配置属性类
spring.datasource.second.hikari.pool-name=HikariCP-Second
#最小空闲连接数
spring.datasource.second.hikari.minimum-idle=5
# 空闲连接存活最大时间，默认10分钟
spring.datasource.second.hikari.idle-timeout=600000
# 连接池最大连接数，默认是10
spring.datasource.second.hikari.maximum-pool-size=10
# 此属性控制从池返回的连接的默认自动提交行为,默认值：true
spring.datasource.second.hikari.auto-commit=true
# 此属性控制池中连接的最长生命周期，值0表示无限生命周期，默认30分钟
spring.datasource.second.hikari.max-lifetime=1800000
# 数据库连接超时时间,默认30秒
spring.datasource.second.hikari.connection-timeout=30000
# 连接测试query,如果你的驱动程序支持JDBC4，强烈建议不要设置此属性。
#spring.datasource.second.hikari.connection-test-query=SELECT 1



## redis 配置
spring.redis.host=127.0.0.1
spring.redis.port=6379
spring.redis.database=0
spring.redis.timeout=10s
spring.redis.lettuce.pool.min-idle=0
spring.redis.lettuce.pool.max-idle=8
spring.redis.lettuce.pool.max-active=8
spring.redis.lettuce.pool.max-wait=-1ms

这里配置了redis，但实际项目中没有使用。

NbigscreenController.java

package com.cqsym.newbig.controller;


import com.cqsym.newbig.primary.entity.TownerBase;
import com.cqsym.newbig.service.BinLogService;
import com.cqsym.newbig.service.TownerBaseService;
import com.cqsym.newbig.service.XianluMingchengService;
import com.cqsym.newbig.service.ZaituDingdanNumSerivce;
import com.cqsym.newbig.service.ZuoyeNumService;
import com.cqsym.newbig.utils.AjaxResult;
import com.cqsym.newbig.vo.DaquNumVo;
import com.cqsym.newbig.vo.OwnerBaseNumVo;
import com.cqsym.newbig.vo.XianluMingchengVo;
import com.cqsym.newbig.vo.ZaituDingdanVo;
import com.cqsym.newbig.vo.ZhanbiVo;
import com.cqsym.newbig.vo.ZuoyeShishiQingkuangVo;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;

import java.util.List;
import java.util.Map;

@Controller
@RequestMapping("/nbigscreen")
public class NbigscreenController {
    private static final Logger log = LoggerFactory.getLogger(NbigscreenController.class);

    @Autowired
    private BinLogService binLogService;
    @Autowired
    private ZuoyeNumService zuoyeNumService;
    @Autowired
    private XianluMingchengService xianluMingchengService;
    @Autowired
    private ZaituDingdanNumSerivce zaituDingdanNumSerivce;
    @Autowired
    private TownerBaseService townerBaseService;




    @ResponseBody
    @RequestMapping("/index")
    public AjaxResult index() {
        log.info("index... ");
        return AjaxResult.success("index");
    }


    @ResponseBody
    @RequestMapping("/on")
    public AjaxResult on() {
        log.info("on... ");
        try {
            binLogService.start();
        } catch (Exception exception) {
            return AjaxResult.error(exception.getMessage());
        }
        return AjaxResult.success();
    }


    @ResponseBody
    @RequestMapping("/off")
    public AjaxResult off() {
        log.info("off... ");
        try {
            binLogService.stop();
        } catch (Exception exception) {
            return AjaxResult.error(exception.getMessage());
        }
        return AjaxResult.success();
    }


    @ResponseBody
    @RequestMapping("/getZuoyeNum")
    public AjaxResult getZuoyeNum() {
        log.info("getZuoyeNum... ");
        Map map = zuoyeNumService.getZuoyeNum();
        return AjaxResult.success(map);
    }


    @ResponseBody
    @RequestMapping("/getBanshichuZuoyeNum")
    public AjaxResult getBanshichuZuoyeNum() {
        log.info("getBanshichuZuoyeNum... ");
        List<OwnerBaseNumVo> list = zuoyeNumService.getBanshichuZuoyeNum();
        return AjaxResult.success(list);
    }


    @ResponseBody
    @RequestMapping("/getDaquZuoyeNum")
    public AjaxResult getDaquZuoyeNum() {
        log.info("getDaquZuoyeNum... ");
        List<DaquNumVo> list = zuoyeNumService.getDaquZuoyeNum();
        return AjaxResult.success(list);
    }


    @ResponseBody
    @RequestMapping("/getShishiZuoyeQingkuang")
    public AjaxResult getShishiZuoyeQingkuang() {
        log.info("getShishiZuoyeQingkuang... ");
        List<ZuoyeShishiQingkuangVo> list = zuoyeNumService.getShishiZuoyeQingkuang();
        return AjaxResult.success(list);
    }


    @ResponseBody
    @RequestMapping("/getXianluMingcheng")
    public AjaxResult getXianluMingcheng() {
        log.info("getXianluMingcheng... ");
        List<XianluMingchengVo> list = xianluMingchengService.getXianluMingcheng();
        return AjaxResult.success(list);
    }


    @ResponseBody
    @RequestMapping("/getZaituDingdanNumForEveryDay")
    public AjaxResult getZaituDingdanNumForEveryDay() throws JsonProcessingException {
        log.info("getZaituDingdanNumForEveryDay... ");
        List<ZaituDingdanVo>  list = zaituDingdanNumSerivce.getZaituDingdanNumForEveryDay();
        return AjaxResult.success(list);
    }


    @ResponseBody
    @RequestMapping("/getTOwnerBaseAll")
    public AjaxResult getTOwnerBaseAll() {
        log.info("getTOwnerBaseAll... ");
        List<TownerBase>  list = townerBaseService.findAll();
        return AjaxResult.success(list);
    }

    @ResponseBody
    @RequestMapping("/getZuoyeZhanbi")
    public AjaxResult getZuoyeZhanbi() {
        log.info("getZuoyeZhanbi... ");
        List<ZhanbiVo> list = zuoyeNumService.getZuoyeZhanbi();
        return AjaxResult.success(list);
    }




}

1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.
72.
73.
74.
75.
76.
77.
78.
79.
80.
81.
82.
83.
84.
85.
86.
87.
88.
89.
90.
91.
92.
93.
94.
95.
96.
97.
98.
99.
100.
101.
102.
103.
104.
105.
106.
107.
108.
109.
110.
111.
112.
113.
114.
115.
116.
117.
118.
119.
120.
121.
122.
123.
124.
125.
126.
127.
128.
129.
130.
131.
132.
133.
134.
135.
136.
137.
138.
139.
140.
141.
142.
143.
144.
145.
146.
147.
148.
149.
150.
151.
152.
153.
154.

二、各种Service

ZuoyeNumService.java

package com.cqsym.newbig.service;



import com.cqsym.newbig.config.DataSourceRepository;
import com.cqsym.newbig.constant.ZuoyeNumConstants;
import com.cqsym.newbig.enums.OperateTypeEnum;
import com.cqsym.newbig.vo.DaquNumVo;
import com.cqsym.newbig.vo.DaquOwnerBaseNumVo;
import com.cqsym.newbig.vo.OwnerBaseNumVo;
import com.cqsym.newbig.vo.ZhanbiVo;
import com.cqsym.newbig.vo.ZuoyeShishiQingkuangVo;
import com.cqsym.newbig.vo.ZuoyeShishiQingkuangVo1;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;

import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.TemporalAdjusters;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static com.cqsym.newbig.cache.AppMap.BanshichuDaquCacheMap;
import static com.cqsym.newbig.cache.AppMap.BanshichuZuoyeNumCacheMap;
import static com.cqsym.newbig.cache.AppMap.OwnerBaseIdNameCacheName;
import static com.cqsym.newbig.cache.AppMap.ZuoyeNumCacheMap;
import static com.cqsym.newbig.cache.AppMap.ZuoyeShishiQingkuang;


@Service
public class ZuoyeNumService {
    private static final Logger log = LoggerFactory.getLogger(ZuoyeNumService.class);

    @Autowired
    @Qualifier(value="secondDataSourceRepository")
    private DataSourceRepository secondDataSourceRepository;

    @Autowired
    private ObjectMapper objectMapper;

    public void init() {
        log.info("ZuoyeNumService init() 开始 ... ");
        log.info("initZuoyeNumCacheMap ... ");
        initZuoyeNumCacheMap();
        log.info("getDayZuoyeNum ... ");
        getDayZuoyeNum();
        log.info("getMonthZuoyeNum ... ");
        getMonthZuoyeNum();
        log.info("getYearZuoyeNum ... ");
        getYearZuoyeNum();
        log.info("getGongsiYearNum ... ");
        getGongsiYearNum();
        log.info("getGongsiMonthNum ... ");
        getGongsiMonthNum();
        log.info("getGongsiDayNum ... ");
        getGongsiDayNum();
        log.info("ZuoyeNumService init() complate完成 ... ");
    }

    public void initZuoyeNumCacheMap() {
        ZuoyeNumCacheMap.clear();

        ZuoyeNumCacheMap.put(ZuoyeNumConstants.GongsiYearTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.GongsiMonthTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.GongsiDayTotal, 0L);

        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JiecheduanboYearTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JiecheduanboMonthTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JiecheduanboDayTotal, 0L);

        ZuoyeNumCacheMap.put(ZuoyeNumConstants.ZhuangcheYearTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.ZhuangcheMonthTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.ZhuangcheDayTotal, 0L);

        ZuoyeNumCacheMap.put(ZuoyeNumConstants.XiecheYearTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.XiecheMonthTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.XiecheDayTotal, 0L);

        ZuoyeNumCacheMap.put(ZuoyeNumConstants.PeisongYearTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.PeisongMonthTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.PeisongDayTotal, 0L);

        ZuoyeNumCacheMap.put(ZuoyeNumConstants.GongluzhifaYearTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.GongluzhifaMonthTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.GongluzhifaDayTotal, 0L);

        ZuoyeNumCacheMap.put(ZuoyeNumConstants.YikuduanboYearTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.YikuduanboMonthTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.YikuduanboDayTotal, 0L);

        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JiaguYearTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JiaguMonthTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JiaguDayTotal, 0L);

        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JieguYearTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JieguMonthTotal, 0L);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JieguDayTotal, 0L);
    }
    public void getYearZuoyeNum() {
        DateTimeFormatter datetimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        LocalDateTime yearStart = LocalDateTime.now().withDayOfYear(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        String yearStartStr = yearStart.format(datetimeFormatter);
        List<Map<String, Object>> list = getZuoyeNum(yearStartStr);
        if (list != null && list.size() > 0) {
            for (Map<String, Object> map : list) {
                switch (map.get("tddOperationType").toString()) {
                    case "1":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JiecheduanboYearTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "2":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.ZhuangcheYearTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "3":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.XiecheYearTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "4":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JiaguYearTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "5":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JieguYearTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "10":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.PeisongYearTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "11":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.YikuduanboYearTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "15":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.GongluzhifaYearTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                }
            }
        }

    }

    public void getMonthZuoyeNum() {
        DateTimeFormatter datetimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        LocalDateTime monthStart = LocalDateTime.now().with(TemporalAdjusters.firstDayOfMonth()).withHour(0).withMinute(0).withSecond(0).withNano(0);
        String monthStartStr = monthStart.format(datetimeFormatter);
        List<Map<String, Object>> list = getZuoyeNum(monthStartStr);
        if (list != null && list.size() > 0) {
            for (Map<String, Object> map : list) {
                switch (map.get("tddOperationType").toString()) {
                    case "1":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JiecheduanboMonthTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "2":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.ZhuangcheMonthTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "3":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.XiecheMonthTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "4":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JiaguMonthTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "5":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JieguMonthTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "10":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.PeisongMonthTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "11":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.YikuduanboMonthTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "15":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.GongluzhifaMonthTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                }
            }
        }
    }


    public void getDayZuoyeNum() {
        DateTimeFormatter datetimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        LocalDateTime dayStart = LocalDateTime.now().withHour(0).withMinute(0).withSecond(0).withNano(0);
        String dayStartStr = dayStart.format(datetimeFormatter);
        List<Map<String, Object>> list = getZuoyeNum(dayStartStr);
        if (list != null && list.size() > 0) {
            for (Map<String, Object> map : list) {
                switch (map.get("tddOperationType").toString()) {
                    case "1":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JiecheduanboDayTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "2":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.ZhuangcheDayTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "3":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.XiecheDayTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "4":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JiaguDayTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "5":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.JieguDayTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "10":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.PeisongDayTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "11":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.YikuduanboDayTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                    case "15":
                        ZuoyeNumCacheMap.put(ZuoyeNumConstants.GongluzhifaDayTotal, Long.valueOf(String.valueOf(map.get("operationNum"))));
                        break;
                }
            }
        }
    }

    private List<Map<String, Object>> getZuoyeNum(String zuoyeshijian) {
        try {
            String sql = "SELECT tdd_operation_type AS tddOperationType, COUNT(1) AS operationNum  " +
                    "FROM t_dispatch_detail  " +
                    "WHERE tdd_status = 2  " +
                    "AND tdd_operation_end_time >= ?  " +
                    "AND dr = 0  " +
                    "GROUP BY tdd_operation_type  ";
            List<Map<String, Object>> list = secondDataSourceRepository.queryForListMap(sql, zuoyeshijian);
            return list;
        } catch (Exception e) {
            log.error("getZuoyeNum error", e);
            return null;
        }
    }



    public void getGongsiYearNum() {
        DateTimeFormatter datetimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        LocalDateTime yearStart = LocalDateTime.now().withDayOfYear(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        String yearStartStr = yearStart.format(datetimeFormatter);
        Long gongsiYearNum = getGongsiNum(yearStartStr);
        log.info("gongsiYearNum:{}", gongsiYearNum);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.GongsiYearTotal, gongsiYearNum);
    }

    public void getGongsiMonthNum() {
        DateTimeFormatter datetimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        LocalDateTime monthStart = LocalDateTime.now().with(TemporalAdjusters.firstDayOfMonth()).withHour(0).withMinute(0).withSecond(0).withNano(0);
        String monthStartStr = monthStart.format(datetimeFormatter);
        Long gongsiMonthNum = getGongsiNum(monthStartStr);
        log.info("gongsiMonthNum:{}", gongsiMonthNum);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.GongsiMonthTotal, gongsiMonthNum);
    }

    public void getGongsiDayNum() {
        DateTimeFormatter datetimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        LocalDateTime dayStart = LocalDateTime.now().withHour(0).withMinute(0).withSecond(0).withNano(0);
        String dayStartStr = dayStart.format(datetimeFormatter);
        Long gongsiDayNum = getGongsiNum(dayStartStr);
        log.info("gongsiDayNum:{}", gongsiDayNum);
        ZuoyeNumCacheMap.put(ZuoyeNumConstants.GongsiDayTotal, gongsiDayNum);
    }


    private Long getGongsiNum(String zuoyeshijian) {
        String sql = "SELECT COUNT(1) AS operationNum  " +
                "FROM t_dispatch_detail  " +
                "WHERE tdd_status = 2  " +
                "AND tdd_operation_end_time >= ?  " +
                "AND dr = 0";
        Map map = secondDataSourceRepository.queryForMap(sql, zuoyeshijian);
        if (null != map ) {
            return (Long) map.getOrDefault("operationNum", 0L);
        };
        return 0L;
    }










    public void initBanshichuZuoyeNum() {
        DateTimeFormatter datetimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        LocalDateTime yearStart = LocalDateTime.now().withDayOfYear(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        String yearStartStr = yearStart.format(datetimeFormatter);
        String sql = "SELECT tdd_owner_base AS tddOwnerBase, COUNT(*) AS totalNum    " +
                "FROM t_dispatch_detail   " +
                "WHERE tdd_status = 2  " +
                "AND tdd_operation_end_time >= ?   " +
                "AND dr = 0   " +
                "GROUP BY tdd_owner_base  ";

        List<OwnerBaseNumVo> ownerBaseNumVoList =secondDataSourceRepository.queryForList(sql, OwnerBaseNumVo.class, yearStartStr);
        Map<Integer, Long> ownerBaseNumVoMap = new HashMap<>();
        if (ownerBaseNumVoList != null && ownerBaseNumVoList.size() > 0) {
            ownerBaseNumVoList.forEach(ownerBaseNumVo -> {
                ownerBaseNumVoMap.put(ownerBaseNumVo.getTddOwnerBase(), ownerBaseNumVo.getTotalNum());
            });
        }
        log.info("ownerBaseNumVoMap:{}", objectMapper.valueToTree(ownerBaseNumVoMap));
        try {
            log.info("ownerBaseNumVoMap: " + objectMapper.writeValueAsString(ownerBaseNumVoMap));
        } catch (JsonProcessingException e) {
            log.error("ownerBaseNumVoMap: " + e.getMessage());
        }
        BanshichuZuoyeNumCacheMap.putAll(ownerBaseNumVoMap);
    }



    public List<OwnerBaseNumVo> getBanshichuZuoyeNum() {
        Map<Integer, Long> ownerBaseNumVoMap = BanshichuZuoyeNumCacheMap.asMap();
        List<OwnerBaseNumVo> ownerBaseNumVoList = new ArrayList<>();
        if (ownerBaseNumVoMap != null && ownerBaseNumVoMap.size() > 0) {
            ownerBaseNumVoMap.forEach((key, value) -> {
                OwnerBaseNumVo ownerBaseNumVo = new OwnerBaseNumVo();
                ownerBaseNumVo.setTddOwnerBase(key);
                ownerBaseNumVo.setTotalNum(value);
                String ownerBaseName = OwnerBaseIdNameCacheName.get(String.valueOf(key));
                ownerBaseNumVo.setTddOwnerBaseName(ownerBaseName);
                ownerBaseNumVoList.add(ownerBaseNumVo);
            });
        }
        List<OwnerBaseNumVo> ownerBaseNumVoList1 = ownerBaseNumVoList.stream().sorted(Comparator.comparing(OwnerBaseNumVo::getTotalNum).reversed()).collect(Collectors.toList());
        return ownerBaseNumVoList1;
    }


    public List<DaquNumVo> getDaquZuoyeNum() {
        Map<Integer, Long> ownerBaseNumVoMap = BanshichuZuoyeNumCacheMap.asMap();
        List<DaquOwnerBaseNumVo> daquOwnerBaseNumVoList = new ArrayList<>();
        if (ownerBaseNumVoMap != null && ownerBaseNumVoMap.size() > 0) {
            ownerBaseNumVoMap.forEach((key, value) -> {
                DaquOwnerBaseNumVo daquOwnerBaseNumVo = new DaquOwnerBaseNumVo();
                daquOwnerBaseNumVo.setTddOwnerBase(key);
                daquOwnerBaseNumVo.setTotalNum(value);
                String ownerBaseName = OwnerBaseIdNameCacheName.get(String.valueOf(key));
                daquOwnerBaseNumVo.setTddOwnerBaseName(ownerBaseName);
                String daquName = (String) BanshichuDaquCacheMap.get(ownerBaseName);
                daquOwnerBaseNumVo.setDaquName(daquName);
                daquOwnerBaseNumVoList.add(daquOwnerBaseNumVo);
            });
        }
        Map<String,List<DaquOwnerBaseNumVo>> map = daquOwnerBaseNumVoList.stream().filter(daquOwnerBaseNumVo -> daquOwnerBaseNumVo.getDaquName() != null).collect(Collectors.groupingBy(daquOwnerBaseNumVo -> daquOwnerBaseNumVo.getDaquName()));
        List<DaquNumVo> daquNumVoList = new ArrayList<>();
        map.entrySet().forEach(entry -> {
            DaquNumVo daquNumVo = new DaquNumVo();
            daquNumVo.setDaquName(entry.getKey());
            daquNumVo.setTotalNum(entry.getValue().stream().mapToLong(DaquOwnerBaseNumVo::getTotalNum).sum());
            daquNumVoList.add(daquNumVo);
        });
        List<DaquNumVo> daquNumVoList1 = daquNumVoList.stream().sorted(Comparator.comparing(DaquNumVo::getTotalNum).reversed()).collect(Collectors.toList());
        return daquNumVoList1;
    }



    public Map getZuoyeNum() {
        Map map = new HashMap();
        map = ZuoyeNumCacheMap.asMap();
        return map;
    }




    /** 对实时作业情况进行初始化 */
    public void initShishiZuoyeQingkuang() {
        String sql = "SELECT tdd_owner_base, tdd_operation_type, tdd_vin, tdd_operation_end_time  " +
                "FROM t_dispatch_detail    " +
                "WHERE tdd_status = 2    " +
                "AND dr = 0  " +
                "ORDER BY tdd_id DESC    " +
                "LIMIT 50  ";
        List<ZuoyeShishiQingkuangVo1> zuoyeShishiQingkuangVo1List = secondDataSourceRepository.queryForList(sql, ZuoyeShishiQingkuangVo1.class);
        if (zuoyeShishiQingkuangVo1List != null && zuoyeShishiQingkuangVo1List.size() > 0) {
            zuoyeShishiQingkuangVo1List.forEach(zuoyeShishiQingkuangVo1 -> {
                ZuoyeShishiQingkuangVo zuoyeShishiQingkuangVo = new ZuoyeShishiQingkuangVo();
                String ownerBaseName = OwnerBaseIdNameCacheName.get(String.valueOf(zuoyeShishiQingkuangVo1.getTddOwnerBase()));
                zuoyeShishiQingkuangVo.setOwnerBaseName(ownerBaseName);
                OperateTypeEnum operateTypeEnum = OperateTypeEnum.fromValue(zuoyeShishiQingkuangVo1.getTddOperationType());
                zuoyeShishiQingkuangVo.setOperateType(operateTypeEnum);
                zuoyeShishiQingkuangVo.setOperateTime(zuoyeShishiQingkuangVo1.getTddOperationEndTime());
                zuoyeShishiQingkuangVo.setVin(zuoyeShishiQingkuangVo1.getTddVin());
                ZuoyeShishiQingkuang.offer(zuoyeShishiQingkuangVo);
            });
        }
    }

    /** 获取实时作业情况 */
    public List<ZuoyeShishiQingkuangVo> getShishiZuoyeQingkuang() {
        List<ZuoyeShishiQingkuangVo> zuoyeShishiQingkuangVoList = new ArrayList<>();
        ZuoyeShishiQingkuang.iterator().forEachRemaining(zuoyeShishiQingkuangVo -> {
            zuoyeShishiQingkuangVoList.add(zuoyeShishiQingkuangVo);
        });
        return zuoyeShishiQingkuangVoList;
    }




    /** 获取作业占比 */
    public List<ZhanbiVo> getZuoyeZhanbi() {
        List<ZhanbiVo> zhanbiVoList = new ArrayList<>();
        List<String> nameList = new ArrayList<>();
        nameList.add("接车短驳");
        nameList.add("装车");
        nameList.add("卸车");
        nameList.add("配送");
        nameList.add("公路直发");
        nameList.add("移库");
        nameList.add("加固");
        nameList.add("解固");
        nameList.add("其他");

        ZhanbiVo zhanbiVo0 = new ZhanbiVo();
        zhanbiVo0.setName("接车短驳");
        zhanbiVo0.setValue(ZuoyeNumCacheMap.get(ZuoyeNumConstants.JiecheduanboYearTotal));
        zhanbiVoList.add(zhanbiVo0);

        ZhanbiVo zhanbiVo1 = new ZhanbiVo();
        zhanbiVo1.setName("装车");
        zhanbiVo1.setValue(ZuoyeNumCacheMap.get(ZuoyeNumConstants.ZhuangcheYearTotal));
        zhanbiVoList.add(zhanbiVo1);

        ZhanbiVo zhanbiVo2 = new ZhanbiVo();
        zhanbiVo2.setName("卸车");
        zhanbiVo2.setValue(ZuoyeNumCacheMap.get(ZuoyeNumConstants.XiecheYearTotal));
        zhanbiVoList.add(zhanbiVo2);

        ZhanbiVo zhanbiVo3 = new ZhanbiVo();
        zhanbiVo3.setName("配送");
        zhanbiVo3.setValue(ZuoyeNumCacheMap.get(ZuoyeNumConstants.PeisongYearTotal));
        zhanbiVoList.add(zhanbiVo3);

        ZhanbiVo zhanbiVo4 = new ZhanbiVo();
        zhanbiVo4.setName("公路直发");
        zhanbiVo4.setValue(ZuoyeNumCacheMap.get(ZuoyeNumConstants.GongluzhifaYearTotal));
        zhanbiVoList.add(zhanbiVo4);

        ZhanbiVo zhanbiVo5 = new ZhanbiVo();
        zhanbiVo5.setName("移库");
        zhanbiVo5.setValue(ZuoyeNumCacheMap.get(ZuoyeNumConstants.YikuduanboYearTotal));
        zhanbiVoList.add(zhanbiVo5);

        ZhanbiVo zhanbiVo6 = new ZhanbiVo();
        zhanbiVo6.setName("加固");
        zhanbiVo6.setValue(ZuoyeNumCacheMap.get(ZuoyeNumConstants.JiaguYearTotal));
        zhanbiVoList.add(zhanbiVo6);

        ZhanbiVo zhanbiVo7 = new ZhanbiVo();
        zhanbiVo7.setName("解固");
        zhanbiVo7.setValue(ZuoyeNumCacheMap.get(ZuoyeNumConstants.JieguYearTotal));
        zhanbiVoList.add(zhanbiVo7);

        ZhanbiVo zhanbiVo8 = new ZhanbiVo();
        zhanbiVo8.setName("其他");
        long a1 = ZuoyeNumCacheMap.get(ZuoyeNumConstants.GongsiYearTotal);
        zhanbiVo8.setValue(a1 - zhanbiVo0.getValue() - zhanbiVo1.getValue() - zhanbiVo2.getValue() - zhanbiVo3.getValue() - zhanbiVo4.getValue() - zhanbiVo5.getValue() - zhanbiVo6.getValue() - zhanbiVo7.getValue());
        zhanbiVoList.add(zhanbiVo8);

        String json = "";
        try {
            json = objectMapper.writeValueAsString(zhanbiVoList);
        } catch (JsonProcessingException e) {
            log.error("json转换异常:" + e.getMessage());
            throw new RuntimeException(e);
        }
        log.info("zhanbiVoList: " + json);
        return zhanbiVoList;
    }
}

1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.
72.
73.
74.
75.
76.
77.
78.
79.
80.
81.
82.
83.
84.
85.
86.
87.
88.
89.
90.
91.
92.
93.
94.
95.
96.
97.
98.
99.
100.
101.
102.
103.
104.
105.
106.
107.
108.
109.
110.
111.
112.
113.
114.
115.
116.
117.
118.
119.
120.
121.
122.
123.
124.
125.
126.
127.
128.
129.
130.
131.
132.
133.
134.
135.
136.
137.
138.
139.
140.
141.
142.
143.
144.
145.
146.
147.
148.
149.
150.
151.
152.
153.
154.
155.
156.
157.
158.
159.
160.
161.
162.
163.
164.
165.
166.
167.
168.
169.
170.
171.
172.
173.
174.
175.
176.
177.
178.
179.
180.
181.
182.
183.
184.
185.
186.
187.
188.
189.
190.
191.
192.
193.
194.
195.
196.
197.
198.
199.
200.
201.
202.
203.
204.
205.
206.
207.
208.
209.
210.
211.
212.
213.
214.
215.
216.
217.
218.
219.
220.
221.
222.
223.
224.
225.
226.
227.
228.
229.
230.
231.
232.
233.
234.
235.
236.
237.
238.
239.
240.
241.
242.
243.
244.
245.
246.
247.
248.
249.
250.
251.
252.
253.
254.
255.
256.
257.
258.
259.
260.
261.
262.
263.
264.
265.
266.
267.
268.
269.
270.
271.
272.
273.
274.
275.
276.
277.
278.
279.
280.
281.
282.
283.
284.
285.
286.
287.
288.
289.
290.
291.
292.
293.
294.
295.
296.
297.
298.
299.
300.
301.
302.
303.
304.
305.
306.
307.
308.
309.
310.
311.
312.
313.
314.
315.
316.
317.
318.
319.
320.
321.
322.
323.
324.
325.
326.
327.
328.
329.
330.
331.
332.
333.
334.
335.
336.
337.
338.
339.
340.
341.
342.
343.
344.
345.
346.
347.
348.
349.
350.
351.
352.
353.
354.
355.
356.
357.
358.
359.
360.
361.
362.
363.
364.
365.
366.
367.
368.
369.
370.
371.
372.
373.
374.
375.
376.
377.
378.
379.
380.
381.
382.
383.
384.
385.
386.
387.
388.
389.
390.
391.
392.
393.
394.
395.
396.
397.
398.
399.
400.
401.
402.
403.
404.
405.
406.
407.
408.
409.
410.
411.
412.
413.
414.
415.
416.
417.
418.
419.
420.
421.
422.
423.
424.
425.
426.
427.
428.
429.
430.
431.
432.
433.
434.
435.
436.
437.
438.
439.
440.
441.
442.
443.
444.
445.
446.
447.
448.
449.
450.
451.
452.
453.
454.
455.
456.
457.
458.
459.
460.
461.
462.
463.
464.
465.
466.
467.
468.
469.
470.
471.
472.
473.
474.
475.
476.
477.
478.
479.

ZaituDingdanNumSerivce.java

package com.cqsym.newbig.service;



import com.cqsym.newbig.config.DataSourceRepository;
import com.cqsym.newbig.vo.ZaituDingdanVo;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;

import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;

import static com.cqsym.newbig.cache.AppMap.ZaituDingdanNumForEveryDayCacheList;

@Service
public class ZaituDingdanNumSerivce {
    private static final Logger log = LoggerFactory.getLogger(ZaituDingdanNumSerivce.class);
    @Autowired
    @Qualifier(value="secondDataSourceRepository")
    private DataSourceRepository secondDataSourceRepository;
    @Autowired
    private ObjectMapper objectMapper;

    public void initZaituDingdanNumForEveryDay() {
        log.info("initZaituDingdanNumForEveryDay... ");
        DateTimeFormatter datetimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        LocalDateTime yearStart = LocalDateTime.now().withDayOfYear(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        String yearStartStr = yearStart.format(datetimeFormatter);
        String sql = "SELECT to_start_province, to_delivery_province , COUNT(*) AS count_num  " +
                "FROM t_order  " +
                "WHERE to_status = 3  " +
                "AND to_order_date >= ?  " +
                "AND to_start_province != to_delivery_province  " +
                "GROUP BY to_start_province, to_delivery_province  " +
                "ORDER BY count_num DESC  ";
        List<ZaituDingdanVo> zaituDingdanVoList =secondDataSourceRepository.queryForList(sql, ZaituDingdanVo.class, yearStartStr);
        String json = "";
        try {
            json = objectMapper.writeValueAsString(zaituDingdanVoList);
        } catch (JsonProcessingException e) {
            e.printStackTrace();
        }
        log.info("zaituDingdanVoList: " + json);
        ZaituDingdanNumForEveryDayCacheList.clear();
        ZaituDingdanNumForEveryDayCacheList.addAll(zaituDingdanVoList);
        log.info("zaituDingdanVoList: " + json);
    }


    public List<ZaituDingdanVo> getZaituDingdanNumForEveryDay() throws JsonProcessingException {
        log.info("getZaituDingdanNumForEveryDay... ");
        List<ZaituDingdanVo> zaituDingdanVoListResult = new ArrayList<>();
        List<ZaituDingdanVo> zaituDingdanVoList = ZaituDingdanNumForEveryDayCacheList;
        zaituDingdanVoList.stream().forEach(zaituDingdanVo -> {
            ZaituDingdanVo zaituDingdanVo1 = new ZaituDingdanVo();
            zaituDingdanVo1.setCountNum(zaituDingdanVo.getCountNum());
            String tosp = zaituDingdanVo.getToStartProvince().replace("省", "").replace("市", "").replace("自治区", "").replace("维吾尔", "").replace("壮族", "").replace("回族", "");
            String todp = zaituDingdanVo.getToDeliveryProvince().replace("省", "").replace("市", "").replace("自治区", "").replace("维吾尔", "").replace("壮族", "").replace("回族", "");
            zaituDingdanVo1.setToStartProvince(tosp);
            zaituDingdanVo1.setToDeliveryProvince(todp);
            zaituDingdanVoListResult.add(zaituDingdanVo1);
        });
        log.info("zaituDingdanVoList: " + objectMapper.writeValueAsString(zaituDingdanVoList));
        return zaituDingdanVoListResult;
    }




}

XianluMingchengService.java

package com.cqsym.newbig.service;



import com.cqsym.newbig.config.DataSourceRepository;
import com.cqsym.newbig.vo.XianluMingchengVo;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;

import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static com.cqsym.newbig.cache.AppMap.XianluMingchengCacheMap;


@Service
public class XianluMingchengService {
    private static final Logger log = LoggerFactory.getLogger(XianluMingchengService.class);
    @Autowired
    private ObjectMapper objectMapper;
    @Autowired
    @Qualifier(value="secondDataSourceRepository")
    private DataSourceRepository secondDataSourceRepository;

    public void init() {
        log.info("XianluMingchengService init() ... ");
        initXianluMingcheng();
    }



    public void initXianluMingcheng() {
        DateTimeFormatter datetimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        LocalDateTime yearStart = LocalDateTime.now().withDayOfYear(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        String yearStartStr = yearStart.format(datetimeFormatter);

        String sql = "SELECT to_start_province AS toStartProvince, to_delivery_province AS toDeliveryProvince, COUNT(*) AS toTotalNum  " +
                "FROM t_order  " +
                "WHERE dr = 0  " +
                "AND to_status != 5  " +
                "AND createtime >= ?  " +
                "AND to_start_province IS NOT NULL  " +
                "AND to_delivery_province IS NOT NULL  " +
                "GROUP BY to_start_province,to_delivery_province  " +
                "ORDER BY toTotalNum DESC ";
        List<XianluMingchengVo> list = secondDataSourceRepository.queryForList(sql, XianluMingchengVo.class, yearStartStr);
        if (null != list && list.size() > 0) {
            for (XianluMingchengVo xianluMingchengVo : list) {
                log.info(xianluMingchengVo.getToStartProvince() + "--" + xianluMingchengVo.getToDeliveryProvince() + "--" + xianluMingchengVo.getToTotalNum());
                XianluMingchengCacheMap.put(xianluMingchengVo.getToStartProvince() + "--" + xianluMingchengVo.getToDeliveryProvince(), xianluMingchengVo.getToTotalNum());
            }
        }
        String mapStr = "";
        try {
            mapStr = objectMapper.writeValueAsString(XianluMingchengCacheMap);
        } catch (Exception exception) {
            exception.printStackTrace();
            log.error("XianluMingchengService getXianluMingcheng() 发生了异常！ " + exception.getMessage());
        }
        log.info("XianluMingchengCacheMap: {}", mapStr );
    }



    public List<XianluMingchengVo> getXianluMingcheng() {
        Map<String, Long> map = XianluMingchengCacheMap.asMap();
        List<XianluMingchengVo> list = new ArrayList<>();
        map.entrySet().forEach(entry -> {
            XianluMingchengVo xianluMingchengVo = new XianluMingchengVo();
            String key1 = entry.getKey().replace("省", "").replace("市", "").replace("自治区", "").replace("维吾尔", "").replace("壮族", "").replace("回族", "");
            log.info("key:{},value:{}", key1, entry.getValue());
            xianluMingchengVo.setXianluMingcheng(key1);
            xianluMingchengVo.setToTotalNum(entry.getValue());
            list.add(xianluMingchengVo);
        });
        List<XianluMingchengVo> list1 = list.stream().sorted((o1, o2) -> o2.getToTotalNum().compareTo(o1.getToTotalNum())).collect(Collectors.toList());
        log.info("list1:{}", list1);
        return list1;
    }







}

TownerBaseService.java

package com.cqsym.newbig.service;


import com.cqsym.newbig.primary.entity.TownerBase;
import com.cqsym.newbig.primary.repository.TownerBaseRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.util.List;

@Service
public class TownerBaseService {
    private static final Logger log = LoggerFactory.getLogger(TownerBaseService.class);
    @Autowired
    private TownerBaseRepository townerBaseRepository;

    public List<TownerBase> findAll() {
        return townerBaseRepository.findAll();
    }

}

BinLogService.java

package com.cqsym.newbig.service;

import com.cqsym.newbig.dese.DataChangeInfo;
import com.cqsym.newbig.dese.DataChangeSink;
import com.cqsym.newbig.dese.MysqlDeserialization;
import com.ververica.cdc.connectors.mysql.source.MySqlSource;
import com.ververica.cdc.connectors.mysql.table.StartupOptions;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.RestOptions;
import org.apache.flink.streaming.api.datastream.DataStreamSink;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

@Service
public class BinLogService {
    private static final Logger log = LoggerFactory.getLogger(BinLogService.class);

    private StreamExecutionEnvironment env;
    private JobExecutionResult jobExecutionResult;

    @Value("${spring.datasource.second.host}")
    private String host;
    @Value("${spring.datasource.second.port}")
    private Integer port;
    @Value("${spring.datasource.second.username}")
    private String username;
    @Value("${spring.datasource.second.password}")
    private String password;


    public void start() {
        log.info("BinLogService start ... ");
        MySqlSource<DataChangeInfo> source = MySqlSource.<DataChangeInfo>builder()
                .hostname(host)
                .port(port)
                .databaseList("twms")
                .tableList("twms.t_order","twms.t_dispatch_detail")
                .username(username)
                .password(password)
                // initial:初始化快照,即全量导入后增量导入(检测更新数据写入)
                .startupOptions(StartupOptions.latest())
                .deserializer(new MysqlDeserialization())
                .includeSchemaChanges(true)
                .serverTimeZone("GMT+8")
                .build();

        Configuration configuration =  new Configuration();
        configuration.setInteger(RestOptions.PORT, 8081);
        env = StreamExecutionEnvironment.getExecutionEnvironment(configuration);

        env.enableCheckpointing(5000);
        //DataStreamSink<String> sink = env.fromSource(source, WatermarkStrategy.noWatermarks(), "MySQL Source").addSink(new CustomSink());
        DataStreamSink<DataChangeInfo> sink = env.fromSource(source, WatermarkStrategy.noWatermarks(), "MySQL Source").addSink(new DataChangeSink());
        try {
            log.info("开始执行 env.execute() ... ");
            jobExecutionResult = env.execute("BinLogService1");
        } catch (Exception e) {
            log.info("env.execute() 执行异常 ... " + e.getMessage());
            e.printStackTrace();
        }
    }



    public void stop() {
        // TODO:: 关闭任务;
        log.info("BinLogService stop ... ");
    }

}

三、这里有2个业务相关多RUNNABLE

TDispatchDetailListenerTaskRunnable.java

package com.cqsym.newbig.listener;


import com.cqsym.newbig.cache.AppMap;
import com.cqsym.newbig.constant.AppConstants;
import com.cqsym.newbig.constant.ZuoyeNumConstants;
import com.cqsym.newbig.dese.DataChangeInfo;
import com.cqsym.newbig.enums.OperateTypeEnum;
import com.cqsym.newbig.vo.ZuoyeShishiQingkuangVo;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.HashMap;
import java.util.Map;

public class TDispatchDetailListenerTaskRunnable implements Runnable {
    private static final Logger log = LoggerFactory.getLogger(TDispatchDetailListenerTaskRunnable.class);
    private ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule());

    private DataChangeInfo dataChangeInfo;
    public TDispatchDetailListenerTaskRunnable(DataChangeInfo dataChangeInfo) {
        this.dataChangeInfo = dataChangeInfo;
    }

    @Override
    public void run() {
        log.info("--------TDispatchDetailListenerTaskRunnable---run------");
        log.info(dataChangeInfo.toString());
        handleBinlogRowData(dataChangeInfo);
    }

    private void handleBinlogRowData(DataChangeInfo dataChangeInfo) {
        if (!(dataChangeInfo.getDatabase().equals("twms") && dataChangeInfo.getTableName().equals("t_dispatch_detail"))) {
            log.error("数据库名:{}，表名:{}，不是twms.t_dispatch_detail", dataChangeInfo.getDatabase(), dataChangeInfo.getTableName());
            return;
        }
        log.info("TDispatchDetailListenerTaskRunnable dataChangeInfo: " + dataChangeInfo);
        if (dataChangeInfo.getOperatorType().equals(1)) {
            String afterData = dataChangeInfo.getAfterData();
            log.info("TDispatchDetailListenerTaskRunnable 新增 afterData:{}", afterData);
            Map row = new HashMap();
            try {
                row = objectMapper.readValue(afterData, Map.class);
            } catch (JsonProcessingException e) {
                log.info("afterData转换成对象发生异常：" + e.getMessage());
            }
            log.info("TDispatchDetailListenerTaskRunnable row: " + row);
            if (row.get("tdd_status").toString().equals("2") ) {
                log.info("作业tdd_operation_type:{}", row.get("tdd_operation_type").toString());
                /** 完成作业数据量的更新 */
                handleZuoyeNum(row);
                /** 添加作业实时情况 */
                handleZuoyeShishiQiankuang(row);
                /** 添加作业到办事处 */
                handleZuoyeNumAddToOwnerBase(row);
            }
        } else if (dataChangeInfo.getOperatorType().equals(2)) {
            String afterData = dataChangeInfo.getAfterData();
            log.info("TDispatchDetailListenerTaskRunnable 修改 afterData:{}", afterData);
            Map row = new HashMap();
            try {
                row = objectMapper.readValue(afterData, Map.class);
            } catch (JsonProcessingException e) {
                log.info("afterData转换成对象发生异常：" + e.getMessage());
            }
            log.info("TDispatchDetailListenerTaskRunnable row: " + row);
            if (row.get("tdd_status").toString().equals("2") ) {
                log.info("作业tdd_operation_type:{}", row.get("tdd_operation_type").toString());
                /** 完成作业数据量的更新 */
                handleZuoyeNum(row);
                /** 添加作业实时情况 */
                handleZuoyeShishiQiankuang(row);
                /** 添加作业到办事处 */
                handleZuoyeNumAddToOwnerBase(row);
            }
        } else if (dataChangeInfo.getOperatorType().equals(3)) {
            String afterData = dataChangeInfo.getAfterData();
            log.info("TDispatchDetailListenerTaskRunnable 删除 afterData:{}", afterData);
        }

    }





    /** 完成作业数据量的更新 */
    private void handleZuoyeNum(Map row) {
        switch (row.get("tdd_operation_type").toString()) {
            case "1":
                handleZuoyeCacheNum(ZuoyeNumConstants.JiecheduanboDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.JiecheduanboMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.JiecheduanboYearTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiYearTotal);
                break;
            case "2":
                handleZuoyeCacheNum(ZuoyeNumConstants.ZhuangcheDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.ZhuangcheMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.ZhuangcheYearTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiYearTotal);
                break;
            case "3":
                handleZuoyeCacheNum(ZuoyeNumConstants.XiecheDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.XiecheMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.XiecheYearTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiYearTotal);
                break;
            case "4":
                handleZuoyeCacheNum(ZuoyeNumConstants.JiaguDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.JiaguMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.JiaguYearTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiYearTotal);
                break;
            case "5":
                handleZuoyeCacheNum(ZuoyeNumConstants.JieguDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.JieguMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.JieguYearTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiYearTotal);
                break;
            case "10":
                handleZuoyeCacheNum(ZuoyeNumConstants.PeisongDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.PeisongMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.PeisongYearTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiYearTotal);
                break;
            case "11":
                handleZuoyeCacheNum(ZuoyeNumConstants.YikuduanboDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.YikuduanboMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.YikuduanboYearTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiYearTotal);
                break;
            case "15":
                handleZuoyeCacheNum(ZuoyeNumConstants.GongluzhifaDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongluzhifaMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongluzhifaYearTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiYearTotal);
                break;
            default:
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiDayTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiMonthTotal);
                handleZuoyeCacheNum(ZuoyeNumConstants.GongsiYearTotal);
                //break;
        }
    }

    private void handleZuoyeCacheNum(String str) {
        AppMap.ZuoyeNumCacheMap.incrementAndGet(str);
    }



    /** 添加作业实时情况 */
    private void handleZuoyeShishiQiankuang(Map row) {
        /** 添加作业实时情况 */
        log.info("作业实时情况after row:{}", row.toString());
//        log.info("作业实时情况tdd_operation_type: " + row.get("tdd_operation_type").toString());
//        log.info("作业实时情况tdd_owner_base: " + row.get("tdd_owner_base").toString());
//        log.info("作业实时情况tdd_vin: " + row.get("tdd_vin").toString());
//        log.info("作业实时情况tdd_operation_end_time: " + row.get("tdd_operation_end_time").toString());
        // 28,800 8小时，要减去
        String a1 = row.get("tdd_owner_base").toString();
        String a2 = row.get("tdd_operation_type").toString();
        String a3 = row.get("tdd_vin").toString();
        Long a40 = Long.valueOf(row.get("tdd_operation_end_time").toString())-Long.valueOf(28800000);
        LocalDateTime localDateTimea4 = LocalDateTime.ofInstant(Instant.ofEpochMilli(Long.valueOf(row.get("tdd_operation_end_time").toString())), ZoneOffset.of("+0"));
//        log.info("作业实时情况a1: " + a1);
//        log.info("作业实时情况a2: " + a2);
//        log.info("作业实时情况a3: " + a3);
//        log.info("作业实时情况a4: " + localDateTimea4);
//        log.info("作业实时情况now: " + LocalDateTime.now());

//        Long l1 = localDateTimea4.toEpochSecond(ZoneOffset.of("+0"));
//        Long l2 = LocalDateTime.now().toEpochSecond(ZoneOffset.of("+0"));
//        Long l3 = Math.abs(l2 - l1);
//        log.info("作业实时情况l2-l1时间差异: " + l3);
//        if (l3 >= AppConstants.ZuoyeShijianCha) {
//            log.info("作业实时情况4：" + "时间差异太大，不要了");
//            return;
//        }
//        log.info("作业实时情况3：" + "算正常情况");

        ZuoyeShishiQingkuangVo zuoyeShishiQingkuangVo = new ZuoyeShishiQingkuangVo();
        String ownerBaseNamea1 = "";
        ownerBaseNamea1 = AppMap.OwnerBaseIdNameCacheName.get(a1);
        zuoyeShishiQingkuangVo.setOwnerBaseName(ownerBaseNamea1);
        zuoyeShishiQingkuangVo.setOperateType(OperateTypeEnum.fromValue(Integer.valueOf(a2)));
        zuoyeShishiQingkuangVo.setVin(a3);
        //zuoyeShishiQingkuangVo.setOperateTime(localDateTimea4);
        zuoyeShishiQingkuangVo.setOperateTime(LocalDateTime.now());
        log.info("作业实时情况1：" + zuoyeShishiQingkuangVo);
        String temp00 = "";
        try {
            temp00 = objectMapper.writeValueAsString(zuoyeShishiQingkuangVo);
        } catch (JsonProcessingException e) {
            log.info("作业实时情况2报错：报错信息" + e.getMessage());
        }
        log.info("作业实时情况2：" + temp00);
        AppMap.ZuoyeShishiQingkuang.offer(zuoyeShishiQingkuangVo);
    }


    /** 添加作业到办事处 */
    private void handleZuoyeNumAddToOwnerBase(Map row) {
        /** 添加作业到办事处 */
        log.info("after row:{}", row.toString());
        if (row.get("tdd_owner_base") != null) {
            handleZuoyeNumAddToOwnerBase(Integer.valueOf(row.get("tdd_owner_base").toString()));
        }
    }

    private void handleZuoyeNumAddToOwnerBase(Integer str) {
        AppMap.BanshichuZuoyeNumCacheMap.incrementAndGet(str);
    }



}

1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.
72.
73.
74.
75.
76.
77.
78.
79.
80.
81.
82.
83.
84.
85.
86.
87.
88.
89.
90.
91.
92.
93.
94.
95.
96.
97.
98.
99.
100.
101.
102.
103.
104.
105.
106.
107.
108.
109.
110.
111.
112.
113.
114.
115.
116.
117.
118.
119.
120.
121.
122.
123.
124.
125.
126.
127.
128.
129.
130.
131.
132.
133.
134.
135.
136.
137.
138.
139.
140.
141.
142.
143.
144.
145.
146.
147.
148.
149.
150.
151.
152.
153.
154.
155.
156.
157.
158.
159.
160.
161.
162.
163.
164.
165.
166.
167.
168.
169.
170.
171.
172.
173.
174.
175.
176.
177.
178.
179.
180.
181.
182.
183.
184.
185.
186.
187.
188.
189.
190.
191.
192.
193.
194.
195.
196.
197.
198.
199.
200.
201.
202.
203.
204.
205.
206.
207.
208.
209.
210.
211.
212.
213.
214.
215.
216.
217.
218.
219.
220.
221.
222.
223.
224.
225.
226.
227.
228.
229.
230.
231.
232.
233.
234.
235.
236.
237.
238.
239.

TOrderListenerTaskRunnable.java

package com.cqsym.newbig.listener;


import com.cqsym.newbig.cache.AppMap;
import com.cqsym.newbig.dese.DataChangeInfo;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.Map;

public class TOrderListenerTaskRunnable implements Runnable {
    private static final Logger log = LoggerFactory.getLogger(TOrderListenerTaskRunnable.class);
    private ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule());
    private DataChangeInfo dataChangeInfo;
    public TOrderListenerTaskRunnable(DataChangeInfo dataChangeInfo) {
        this.dataChangeInfo = dataChangeInfo;
    }

    @Override
    public void run() {
        log.info("--------TOrderListenerTaskRunnable---run------");
        log.info(dataChangeInfo.toString());
        handleBinlogRowData(dataChangeInfo);
    }


    private void handleBinlogRowData(DataChangeInfo dataChangeInfo) {
        if (!(dataChangeInfo.getDatabase().equals("twms") && dataChangeInfo.getTableName().equals("t_order"))) {
            log.error("数据库名:{}，表名:{}，不是twms.t_order", dataChangeInfo.getDatabase(), dataChangeInfo.getTableName());
            return;
        }
        log.info("TOrderListenerTaskRunnable dataChangeInfo: " + dataChangeInfo);
        if (dataChangeInfo.getOperatorType().equals(1)) {
            String afterData = dataChangeInfo.getAfterData();
            log.info("TOrderListenerTaskRunnable 新增 afterData:{}", afterData);
            Map row = new HashMap();
            try {
                row = objectMapper.readValue(afterData, Map.class);
            } catch (JsonProcessingException e) {
                log.info("afterData转换成对象发生异常：" + e.getMessage());
            }
            log.info("TOrderListenerTaskRunnable row: " + row);
            Long num = 0L;
            num = (Long) AppMap.XianluMingchengCacheMap.get(row.get("to_start_province").toString()+"--"+row.get("to_delivery_province").toString());
            if (num == null && num.equals(0)) {
                num = AppMap.XianluMingchengCacheMap.addAndGet(row.get("to_start_province").toString()+"--"+row.get("to_delivery_province").toString(), 1L);
                log.info("新增了一条线路： 【{}--{}】 数量为{}。 ", row.get("to_start_province").toString(), row.get("to_delivery_province").toString(), num);
            } else {
                num = AppMap.XianluMingchengCacheMap.incrementAndGet(row.get("to_start_province").toString()+"--"+row.get("to_delivery_province").toString());
                log.info("线路： 【{}--{}】 数量为{}。 ", row.get("to_start_province").toString(), row.get("to_delivery_province").toString(), num);
            }
        } else if (dataChangeInfo.getOperatorType().equals(2)) {
            String afterData = dataChangeInfo.getAfterData();
            log.info("TOrderListenerTaskRunnable 修改 afterData:{}", afterData);
            Map row = new HashMap();
            try {
                row = objectMapper.readValue(afterData, Map.class);
            } catch (JsonProcessingException e) {
                log.info("afterData转换成对象发生异常：" + e.getMessage());
            }
            log.info("TOrderListenerTaskRunnable row: " + row);

        } else if (dataChangeInfo.getOperatorType().equals(3)) {
            String afterData = dataChangeInfo.getAfterData();
            log.info("TOrderListenerTaskRunnable 删除 afterData:{}", afterData);
        }

    }







}

四、这里是Flink-CDC技术，我把所有用到的都放到了dese包里了。

dese包图片

使用flink-cdc技术（2.3.0）解析binlog实现实时数据大屏_apache

AggregationDataChangSinkUtils.java

这个主要用来实现对mysql-binlog解析后按照数据库名和表名选择不同的处理方式，这里虽然使用了静态类，但是在多线程环境下，需要在每个处理业务的都调用下initRegister()不然会获取不到对应的listener 接口。

package com.cqsym.newbig.dese;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ThreadPoolExecutor;


public class AggregationDataChangSinkUtils {
    private static final Logger log = LoggerFactory.getLogger(AggregationDataChangSinkUtils.class);
    public static Map<String, Ilistener> listenerMap = new HashMap<>();

    public static String genKey(String dbName, String tableName) {
        return dbName + ":" + tableName;
    }

    public static void register(String _dbName, String _tableName,
                         Ilistener ilistener) {
        log.info("register : {}-{}", _dbName, _tableName);
        listenerMap.put(genKey(_dbName, _tableName), ilistener);
    }

    public static Ilistener getListener(String _dbName, String _tableName) {
        return listenerMap.get(genKey(_dbName, _tableName));
    }

    public static Ilistener getListener(String getKey) {
        return listenerMap.get(getKey);
    }



    public static void initRegister() {
        register("twms", "t_dispatch_detail", new TDispatchDetailListener());
        register("twms", "t_order", new TOrderListener());
    }

    public static void printRegister() {
        Ilistener ilistener1 = AggregationDataChangSinkUtils.getListener("twms", "t_order");
        Ilistener ilistener2 = AggregationDataChangSinkUtils.getListener("twms", "t_dispatch_detail");
        log.info("ilistener1:" + ilistener1);
        log.info("ilistener2:" + ilistener2);
    }






    public static final ThreadPoolTaskExecutor aggregationListenerThreadPoolTaskExecutor;
    static {
        aggregationListenerThreadPoolTaskExecutor = new ThreadPoolTaskExecutor();
        aggregationListenerThreadPoolTaskExecutor.setCorePoolSize(40);
        aggregationListenerThreadPoolTaskExecutor.setMaxPoolSize(100);
        aggregationListenerThreadPoolTaskExecutor.setQueueCapacity(500);
        aggregationListenerThreadPoolTaskExecutor.setKeepAliveSeconds(60);
        aggregationListenerThreadPoolTaskExecutor.setThreadNamePrefix("aggregationListenerThreadPoolTaskExecutor--");
        aggregationListenerThreadPoolTaskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
        aggregationListenerThreadPoolTaskExecutor.initialize(); // 初始化线程池
    }

}

DataChangeInfo.java

flink-cdc获取到的binlog解析后的数据。

package com.cqsym.newbig.dese;

import lombok.Data;

@Data
public class DataChangeInfo {
    /**
     * 变更类型： 0 初始化 1新增 2修改 3删除 4导致源中的现有表被截断的操作
     */
    private Integer operatorType;
    /**
     * 变更前数据
     */
    private String beforeData;
    /**
     * 变更后数据
     */
    private String afterData;

    /**
     * 操作的数据
     */
    private String data;
    /**
     * binlog文件名
     */
    private String fileName;
    /**
     * binlog当前读取点位
     */
    private Integer filePos;
    /**
     * 数据库名
     */
    private String database;
    /**
     * 表名
     */
    private String tableName;
    /**
     * 变更时间
     */
    private Long operatorTime;
}

DataChangeSink.java

这个是cdc用于处理数据的sink。

package com.cqsym.newbig.dese;

import com.cqsym.newbig.utils.SpringContextUtils;
import com.cqsym.newbig.utils.SpringbootContextUtils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

import java.util.concurrent.ThreadPoolExecutor;


public class DataChangeSink implements SinkFunction<DataChangeInfo> {
    private static final Logger log = LoggerFactory.getLogger(DataChangeSink.class);
    private static final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule());

    public static final ThreadPoolTaskExecutor aggregationListenerThreadPoolTaskExecutor;
    static {
        aggregationListenerThreadPoolTaskExecutor = new ThreadPoolTaskExecutor();
        aggregationListenerThreadPoolTaskExecutor.setCorePoolSize(40);
        aggregationListenerThreadPoolTaskExecutor.setMaxPoolSize(100);
        aggregationListenerThreadPoolTaskExecutor.setQueueCapacity(500);
        aggregationListenerThreadPoolTaskExecutor.setKeepAliveSeconds(60);
        aggregationListenerThreadPoolTaskExecutor.setThreadNamePrefix("aggregationListenerThreadPoolTaskExecutor--");
        aggregationListenerThreadPoolTaskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
        aggregationListenerThreadPoolTaskExecutor.initialize(); // 初始化线程池
    }

    @Override
    public void invoke(DataChangeInfo dataChangeInfo, Context context) throws JsonProcessingException {
//        log.info("DataChangeSink invoke1: ... ");
//        log.info("DataChangeSink invoke3: " + SpringbootContextUtils.containsBean("aggregationListenerThreadPoolTaskExecutor"));
//        log.info("DataChangeSink invoke2: " + SpringContextUtils.containsBean("aggregationListenerThreadPoolTaskExecutor"));


        //ThreadPoolTaskExecutor threadPoolTaskExecutor = (ThreadPoolTaskExecutor) SpringContextUtils.getBean("aggregationListenerThreadPoolTaskExecutor");
        log.info("DataChangeSink invoke dataChangeInfo: " + dataChangeInfo);
        aggregationListenerThreadPoolTaskExecutor.submit(new DataChangeSinkRunnable(dataChangeInfo));
    }

}

DataChangeSinkRunnable.java

这里需要执行

AggregationDataChangSinkUtils.initRegister();

AggregationDataChangSinkUtils.printRegister();

不然获取不到注册的Listener

package com.cqsym.newbig.dese;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class DataChangeSinkRunnable implements Runnable {
    private static final Logger log = LoggerFactory.getLogger(DataChangeSinkRunnable.class);
    private static ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule());
    private DataChangeInfo dataChangeInfo;
    public DataChangeSinkRunnable(DataChangeInfo dataChangeInfo) {
        this.dataChangeInfo = dataChangeInfo;
    }

    @Override
    public void run() {
        AggregationDataChangSinkUtils.initRegister();
        AggregationDataChangSinkUtils.printRegister();

        log.info("DataChangeSinkRunnable run:");
        log.info("DataChangeSinkRunnable run:{}-{}", dataChangeInfo.getDatabase(), dataChangeInfo.getTableName());
        Ilistener ilistener = AggregationDataChangSinkUtils.getListener(dataChangeInfo.getDatabase(), dataChangeInfo.getTableName());
        log.info("DataChangeSinkRunnable run:ilistener: {}", ilistener);
        log.info("DataChangeSinkRunnable run:dataChangeInfo: {}", dataChangeInfo);
        if (ilistener != null) {
            ilistener.onEvent(dataChangeInfo);
        } else {
            log.warn("No listener registered for: {}:{}", dataChangeInfo.getDatabase(), dataChangeInfo.getTableName());
        }
    }

}

Ilistener.java

Ilistener for 为了后续扩展不同的实现，为了不同的表实现不同的功能。

package com.cqsym.newbig.dese;

/**
 * Ilistener for 为了后续扩展不同的实现
 */
public interface Ilistener {
    void onEvent(DataChangeInfo dataChangeInfo);
}

MysqlDeserialization.java

MysqlDeserialization主要是用于解析mysql-binlog的。

package com.cqsym.newbig.dese;

import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.ImmutableMap;
import com.ververica.cdc.debezium.DebeziumDeserializationSchema;
import io.debezium.data.Envelope;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.util.Collector;
import org.apache.kafka.connect.data.Field;
import org.apache.kafka.connect.data.Schema;
import org.apache.kafka.connect.data.Struct;
import org.apache.kafka.connect.source.SourceRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;
import java.util.Map;
import java.util.Optional;

public class MysqlDeserialization implements DebeziumDeserializationSchema<DataChangeInfo> {
    private static final Logger log = LoggerFactory.getLogger(MysqlDeserialization.class);

    public static final String TS_MS = "ts_ms";
    public static final String BIN_FILE = "file";
    public static final String POS = "pos";
    public static final String BEFORE = "before";
    public static final String AFTER = "after";
    public static final String SOURCE = "source";

    /**
     * 获取操作类型 READ CREATE UPDATE DELETE TRUNCATE;
     * 变更类型： 0 初始化 1新增 2修改 3删除 4导致源中的现有表被截断的操作
     */
    private static final Map<String, Integer> OPERATION_MAP = ImmutableMap.of(
            "READ", 0,
            "CREATE", 1,
            "UPDATE", 2,
            "DELETE", 3,
            "TRUNCATE", 4);



    @Override
    public void deserialize(SourceRecord sourceRecord, Collector<DataChangeInfo> collector) throws Exception {
        String topic = sourceRecord.topic();
        String[] fields = topic.split("\\.");
        String database = fields[1];
        String tableName = fields[2];
        Struct struct = (Struct) sourceRecord.value();
        final Struct source = struct.getStruct(SOURCE);
        DataChangeInfo dataChangeInfo = new DataChangeInfo();
        // 获取操作类型 READ CREATE UPDATE DELETE TRUNCATE;
        Envelope.Operation operation = Envelope.operationFor(sourceRecord);
        String type = operation.toString().toUpperCase();
        int eventType = OPERATION_MAP.get(type);
        // fixme 一般情况是无需关心其之前之后数据的,直接获取最新的数据即可,但这里为了演示,都进行输出
        dataChangeInfo.setBeforeData(getJsonObject(struct, BEFORE).toJSONString());
        dataChangeInfo.setAfterData(getJsonObject(struct, AFTER).toJSONString());
        if (eventType == 3) {
            dataChangeInfo.setData(getJsonObject(struct, BEFORE).toJSONString());
        } else {
            dataChangeInfo.setData(getJsonObject(struct, AFTER).toJSONString());
        }
        dataChangeInfo.setOperatorType(eventType);
        dataChangeInfo.setFileName(Optional.ofNullable(source.get(BIN_FILE)).map(Object::toString).orElse(""));
        dataChangeInfo.setFilePos(
                Optional.ofNullable(source.get(POS))
                        .map(x -> Integer.parseInt(x.toString()))
                        .orElse(0)
        );
        dataChangeInfo.setDatabase(database);
        dataChangeInfo.setTableName(tableName);
        dataChangeInfo.setOperatorTime(Optional.ofNullable(struct.get(TS_MS))
                .map(x -> Long.parseLong(x.toString())).orElseGet(System::currentTimeMillis));
        // 输出数据
        collector.collect(dataChangeInfo);
    }

    @Override
    public TypeInformation<DataChangeInfo> getProducedType() {
        return TypeInformation.of(DataChangeInfo.class);
    }


    /**
     * 从元素数据获取出变更之前或之后的数据
     *
     * @param value        value
     * @param fieldElement fieldElement
     * @return JSONObject
     */
    private JSONObject getJsonObject(Struct value, String fieldElement) {
        Struct element = value.getStruct(fieldElement);
        JSONObject jsonObject = new JSONObject();
        if (element != null) {
            Schema afterSchema = element.schema();
            List<Field> fieldList = afterSchema.fields();
            for (Field field : fieldList) {
                Object afterValue = element.get(field);
                jsonObject.put(field.name(), afterValue);
            }
        }
        return jsonObject;
    }

}

1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.
72.
73.
74.
75.
76.
77.
78.
79.
80.
81.
82.
83.
84.
85.
86.
87.
88.
89.
90.
91.
92.
93.
94.
95.
96.
97.
98.
99.
100.
101.
102.
103.
104.
105.
106.

TDispatchDetailListener.java

这个有2个功能，一个是用于提交到其他线程池进行真正的业务处理。他还有个被用的就是在Sink的可以根据获取的数据库名和表名，来进行不用的listener。

package com.cqsym.newbig.dese;

import com.cqsym.newbig.listener.TDispatchDetailListenerTaskRunnable;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

import java.util.concurrent.ThreadPoolExecutor;


public class TDispatchDetailListener implements Ilistener {
    private static final Logger log = LoggerFactory.getLogger(TDispatchDetailListener.class);
    private static final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule());
    private static ThreadPoolTaskExecutor tDispatchDetailListenerThreadPoolTaskExecutor;
    static {
        tDispatchDetailListenerThreadPoolTaskExecutor = new ThreadPoolTaskExecutor();
        tDispatchDetailListenerThreadPoolTaskExecutor.setCorePoolSize(20);
        tDispatchDetailListenerThreadPoolTaskExecutor.setMaxPoolSize(500);
        tDispatchDetailListenerThreadPoolTaskExecutor.setQueueCapacity(500);
        tDispatchDetailListenerThreadPoolTaskExecutor.setKeepAliveSeconds(60);
        tDispatchDetailListenerThreadPoolTaskExecutor.setThreadNamePrefix("tDispatchDetailListenerThreadPoolTaskExecutor--");
        tDispatchDetailListenerThreadPoolTaskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
        tDispatchDetailListenerThreadPoolTaskExecutor.initialize(); // 初始化线程池
    }

    @Override
    public void onEvent(DataChangeInfo dataChangeInfo) {
        log.info("--------TDispatchDetailListener---onEvent------");
        log.info(dataChangeInfo.toString());
        tDispatchDetailListenerThreadPoolTaskExecutor.submit(new TDispatchDetailListenerTaskRunnable(dataChangeInfo));
    }

}

TOrderListener.java

package com.cqsym.newbig.dese;

import com.cqsym.newbig.listener.TOrderListenerTaskRunnable;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

import java.util.concurrent.ThreadPoolExecutor;


public class TOrderListener implements Ilistener {
    private static final Logger log = LoggerFactory.getLogger(TOrderListener.class);
    private static final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule());

    private static ThreadPoolTaskExecutor tOrderListenerThreadPoolTaskExecutor;
    static {
        tOrderListenerThreadPoolTaskExecutor = new ThreadPoolTaskExecutor();
        tOrderListenerThreadPoolTaskExecutor.setCorePoolSize(20);
        tOrderListenerThreadPoolTaskExecutor.setMaxPoolSize(500);
        tOrderListenerThreadPoolTaskExecutor.setQueueCapacity(500);
        tOrderListenerThreadPoolTaskExecutor.setKeepAliveSeconds(60);
        tOrderListenerThreadPoolTaskExecutor.setThreadNamePrefix("tOrderListenerThreadPoolTaskExecutor--");
        tOrderListenerThreadPoolTaskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
        tOrderListenerThreadPoolTaskExecutor.initialize(); // 初始化线程池
    }
    @Override
    public void onEvent(DataChangeInfo dataChangeInfo) {
        log.info("--------TOrderListener---onEvent------");
        log.info(dataChangeInfo.toString());
        tOrderListenerThreadPoolTaskExecutor.execute(new TOrderListenerTaskRunnable(dataChangeInfo));
    }

}

五、flink-cdc技术封装的dese包里，主要使用步骤如下

前面的dese包的调用还是主要由binLogService该类的start方法启动。

使用flink-cdc技术（2.3.0）解析binlog实现实时数据大屏_spring_02

六、介绍下用到的3个缓存工具

使用flink-cdc技术（2.3.0）解析binlog实现实时数据大屏_spring_03

AppCacheMap.java

好像这个项目这里没有使用到。

package com.cqsym.newbig.cache;

import java.util.HashMap;
import java.util.Map;

public class AppCacheMap {
    public static Map cache = new HashMap();

    public static Object get(Object key) {
        return cache.get(key);
    }

    public static void put(Object key, Object value) {
        cache.put(key, value);
    }

}

AppMap.java

package com.cqsym.newbig.cache;


import com.cqsym.newbig.vo.ZaituDingdanVo;
import com.cqsym.newbig.vo.ZuoyeShishiQingkuangVo;
import com.google.common.util.concurrent.AtomicLongMap;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class AppMap {
    public static Map<String, String> BanshichuDaquCacheMap = new HashMap();
    public static final AtomicLongMap<Integer> BanshichuZuoyeNumCacheMap = AtomicLongMap.create();
    public static final AtomicLongMap<String> ZuoyeNumCacheMap = AtomicLongMap.create();
    public static final AtomicLongMap<String> XianluMingchengCacheMap = AtomicLongMap.create();
    public static final FixedSizeConcurrentQueue<ZuoyeShishiQingkuangVo> ZuoyeShishiQingkuang = new FixedSizeConcurrentQueue<ZuoyeShishiQingkuangVo>(50);
    public static Map<String, String> OwnerBaseIdNameCacheName = new HashMap();
    public static List<ZaituDingdanVo> ZaituDingdanNumForEveryDayCacheList = new ArrayList<>();



    static {
        initBanshichuDAquCacheMap();
    }



    private static void initBanshichuDAquCacheMap() {
        BanshichuDaquCacheMap.put("包头作业点", "华北");
        BanshichuDaquCacheMap.put("保定办事处", "华北");
        BanshichuDaquCacheMap.put("北京办事处", "华北");
        BanshichuDaquCacheMap.put("成都办事处", "西南");
        BanshichuDaquCacheMap.put("定州办事处", "华北");
        BanshichuDaquCacheMap.put("福州作业点", "华南");
        BanshichuDaquCacheMap.put("格尔木作业点", "西北");
    }


}

FixedSizeConcurrentQueue.java

FixedSizeConcurrentQueues是基于ConcurrentLinkedQueue封装的固定长度的，可以用于多线程环境的一个队列（Queue）。

package com.cqsym.newbig.cache;

import java.util.Collection;
import java.util.Iterator;
import java.util.concurrent.ConcurrentLinkedQueue;

public class FixedSizeConcurrentQueue<T> {
    private final ConcurrentLinkedQueue<T> queue;
    private final int maxSize;
    public FixedSizeConcurrentQueue(int maxSize) {
        this.maxSize = maxSize;
        this.queue = new ConcurrentLinkedQueue<>();
    }

    public boolean add(T element) {
        while (queue.size() >= maxSize) {
            // 队列已满，移除队头元素
            T polled = queue.poll();
            if (polled == null) {
                // poll() 返回 null 表示队列为空，这不应该发生，因为我们已经检查了队列大小
                throw new IllegalStateException("Queue unexpectedly empty");
            }
        }
        return queue.add(element);
    }

    public boolean offer(T element) {
        while (queue.size() >= maxSize) {
            // 队列已满，移除队头元素
            T polled = queue.poll();
            if (polled == null) {
                // poll() 返回 null 表示队列为空，这不应该发生，因为我们已经检查了队列大小
                throw new IllegalStateException("Queue unexpectedly empty");
            }
        }
        // 添加新元素到队列
        return queue.offer(element);
    }


    public T poll() {
        return queue.poll();
    }

    public T peek() {
        return queue.peek();
    }

    public int size() {
        return queue.size();
    }

    public boolean isEmpty() {
        return queue.isEmpty();
    }

    public boolean contains(Object o) {
        return queue.contains(o);
    }

    public boolean remove(Object o) {
        return queue.remove(o);
    }

    public boolean addAll(Collection<? extends T> c) {
        return queue.addAll(c);
    }

    public Iterator<T> iterator() {
        return queue.iterator();
    }

    public Object[] toArray() {
        return queue.toArray();
    }

    public <T> T[] toArray(T[] a) {
        return queue.toArray(a);
    }

    @Override
    public String toString() {
        return queue.toString();
    }


}

原创作者: lenglingx 转载于: https://blog.51cto.com/lenglingx/11579625

日记塔

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
使用flink-cdc技术（2.3.0）解析binlog实现实时数据大屏

使用flink-cdc技术（2.3.0）解析binlog实现实时数据大屏。一、项目环境该项目主要为一个数据大屏，采用了flink-cdc技术（2.3.0），flink版本为（1.14.6），利用他，实现自动获取并解析Mysql的binlog，来实时把对应的数据解析出来用于大屏展示。登录后复制 <!-- https:...
复制链接

扫一扫