flink cdc业务逻辑层处理

DataChangeSink类

package com.hbos.cdc.listener;

import com.google.gson.Gson;
import com.google.gson.internal.LinkedTreeMap;
import com.hbos.cdc.config.SpringBeanUtil;
import com.hbos.cdc.entity.TargetDataProperties;
import com.hbos.cdc.server.SqlServerCommonService;
import lombok.extern.slf4j.Slf4j;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.ibatis.jdbc.SqlRunner;

import java.sql.*;
import java.text.SimpleDateFormat;
import java.util.*;

/**
 * 自定义sink(接收器) 交由spring管理
 * 处理变更数据
 **/

@Slf4j
public class DataChangeSink extends RichSinkFunction<String> {

    Connection connection;

    //目标表连接
    private Connection getConnection() {
        TargetDataProperties targetDataProperties = SpringBeanUtil.getBean(TargetDataProperties.class);
        Connection conn = null;
        try {
            Class.forName(targetDataProperties.getDriverClassName());
            String url = targetDataProperties.getUrl();
            conn = DriverManager.getConnection(url, targetDataProperties.getUsername(), targetDataProperties.getPassword());
        } catch (Exception e) {
            e.printStackTrace();
        }
        return conn;
    }

    /**
     * 在项目初始化时执行一些特定的逻辑，比如建立与外部系统的连接、初始化资源等
     * @param parameters
     * @throws Exception
     */
    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
        connection = getConnection();
    }


    // 每条记录捕获时调用一次
    public void invoke(String value, Context context) throws Exception {
        //将Java对象序列化为JSON字符串,或将JSON字符串反序列化为Java对象。
        Gson gson = new Gson();
        //获取捕获原表的数据信息
        HashMap<String, Object> hashMap = gson.fromJson(value, HashMap.class);
        LinkedTreeMap<String,Object> source = (LinkedTreeMap<String,Object>)hashMap.get("source");
//        String database = (String) source.get("db"); //库名

        TargetDataProperties targetDataProperties = SpringBeanUtil.getBean(TargetDataProperties.class);
        //原表
        String sourceSchema = (String) source.get("schema"); // schema
        String sourceTable = (String) source.get("table"); // 表名
        String op = (String) hashMap.get("op");// 操作类型（增（c）、删(d)、改(u)、查(r)）

        SqlRunner sqlRunner = new SqlRunner(connection);

        List<String> primarykeysList = new ArrayList();
        //获取数据库元数据
        DatabaseMetaData metaData = connection.getMetaData();

        //TODO 获取所有字段
        List<String> allColumnsList = new ArrayList();

        //TODO 获取主键
        if (metaData != null){
            ResultSet keys = null;
            if (targetDataProperties.getUrl().contains("sqlserver")){
                keys = metaData.getPrimaryKeys(targetDataProperties.getDatabase(), sourceSchema, sourceTable);
            }else {
                keys = metaData.getPrimaryKeys(sourceSchema, "", sourceTable);
            }

            while (keys.next()) {
                String pkColumnName = keys.getString("COLUMN_NAME");
                primarykeysList.add(pkColumnName);
            }
        }

        //TODO 组织主键以外的字段
        ResultSet columns=null;
        if (targetDataProperties.getUrl().contains("sqlserver")){
            columns = metaData.getColumns(targetDataProperties.getDatabase(), sourceSchema, sourceTable, "%");
        }else {
            columns = metaData.getColumns(sourceSchema, sourceSchema, sourceTable, "%");
        }


        List<String> columnNamesList = new ArrayList();
        StringBuilder columnNames = new StringBuilder();
        StringBuilder primarykeys = new StringBuilder();
        //存储字段名对应的类型
        Map<String,Object> nameAndTypeMap = new HashMap();

        while (columns.next()) {
            //获取字段
            String columnName = columns.getString("COLUMN_NAME");
            //获取字段的类型
            Object columnType = columns.getString("TYPE_NAME");
            nameAndTypeMap.put(columnName,columnType);

            // 所有字段集合
            allColumnsList.add(columnName);
            // 过滤掉主键列集合
            if (!primarykeysList.contains(columnName)) {
                columnNamesList.add(columnName);
            }
        }

        Set<String> validTypes = new HashSet(Arrays.asList("v