Flink-Kafka-MySQL

2018年开始处理大数据相关的业务，Flink作为流处理新秀，在实时计算领域发挥着越来越大作用，本文主要整理在以往开发中Flink使用Kafka作为数据源，计算处理之后，再将数据存到MySQL的处理过程。

前置条件
启动zookeeper，启动kafka

业务系统发送消息到Kafka，使之作为数据源

import com.alibaba.fastjson.JSON;
import com.example.flinkdemo.model.User;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.util.Properties;

@Slf4j
public class KafkaSender {

    private static final String kafkaTopic = "flink-kafka";
    private static final String brokerAddress = "192.168.100.10:9092";
    private static Properties properties;
    private static void init() {
        properties = new Properties();
        properties.put("bootstrap.servers", brokerAddress);
        properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    }
    public static void main(String[] args) throws InterruptedException {
        init();
        sendUrlToKafka(); // 发送kafka消息
    }
    private static void sendUrlToKafka() {
        KafkaProducer producer = new KafkaProducer<String, String>(properties);
        long currentMills = System.currentTimeMillis();
        User user = new User();
        for (int i = 0; i <100000 ; i++) {
            user.setId(i+"");
            user.setName("test-flink-kafka-mysql"+i);
            user.setAge(i+"");
            // 确保发送的消息都是string类型
            String msgContent = JSON.toJSONString(user);
            ProducerRecord record = new ProducerRecord<String, String>(kafkaTopic, null, null, msgContent);
            producer.send(record);
            log.info("send msg:" + msgContent);
        }
        producer.flush();
    }
}

Flink从Kafka取数据进行计算

import com.alibaba.fastjson.JSON;
import com.example.flinkdemo.flink1.MySQLSink;
import com.example.flinkdemo.model.User;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.functions.FilterFunction;
im