flink 读取kafka 入到mysql案例

1、mysql结果：

2、maven文件

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.38</version>
        </dependency>

3、代码如下：

import java.util.Properties
import java.util.regex.Pattern

import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.scala.{DataStream, KeyedStream, StreamExecutionEnvironment}
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer
import org.apache.flink.api.scala._
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows
import org.apache.flink.streaming.api.windowing.time.Time


object StreamingApiApp {
  private val patternBd = Pattern.compile("^(.*),[0-9]{1,3}(.*)  - (.*)")

  def main(args: Array[String]): Unit = {
    val environment: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    environment.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
    environment.setParallelism(1)

    // configure Kafka consumer
    val kafkaProps = new Properties()
    kafkaProps.setProperty("bootstrap.servers", "k092")
    kafkaProps.setProperty("group.id", "flink_201909260905")

    val dStream: DataStream[String] = environment.
      addSource(new FlinkKafkaConsumer[String]("video",new SimpleStringSchema(),kafkaProps))
    val log: DataStream[String] = dStream.map((log) => {
      val matcher = patternBd.matcher(log)
      var result = ""
      if (matcher.matches()) {
        result=matcher.group(3).toString
      }
      result
    })
    log.map(value=>{
      val strings: Array[String] = value.split("\\|")
      strings(0).substring(0,10)
    }).addSink(new SinkToMySQL())

//    val socTextStream = environment.socketTextStream("10.18.34.199",1111)
//    socTextStream
//      .flatMap(_.split(" ")).map((_, 1)).keyBy(0).
//      timeWindow(Time.seconds(30)).sum(1).print()
    environment.execute()

  }
}

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;

public class SinkToMySQL extends RichSinkFunction<String>{

    Connection connection;
    PreparedStatement pstmt;


    private Connection getConnection() {
        Connection conn = null;
        try {
            Class.forName("com.mysql.jdbc.Driver");

            String url = "jdbc:mysql://1";

            conn = DriverManager.getConnection(url,"root","oop");

        } catch (Exception e) {
            e.printStackTrace();
        }

        return conn;
    }

    /**
     * 在open方法中建立connection
     * @param parameters
     * @throws Exception
     */
    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);

        connection = getConnection();
        String sql = "insert into flink_mysql_test_20190926(name) values (?)";
        pstmt = connection.prepareStatement(sql);


        System.out.println("open");

    }

    // 每条记录插入时调用一次
    public void invoke(String value, Context context) throws Exception {
        System.out.println("invoke~~~~~~~~~");
        // 未前面的占位符赋值
        pstmt.setString(1, value);
//        pstmt.setString(2, value.getName());
//        pstmt.setInt(3, value.getAge());

        pstmt.executeUpdate();

    }

    /**
     * 在close方法中要释放资源
     * @throws Exception
     */
    @Override
    public void close() throws Exception {
        super.close();

        if(pstmt != null) {
            pstmt.close();
        }

        if(connection != null) {
            connection.close();
        }
    }
}