Flink从kafka读取数据写入到mysql中，然后任务提交到集群

1、maven依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>flinktest</groupId>
    <artifactId>flink</artifactId>
    <version>1.0-SNAPSHOT</version>
<!--    <build>-->
<!--        <plugins>-->
<!--            <plugin>-->
<!--                <groupId>org.apache.maven.plugins</groupId>-->
<!--                <artifactId>maven-compiler-plugin</artifactId>-->
<!--                <configuration>-->
<!--                    <source>8</source>-->
<!--                    <target>8</target>-->
<!--                </configuration>-->
<!--            </plugin>-->
<!--        </plugins>-->
<!--&lt;!&ndash;    </build>&ndash;&gt;-->
    <dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_2.11</artifactId>
            <version>1.9.0</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>1.9.0</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.11</artifactId>
            <version>1.9.0</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_2.11</artifactId>
            <version>1.9.0</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.kudu</groupId>
            <artifactId>kudu-client</artifactId>
            <version>1.7.0</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>1.2.17</version>
        </dependency>
<!--        <dependency>-->
<!--            <groupId>org.apache.flink</groupId>-->
<!--            <artifactId>flink-connector-kafka_2.11</artifactId>-->
<!--            <version>1.10.0</version>-->
<!--        </dependency>-->
        <!-- https://mvnrepository.com/artifact/com.alibaba.otter/canal.client -->
        <dependency>
            <groupId>com.alibaba.otter</groupId>
            <artifactId>canal.client</artifactId>
            <version>1.1.4</version>
            <scope>provided</scope>
        </dependency>

<!--        <dependency>-->
<!--            <groupId>org.apache.flink</groupId>-->
<!--            <artifactId>flink-table-planner-blink_2.11</artifactId>-->
<!--            <version>1.10.0</version>-->
<!--&lt;!&ndash;            <scope>provided</scope>&ndash;&gt;-->
<!--        </dependency>-->

<!--        <dependency>-->
<!--            <groupId>org.apache.flink</groupId>-->
<!--            <artifactId>flink-table-common</artifactId>-->
<!--            <version>1.10.0</version>-->
<!--&lt;!&ndash;            <scope>provided</scope>&ndash;&gt;-->
<!--        </dependency>-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_2.11</artifactId>
            <version>1.9.0</version>
                        <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner_2.11</artifactId>
            <version>1.9.0</version>
                        <scope>provided</scope>
        </dependency>
<!--        <dependency>-->
<!--            <groupId>com.alibaba.blink</groupId>-->
<!--            <artifactId>flink-table_2.11</artifactId>-->
<!--            <version>blink-3.2.2</version>-->
<!--        </dependency>-->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.47</version>
        </dependency>

    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>8</source>
                    <target>8</target>
                </configuration>
            </plugin>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <appendAssemblyId>false</appendAssemblyId>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                    <archive>
                        <manifest>
                            <!-- 此处指定main方法入口的class -->
                            <!--                            <mainClass>com.cainiao.ts.udf.com.cainiao.ts.udf.RowkeyGenerater</mainClass>-->
                        </manifest>
                    </archive>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>assembly</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>


    </build>



</project>

2、相关代码

package TestKudu;

import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.java.StreamTableEnvironment;
import org.apache.flink.types.Row;

import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

import static org.apache.calcite.sql.fun.SqlStdOperatorTable.JSON_EXISTS;

/**
 * @author 
 * @date 2020/3/19 16:57
 * @explain
 */
public class KafkaSinkTest {
    public static void main(String[] args) throws Exception{
        // 初始化 flink 执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        EnvironmentSettings fsSettings = EnvironmentSettings.newInstance().useOldPlanner().inStreamingMode().build();
//        StreamExecutionEnvironment fsEnv = StreamExecutionEnvironment.getExecutionEnvironment();
        StreamTableEnvironment fsTableEnv = StreamTableEnvironment.create(env, fsSettings);

        // 生成数据源
        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers", ":9092");
// only required for Kafka 0.8
//        properties.setProperty("zookeeper.connect", ":2181");
        properties.setProperty("group.id", "test1");
        FlinkKafkaConsumer<String> consumer = new FlinkKafkaConsumer<>("user_behavior3", new SimpleStringSchema(), properties);
        consumer.setStartFromEarliest();
        DataStream<String> stream = env
                .addSource(consumer);

         fsTableEnv.registerDataStream("table1",stream,"a");
//         fsTableEnv.registerFunction("testlength",new StringLengthUdf());
        fsTableEnv.registerFunction("json_value1",new GetJsonObject());
//        Table query = fsTableEnv.sqlQuery("select testlength(a) from table1");
        Table query = fsTableEnv.sqlQuery("select json_value1(a,'$.id'),json_value1(a,'$.name'),json_value1(a,'$.sex') from table1");


        DataStream<Row> dsRow = fsTableEnv.toAppendStream(query, Row.class);
//        dsRow.print();

        dsRow.addSink(new MysqlSink());

//
        env.execute("sink-test");
    }

}

3、自定义mysqlsink

package TestKudu;

import com.mysql.jdbc.Connection;
import com.mysql.jdbc.PreparedStatement;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.flink.types.Row;

import java.sql.DriverManager;


/**
 * @author 
 * @date 2020/5/14 13:43
 * @explain
 */
public class MysqlSink extends RichSinkFunction<Row> {
    private Connection connection;
    private PreparedStatement preparedStatement;

    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
        String className = "com.mysql.jdbc.Driver";
        Class.forName(className);
        String url = "jdbc:mysql://:3306/database";
        String user = "";
        String password = "";
        connection = (Connection) DriverManager.getConnection(url, user, password);
        String sql = "replace into flinkjsontest(id,name,sex) values(?,?,?)";
        preparedStatement = (PreparedStatement) connection.prepareStatement(sql);
        super.open(parameters);
    }

    @Override
    public void close() throws Exception {
        super.close();
        if (preparedStatement != null) {
            preparedStatement.close();
        }
        if (connection != null) {
            connection.close();
        }
        super.close();
    }

    public void invoke(Row value, Context context) throws Exception {

        String   id = (String)value.getField(0);
        String   name = (String)value.getField(1);
        String   sex = (String)value.getField(2);

        System.out.print(id+":"+name+":"+sex);
        preparedStatement.setString(1, id);
        preparedStatement.setString(2, name);
        preparedStatement.setString(3,sex);
        int i = preparedStatement.executeUpdate();
        if (i > 0) {
            System.out.println("value=" + value);
        }
    }
}

4、提交任务

flink run -c TestKudu.KafkaSinkTest -p 2 flink-1.0-SNAPSHOT.jar -m cdh01:8081 

5、往kafka打入数据

kafka-console-producer --broker-list :9092 --topic user_behavior3