1、maven依赖
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>flinktest</groupId>
<artifactId>flink</artifactId>
<version>1.0-SNAPSHOT</version>
<!-- <build>-->
<!-- <plugins>-->
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-compiler-plugin</artifactId>-->
<!-- <configuration>-->
<!-- <source>8</source>-->
<!-- <target>8</target>-->
<!-- </configuration>-->
<!-- </plugin>-->
<!-- </plugins>-->
<!--<!– </build>–>-->
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka_2.11</artifactId>
<version>1.9.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>1.9.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.11</artifactId>
<version>1.9.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-java-bridge_2.11</artifactId>
<version>1.9.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
<version>1.7.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.apache.flink</groupId>-->
<!-- <artifactId>flink-connector-kafka_2.11</artifactId>-->
<!-- <version>1.10.0</version>-->
<!-- </dependency>-->
<!-- https://mvnrepository.com/artifact/com.alibaba.otter/canal.client -->
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client</artifactId>
<version>1.1.4</version>
<scope>provided</scope>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.apache.flink</groupId>-->
<!-- <artifactId>flink-table-planner-blink_2.11</artifactId>-->
<!-- <version>1.10.0</version>-->
<!--<!– <scope>provided</scope>–>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.apache.flink</groupId>-->
<!-- <artifactId>flink-table-common</artifactId>-->
<!-- <version>1.10.0</version>-->
<!--<!– <scope>provided</scope>–>-->
<!-- </dependency>-->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-java-bridge_2.11</artifactId>
<version>1.9.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner_2.11</artifactId>
<version>1.9.0</version>
<scope>provided</scope>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.alibaba.blink</groupId>-->
<!-- <artifactId>flink-table_2.11</artifactId>-->
<!-- <version>blink-3.2.2</version>-->
<!-- </dependency>-->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<!-- 此处指定main方法入口的class -->
<!-- <mainClass>com.cainiao.ts.udf.com.cainiao.ts.udf.RowkeyGenerater</mainClass>-->
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>assembly</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
2、相关代码
package TestKudu;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import static org.apache.calcite.sql.fun.SqlStdOperatorTable.JSON_EXISTS;
/**
* @author
* @date 2020/3/19 16:57
* @explain
*/
public class KafkaSinkTest {
public static void main(String[] args) throws Exception{
// 初始化 flink 执行环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
EnvironmentSettings fsSettings = EnvironmentSettings.newInstance().useOldPlanner().inStreamingMode().build();
// StreamExecutionEnvironment fsEnv = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment fsTableEnv = StreamTableEnvironment.create(env, fsSettings);
// 生成数据源
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", ":9092");
// only required for Kafka 0.8
// properties.setProperty("zookeeper.connect", ":2181");
properties.setProperty("group.id", "test1");
FlinkKafkaConsumer<String> consumer = new FlinkKafkaConsumer<>("user_behavior3", new SimpleStringSchema(), properties);
consumer.setStartFromEarliest();
DataStream<String> stream = env
.addSource(consumer);
fsTableEnv.registerDataStream("table1",stream,"a");
// fsTableEnv.registerFunction("testlength",new StringLengthUdf());
fsTableEnv.registerFunction("json_value1",new GetJsonObject());
// Table query = fsTableEnv.sqlQuery("select testlength(a) from table1");
Table query = fsTableEnv.sqlQuery("select json_value1(a,'$.id'),json_value1(a,'$.name'),json_value1(a,'$.sex') from table1");
DataStream<Row> dsRow = fsTableEnv.toAppendStream(query, Row.class);
// dsRow.print();
dsRow.addSink(new MysqlSink());
//
env.execute("sink-test");
}
}
3、自定义mysqlsink
package TestKudu;
import com.mysql.jdbc.Connection;
import com.mysql.jdbc.PreparedStatement;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.flink.types.Row;
import java.sql.DriverManager;
/**
* @author
* @date 2020/5/14 13:43
* @explain
*/
public class MysqlSink extends RichSinkFunction<Row> {
private Connection connection;
private PreparedStatement preparedStatement;
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
String className = "com.mysql.jdbc.Driver";
Class.forName(className);
String url = "jdbc:mysql://:3306/database";
String user = "";
String password = "";
connection = (Connection) DriverManager.getConnection(url, user, password);
String sql = "replace into flinkjsontest(id,name,sex) values(?,?,?)";
preparedStatement = (PreparedStatement) connection.prepareStatement(sql);
super.open(parameters);
}
@Override
public void close() throws Exception {
super.close();
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
super.close();
}
public void invoke(Row value, Context context) throws Exception {
String id = (String)value.getField(0);
String name = (String)value.getField(1);
String sex = (String)value.getField(2);
System.out.print(id+":"+name+":"+sex);
preparedStatement.setString(1, id);
preparedStatement.setString(2, name);
preparedStatement.setString(3,sex);
int i = preparedStatement.executeUpdate();
if (i > 0) {
System.out.println("value=" + value);
}
}
}
4、提交任务
flink run -c TestKudu.KafkaSinkTest -p 2 flink-1.0-SNAPSHOT.jar -m cdh01:8081
5、往kafka打入数据
kafka-console-producer --broker-list :9092 --topic user_behavior3