**
Flink底层API写入clickhouse——如果遇到clickhouse中有map类型
**
首先在clickhouse中建表语句如下:
//在clickhouse中 Map 类型在当前的版本中是默认禁用的,所以首先需要开启它:
set allow_experimental_map_type = 1;
CREATE TABLE reclicks ( test_value Map(String, String)) ENGINE = MergeTree ORDER BY test_value
flink底层API实现
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
public class IntoClickhouse2 {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env
.addSource(new Example1.ClickSource())
.addSink(new RichSinkFunction<Example1.Event>() {
private Connection conn;
private PreparedStatement insertStmt;
private PreparedStatement updateStmt;
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
conn = DriverManager.getConnection(
"jdbc:clickhouse://hadoop102:8123/test"//jdbc:clickhouse://IP地址:端口号/数据库名字
);
insertStmt = conn.prepareStatement("INSERT INTO reclicks (test_value) VALUES (?)");
}
// 来一条数据触发调用一次
@Override
public void invoke(Example1.Event value, Context context) throws Exception {
//clickhouse里的数据类型是map,如此处理
insertStmt.setString(1,"{ '"+value.user+"':'"+value.url+"'}");
insertStmt.execute();
}
@Override
public void close() throws Exception {
super.close();
insertStmt.close();
conn.close();
}
});
env.execute();
}
}
用到的自定义数据源
```java
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import java.sql.Timestamp;
import java.util.Calendar;
import java.util.Random;
// 自定义数据源
public class Example1 {
public static void main(String[] args) throws Exception{
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.addSource(new ClickSource()).print();
env.execute();
}
public static class Event{
public String user;
public String url;
public Long timestamp;
public Event() {
}
public Event(String user, String url, Long timestamp) {
this.user = user;
this.url = url;
this.timestamp = timestamp;
}
@Override
public String toString() {
return "Event{" +
"user='" + user + '\'' +
", url='" + url + '\'' +
", timestamp=" + new Timestamp(timestamp) +
'}';
}
}
// 自定义数据源
// SourceFunction<T>, T是数据流中的元素类型
// SourceFunction只能用来产生并行度为1的数据源
// ParallelSourceFunction可以产生并行数据源
public static class ClickSource implements SourceFunction<Event> {
private boolean running=true;
private Random random=new Random();
private String[] userArr={"Mary", "Bob", "Alice", "Liz"};
private String[] urlArr={"./home", "./cart", "./fav", "./prod?id=1", "./prod?id=2"};
// 任务开始时触发run的调用
@Override
public void run(SourceContext<Event> sourceContext) throws Exception {
while (running){
String user=userArr[random.nextInt(userArr.length)];
String url=urlArr[random.nextInt(urlArr.length)];
long timestamp = Calendar.getInstance().getTimeInMillis();// 毫秒时间戳(机器时间)
Event event=new Event(user,url,timestamp);
// 使用collect方法发射数据
sourceContext.collect(event);
Thread.sleep(100L);
}
}
// 在取消任务时触发调用cancel,例如在web ui点击任务的cancel按钮
@Override
public void cancel() {
running=false;
}
}
}