1、引入依赖
<dependency>
<groupId>com.starrocks</groupId>
<artifactId>flink-connector-starrocks</artifactId>
<version>1.2.3_flink-1.14_2.11</version>
<!-- <scope>provided</scope>-->
</dependency>
2、创建StarRocks表
create table flink.json_test1 (
id bigint,
name string comment '集团名称',
json_tex json comment '集团'
)
primary key (id)
distributed by hash(id)
properties("replication_num" = "3",
"enable_persistent_index" = "true");
3、flink-connector-starrocks读取代码
import com.starrocks.connector.flink.StarRocksSource;
import com.starrocks.connector.flink.table.source.StarRocksSourceOptions;
import lombok.extern.slf4j.Slf4j;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.data.RowData;
import java.util.HashMap;
/**
* StarrocksConnector Reader插件读取StarRocks中json字段
*/
@Slf4j
public class UserStarrocksConnectorReader {
public static void main(String[] args) throws Exception {
StarRocksSourceOptions options = StarRocksSourceOptions.builder()
.withProperty("scan-url", "192.168.1.1:8030")
.withProperty("jdbc-url", "jdbc:mysql://192.168.1.1:9030")
.withProperty("username", "root")
.withProperty("password", "")
.withProperty("table-name", "json_test")
.withProperty("database-name", "flink")
.build();
TableSchema tableSchema = TableSchema.builder()
.field("id", DataTypes.BIGINT())
.field("name", DataTypes.STRING())
.field("json_tex", DataTypes.STRING())
.build();
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource<RowData> dataSource = env.addSource(StarRocksSource.source(tableSchema, options));
dataSource.setParallelism(1);
HashMap<String, String> map = new HashMap<>();
dataSource.map(new MapFunction<RowData, String>() {
@Override
public String map(RowData value) throws Exception {
String json = value.getString(2).toString();
return json ;
}
}).print();
env.execute("StarRocks flink source");
}
}
4、json字段类型获取异常
类:StarRocksSchema.genSchema方法空指针异常?
public static StarRocksSchema genSchema(List<TScanColumnDesc> tscanColumnDescs) {
StarRocksSchema schema = new StarRocksSchema();
tscanColumnDescs.stream().forEach(desc -> schema.put(
new Column(desc.getName(), desc.getType().name(), "", 0, 0))
);
return schema;
}
发现 desc.getType().name()空指针异常,debug发现json字段,获取的类型为null,为了解决这一问题,将该类重写,相同包名和类名,修改为如下:
public static StarRocksSchema genSchema(List<TScanColumnDesc> tscanColumnDescs) {
StarRocksSchema schema = new StarRocksSchema();
tscanColumnDescs.stream().forEach(desc -> schema.put(
new Column(desc.getName(), desc.getType()==null? "VARCHAR": desc.getType().name(), "", 0, 0))
);
return schema;
}
即可获取json字段为字符串类型,然后映射为具体的bean处理即可。
你学会了吗?
更多内容来自 公众号: