在flink程序流程中属于第二步
从集合或文件中读取
// 从集合中读取
env.fromCollection(Arrays.asList(
"hello java",
"hello spark",
"hello flink",
"hello scala"));
// 从文件中读取
env.readTextFile("<filePath>");
从kafka中读取
添加第三方包
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka-0.11_2.11</artifactId>
<version>1.10.1</version>
</dependency>
连接代码
// 定义链接kafka的对应配置,告诉程序如何连接kafka,即kafka地址,offset的机制等
Properties properties = new Properties();
properties.setProperty("bootstrap.servers","localhost:9092");
properties.setProperty("group_id","consumer-group");
properties.setProperty("key.deserializer","org.apache.kafka.common.serialization.StrinDeserializer");
properties.setProperty("value.deserializer","org.apache.kafka.common.serialization.StrinDeserializer");
properties.setProperty("auto.offset.reset","latest");
DataStreamSource<String> =
env.addSource(new FlinkKafkaConsumer010<String>(
// kafka的topic
"test",
new SimpleStringSchema(),
properties));
自定义source 数据源
package com.flinkTest.source;
import com.flinkTest.POJO.SensorReading;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import java.util.HashMap;
import java.util.Random;
public class ModifyMySource {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource inputStream = env.addSource(new MySource()); // 从自定义数据源读取数据
inputStream.print(); // 没有逻辑处理,直接打印
env.execute();
}
/** 自定义数据源, 数据是 生产1组 10个温度传感器的对象
*
* */
public static class MySource<i> implements SourceFunction<SensorReading> {
private boolean running = true;
@Override
public void run(SourceContext sourceContext) throws Exception {
Random random = new Random();
//生成10个初始sensor温度
// 设置初始温度值(10个)
HashMap<String,Double> tempMap = new HashMap<>();
for (int i =0; i<10 ; i++){
tempMap.put("sensor_" + (i+1), 60+random.nextGaussian()*20);
}
while(running){
for(String id: tempMap.keySet()){
Double newTemp = tempMap.get(id) + random.nextGaussian();
tempMap.put(id,newTemp);
sourceContext.collect(new SensorReading(id,System.currentTimeMillis(),newTemp));
}
Thread.sleep(1000);
}
}
@Override
public void cancel() {
running = false;
}
}
// 温度传感器 内部类
public static class SensorReading {
private String id;
private Long timestamp;
private Double temperature;
public SensorReading() {
}
public SensorReading(String id, Long timestamp, Double temperature) {
this.id = id;
this.timestamp = timestamp;
this.temperature = temperature;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public Long getTimestamp() {
return timestamp;
}
public void setTimestamp(Long timestamp) {
this.timestamp = timestamp;
}
public Double getTemperature() {
return temperature;
}
public void setTemperature(Double temperature) {
this.temperature = temperature;
}
@Override
public String toString() {
return "SensorReading{" +
"id='" + id + '\'' +
", timestamp=" + timestamp +
", temperature=" + temperature +
'}';
}
}
}