Flink示例——Source

Flink示例——Source

版本信息

产品	版本
Flink	1.7.2
Java	1.8.0_231
Scala	2.11.12

Mavan依赖

• pom.xml 依赖部分

  <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-java</artifactId>
      <version>${flink.version}</version>
  </dependency>
  <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-streaming-java_2.11</artifactId>
      <version>${flink.version}</version>
  </dependency>
  <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-clients_2.11</artifactId>
      <version>${flink.version}</version>
  </dependency>
  

Collection/Elements Source 示例

• 代码 CollectionElementsSourceDemo

  public class CollectionElementsSourceDemo {
  
      public static void main(String[] args) {
          StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  
          ArrayList<String> list = new ArrayList<>(
                  Arrays.asList("name1", "name2", "name3")
          );
  
          env.fromCollection(list)
                  .print("stream");
  
  //        env.fromElements("name", 18, 20.1);
  
          try {
              env.execute();
          } catch (Exception e) {
              e.printStackTrace();
          }
      }
  
  }
  

Text Source 示例

• 代码 TextSourceDemo

  public class TextSourceDemo {
  
      public static void main(String[] args) {
          StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  
          DataStreamSource<String> dataDS = env.readTextFile("C:\\Users\\Skey\\Projects\\Java\\flink-demo\\src\\main\\resources\\README.txt");
  
          dataDS.print("stream");
  
          try {
              env.execute();
          } catch (Exception e) {
              e.printStackTrace();
          }
      }
  
  }
  

Socket Source 示例

• 测试建议
  • 下载 netcat(windows)
  • 启动命令 nc.exe -L -v -p 23333(windows)
  • 启动Flink应用
  • 在启动的nc程序中发送字符串
• 代码 SocketSourceDemo

  public class SocketSourceDemo {
  
      public static void main(String[] args) {
          StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  
          DataStreamSource<String> dataDS = env.socketTextStream("localhost", 23333);
  
          dataDS.print("stream");
  
          try {
              env.execute();
          } catch (Exception e) {
              e.printStackTrace();
          }
      }
  
  }
  

Kafka Source 示例

• Maven导包 pom.xml

  <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-kafka-0.11_2.11</artifactId>
      <version>${flink.version}</version>
  </dependency>
  

• 代码 KafkaSourceDemo

  public class KafkaSourceDemo {
  
      public static void main(String[] args) {
          StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  
          // 创建FlinkKafkaConsumer
          FlinkKafkaConsumer011<String> kafkaConsumer011 = generateKafkaConsumer();
  
          // 创建KafkaSource
          DataStreamSource<String> kafkaDS = env.addSource(kafkaConsumer011);
  
          // 处理
          kafkaDS.print();
  
          try {
              env.execute();
          } catch (Exception e) {
              e.printStackTrace();
          }
      }
  
      /**
       * 生成FlinkKafkaConsumer
       */
      private static FlinkKafkaConsumer011<String> generateKafkaConsumer() {
          // Kafka配置
          Properties props = new Properties();
          props.setProperty("bootstrap.servers", "192.168.0.101:9092,192.168.0.102:9092,192.168.0.103:9092");
          props.setProperty("group.id", "consumer-group-1");
          props.setProperty("auto.offset", "latest");
  
          // 创建FlinkKafkaConsumer
          FlinkKafkaConsumer011<String> kafkaConsumer011 = new FlinkKafkaConsumer011<>(
                  // SimpleStringSchema 默认使用UTF-8解码
                  "topic_1", new SimpleStringSchema(), props
          );
  
          // 自定义反序列化 - 示例
          /*
          KeyedDeserializationSchema<String> keyedDeserializationSchema = new KeyedDeserializationSchema<String>() {
              @Override
              public String deserialize(byte[] messageKey, byte[] message, String topic, int partition, long offset) throws IOException {
                  // 自定义解码byte[]
                  return new String(message, StandardCharsets.UTF_8);
              }
  
              @Override
              public boolean isEndOfStream(String nextElement) {
                  return false;
              }
  
              @Override
              public TypeInformation<String> getProducedType() {
                  return BasicTypeInfo.STRING_TYPE_INFO;
              }
          };
          */
  
          return kafkaConsumer011;
      }
  
  }
  

自定义 Source 示例

• 代码 CustomSourceFunction

  public class CustomSourceFunction extends RichSourceFunction<Tuple2<String, Long>> {
  
      private boolean flag = true;
  
      @Override
      public void run(SourceContext<Tuple2<String, Long>> ctx) throws Exception {
          List<String> data = Arrays.asList("a", "b", "c", "d", "e", "f", "g");
          Random random = new Random();
          while (flag) {
              Thread.sleep(100);
              // 随机取一个值
              String key = data.get(random.nextInt(data.size()));
              long value = System.currentTimeMillis();
              ctx.collect(Tuple2.of(key, value));
          }
      }
  
      @Override
      public void cancel() {
          flag = false;
      }
  
  }
  

• 代码 CustomSourceDemo

  public class CustomSourceDemo {
  
      public static void main(String[] args) {
          // 创建env
          StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  
          // 自定义SourceFunction
          CustomSourceFunction sourceFunction = new CustomSourceFunction();
  
          // 创建数据源
          DataStreamSource<Tuple2<String, Long>> customDS = env.addSource(sourceFunction);
          // 开始处理
          customDS.print();
  
          try {
              env.execute();
          } catch (Exception e) {
              e.printStackTrace();
          }
  
      }
  
  }