02_Flink Streaming SourceFunction

最新推荐文章于 2024-09-02 09:34:23 发布

codemosi

最新推荐文章于 2024-09-02 09:34:23 发布

阅读量3.8k

点赞数

分类专栏： flink streaming

本文链接：https://blog.csdn.net/codemosi/article/details/51576756

版权

本文深入探讨了Flink Streaming中如何通过env对象的addSource方法添加数据源，重点解析SourceFunction的作用和使用，阐述其作为数据接入接口的关键特性。

摘要由CSDN通过智能技术生成

env对象的addSource（SourceFunction）。需要传入一个SourceFunction对象。这个对象作为接入数据源的接口

package com.alibaba.flink.train.streaming;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;

public class MemSource implements SourceFunction<String> {

	/**
	 * 产生数据
	 */
	@Override
	public void run(SourceContext<String> sourceContext) throws Exception {
		while (true) {
			sourceContext.collect("flink spark storm");
		}
	}

	/**
	 * 关闭资源
	 */
	@Override
	public void cancel() {
	}

}

class RSource extends RichSourceFunction<String> {
	@Override
	public void open(Configuration parameters) throws Exception {
		super.open(parameters);
	}

	@Override
	public void run(SourceFunction.SourceContext<String> ctx) throws Exception {

	}

	@Override
	public void cancel() {

	}

	@Override
	public void close() throws Exception {
		super.close();
	}

}

package com.alibaba.flink.train.streaming;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

public class HelloWorld {
	public static void main(String[] args) throws Exception {
		StreamExecutionEnvironment env = StreamExecutionEnvironment
				.getExecutionEnvironment();
		// env.setParallelism(4);//并发度
		DataStream<String> dataStream = env
				.readTextFile("D:/flinkdata/helloworld"); // 1:(flink storm
															// )(hadoop hive)
		dataStream = env.addSource(new MemSource());
		dataStream
				.flatMap(
						new FlatMapFunction<String, Tuple2<String, Integer>>() {
							@Override
							public void flatMap(String input,
									Collector<Tuple2<String, Integer>> collector)
									throws Exception {
								String[] objs = input.split(" ");
								for (String obj : objs) {
									collector
											.collect(new Tuple2<String, Integer>(
													obj, 1));// (这里很关键，表示0位置是word，1的位置是1次数)
								}
							}
						})// 2:(flink 1)(storm 1)
				.keyBy(0)// 3:以第0个位置的值，做分区。
				.sum(1)// (flink:8)(storm:5)，对第1个位置的值做sum的操作。
				.printToErr();
		env.execute();// 启动任务
		while (true) {

		}
	}

}

需要关注SourceFunction

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not