简述:
Flink job中的环境配置信息全部从代码里面抽出来放到配置文件中去。 主要讲一下对flink 自带的ParameterTool 的使用细节,以及遇到的问题,如何解决。
有三种方案:
-
使用flink 的 Configuration
-
采用广播变量
-
使用flink 自带的ParameterTool读取外部配置文件
我这里介绍的是第三种方案: 使用flink 自带的ParameterTool读取外部配置文件
Flink 文档地址:https://github.com/apache/flink/blob/master/docs/dev/application_parameters.zh.md
使用过程1:
flink 启动命令: flink-1.9.0/bin/flink run -c com.application.TestApplication -m yarn-cluster
-config_path /home/hadoop/config.properties
注释: -config_path /home/hadoop/config.properties (从外部配置文件读取配置信息)
在Flink代码刚开始启动的时候去读取外部文件配置信息, 逻辑如下:
package com.application
import com.ProjectConfig
import org.apache.flink.api.java.utils.ParameterTool
object TestApplication extends App {
// 开始flink 数据流
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
//启动Flink任务时,需要添加参数config_path 举例: flink run Test.jar -config_path /config.properties
val parameters = ParameterTool.fromArgs(args)
val propertiesFilePath= parameters.get("config_path")
// 初始化配置信息, 形成静态变量
val parameters = ProjectConfig.initConfig(propertiesFilePath)
// 注册全局变量
env.getConfig.setGlobalJobParameters(parameters)
}
设置静态变量
package com.util
import java.util.Properties
import org.apache.flink.api.java.utils.ParameterTool
object ProjectConfig {
//#Hbase配置
var HBASE_ZOOKEEPER_QUORUM = ""
var HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT ="2181"
var KAFKA_QUORUM=""
// topic命名 需要带_topic 后缀
//kafka topic config topic
var KAFKA_CONTEXT_CONFIG_TOPIC: String = "mainfab_context_config_topic"
var KAFKA_WINDOW_CONFIG_TOPIC: String = "mainfab_window_config_topic"
var KAFKA_INDICATOR_CONFIG_TOPIC = "mainfab_indicator_config_topic"
var KAFKA_AUTO_LIMIT_CONFIG_TOPIC = "mainfab_autoLimit_config_topic"
var KAFKA_VIRTUAL_SENSOR_CONFIG_TOPIC = "mainfab_virtual_sensor_config_topic"
var KAFKA_ALARM_CONFIG_TOPIC = "mainfab_alarm_config_topic"
/**
* 初始化
*/
def initConfig(PROPERTIES_FILE_PATH: String): ParameterTool = {
var configname: ParameterTool = null
try {
configname = ParameterTool.fromPropertiesFile(PROPERTIES_FILE_PATH)
Config(configname)
} catch {
case e: Exception =>
e.printStackTrace()
}
configname
}
/**
* 获取变量
*/
def getConfig(configname: ParameterTool): ParameterTool = {
Config(configname)
configname
}
def Config(configname: ParameterTool): ParameterTool = {
/**
* 配置文件 配置名 全小写 用 . 分割
*/
ROCKSDB_HDFS_PATH = configname.get("rocksdb.hdfs.path")
try{
RUN_ALARM_TIME_OUT = configname.get("run.alarm.time.out")
OFFLINE_TASK_TIME_OUT = configname.get("offline.task.time.out")
}catch {
case e: Exception => None
}
JOB_VERSION = configname.get("job.version")
RUN_MAX_LENGTH=configname.getInt("job.run.max.length")
RAW_DATA_WRITE_BATCH_COUNT=configname.getInt("raw.data.write.batch.count")
RUN_DATA_MAX_TIME = configname.getInt("job.run.data.max.time")
//Oracle
MAIN_FAB_CORE_ORACLE_URL = configname.get("main.fab.core.oracle.url")
MAIN_FAB_CORE_ORACLE_USER = configname.get("main.fab.core.oracle.user")
MAIN_FAB_CORE_ORACLE_PASSWORD = configname.get("main.fab.core.oracle.password")
//Hbase
HBASE_ZOOKEEPER_QUORUM = configname.get("hbase.zookeeper.quorum")
HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT = configname.get("hbase.zookeeper.property.clientport")
HBASE_MAINFAB_RUNDATA_TABLE = configname.get("hbase.rundata.table")
HBASE_INDICATOR_DATA_TABLE = configname.get("hbase.indicator.data.table")
HBASE_OFFLINE_INDICATOR_DATA_TABLE = configname.get("hbase.offline.indicator.data.table")
}
/home/hadoop/config.properties文件样例
########rocksdb
rocksdb.hdfs.path=hdfs://nameservice1/flink-checkpoints
########hdfs
hdfs.dir=hdfs://nameservice1/data/base_alarm_report/
hdfs.parallelism=1
rawtrace.file.path=/rawtrace
indicator.file.path=/indicator
fs.url=hdfs://nameservice1:8020
########Oracle
main.fab.core.oracle.url=jdbc:oracle:thin:@192.168.0.98:1522:ORCLCDB
main.fab.core.oracle.user=mainfabcoretest
main.fab.core.oracle.password=mainfabcoretest
########kafka地址
kafka.zookeeper.connect=fdc12:9092,fdc13:9092,fdc14:9092
########opentsdb rawData地址
opentsdb.url=http://fdc11
opentsdb.port=4242
opentsdb.rawdata.url=http://fdc11:4242/api/query
########支持的最大run的长度,类型Int,单位:小时
job.run.max.length=20
offline.task.time.out=10000
raw.data.write.batch.count=200
####runData超时时间,单位:小时
job.run.data.max.time=5
########分流topic A流
job.version=0
kafka.topic.update=mainfab_update_topic
##
kafka.topic.data.window=mainfab_A_window_data_topic
kafka.topic.indicator.result=mainfab_A_indicator_topic
kafka.topic.drift.indicator=mainfab_A_drift_indicator_topic
kafka.topic.calculated.indicator=mainfab_A_calculated_indicator_topic
kafka.topic.logistic.indicator=mainfab_A_logistic_indicator_topic
在on yarn模式下跑flink job, 但是出现了一个问题。
flink job中存在两个RichSourceFunction,一个是flink自带的从kafka获取数据流, 另一个是自定义的从hbase获取维表数据。
在on yarn运行的时候失败了, 原因: flink从kafka获取数据流是正常运行了, 但是从hbase获取维表的RichSourceFunction,显示不能获取配置信息。
使用过程2:
使用ParameterTool 的注册全局变量
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
// 注册全局变量
env.getConfig.setGlobalJobParameters(parameters)
hbase获取维表的RichSourceFunction的open函数
override def open(parameters: Configuration): Unit = {
// 获取全局变量
val parameters = getRuntimeContext.getExecutionConfig.getGlobalJobParameters.asInstanceOf[ParameterTool]
ProjectConfig.initConfig(parameters)
val conf = HBaseConfiguration.create()
conf.set("hbase.zookeeper.quorum", ProjectConfig.HBASE_ZOOKEEPER_QUORUM)
conf.set("hbase.zookeeper.property.clientPort", ProjectConfig.HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT)
connection = ConnectionFactory.createConnection(conf)
}
flink 在on yarn 上运行成功
参考:http://ytluck.github.io/big-data/my-bigdata-post-59.html
https://blog.csdn.net/smileyan9/article/details/99587572
https://www.cnblogs.com/tonglin0325/p/14115069.html