flink 写入到es_scala Flink读取kafka 写入es(elasticsearch) 简单实现

scala Flink读取kafka 写入elasticsearch 简单实现

引入pom依赖

es的Mapping

读取kafka 写入es 代码实现

scala 构建kafka生产者

scala 构建kafka消费者

引入pom依赖

org.apache.kafka

kafka_2.11

1.1.0

org.apache.kafka

kafka-clients

1.1.0

org.apache.flink

flink-connector-kafka_2.11

1.9.0

org.apache.flink

flink-connector-elasticsearch6_2.11

1.7.0

org.apache.flink

flink-streaming-scala_2.11

1.7.0

es的Mapping

{

"order": 0,

"index_patterns": [

"xiaojin_*"

],

"settings": {

"index": {

"number_of_shards": "3",

"translog": {

"flush_threshold_size": "1g"

},

"number_of_replicas": "1"

}

},

"mappings": {

"_default_": {

"dynamic_templates": [

{

"string_as_keyword": {

"mapping": {

"type": "keyword"

},

"match_mapping_type": "string",

"match": "*"

}

}

],

"_all": {

"enabled": false

},

"properties": {

"originalMsg": {

"index": false,

"type": "text"

},

"index": {

"index": false,