Hive整合ElasticSearch

Hive整合ElasticSearch

作用：直接将Hive数据入到Es中

1. 创建Index

PUT bank_2 
{
  "settings": {
    "number_of_shards": 5
    , "number_of_replicas": 1
  }
}

2. 创建Type的Mapping

PUT bank_2/_doc/_mapping
{
  "properties": {
    "fid":{
      "type": "keyword"
    },
    "tid":{
      "type": "keyword"
    },
    "counts":{
      "type": "long"
    },
    "sums":{
      "type": "double"
    },
    "years":{
      "type": "integer"
    }
  }
}

3. 下载es-hive相关Jar包,入到HDFS中

wget https://artifacts.elastic.co/downloads/elasticsearch-hadoop/elasticsearch-hadoop-6.6.0.zip

4. Hive添加相关JAR包

hive > add jar hdfs://gl0:9000/jars/elasticsearch-hadoop-hive-6.5.1.jar;
hive > add jar hdfs://gl0:9000/jars/commons-httpclient-3.1.jar;

5. 创建ES外部表

drop table yin_hang_ce_shi.elastic_bank_2;
CREATE EXTERNAL TABLE yin_hang_ce_shi.elastic_bank_2(
   idcard string,
   fid string,
   tid string,
   counts bigint,
   sums double,
   years int
)
STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler'
TBLPROPERTIES('es.resource'='${index}',
'es.nodes'='${ips}',
'es.port'='9200',
'es.index.auto.create'='TRUE',
'es.mapping.id' = '${索引ID字段}'
);

6. 添加数据

insert into yin_hang_ce_shi.elastic_bank_2 (select * from yin_hang_ce_shi.table);