所有操作使用shell脚本方式执行
1. 创建指定mapping文件
more ./create/es_reg_user_goods_article_recommend_mapping.json
{
"et_reg_user_goods_article_recommend" : {
"properties" : {
"user_id" : {"type" : "long", "store": "yes"},
"article_ids" : {"type" : "string", "store": "yes", "index": "not_analyzed"},
"create_date" : {"type" : "string", "store": "yes", "index": "not_analyzed"}
}
}
}
2. 在ES中创建索引
# del index
# 删除索引
curl -XDELETE http://192.168.1.1:9200/es_reg_user_article_recommend
# create index
# 创建索引
curl -XPUT http://192.168.1.1:9200/es_reg_user_goods_article_recommend/ -d'
{
"mappings" : {
"_default_" : {
"dynamic" : "strict"
}
}
}'
# create alias
# 创建别名
curl -XPOST 'http://192.168.1.1:9200/_aliases' -d'
{
"actions" : [{ "add" : { "index" : "es_reg_user_goods_article_recommend", "alias" : "reg_user_goods_article_recommend_index"}}]
}'
# modify alias
# 修改别名
curl -XPOST 'http://192.168.1.1:9200/_aliases' -d'
{
"actions" : [
{ "remove" : { "index" : "es_reg_user_article_recommend", "alias" : "reg_user_article_recommend_index" }},
{ "add" : { "index" : "es_reg_user_article_recommend_new", "alias" : "reg_user_article_recommend_index"}}
]
}'
# create mapping
# 创建mapping
curl -XPUT http://192.168.1.1:9200/es_reg_user_goods_article_recommend/et_reg_user_goods_article_recommend/_mapping?pretty -d @/opt/DATA/goldmine/src/mining/user_recommend_mining/goods_article_recommend/create/es_reg_user_goods_article_recommend_mapping.
json
3. 创建 HIVE 表
more ./create/hive_to_es_reg_user_goods_article_recommend.sh
#!/bin/sh
###
# 1. del jar
hadoop fs -rm /user/hive/warehouse/jars/elasticsearch-hadoop-2.4.1.jar
# 2. put jar
hadoop fs -put /usr/local/goldmine/hive/default/dg-lib/elasticsearch-hadoop-2.4.1.jar /user/hive/warehouse/jars/
# 3. create index
sh create_es_reg_user_article_recommend_mapping
###
#--org.apache.commons.httpclient_3.1.0.v201012070820.jar这个jar包一般是不用加的,除非在报httpclient相关错误的时候才加
#add jar hdfs:/opt/data/jar/org.apache.commons.httpclient_3.1.0.v201012070820.jar;\
# 4. create table
# /usr/local/goldmine/hive/default/dg-lib/elasticsearch-hadoop-2.4.1.jar
reg_user_goods_article_recommend_create_table="add jar hdfs:/user/hive/warehouse/jars/elasticsearch-hadoop-2.4.1.jar;\
create external table if not exists des_reg_user_goods_article_recommend(
user_id bigint,
article_ids string,
create_date string
)
STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler'
TBLPROPERTIES(
'es.nodes'='192.168.1.1:9200',
'es.resource' = 'es_reg_user_goods_article_recommend/et_reg_user_goods_article_recommend',
'es.index.auto.create'='false',
'es.mapping.id'='user_id',
'es.mapping.names'='article_ids:article_ids,create_date:create_date');"
# --es.resource斜线前面为索引,斜线后面为mapping名称。mapping可以不用事先建好,当数据插入时会自动根据数据类型创建mapping。
hive -e "${reg_user_goods_article_recommend_create_table}"