11、es---java api

一、client集群自动探查

1、默认情况下，是根据我们手动指定的所有节点，依次轮询这些节点，来发送各种请求的，如下面的代码，我们可以手动为client指定多个节点

Settings settings = Settings.builder()
                .put("cluster.name", "elasticsearch")
                .put("client.transport.sniff", true)---->开启自动探查功能
                .build();
TransportClient client = new PreBuiltTransportClient(settings)
                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost1"), 9300))
                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost2"), 9300))
                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost3"), 9300));

2、自动探查（sniff）

es client提供了一种集群节点自动探查的功能，打开这个自动探查机制以后，es client会根据我们手动指定的几个节点连接过去，然后通过集群状态自动获取当前集群中的所有data node，然后用这份完整的列表更新自己内部要发送请求的node list。默认每隔5秒钟，就会更新一次node list。

 

二、upsert

IndexRequest indexRequest = new IndexRequest("car_shop", "cars", "1")
        .source(jsonBuilder()
            .startObject()
                .field("brand", "宝马")
                .field("name", "宝马320")
                .field("price", 320000)
                .field("produce_date", "2017-01-01")
            .endObject());
UpdateRequest updateRequest = new UpdateRequest("car_shop", "cars", "1")
        .doc(jsonBuilder()
            .startObject()
                .field("price", 320000)
            .endObject())
        .upsert(indexRequest);                 
client.update(updateRequest).get();

 

三、mget

PUT /car_shop/cars/2
{
    "brand": "奔驰",
    "name": "奔驰C200",
    "price": 350000,
    "produce_date": "2017-01-05"
}

MultiGetResponse multiGetItemResponses = client.prepareMultiGet()
    .add("car_shop", "cars", "1")           
    .add("car_shop", "cars", "2")        
    .get();

for (MultiGetItemResponse itemResponse : multiGetItemResponses) { 
    GetResponse response = itemResponse.getResponse();
    if (response.isExists()) {                      
        String json = response.getSourceAsString(); 
    }
}

 

四、bulk

PUT /car_shop/sales/1
{
    "brand": "宝马",
    "name": "宝马320",
    "price": 320000,
    "produce_date": "2017-01-01",
    "sale_price": 300000,
    "sale_date": "2017-01-21"
}

PUT /car_shop/sales/2
{
    "brand": "宝马",
    "name": "宝马320",
    "price": 320000,
    "produce_date": "2017-01-01",
    "sale_price": 300000,
    "sale_date": "2017-01-21"
}

BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
        
IndexRequestBuilder indexRequestBuilder = client.prepareIndex("car_shop", "sales", "3") 
        .setSource(XContentFactory.jsonBuilder()
                    .startObject()
                        .field("brand", "奔驰")
                        .field("name", "奔驰C200")
                        .field("price", 350000)
                        .field("produce_date", "2017-01-20")
                        .field("sale_price", 320000)
                        .field("sale_date", "2017-01-25")
                    .endObject());
bulkRequestBuilder.add(indexRequestBuilder);

UpdateRequestBuilder updateRequestBuilder = client.prepareUpdate("car_shop", "sales", "1")
        .setDoc(XContentFactory.jsonBuilder()
                .startObject()
                    .field("sale_price", 290000)
                .endObject());
bulkRequestBuilder.add(updateRequestBuilder);

DeleteRequestBuilder deleteReqeustBuilder = client.prepareDelete("car_shop", "sales", "2"); 
bulkRequestBuilder.add(deleteReqeustBuilder);

BulkResponse bulkResponse = bulkRequestBuilder.get();

for(BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
    System.out.println("version: " + bulkItemResponse.getVersion()); 
}

 

五、批量查询：scroll

TransportClient client = new PreBuiltTransportClient(settings)
        .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300)); 

SearchResponse searchResponse = client.prepareSearch("car_shop") 
        .setTypes("sales")
        .setQuery(QueryBuilders.termQuery("brand.keyword", "宝马"))
        .setScroll(new TimeValue(60000))
        .setSize(1)
        .get();

int batchCount = 0;

do {
    for(SearchHit searchHit : searchResponse.getHits().getHits()) {
        System.out.println("batch: " + ++batchCount); 
        System.out.println(searchHit.getSourceAsString());  
        
        // 每次查询一批数据，比如1000行，然后写入本地的一个excel文件中
        // 如果说你一下子查询几十万条数据，不现实，jvm内存可能都会爆掉
    }
    
    searchResponse = client.prepareSearchScroll(searchResponse.getScrollId())
            .setScroll(new TimeValue(60000))
            .execute()
            .actionGet();
} while(searchResponse.getHits().getHits().length != 0);

 

六、利用模板搜索

1、在es/config/scripts下建立page_query_by_brand.mustache文件，内容为：

{
  "from": {{from}},
  "size": {{size}},
  "query": {
    "match": {
      "brand.keyword": "{{brand}}" 
    }
  }
}

Map<String, Object> scriptParams = new HashMap<String, Object>();
scriptParams.put("from", 0);
scriptParams.put("size", 1);
scriptParams.put("brand", "宝马");

SearchResponse searchResponse = new SearchTemplateRequestBuilder(client)
        .setScript("page_query_by_brand")
        .setScriptType(ScriptType.FILE)
        .setScriptParams(scriptParams)
        .setRequest(new SearchRequest("car_shop").types("sales"))
        .get()
        .getResponse();

for(SearchHit searchHit : searchResponse.getHits().getHits()) {
    System.out.println(searchHit.getSourceAsString());  
}

 

七、各种query

SearchResponse response = client.prepareSearch("car_shop")
        .setTypes("cars")
        .setQuery(QueryBuilders.matchQuery("brand", "宝马"))                
        .get();

SearchResponse response = client.prepareSearch("car_shop")
        .setTypes("cars")
        .setQuery(QueryBuilders.multiMatchQuery("宝马", "brand", "name"))                
        .get();

SearchResponse response = client.prepareSearch("car_shop")
        .setTypes("cars")
        .setQuery(QueryBuilders.commonTermsQuery("name", "宝马320"))                
        .get();

SearchResponse response = client.prepareSearch("car_shop")
        .setTypes("cars")
        .setQuery(QueryBuilders.prefixQuery("name", "宝"))                
        .get();

for(SearchHit searchHit : searchResponse.getHits().getHits()) {
    System.out.println(searchHit.getSourceAsString());  
}

 

八、多条件查询：bool

QueryBuilder queryBuilder = QueryBuilders.boolQuery()
        .must(QueryBuilders.matchQuery("brand", "宝马"))
        .mustNot(QueryBuilders.termQuery("name.raw", "宝马318"))
        .should(QueryBuilders.rangeQuery("produce_date").gte("2017-01-01").lte("2017-01-31"))
        .filter(QueryBuilders.rangeQuery("price").gte(280000).lte(350000));    

SearchResponse searchResponse = client.prepareSearch("car_shop")  
        .setTypes("cars")
        .setQuery(queryBuilder)
        .get();

for(SearchHit searchHit : searchResponse.getHits().getHits()) {
    System.out.println(searchHit.getSourceAsString());  
}

 

九、基于地理位置搜索：geo_point

1、加依赖

<dependency>
    <groupId>org.locationtech.spatial4j</groupId>
    <artifactId>spatial4j</artifactId>
    <version>0.6</version>                        
</dependency>

<dependency>
    <groupId>com.vividsolutions</groupId>
    <artifactId>jts</artifactId>
    <version>1.13</version>                         
    <exclusions>
        <exclusion>
            <groupId>xerces</groupId>
            <artifactId>xercesImpl</artifactId>
        </exclusion>
    </exclusions>
</dependency>

2、添加数据

POST /car_shop/_mapping/shops
{
  "properties": {
      "pin": {
          "properties": {
              "location": {
                  "type": "geo_point"
              }
          }
      }
  }
}

PUT /car_shop/shops/1
{
    "name": "上海至全宝马4S店",
    "pin" : {
        "location" : {
            "lat" : 40.12,
            "lon" : -71.34
        }
    }
}

3、查询

SearchResponse searchResponse = client.prepareSearch("car_shop")
        .setTypes("shops")
        .setQuery(QueryBuilders.geoBoundingBoxQuery("pin.location")
                        .setCorners(40.73, -74.1, 40.01, -71.12))
        .get();

for(SearchHit searchHit : searchResponse.getHits().getHits()) {
    System.out.println(searchHit.getSourceAsString());  
}

System.out.println("=============第一个需求：搜索两个坐标点组成的一个区域========================");

List<GeoPoint> points = new ArrayList<GeoPoint>();             
points.add(new GeoPoint(40.73, -74.1));
points.add(new GeoPoint(40.01, -71.12));
points.add(new GeoPoint(50.56, -90.58));

searchResponse = client.prepareSearch("car_shop")
        .setTypes("shops")
        .setQuery(QueryBuilders.geoPolygonQuery("pin.location", points))  
        .get();

for(SearchHit searchHit : searchResponse.getHits().getHits()) {
    System.out.println(searchHit.getSourceAsString());  
}

System.out.println("==第二个需求：指定一个区域，由三个坐标点，组成，比如上海大厦，东方明珠塔，上海火车站====");

searchResponse = client.prepareSearch("car_shop")
        .setTypes("shops")
        .setQuery(QueryBuilders.geoDistanceQuery("pin.location")
                .point(40, -70)
                .distance(200, DistanceUnit.KILOMETERS))  
        .get();

for(SearchHit searchHit : searchResponse.getHits().getHits()) {
    System.out.println(searchHit.getSourceAsString());  
}
System.out.println("==第三个需求：搜索距离当前位置在200公里内的4s店====");

 

十、es通过BulkProcessor批量操作

package com.share.es;

import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.InetAddress;

public class EsBulkProcessorClient {
    private BulkProcessor bulkProcessor;
    private final static Logger LOGGER = LoggerFactory.getLogger(EsBulkProcessorClient.class);

    public EsBulkProcessorClient(String esClusterName, String[] address){
        try{
            prepareBulkProcessor(buildClient(esClusterName,address));
        }catch (Exception e){
            LOGGER.error(e.toString());
        }

    }

    private Client buildClient(String esClusterName, String[] address) throws Exception{
        if(esClusterName == null || esClusterName.isEmpty() || address == null || address.length==0 ){
            throw new IllegalArgumentException("Illegal cluster name or the corresponding ip address.");
        }
        TransportClient client = null;
        try {
            Settings settings = Settings.settingsBuilder()
                    .put("cluster.name", esClusterName)
                    .put("client.transport.sniff", false)
                    .build();

            client = TransportClient.builder().settings(settings).build();
            for (String addr : address) {
                String [] str = addr.split(":");
                if(str.length==2){
                    client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(str[0]), Integer.valueOf(str[1])));
                }

            }
        }catch (Exception e){
            throw new Exception(e);
        }

        return client;
    }

    public void bulkCreate(IndexRequest request){
        bulkProcessor.add(request);
    }

    private void prepareBulkProcessor(Client client) throws Exception {
        bulkProcessor = BulkProcessor.builder(
                client,
                new BulkProcessor.Listener() {
                    @Override
                    public void beforeBulk(long executionId,
                                           BulkRequest request) {

                    }

                    @Override
                    public void afterBulk(long executionId,
                                          BulkRequest request,
                                          BulkResponse response) {
                        if(response.hasFailures()){
                            LOGGER.error(response.buildFailureMessage());
                        }
                    }

                    @Override
                    public void afterBulk(long executionId,
                                          BulkRequest request,
                                          Throwable failure) {
                        if(failure!=null){
                            LOGGER.error(failure.getMessage());
                        }

                    }
                })
//                .setBulkActions(100)
                .setBulkSize(new ByteSizeValue(10, ByteSizeUnit.MB))
                .setFlushInterval(TimeValue.timeValueSeconds(5))
                .setConcurrentRequests(1)
                .build();
    }

    public static void main(String[] args) {
        String[] servers = {"ip1", "ip2"};
        String esClusterName = "esClusterName";
        EsBulkProcessorClient client = new EsBulkProcessorClient(esClusterName,servers);
        client.bulkCreate(new IndexRequest("index", "type").routing(String.valueOf("data-id")).source("data-content"));

    }
}