ElasticSearch--java操作api

1.解决虚拟机占满宿主机磁盘的问题

     本篇博客是在ES集群搭建的基础上而来

    首先解决一个问题，就是虚拟机把宿主机磁盘占满的问题，导致虚拟机无法正常运行，首先楼主的虚拟机安装在了D盘，每台虚拟机分配20G的磁盘空间，结果跑着跑着随机虚拟机占用磁盘的增大，沾满了D盘，导致无法运行虚拟机，

   下面将解决办法：现在需要对其他盘的空闲空间合并一部分到D盘

    首先打开计算机管理：

    

 右击C盘，点击压缩卷，输入需要分出来的磁盘的大小

 

压缩成功，成功分配了空闲的磁盘出来

接下来下载安装分区助手

就是这个玩意儿，也可以看到分区的情况，右击合并分区选择合并到D盘再提交

ok,这样就为D盘扩容了

然后在此路径中找到.vmx文件，在文件中查找（Ctrl+F快速查找）vmci0.present，此时会看到“vmci0.present = "TRUE"”

修改为FALSE即可。

如果还是有问题，可以执行下面的操作：

删除以.lck为后缀名的文件，然后重新打开虚拟机

2.ES的java api操作

    新建maven工程，添加依赖

<properties>
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
        <encoding>UTF-8</encoding>
    </properties>

    <dependencies>

        <!-- es的客户端-->
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>transport</artifactId>
            <version>5.4.3</version>
        </dependency>

        <!-- 依赖2.x的log4j -->
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-api</artifactId>
            <version>2.8.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.8.2</version>
        </dependency>
        <!-- 单元测试 -->
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>

    </dependencies>

  log4j2

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN">
    <Appenders>
        <Console name="Console" target="SYSTEM_OUT">
            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
        </Console>
    </Appenders>
    <Loggers>
        <Root level="error">
            <AppenderRef ref="Console"/>
        </Root>
    </Loggers>
</Configuration>

  第一个简单操作：

package com.wx.es1;

import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.junit.Test;
import java.net.InetAddress;
public class HelloEs {
    /*
      1.连接es集群，通过id查询数据
     */
    @Test
    public void helloES()
    {
        TransportClient client=null;
        try {
            //1.设置集群的配置信息
            Settings settings=Settings.builder()
                    .put("cluster.name","my-es")
                    .build();
            //2.连接集群,创建客户端
            client=new PreBuiltTransportClient(settings).addTransportAddresses(
                    new InetSocketTransportAddress(InetAddress.getByName("192.168.203.128"),9300),
                    new InetSocketTransportAddress(InetAddress.getByName("192.168.203.129"),9300),
                    new InetSocketTransportAddress(InetAddress.getByName("192.168.203.130"),9300));
            //3.使用客户端操作写操作语句，操作集群 .actionGet()方法是同步的，没有返回就等待
            GetResponse response = client.prepareGet("news", "fulltext", "2").execute().actionGet();
            //4.打印执行结果
            System.out.print(response);
        }
        catch (Exception e)
        {
            e.printStackTrace();
        }
        finally {
            client.close();
        }
    }
}

 CRUD操作：

package com.wx.es1;

import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetItemResponse;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.DeleteByQueryAction;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.InetAddress;
import java.util.Date;
import java.util.Iterator;
import java.util.Map;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;

public class CRUDEs {
    private TransportClient client=null;
    @Before
    public void init()
    {
      try {
          Settings settings=Settings.builder()
                  .put("cluster.name","my-es")
                  .build();
          client=new PreBuiltTransportClient(settings).addTransportAddresses(
                  new InetSocketTransportAddress(InetAddress.getByName("192.168.203.128"),9300),
                  new InetSocketTransportAddress(InetAddress.getByName("192.168.203.129"),9300),
                  new InetSocketTransportAddress(InetAddress.getByName("192.168.203.130"),9300)
          );
      }catch (Exception e)
      {
          e.printStackTrace();
      }
    }
     /*
       关闭客户端
     */
    @After
    public void closeClient()
    {
        if (client!=null)
        {
            client.close();
        }
    }
    //插入一条数据
    @Test
    public void createIndex () throws Exception
    {
        IndexResponse response = client.prepareIndex("gamelog", "users", "2")
                .setSource(
                     jsonBuilder()
                                .startObject()
                                .field("username", "李四")
                                .field("gender", "male")
                                .field("birthday", new Date())
                                .field("fv", 9999)
                                .field("message", "trying out Elasticsearch")
                                .endObject()
                ).get();
    }
    /*
    查询一条数据
     */
    @Test
    public void getOne()
    {
        GetResponse response = client.prepareGet("gamelog", "users", "1").get();
        System.out.print(response);
    }
    /*
    查找多条数据
    */
    @Test
    public void getMulti() throws IOException
    {
        MultiGetResponse multiGetItemResponses = client.prepareMultiGet()
                .add("gamelog", "users", "1")
                .add("news", "fulltext", "2","3")
                .get();
        for(MultiGetItemResponse  responses: multiGetItemResponses) {
            GetResponse response = responses.getResponse();
            System.out.print(response);

        }
    }
    /*
    修改
     */
    @Test
    public void testUpdate() throws Exception {
        UpdateRequest updateRequest = new UpdateRequest();
        updateRequest.index("gamelog");
        updateRequest.type("users");
        updateRequest.id("1");
        updateRequest.doc(
                jsonBuilder()
                        .startObject()
                        .field("fv", 999.9)
                        .endObject());
        client.update(updateRequest).get();
    }
    /*
    删除
     */
    @Test
    public void testDelete() {
        DeleteResponse response = client.prepareDelete("gamelog", "users", "2").get();
        System.out.println(response);
    }
    /*
    指定条件删除
     */
    @Test
    public void testDeleteByQuery() {
        BulkByScrollResponse response =
                DeleteByQueryAction.INSTANCE.newRequestBuilder(client)
                        //指定查询条件
                        .filter(QueryBuilders.matchQuery("username", "李四"))
                        //指定索引名称
                        .source("gamelog")
                        .get();
        long deleted = response.getDeleted();
        System.out.println(deleted);
    }
    /*
    异步删除
     */
    //异步删除
    @Test
    public void testDeleteByQueryAsync() {
        DeleteByQueryAction.INSTANCE.newRequestBuilder(client)
                .filter(QueryBuilders.matchQuery("gender", "male"))
                .source("gamelog")
                .execute(new ActionListener<BulkByScrollResponse>() {
                    @Override
                    public void onResponse(BulkByScrollResponse response) {
                        long deleted = response.getDeleted();
                        System.out.println("数据删除了");
                        System.out.println(deleted);
                    }
                    @Override
                    public void onFailure(Exception e) {
                        e.printStackTrace();
                    }
                });
        try {
            System.out.println("异步删除");
            Thread.sleep(10000);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    /*
    范围内查询
     */
    @Test
    public void testRange() {

        QueryBuilder qb = rangeQuery("fv")
                // [88.99, 10000)
                .from(88.99)
                .to(10000)
                .includeLower(true)
                .includeUpper(false);

        SearchResponse response = client.prepareSearch("gamelog").setQuery(qb).get();

        System.out.println(response);
    }
    /*
        进行聚合查询 select team, count(*) as player_count from player group by team;
     */
    @Test
    public void testAgg1() {

        //指定索引和type
        SearchRequestBuilder builder = client.prepareSearch("player_info").setTypes("player");
        //按team分组然后聚合，但是并没有指定聚合函数
        TermsAggregationBuilder teamAgg = AggregationBuilders.terms("player_count").field("team");
        //添加聚合器
        builder.addAggregation(teamAgg);
        //触发
        SearchResponse response = builder.execute().actionGet();
        //System.out.println(response);
        //将返回的结果放入到一个map中
        Map<String, Aggregation> aggMap = response.getAggregations().getAsMap();
//        Set<String> keys = aggMap.keySet();
//
//        for (String key: keys) {
//            System.out.println(key);
//        }

//        //取出聚合属性
        StringTerms terms = (StringTerms) aggMap.get("player_count");


        //
        //依次迭代出分组聚合数据
//        for (Terms.Bucket bucket : terms.getBuckets()) {
//            //分组的名字
//            String team = (String) bucket.getKey();
//            //count，分组后一个组有多少数据
//            long count = bucket.getDocCount();
//            System.out.println(team + " " + count);
//        }

        Iterator<Terms.Bucket> teamBucketIt = terms.getBuckets().iterator();
        while (teamBucketIt .hasNext()) {
            Terms.Bucket bucket = teamBucketIt.next();
            String team = (String) bucket.getKey();

            long count = bucket.getDocCount();

            System.out.println(team + " " + count);
        }

    }

    //select team, position, count(*) as pos_count from player group by team, position;
    /*
      查找相同球队相同位置球员的数量
    */
    @Test
    public void testAgg2() {
        SearchRequestBuilder builder = client.prepareSearch("player_info").setTypes("player");
        //指定别名和分组的字段
        TermsAggregationBuilder teamAgg = AggregationBuilders.terms("team_name").field("team");
        TermsAggregationBuilder posAgg= AggregationBuilders.terms("pos_count").field("position");
        //添加两个聚合构建器
        builder.addAggregation(teamAgg.subAggregation(posAgg));
        //执行查询
        SearchResponse response = builder.execute().actionGet();
        //将查询结果放入map中
        Map<String, Aggregation> aggMap = response.getAggregations().getAsMap();
        //根据属性名到map中查找
        StringTerms teams = (StringTerms) aggMap.get("team_name");
        //循环查找结果
        for (Terms.Bucket teamBucket : teams.getBuckets()) {
            //先按球队进行分组
            String team = (String) teamBucket.getKey();
            Map<String, Aggregation> subAggMap = teamBucket.getAggregations().getAsMap();
            StringTerms positions = (StringTerms) subAggMap.get("pos_count");
            //因为一个球队有很多位置，那么还要依次拿出位置信息
            for (Terms.Bucket posBucket : positions.getBuckets()) {
                //拿到位置的名字
                String pos = (String) posBucket.getKey();
                //拿出该位置的数量
                long docCount = posBucket.getDocCount();
                //打印球队，位置，人数
                System.out.println(team + " " + pos + " " + docCount);
            }


        }
    }

}

 AdminAPI:
 

package com.wx.es1;

import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.client.Requests;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.avg.InternalAvg;
import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.sum.InternalSum;
import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.InetAddress;
import java.util.HashMap;
import java.util.Map;

public class AdminAPI {
    /*
    AdminApi主要就是用来操作创建索引分片之类的admin操作
     */
   //初始化集群设置
    private TransportClient client;
    @Before
    public void init()throws Exception{
        Settings settings=Settings.builder().put("cluster.name","my-es").build();
        client=new PreBuiltTransportClient(settings).addTransportAddresses(
                new InetSocketTransportAddress(InetAddress.getByName("192.168.203.128"),9300),
                new InetSocketTransportAddress(InetAddress.getByName("192.168.203.129"),9300),
                new InetSocketTransportAddress(InetAddress.getByName("192.168.203.130"),9300)
        );
    }

    //创建索引，并配置一些参数
    @Test
    public void createIndexWithSettings() {
        //获取Admin的API
        AdminClient admin = client.admin();
        //使用Admin API对索引进行操作
        IndicesAdminClient indices = admin.indices();
        //准备创建索引
        indices.prepareCreate("gamelog")
                //配置索引参数
                .setSettings(
                        //参数配置器
                        Settings.builder()//指定索引分区的数量
                                .put("index.number_of_shards", 4)
                                //指定索引副本的数量（注意：不包括本身，如果设置数据存储副本为2，实际上数据存储了3份）
                                .put("index.number_of_replicas", 2)
                )
                //真正执行
                .get();
    }
    /**
     * 你可以通过dynamic设置来控制这一行为，它能够接受以下的选项：
     * true：默认值。动态添加字段
     * false：忽略新字段
     * strict：如果碰到陌生字段，抛出异常
     * @throws
     */
    //创建索引和mapping映射
    @Test
    public void createSettingsMappings() throws Exception {
        //1.设置Settings
        HashMap<String, Object> settings_map = new HashMap<String, Object>();
        //设置分片的数量
        settings_map.put("number_of_shards", 3);
        //设置副本的数量，如果设置两个副本就代表会存三份数据
        settings_map.put("number_of_replicas", 2);
        //2.这是mapping映射,这相当于配置一个域信息
        XContentBuilder contentBuilder = XContentFactory.jsonBuilder()
                .startObject()
                .field("dynamic", true)
                //设置type中的属性
                .startObject("properties")
                //设置num属性的信息
                .startObject("num")
                //类型是Integer
                .field("type", "integer")
                //不分词，但是建索引
                .field("index", "not_analyzed")
                //在文档中存储
                .field("store", "yes")
                .endObject()
                //设置name属性的信息
                .startObject("name")
                //类型为string
                .field("type", "string")
                //在文档中存储
                .field("store", "yes")
                //建立索引，分词
                .field("index", "analyzed")
                //分词器使用ik
                .field("analyzer", "ik_max_word")
                .endObject()
                .endObject()
                .endObject();
        //创建索引,名字叫user_info
        CreateIndexRequestBuilder builder = client.admin().indices().prepareCreate("user_info");
        //管理索引（user_info）然后关联type（user）
        builder.setSettings(settings_map).addMapping( "user",contentBuilder).get();
    }
    /**
     * index这个属性，no代表不建索引
     * not_analyzed，建索引不分词
     * analyzed 即分词，又建立索引
     * expected [no], [not_analyzed] or [analyzed]
     * @throws IOException
     */
    @Test
    public void testSettingsPlayerMappings() throws IOException {
        //1:settings
        HashMap<String, Object> settings_map = new HashMap<String, Object>(2);
        settings_map.put("number_of_shards", 3);
        settings_map.put("number_of_replicas", 1);

        //2:mappings
        XContentBuilder builder = XContentFactory.jsonBuilder()
                .startObject()//
                .field("dynamic", "true")
                .startObject("properties")
                .startObject("id")
                .field("type", "integer")
                .field("store", "yes")
                .endObject()
                .startObject("name")
                .field("type", "string")
                .field("index", "not_analyzed")
                .endObject()
                .startObject("age")
                .field("type", "integer")
                .endObject()
                .startObject("salary")
                .field("type", "integer")
                .endObject()
                .startObject("team")
                .field("type", "string")
                .field("index", "not_analyzed")
                .endObject()
                .startObject("position")
                .field("type", "string")
                .field("index", "not_analyzed")
                .endObject()
                .startObject("description")
                .field("type", "string")
                .field("store", "no")
                .field("index", "analyzed")
                .field("analyzer", "ik_smart")
                .endObject()
                .startObject("addr")
                .field("type", "string")
                .field("store", "yes")
                .field("index", "analyzed")
                .field("analyzer", "ik_smart")
                .endObject()
                .endObject()
                .endObject();

        CreateIndexRequestBuilder prepareCreate = client.admin().indices().prepareCreate("player_info");
        prepareCreate.setSettings(settings_map).addMapping("player", builder).get();
    }
    //select team, max(age) as max_age from player group by team;查询每个队年龄最大的球员
    @Test
    public void testAgg3() {
        SearchRequestBuilder builder = client.prepareSearch("player_info").setTypes("player");
        //指定安球队进行分组
        TermsAggregationBuilder teamAgg = AggregationBuilders.terms("team_name").field("team");
        //指定分组求最大值
        MaxAggregationBuilder maxAgg = AggregationBuilders.max("max_age").field("age");
        //分组后求最大值
        builder.addAggregation(teamAgg.subAggregation(maxAgg));
        //查询
        SearchResponse response = builder.execute().actionGet();
        Map<String, Aggregation> aggMap = response.getAggregations().getAsMap();
        //根据team属性，获取map中的内容
        StringTerms teams = (StringTerms) aggMap.get("team_name");
        for (Terms.Bucket teamBucket : teams.getBuckets()) {
            //分组的属性名
            String team = (String) teamBucket.getKey();
            //在将聚合后取最大值的内容取出来放到map中
            Map<String, Aggregation> subAggMap = teamBucket.getAggregations().getAsMap();
            //取分组后的最大值
            InternalMax ages = (InternalMax)subAggMap.get("max_age");
            double max = ages.getValue();
            System.out.println(team + " " + max);
        }
    }
    //select team, avg(age) as avg_age, sum(salary) as total_salary from player group by team;
    //查询每个队的平均年龄和平均工资
    @Test
    public void testAgg4() {
        SearchRequestBuilder builder = client.prepareSearch("player_info").setTypes("player");
        //指定分组字段
        TermsAggregationBuilder termsAgg = AggregationBuilders.terms("team_name").field("team");
        //指定聚合函数是求平均数据
        AvgAggregationBuilder avgAgg = AggregationBuilders.avg("avg_age").field("age");
        //指定另外一个聚合函数是求和
        SumAggregationBuilder sumAgg = AggregationBuilders.sum("total_salary").field("salary");
        //分组的聚合器关联了两个聚合函数
        builder.addAggregation(termsAgg.subAggregation(avgAgg).subAggregation(sumAgg));
        SearchResponse response = builder.execute().actionGet();
        Map<String, Aggregation> aggMap = response.getAggregations().getAsMap();
        //按分组的名字取出数据
        StringTerms teams = (StringTerms) aggMap.get("team_name");
        for (Terms.Bucket teamBucket : teams.getBuckets()) {
            //获取球队名字
            String team = (String) teamBucket.getKey();
            Map<String, Aggregation> subAggMap = teamBucket.getAggregations().getAsMap();
            //根据别名取出平均年龄
            InternalAvg avgAge = (InternalAvg)subAggMap.get("avg_age");
            //根据别名取出薪水总和
            InternalSum totalSalary = (InternalSum)subAggMap.get("total_salary");
            double avgAgeValue = avgAge.getValue();
            double totalSalaryValue = totalSalary.getValue();
            System.out.println(team + " " + avgAgeValue + " " + totalSalaryValue);
        }
    }

    //select team, sum(salary) as total_salary from player group by team order by total_salary desc;
    //查询每个队的队员的总工资并按升序排列
    @Test
    public void testAgg5() {
        SearchRequestBuilder builder = client.prepareSearch("player_info").setTypes("player");
        //按team进行分组，然后指定排序规则
        TermsAggregationBuilder termsAgg = AggregationBuilders.terms("team_name").field("team").order(Terms.Order.aggregation("total_salary ", true));
        SumAggregationBuilder sumAgg = AggregationBuilders.sum("total_salary").field("salary");
        builder.addAggregation(termsAgg.subAggregation(sumAgg));
        SearchResponse response = builder.execute().actionGet();
        Map<String, Aggregation> aggMap = response.getAggregations().getAsMap();
        StringTerms teams = (StringTerms) aggMap.get("team_name");
        for (Terms.Bucket teamBucket : teams.getBuckets()) {
            String team = (String) teamBucket.getKey();
            Map<String, Aggregation> subAggMap = teamBucket.getAggregations().getAsMap();
            InternalSum totalSalary = (InternalSum)subAggMap.get("total_salary");
            double totalSalaryValue = totalSalary.getValue();
            System.out.println(team + " " + totalSalaryValue);
        }
    }
}

ES的sql插件安装：所有的机器都要安装，在安装目录下执行

./bin/elasticsearch-plugin install https://github.com/NLPchina/elasticsearch-sql/releases/download/5.4.3.0/elasticsearch-sql-5.4.3.0.zip

重启ES：

简单查询：http://192.168.203.128:9200/_sql?sql=select * from player_info limit 10

聚合查询：http://192.168.203.128:9200/_sql?sql=select team,avg(salary) avg_sal  from player_info group by team

安装sql的ui插件：

    

  修改配置：

  

步骤命令：

用npm编译安装

unzip es-sql-site-standalone.zip

cd site-server/

npm install express --save

 

修改SQL的Server的端口

vi site_configuration.json

启动服务

node node-server.js &

Select team,avg(salary) from player_info group by team

  接下解决ES分片的问题