es搜索引擎使用

最新推荐文章于 2024-04-15 04:16:13 发布

weixin_47074036

最新推荐文章于 2024-04-15 04:16:13 发布

阅读量452

点赞数

文章标签： elasticsearch 搜索引擎大数据

本文链接：https://blog.csdn.net/weixin_47074036/article/details/127621114

版权

1.下载es

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.15.1-linux-x86_64.tar.gz

# 解压
tar -zxvf elasticsearch-7.15.1-linux-x86_64.tar.gz

2.es默认不允许root用户启动，所以需要创建普通用户

useradd es

passwd tssp#2022
需要设置8位数密码：tssp#2022

3.es安装配置

mkdir /home/es/data
mkdir /home/es/logs

cd /home/es/elasticsearch-7.15.1/config

修改yml配置
vi elasticsearch.yml

cluster.name: my-application
node.name: node-1
network.host: 0.0.0.0
xpack.security.enabled: false
path.data: /home/es/data
path.logs: /home/es/logs
cluster.initial_master_nodes: node-1
修改完毕后，:wq! 保存退出

修改jvm大小，默认4个g
vi jvm.options

 -Xms512m
 -Xmx512m

在文件底部添加
vi /etc/security/limits.conf

 es soft nofile 65536
 es hard nofile 65536

vi /etc/sysctl.conf

vm.max_map_count = 655360
:wq! 保存退出

sysctl -p

4.切换es用户，启动

#切换es用户
su es

cd /home/es/elasticsearch-7.15.1/bin

后台启动
./elasticsearch -d

5.开发问题：
1.最大查询量受限制：

修改所有索引的最大查询量
http://ip:9200/_all/_settings
body请求体
{         
"index.max_result_window":400000；修改最大数量
}

2.java使用es

1.引入pom依赖
  <elasticsearch.version>7.15.1</elasticsearch.version>
  <dependency>
      <groupId>org.elasticsearch.client</groupId>
      <artifactId>elasticsearch-rest-high-level-client</artifactId>
      <version>${elasticsearch.version}</version>
  </dependency>
  
2.写一个配置类
@Configuration
public class ElasticSearchClientConfig {

    @Value("${es.ip}")
    public String ip;
    @Value("${es.port}")
    public String port;
    @Bean
    // 硬编码的值可以设置到配置文件，通过@Value读取
    public RestHighLevelClient restHighLevelClient(){
        RestHighLevelClient client = new RestHighLevelClient(RestClient.builder(new HttpHost(ip, Integer.parseInt(port), "http")));
        return client;
    }
}

package com.iflytek.site.es;

import cn.smallbun.screw.core.util.CollectionUtils;
import cn.smallbun.screw.core.util.StringUtils;
import com.alibaba.fastjson.JSON;
import com.iflytek.site.buildscan.dto.BuildScanComResultDTO;
import com.iflytek.site.buildscan.entity.BuildScanCom;
import com.iflytek.site.buildscan.mapper.BuildScanComMapper;
import com.iflytek.site.buildscan.service.BuildScanComService;
import com.iflytek.site.buildscan.vo.BuildScanComVO;
import com.iflytek.site.common.utils.JackSonUtils;
import com.iflytek.site.packagetask.entity.PatrolPackageTask;
import com.iflytek.site.packagetask.service.PatrolPackageTaskService;
import lombok.extern.slf4j.Slf4j;
import org.apache.lucene.search.TotalHits;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.*;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.IdsQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

/**
 *  es7.6.x 高级客户端测试 API
 */
@RestController
@RequestMapping("/es")
@Slf4j
public class ElasticsearchJdApplicationTests {
    // 面向对象来操作
    @Autowired
    @Qualifier("restHighLevelClient")
    private RestHighLevelClient client;

    @Value("${es.task_index}")
    private String taskIndex;
    @Autowired
    private BuildScanComMapper scanComMapper;
    @Autowired
    private PatrolPackageTaskService taskService;
    @Autowired
    private BuildScanComService buildScanComService;

    // 测试索引的创建 Request PUT 
    @RequestMapping("/testCreateIndex")
    void testCreateIndex(String index) throws IOException {
        // 1、创建索引请求
        CreateIndexRequest request = new CreateIndexRequest(index);
        // 2、客户端执行请求 IndicesClient,请求后获得响应
        CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT);
    }

    // 测试获取索引,判断其是否存在
    @RequestMapping("/testExistIndex")
    void testExistIndex(String index) throws IOException {
        GetIndexRequest request = new GetIndexRequest(index);
        boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
        if(!exists){
            //不存在索引创建
            testCreateIndex(index);
        }
    }

    // 测试删除索引
    @RequestMapping("/testDeleteIndex") 
    String testDeleteIndex(String taskId) throws IOException {
        final String[] split = taskId.split(",");
        for (int i = 0; i < split.length; i++) {
            DeleteIndexRequest request = new DeleteIndexRequest(taskIndex+split[i]);
            // 删除
            AcknowledgedResponse delete = client.indices().delete(request, RequestOptions.DEFAULT);
        }
        log.info("删除成功");
        //System.out.println(delete.isAcknowledged());
        return "删除成功";
    }

    // 测试添加文档
    @RequestMapping("/testAddDocument")
    void testAddDocument(Long taskId) throws IOException {
        // 创建对象
        BuildScanComVO vo=new BuildScanComVO();
        vo.setTaskId(String.valueOf(taskId));
        List<BuildScanCom> list = scanComMapper.listByES(vo);

        // 将我们的数据放入请求 json
        AtomicInteger i= new AtomicInteger();
        //y计算数据还剩多少没有添加到索引中
        AtomicInteger y= new AtomicInteger(1);
        BulkRequest[] bulkRequest = {new BulkRequest()};
        AtomicInteger size = new AtomicInteger(list.size() / 1000);
        list.forEach(x->{
        	//i++;
            i.getAndIncrement();
            bulkRequest[0].timeout("500s");
            bulkRequest[0].add(new IndexRequest(taskIndex+taskId).id(String.valueOf(x.getId()))
                    .source(JSON.toJSONString(x), XContentType.JSON));
            if(y.get() *1000==i.get() || i.get()==list.size()){
            	//如果i==1000就添加索引数据
                y.getAndIncrement();
                try {
                	//索引一次添加数据量不能太大，会报错。1000目前可以
                    BulkResponse bulkResponse = client.bulk(bulkRequest[0], RequestOptions.DEFAULT);
                    log.info("添加数据成功"+size.getAndDecrement());
                } catch (IOException e) {
                    e.printStackTrace();
                }
                BulkRequest bulkRequest1 = new BulkRequest();
                bulkRequest[0] =bulkRequest1;
            }
        });
    }

    // 获取文档，判断是否存在 get /index/doc/1
    @RequestMapping("/testIsExists")
    void testIsExists() throws IOException {
        GetRequest getRequest = new GetRequest(taskIndex, "1");
        // 不获取返回的 _source 的上下文了
        getRequest.fetchSourceContext(new FetchSourceContext(false));
        getRequest.storedFields("_none_");
        boolean exists = client.exists(getRequest, RequestOptions.DEFAULT);
        System.out.println(exists);
    }

    // 获得文档的信息
    @RequestMapping("/testGetDocument")
   public List<BuildScanCom> testGetDocument(String taskId, String id, String name, Integer from, Integer size) throws IOException {
        SearchRequest searchRequest = new SearchRequest(taskIndex + taskId);
        //2,构建SearchSourceBuilder查询对象
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        //3，构建QueryBuilder对象指定查询方式和查询条件
        if (StringUtils.isNotBlank(name)) {
            TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("name", name);
            //4,将QuseryBuilder对象设置到SearchSourceBuilder对象中
            sourceBuilder.query(termQueryBuilder);
        }

        // 查询条件，我们可以使用 QueryBuilders 工具来实现
        // QueryBuilders.termQuery 精确
        // QueryBuilders.matchAllQuery() 匹配所有
        sourceBuilder.sort("risk", SortOrder.DESC);
        sourceBuilder.sort("createTime", SortOrder.DESC);
        sourceBuilder.trackTotalHits(true);
        //分页
        sourceBuilder.from(from);
        sourceBuilder.size(size);
        searchRequest.source(sourceBuilder);
        List<BuildScanCom> buildScanComs = new ArrayList<>();
        try {
            //6,调用方法查询数据
            SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
            //7,解析返回结果
            TotalHits totalHits = searchResponse.getHits().getTotalHits();
            System.out.println("总数：" + totalHits.value);
            SearchHit[] hits = searchResponse.getHits().getHits();
            for (int i = 0; i < hits.length; i++) {
                System.out.println("返回的结果： " + hits[i].getSourceAsString());
                BuildScanCom buildScanCom = JackSonUtils.jsonToTransfer(hits[i].getSourceAsString(), BuildScanCom.class);
                buildScanComs.add(buildScanCom);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return buildScanComs;
    }

    // 获得文档的信息
    @RequestMapping("/testGetDocument1")
    //正在使用，分页查询
    public BuildScanComResultDTO testGetDocument1(BuildScanComVO vo,SearchSourceBuilder builder) throws IOException {
        BuildScanComResultDTO resultDTO = new BuildScanComResultDTO();
        SearchRequest searchRequest = new SearchRequest(taskIndex + vo.getTaskId());
        builder.sort("risk", SortOrder.DESC);
        builder.sort("createTime", SortOrder.DESC);
        builder.trackTotalHits(true);//防止查询数据太多报错
        //分页
        builder.from((vo.getCurrent()-1)*10);
        builder.size(vo.getSize());
        searchRequest.source(builder);
        List<BuildScanCom> buildScanComs = new ArrayList<>();
        try {
            //6,调用方法查询数据
            SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
            //7,解析返回结果
            TotalHits totalHits = searchResponse.getHits().getTotalHits();
            resultDTO.setTotal(totalHits.value);//返回总数
            SearchHit[] hits = searchResponse.getHits().getHits();
            for (int i = 0; i < hits.length; i++) {
                BuildScanCom buildScanCom = JackSonUtils.jsonToTransfer(hits[i].getSourceAsString(), BuildScanCom.class);
                buildScanComs.add(buildScanCom);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        resultDTO.setRecords(buildScanComs);
        resultDTO.setCurrent(vo.getCurrent());
        resultDTO.setSize(vo.getSize());
        return resultDTO;
    }
    // 更新文档的信息
    @RequestMapping("/testUpdateRequest")
    void testUpdateRequest() throws IOException {
        UpdateRequest updateRequest = new UpdateRequest(taskIndex, "1");
        updateRequest.timeout("1s");
        BuildScanCom buildScanCom=new BuildScanCom();
        buildScanCom.setName("maven");
        buildScanCom.setVersion("1.23");
        updateRequest.doc(JSON.toJSONString(buildScanCom), XContentType.JSON);
        UpdateResponse updateResponse = client.update(updateRequest, RequestOptions.DEFAULT);
        System.out.println(updateResponse.status());
    }

    // 删除文档记录
    @RequestMapping("/testDeleteRequest")
    void testDeleteRequest() throws IOException {
        DeleteRequest request = new DeleteRequest(taskIndex, "1");
        request.timeout("1s");
        DeleteResponse deleteResponse = client.delete(request, RequestOptions.DEFAULT);
        System.out.println(deleteResponse.status());
    }

    // 特殊的，真的项目一般都会批量插入数据！
    @RequestMapping("/testBulkRequest") //创建和插入数据：1//str等于1代表查小于id的数，等于2代表等于id的数
    BulkResponse testBulkRequest(Long id,String str) throws IOException {
        long startTime = System.currentTimeMillis();
        List<PatrolPackageTask> list1 = taskService.lambdaQuery()
                .orderByDesc(PatrolPackageTask::getCreateTime)
                .func(x->{
                    if(id!=null && str.equals("1")){
                        x.lt(PatrolPackageTask::getId,id);
                    }else if(id!=null && str.equals("2")){
                        x.eq(PatrolPackageTask::getId,id);
                    }
                })
                .list();
        for (int j = 0; j < list1.size(); j++) {
            PatrolPackageTask task = list1.get(j);
            try {
                String index=taskIndex+task.getId();
                testExistIndex(index);//是否存在索引，不存在添加
                //是否存在索引数据
                boolean flag = testSearch(task.getId(),String.valueOf(com.getId()));
                if (flag) {
                    //如果为true存在数据
                    continue;
                }
                testAddDocument(task.getId());
                int z=collect.size() - j;
                System.out.println("还剩多少个"+z);
            }catch (Exception e){
                log.info("创建索引失败id:{}=",task.getId());
                errId.add(task.getId());
            }
        }
        long endTime = System.currentTimeMillis();
        log.info("执行时间为:{}",endTime-startTime);
        log.info("错的名称:{}",errName);
        log.info("错的id:{}",errId);
        return null;// 是否失败，返回 false 代表 成功！
    }

    // 查询
    // SearchRequest 搜索请求
    // SearchSourceBuilder 条件构造
    // HighlightBuilder 构建高亮
    // TermQueryBuilder 精确查询
    // MatchAllQueryBuilder
    // xxx QueryBuilder 对应我们刚才看到的命令！
    @RequestMapping("/testSearch")
    boolean testSearch(Long taskId,String id) throws IOException {
        SearchRequest searchRequest = new SearchRequest(taskIndex+taskId);
        // 构建搜索条件
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
       // sourceBuilder.highlighter();
        // 查询条件，我们可以使用 QueryBuilders 工具来实现
        // QueryBuilders.termQuery 精确
        // QueryBuilders.matchAllQuery() 匹配所有
        IdsQueryBuilder idsQueryBuilder = QueryBuilders.idsQuery().addIds(id);
        //TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("name", "qinjiang1");
        // MatchAllQueryBuilder matchAllQueryBuilder =
        //QueryBuilders.matchAllQuery();
        sourceBuilder.query(idsQueryBuilder);
       // sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchRequest.source(sourceBuilder);
        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        System.out.println(JSON.toJSONString(searchResponse.getHits()));
        System.out.println("=================================");
        TotalHits totalHits = searchResponse.getHits().getTotalHits();
        if(totalHits.value>0){
            return true;
        }
        return false;
    }
}

5.查询通用方法

    public BuildScanComResultDTO show(BuildScanComVO vo) {
        BuildScanComResultDTO resultDTO = new BuildScanComResultDTO();
        //4,将QuseryBuilder对象设置到SearchSourceBuilder对象中
        SearchSourceBuilder builder = new SearchSourceBuilder();
        WildcardQueryBuilder likeBuilder = null;
        TermQueryBuilder queryBuilder = null;
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        //must表示and，termQuery表示精确，wildcardQuery代表模糊
        if (StringUtils.isNotBlank(vo.getName())) {
            likeBuilder = QueryBuilders.wildcardQuery("name.keyword", "*" + vo.getName() + "*");
            boolQueryBuilder.must(likeBuilder);
        }
        if (StringUtils.isNotBlank(vo.getVersion())) {
            likeBuilder = QueryBuilders.wildcardQuery("version.keyword", "*" + vo.getVersion() + "*");
            boolQueryBuilder.must(likeBuilder);
        }
        if (Objects.nonNull(vo.getRisk())) {
            queryBuilder = QueryBuilders.termQuery("risk", vo.getRisk());
            boolQueryBuilder.must(queryBuilder);
        }
        if (StringUtils.isNotBlank(vo.getTechnicalClass())) {
            likeBuilder = QueryBuilders.wildcardQuery("technicalClass.keyword", "*" + vo.getTechnicalClass() + "*");
            boolQueryBuilder.must(likeBuilder);
        }
        if (StringUtils.isNotBlank(vo.getStoreStatus())) {
            queryBuilder = QueryBuilders.termQuery("storeStatus", vo.getStoreStatus());
            boolQueryBuilder.must(queryBuilder);//
        }
        builder.query(boolQueryBuilder);
        try {
            return elTest.testGetDocument1(vo, builder);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return resultDTO;
    }