ElasticSearch部署

ElasticSearch部署

1、创建普通用户（已有非root用户，可略过），elasticsearch5.0开始不允许root用户启动

# root下创建elastic用户组和elastic用户，并将elastic用户添加到elastic用户组中
groupadd elastic
useradd elastic -g elastic

2、解压缩

tar zxvf elasticsearch-7.9.3-linux-x86_64.tar.gz

3、elasticsearch7.0开始需要jdk11，elasticsearch自带jdk

# 复制elasticsearch目录下jdk路径，切换到bin目录下
vim elasticsearch-env
# 添加JAVA_HOME
set -e -o pipefail
JAVA_HOME="/usr/elk/elasticsearch-7.9.3/jdk/"

CDPATH=""

SCRIPT="$0"

4、启动elasticsearch

# 更改解压后文件夹及内部文件的所属用户和组
chown -Rf elastic:elastic /usr/elk/elasticsearch-7.9.3
# 切换用户，bin目录下启动
su elastic
./elasticsearch
# 测试
curl 'http://localhost:9200/?pretty'

**错误1：**elasticsearch进程的最大文件描述符[4096]太低，请至少增加到[65535]

**错误2：**用户elastic可以创建的最大线程数[3795]太低，请至少增加到[4096]

解决

vim /etc/security/limits.conf
# 在文件末尾添加
elastic soft nofile 65535
elastic hard nofile 65535
elastic soft nproc 4096
elastic hard nproc 4096

**错误3：**最大虚拟内存区域vm.max_map_count[65530]太低，请至少增加到[262144]

解决

vim /etc/sysctl.conf
# 在文件末尾添加
vm.max_map_count = 262144
# 重新加载虚拟内存配置
sysctl -p

5、elasticsearch配置

vim config/elasticsearch.yml
修改配置后需重启

单机

# 允许访问的地址
network.host: 0.0.0.0
# 端口号
http.port: 9200
# 允许跨域，搭配es-head
http.cors.enabled: true
http.cors.allow-origin: "*"
discovery.seed_hosts: ["192.168.1.104"]
cluster.initial_master_nodes: ["192.168.1.104"]

集群

# 集群名称，同一集群下保持一致
cluster.name: my-application
# 集群下各节点名称，此时cluster.initial_master_nodes需更改为节点名称
node.name: node-1
# 集群发现配置
discovery.seed_hosts: ["192.168.1.104:9200","192.168.1.105:9200","192.168.1.106:9200"]
cluster.initial_master_nodes: [node-1,node-2,node-3]
discovery.zen.ping_timeout: 60s

es-head安装

# 可以在外部下载、编译完成，移动到服务器，直接启动
git clone git://github.com/mobz/elasticsearch-head.git
# 编译启动，cnpm为淘宝源
cnpm install
cnpm run start

docker安装存在缺陷，可直接部署在外部，且需要nodejs支持

elasticsearch7.x版本以后对请求头做了严格限制，需设置contentType === “application/json;charset=UTF-8”

编辑内部文件_site/vendor.js

6886行和7574行

“application/x-www-form-urlencoded改成"application/json;charset=UTF-8”

Logstash部署

1、解压缩

tar zxvf logstash-7.9.3.tar.gz

2、logstash配置oracle

复制oracle驱动包至/usr/elk/logstash-7.9.3/logstash-core/lib/jars/目录下

新建配置文件jdbc.sql，

input {
 stdin {
 }
 jdbc {
  type => "bdcqzs"
  # 数据库配置
  jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
  jdbc_connection_string => "jdbc:oracle:thin:@192.168.1.33:1521:njyzt"
  jdbc_user => "netobdc"
  jdbc_password => "netobdc"
  jdbc_default_timezone =>"Asia/Shanghai"
  # 是否清除sql_last_value的记录，需要增量同步时此字段必须为false
  clean_run => false
  # 同步频率(分 时 天 月 年)，默认每分钟同步一次
  # 104系统时间与北京时间相差8小时，故+8取实际时间，此时cron表达式表示每天23点执行一次
  schedule => "0 7 * * *"
  statement => "select bsm, iid, to_char(djsj,'yyyy/MM/dd HH24:mm:ss') djsj, bdcqzh, bh, qlrmc, qlrzjhm, zl, bdcdyh, qllx, qlxz, yt, mj, syqx, qlqtqk, sf, 0 as isdelete, 0 v from bdcqzs where djsj > trunc(sysdate) and djsj is not null order by djsj asc"
 }
 jdbc {
  type => "bdcqzm"
  # 数据库配置
  jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
  jdbc_connection_string => "jdbc:oracle:thin:@192.168.1.33:1521:njyzt"
  jdbc_user => "netobdc"
  jdbc_password => "netobdc"
  jdbc_default_timezone =>"Asia/Shanghai"
  # 是否清除sql_last_value的记录，需要增量同步时此字段必须为false
  clean_run => false
  # 同步频率(分 时 天 月 年)，默认每分钟同步一次
  # 104系统时间与北京时间相差8小时，故+8取实际时间，此时cron表达式表示每天23点执行一次
  schedule => "0 7 * * *"
  statement => "select bsm, iid, to_char(djsj,'yyyy/MM/dd HH24:mm:ss') djsj, bdcqzmh, bh, zmql, qlrmc, ywrmc, zl, bdcdyh, qt, sf, 0 as isdelete, 1 v from bdcqzm where djsj > trunc(sysdate) and djsj is not null order by djsj asc"
 }
}
filter {
}
output {
 stdout {
  codec => rubydebug
 }
 if[type]=="bdcqzs" {
   elasticsearch {
    # 集群地址以逗号隔开
    hosts => ["192.168.1.104:9200"]
    # 索引
    index => "bdcqzs"
    document_id => "%{bsm}"
  }
 }
 if[type]=="bdcqzm" {
   elasticsearch {
    # 集群地址以逗号隔开
    hosts => ["192.168.1.104:9200"]
    # 索引
    index => "bdcqzm"
    document_id => "%{bsm}"
  }
 }
}

3、启动

# 测试文件是否正确，运行需要jdk支持
bin/logstash -f /usr/elk/logstash-7.9.3/jdbc.sql -t
启动
bin/logstash -f /usr/elk/logstash-7.9.3/jdbc.sql

分页问题

查询10000条以后的结果，需设置该索引的窗口最大返回数

postman PUT

http://[ip]:[port]/[indices]/_settings
{
    "index": {
        "max_result_window": 100000000
    }
}

常用请求

Java-ES-Api

1、RestHighLevelClient

创建索引

    @Test
    void testCreateIndex() throws IOException {
        // 1、创建索引请求
        CreateIndexRequest createIndexRequest = new CreateIndexRequest("test");
        // 2、客户端执行请求IndicesClient，请求后获得响应
        CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
        System.out.println(createIndexResponse);
    }

获取索引

    @Test
    void testExistIndex() throws IOException {
        GetIndexRequest getIndexRequest = new GetIndexRequest("test");
        boolean exists = restHighLevelClient.indices().exists(getIndexRequest, RequestOptions.DEFAULT);
        System.out.println(exists);
    }

删除索引

    @Test
    void testDeleteIndex() throws IOException {
        DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest("jd_goods");
        AcknowledgedResponse delete = restHighLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT);
        System.out.println(delete.isAcknowledged());
    }

添加文档

    @Test
    void testAddDocument() throws IOException {
        // 创建对象
        User user = new User("tom", 12);
        // 创建请求
        IndexRequest indexRequest = new IndexRequest("test");
        // 规则 PUT /antu_index/_doc/1
        indexRequest.id("1");
        indexRequest.timeout(TimeValue.timeValueSeconds(1));
        // 将数据放入请求 JSON
        indexRequest.source(JSON.toJSONString(user), XContentType.JSON);
        // 客户端发送请求,获取响应结果
        IndexResponse indexResponse = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
        System.out.println(indexResponse.toString());
        // 对应命令返回的状态 CREATED
        System.out.println(indexResponse.status());
    }

获取文档

    @Test
    void testExistDocument() throws IOException {
        GetRequest getRequest = new GetRequest("test", "1");
        // 不获取返回的_source上下文
        getRequest.fetchSourceContext(new FetchSourceContext(false));
        getRequest.storedFields("_none_");
        boolean exists = restHighLevelClient.exists(getRequest, RequestOptions.DEFAULT);
        System.out.println(exists);
    }

获取返回的文档信息

    @Test
    void testGetDocument() throws IOException {
        GetRequest getRequest = new GetRequest("test", "1");
        GetResponse getResponse = restHighLevelClient.get(getRequest, RequestOptions.DEFAULT);
        // 打印文档内容
        System.out.println(getResponse.getSourceAsString());
        // 返回的全部内容和命令一样
        System.out.println(getResponse);
    }

更新文档信息

    @Test
    void testUpdateDocument() throws IOException {
        UpdateRequest updateRequest = new UpdateRequest("test", "1");
        updateRequest.timeout("1s");
        User user = new User("lucy", 20);
        updateRequest.doc(JSON.toJSONString(user), XContentType.JSON);
        UpdateResponse updateResponse = restHighLevelClient.update(updateRequest, RequestOptions.DEFAULT);
        System.out.println(updateResponse.status());
    }

删除文档记录

    @Test
    void testDeleteDocument() throws IOException {
        DeleteRequest deleteRequest = new DeleteRequest("test", "1");
        deleteRequest.timeout("1s");
        DeleteResponse deleteResponse = restHighLevelClient.delete(deleteRequest, RequestOptions.DEFAULT);
        System.out.println(deleteResponse.status());
    }

批量插入

    @Test
    void testBulkRequest() throws IOException {
        BulkRequest bulkRequest = new BulkRequest();
        bulkRequest.timeout("10s");
        List<User> userList = new ArrayList<>();
        userList.add(new User("tom1", 2));
        userList.add(new User("tom2", 4));
        userList.add(new User("tom3", 5));
        userList.add(new User("tom4", 7));
        // 批量处理请求
        for (User user : userList) {
            bulkRequest.add(
                    new IndexRequest("test")
                            .source(JSON.toJSONString(user), XContentType.JSON)
            );
        }
        BulkResponse bulkResponse = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
        System.out.println(bulkResponse.hasFailures());
    }

查询

   /**
     * @author: zhangBin
     * @description: 返回搜索结果
     * @date: 2021/1/4 9:30
     * @param: resultMap 返回结果
     * @param: table 查询索引
     * @param: queryMap 查询条件
     * @param: pageSize 每页显示条
     * @param: pageNum 当前页
     * @param: resultList 返回结果列
     * @param: ignoreList 忽略返回结果列
     * @return: java.util.Map<java.lang.String, java.lang.Object>
     */
    @Override
    public Map<String, Object> listSource(Map<String, Object> resultMap, String table, Map<String, Object> queryMap, int pageSize, int pageNum, List<String> resultList, List<String> ignoreList) throws IOException {
        String indices = table.replace(".", "_");

        // 搜索请求对象
        SearchRequest searchRequest = new SearchRequest(indices);

        // 搜索源构建对象
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().trackTotalHits(true);
        // QueryBuilders.termQuery 精确匹配,不能匹配中文
        // QueryBuilders.matchAllQuery 匹配所有

        // 布尔查询
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();

        // 查询条件为空，则匹配所有
        if (queryMap.size() == 0) {
            boolQueryBuilder.must(QueryBuilders.matchAllQuery());
        }

        // 过滤已删除的数据
        boolQueryBuilder.must(QueryBuilders.termQuery("isdelete", 0));

        BoolQueryBuilder shouldQuery = QueryBuilders.boolQuery();

        // 定义匹配查询  matchPhraseQuery可代替精确匹配  matchQuery相似度匹配
        queryMap.forEach((key, value) -> {
            shouldQuery.should(QueryBuilders.matchPhraseQuery(key, value).slop(value.toString().length()));
        });

        // 在must下，满足should  即 A must (B should C should D)
        boolQueryBuilder.must(shouldQuery);

        // 分页
        if (0 != pageSize) {
            searchSourceBuilder.size(pageSize);
            if (0 != pageNum) {
                searchSourceBuilder.from((pageNum - 1) * pageSize);
            }
        }

        // source源字段过虑，第一个参数结果集包括哪些字段，第二个参数表示结果集不包括哪些字段
        searchSourceBuilder.fetchSource(resultList.toArray(new String[0]), ignoreList.toArray(new String[0]));

        // 搜索方式 布尔查询
        searchSourceBuilder.query(boolQueryBuilder);

        // 搜索字段高亮显示 多字段高亮
        HighlightBuilder highlightBuilder = new HighlightBuilder();
        queryMap.forEach((key, value) -> {
            highlightBuilder.field(key);
        });
        highlightBuilder.requireFieldMatch(true);
        highlightBuilder.preTags("<span style='color:red'>");
        highlightBuilder.postTags("</span>");
        searchSourceBuilder.highlighter(highlightBuilder);

        // 向搜索请求对象中添加搜索源
        searchRequest.source(searchSourceBuilder);

        // 执行搜索，向es发出http请求
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);

        // 获取搜索结果
        SearchHits hits = searchResponse.getHits();

        // 得到匹配度高的文档
        SearchHit[] searchHits = hits.getHits();
        List<Map<String, Object>> list = new ArrayList<>();
        for (SearchHit hit : searchHits) {
            Map<String, HighlightField> highlightFields = hit.getHighlightFields();
            Map<String, Object> sourceAsMap = hit.getSourceAsMap();
            queryMap.forEach((key, value) -> {
                HighlightField field = highlightFields.get(key);
                if (null != field) {
                    Text[] fragments = field.fragments();
                    StringBuilder highLightField = new StringBuilder();
                    for (Text text : fragments) {
                        highLightField.append(text);
                    }
                    if (sourceAsMap.containsKey(key)) {
                        sourceAsMap.put(key, highLightField.toString());
                    }
                }
            });
            list.add(sourceAsMap);
        }

        resultMap.put(table + ".source", list);
        return resultMap;
    }

2、ElasticsearchRepository

3、ElasticsearchRestTemplate