ELK

ELK

1、E：Elasticsearch

1.1 : 下载

• ElasticSearch
• logstash
• kibana

1.2 : QueryDSL

1.2.1 增

• 新建索引

  7版本以后创建索引需要设置切片和副本，否则默认都是1

  # 新增/设置索引
    PUT /dm
    {
      "settings": {
        "number_of_shards": 5,
        "number_of_replicas": 1
      },
      "mappings": {
        "properties": {
          "name": {
            "type": "text",
            "analyzer": "standard"
          },
          "age":{
            "type": "keyword"
          }
        }
      }
    }
  

• 新建文档

  在7.x版本中弃用了文档类型采用通用类型_doc/1 文档名称

  PUT /dm/_doc/1
  {
    "name": "法外狂徒",
    "age": 12
  }
  PUT /dm/_doc/2
  {
    "name":"法外逍遥",
    "age":20
  }
  
  

1.2.2 改

• 修改文档

  /index/_update/document

  # 改
  POST /dm/_update/1
  {
    "doc": {
      "name": "法外狂徒2"
    }
  }
  

1.2.3 删

• 删除索引

  在es中的删除是将索引放在了类似于windows系统中的回收站一样
  重新执行添加语句还是会回来的 速度要比第一次执行创建的速度要快很多

  DELETE /dm
  

• 删除文档

  /index/_cod/documentID

  DELETE /dm/_doc/1
  

1.2.4 查

• 分词查询

  出现两个结果

    #分词查询
  GET /dm/_search
  {
    "query": {
      "match": {
        "name": "法外"
      	}
   	 }
  }
  

• 筛选结果

  #模糊查询  结果过滤[显示指定的field]
    GET /dm/_search
    {
      "query": {
        "match": {
          "name": "法外狂徒"
        }
      }
      , "_source": ["name"]
    }
  

• 排序

  #排序
  GET /dm/_search
  {
    "query": {
      "match": {
        "name": "法外狂徒"
      }
    }
    , "sort": [
      {
        "age": {
          "order": "asc"
        }
      }
    ]
  }
  

• 分页

  #分页
  GET /dm/_search
  {
    "query": {
      "match": {
        "name": "法外狂徒"
      }
    },
    "from": 0,
    "size": 1
  }
  

• 多条件查询

  #多条件查询 must == and
  GET /dm/_search
  {
    "query": {
      "bool": {
        "must": [
          {
            "match": {
              "name": "法外逍遥2"
            }
          },{
            "match": {qq
              "age": "10"
            }
          }
        ]
      }
    }  
  }
  #多条件查询 should 不会影响结果 只会对结果的_score进行打分
  GET /dm/_search
  {
    "query": {
      "bool": {
        "should": [
          {
            "match": {
              "name": "法外逍遥2"
            }
          },{
            "match": {
              "age": "10"
            }
          }
        ]
      }
    } 
  }
  

• not查询

  #not查询
  GET /dm/_search
  {
    "query": {
      "bool": {
        "must_not": [
         {
            "match": {
              "age": "10"
            }
          }
        ]
      }
    }
    
  }
  

• filter筛选

  #filter过滤
  # gt 大于
  # get 大于等于
  # lt 小于
  # lte 小于等于
  GET /dm/_search
  {
    "query": {
      "bool": {
        "must": [
         {
            "match": {
              "name": "法外"
            }
          }
        ],
        "filter": {
          "range": {
            "age": {
              "gte": 10
            }
          }
        }
      }
    }
  }
  

• 高亮

  #查询关键字高亮显示
  GET /dm/_search
  {
    "query": {
      "match": {
        "name": "法外"
      }
    },
    "highlight": {
      "fields": {
        "name":{}
      }
    }
  }
  

• 自定义高亮

  GET /dm/_search
  {
    "query": {
      "match": {
        "name": "法外"
      }
    },
    "highlight": {
      "pre_tags": "<p class='key' style='color: red'>",
      "post_tags": "</p>", 
      "fields": {
        "name":{}
      }
    }
  }
  
  

1.3 RestHighLevelClient

RestHighLevelClient
该对象是Elasticsearch退出的java高级对象

1.3.1 Index

• 依赖

  <dependency>
          <groupId>org.elasticsearch.client</groupId>
          <artifactId>elasticsearch-rest-high-level-client</artifactId>
          <version>7.6.1</version>
      </dependency>
  

• 创建索引

   //1、创建请求
      CreateIndexRequest request = new CreateIndexRequest("book");
  
      //2、设置索引映射
      Map<String, Object> title = new HashMap<>();
      title.put("type","text");
      title.put("analyzer","ik_max_word");
  
      Map<String, Object> properties = new HashMap<>();
      properties.put("title",title);
  
      Map<String, Object> mapping = new HashMap<>();
      mapping.put("properties", properties);
  
      request.mapping(mapping);
  
      //3、发起请求
      CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(request, RequestOptions.DEFAULT);
  

• 判断索引是否存在

    GetIndexRequest request = new GetIndexRequest("book");
    boolean exists = restHighLevelClient.indices().exists(request, RequestOptions.DEFAULT);
    System.out.println(exists);
  

• 删除索引

  DeleteIndexRequest request = new DeleteIndexRequest("book");
   AcknowledgedResponse delete = restHighLevelClient.indices().delete(request, RequestOptions.DEFAULT);
  System.out.println(delete.isAcknowledged());
  

1.3.2 Document

• 添加文档

  User user = new User("李垚",21);
  
      //创建请求 入参是index
      IndexRequest request = new IndexRequest("book");
      //文档ID
      request.id("1");
      //超时时间
      request.timeout("1m");
      //封装请求
      request.source(JSON.toJSONString(user), XContentType.JSON);
  
      //客户端发送请求
      IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT);
      System.out.println(indexResponse.status());
  

• 批量添加文档

   //1、该数据爬虫爬下来的数据
      List<Book> bookList = ParseHTMLUtil.parse(keyword);
  
      //2、发起批量添加文档请求
      BulkRequest bulkRequest = new BulkRequest();
  
      for (int i = 0; i < bookList.size(); i++) {
          bulkRequest.add(
                  //添加数据
                  new IndexRequest("book").source(
                          JSON.toJSONString(bookList.get(i)), XContentType.JSON
                  )
          );
      }
  
      //3、执行请求
      BulkResponse responses = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
  

• 判断文档是否存在

  //book:索引
  //1：文档id
  GetRequest request = new GetRequest("book,"1");
      boolean exists = restHighLevelClient.exists(request, RequestOptions.DEFAULT);
      System.out.println(exists);
  

• 更新文档

  //1、发送更新请求
      UpdateRequest updateRequest = new UpdateRequest("book","2");
      //2、封装更新数据
      User user = new User();
      user.setName("法外狂徒");
      user.setAge(12);
      updateRequest.doc(JSON.toJSONString(user),XContentType.JSON);
      //3、执行更新操作
      UpdateResponse update = restHighLevelClient.update(updateRequest,RequestOptions.DEFAULT);
      System.out.println(update.status());
  

• 根据id查询文档

   //1、发送请求
      GetRequest request = new GetRequest("book","1");
      //2、执行请求
      GetResponse document = restHighLevelClient.get(request, RequestOptions.DEFAULT);
      //3、将相应结果封装成json对象
      String str = document.getSourceAsString();
      //4、将json对象转换为模型对象
      User user = (User) JSON.parse(str);
      System.out.println(user);
  

• 单条件查询文档

  	//1、创建请求
      SearchRequest request = new SearchRequest("dm");
      //2、构建请求条件
      SearchSourceBuilder builder = new SearchSourceBuilder();
      //3、精准匹配
      TermQueryBuilder age = QueryBuilders.termQuery("age", "12");
  
      builder.query(age);
  
      //3、发送请求
      SearchResponse search = restHighLevelClient.search(request, RequestOptions.DEFAULT);
      System.out.println(search);
  

• 高亮查询

   //1、创建请求
      SearchRequest searchRequest = new SearchRequest();
      SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
  
      //2、构建查询条件
  
      //条件高亮
      HighlightBuilder highlightBuilder = new HighlightBuilder();
      highlightBuilder.field("title");
      highlightBuilder.preTags("<span style='color:red'>");
      highlightBuilder.postTags("</span>");
      sourceBuilder.highlighter(highlightBuilder);
  
      //分页
      sourceBuilder.from(offset);
      sourceBuilder.size(pagesize);
  
      //精准查询
      TermQueryBuilder queryBuilder = QueryBuilders.termQuery("title", keyword);
  
      //3、向请求添加条件
      SearchSourceBuilder query = sourceBuilder.query(queryBuilder);
      searchRequest.source(query);
  
      //4、发送请求
      SearchResponse search = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
  
  
  
      List<Map<String,Object>> bookList = new ArrayList<>();
      //5、解析条件封装数据
      for (SearchHit hit : search.getHits().getHits()) {
          //获取高亮字段
          Map<String, HighlightField> highlightFields = hit.getHighlightFields();
          HighlightField title = highlightFields.get("title");
          //获取所有数据
          Map<String, Object> sourceAsMap = hit.getSourceAsMap();
          if (title != null){
              //将原来的title替换成高亮字段
              Text[] fragments = title.fragments();
              String newTitle = "";
              for (Text fragment : fragments) {
                  newTitle += fragment;
              }
           //替换
           sourceAsMap.put("title",newTitle);
          }
          bookList.add(sourceAsMap);
      }
  

1.4 ElasticsearchRestTemplate

该对象是Springboot-data-elasticsearch提供的，是对RestHighLevelClient的封装
因为采用了面向对象的思想，所以减少了我们很多操作

• 依赖

  <dependency>
          <groupId>org.springframework.boot</groupId>
          <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
   </dependency>
  

1.4.1 Index

• pojo

  @Data
  @AllArgsConstructor
  @NoArgsConstructor
  @Addressing
  //索引name
  @Document(indexName = "dm_item")
  public class DmItem {
  
      //文档id
      @Id
      private Integer id;
  
      @Field(name = "name",
              type = FieldType.Text,
              analyzer = "ik_max_word"
              )
      private String name;
  
      @Field(
              name = "areaname",
              type = FieldType.Keyword
      )
      private String areaname;
  
      @Field(
              name = "starttime",
              type = FieldType.Date,
              format = DateFormat.date_optional_time
      )
      private Date starttime;
  
      @Field(
              name = "address",
              type = FieldType.Text,
              analyzer = "ik_max_word"
      )
      private String address;
  }
  

• service

   //1、构件查询条件
      BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
      
      queryBuilder.must(QueryBuilders.matchQuery("address",name));
      queryBuilder.must(QueryBuilders.matchQuery("name",name));
  
  
  
      //2、封装查询条件
      NativeSearchQuery query = new NativeSearchQueryBuilder()
              //查询条件
              .withQuery(queryBuilder)
              //分页
              .withPageable(PageRequest.of(0,10))
              //高亮
              .withHighlightFields(
                      new HighlightBuilder.Field("address")
              )
              .withHighlightBuilder(
                      new HighlightBuilder()
                      .preTags("<span style='color:red'>")
                      .postTags("</span>")
              )
              .build();
  
      //3、执行查询
      SearchHits<DmItem> search = elasticsearchRestTemplate.search(query, DmItem.class);
      List<SearchHit<DmItem>> searchHits = search.getSearchHits();
  
      //4、封装查询结果
      List<DmItem> list = new ArrayList<>();
      searchHits.forEach(dmItemSearchHit -> {
          Map<String, List<String>> highlightFields = dmItemSearchHit.getHighlightFields();
          dmItemSearchHit.getContent().setAddress(
                  highlightFields.get("address") == null ? dmItemSearchHit.getContent().getAddress(): highlightFields.get("address").get(0)
          );
          list.add(dmItemSearchHit.getContent());
      });
      System.out.println(list.size());
  

2、L：Logstash

2.1 ：概论

Logstash is an open source data collection engine with real-time pipelining capabilities。

简单来说logstash就是一根具备实时数据传输能力的管道，负责将数据信息从管道的输入端传输到管道的输出端；与此同时这根管道还可以让你根据自己的需求在中间加上滤网，Logstash提供里很多功能强大的滤网以满足你的各种应用场景。

Logstash包含三个plugin：input,filter,output。
官方的配置表

Input plugin
https://www.elastic.co/guide/en/logstash/6.4/plugins-inputs-jdbc.html

Output plugin
https://www.elastic.co/guide/en/logstash/6.4/plugins-outputs-elasticsearch.html

Filter plugin
https://www.elastic.co/guide/en/logstash/6.4/plugins-filters-elasticsearch.html

2.2 ：安装配置

• 下载地址
  https://www.elastic.co/downloads/logstash
• 创建自己的配置文件

  cd /logstash/config 创建conf自己的配置文件 mkdir conf/jdbc/my.conf

  #输入方: jdbc、redis、rabbitMQ
  input{
      stdin{
      }
      jdbc{
      	#数据库连接地址
          jdbc_connection_string => "jdbc:mysql://localhost:3306/dm_item?characterEncoding=UTF-8&useSSL=false&useUnicode=true&serverTimezone=UTC"
          #数据库账号
          jdbc_user => "root"
          #数据库密码
          jdbc_password => "root"
          #要在这个路径下复制粘贴一个数据库连接的jar包
          jdbc_driver_library => "D:\elasicSearch\logstash-7.6.1\config\conf\jdbc\mysql-connector-java-8.0.22.jar"
          #数据库驱动
          jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
          #是否开启分页
          jdbc_paging_enabled => "true"
          #每页显示数据
          jdbc_page_size => "50000"
          #数据转换中的编码格式
          codec => plain { charset => "UTF-8"}
          #数据增量监听的字段
          tracking_column => "updatedTime"
          #是否记录最后一次执行的结果
          record_last_run => true
          #记录结果保存的路径
          last_run_metadata_path => "D:\elasicSearch\logstash-7.6.1\config\conf\jdbc\lastrun\.logstash_jdbc_last_run"
          #设置时区
          jdbc_default_timezone => "Asia/Shanghai"
          #要执行的sql文件
          statement_filepath => "D:\elasicSearch\logstash-7.6.1\config\conf\jdbc\dm_item.sql"
          #是否清空上次执行的语句缓存
          clean_run => false
          #定时器 设置多久执行一次sql：[分 时 天 月 年]
          schedule => "* * * * *"
      }
  }
  #输出到 Elasticsearch
  output{
      elasticsearch{
      	#必须写ip地址不能写localhost
          hosts => "127.0.0.1:9200"
          #索引名称
          index => "dm_item"
          #文档的名称  这里使用一个表达式代表使用数据库里面的id字段作为文档的名称
          document_id => "%{id}"
      }
      #标准输出执行的过程
      stdout{
          codec => json_lines
      }
  }
  

• 配置logstash.yml

  #
  http.host: "0.0.0.0"
  #要输入到的es的地址
  xpack.monitoring.elasticsearch.hosts: [ "http://localhost:9200" ]
  #自己的配置文件地址
  path.config: D:\elasicSearch\logstash-7.6.1\config\conf\jdbc\*.conf
  

• 启动

  进入bin目录 ：-f 代表配置文件启动

  logstash.bat -f ../config/conf/jdbc/dm_item.conf