ELK环境搭建与Springboot集成

安装JDK

略

Elasticsearch安装

• 下载
  wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz

• 安装插件
  /home/elasticsearch/bin/plugin install mobz/elasticsearch-head

• 启动
  /home/elasticsearch/bin/elasticsearch -Des.insecure.allow.root=true
  注意:启动elasticsearch默认情况下不能用root账号，需要添加参数：Des.insecure.allow.root=true

对外服务的HTTP端口是9200(IP:9200/_plugin/head/)，节点间交互的TCP端口是9300

Logstash安装

• 下载
  wget https://download.elastic.co/logstash/logstash/logstash-2.3.1.tar.gz

• 新建配置文件：vim simple.conf

  input {
    tcp {
      port => 9250
      mode => "server"
      tags => ["tags"]
      codec => "json_lines"
    }
  }
  output {
    stdout{codec =>rubydebug}
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      flush_size => 1000
    }
  }
  

• 启动
  /home/logstash/bin/logstash -f /home/logstash/simple.conf

Kibana安装

• 下载
  wget https://download.elastic.co/kibana/kibana/kibana-4.1.1-linux-x64.tar.gz

• 启动
  /home/kibana/bin/kibana

访问端口：5601

项目实战

• 引入Maven

  <dependency>
      <groupId>net.logstash.logback</groupId>
      <artifactId>logstash-logback-encoder</artifactId>
      <version>5.1</version>
  </dependency>
  

• Logback

  <appender name="elk" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
      <destination>192.168.1.200:9250</destination>
      <encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder" />
  </appender>
  
  <root level="info">
      <appender-ref ref="elk"/>
  </root>
  

Docker环境搭建

下载镜像

docker pull docker.elastic.co/kibana/kibana:7.2.0
docker pull docker.elastic.co/elasticsearch/elasticsearch:7.2.0

Linux：sysctl -w vm.max_map_count=262144

docker-compose.yml

创建网络：docker network create --subnet=172.16.30.0/24 es

version: '3'
services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0
    container_name: es01
    environment:
      - node.name=es01
      - discovery.seed_hosts=es02
      - cluster.initial_master_nodes=es01,es02
      - cluster.name=docker-cluster
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata01:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
      - 9300:9300
 
  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0
    container_name: es02
    environment:
      - node.name=es02
      - discovery.seed_hosts=es01
      - cluster.initial_master_nodes=es01,es02
      - cluster.name=docker-cluster
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata02:/usr/share/elasticsearch/data

  kibana:
    image: docker.elastic.co/kibana/kibana:7.2.0
    container_name: kibana
    environment:
      SERVER_NAME: kibana
      ELASTICSEARCH_HOSTS: http://es01:9200
    volumes:
      - kibana:/usr/share/kibana/config
    ports:
      - 5601:5601
    
volumes:
  esdata01:
    driver: local
  esdata02:
    driver: local
  kibana:
    driver: local
# /var/lib/docker/volumes/

networks:
  default:
    external:
      name: es

测试

http://192.168.40.200:5601
http://192.168.40.200:9200/_cat/health?v

日志插件

curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.2.0-linux-x86_64.tar.gz
./filebeat modules enable nginx
./filebeat setup
./filebeat -e

vi filebeat.yml

output.elasticsearch:
  hosts: ["localhost:9200"]
setup.kibana:
  host: "localhost:5601"

Nginx模块
vi modules.d/nginx.yml
var.paths: ["/home/docker/nginx/logs/access.log*"]

插件

拼音：https://github.com/medcl/elasticsearch-analysis-pinyin
中文分词：https://github.com/medcl/elasticsearch-analysis-ik

测试

GET http://192.168.40.200:9200/_analyze
{
  "analyzer" : "ik_smart",
  "text" : "中华人民共和国国歌"
}

{
  "analyzer" : "pinyin",
  "text" : "中华人民共和国国歌"
}

图形化界面：https://github.com/mobz/elasticsearch-head

cd elasticsearch-head
npm install
npm run start
open http://ip:9100/

与SpringBoot集成

注意版本：org.elasticsearch.client版本需要与安装es的版本一致，springboot2.1推荐安装6.4.3版本。

docker-compose

version: '3'
services:
  es11:
    image: docker.elastic.co/elasticsearch/elasticsearch:6.4.3
    container_name: es11
    environment:
      - cluster.name=docker-cluster
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata11:/usr/share/elasticsearch
    ports:
      - 9200:9200
      - 9300:9300

  es12:
    image: docker.elastic.co/elasticsearch/elasticsearch:6.4.3
    container_name: es12
    environment:
      - cluster.name=docker-cluster
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - "discovery.zen.ping.unicast.hosts=es11"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata12:/usr/share/elasticsearch

volumes:
  esdata11:
    driver: local
  esdata12:
    driver: local

networks:
  default:
    external:
      name: es

Maven

<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>

Java

application.yml

spring:
  data: # es配置
    elasticsearch:
      cluster-name: docker-cluster
      cluster-nodes: 192.168.40.200:9300
      repositories:
        enabled: true

cluster-name可通过http://ip:9200查看

文档

@Document(indexName = "project", type = "article")
public class Article {

    @Id
    private long id;
    private String title;
    private String content;
    ...

Repository

public interface ArticleRepository extends ElasticsearchRepository<Article, Long> {

    @Override
    Page<Article> findAll(Pageable pageable);

    Page<Article> findByTitle(String title, Pageable pageable);

    Page<Article> findByTitleOrContent(String title, String content, Pageable pageable);

}

Service

@Service
public class ArticleService {

@Autowired
private ArticleRepository articleRepository;

 public Page<Article> search(String title, String content, Pageable pageable) {
    return articleRepository.findByTitleOrContent(title, content, pageable);
 }
}

API

匹配查询(MatchQuery)

进行模糊匹配查询
MatchQueryBuilder matchQuery = QueryBuilders.matchQuery("title", "中国");

项查询(TermQuery)

完全匹配查询
TermQueryBuilder termQuery = QueryBuilders.termQuery("id", 2);

范围查询(Range Query)

范围查询，字数在 0-30w 之间的作品
RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("words").gt(0).lt(300000);

复合查询

BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();

MultiMatchQueryBuilder matchQuery = QueryBuilders.multiMatchQuery(query.getQueryString(), "name", "intro", "author");
boolQuery.must(matchQuery);
RangeQueryBuilder wordsQuery = QueryBuilders.rangeQuery("words").gt(query.getWordsBegin()).lt(query.getWordsEnd());
boolQuery.filter(wordsQuery);
TermQueryBuilder siteQuery = QueryBuilders.termQuery("site", query.getSite());
boolQuery.filter(siteQuery);

分词、拼音、高亮查询参见项目