elasticsearch+logstash+kibana(ELK)入门

Elasticsearch官方文档地址:https://www.elastic.co/guide/en/elasticsearch/client/java-rest/7.x/java-rest-high-document-index.html
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Logstash官网文档地址:https://www.elastic.co/guide/en/logstash/7.x/index.html

一、Elasticsearch基础入门

Java SpringBoot使用Elasticsearch

1. Config

package com.ola.elasticsearchstudy.config;

import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class ElasticSearchConfig {
    @Bean
    public RestHighLevelClient restHighLevelClient() {
        RestHighLevelClient client = new RestHighLevelClient(RestClient.builder(
                        new HttpHost("127.0.0.1", 31174, "http")));
        return client;
    }
}

2. 对index的操作

package com.ola.elasticsearchstudy;

import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.client.indices.GetIndexResponse;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.test.context.SpringBootTest;

import java.io.IOException;

@SpringBootTest
class ElasticSearchIndexTest {

    @Autowired
    @Qualifier("restHighLevelClient")
    private RestHighLevelClient client;

    /**
     * 创建索引
     */
    @Test
    void createIndex() throws IOException {
        CreateIndexRequest request = new CreateIndexRequest("ola_test_index");
        CreateIndexResponse response = client.indices().create(request, RequestOptions.DEFAULT);
        System.out.println(response);
    }

    /**
     * 获取索引(如果存在)
     */
    @Test
    void getIndex() throws IOException {
        GetIndexRequest request = new GetIndexRequest("ola_test_index");
        boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
        if (exists){
            GetIndexResponse response = client.indices().get(request, RequestOptions.DEFAULT);
            System.out.println(response);
        }else {
            System.out.println("索引不存在");
        }
    }

    /**
     * 删除索引
     * @throws IOException
     */
    @Test
    void deleteIndex() throws IOException {
        DeleteIndexRequest request = new DeleteIndexRequest("ola_test_index");
        AcknowledgedResponse response = null;
        try {
            response = client.indices().delete(request, RequestOptions.DEFAULT);
            if (response.isAcknowledged()) {
                System.out.println("删除成功");
            }else {
                System.out.println("删除失败");
            }
            System.out.println(response);
        } catch (ElasticsearchStatusException e) {
            System.out.println("该索引不存在");
        }
    }
}

3. 对document的操作

package com.ola.elasticsearchstudy;

import com.alibaba.fastjson.JSON;
import com.ola.elasticsearchstudy.pojo.User;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.test.context.SpringBootTest;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

@SpringBootTest
class ElasticSearchDocumentTest {
    @Autowired
    @Qualifier("restHighLevelClient")
    private RestHighLevelClient client;

    /**
     * 添加文档
     *
     * @throws IOException
     */
    @Test
    void addDocument() throws IOException {
        User user = new User("ola", 22, "handsome");
        IndexRequest request = new IndexRequest("ola_test_index");
        request.id("1");
        request.source(JSON.toJSONString(user), XContentType.JSON);
        IndexResponse response = client.index(request, RequestOptions.DEFAULT);
        System.out.println(response.toString());
        System.out.println("response.status() = " + response.status());
    }

    /**
     * 批量添加文档
     *
     * @throws IOException
     */
    @Test
    void addDocumentBulk() throws IOException {
        BulkRequest request = new BulkRequest();
        List<User> userList = new ArrayList<>();
        userList.add(new User("ola", 22, "handsome"));
        userList.add(new User("xiaoming", 21, "clever"));
        userList.add(new User("xiaohong", 12, "pretty"));
        for (User user : userList) {
            request.add(new IndexRequest("ola_test_index").source(JSON.toJSONString(user), XContentType.JSON));
        }
        BulkResponse response = client.bulk(request, RequestOptions.DEFAULT);
        System.out.println("response = " + response.toString());
        System.out.println("response.status() = " + response.status());
    }

    /**
     * 获取文档信息(如果存在)
     *
     * @throws IOException
     */
    @Test
    void getDocument() throws IOException {
        GetRequest request = new GetRequest("ola_test_index", "1");
        // 是否返回_source
        // request.fetchSourceContext(new FetchSourceContext(false));
        // request.storedFields("_none_");
        boolean exists = client.exists(request, RequestOptions.DEFAULT);
        if (exists) {
            GetResponse response = client.get(request, RequestOptions.DEFAULT);
            System.out.println(response.toString());
            System.out.println("response.getIndex() = " + response.getIndex());
            System.out.println("response.getType() = " + response.getType());
            System.out.println("response.getId() = " + response.getId());
            System.out.println("response.getVersion() = " + response.getVersion());
            System.out.println("response.getSeqNo() = " + response.getSeqNo());
            System.out.println("response.getPrimaryTerm() = " + response.getPrimaryTerm());
            System.out.println("response.getSource() = " + response.getSource());
            System.out.println("response.getSourceAsString() = " + response.getSourceAsString());
            Map<String, Object> map = response.getSourceAsMap();
            System.out.println("map = " + map);
        } else {
            System.out.println("没有该条文档");
        }
    }

    /**
     * 更新文档
     *
     * @throws IOException
     */
    @Test
    void updateDocument() throws IOException {
        UpdateRequest request = new UpdateRequest("ola_test_index", "1");
        User user = new User("ola", 22, "handsome and clever");
        request.doc(JSON.toJSONString(user), XContentType.JSON);
        UpdateResponse response = client.update(request, RequestOptions.DEFAULT);
        System.out.println("response = " + response.toString());
        System.out.println("response.status() = " + response.status());
    }

    /**
     * 删除文档
     *
     * @throws IOException
     */
    @Test
    void deleteDocument() throws IOException {
        DeleteRequest request = new DeleteRequest("ola_test_index", "1");
        DeleteResponse response = client.delete(request, RequestOptions.DEFAULT);
        System.out.println("response = " + response.toString());
        System.out.println("response.status() = " + response.status());
    }

    /**
     * 查询文档
     *
     * @throws IOException
     */
    @Test
    void searchDocument() throws IOException {
        SearchRequest request = new SearchRequest("ola_test_index");
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        // 精确搜索
        TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("name", "ola");
        MatchAllQueryBuilder matchAllQueryBuilder = QueryBuilders.matchAllQuery();
        sourceBuilder.query(matchAllQueryBuilder);
        sourceBuilder.from(0);
        sourceBuilder.size(2);
        request.source(sourceBuilder);
        SearchResponse response = client.search(request, RequestOptions.DEFAULT);
        System.out.println("response = " + response.toString());
        System.out.println("response.getHits() = " + JSON.toJSONString(response.getHits()));
        for (SearchHit hit : response.getHits().getHits()) {
            Map<String, Object> sourceAsMap = hit.getSourceAsMap();
            System.out.println(sourceAsMap);
        }
    }
}

二、Logstash入门

input{
   rabbitmq{
	host=>"ip"                             # 这里填写Rabbitmq的地址，确保可以ping通
	port=> 5672                            # 这里填写Rabbitmq的端口
	user=>"guest"                          # 这里填写Rabbitmq的用户名
	password=>"guest"                      # 这里填写Rabbitmq的密码
	queue=>"queuename"                     # 这里填写Rabbitmq的队列的名称
	durable=> true                         # 这里填写Rabbitmq的队列的durable属性
	codec=>json                            # 这里填写Rabbitmq的队列的内容是什么格式
	type=> "result"                        # 这里选填
  }
}
filter{
	if([messageType] not in "2,3"){        # 过滤条件，可以不要
		drop{}	
	}		
}
output {
	elasticsearch {
	hosts => ["127.0.0.1:9200"]          # ElasticSearch的地址加端口
 	index => "position-%{+YYYYMMdd}"        # ElasticSearch的保存文档的index名称，
	document_type=>"%{messageType}"         # ElasticSearch的保存文档的type
	document_id=>"%{mark_uuid}"             # ElasticSearch的保存文档的id
	flush_size => 500                       # ElasticSearch的保存文档的多少条提交保存
  	idle_flush_time => 10                   # ElasticSearch的保存文档的多少秒提交保存
  }
}