909422229_ELK使用

ElasticSearch是一个基于Lucene的搜索服务器。它提供了一个分布式多用户能力的全文搜索引擎，基于RESTful web接口。Elasticsearch是用Java开发的，并作为Apache许可条款下的开放源码发布，是当前流行的企业级搜索引擎。设计用于云计算中，能够达到实时搜索，稳定，可靠，快速，安装使用方便，主要是可以存储特别大的数据量，查询速度特别快。利用logstash 进行数据同步，达到实时。Kibana图形化界面，方便我们进行数据查询，编写elk语句、创建index、type等。

我在项目中没有使用logstash,使用了一个定时器代替logstash，每次查询结束后在配置文件中定义标志位，定时器到了之后，下次数据库查询从标致位到目前时间的数据。将数据同步到es中，统计es中的数据，使用jest方法分组查询

概念直达：https://baike.so.com/doc/6990701-7213557.html

es：elasticsearch-5.6.3 

下载地址：https://download.csdn.net/download/a909422229/10434208

kibana：kibana-5.6.3-windows-x86

下载地址：https://download.csdn.net/download/a909422229/10434239

logstash ：logstash-5.6.3-

下载地址：官网。

package com.jarry.elasticsearch;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.SerializeConfig;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.fastjson.serializer.SimpleDateFormatSerializer;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.transport.client.PreBuiltTransportClient;

import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
import java.util.concurrent.TimeUnit;

/**
 * elasticsearch工具类
 *
 **/
public class ElasticSearchUtils {

	/**
	 * es集群名称，默认为elasticsearch
	 */
	private static String clusterName;

	/**
	 * es节点ip
	 */
	private static String esIpAddress;

	public static void setClusterName(String clusterName) {
		ElasticSearchUtils.clusterName = clusterName;
	}

	public static void setEsIpAddress(String esIpAddress) {
		ElasticSearchUtils.esIpAddress = esIpAddress;
	}

	private static TransportClient client = null;

	private static TimeValue searchTimeOut = new TimeValue(2, TimeUnit.SECONDS);
	private static TimeValue updateTimeOut = new TimeValue(10, TimeUnit.SECONDS);

	private static void initConnection() {
		if (client == null) {
			if (clusterName == null) {
				// 从配置文件加载相关配置
				ResourceBundle resourceBundle = ResourceBundle.getBundle("application_es");
				clusterName = resourceBundle.getString("ES.clusterName");
				esIpAddress = resourceBundle.getString("ES.ipAddress");
			}
			// 集群名只要不等于elasticsearch必须设置
			// Settings settings = Settings.builder().put("cluster.name",
			// clusterName).build();
			try {
				synchronized (ElasticSearchUtils.class) {
					/*
					 * 9300端口号是本机客户端与ES通信的端口
					 * 本机和es通信有两种方式1：把本机当成一个es节点，2：直接通过客户端client通信 我使用的是第二中
					 */
					if (client == null) {
						// 可以通过addTransportAddress(new
						// InetSocketTransportAddress(InetAddress.getByName(esIpAddress),
						// 9300))继续添加节点
						client = new PreBuiltTransportClient(Settings.EMPTY).addTransportAddress(
								new InetSocketTransportAddress(InetAddress.getByName(esIpAddress), 9300));
					}
				}
			} catch (Exception e) {
				System.out.println(e.getMessage());
			}
		}
	}

	/**
	 * 写入数据
	 * 
	 * @param index
	 * @param type
	 * @param id
	 * @param json
	 * @return
	 * @throws Exception
	 */
	public static String putDocument(String index, String type, String id, String json) throws Exception {
		initConnection();
		if (index == null || type == null || id == null || json == null || "".equals(index) || "".equals(type)
				|| "".equals(id) || "".equals(json)) {
			throw new IllegalArgumentException("参数不对");
		}
		IndexResponse response = client.prepareIndex(index, type).setTimeout(updateTimeOut).setId(id)
				.setSource(json, XContentType.JSON).get();
		if (RestStatus.CREATED.getStatus() == response.status().getStatus()) {
			return response.getId();
		} else {
			throw new RuntimeException("创建文档失败");
		}
	}

	/**
	 * 批量写入数据
	 * 
	 * @param index 索引
	 * @param type 类型
	 * @param objectList 集合数据
	 * @return
	 * @throws Exception
	 */
	public static String putListDocument(String index, String type, List objectList) throws Exception {
		initConnection();
		if (index == null || type == null || objectList.isEmpty() || "".equals(index) || "".equals(type)) {
			throw new IllegalArgumentException("参数不对");
		}
		BulkRequestBuilder bulkRequest = client.prepareBulk().setTimeout(updateTimeOut);
		SerializeConfig config = new SerializeConfig();
		config.put(Date.class, new SimpleDateFormatSerializer("yyyy-MM-dd HH:mm:ss"));
		for (Object item : objectList) {
			String jsonStr = JSONObject.toJSONString(item, config, SerializerFeature.WriteMapNullValue);
			JSONObject jObj = JSONObject.parseObject(jsonStr);
			if (jObj.get("id") == null || "".equals(jObj.get("id"))) {
				throw new RuntimeException("对象没有id信息");
			}
			bulkRequest.add(client.prepareIndex(index, type, jObj.get("id").toString()).setTimeout(updateTimeOut)
					.setSource(jsonStr, XContentType.JSON));
		}
		BulkResponse bulkResponse = bulkRequest.setTimeout(updateTimeOut).get();
		if (bulkResponse.hasFailures()) {
			throw new RuntimeException("存在插入失败情况!");
		} else {
			return "success";
		}
	}

	/**
	 * 根据ID查询数据 
	 * @param index 索引：库
	 * @param type 类型：表
	 * @param id ID：唯一ID
	 * @return
	 */
	
	public static Map<String, Object> findSOBangById(String index, String type, String id) {
		
		GetRequest rq = new GetRequest(index, type, id);
		GetResponse response = client.get(rq).actionGet();
		Map<String,Object> data = null;
		// 判断非空
		if (!response.isSourceEmpty()) {
			data = response.getSource();
		}
		return data;
	}

	public static void searchAll() {
		SearchResponse response = client.prepareSearch().get();
		response.getHits().forEach(hit -> {
			System.out.println(hit.getScore());
		});
	}
	
	public static boolean bulkIndex(String index, String type, List<Map<String, Object>> list) {
        BulkRequestBuilder bulkRequest = client.prepareBulk();
        list.stream().forEach(map -> {
            bulkRequest.add(
                    client.prepareIndex(index, type, String.valueOf(map.get("id"))).setSource(map));
        });
        System.out.println(list.size());
        return !bulkRequest.get().hasFailures();
    }
	
	 public static void searchByIndexType(String index, String type) {
	        SearchResponse response = client.prepareSearch(index).setTypes(type).get();
//	        logger.info(index + "-" + type);
	        System.out.println(index + "-" + type);
//	        logger.info(response);
	        System.out.println(response);
	    }

	public static void main(String[] args) throws Exception {
		initConnection();
//		JSONObject jo = new JSONObject();
//		jo.put("id", "150");
//		jo.put("name", "xiaoming");
//		JSONArray ja = new JSONArray();
//		ja.add(jo);
//		
//		putDocument("a","a1","1",jo.toJSONString());
//		
		//批量插入到es中,不需要额外传入ID。
//		Map<String, Object> map = findSOBangById("a","b","c");
//		System.out.println(map);
//		Map<String, Object> map2 = findSOBangById("a","b","c1");
		Map<String, Object> map3= findSOBangById("a","b","c2");
//		Map<String, Object> map3 =new HashMap<String, Object>();
//		map3.put("c3", "666");
//		List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
//		list.add(map);
//		list.add(map2);
//		list.add(map3);
//		boolean bulkIndex = bulkIndex("a","b",list);
//		System.out.println(bulkIndex);
		
		searchByIndexType("qh_query_index","qh_query_type");

//		Map<String, Object> findSOBangById = findSOBangById("a","a1","1");
//		System.out.println(findSOBangById);
		
		//要求json中必须有ID属性
//		JSONObject jo = new JSONObject();
//		jo.put("id", "id");
//		jo.put("a", "aaa");
//		jo.put("b", "bbb");
//		List<JSONObject> list = new ArrayList<JSONObject>();
//		list.add(jo);
//		
//		String putListDocument = putListDocument("a","b",list);
//		System.out.println(putListDocument);
		
	}
}