ElasticSearch是一个基于Lucene的搜索服务器。它提供了一个分布式多用户能力的全文搜索引擎,基于RESTful web接口。Elasticsearch是用Java开发的,并作为Apache许可条款下的开放源码发布,是当前流行的企业级搜索引擎。设计用于云计算中,能够达到实时搜索,稳定,可靠,快速,安装使用方便,主要是可以存储特别大的数据量,查询速度特别快。利用logstash 进行数据同步,达到实时。Kibana图形化界面,方便我们进行数据查询,编写elk语句、创建index、type等。
我在项目中没有使用logstash,使用了一个定时器代替logstash,每次查询结束后在配置文件中定义标志位,定时器到了之后,下次数据库查询从标致位到目前时间的数据。将数据同步到es中,统计es中的数据,使用jest方法分组查询
概念直达:https://baike.so.com/doc/6990701-7213557.html
es:elasticsearch-5.6.3
下载地址:https://download.csdn.net/download/a909422229/10434208
kibana:kibana-5.6.3-windows-x86
下载地址:https://download.csdn.net/download/a909422229/10434239
logstash :logstash-5.6.3-
下载地址:官网。
package com.jarry.elasticsearch;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.SerializeConfig;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.fastjson.serializer.SimpleDateFormatSerializer;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
import java.util.concurrent.TimeUnit;
/**
* elasticsearch工具类
*
**/
public class ElasticSearchUtils {
/**
* es集群名称,默认为elasticsearch
*/
private static String clusterName;
/**
* es节点ip
*/
private static String esIpAddress;
public static void setClusterName(String clusterName) {
ElasticSearchUtils.clusterName = clusterName;
}
public static void setEsIpAddress(String esIpAddress) {
ElasticSearchUtils.esIpAddress = esIpAddress;
}
private static TransportClient client = null;
private static TimeValue searchTimeOut = new TimeValue(2, TimeUnit.SECONDS);
private static TimeValue updateTimeOut = new TimeValue(10, TimeUnit.SECONDS);
private static void initConnection() {
if (client == null) {
if (clusterName == null) {
// 从配置文件加载相关配置
ResourceBundle resourceBundle = ResourceBundle.getBundle("application_es");
clusterName = resourceBundle.getString("ES.clusterName");
esIpAddress = resourceBundle.getString("ES.ipAddress");
}
// 集群名只要不等于elasticsearch必须设置
// Settings settings = Settings.builder().put("cluster.name",
// clusterName).build();
try {
synchronized (ElasticSearchUtils.class) {
/*
* 9300端口号是本机客户端与ES通信的端口
* 本机和es通信有两种方式1:把本机当成一个es节点,2:直接通过客户端client通信 我使用的是第二中
*/
if (client == null) {
// 可以通过addTransportAddress(new
// InetSocketTransportAddress(InetAddress.getByName(esIpAddress),
// 9300))继续添加节点
client = new PreBuiltTransportClient(Settings.EMPTY).addTransportAddress(
new InetSocketTransportAddress(InetAddress.getByName(esIpAddress), 9300));
}
}
} catch (Exception e) {
System.out.println(e.getMessage());
}
}
}
/**
* 写入数据
*
* @param index
* @param type
* @param id
* @param json
* @return
* @throws Exception
*/
public static String putDocument(String index, String type, String id, String json) throws Exception {
initConnection();
if (index == null || type == null || id == null || json == null || "".equals(index) || "".equals(type)
|| "".equals(id) || "".equals(json)) {
throw new IllegalArgumentException("参数不对");
}
IndexResponse response = client.prepareIndex(index, type).setTimeout(updateTimeOut).setId(id)
.setSource(json, XContentType.JSON).get();
if (RestStatus.CREATED.getStatus() == response.status().getStatus()) {
return response.getId();
} else {
throw new RuntimeException("创建文档失败");
}
}
/**
* 批量写入数据
*
* @param index 索引
* @param type 类型
* @param objectList 集合数据
* @return
* @throws Exception
*/
public static String putListDocument(String index, String type, List objectList) throws Exception {
initConnection();
if (index == null || type == null || objectList.isEmpty() || "".equals(index) || "".equals(type)) {
throw new IllegalArgumentException("参数不对");
}
BulkRequestBuilder bulkRequest = client.prepareBulk().setTimeout(updateTimeOut);
SerializeConfig config = new SerializeConfig();
config.put(Date.class, new SimpleDateFormatSerializer("yyyy-MM-dd HH:mm:ss"));
for (Object item : objectList) {
String jsonStr = JSONObject.toJSONString(item, config, SerializerFeature.WriteMapNullValue);
JSONObject jObj = JSONObject.parseObject(jsonStr);
if (jObj.get("id") == null || "".equals(jObj.get("id"))) {
throw new RuntimeException("对象没有id信息");
}
bulkRequest.add(client.prepareIndex(index, type, jObj.get("id").toString()).setTimeout(updateTimeOut)
.setSource(jsonStr, XContentType.JSON));
}
BulkResponse bulkResponse = bulkRequest.setTimeout(updateTimeOut).get();
if (bulkResponse.hasFailures()) {
throw new RuntimeException("存在插入失败情况!");
} else {
return "success";
}
}
/**
* 根据ID查询数据
* @param index 索引:库
* @param type 类型:表
* @param id ID:唯一ID
* @return
*/
public static Map<String, Object> findSOBangById(String index, String type, String id) {
GetRequest rq = new GetRequest(index, type, id);
GetResponse response = client.get(rq).actionGet();
Map<String,Object> data = null;
// 判断非空
if (!response.isSourceEmpty()) {
data = response.getSource();
}
return data;
}
public static void searchAll() {
SearchResponse response = client.prepareSearch().get();
response.getHits().forEach(hit -> {
System.out.println(hit.getScore());
});
}
public static boolean bulkIndex(String index, String type, List<Map<String, Object>> list) {
BulkRequestBuilder bulkRequest = client.prepareBulk();
list.stream().forEach(map -> {
bulkRequest.add(
client.prepareIndex(index, type, String.valueOf(map.get("id"))).setSource(map));
});
System.out.println(list.size());
return !bulkRequest.get().hasFailures();
}
public static void searchByIndexType(String index, String type) {
SearchResponse response = client.prepareSearch(index).setTypes(type).get();
// logger.info(index + "-" + type);
System.out.println(index + "-" + type);
// logger.info(response);
System.out.println(response);
}
public static void main(String[] args) throws Exception {
initConnection();
// JSONObject jo = new JSONObject();
// jo.put("id", "150");
// jo.put("name", "xiaoming");
// JSONArray ja = new JSONArray();
// ja.add(jo);
//
// putDocument("a","a1","1",jo.toJSONString());
//
//批量插入到es中,不需要额外传入ID。
// Map<String, Object> map = findSOBangById("a","b","c");
// System.out.println(map);
// Map<String, Object> map2 = findSOBangById("a","b","c1");
Map<String, Object> map3= findSOBangById("a","b","c2");
// Map<String, Object> map3 =new HashMap<String, Object>();
// map3.put("c3", "666");
// List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
// list.add(map);
// list.add(map2);
// list.add(map3);
// boolean bulkIndex = bulkIndex("a","b",list);
// System.out.println(bulkIndex);
searchByIndexType("qh_query_index","qh_query_type");
// Map<String, Object> findSOBangById = findSOBangById("a","a1","1");
// System.out.println(findSOBangById);
//要求json中必须有ID属性
// JSONObject jo = new JSONObject();
// jo.put("id", "id");
// jo.put("a", "aaa");
// jo.put("b", "bbb");
// List<JSONObject> list = new ArrayList<JSONObject>();
// list.add(jo);
//
// String putListDocument = putListDocument("a","b",list);
// System.out.println(putListDocument);
}
}