进击的大数据（十三）数据工厂模块开发——java下的elasticsearch操作

项目我上传到github中，感兴趣的可以下载下来看看，希望大神可以在以后多多指点。

github代码地址：https://github.com/sjctt/spartacus

数据工厂模块主要功能是：将接收模块缓存到redis中的数据，进行基础分析后生成elasticsearch索引。

模块命名：spartacus_datafactory

关于springboot项目的创建，可以参考之前的文章：https://blog.csdn.net/FormatWindowsXP/article/details/108284275

1. pom.xml添加ES驱动

<!-- elasticsearch需要的jar -->
<dependency>
    <groupId>org.elasticsearch</groupId>
    <artifactId>elasticsearch</artifactId>
    <version>7.2.0</version>
</dependency>
<dependency>
    <groupId>org.elasticsearch.client</groupId>
    <artifactId>elasticsearch-rest-high-level-client</artifactId>
    <version>7.2.0</version>
</dependency>
		
<dependency>  
    <groupId>org.nlpcn</groupId>  
    <artifactId>elasticsearch-sql</artifactId>  
    <version>2.3.5.0</version>  
</dependency>

<dependency>
    <groupId>org.apache.logging.log4j</groupId>
    <artifactId>log4j-core</artifactId>
    <version>2.7</version>
</dependency>
<dependency>
    <groupId>org.apache.logging.log4j</groupId>
    <artifactId>log4j-api</artifactId>
    <version>2.7</version>
</dependency>
<!-- elasticsearch需要的jar结束 -->

2.ES操作对象封装

工厂模块只需要创建索引

package com.data.operate;

import java.io.InputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Properties;

import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig.Builder;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;

import spartacus_public.method.spartacus_debug;

import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback;
import org.elasticsearch.client.RestClientBuilder.RequestConfigCallback;

/**
 * @author Song
 * @category
 * @serial
 *【2019年8月12日】	建立对象
 */
public class es_dop 
{
	//#region 连接搜索引擎
	public RestHighLevelClient es_connection()
	{
		try 
		{
			InputStream is = getClass().getClassLoader().getResourceAsStream("config/es.Properties");
			Properties properties = new Properties();
			properties.load(is);
			String es_host = properties.getProperty("es_host");
			int es_port = Integer.parseInt(properties.getProperty("es_port"));
			String es_schema = properties.getProperty("es_schema");
			final int es_connectTimeOut = Integer.parseInt(properties.getProperty("es_connectTimeOut"));
			final int es_socketTimeOut = Integer.parseInt(properties.getProperty("es_socketTimeOut"));
			final int es_connectionRequestTimeOut = Integer.parseInt(properties.getProperty("es_connectionRequestTimeOut"));
			final int es_maxConnectNum = Integer.parseInt(properties.getProperty("es_maxConnectNum"));
			final int es_maxConnectPerRoute = Integer.parseInt(properties.getProperty("es_maxConnectPerRoute"));
			
			ArrayList<HttpHost> hostList = new ArrayList<HttpHost>();
			String[] hosts = es_host.split(",");
			for (String host : hosts) 
			{
				hostList.add(new HttpHost(host,es_port,es_schema));
			}
			
			RestClientBuilder builder = RestClient.builder(hostList.toArray(new HttpHost[0]));
			//异步httpclient连接延时配置
			builder.setRequestConfigCallback(new RequestConfigCallback() 
			{
				public Builder customizeRequestConfig(Builder requestConfigBuilder) 
				{
					requestConfigBuilder.setConnectTimeout(es_connectTimeOut);
					requestConfigBuilder.setSocketTimeout(es_socketTimeOut);
					requestConfigBuilder.setConnectionRequestTimeout(es_connectionRequestTimeOut);
					return requestConfigBuilder;
				}
			});
			// 异步httpclient连接数配置
			builder.setHttpClientConfigCallback(new HttpClientConfigCallback() 
			{
				public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) 
				{
					httpClientBuilder.setMaxConnTotal(es_maxConnectNum);
					httpClientBuilder.setMaxConnPerRoute(es_maxConnectPerRoute);
					return httpClientBuilder;
				}
			});
			RestHighLevelClient client = new RestHighLevelClient(builder);
			return client;
		} 
		catch (Exception e)
		{
			if(spartacus_debug.isdebug)
			{
				//记录debug日志
				spartacus_debug.writelog_txt("spartacus_datafactory[datafactory_service]:ES连接失败，"+e.getMessage());
			}
			return null;
		}
	}
	//#endregion
	//#region 单条创建索引
	public void createIndex(RestHighLevelClient client,String logJson)
	{
		try 
		{
			IndexRequest indexRequest = new IndexRequest();
			indexRequest.index("logsys_index");
			indexRequest.source(logJson,XContentType.JSON);
			client.index(indexRequest,RequestOptions.DEFAULT);
		} 
		catch (Exception e) 
		{
			System.out.println("触发catch："+e.getMessage());
			e.printStackTrace();
		}
	}
	//#endregion
	//#region 批量创建索引
	public void createIndex_many(RestHighLevelClient client,ArrayList<String> jsonlist)
	{
		try 
		{
			BulkRequest request = new BulkRequest();
			for (String jsonstr : jsonlist) 
			{
				IndexRequest indexRequest = new IndexRequest();
				indexRequest.index("logsys_index");
//				
//				String logJson = JSONObject.fromObject(logentity).toString();
				indexRequest.source(jsonstr,XContentType.JSON);
				request.add(indexRequest);
			}
			client.bulk(request, RequestOptions.DEFAULT);
		}
		catch (Exception e) 
		{
			System.out.println("触发catch："+e.getMessage());
		}
	}
	//#endregion
}

项目我上传到github中，感兴趣的可以下载下来看看，希望大神可以在以后多多指点。

github代码地址：https://github.com/sjctt/spartacus