java returnAddres_java读取elasticsearch全量数据并导出json文件

一：创建ES连接

import org.elasticsearch.client.transport.TransportClient;

import org.elasticsearch.common.settings.Settings;

import org.elasticsearch.common.transport.TransportAddress;

import org.elasticsearch.transport.client.PreBuiltTransportClient;

import java.io.IOException;

import java.net.InetAddress;

public class ElasticSearchUtil {

private static TransportClient client;

static {

try {

/*此处可穿件配置文件自行配置，代码中为方便直接写死

InputStream resourceAsStream = ElasticSearchUtil.class.getClassLoader().getResourceAsStream("elasticsearch.conf");*/

String host = "10.0.221.73";

int port = Integer.parseInt("9300");

String clusterName = "my-application";

Settings settings = Settings.builder()

.put("cluster.name", clusterName)

//.put("client.transport.sniff", true)

.build();

client = new PreBuiltTransportClient(settings);

//这里可以有多个，集群模式

TransportAddress ta = new TransportAddress(

InetAddress.getByName(host), port);

//addTransportAddresses(TransportAddress... transportAddress)，参数为一个可变参数

client.addTransportAddresses(ta);

} catch (IOException e) {

e.printStackTrace();

}

}

public static TransportClient getTransportClient() {

return client;

}

public static void close(TransportClient client) {

if (client != null) {

client.close();

}

}

}

二： 创建文件输出流

import java.io.FileOutputStream;

import java.io.IOException;

public class JsonFormatTool {

/**

* 将字符串追加到文件已有内容后面

*

* @param fileFullPath 文件完整地址：/es.json

* @param

content 需要写入的

*/

public static void writeFile(String fileFullPath,String content) {

FileOutputStream fos = null;

try {

//true不覆盖已有内容

fos = new FileOutputStream(fileFullPath, true);

//写入

fos.write(content.getBytes());

// 写入一个换行

fos.write("\r\n".getBytes());

} catch (IOException e) {

e.printStackTrace();

}finally{

if(fos != null){

try {

fos.flush();

fos.close();

} catch (IOException e) {

e.printStackTrace();

}

}

}

}

}

三：读取ES数据并调用输出流输出json文件

import org.elasticsearch.action.search.*;

import org.elasticsearch.client.Client;

import org.elasticsearch.common.unit.TimeValue;

import org.elasticsearch.index.query.QueryBuilders;

import org.elasticsearch.search.SearchHit;

import org.elasticsearch.search.sort.SortOrder;

import java.text.DateFormat;

import java.text.SimpleDateFormat;

import java.util.Date;

public class ScrollAll {

public static void main(String[] args) {

ScroolSearchID();

}

public static String ScroolSearchID() {

//动态获取日期作为生成json文件的后缀

Date date = new Date();

DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");

String suffix = dateFormat.format(date);

//获取ES连接

Client client = ElasticSearchUtil.getTransportClient();

//指定一个index和type

SearchRequestBuilder search = client.prepareSearch("binloghash").setTypes("binloghash");

//使用原生排序优化性能

search.addSort("_doc", SortOrder.ASC);

//设置每批读取的数据量,此处为测试环境为了测试批次设置值较小，生产可根据实际情况调参

search.setSize(10);

//默认是查询所有

search.setQuery(QueryBuilders.queryStringQuery("*:*"));

//设置 search context 维护1分钟的有效期

search.setScroll(TimeValue.timeValueMinutes(1));

//获得首次的查询结果

SearchResponse scrollResp = search.get();

//打印命中数量

System.out.println("命中总数量：" + scrollResp.getHits().getTotalHits());

//打印计数

int count = 1;

do {

System.out.println("第" + count + "次打印数据：");

//读取结果集数据

for (SearchHit hit : scrollResp.getHits().getHits()) {

//System.out.println(hit.getSourceAsString());

//获取ES数据

String esData = hit.getSourceAsString();

//按日期动态生成文件

String fileFullPath = "/tmp/es"+suffix+".json";

//调用输出流输出文件

JsonFormatTool.writeFile(fileFullPath,esData);

}

count++;

//将scorllId循环传递

scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(TimeValue.timeValueMinutes(1)).execute().actionGet();

//当searchHits的数组为空的时候结束循环，至此数据全部读取完毕

} while (scrollResp.getHits().getHits().length != 0);

return null;

}

}

四：maven依赖

4.0.0

com.my.es

EsToFile

jar

1.0-SNAPSHOT

EsToFile Maven Webapp

http://maven.apache.org

UTF-8

junit

junit

4.10

org.elasticsearch.client

transport

6.4.2

org.json

json

20180813

org.elasticsearch

elasticsearch-hadoop

6.4.2

org.apache.maven.plugins

maven-compiler-plugin

1.8

1.8

org.apache.maven.plugins

maven-jar-plugin

true

false

lib/

cn.mymaven.test.TestMain

总结：需要注意的点1. es7以后 TransportClient被废弃，此代码不适用7版本以后，es7的java代码，只能使用restclient，建议采用 High-level-rest-client 的方式操作ES集群。

2. 以上java连接ES,注意在ES集群中的elasticsearch.yml添加开放9300端口配置