java es游标查询，导出多个文件到压缩文件

1.资料1

package com.smk.es.servicce;
import org.elasticsearch.action.search.SearchResponse;

import org.elasticsearch.client.transport.TransportClient;

import org.elasticsearch.common.settings.Settings;

import org.elasticsearch.common.transport.TransportAddress;

import org.elasticsearch.common.unit.TimeValue;

import org.elasticsearch.index.query.BoolQueryBuilder;

import org.elasticsearch.index.query.QueryBuilders;

import org.elasticsearch.search.SearchHit;

import org.elasticsearch.transport.client.PreBuiltTransportClient;

import java.net.InetAddress;

import java.util.HashMap;

import java.util.Map;

public class TestEs {

private String clusterName ="es-smk-sit";

private String clusterNode = "192.168.23.10";

private String clusterPort ="9301";

private String poolSize = "10";

private boolean snf = true;

private String index = "smk-label";

private String type = "label";

public TransportClient transportClient() {

TransportClient transportClient = null;

try {

Settings esSetting = Settings.builder()

.put("cluster.name", clusterName) //集群名字

.put("client.transport.sniff", snf)//增加嗅探机制，找到ES集群

.put("thread_pool.search.size", Integer.parseInt(poolSize))//增加线程池个数，暂时设为5

.build();

//配置信息Settings自定义

transportClient = new PreBuiltTransportClient(esSetting);

TransportAddress transportAddress = new TransportAddress(InetAddress.getByName(clusterNode), Integer.valueOf(clusterPort));

transportClient.addTransportAddresses(transportAddress);

} catch (Exception e) {

e.printStackTrace();

System.out.println("elasticsearch TransportClient create error!!");

}

System.out.println("es客户端创建成功");

return transportClient;

}

public static String scrollId = "";

/**

* 第一次查询的方式

* @param client

* @return

*/

private Mapmy(TransportClient client){

BoolQueryBuilder mustQuery = QueryBuilders.boolQuery();

//设置查询条件

mustQuery.must(QueryBuilders.matchQuery("sex","男"));

mustQuery.must(QueryBuilders.matchQuery("city","杭州市"));

SearchResponse rep = client.prepareSearch()

.setIndices(index) // 索引

.setTypes(type) //类型

.setQuery(mustQuery)

.setScroll(TimeValue.timeValueMinutes(2)) //设置游标有效期

.setSize(100) //每页的大小

.execute()

.actionGet();

Mapm = new HashMap();

m.put("scrollId",rep.getScrollId());//获取返回的 游标值

m.put("id", (rep.getHits().getHits())[0].getId());

return m;

}

private Mapmy2(String scrollId,TransportClient client){

SearchResponse rep1 = client.prepareSearchScroll(scrollId) //设置游标

.setScroll(TimeValue.timeValueMinutes(2)) //设置游标有效期

.execute()

.actionGet();

Mapm = new HashMap();

m.put("scrollId",rep1.getScrollId());

SearchHit[] s = rep1.getHits().getHits();

if(s == null || s.length == 0){

return null;

}

m.put("id", (rep1.getHits().getHits())[0].getId());

return m;

}

public static void main(String[] args) {

TestEs t = new TestEs();

TransportClient client = t.transportClient();

Mapm1 = t.my(client);

System.out.println("first:"+m1.get("id"));

String s = m1.get("scrollId").toString();

System.out.println("first:"+s);

int i = 0;

while (true){

i++;

Mapm2 = t.my2(s,client);

// 查询不到数据了，就表示查询完了

if(m2 == null){

break;

}

System.out.println("insert to mysql");

}

System.out.println("总次数："+i);

System.out.println("end");

}

}

资料2

<dependency>
    <groupId>org.elasticsearch.client</groupId>
    <artifactId>elasticsearch-rest-high-level-client</artifactId>
    <version>7.4.2</version>
</dependency>

public static boolean getEs(String beginDate, String endDate) throws IOException {
        RestHighLevelClient client = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.100.100", 9201, "http")));
        //设定滚动时间间隔
        final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
        SearchRequest searchRequest = new SearchRequest("index");//索引库
        searchRequest.types("type");//索引表
        searchRequest.scroll(scroll);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        //设定每次返回多少条数据
        searchSourceBuilder.size(pageNum);
        //返回字段信息
        searchSourceBuilder.fetchSource(new String[]{"id", "x_dip","i_dport"}, new String[]{});

        //x_begintime 字段范围值过滤
        RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("x_begintime");
        rangeQueryBuilder.gte(beginDate);//开始时间
        rangeQueryBuilder.lte(endDate);//结束时间
        BoolQueryBuilder boolBuilder = QueryBuilders.boolQuery();
        boolBuilder.must(rangeQueryBuilder);
        searchSourceBuilder.query(boolBuilder);
        searchRequest.source(searchSourceBuilder);

        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        String scrollId = searchResponse.getScrollId();
        SearchHit[] searchHits = searchResponse.getHits().getHits();
        //把导出的结果以JSON的格式写到文件里,filePath=写入文件路径
        BufferedWriter out = new BufferedWriter(new FileWriter(filePath, true));
        
        System.out.println("-----"+searchHits.length+"-----");
        for (SearchHit searchHit : searchHits) {
            String json = searchHit.getSourceAsString();
            System.out.println(json);
            out.write(json);
            out.write("\r\n");
        }
        //遍历搜索命中的数据，直到没有数据
        while (searchHits != null && searchHits.length > 0) {
            SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
            scrollRequest.scroll(scroll);
            searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
            scrollId = searchResponse.getScrollId();
            searchHits = searchResponse.getHits().getHits();
            if (searchHits != null && searchHits.length > 0) {
                System.out.println("-----"+searchHits.length+"-----");
                for (SearchHit searchHit : searchHits) {
                    String json = searchHit.getSourceAsString();
                    System.out.println(json);
                    out.write(json);
                    out.write("\r\n");
                }
            }
        }

        //清除滚屏
        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
        //也可以选择setScrollIds()将多个scrollId一起使用
        clearScrollRequest.addScrollId(scrollId);
        ClearScrollResponse clearScrollResponse = null;
        try {
            clearScrollResponse = client.clearScroll(clearScrollRequest,RequestOptions.DEFAULT);
        } catch (IOException e) {
            e.printStackTrace();
        }
        boolean succeeded = clearScrollResponse.isSucceeded();
        if(succeeded){
            System.out.println(filePath);
        }
        out.close();
        client.close();
        //成功返回true
        return succeeded;
    }

文件压缩成文件夹

import java.io.*;
import java.util.zip.*;

public class ZipFiles {
    public static void main(String[] args) throws IOException {
        // 创建一个输出流，指向压缩文件
        FileOutputStream fos = new FileOutputStream("result.zip");
        // 创建一个 ZipOutputStream 对象，将输出流传递进去
        ZipOutputStream zos = new ZipOutputStream(fos);

        // 遍历要压缩的文件列表
        String[] filesToZip = new String[]{"file1.txt", "file2.txt", "file3.txt"};
        for (String file : filesToZip) {
            // 创建一个输入流，读取文件内容
            FileInputStream fis = new FileInputStream(file);
            // 创建一个 ZipEntry 对象，表示压缩文件中的一个条目
            ZipEntry zipEntry = new ZipEntry(file);
            // 将条目写入压缩文件
            zos.putNextEntry(zipEntry);

            // 将文件内容写入输出流
            byte[] buffer = new byte[1024];
            int len;
            while ((len = fis.read(buffer)) > 0) {
                zos.write(buffer, 0, len);
            }

            // 关闭当前条目
            zos.closeEntry();
            // 关闭输入流
            fis.close();
        }

        // 关闭 ZipOutputStream
        zos.close();
    }
}