异步入库es方法的编写

Survive by day and develop by night.
talk for import biz , show your perfect code,full busy，skip hardness,make a better result,wait for change,challenge Survive.
happy for hardess to solve denpendies.

目录

概述

网络爬虫的是一个非常常见的需求。

需求：

设计思路

实现思路分析

1.加上异步注解

2.控制层调用

下载器我们可以使用建立HTTP请求把界面的URL元素下载下来。实质就是
下载器。

3.注入Spring的bean

爬虫调度器就是可以利用多线程机制，进行调度似的更快的进行网页爬取。

4.常见代码：

package com.georsoft.search.controller.provider;


import com.alibaba.fastjson.JSONArray;
import com.georsoft.common.core.utils.DateUtils;

import com.georsoft.search.es.entity.ArcDocument;
import com.georsoft.search.index.convert.ArcDocCreateConvert;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;

import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Component
@Slf4j
public class AsyncTaskEs {

    @Autowired
    private ArcDocCreateConvert arcDocumentConvert;

    @SneakyThrows
    @Async
    public void doTask1(List<Map<String, Object>> addEsDataMapList) {
        long t1 = System.currentTimeMillis();
        addElasticsearchData(addEsDataMapList);
        long t2 = System.currentTimeMillis();
        log.info("task1 cost {} ms" , t2-t1);

    }


    @Value("${spring.elasticsearch.rest.nodes:10.100.13.105}")
    private String host;
    @Value("${spring.elasticsearch.rest.port:8083}")
    private String port;
    @Value("${spring.elasticsearch.rest.indexName:test}")
    private String arc_document;//索引名称

    /**
     * 创建连接 高级客户端
     */
    public RestHighLevelClient restHighLevelClient() {
//        RestHighLevelClient client = new RestHighLevelClient(
//                RestClient.builder(new HttpHost(host, Integer.parseInt(port), "http")));
        HttpHost host = new HttpHost("10.100.13.105",  8083, HttpHost.DEFAULT_SCHEME_NAME);

        RestClientBuilder builder = RestClient.builder(host);

        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("es", "111111"));
        builder.setHttpClientConfigCallback(f -> f.setDefaultCredentialsProvider(credentialsProvider));
        RestHighLevelClient client = new RestHighLevelClient(builder);
        return  client;
    }



        /**
         * Elasticsearch数据导入
         */
        public void addElasticsearchData(List<Map<String, Object>> addEsDataMapList) {
            //获取连接
            RestHighLevelClient client = restHighLevelClient();
            try {
                //创建请求
                BulkRequest bulkRequest = new BulkRequest();
                //创建index请求 千万注意，这个写在循环外侧，否则UDP协议会有丢数据的情况，看运气
                IndexRequest requestData = null;
                Map<Object,Object> dataMap=new HashMap<>();
                for (Map<String, Object> addEsDataMap : addEsDataMapList) {//添加数据
                        for (Map.Entry<String, Object> entry : addEsDataMap.entrySet()) {
                            dataMap.put(lineToHump(entry.getKey()),  entry.getValue());
                            dataMap.put("description","描述");
                            dataMap.put("year",2008);
                            dataMap.put("trxId","交易ID");
                            dataMap.put("contractNo","12332131");
                            dataMap.put("deadline",12332L);
                        }

                    ArcDocument arcDocument =arcDocumentConvert.convert(dataMap);

                    requestData=new IndexRequest(arc_document, "_doc", dataMap.get("arcId").toString()).source(arcDocument, XContentType.JSON);
                    bulkRequest.add(requestData);
                }
                log.info("es同步数据数量:{}", bulkRequest.numberOfActions());
                //设置索引刷新规则
                bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
                //分批次提交，数量控制
                if (bulkRequest.numberOfActions() >= 1) {
    //                BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
    //                log.info("es同步数据结果:{}", bulkResponse.hasFailures());

                    BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
                    if(bulkResponse.hasFailures()){
                        log.info("数据写入失败：{}",bulkResponse.buildFailureMessage());
                    }else {
                        log.info("实时消息es写入成功");

                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
                log.error("es同步数据执行失败:{}", addEsDataMapList);
            } finally {
                try {
                    client.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }


    public static String lineToHump(String str) {
        str = str.toLowerCase();
        Pattern linePattern = Pattern.compile("_(\\w)");
        Matcher matcher = linePattern.matcher(str);
        StringBuffer sb = new StringBuffer();
        while (matcher.find()) {
            matcher.appendReplacement(sb, matcher.group(1).toUpperCase());
        }
        matcher.appendTail(sb);
        return sb.toString();
    }


}

性能参数测试：

参考资料和推荐阅读

1. 暂无

欢迎阅读，各位老铁，如果对你有帮助，点个赞加个关注呗！~