doris stream load

最新推荐文章于 2024-09-26 10:31:07 发布

有机合成实验员

最新推荐文章于 2024-09-26 10:31:07 发布

阅读量608

点赞数

分类专栏： Doris 文章标签：大数据

本文链接：https://blog.csdn.net/doublezsx/article/details/104279309

版权

Doris 专栏收录该内容

1 篇文章 0 订阅

订阅专栏

package uhp;


import java.io.IOException;
import java.nio.charset.StandardCharsets;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.*;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;




public class HttpClientUtilStreamLoadDoris {


    private final static String DORIS_HOST_DEV = "xxx";

    //public static Logger logger = LoggerFactory.getLogger(HttpClientUtilStreamLoadDoris.class);

    //    private final static String DORIS_TABLE = "join_test";
    private final static String DORIS_USER = "xxx";
    private final static String DORIS_PASSWORD = "xxx";
    private final static int DORIS_HTTP_PORT = 8030;

    public void sendData(String content, String db, String table) throws Exception {

        final String loadUrl = String.format("http://%s:%s/api/%s/%s/_stream_load",
                DORIS_HOST_DEV,
                DORIS_HTTP_PORT,
                db,
                table);
        final HttpClientBuilder httpClientBuilder = HttpClients
                .custom().addInterceptorFirst(new ContentLengthHeaderRemover())
                .setRedirectStrategy(new DefaultRedirectStrategy() {
                    @Override
                    protected boolean isRedirectable(String method) {
                        return true;
                    }
                });
        CloseableHttpClient client = httpClientBuilder.build();


        HttpPut put = new HttpPut(loadUrl);
        StringEntity entity = new StringEntity(content, "UTF-8");
        put.setHeader(HttpHeaders.EXPECT, "100-continue");
        put.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader(DORIS_USER, DORIS_PASSWORD));

        //put.setHeader("strip_outer_array", "true");
        put.setHeader("format", "json");
        //put.setHeader("merge_type", "MERGE");
        //put.setHeader("delete", "canal_type=\"DELETE\"");
        put.setHeader("label", "39c25a5c-7000-496e-a98e-348a264c81dd");
        put.setEntity(entity);
        put.removeHeaders("Content-Length");


        reConnect(client, put);

    }

    private void reConnect(CloseableHttpClient client, HttpPut put) throws Exception {


        String loadResult = "";
        CloseableHttpResponse response = client.execute(put);
        //todo 调用方法
        if (response.getEntity() != null) {

            loadResult = EntityUtils.toString(response.getEntity());
        }
        final int statusCode = response.getStatusLine().getStatusCode();
        if (statusCode != 200) {
            System.out.println("写入失败");
        } else {
            if (loadResult.contains("OK") && loadResult.contains("Success")) {
                System.out.println(loadResult);
            } else if (loadResult.contains("Fail")) {
                throw new Exception(loadResult + ",抛出异常,任务失败，当前时间: " + System.currentTimeMillis());
            } else {
                throw new Exception(loadResult + ",抛出异常,任务失败，当前时间: " + System.currentTimeMillis());
            }

        }


    }

    private String basicAuthHeader(String username, String password) {
        final String tobeEncode = username + ":" + password;
        byte[] encoded = Base64.encodeBase64(tobeEncode.getBytes(StandardCharsets.UTF_8));
        return "Basic " + new String(encoded);
    }

    public static void main(String[] args) throws Exception {
        HttpClientUtilStreamLoadDoris httpClientUtilStreamLoadDoris = new HttpClientUtilStreamLoadDoris();
        httpClientUtilStreamLoadDoris.sendData("{\"event_day\":\"2021-03-24\",\"siteid\":1,\"citycode\":1,\"username\":\"杰a森\",\"pv\":23,\"uv\":66}","aac_test","aac_test_table2");

    }

    private static class ContentLengthHeaderRemover implements HttpRequestInterceptor{
        @Override
        public void process(HttpRequest request, HttpContext context) throws HttpException, IOException {
            request.removeHeaders("Content-Length");// fighting org.apache.http.protocol.RequestContent's ProtocolException("Content-Length header already present");
        }
    }
}