Doris数据同步：stream load Java工具类实现

最新推荐文章于 2024-05-13 11:59:18 发布

life is ftc

最新推荐文章于 2024-05-13 11:59:18 发布

阅读量2.1k

点赞数

文章标签： java 数据库开发语言 doris

本文链接：https://blog.csdn.net/StringBuff/article/details/126640421

版权

导入原理自己百度吧，工具类需要引入一个依赖

<dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>${httpcomponents.version}</version>
        </dependency>

工具类：

1.导入csv文件，文件格式如下：

1,2,,3,4

1,2,3,4,5

public class DorisStreamLoader {
    /**
     * 用户名
     */
    private final String user ;
    /**
     * 密码
     */
    private final String password;

    /**
     * doris stream load url
     */
    private final String loadUrl ;

    /**
     *
     * @param host FE ip地址
     * @param port FE 端口
     * @param database 数据库名
     * @param table 表名
     * @param user 用户名
     * @param password 密码
     */
    public DorisStreamLoader(String host, int port, String database, String table, String user, String password) {
        this.user = user;
        this.password = password;
        this.loadUrl = String.format("http://%s:%s/api/%s/%s/_stream_load", host, port, database, table);
    }

    /**
     * 构建HTTP客户端
     */
    private final static HttpClientBuilder httpClientBuilder = HttpClients
            .custom()
            .setRedirectStrategy(new DefaultRedirectStrategy() {
                @Override
                protected boolean isRedirectable(String method) {
                    // If the connection target is FE, you need to deal with 307 redirect。
                    return true;
                }
            });

    /**
     * 文件数据导入
     * @param inputStream 要导入的文件流
     * @throws Exception Exception
     */
    public void load(InputStream inputStream,String label ) throws Exception {
        try (CloseableHttpClient client = httpClientBuilder.build()) {
            HttpPut put = new HttpPut(loadUrl);
            put.removeHeaders(HttpHeaders.CONTENT_LENGTH);
            put.removeHeaders(HttpHeaders.TRANSFER_ENCODING);
            put.setHeader(HttpHeaders.EXPECT, "100-continue");
            put.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader(user, password));

            // You can set stream load related properties in the Header, here we set label and column_separator.
            put.setHeader("label", label);
            put.setHeader("column_separator", ",");
            // Set up the import file. Here you can also use StringEntity to transfer arbitrary data.
            InputStreamEntity entity = new InputStreamEntity(inputStream);
            put.setEntity(entity);
            log.info("开始同步doris,url={},label={}",loadUrl,label);
            try (CloseableHttpResponse response = client.execute(put)) {
                String loadResult = "";
                if (response.getEntity() != null) {
                    loadResult = EntityUtils.toString(response.getEntity());
                }

                final int statusCode = response.getStatusLine().getStatusCode();
                if (statusCode != 200) {
                    throw new IOException(String.format("Stream load failed. status: %s load result: %s", statusCode, loadResult));
                }
                log.info("Get load result: {}" , loadResult);
            }
        }
    }

    /**
     * JSON格式的数据导入
     * @param jsonData String
     * @throws Exception Exception
     */
    public void loadJson(String jsonData) throws Exception {
        try (CloseableHttpClient client = httpClientBuilder.build()) {
            HttpPut put = new HttpPut(loadUrl);
            put.removeHeaders(HttpHeaders.CONTENT_LENGTH);
            put.removeHeaders(HttpHeaders.TRANSFER_ENCODING);
            put.setHeader(HttpHeaders.EXPECT, "100-continue");
            put.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader(user, password));

            // You can set stream load related properties in the Header, here we set label and column_separator.
            put.setHeader("label", UUID.randomUUID().toString());
            put.setHeader("column_separator", ",");
            put.setHeader("format", "json");

            // Set up the import file. Here you can also use StringEntity to transfer arbitrary data.
            StringEntity entity = new StringEntity(jsonData);
            put.setEntity(entity);

            try (CloseableHttpResponse response = client.execute(put)) {
                String loadResult = "";
                if (response.getEntity() != null) {
                    loadResult = EntityUtils.toString(response.getEntity());
                }

                final int statusCode = response.getStatusLine().getStatusCode();
                if (statusCode != 200) {
                    throw new IOException(String.format("Stream load failed. status: %s load result: %s", statusCode, loadResult));
                }
                log.info("Get load result: {}" , loadResult);
            }
        }
    }

    /**
     * 封装认证信息
     * @param username String
     * @param password String
     * @return String
     */
    private String basicAuthHeader(String username, String password) {
        final String tobeEncode = username + ":" + password;
        byte[] encoded = Base64.encodeBase64(tobeEncode.getBytes(StandardCharsets.UTF_8));
        return "Basic " + new String(encoded);
    }


    public static void main(String[] args) throws Exception {
        DorisStreamLoader streamLoader = new DorisStreamLoader("zzxczxc", 8888, "database", "table", "user", "password");

    }
}