Kudu之java操作

最新推荐文章于 2024-08-02 03:51:30 发布

风情客家__

最新推荐文章于 2024-08-02 03:51:30 发布

阅读量940

点赞数

分类专栏： Kudu 文章标签： kudu java

原文链接：https://blog.csdn.net/a376554764/article/details/89445319

版权

Kudu 专栏收录该内容

22 篇文章 5 订阅

订阅专栏

参考：kudu原理与使用

java操作kudu数据库

导入依赖

<!--
<dependency>
  <groupId>org.apache.kudu</groupId>
  <artifactId>kudu-client</artifactId>
  <version>${kudu.version}</version>
  <scope>test</scope>
</dependency>
-->

<!--导入kudu的客户端工具-->
<dependency>
  <groupId>org.apache.kudu</groupId>
  <artifactId>kudu-client-tools</artifactId>
  <version>${kudu.version}</version>
</dependency>

方案一(use)

KuduUtils

package org.fiend.kudutest;

import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import org.apache.kudu.ColumnSchema;
import org.apache.kudu.Schema;
import org.apache.kudu.Type;
import org.apache.kudu.client.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;

/**
 * @author langpf 2020-05-07 21:57:47
 */
public class KuduUtils {
    private static final Logger log = LoggerFactory.getLogger(KuduUtils.class);

    private static final ThreadLocal<KuduSession> threadLocal = new ThreadLocal<>();

    /**
     * @param tableName      tableName
     * @param kuduColumnList column
     * @param buckets        分区数目
     * @param copies         副本数
     * @throws KuduException e
     */
    public static boolean createTable(String tableName,
                                      List<KuduColumn> kuduColumnList,
                                      int buckets,
                                      int copies) throws KuduException {
        // 1、获取client
        KuduClient client = Kudu.INSTANCE.client();

        if (client.tableExists(tableName)) {
            log.warn("表" + tableName + "已存在, 放弃创建表。");
            return false;
        }

        // 2、创建schema信息
        List<ColumnSchema> columns = Lists.newArrayList();

        // 3、指定分区字段
        List<String> partitions = Lists.newArrayList();

        kuduColumnList.forEach(x -> {
            // columns.add(new ColumnSchema.ColumnSchemaBuilder("id", Type.INT32).key(true).nullable(false).build());
            columns.add(
                    new ColumnSchema
                            .ColumnSchemaBuilder(x.getName(), x.getType())
                            .key(x.isKey())
                            .nullable(x.isNullable())
                            .build()
            );

            if (x.isPartitionKey()) {
                partitions.add(x.getName());
            }
        });
        Schema schema = new Schema(columns);

        //4、指定分区方式为hash分区、6个分区，一个副本
        // CreateTableOptions options = new CreateTableOptions().addHashPartitions(partitions, buckets).setNumReplicas(1);
        CreateTableOptions options = new CreateTableOptions()
                .addHashPartitions(partitions, buckets)
                .setNumReplicas(copies);

        //5、创建表，
        client.createTable(tableName, schema, options);

        return true;
    }

    public static void deleteTable(String tableName) throws KuduException {
        KuduClient client = Kudu.INSTANCE.client();
        if (!client.tableExists(tableName)) {
            return;
        }
        client.deleteTable(tableName);
    }

    /**
     * 添加空列
     * @param tableName table name
     * @param colName column name
     * @param type column type
     * @throws KuduException e
     * @throws InterruptedException i
     */
    public static boolean addNullableColumn(String tableName,
                                            String colName,
                                            Type type) throws KuduException, InterruptedException {
        if (existColumn(tableName, colName)) {
            log.error("表{}中的columnName: {}已存在!", tableName, colName);
            return false;
        }

        KuduClient client = Kudu.INSTANCE.client();
        AlterTableOptions ato = new AlterTableOptions();
        ato.addNullableColumn(colName, type);
        client.alterTable(tableName, ato);
        Kudu.INSTANCE.flushTables(tableName);

        int totalTime = 0;
        while (!client.isAlterTableDone(tableName)) {
            Thread.sleep(200);
            if (totalTime > 20000) {
                log.warn("Alter table is Not Done!");
                return false;
            }
            totalTime += 200;
        }

        return true;
    }

    /**
     * 添加有默认值的列
     */
    public static boolean addDefaultValColumn(String tableName,
                                    String colName,
                                    Type type,
                                    Object defaultVal) throws KuduException, InterruptedException {
        if (existColumn(tableName, colName)) {
            log.error("表{}中的columnName: {}已存在!", tableName, colName);
            return false;
        }

        KuduClient client = Kudu.INSTANCE.client();
        AlterTableOptions ato = new AlterTableOptions();
        ato.addColumn(colName, type, defaultVal);
        client.alterTable(tableName, ato);
        Kudu.INSTANCE.flushTables(tableName);

        int totalTime = 0;
        while (!client.isAlterTableDone(tableName)) {
            Thread.sleep(200);
            if (totalTime > 20000) {
                log.warn("Alter table is Not Done!");
                return false;
            }
            totalTime += 200;
        }

        return true;
    }

    public static boolean existColumn(String tableName, String colName) throws KuduException {
        Kudu.INSTANCE.flushTables(tableName);
        KuduTable ktable = Kudu.INSTANCE.getTable(tableName);

        Schema schema = ktable.getSchema();
        List<ColumnSchema> columns = schema.getColumns();
        for (ColumnSchema c : columns) {
            if (c.getName().equals(colName)) {
                return true;
            }
        }

        return false;
    }

    public static void insert(String tableName, JSONObject data) throws KuduException {
        Insert insert = createInsert(tableName, data);

        KuduSession session = getSession();
        session.apply(insert);
        session.flush();

        closeSession();
    }

    public static void upsert(String tableName, JSONObject data) throws KuduException {
        Upsert upsert = createUpsert(tableName, data);
        KuduSession session = getSession();
        session.apply(upsert);
        session.flush();

        closeSession();
    }

    public static void update(String tableName, JSONObject data) throws KuduException {
        Update update = createUpdate(tableName, data);
        KuduSession session = getSession();
        session.apply(update);
        session.flush();

        closeSession();
    }

    /**
     * @param tableName        表名
     * @param selectColumnList 查询字段名 为空时返回全部字段
     * @param columnCondList   条件列 可为空
     * @return data
     * @throws KuduException k
     */
    public static List<JSONObject> query(String tableName,
                                         List<String> selectColumnList,
                                         List<ColumnCond> columnCondList) throws KuduException {
        List<JSONObject> dataList = Lists.newArrayList();

        KuduTable ktable = Kudu.INSTANCE.getTable(tableName);
        if (null == ktable) {
            return null;
        }

        if (null == selectColumnList) {
            selectColumnList = Lists.newArrayList();
        }
        if (selectColumnList.size() < 1) {
            selectColumnList = getColumnList(ktable);
        }

        KuduScanner.KuduScannerBuilder kuduScannerBuilder = 
                Kudu.INSTANCE.client().newScannerBuilder(ktable);
        kuduScannerBuilder.setProjectedColumnNames(selectColumnList);

        /*
         * 设置搜索的条件 where 条件过滤字段名
         * 如果不设置，则全表扫描
         */
        if ((null != columnCondList) && (columnCondList.size() > 0)) {
            KuduPredicate predicate;
            for (ColumnCond cond : columnCondList) {
                predicate = getKuduPredicate(ktable, cond);
                if (null != predicate) {
                    kuduScannerBuilder.addPredicate(predicate);
                }
            }
        }

        KuduScanner scanner = kuduScannerBuilder.build();
        while (scanner.hasMoreRows()) {
            RowResultIterator rows = scanner.nextRows();

            // 每次从tablet中获取的数据的行数, 如果查询不出数据返回0
            int numRows = rows.getNumRows();
            if (numRows > 10000) {
                log.error("查询数据条数: {}, 大于10000条, 数据量过载!", numRows);
                break;
            }

            while (rows.hasNext()) {
                dataList.add(getRowData(rows.next(), selectColumnList));
            }
        }

        //7、关闭client
        Kudu.INSTANCE.client().close();

        return dataList;
    }

    public static void delete(String tableName, JSONObject data) throws KuduException {
        Delete delete = createDelete(tableName, data);
        KuduSession session = getSession();

        session.apply(delete);
        session.flush();
        closeSession();

        Kudu.INSTANCE.client().close();
    }

    private static Insert createInsert(String tableName, JSONObject data) throws KuduException {
        KuduTable ktable = Kudu.INSTANCE.getTable(tableName);

        Insert insert = ktable.newInsert();
        PartialRow row = insert.getRow();

        Schema schema = ktable.getSchema();
        for (String colName : data.keySet()) {
            ColumnSchema colSchema = schema.getColumn(colName);
            fillRow(row, colSchema, data);
        }

        return insert;
    }

    private static Insert createEmptyInsert(String tableName) throws KuduException {
        KuduTable ktable = Kudu.INSTANCE.getTable(tableName);
        return ktable.newInsert();
    }

    private static Upsert createUpsert(String tableName, JSONObject data) throws KuduException {
        KuduTable ktable = Kudu.INSTANCE.getTable(tableName);

        Upsert upsert = ktable.newUpsert();
        PartialRow row = upsert.getRow();
        Schema schema = ktable.getSchema();
        for (String colName : data.keySet()) {
            ColumnSchema colSchema = schema.getColumn(colName);
            fillRow(row, colSchema, data);
        }

        return upsert;
    }

    private static Upsert createEmptyUpsert(String tableName) throws KuduException {
        KuduTable ktable = Kudu.INSTANCE.getTable(tableName);
        return ktable.newUpsert();
    }

    private static Update createUpdate(String tableName, JSONObject data) throws KuduException {
        KuduTable ktable = Kudu.INSTANCE.getTable(tableName);

        Update update = ktable.newUpdate();
        PartialRow row = update.getRow();
        Schema schema = ktable.getSchema();
        for (String colName : data.keySet()) {
            ColumnSchema colSchema = schema.getColumn(colName);
            fillRow(row, colSchema, data);
        }

        return update;
    }

    private static Update createEmptyUpdate(String tableName) throws KuduException {
        KuduTable ktable = Kudu.INSTANCE.getTable(tableName);
        return ktable.newUpdate();
    }

    private static Delete createDelete(String tableName, JSONObject data) throws KuduException {
        KuduTable ktable = Kudu.INSTANCE.getTable(tableName);
        Delete delete = ktable.newDelete();
        PartialRow row = delete.getRow();

        Schema schema = ktable.getSchema();
        for (String colName : data.keySet()) {
            ColumnSchema colSchema = schema.getColumn(colName);
            fillRow(row, colSchema, data);
        }

        return delete;
    }

    private static Delete createEmptyDelete(String tableName) throws KuduException {
        KuduTable ktable = Kudu.INSTANCE.getTable(tableName);
        return ktable.newDelete();
    }

    private static void fillRow(PartialRow row, ColumnSchema colSchema, JSONObject data) {
        String name = colSchema.getName();
        if (data.get(name) == null) {
            return;
        }
        Type type = colSchema.getType();
        switch (type) {
            case STRING:
                row.addString(name, data.getString(name));
                break;
            case INT64:
            case UNIXTIME_MICROS:
                row.addLong(name, data.getLongValue(name));
                break;
            case DOUBLE:
                row.addDouble(name, data.getDoubleValue(name));
                break;
            case INT32:
                row.addInt(name, data.getIntValue(name));
                break;
            case INT16:
                row.addShort(name, data.getShortValue(name));
                break;
            case INT8:
                row.addByte(name, data.getByteValue(name));
                break;
            case BOOL:
                row.addBoolean(name, data.getBooleanValue(name));
                break;
            case BINARY:
                row.addBinary(name, data.getBytes(name));
                break;
            case FLOAT:
                row.addFloat(name, data.getFloatValue(name));
                break;
            default:
                break;
        }
    }

    private static JSONObject getRowData(RowResult row, List<String> selectColumnList) {
        JSONObject dataJson = new JSONObject();

        selectColumnList.forEach(x -> {
            Type type = row.getColumnType(x);
            switch (type) {
                case STRING:
                    dataJson.put(x, row.getString(x));
                    break;
                case INT64:
                case UNIXTIME_MICROS:
                    dataJson.put(x, row.getLong(x));
                    break;
                case DOUBLE:
                    dataJson.put(x, row.getDouble(x));
                    break;
                case FLOAT:
                    dataJson.put(x, row.getFloat(x));
                    break;
                case INT32:
                case INT16:
                case INT8:
                    dataJson.put(x, row.getInt(x));
                    break;
                case BOOL:
                    dataJson.put(x, row.getBoolean(x));
                    break;
                case BINARY:
                    dataJson.put(x, row.getBinary(x));
                    break;

                default:
                    break;
            };
        });

        return dataJson;
    }

    private static KuduPredicate getKuduPredicate(KuduTable ktable, ColumnCond cond) {
        String colName = cond.getColName();
        KuduPredicate.ComparisonOp op = cond.getOp();
        Object objVal = cond.getValue();

        if (objVal instanceof Boolean) {
            return KuduPredicate.newComparisonPredicate(
                    ktable.getSchema().getColumn(colName),
                    op,
                    (boolean) cond.getValue());
        }
        if (objVal instanceof Integer) {
            return KuduPredicate.newComparisonPredicate(
                    ktable.getSchema().getColumn(colName),
                    op, ((Integer)cond.getValue()).longValue());
        }
        if (objVal instanceof Long) {
            return KuduPredicate.newComparisonPredicate(
                    ktable.getSchema().getColumn(colName),
                    op, (Long) cond.getValue());
        }
        if (objVal instanceof Short) {
            return KuduPredicate.newComparisonPredicate(
                    ktable.getSchema().getColumn(colName),
                    op, ((Short) cond.getValue()).longValue());
        }
        if (objVal instanceof Byte) {
            return KuduPredicate.newComparisonPredicate(
                    ktable.getSchema().getColumn(colName),
                    op, ((Byte) cond.getValue()).longValue());
        }
        if (objVal instanceof Float) {
            return KuduPredicate.newComparisonPredicate(
                    ktable.getSchema().getColumn(colName),
                    op,
                    (float) cond.getValue());
        }
        if (objVal instanceof Double) {
            return KuduPredicate.newComparisonPredicate(
                    ktable.getSchema().getColumn(colName),
                    op,
                    (double) cond.getValue());
        }
        if (objVal instanceof String) {
            return KuduPredicate.newComparisonPredicate(
                    ktable.getSchema().getColumn(colName),
                    op,
                    (String) cond.getValue());
        }
        if (objVal instanceof byte[]) {
            return KuduPredicate.newComparisonPredicate(
                    ktable.getSchema().getColumn(colName),
                    op,
                    (byte[]) cond.getValue());
        }

        return null;
    }

    private static KuduSession getSession() throws KuduException {
        KuduSession session = threadLocal.get();
        if (session == null) {
            session = Kudu.INSTANCE.newSession();
            threadLocal.set(session);
        }
        return session;
    }

    private static KuduSession getAsyncSession() {
        KuduSession session = threadLocal.get();
        if (session == null) {
            session = Kudu.INSTANCE.newAsyncSession();
            threadLocal.set(session);
        }
        return session;
    }

    private static void closeSession() {
        KuduSession session = threadLocal.get();
        threadLocal.set(null);
        Kudu.INSTANCE.closeSession(session);
    }

    private static List<String> getColumnList(KuduTable ktable) {
        List<String> columns = Lists.newArrayList();

        List<ColumnSchema> columnSchemaList = ktable.getSchema().getColumns();
        columnSchemaList.forEach(x -> {
            columns.add(x.getName());
        });

        return columns;
    }
}

Kudu

package org.fiend.kudutest;

import org.apache.kudu.client.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.Map;

public enum Kudu {
    INSTANCE;

    private KuduClient client = null;
    private Map<String, KuduTable> tables = new HashMap<>();
    private Logger logger = LoggerFactory.getLogger(Kudu.class);

    private Kudu() {
        init();
        addShutdownHook();
    }

    private void init() {
        client = new KuduClient
                .KuduClientBuilder("192.168.1.132, 192.168.1.133, 192.168.1.134")
                .defaultOperationTimeoutMs(60000)
                .defaultSocketReadTimeoutMs(30000)
                .defaultAdminOperationTimeoutMs(60000)
                .build();
    }

    private void addShutdownHook() {
        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            if (client != null) {
                try {
                    client.close();
                } catch (Exception e) {
                    logger.error("ShutdownHook Close KuduClient Error!", e);
                }
            }
        }));
    }

    public KuduClient client() {
        return client;
    }

    public KuduTable getTable(String name) throws KuduException {
        KuduTable table = tables.get(name);
        if (table == null) {
            table = client.openTable(name);
            tables.put(name, table);
        }

        return table;
    }

    /**
     * FlushMode:AUTO_FLUSH_BACKGROUND
     * @return session
     * @throws KuduException
     */
    public KuduSession newAsyncSession() throws KuduException {
        KuduSession session = client.newSession();
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
        session.setFlushInterval(500);
        session.setMutationBufferSpace(5000);

        return session;
    }

    /**
     * FlushMode:AUTO_FLUSH_SYNC
     * @return
     * @throws KuduException
     */
    public KuduSession newSession() throws KuduException {
        KuduSession session = client.newSession();
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
        session.setMutationBufferSpace(5000);
        return session;
    }

    public void closeSession(KuduSession session) {
        if (session != null && !session.isClosed()) {
            try {
                session.close();
            } catch (KuduException e) {
                logger.error("Close KuduSession Error!", e);
            }
        }
    }

    public KuduScanner.KuduScannerBuilder scannerBuilder(String table) {
        return client.newScannerBuilder(tables.get(table));
    }
}

KuduColumn

package org.fiend.kudutest;

import org.apache.kudu.Type;

/**
 * @author lpf 2020-05-08 9:10:20
 */
public class KuduColumn {
    private String name;
    private Type type;
    private boolean isKey;
    private boolean nullable;
    private boolean isPartitionKey;

    public boolean isPartitionKey() {
        return isPartitionKey;
    }

    public void setPartitionKey(boolean partitionKey) {
        isPartitionKey = partitionKey;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public Type getType() {
        return type;
    }

    public void setType(Type type) {
        this.type = type;
    }

    public boolean isKey() {
        return isKey;
    }

    public void setKey(boolean key) {
        isKey = key;
    }

    public boolean isNullable() {
        return nullable;
    }

    public void setNullable(boolean nullable) {
        this.nullable = nullable;
    }
}

ColumnCond

package org.fiend.kudutest;

import org.apache.kudu.client.KuduPredicate;

/**
 * @author lpf 2020-05-08 9:10:20
 */
public class ColumnCond {
    // ColumnSchema column, KuduPredicate.ComparisonOp op, long value
    private String colName;
    private KuduPredicate.ComparisonOp op;
    private Object value;

    public String getColName() {
        return colName;
    }

    public void setColName(String colName) {
        this.colName = colName;
    }

    public KuduPredicate.ComparisonOp getOp() {
        return op;
    }

    public void setOp(KuduPredicate.ComparisonOp op) {
        this.op = op;
    }

    public Object getValue() {
        return value;
    }

    public void setValue(Object value) {
        this.value = value;
    }
}

方案二

表创建

/**
 * 创建表
 * @throws Exception
 */
public static void createTable() throws Exception{
  //1、创建一个client
  KuduClient client = new KuduClientBuilder(KUDU_MASTER).build();
  
  //2、创建schema信息
  List<ColumnSchema> columns = new ArrayList<ColumnSchema>();
  columns.add(new ColumnSchema.ColumnSchemaBuilder("id", Type.INT32).key(true).nullable(false).build());
  columns.add(new ColumnSchema.ColumnSchemaBuilder("name", Type.STRING).key(false).nullable(false).build());
  columns.add(new ColumnSchema.ColumnSchemaBuilder("age", Type.INT32).key(false).nullable(false).build());
  Schema schema = new Schema(columns);
  
  //3、指定分区字段
  List<String> partions = new ArrayList<String>();
  partions.add("id");
  
  //4、指定分区方式为hash分区、6个分区，一个副本
  CreateTableOptions options = new CreateTableOptions().addHashPartitions(partions, 6).setNumReplicas(1);
  
  //5、创建表，
  client.createTable("person",schema,options);
  
  client.close();
}

数据插入(Insert)

/**
 * 插入数据
 * @throws Exception
 */
public static void add() throws Exception{
  //1、创建一个client
  KuduClient client = new KuduClientBuilder(KUDU_MASTER).build();
  
  //2、打开表
  KuduTable table = client.openTable("person");
  
  //3、创建一个session会话
  KuduSession session = client.newSession();
  
  //4、创建插入
  Insert insert = table.newInsert();
  
  //5、指定插入数据
  insert.getRow().addInt("id",1);
  insert.getRow().addInt("age",18);
  insert.getRow().addString("name","张三");
  
  //6、应用插入
  session.apply(insert);
  session.flush();

  session.close();
  client.close();
}

数据更新(update)

/**
 * 更新数据
 * @throws Exception
 */
public static void update() throws Exception{
  //1、创建kudu client
  KuduClient client = new KuduClientBuilder(KUDU_MASTER).build();
  
  //2、打开表
  KuduTable table = client.openTable("person");
  
  KuduSession session = client.newSession();
  
  Update update = table.newUpdate();
  update.getRow().addInt("id",1);
  update.getRow().addString("name","李四");
  
  session.apply(update);
  session.flush();
  session.close();
  
  client.close();
}

删除(delete)

/**
 * 删除数据
 * @throws Exception
 */
public static void delete() throws Exception{
  //1、创建kudu client
  KuduClient client = new KuduClientBuilder(KUDU_MASTER).build();
  //2、打开表
  KuduTable table = client.openTable("person");
  
  KuduSession session = client.newSession();
  
  Delete delete = table.newDelete();
  delete.getRow().addInt("id",1);
  
  
  session.apply(delete);
  session.flush();
  session.close();
  
  client.close();
}

查询(query)

/**
 * @throws Exception
 */
public static void query() throws Exception {
    //1、创建kudu client
    KuduClient client = new KuduClient.KuduClientBuilder("").build();

    //2、打开表
    KuduTable table = client.openTable("person");

    //select 查询字段名
    List<String> projectColumns = new ArrayList<String>();
    projectColumns.add("user_id"); //字段名
    projectColumns.add("day");

    // 3、创建scanner扫描器
    // 简单的读取 newScannerBuilder(查询表)  
    // setProjectedColumnNames(指定输出列)  build()开始扫描
    KuduScanner.KuduScannerBuilder kuduScannerBuilder = 
                client.newScannerBuilder(table).setProjectedColumnNames(projectColumns);

    //  4、创建查询条件
    /*
     * 设置搜索的条件 where 条件过滤字段名
     * 如果不设置，则全表扫描
     */
    long userID = 7232560922086310458L;
    int Day = 17889;

    // 比较方法ComparisonOp：GREATER、GREATER_EQUAL、EQUAL、LESS、LESS_EQUAL
    // 下面的条件过滤 where user_id = xxx and day = xxx;
    KuduPredicate predicate1 = KuduPredicate.newComparisonPredicate(
            table.getSchema().getColumn("user_id"),
            KuduPredicate.ComparisonOp.EQUAL,
            userID);
    KuduPredicate predicate2 = KuduPredicate.newComparisonPredicate(
            table.getSchema().getColumn("day"),
            KuduPredicate.ComparisonOp.EQUAL,
            Day);

    // 5、将查询条件加入到scanner中
    kuduScannerBuilder.addPredicate(predicate1);
    kuduScannerBuilder.addPredicate(predicate2);

    // 6. 开始扫描
    KuduScanner scanner = kuduScannerBuilder.build();

    // 7. 获取查询结果
    while (scanner.hasMoreRows()) {
        RowResultIterator rows = scanner.nextRows();

        // 每次从tablet中获取的数据的行数
        int numRows = rows.getNumRows();
        System.out.println("numRows count is : " + numRows);

        while (rows.hasNext()) {
            RowResult row = rows.next();

            Integer id = row.getInt("id");
            String name = row.getString("name");
            int age = row.getInt("age");

            System.out.println(id + "---" + name + "---" + age);
        }
    }

    // 8、关闭client
    client.close();
}