用到impala和KuduClient结合的方式进行操作,这里用impala获取数据库的表字段信息,通过kuduclient实现数据写入操作,(代码有点冗余)
public KuduDbAdapter(String ip, int port, String db, String user, String pwd) {
this.ip = ip;
this.port = port;
this.db = db;
this.user = user;
this.pwd = pwd;
}
@Override
public Connection getConn() {
Connection con = null;
try {
Class.forName("com.cloudera.impala.jdbc41.Driver");
String url = "jdbc:impala://" + ip + ":" + port + "/" + db + "";
try {
con = DriverManager.getConnection(url);
} catch (SQLException e) {
e.printStackTrace();
}
} catch (ClassNotFoundException e) {
System.out.println(e);
}
return con;
}
@Override
public boolean updateInsert(MapData data, String tableName, String keyColumns) throws Exception {
boolean success = false;
KuduClient client = initClient(ip);
KuduTable kuduTable = client.openTable(tableName);
KuduSession kuduSession = client.newSession();
// 设置手动刷新
kuduSession.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
// 设置缓冲区大小
kuduSession.setMutationBufferSpace(commitCount);
// 拿到列名的集合
Set<String> columns = data.keySet();
Map<String, String> columnInfo = getDescribeTable(tableName);
for (String rowDataKey : data.keySet()) {
if ("double".equals(columnInfo.get(rowDataKey.toLowerCase()))) {
data.put(rowDataKey, data.getDouble(rowDataKey));
}
if ("int".equals(columnInfo.get(rowDataKey.toLowerCase()))) {
data.put(rowDataKey, data.getInt(rowDataKey));
}
if ("time".equals(rowDataKey.toLowerCase())) {
String time = data.get(rowDataKey).toString();
Date date = new Date();
if (time.contains(".00")) {
date = DateUtils.parse(time);
} else {
date = (Date) data.get(rowDataKey);
}
Date observerTime = DateUtils.addHour(date, 8);
data.put(rowDataKey, observerTime.getTime() * 1000);
}
if ("shardingid".equals(columnInfo.get(rowDataKey.toLowerCase()))) {
data.remove(rowDataKey);
}
}
try {
Upsert upsert = kuduTable.newUpsert();
for (String column : columns) {
if (data.get(column) instanceof Double) {
upsert.getRow().addDouble(column.toLowerCase(), (Double) data.get(column));
} else if (data.get(column) instanceof String) {
upsert.getRow().addString(column.toLowerCase(), data.get(column).toString());
} else if (data.get(column) instanceof Integer) {
upsert.getRow().addInt(column.toLowerCase(), (Integer) data.get(column));
} else if (data.get(column) instanceof Long) {
upsert.getRow().addLong(column.toLowerCase(), (Long) data.get(column));
} else if (data.get(column) instanceof Float) {
upsert.getRow().addFloat(column.toLowerCase(), (Float) data.get(column));
}
}
kuduSession.apply(upsert);
kuduSession.flush();
success = true;
} catch (Exception e) {
throw e;
} finally {
client.close();
}
return success;
}
- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
- 25.
- 26.
- 27.
- 28.
- 29.
- 30.
- 31.
- 32.
- 33.
- 34.
- 35.
- 36.
- 37.
- 38.
- 39.
- 40.
- 41.
- 42.
- 43.
- 44.
- 45.
- 46.
- 47.
- 48.
- 49.
- 50.
- 51.
- 52.
- 53.
- 54.
- 55.
- 56.
- 57.
- 58.
- 59.
- 60.
- 61.
- 62.
- 63.
- 64.
- 65.
- 66.
- 67.
- 68.
- 69.
- 70.
- 71.
- 72.
- 73.
- 74.
- 75.
- 76.
- 77.
- 78.
- 79.
- 80.
- 81.
- 82.
- 83.
- 84.
- 85.
- 86.