<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.8.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.1.1</version>
<exclusions>
<exclusion>
<artifactId>jetty-util</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId>
<version>3.0.0</version>
<exclusions>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-runner</artifactId>
</exclusion>
<exclusion>
<artifactId>hive-llap-server</artifactId>
<groupId>org.apache.hive</groupId>
</exclusion>
<exclusion>
<artifactId>log4j-slf4j-impl</artifactId>
<groupId>org.apache.logging.log4j</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-util</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>zookeeper</artifactId>
<groupId>org.apache.zookeeper</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.9.3</version>
</dependency>
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
<version>0.1.54</version>
</dependency>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.8.0</version>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>8.3.5</version>
</dependency>
@Override
public List<String> getTableList(Integer id) {
List<String> stringList = new ArrayList<>();
SysDatasource dsDetail = getDatasourceDetail(id);
String datasourceType = dsDetail.getDatasourceType();
String ip = dsDetail.getHost();
Integer port = dsDetail.getPort();
String username = dsDetail.getUsername();
String password = dsDetail.getPassword();
String databaseName = dsDetail.getDatabaseName();
String configParam;
String url = "";
Connection connection = null;
PreparedStatement preparedStatement = null;
ResultSet resultSet = null;
switch (datasourceType) {
case "MYSQL":
case "Apache Doris":
url = "jdbc:mysql://" + ip + ":" + port + "/" + databaseName;
try {
connection = DriverManager.getConnection(url, username, password);
preparedStatement = connection.prepareStatement("select table_name as tableName from information_schema.tables where table_type = 'BASE TABLE'");
resultSet = preparedStatement.executeQuery();
while (resultSet.next()) {
stringList.add(resultSet.getString("tableName"));
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
} finally {
try {
if (resultSet != null) {
resultSet.close();
}
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
}
}
break;
case "ORACLE":
url = "jdbc:oracle:thin:@" + ip + ":" + port + "/" + databaseName;
try {
connection = DriverManager.getConnection(url, username, password);
preparedStatement = connection.prepareStatement("select TABLE_NAME from all_tables WHERE OWNER = '" + username.toUpperCase() + "'");
resultSet = preparedStatement.executeQuery();
while (resultSet.next()) {
stringList.add(resultSet.getString("TABLE_NAME"));
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
} finally {
try {
if (resultSet != null) {
resultSet.close();
}
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
}
}
break;
case "SQLSERVER":
url = "jdbc:sqlserver://" + ip + ":" + port + ";DatabaseName=" + databaseName;
try {
connection = DriverManager.getConnection(url, username, password);
preparedStatement = connection.prepareStatement("select name as tableName from sys.objects WHERE type_desc = 'user_table'");
resultSet = preparedStatement.executeQuery();
while (resultSet.next()) {
stringList.add(resultSet.getString("tableName"));
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
} finally {
try {
if (resultSet != null) {
resultSet.close();
}
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
}
}
break;
case "PostgreSQL":
url = "jdbc:postgresql://" + ip + ":" + port + "/" + databaseName;
try {
connection = DriverManager.getConnection(url, username, password);
preparedStatement = connection.prepareStatement("select tablename as tableName from pg_tables where tableowner = '" + username + "'");
resultSet = preparedStatement.executeQuery();
while (resultSet.next()) {
stringList.add(resultSet.getString("tableName"));
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
} finally {
try {
if (resultSet != null) {
resultSet.close();
}
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
}
}
break;
case "Apache Hive":
if (!"".equals(dsDetail.getAddress()) && null != dsDetail.getAddress()) {
url = dsDetail.getAddress();
}
configParam = dsDetail.getConfigParam();
if (null != configParam && "1".equals(JSONUtil.parseObj(configParam).getStr("isKerberos"))) {
JSONObject cpJson = JSONUtil.parseObj(configParam);
String krb5Path = cpJson.getStr("krb5Path");
String keytabPath = cpJson.getStr("keytabPath");
String principal = cpJson.getStr("principal");
String hadoopWinutilsPath = cpJson.getStr("hadoopHomeDir");
System.setProperty("java.security.krb5.conf", krb5Path);
System.setProperty("hadoop.home.dir", hadoopWinutilsPath);
Configuration config = new Configuration();
config.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(config);
try {
UserGroupInformation.loginUserFromKeytab(principal, keytabPath);
} catch (IOException e) {
LOGGER.error("error", e);
break;
}
url = "".equals(url) ? "jdbc:hive2://" + ip + ":" + port + "/;principal=" + principal : url + "/;principal=" + principal;
} else {
url = "".equals(url) ? "jdbc:hive2://" + ip + ":" + port : url;
}
try {
connection = DriverManager.getConnection(url);
ResultSet tables = connection.getMetaData().getTables(null, "%", "%", new String[]{"TABLE"});
while (tables.next()) {
stringList.add(tables.getString("table_name"));
}
tables.close();
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
} finally {
try {
if (connection != null) {
connection.close();
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
}
}
break;
case "Elasticsearch":
List<String> serverAddresses = dsDetail.getServerAddress();
configParam = dsDetail.getConfigParam();
for (String serverAddress : serverAddresses) {
String[] split;
if (serverAddress.contains("//")) {
split = serverAddress.split("//")[1].split(":");
} else {
split = serverAddress.split(":");
}
ip = split[0];
port = Integer.valueOf(split[split.length - 1]);
RestHighLevelClient client;
if (null != configParam && "1".equals(JSONUtil.parseObj(configParam).getStr("isAuth"))) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
Credentials credentials = new UsernamePasswordCredentials(username, password);
credentialsProvider.setCredentials(AuthScope.ANY, credentials);
client = new RestHighLevelClient(RestClient.builder(new HttpHost(ip, port, "http")).setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
return httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
}));
} else {
client = new RestHighLevelClient(RestClient.builder(new HttpHost(ip, port, "http")).setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
return httpAsyncClientBuilder.setDefaultCredentialsProvider(null);
}
}));
}
try {
GetIndexResponse getIndexResponse = client.indices().get(new GetIndexRequest("*"), RequestOptions.DEFAULT);
String[] indices = getIndexResponse.getIndices();
stringList = Arrays.asList(indices);
} catch (IOException e) {
LOGGER.error("error", e);
} finally {
try {
client.close();
} catch (IOException e) {
LOGGER.error("error", e);
}
}
}
break;
case "FTP":
configParam = dsDetail.getConfigParam();
if (null != configParam && "1".equals(JSONUtil.parseObj(configParam).getStr("isSftp"))) {
Session session = null;
Channel channel = null;
ChannelSftp sftp = null;
port = 22;
try {
JSch jSch = new JSch();
session = jSch.getSession(username, ip, port);
session.setPassword(password);
session.setConfig("StrictHostKeyChecking", "no");
session.connect();
channel = session.openChannel("sftp");
channel.connect();
sftp = (ChannelSftp) channel;
stringList = (Vector<String>) sftp.ls("*");
} catch (JSchException | SftpException e) {
e.printStackTrace();
} finally {
if (null != sftp) {
if (sftp.isConnected()) {
sftp.disconnect();
}
}
if (null != channel) {
channel.disconnect();
}
if (null != session) {
session.disconnect();
}
}
} else {
FTPClient ftpClient = new FTPClient();
try {
ftpClient.connect(ip);
ftpClient.login(username, password);
if (!FTPReply.isPositiveCompletion(ftpClient.getReplyCode())) {
LOGGER.error("连接失败,账号或密码错误!");
break;
}
FTPFile[] ftpFiles = ftpClient.listFiles();
for (FTPFile obj : ftpFiles) {
stringList.add(obj.getName());
}
} catch (IOException e) {
LOGGER.error("error", e);
} finally {
try {
ftpClient.disconnect();
} catch (IOException e) {
LOGGER.error("error", e);
}
}
}
break;
case "minIO":
String[] split;
String address = dsDetail.getAddress();
if (address.contains("//")) {
split = address.split("//")[1].split(":");
} else {
split = address.split(":");
}
ip = split[0];
port = Integer.valueOf(split[split.length - 1]);
url = "http://" + ip + ":" + port;
MinioClient minioClient = null;
List<Bucket> buckets = new ArrayList<>();
try {
minioClient = MinioClient.builder()
.endpoint(url)
.credentials(username, password)
.build();
buckets = minioClient.listBuckets();
for (Bucket bucket : buckets) {
stringList.add(bucket.name());
}
} catch (Exception e) {
LOGGER.error("error", e);
break;
}
break;
}
return stringList;
}
@Override
public List<TableStruct> getTableStruct(Integer id, String tableName) {
List<TableStruct> tableStructs = new ArrayList<>();
SysDatasource dsDetail = getDatasourceDetail(id);
String datasourceType = dsDetail.getDatasourceType();
String ip = dsDetail.getHost();
Integer port = dsDetail.getPort();
String username = dsDetail.getUsername();
String password = dsDetail.getPassword();
String databaseName = dsDetail.getDatabaseName();
String url = "";
Connection connection = null;
PreparedStatement preparedStatement = null;
ResultSet resultSet = null;
switch (datasourceType) {
case "MYSQL":
case "Apache Doris":
url = "jdbc:mysql://" + ip + ":" + port + "/" + databaseName;
try {
connection = DriverManager.getConnection(url, username, password);
String sql = "select COLUMN_NAME , DATA_TYPE , COLUMN_COMMENT from information_schema.COLUMNS where TABLE_NAME = '" + tableName + "'";
preparedStatement = connection.prepareStatement(sql);
resultSet = preparedStatement.executeQuery();
while (resultSet.next()) {
TableStruct tableStruct = new TableStruct();
tableStruct.setField(resultSet.getString("COLUMN_NAME"));
tableStruct.setType(resultSet.getString("DATA_TYPE"));
tableStruct.setComments(resultSet.getString("COLUMN_COMMENT"));
tableStructs.add(tableStruct);
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
} finally {
try {
if (resultSet != null) {
resultSet.close();
}
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
}
}
break;
case "ORACLE":
url = "jdbc:oracle:thin:@" + ip + ":" + port + "/" + databaseName;
try {
connection = DriverManager.getConnection(url, username, password);
String sql = "select t.*, c.COMMENTS from user_tab_columns t join user_col_comments c on t.table_name = c.table_name and t.column_name = c.column_name where t.table_name = '" + tableName + "'";
preparedStatement = connection.prepareStatement(sql);
resultSet = preparedStatement.executeQuery();
while (resultSet.next()) {
TableStruct tableStruct = new TableStruct();
tableStruct.setField(resultSet.getString("COLUMN_NAME"));
tableStruct.setType(resultSet.getString("DATA_TYPE"));
tableStruct.setComments(resultSet.getString("COMMENTS"));
tableStructs.add(tableStruct);
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
} finally {
try {
if (resultSet != null) {
resultSet.close();
}
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
}
}
break;
case "SQLSERVER":
url = "jdbc:sqlserver://" + ip + ":" + port + ";DatabaseName=" + databaseName;
try {
connection = DriverManager.getConnection(url, username, password);
String sql = " select convert(varchar(200), b.name) as column_name, convert(varchar(200), d.DATA_TYPE) as data_type,convert(varchar(200), c.value) as column_desc from sys.columns b " +
"left join sys.extended_properties c on c.major_id = b.object_id and c.minor_id = b.column_id " +
"inner join INFORMATION_SCHEMA.COLUMNS d on d.column_name = b.name " +
"where d.table_name = '" + tableName + "'";
preparedStatement = connection.prepareStatement(sql);
resultSet = preparedStatement.executeQuery();
while (resultSet.next()) {
TableStruct tableStruct = new TableStruct();
tableStruct.setField(resultSet.getString("column_name"));
tableStruct.setType(resultSet.getString("data_type"));
tableStruct.setComments(resultSet.getString("column_desc"));
tableStructs.add(tableStruct);
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
} finally {
try {
if (resultSet != null) {
resultSet.close();
}
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
}
}
break;
case "SAP HANA":
break;
case "PostgreSQL":
url = "jdbc:postgresql://" + ip + ":" + port + "/" + databaseName;
try {
connection = DriverManager.getConnection(url, username, password);
String sql = "SELECT A.attname AS field, T.typname AS TYPE, b.description AS COMMENT " +
"from pg_class C, pg_attribute A " +
"LEFT OUTER JOIN pg_description b ON A.attrelid = b.objoid AND A.attnum = b.objsubid, pg_type T " +
"where C.relname = '" + tableName + "'AND A.attnum > 0 AND A.attrelid = C.oid AND A.atttypid = T.oid";
preparedStatement = connection.prepareStatement(sql);
resultSet = preparedStatement.executeQuery();
while (resultSet.next()) {
TableStruct tableStruct = new TableStruct();
tableStruct.setField(resultSet.getString("field"));
tableStruct.setType(resultSet.getString("TYPE"));
tableStruct.setComments(resultSet.getString("COMMENT"));
tableStructs.add(tableStruct);
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
} finally {
try {
if (resultSet != null) {
resultSet.close();
}
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
}
}
break;
case "Apache Hive":
if (!"".equals(dsDetail.getAddress()) && null != dsDetail.getAddress()) {
url = dsDetail.getAddress();
}
String configParam = dsDetail.getConfigParam();
if (null != configParam && "1".equals(JSONUtil.parseObj(configParam).getStr("isKerberos"))) {
JSONObject cpJson = JSONUtil.parseObj(configParam);
String krb5Path = cpJson.getStr("krb5Path");
String keytabPath = cpJson.getStr("keytabPath");
String principal = cpJson.getStr("principal");
String hadoopWinutilsPath = cpJson.getStr("hadoopHomeDir");
System.setProperty("java.security.krb5.conf", krb5Path);
System.setProperty("hadoop.home.dir", hadoopWinutilsPath);
Configuration config = new Configuration();
config.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(config);
try {
UserGroupInformation.loginUserFromKeytab(principal, keytabPath);
} catch (IOException e) {
LOGGER.error("error", e);
break;
}
url = "".equals(url) ? "jdbc:hive2://" + ip + ":" + port + "/;principal=" + principal : url + "/;principal=" + principal;
} else {
url = "".equals(url) ? "jdbc:hive2://" + ip + ":" + port : url;
}
try {
connection = DriverManager.getConnection(url);
ResultSet columns = connection.getMetaData().getColumns(null, "%", tableName, "%");
while (columns.next()) {
TableStruct tableStruct = new TableStruct();
tableStruct.setField(columns.getString("column_name"));
tableStruct.setType(columns.getString("type_name"));
tableStruct.setComments(columns.getString("remarks"));
tableStructs.add(tableStruct);
}
columns.close();
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
} finally {
try {
if (connection != null) {
connection.close();
}
} catch (SQLException throwables) {
LOGGER.error("error", throwables);
}
}
break;
}
return tableStructs;
}