<dependency>
<groupId>com.qcloud</groupId>
<artifactId>cos_api</artifactId>
<version>5.6.133</version>
</dependency>
package gaei.cn.x5l.x5lhive2cos.utils;
import com.qcloud.cos.COSClient;
import com.qcloud.cos.ClientConfig;
import com.qcloud.cos.auth.BasicCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.exception.CosClientException;
import com.qcloud.cos.exception.CosServiceException;
import com.qcloud.cos.http.HttpProtocol;
import com.qcloud.cos.model.COSObjectSummary;
import com.qcloud.cos.model.ListObjectsRequest;
import com.qcloud.cos.model.ObjectListing;
import com.qcloud.cos.region.Region;
import java.io.*;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.List;
public class CosFileCountAndDataSize {
transient static String dbName = "database";
transient static String tableName = "table_name";
transient static String preDir = "/user/x5l/hive/";
transient static String sufFile = ".txt";
transient static String separator = "/";
transient static String localPreDir = "C:/Users/MECHREVO/Desktop/新建文件夹/history_partitions/";
private final static String MYSQL_URL = "jdbc:mysql://10.1.1.1:3316/test?useSSL=false";
private final static String MYSQL_USERNAME = "root";
private final static String MYSQL_PASSWORD = "123456";
private final static String MYSQL_TABLE = "`test`.`test`";
private static Connection conn = null;
private static ResultSet rs = null;
private static PreparedStatement ps = null;
private final static String BUCKET_NAME = "pro-x5l-1111111111";
private final static String SECRET_ID = "****************************";
private final static String SECRET_KEY = "****************************";
private final static String REGION = "ap-guangzhou";
private final static int MAXKEYS = 10000;
private final static String CHART_SET = "GBK";
transient static COSClient cosClient = null;
transient static ListObjectsRequest listObjectsRequest = null;
public static void main(String[] args) {
BufferedReader reader = null;
FileWriter fw = null;
String partitionDir = null;
String sampleDate = null;
int line = 1;
long startTime = 0L;
List<String> sampleDataList = new ArrayList<>();
try {
reader = new BufferedReader(new InputStreamReader(new FileInputStream(new File(localPreDir + "需要统计的sample_date" + separator + tableName + sufFile)), CHART_SET));
while ((sampleDate = reader.readLine()) != null) {
sampleDataList.add(sampleDate);
line++;
}
System.out.println(sampleDataList);
fw = new FileWriter(new File(localPreDir + "cos数据量统计" + separator + tableName + sufFile));
COSCredentials cred = new BasicCOSCredentials(SECRET_ID, SECRET_KEY);
Region region = new Region(REGION);
ClientConfig clientConfig = new ClientConfig(region);
clientConfig.setHttpProtocol(HttpProtocol.https);
cosClient = new COSClient(cred, clientConfig);
listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.setBucketName(BUCKET_NAME);
for (String date : sampleDataList) {
startTime = System.currentTimeMillis();
long fileSize = 0L;
int count = -1;
reader = new BufferedReader(new InputStreamReader(new FileInputStream(new File(localPreDir + "分区详情" + separator + tableName + sufFile)), CHART_SET));
while ((partitionDir = reader.readLine()) != null) {
if (partitionDir.contains(date)) {
listObjectsRequest.setPrefix(preDir + dbName + separator + tableName + separator + partitionDir + separator);
listObjectsRequest.setDelimiter(separator);
listObjectsRequest.setMaxKeys(MAXKEYS);
ObjectListing objectListing = null;
do {
try {
objectListing = cosClient.listObjects(listObjectsRequest);
} catch (CosServiceException e) {
e.printStackTrace();
return;
} catch (CosClientException e) {
e.printStackTrace();
return;
}
List<COSObjectSummary> cosObjectSummaries = objectListing.getObjectSummaries();
for (COSObjectSummary cosObjectSummary : cosObjectSummaries) {
fileSize += cosObjectSummary.getSize();
count++;
}
String nextMarker = objectListing.getNextMarker();
listObjectsRequest.setMarker(nextMarker);
} while (objectListing.isTruncated());
}
line++;
}
fw.write(date + separator + count + separator + fileSize + "\n");
fw.flush();
System.out.println("计算日期:" + date + " - 文件总数:" + count + " - 数据量大小:" + fileSize + " Bytes " + "(" + new DecimalFormat("0.00").format((double) fileSize / 1024 / 1024 / 1024) + " Gb - " + new DecimalFormat("0.00").format((double) fileSize / 1024 / 1024 / 1024 / 1024) + " Tb" + ")" + " - 用时:" + new DecimalFormat("0.00").format((double) (System.currentTimeMillis() - startTime) / 1000) + "s");
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (reader != null) {
reader.close();
System.out.println("reader资源释放完成");
}
if (fw != null) {
fw.close();
System.out.println("fw资源释放完成");
}
cosClient.shutdown();
System.out.println("cosClient资源释放完成");
} catch (IOException e) {
e.printStackTrace();
}
}
}
}