Amazon S3 API java uitl

最新推荐文章于 2023-11-11 21:20:58 发布

偷偷学习被我发现

最新推荐文章于 2023-11-11 21:20:58 发布

阅读量684

点赞数 2

分类专栏：大数据文章标签： java 开发语言

本文链接：https://blog.csdn.net/qq_37432174/article/details/129114871

版权

大数据专栏收录该内容

14 篇文章 1 订阅

订阅专栏

依赖：

<!-- aws 1.11.486 -->
<dependency>
    <groupId>com.amazonaws</groupId>
    <artifactId>aws-java-sdk-core</artifactId>
    <version>${awsjavasdk.version}</version>
    <scope>compile</scope>
</dependency>
<dependency>
    <groupId>com.amazonaws</groupId>
    <artifactId>aws-java-sdk-kms</artifactId>
    <version>${awsjavasdk.version}</version>
    <scope>compile</scope>
</dependency>
<dependency>
    <groupId>com.amazonaws</groupId>
    <artifactId>aws-java-sdk-s3</artifactId>
    <version>${awsjavasdk.version}</version>
    <scope>compile</scope>
</dependency>

AwsS3Client

import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;

@Repository
public class AmazonS3Client {
	//连接云平台ip端口
	@Value("${endpoint}")
	private  String endpoint;
	

	//创建AmazonS3Client
    public  AmazonS3 createS3Client(String accessKey,String secretKey){
        try {
            AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
            /*ClientConfiguration cfg = new ClientConfiguration();
            cfg.setProtocol(Protocol.HTTP);*/
            AmazonS3 amazonS3 = AmazonS3ClientBuilder.standard()
                    .withCredentials(new AWSStaticCredentialsProvider(credentials))
                    .withEndpointConfiguration(
                            new AwsClientBuilder.EndpointConfiguration(
                            		endpoint, ""
                            )
                    )
                    .build();
            return amazonS3;
        }catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }
    
    //获取AmazonS3Client 
    public  AmazonS3 get_s3_client(String lisk_addr,String accessKey,String secretKey){
        try {
            AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
			ClientConfiguration cfg = new ClientConfiguration();
			cfg.setProtocol(Protocol.HTTP);//设置客户端协议
//			cfg.setMaxConnections(10);//设置最大连接数10
//			cfg.setConnectionTimeout(5000);//设置连接超时5秒
//			cfg.setSocketTimeout(2000);//设置传输数据超时时间为2秒
            AmazonS3 amazonS3 = AmazonS3ClientBuilder.standard()
                    .withCredentials(new AWSStaticCredentialsProvider(credentials))
					.withClientConfiguration(cfg)
                    .withEndpointConfiguration(
                            new AwsClientBuilder.EndpointConfiguration(
                            		lisk_addr, ""
                            )
                    )
                    .build();
            return amazonS3;
        }catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }
}

s3桶常用操作类

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.*;
import com.csg.dataprocess.baidubce.service.CephPublicService;
import org.springframework.stereotype.Service;

public class AwsS3Bucket extends AwsS3Client {
   
    /**
     * 创建公有bucket
     */
    public boolean createPublicBucket(AmazonS3 s3Client ,String bucketName) {
        if (!s3Client.doesBucketExistV2(bucketName)) {
            CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName);
            createBucketRequest.withCannedAcl(CannedAccessControlList.PublicReadWrite);
            s3Client.createBucket(createBucketRequest);
            String bucketLocation = s3Client.getBucketLocation(new GetBucketLocationRequest(bucketName));
            System.out.println("Bucket location: " + bucketLocation);
            return true;
        }else {
        	return false;
		}
    }

   /**
     * 创建私有bucket
     */
	public boolean createPrivateBucket(AmazonS3 s3Client, String bucketName) {
        if (!s3Client.doesBucketExistV2(bucketName)) {
            CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName);
            createBucketRequest.withCannedAcl(CannedAccessControlList.Private);
            s3Client.createBucket(createBucketRequest);
            String bucketLocation = s3Client.getBucketLocation(new GetBucketLocationRequest(bucketName));
            System.out.println("Bucket location: " + bucketLocation);
            return true;
        }else {
        	return false;
		}
	}

	/**
	 * 开启bucket的多版本
	 */
    public String setBucketVersionOpen(AmazonS3 s3Client, String bucketName) {
        BucketVersioningConfiguration configuration = new BucketVersioningConfiguration().withStatus(BucketVersioningConfiguration.ENABLED);//Enabled   SUSPENDED
        SetBucketVersioningConfigurationRequest setBucketVersioningConfigurationRequest = new SetBucketVersioningConfigurationRequest(bucketName, configuration);
        s3Client.setBucketVersioningConfiguration(setBucketVersioningConfigurationRequest);
        BucketVersioningConfiguration conf = s3Client.getBucketVersioningConfiguration(bucketName);
        return conf.getStatus();
    }
    
	 /**
     * 暂停bucket的多版本
     */
    public String setBucketVersionSupended(AmazonS3 s3Client, String bucketName) {
        BucketVersioningConfiguration configuration = new BucketVersioningConfiguration().withStatus(BucketVersioningConfiguration.SUSPENDED);//Enabled   SUSPENDED
        SetBucketVersioningConfigurationRequest setBucketVersioningConfigurationRequest = new SetBucketVersioningConfigurationRequest(bucketName, configuration);
        s3Client.setBucketVersioningConfiguration(setBucketVersioningConfigurationRequest);
        BucketVersioningConfiguration conf = s3Client.getBucketVersioningConfiguration(bucketName);
        return conf.getStatus();
    }
    
	/**
     * bucket 列表
     */
    public List<Bucket> listBuckets(AmazonS3 s3Client ) {
        List<Bucket> listBuckets = s3Client.listBuckets();
        return listBuckets;
    }
 
    /**
     * 创建一个s3桶-默认
     */
    public void createBucketBase() {
        Bucket bucket = s3.createBucket(bucketName);
        System.out.println(bucket.toString());
    }
 
    /**
     * 创建一个s3桶-带参数
     */
    public void createBucketWithParams() {
        //指定名称和区域
        CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName, region);
        //是否启用对象锁-启用后，阻止删除对象
        createBucketRequest.setObjectLockEnabledForBucket(true);
        Bucket bucket = s3.createBucket(createBucketRequest);
        System.out.println(bucket.toString());
    }
 
    /**
     * 删除一个s3桶
     */
  public void delBucket(AmazonS3 s3Client ,String bucketName) {
        s3Client.deleteBucket(bucketName);
    }
    
    /**
     * 判断s3桶是否存在
     */  
 public boolean doesBucketExist(AmazonS3 s3Client ,String bucketName) {
        return s3Client.doesBucketExistV2(bucketName);
    }
    
   /**
     * s3桶配置
     */
    public void configBucket() {
    
        /**
         * 加速配置
         */
        BucketAccelerateConfiguration bucketAccelerateConfiguration = new BucketAccelerateConfiguration(BucketAccelerateStatus.Enabled);
        s3.setBucketAccelerateConfiguration(bucketName, bucketAccelerateConfiguration);
      
        /**
         * 权限配置
         */
        //公共访问权限
        SetBucketAclRequest setBucketAclRequest = new SetBucketAclRequest(bucketName, CannedAccessControlList.Private);
        s3.setBucketAcl(setBucketAclRequest);
        //访问控制列表
        AccessControlList accessControlList = new AccessControlList();
        accessControlList.setRequesterCharged(true);
        accessControlList.setOwner(null);
        SetBucketAclRequest setBucketAclRequest2 = new SetBucketAclRequest(bucketName, accessControlList);
        s3.setBucketAcl(setBucketAclRequest2);
       
        /**
         * 分析配置
         */
        AnalyticsConfiguration analyticsConfiguration = new AnalyticsConfiguration();
        analyticsConfiguration.setId(null);
        SetBucketAnalyticsConfigurationRequest setBucketAnalyticsConfigurationRequest = new SetBucketAnalyticsConfigurationRequest(bucketName, analyticsConfiguration);
        s3.setBucketAnalyticsConfiguration(setBucketAnalyticsConfigurationRequest);
        
        /**
         * 生命周期配置
         */
        BucketLifecycleConfiguration bucketLifecycleConfiguration = new BucketLifecycleConfiguration();
        List<BucketLifecycleConfiguration.Rule> rules = new ArrayList();
        //需要预先制定规则
        BucketLifecycleConfiguration.Rule rule = new BucketLifecycleConfiguration.Rule().withId(null);
        rules.add(rule);
        bucketLifecycleConfiguration.setRules(rules);
        SetBucketLifecycleConfigurationRequest setBucketLifecycleConfigurationRequest = new SetBucketLifecycleConfigurationRequest(bucketName, bucketLifecycleConfiguration);
        s3.setBucketLifecycleConfiguration(setBucketLifecycleConfigurationRequest);
       
        /**
         * 加密配置
         * 当对象存储在s3时默认是加密的
         */
        SetBucketEncryptionRequest setBucketEncryptionRequest = new SetBucketEncryptionRequest();
        setBucketEncryptionRequest.setBucketName(bucketName);
        ServerSideEncryptionConfiguration serverSideEncryptionConfiguration = new ServerSideEncryptionConfiguration();
        //同样，需要预先制定规则
        serverSideEncryptionConfiguration.setRules(null);
        setBucketEncryptionRequest.setServerSideEncryptionConfiguration(serverSideEncryptionConfiguration);
        s3.setBucketEncryption(setBucketEncryptionRequest);
       
        /**
         * 版本控制配置
         */
        BucketVersioningConfiguration bucketVersioningConfiguration = new BucketVersioningConfiguration();
        bucketVersioningConfiguration.setMfaDeleteEnabled(true);
        bucketVersioningConfiguration.setStatus(BucketVersioningConfiguration.ENABLED);
        SetBucketVersioningConfigurationRequest setBucketVersioningConfigurationRequest = new SetBucketVersioningConfigurationRequest(bucketName, bucketVersioningConfiguration);
        s3.setBucketVersioningConfiguration(setBucketVersioningConfigurationRequest);
 
        /**
         * 为s3指定一个策略-s3的策略是唯一的
         */
        s3.setBucketPolicy(null);
      
        /**
         * 日志记录配置
         */
        s3.setBucketLoggingConfiguration(null);
      
        /**
         * 通知配置
         */
        s3.setBucketNotificationConfiguration(null);
       
        /**
         * 复制配置
         */
        s3.setBucketReplicationConfiguration(null);
    
        /**
         * 标签配置
         */
        s3.setBucketTaggingConfiguration(null);
       
        /**
         * 静态网站托管配置
         */
        s3.setBucketWebsiteConfiguration(null);
       
        /**
         * 指标配置
         */
        s3.setBucketMetricsConfiguration(null);
 
    }

//创建桶
 public Map<String,Object> create_bucket(String bucket_name,String accessKey,String secretKey,String endpoint,Integer state){
		Map<String,Object>  map = new HashMap<String, Object>();
		try {
			boolean result = false;
			AmazonS3 s3Client =createS3Client(accessKey, secretKey,endpoint);
			if(state!=null&&state==1)
				result = createPublicBucket(s3Client, bucket_name);
			else
				result = createPrivateBucket(s3Client, bucket_name);
			if(result){
				CephConfig.setCephConfig(s3Client,bucket_name);
			    map.put("state","桶 "+bucket_name+" 创建成功");
			    map.put("user_id",s3Client.getS3AccountOwner().getDisplayName());
			    map.put("bucket_name",bucket_name);
			}else
				throw new Exception("创建桶出错了");
		} catch (Exception e) {
			e.printStackTrace();
		    map.put("state","桶 "+bucket_name+" 创建失败");
		    map.put("user_id","空");
		    map.put("bucket_name",bucket_name);
		    map.put("Exception",e.getMessage());
		}
		return map;
	}
}

s3桶文件操作类

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.*;
import com.csg.dataprocess.baidubce.service.CephPublicService;
import org.springframework.stereotype.Service;
import java.io.File;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

public class AwsS3File extends AwsS3Client {

	/**
     * 列出一个s3桶内的文件
     */
public List<S3ObjectSummary> listObjects(AmazonS3 s3Client ,String bucketName) {
        return s3Client.listObjects(bucketName).getObjectSummaries();
    }
}

	/**
     * 列出一个s3桶内的文件
     */
public ObjectListing listObjects(AmazonS3 s3Client ,String bucketName, String prefix) {
        ObjectListing objects = s3Client.listObjects(bucketName, prefix);
        return objects;
    }

	/**
     * 分页列出一个s3桶内的文件
     */
public List<List<S3ObjectSummary>> listBucketObjPage(AmazonS3 s3Client ,String bucketName, int maxKey) {
        // 用户可设置每页最多条记录maxKey
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
        listObjectsRequest.setBucketName(bucketName);
        listObjectsRequest.withMaxKeys(maxKey);
        ArrayList<List<S3ObjectSummary>> listall = new ArrayList<>();
        boolean isTruncated = true;
        while (isTruncated) {
            ObjectListing listObjectsResponse = s3Client.listObjects(listObjectsRequest);
            isTruncated = listObjectsResponse.isTruncated();
            if (listObjectsResponse.getNextMarker() != null) {
                listObjectsRequest.withMarker(listObjectsResponse.getNextMarker());
            }
            //加入listall
            List<S3ObjectSummary> objectSummaries = listObjectsResponse.getObjectSummaries();
            listall.add(objectSummaries);
        }
        return listall;
    }

	/**
     * 上传一个文件到s3桶
     * 注意1：如果key相同，则替换
     * 注意2：如果key是一个层次路径，那么s3会自动创建相应文件夹
     */
     public void putObj(AmazonS3 s3Client ,String bucketName, String objectKey, String filePath) {
        s3Client.putObject(bucketName, objectKey, new File(filePath));

    }
    
	/**
     * 上传一个文件到s3桶
     * 注意1：如果key相同，则替换
     * 注意2：如果key是一个层次路径，那么s3会自动创建相应文件夹
     */
    public void putObj(AmazonS3 s3Client ,String bucketName,String objectKey,File file) {
        s3Client.putObject(bucketName, objectKey, file);
//        s3Client.putObject(bucketName, key, input, metadata);

    }

	/**
     * 下载对象
     */
    public void downObj(AmazonS3 s3Client ,String bucketName, String objectKey, String saveFilePath) {
        s3Client.getObject(new GetObjectRequest(bucketName, objectKey), new File(saveFilePath));
    }


    /**
     * 删除对象
     */
    public void delObj(AmazonS3 s3Client ,String bucketName, String objectKey) {
        s3Client.deleteObject(bucketName, objectKey);
    }

    /**
     * 输出对象下载URL
     */
    public URL getURLS(AmazonS3 s3Client ,String bucketName, String objectKey) {
        GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, objectKey);
        URL url = s3Client.generatePresignedUrl(request);
        return url;
    }
    
    /**
     * 输出从重命名后的对象下载URL
     */
    public URL getURLS_rename(AmazonS3 s3Client ,String bucketName, String objectKey,String file_name) {
        GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, objectKey);
        request.addRequestParameter("response-content-disposition","filename="+file_name+";");
        URL url = s3Client.generatePresignedUrl(request);
        return url;
    }

	/**
     * 获取附件下载URL
     */
	 public String getFileUrl(AmazonS3 s3,String bucketName,String fileName) {
        GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName,fileName);
        request.addRequestParameter("response-content-disposition","attachment;");
        URL url = s3.generatePresignedUrl(request);
        return url.toString();
    }

	/**
     * 获取附件下载URL并重命名
     */
	public String getFileUrl_rename(AmazonS3 s3,String bucketName,String ceph_key,String file_name) {
        GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName,ceph_key);
        request.addRequestParameter("response-content-disposition","attachment;filename="+file_name+";");
        URL url = s3.generatePresignedUrl(request);
        return url.toString();
    }

	/**
     * 分片上传文件
     */
	public String uploadAppenderFile(byte[] fileData,AmazonS3 s3,String bucketName,String filepath) {
		
	       //生成上传文件的随机序号
		    String fileName = UUID.randomUUID().toString().replaceAll("-", "");
		    List<PartETag> partETags = new ArrayList<PartETag>();
		    InitiateMultipartUploadRequest initRequest = new  InitiateMultipartUploadRequest(bucketName, fileName);
		    InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest);
		     try {
		     File file = new File(filepath);//文件路径
		     long contentLength = file.length();
			 long partSize = 5242880; // 设置分片大小. 测试采用5M分片便于测试
			 long filePosition = 0;
		        for (int i = 1; filePosition < contentLength; i++) {
		            partSize = Math.min(partSize, (contentLength - filePosition));
		            // 创建分片上传请求
		            UploadPartRequest uploadRequest = new UploadPartRequest()
		                .withBucketName(bucketName).withKey(fileName)
		                .withUploadId(initResponse.getUploadId()).withPartNumber(i)
		                .withFileOffset(filePosition)
		                .withPartSize(partSize)
		                .withFile(file);
		                // 将分片添加到列表.
		            partETags.add(s3.uploadPart(uploadRequest).getPartETag());
		            filePosition += partSize;
		        }
		        CompleteMultipartUploadRequest compRequest = new 
		                     CompleteMultipartUploadRequest(
		                    		 bucketName, 
		                    		 fileName, 
		                             initResponse.getUploadId(), 
		                             partETags);
		        s3.completeMultipartUpload(compRequest);
		    } 
		    catch (Exception e) 
		    {
		    	  e.printStackTrace();
		    	   }

	        return fileName;
	}
}