Amazon S3 多部分上传

最近项目需要备份文件到Amazon S3，使用java SDK进行了一些尝试，
maven配置：

<!--aws SDK -->
<dependency>
    <groupId>com.amazonaws</groupId>
    <artifactId>aws-java-sdk-bom</artifactId>
    <version>1.10.43</version>
    <type>pom</type>
    <scope>import</scope>
</dependency>

<!--指定使用 aws-sdk java模块 -->
<dependency>
    <groupId>com.amazonaws</groupId>
    <artifactId>aws-java-sdk-s3</artifactId>
    <version>1.10.43</version>
</dependency>

构建client对象:

public static AmazonS3 getS3(){     
    String accessKey = "your AmazonS3 accessKey ";
    String secretKey = "your AmazonS3 secretKey ";
    AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.setProtocol(Protocol.HTTP);
    clientConfig.setSignerOverride("S3SignerType"); // 用于兼容 ceph API
    AmazonS3 s3Client = new AmazonS3Client(credentials, clientConfig);
    s3Client.setEndpoint("your AmazonS3 endpoint");
    return s3Client;
}

上传文件（upload. waitForCompletion()对于大文件默认采用多部分上传，不会出现文件过大导致上传失败）：

/**
    * 大文件分片上传
    *
    * @param bucketName s3的buckename
    * @param fileName 上传创建的文件名称
    * @param filePath 源文件磁盘路径
    */
public static void partUpload(String bucketName, String fileName, String filePath){

    PutObjectRequest request = new PutObjectRequest(bucketName, fileName, new File(filePath));
    //- 监听上传过程
    request.setGeneralProgressListener(new ProgressListener() {
        public void progressChanged(ProgressEvent progressEvent) {
            System.out.println("Transferred bytes: "+progressEvent.getBytesTransferred());
        }
    });
    TransferManager tm = new TransferManager(getS3());
    Upload upload = tm.upload(request);
    try {
        //- You can block and wait for the upload to finish
        upload.waitForCompletion();
    } catch (AmazonClientException amazonClientException) {
        amazonClientException.printStackTrace();
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    //- 释放资源
    tm.shutdownNow();
}

对于标准的多部分上传，尝试了直接从下载链接URL中读取文件流进行上传，上传一部分后总会报错，定位文件流位置失败，debug发现可能是网络传递过程中丢包过多导致的（待定），所以替换采用把文件下载下来再进行上传，上传完毕删除文件。

标准多部分上传示例（失败例子，仅供参考）：

/**
 * 上传（从url读取文件流）,<失败例子，仅供参考>
 *
 * @param bucketName s3的buckename
 * @param fileName 上传创建的文件名称
 * @param url 文件下载url
 */
public static void uploadFileFromUrl(String bucketName, String fileName, String url){
        List<PartETag> partETags = new ArrayList<PartETag>();
        //- Step 1: Initialize.
        InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, fileName);
        initRequest.getRequestClientOptions().setReadLimit(5 * 1024 * 1024);
        InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);

        File file = new File(fileName);
        try {
            URLConnection uc = new URL(url).openConnection();
            uc.setConnectTimeout(1000000);
            uc.setDoOutput(true);
            InputStream in = new BufferedInputStream(uc.getInputStream());

            //- 返回文件所占字节大小（如果是文件）
            long contentLength = uc.getContentLengthLong();
            long partSize =5 * 1024 * 1024; // Set part size to 10 MB.
            //- Step 2: Upload parts.
            long filePosition = 0;

            for (int i = 1; filePosition < contentLength; i++) {
                //- Last part can be less than 5 MB. Adjust part size.
                partSize = Math.min(partSize, (contentLength - filePosition));
                //- Create request to upload a part.
                UploadPartRequest uploadRequest = new UploadPartRequest()
                        .withBucketName(bucketName).withKey(fileName)
                        .withUploadId(initResponse.getUploadId()).withPartNumber(i)
                        .withInputStream(in)
                        .withFileOffset(filePosition)
                        .withPartSize(partSize);
                //- Upload part and add response to our list.
                try {
                    partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());
                }catch (Exception e){
                    e.printStackTrace();
                    System.err.println("上传失败的子部分序号："+i);
                }
                filePosition += partSize;
                System.err.println("已上传字节："+filePosition);
            }
            //- Step 3: Complete.
            CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(
                    bucketName, fileName, initResponse.getUploadId(), partETags);
            s3Client.completeMultipartUpload(compRequest);
        } catch (Exception e) {
            //- 终止多部分上传
            s3Client.abortMultipartUpload(new AbortMultipartUploadRequest(
                    bucketName, fileName, initResponse.getUploadId()));
            e.printStackTrace();
        }
        file.delete();
    }

标准多部分上传官方范例：
http://docs.aws.amazon.com/AmazonS3/latest/dev/llJavaUploadFile.html
s3 java SDK官方文档：
http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/index.html