使用AWS S3 PythonSDK连接私有S3节点

AWS 提供多语言SDK连接S3存储服务，本文主要描述使用pythonSDK连接私有S3节点

# _*_ coding: utf-8 _*_
import os

from boto3.session import Session
from botocore.exceptions import ClientError

from Common.tools.logger import Logger

logger = Logger(logger=__name__).getlog()


class Boto3Client:
    def __init__(self, access_key, secret_key, url, source=False):
        """
        access_key = "xxx"
        secret_key = "xxx"
        url = "https://s3.ap-east-1.amazonaws.com"  # 也可以是自己节点的地址
        """
        self.access_key = access_key
        self.secret_key = secret_key
        self.url = url
        self.source = source
        if self.source:
            self.url = "https://" + url
        else:
            self.url = "http://" + url
        session = Session(self.access_key, self.secret_key)
        self.s3_client = session.client('s3', endpoint_url=self.url)
        self.s3 = session.resource('s3', endpoint_url=self.url)
        # self.urlR = url

    def make_bucket(self, bucketName):
        self.s3_client.create_bucket(Bucket=bucketName)
        return bucketName

    def existed_bucket(self, bucketName):
        try:
            self.s3_client.meta.client.head_bucket(Bucket=bucketName)
        except ClientError:
            return False
        return True

    def remove_bucket(self, bucketName):
        return self.s3_client.delete_bucket(Bucket=bucketName)

    def list_buckets(self):
        res = self.s3_client.list_buckets()
        logger.debug(res)
        return [i['Name'] for i in res['Buckets']]

    def put_object(self, bucketName, file, objectName=None):
        if objectName is None:
            objectName=file.split('/')[-1]
        with open (file,'rb')as f:
            res = self.s3_client.put_object(
                Body=f,
                Bucket=bucketName,
                Key=objectName
            )
            logger.debug(f'put object:{res}')
            return objectName

    def list_objects(self, bucketName):
        res = self.s3_client.list_objects(Bucket=bucketName)
        if 'Contents' not in res:
            return []
        else:
            return [i['Key'] for i in res['Contents']]

    def list_object_v2(self, bucketName):
        res = self.s3_client.list_objects_v2(Bucket=bucketName)
        return [i['Key'] for i in res['Contents']]

    def get_object(self, bucket_name, object_name, file_name):
        self.s3_client.download_file(bucket_name, object_name, file_name)
        return os.path.abspath(file_name)

    def delete_obj(self, bucket_name, object_name):
        res = self.s3_client.delete_object(
            Bucket=bucket_name,
            Key=object_name,
        )
        return res

    def get_bucket_location(self, bucket_name):
        location_info = self.s3_client.get_bucket_location(Bucket=bucket_name)
        logger.debug(f'location_info:{location_info}')
        return location_info['LocationConstraint']

    def head_bucket(self, bucket_name):
        resp = self.s3_client.head_bucket(Bucket=bucket_name)
        resp['ResponseMetadata']['HTTPHeaders']['x-amz-request-id'] = 0
        resp['ResponseMetadata']['HTTPHeaders']['date'] = 0
        resp['ResponseMetadata']['RequestId'] = 0
        return resp

    def head_object(self, bucket_name, object_name):
        return self.s3_client.head_object(
            Bucket=bucket_name,
            Key=object_name,
        )

    def get_bucket_acl(self, bucket_name):
        resp = self.s3_client.get_bucket_acl(Bucket=bucket_name)
        resp['ResponseMetadata']['HTTPHeaders']['x-amz-request-id'] = 0
        resp['ResponseMetadata']['RequestId'] = 0
        return resp

    def get_object_acl(self, bucket_name, object_name):
        resp = self.s3_client.get_object_acl(Bucket=bucket_name, Key=object_name)
        resp['ResponseMetadata']['HTTPHeaders']['x-amz-request-id'] = 0
        resp['ResponseMetadata']['RequestId'] = 0
        return resp

    def put_object_Acl(self, bucket_name, object_name, Permission='FULL_CONTROL'):
        """Permission:'FULL_CONTROL'|'WRITE'|'WRITE_ACP'|'READ'|'READ_ACP'"""
        return self.s3_client.put_object_acl(
            ACL='private',
            AccessControlPolicy={
                'Grants': [
                    {
                        'Grantee': {
                            'DisplayName': 'string',
                            'EmailAddress': 'string',
                            'ID': 'string',
                            'Type': 'CanonicalUser',
                            'URI': 'string'
                        },
                        'Permission': Permission
                    },
                ],
                'Owner': {
                    'DisplayName': 'string',
                    'ID': 'string'
                }
            },
            Bucket=bucket_name,
            GrantFullControl='string',
            GrantRead='string',
            GrantReadACP='string',
            GrantWrite='string',
            GrantWriteACP='string',
            Key=object_name,
            RequestPayer='requester',
            VersionId='string',
            ExpectedBucketOwner='string'
        )

    def create_multipart_upload(self, bucket_name, object_name):
        return self.s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_name)['UploadId']

    def complete_multipart_upload(self, bucket_name, object_name, upload_id, part: list):
        # part : 传 Parts结构
        """[
            {
                'ETag': file_md5,
                'PartNumber': part_nums
            },
            ...
            ]"""
        return self.s3_client.complete_multipart_upload(
            Bucket=bucket_name,
            Key=object_name,
            UploadId=upload_id,
            MultipartUpload={
                'Parts': part
            },
        )

    def upload_part(self, fileIO, bucket_name, object_name, part_num: int, upload_id):
        return self.s3_client.upload_part(
            Body=fileIO,
            Bucket=bucket_name,
            Key=object_name,
            PartNumber=part_num,
            UploadId=upload_id
        )

    def clearAll(self):
        logger.debug('start clear')
        buckets = self.list_buckets()
        for i in buckets:
            logger.debug(f'bucket:{i}')
            for j in self.list_objects(bucketName=i):
                logger.debug('bucket:{} \t object:{}'.format(i, j))
                self.delete_obj(i, j)
            self.remove_bucket(i)