boto3相关（aws dynamodb/s3/sqs python用法）

１、安装boto3

pip install boto3
python -m pip install boto3

---

2、dynamodb使用

官方例子：Amazon DynamoDB — Boto3 Docs 1.24.60 documentation

#! /usr/bin/env python
#coding=utf-8

import boto3
from boto3.dynamodb.conditions import Key, Attr
from botocore.exceptions import ClientError

table_name = "alan_test"

def print_item(item):
  # id
  id_ = item['id'] if 'id' in item else 0
  # name
  name_ = item['name'] if 'name' in item else ''
  # age
  age_ = item['age'] if 'age' in item else 0
  # class
  class_ = item['class'] if 'class' in item else 0
  print("id: %s, name: %s, age: %s, class: %s" % (id_, name_, age_, class_))

def run():
  # init dynamodb
  if(raw_input("init dynamodb: [y/n]") != 'y'):
    return
  session_args = {} # Found credentials in ~/.aws/credentials [default]
  #session_args = {'profile_name': 'alan'}
  #session_args = {
  #  'aws_access_key_id': 'A1b1C1',
  #  'aws_secret_access_key': 'A+B2c2',
  #}
  session = boto3.session.Session(**session_args)
  dynamodb = session.resource('dynamodb', region_name='cn-northwest-1') # region_name 使用哪个区域的aws服务
  # dynamodb = session.resource('dynamodb', region_name='cn-northwest-1', endpoint_url='http://localhost:8000') # endpoint_url dynamodb位置
  

  # list table
  if(raw_input("list table: [y/n]") != 'y'):
    return
  for table in dynamodb.tables.all():
    print(table.name)


  # create table
  if(raw_input("create table: [y/n]") != 'y'):
    return
  try:
    dynamodb.create_table(
        TableName=table_name, # 表名
        KeySchema=[ # 主键
            {
                'AttributeName': 'id',
                'KeyType': 'HASH'  # Partition key 分区键
            },
            {
                'AttributeName': 'name',
                'KeyType': 'RANGE'  # Sort key 排序键
            }
        ],
        AttributeDefinitions=[ # 主键数据类型
            {
                'AttributeName': 'id',
                'AttributeType': 'N' # Number
            },
            {
                'AttributeName': 'name',
                'AttributeType': 'S' # String
            },

        ],
        BillingMode='PAY_PER_REQUEST',
        Tags=[
            {
                "Key": "Owner",
                "Value": "alan"
            }
        ]
    )
    # Wait until the table exists.
    dynamodb.Table(table_name).meta.client.get_waiter(
        'table_exists').wait(TableName=table_name)
  except Exception, e:
    print('failed to create table [%s].' % table_name)


  # check table
  if(raw_input("check table: [y/n]") != 'y'):
    return
  table = dynamodb.Table(table_name)
  try:
    if table.table_status == 'ACTIVE':
      print('table [%s] already exists.' % table_name)
  except ClientError as e:
    if e.response['Error']['Code'] == 'ResourceNotFoundException':
      print('Table [%s] does not exist. Create the table first.' % table_name)
    else:
      print('Unknown exception occurred while querying for table [%s]' % table_name)


  # put item
  if(raw_input("put item: [y/n]") != 'y'):
    return
  item = {'id': 1, 'name': 'alan', 'age': 20}
  try:
    table = dynamodb.Table(table_name)
    response = table.put_item(Item=item)
  except ClientError as e:
    print('failed to put item: %s' % e.response['Error']['Message'])


  # batch write item
  if(raw_input("batch write item: [y/n]") != 'y'):
    return
  item1 = {'id': 1, 'name': 'alan', 'age': 20}
  item2 = {'id': 2, 'name': 'ayun', 'age': 18}
  items = [item1, item2]
  try:
    table = dynamodb.Table(table_name)
    with table.batch_writer() as batch:
      for item in items:
        batch.put_item(Item=item)
  except ClientError as e:
    print('failed to batch write item: %s' % e.response['Error']['Message'])


  # update item
  if(raw_input("update item: [y/n]") != 'y'):
    return
  try:
    table = dynamodb.Table(table_name)
    response = table.update_item(
      Key={'id': 1, 'name': 'alan'},
      ExpressionAttributeNames={"#class": "class"}, # class是保留的关键字，需要特殊处理
      UpdateExpression="SET age=:age, #class=:class",
      ExpressionAttributeValues={':age': 25, ':class': 4},
      ReturnValues="UPDATED_NEW")
  except ClientError as e:
    print('failed to update item: %s' % e.response['Error']['Message'])


  # get item
  if(raw_input("get item: [y/n]") != 'y'):
    return
  try:
    table = dynamodb.Table(table_name)
    response = table.get_item(Key={'id': 1, 'name': 'alan'})
    if 'Item' in response:
      item = response['Item']
      print_item(item)
  except ClientError as e:
    print('failed to get item: %s' % e.response['Error']['Message'])


  # query items
  if(raw_input("query items: [y/n]") != 'y'):
    return
  try:
    table = dynamodb.Table(table_name)
    response = table.query(KeyConditionExpression=Key('id').eq(1))
    # response = table.query(KeyConditionExpression=Key('id').eq(1) & Key('name').eq('alan'))
    if 'Items' in response:
      items = response['Items']
    while 'LastEvaluatedKey' in response:
      response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
      if 'Items' in response:
        items.extend(response['Items'])
    for item in items:
      print_item(item)
  except ClientError as e:
    print('failed to query items: %s' % e.response['Error']['Message'])


  # scan table
  if(raw_input("scan table: [y/n]") != 'y'):
    return
  try:
    table = dynamodb.Table(table_name)
    response = table.scan(FilterExpression=Attr('age').lt(20))
    items = response['Items']
    for item in items:
      print_item(item)
  except ClientError as e:
    print('failed to scan table: %s' % e.response['Error']['Message'])


  # delete item
  if(raw_input("delete item: [y/n]") != 'y'):
    return
  try:
    table = dynamodb.Table(table_name)
    response = table.delete_item(Key={'id': 1, 'name': 'alan'})
  except ClientError as e:
    print('failed to delete item: %s' % e.response['Error']['Message'])


  # delete table
  if(raw_input("delete table: [y/n]") != 'y'):
    return
  try:
    table = dynamodb.Table(table_name)
    table.delete()
  except Exception, e:
    print('failed to delete table: %s' % e.message.split(':')[-1])

if __name__ == '__main__':
  run()

---

３、s3使用

官方例子：Amazon S3 examples — Boto3 Docs 1.24.60 documentation 

#! /usr/bin/env python
#coding=utf-8

import boto3
import botocore
from botocore.exceptions import ClientError

bucket_name = 'alan'
key = 'data/test.json'
s3_uri = 's3://alan/data/test.json'
#bucket_name = s3_uri.strip().split('s3://')[-1].split('/')[0]
#key = s3_uri.strip().replace('s3://' + bucket, '').lstrip('/')

def run():
  # init s3
  if(raw_input("init s3: [y/n]") != 'y'):
    return
  session_args = {} # Found credentials in ~/.aws/credentials [default]
  #session_args = {'profile_name': 'alan'}
  #session_args = {
  #  'aws_access_key_id': 'A1b1C1',
  #  'aws_secret_access_key': 'A+B2c2',
  #}
  session = boto3.session.Session(**session_args)
  s3 = session.resource('s3', region_name='cn-northwest-1') # region_name 使用哪个区域的aws服务
  # s3 = session.resource('s3', region_name='cn-northwest-1', endpoint_url='http://localhost:8000') # endpoint_url dynamodb位置
  

  # list buckets
  if(raw_input("list buckets: [y/n]") != 'y'):
    return
  for bucket in s3.buckets.all():
    print(bucket.name)


  # check object exists
  if(raw_input("check object exists: [y/n]") != 'y'):
    return
  try:
    s3.Object(bucket_name, key).load()
    print('object %s is exists' % key)
  except ClientError as e:
    if e.response['Error']['Code'] == "404":
      print('object %s is not exists' % key)
    else:
      print('error occour when checking object exists: %s' % e)
      raise


  # upload object
  if(raw_input("upload object: [y/n]") != 'y'):
    return
  bucket = s3.Bucket(bucket_name)
  try:
    local_file = '/tmp/test.json'
    bucket.upload_file(local_file, key)
    # bucket.upload_file(local_file, key, ExtraArgs={'ACL': 'public-read'})
    print('uploaded %s to %s/%s' % (local_file, bucket_name, key))
  except ClientError as e:
    print('fail to upload object: %s' % e)


  # copy object
  if(raw_input("copy object: [y/n]") != 'y'):
    return
  try:
    dst_bucket_name = bucket_name
    dst_object_key = 'data/test2.json'
    copy_src = '{}/{}'.format(bucket_name, key)
    copy_dst = '{}/{}'.format(dst_bucket_name, dst_object_key)
    s3.Object(dst_bucket_name, dst_object_key).copy_from(CopySource=copy_src)
    print('copied %s to %s' % (copy_src, copy_dst))
  except ClientError as e:
    print('fail to copy object: %s' % e)


  # list object
  if(raw_input("list object: [y/n]") != 'y'):
    return
  bucket = s3.Bucket(bucket_name)
  objects = bucket.objects.filter(Prefix='data') # objects = bucket.objects.all()
  for obj in objects:
    print('name: %s, last_modified: %s' % (obj.key, obj.last_modified))

  # list object
  if(raw_input("list object: [y/n]") != 'y'):
    return
  s3_client = boto3.client('s3')
  result = s3_client.list_objects(Bucket=bucket_name, Prefix='data', Delimiter='/')
  for o in result.get('CommonPrefixes'):
    print(o.get('Prefix'))


  # check object size
  if(raw_input("check object size: [y/n]") != 'y'):
    return
  s3_object = s3.Object(bucket_name, key)
  object_size = s3_object.content_length
  print("object_size: %s B" % object_size)


  # download object
  if(raw_input("download object: [y/n]") != 'y'):
    return
  bucket = s3.Bucket(bucket_name)
  try:
    local_file = '/tmp/test3.json'
    bucket.download_file(key, local_file)
    print('downloaded %s/%s to %s' % (bucket_name, key, local_file))
  except ClientError as e:
    print('fail to download object: %s' % e)


  # delete object
  if(raw_input("delete object: [y/n]") != 'y'):
    return
  try:
    s3.Object(bucket_name, key).delete()
    print('deleted %s/%s' % (bucket_name, key))
  except ClientError as e:
    print('fail to delete object: %s' % e)


  # batch delete objects
  if(raw_input("batch delete objects: [y/n]") != 'y'):
    return
  objects = []
  objects.append({'Key': 'data/test.json'})
  objects.append({'Key': 'data/test2.json'})
  bucket = s3.Bucket(bucket_name)
  try:
    response = bucket.delete_objects(Delete={'Objects': objects})
    for item in response['Deleted']:
      print('deleted %s/%s' % (bucket_name, item['Key']))
  except ClientError as e:
    print('fail to batch delete objects: %s' % e)

if __name__ == '__main__':
  run()

---

4、sqs使用

官方例子：Amazon SQS examples — Boto3 Docs 1.24.59 documentation 

#! /usr/bin/env python
#coding=utf-8

import boto3
import json

messages = [
  {
    "name": "alan",
    "age": 25
  },
  {
    "name": "ayun",
    "age": 18
  },
]

queue_name = "alan_test"

receipt_handles = []

def run():
  # init sqs
  if(raw_input("init sqs: [y/n]") != 'y'):
    return
  session_args = {} # Found credentials in shared credentials file: ~/.aws/credentials
  #session_args = {'profile_name': 'alan'}
  #session_args = {
  #  'aws_access_key_id': 'A1b1C1',
  #  'aws_secret_access_key': 'A+B2c2',
  #}
  session = boto3.session.Session(**session_args)
  sqs = session.resource('sqs', region_name='cn-northwest-1')
  # sqs = session.resource('sqs', region_name='cn-northwest-1', endpoint_url='https://xxx.xxx.com')

  # list queues
  if(raw_input("list queues: [y/n]") != 'y'):
    return
  queues = [q.url for q in sqs.queues.all()]
  for q in queues:
    print(q)

  # create queue
  if(raw_input("create queue: [y/n]") != 'y'):
    return
  attributes = {
    'DelaySeconds': '0',
    'VisibilityTimeout': '18000',
    'MessageRetentionPeriod': '1209600'
  }
  queue = sqs.create_queue(QueueName=queue_name, Attributes=attributes, tags={'Owner': 'alan'})
  queue_url = sqs.get_queue_by_name(QueueName=queue_name).url
  print(queue_url) # https://cn-northwest-1.queue.amazonaws.com.cn/111222333444/alan_test

  # send message
  if(raw_input("send message: [y/n]") != 'y'):
    return
  queue = sqs.get_queue_by_name(QueueName=queue_name)
  for msg in messages:
    response = queue.send_message(MessageBody=json.dumps(msg))

  # batch send message
  if(raw_input("batch send message: [y/n]") != 'y'):
    return
  entries = [{'Id': str(idx),
              'MessageBody': json.dumps(msg)}
              for idx, msg in enumerate(messages)]
  sqs_client = boto3.client('sqs', region_name='cn-northwest-1')
  response = sqs_client.send_message_batch(QueueUrl=queue_url, Entries=entries)

  # read message
  if(raw_input("read message: [y/n]") != 'y'):
    return
  queue = sqs.get_queue_by_name(QueueName=queue_name)
  while True:
    msgs = queue.receive_messages(MaxNumberOfMessages=1, MessageAttributeNames=['All'])
    if len(msgs) == 0:
      break
    msg = msgs[0]
    msg_body = json.loads(msg.body)
    receipt_handle = msg.receipt_handle
    print(json.dumps(msg_body, indent=2))
    receipt_handles.append(receipt_handle)

  # delete message
  if(raw_input("delete message: [y/n]") != 'y'):
    return
  entries = [{'Id': str(idx), 'ReceiptHandle': rh}
              for idx, rh in enumerate(receipt_handles)]
  queue.delete_messages(QueueUrl=queue_url, Entries=entries)
  for rh in receipt_handles:
    print('deleted message: %s', rh[:30])

  # delete queue
  if(raw_input("delete queue: [y/n]") != 'y'):
    return
  sqs.meta.client.delete_queue(QueueUrl=queue_url)

if __name__ == '__main__':
  run()