S3奇淫技巧:一步完成上传及ACL设置

最新推荐文章于 2023-04-08 18:03:45 发布

weixin_34107739

最新推荐文章于 2023-04-08 18:03:45 发布

阅读量1k

点赞数

文章标签： python java

原文链接：https://my.oschina.net/diluga/blog/1057730

版权

为什么80%的码农都做不了架构师？>>>

需求描述

用户A需要在上传文件到S3的同时设置允许用户test1可读，上传数据和设置ACL需要一步完成。其目的是尽量减少http请求次数，提高并发效率。

python用例

from boto.s3.connection import S3Connection
import boto
import os

endpoint = 's3.ceph.work'
bucket_name = 'test3'
access_key = ''
secret_key = ''
key_name = 'abc'

conn = boto.connect_s3(
    aws_access_key_id=access_key,
    aws_secret_access_key=secret_key,
    host=endpoint,
    is_secure=False,
    calling_format=boto.s3.connection.SubdomainCallingFormat(),
    validate_certs=False,
)

bucket = conn.get_bucket(bucket_name)
key_ = bucket.new_key(key_name)

#方法1，通常一般用这个方法
key_.set_contents_from_string('aa') #第一次http请求，PUT操作，上传数据
key_.add_user_grant('READ','test1') #第二次http请求，PUT操作，设置ACL

#方法2，设置http请求头
headers = {"x-amz-grant-read":"id=test1"} #这里切记要在id(type)后面加上"="，不然就踩坑了
key_.set_contents_from_string('aa', headers=headers) #一次http请求，PUT操作，上传数据同时设置Object的ACL

原理说明

方法2其实也是遵循了S3标准的操作，只需要在上传Object的HTTP请求header中加入"x-amz-acl"或“x-amz-grant-permission”,目前ceph已经实现了emailAddress、id、url三种类型认证方式，但是注意提交的value，一定要用 type=value的形式，下面介绍一下rgw中的源码具体实现。

S3中关于上传Object的具体标准请参考如下 http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUTacl.html

RGW的实现

第一步根据之前源码介绍的找入口方法，定位到以下函数

#src/rgw/rgw_rest_s3.cc
RGWOp *RGWHandler_REST_Obj_S3::op_put()
{
  if (is_acl_op()) {
    return new RGWPutACLs_ObjStore_S3;
  }
  if (s->init_state.src_bucket.empty())
    return new RGWPutObj_ObjStore_S3; #这里对应的是PUT方式上传Object
  else
    return new RGWCopyObj_ObjStore_S3;
}

再看一下这个类的定义如下

#src/rgw/rgw_rest_s3.cc
class RGWPutObj_ObjStore_S3 : public RGWPutObj_ObjStore {
public:
  RGWPutObj_ObjStore_S3() {}
  ~RGWPutObj_ObjStore_S3() {}

  int get_params(); #这里实现HTTP请求头的数据处理
  int get_data(bufferlist& bl);
  void send_response();

  int validate_aws4_single_chunk(char *chunk_str,
                                 char *chunk_data_str,
                                 unsigned int chunk_data_size,
                                 string chunk_signature);
  int validate_and_unwrap_available_aws4_chunked_data(bufferlist& bl_in,
                                                      bufferlist& bl_out);
};

在get_params的实现中通过create_s3_policy来实现对policy的构建

#src/rgw/rgw_rest_s3.cc

int RGWPutObj_ObjStore_S3::get_params()
{
  RGWObjectCtx& obj_ctx = *static_cast<RGWObjectCtx *>(s->obj_ctx);
  map<string, bufferlist> src_attrs;
  size_t pos;
  int ret;

  RGWAccessControlPolicy_S3 s3policy(s->cct);
  if (!s->length)
    return -ERR_LENGTH_REQUIRED;

  ret = create_s3_policy(s, store, s3policy, s->owner); #创建policy
  if (ret < 0)
    return ret;

  policy = s3policy;

再看一下create_s3_policy有两类s3policy的构建方法，一类是create_from_headers，另外一类是create_canned，我们这里是通过http header设置，所以选create_from_headers

#src/rgw/rgw_rest_s3.cc
static int create_s3_policy(struct req_state *s, RGWRados *store,
			    RGWAccessControlPolicy_S3& s3policy,
			    ACLOwner& owner)
{
  if (s->has_acl_header) {
    if (!s->canned_acl.empty())
      return -ERR_INVALID_REQUEST;

    return s3policy.create_from_headers(store, s->info.env, owner);
  }

  return s3policy.create_canned(owner, s->bucket_owner, s->canned_acl);
}

再看一下create_from_headers的实现，其基本流程是遍历s3_acl_header获得对应HTTP的ACL类型，最后通过parse_grantee_str生成对应的规则

#src/rgw/rgw_acl_s3.cc
static const s3_acl_header acl_header_perms[] = {
  {RGW_PERM_READ, "HTTP_X_AMZ_GRANT_READ"},
  {RGW_PERM_WRITE, "HTTP_X_AMZ_GRANT_WRITE"},
  {RGW_PERM_READ_ACP,"HTTP_X_AMZ_GRANT_READ_ACP"},
  {RGW_PERM_WRITE_ACP, "HTTP_X_AMZ_GRANT_WRITE_ACP"},
  {RGW_PERM_FULL_CONTROL, "HTTP_X_AMZ_GRANT_FULL_CONTROL"},
  {0, NULL}
};

int RGWAccessControlPolicy_S3::create_from_headers(RGWRados *store, RGWEnv *env, ACLOwner& _owner)
{
  std::list<ACLGrant> grants;
 
  for (const struct s3_acl_header *p = acl_header_perms; p->rgw_perm; p++) {
    if (parse_acl_header(store, env, p, grants) < 0) #根据匹配到的header生成对应acl
      return false;
  }

  RGWAccessControlList_S3& _acl = static_cast<RGWAccessControlList_S3 &>(acl);
  int r = _acl.create_from_grants(grants);

  owner = _owner;

  return r;
}


static int parse_acl_header(RGWRados *store, RGWEnv *env,
         const struct s3_acl_header *perm, std::list<ACLGrant>& _grants)
{
  std::list<string> grantees;
  std::string hacl_str;

  const char *hacl = get_acl_header(env, perm);
  if (hacl == NULL)
    return 0;

  hacl_str = hacl;
  get_str_list(hacl_str, ",", grantees);

  for (list<string>::iterator it = grantees.begin(); it != grantees.end(); ++it) {
    ACLGrant grant;
    int ret = parse_grantee_str(store, *it, perm, grant); #最终实现acl的匹配在这里面
    if (ret < 0)
      return ret;

    _grants.push_back(grant);
  }

  return 0;
}

static int parse_grantee_str(RGWRados *store, string& grantee_str,
        const struct s3_acl_header *perm, ACLGrant& grant)
{
  string id_type, id_val_quoted;
  int rgw_perm = perm->rgw_perm;
  int ret;

  RGWUserInfo info;

  ret = parse_key_value(grantee_str, id_type, id_val_quoted); #注意这里对header中传进来的type=value进行处理
  if (ret < 0)
    return ret;

  string id_val = rgw_trim_quotes(id_val_quoted);

  if (strcasecmp(id_type.c_str(), "emailAddress") == 0) {
    ret = rgw_get_user_info_by_email(store, id_val, info);
    if (ret < 0)
      return ret;

    grant.set_canon(info.user_id, info.display_name, rgw_perm);
  } else if (strcasecmp(id_type.c_str(), "id") == 0) {
    rgw_user user(id_val);
    ret = rgw_get_user_info_by_uid(store, user, info);
    if (ret < 0)
      return ret;

    grant.set_canon(info.user_id, info.display_name, rgw_perm);
  } else if (strcasecmp(id_type.c_str(), "uri") == 0) {
    ACLGroupTypeEnum gid = grant.uri_to_group(id_val);
    if (gid == ACL_GROUP_NONE)
      return -EINVAL;

    grant.set_group(gid, rgw_perm);
  } else {
    return -EINVAL;
  }

  return 0;
}

这里需要重点提到的一点就是在parse_grantee_str中parse_key_value方法，如果你传进来的header字段的value不包含"="，那么你就悲剧了。

#src/rgw/rgw_common.cc
int parse_key_value(string& in_str, const char *delim, string& key, string& val)
{
  if (delim == NULL)
    return -EINVAL;

  int pos = in_str.find(delim);
  if (pos < 0)
    return -EINVAL;

  trim_whitespace(in_str.substr(0, pos), key);
  pos++;

  trim_whitespace(in_str.substr(pos), val);

  return 0;
}

int parse_key_value(string& in_str, string& key, string& val)
{
  return parse_key_value(in_str, "=", key,val); #将header里的value按"="进行拆分
}

转载于:https://my.oschina.net/diluga/blog/1057730