需求描述
用户A需要在上传文件到S3的同时设置允许用户test1可读,上传数据和设置ACL需要一步完成。其目的是尽量减少http请求次数,提高并发效率。
python用例
from boto.s3.connection import S3Connection
import boto
import os
endpoint = 's3.ceph.work'
bucket_name = 'test3'
access_key = ''
secret_key = ''
key_name = 'abc'
conn = boto.connect_s3(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
host=endpoint,
is_secure=False,
calling_format=boto.s3.connection.SubdomainCallingFormat(),
validate_certs=False,
)
bucket = conn.get_bucket(bucket_name)
key_ = bucket.new_key(key_name)
#方法1,通常一般用这个方法
key_.set_contents_from_string('aa') #第一次http请求,PUT操作,上传数据
key_.add_user_grant('READ','test1') #第二次http请求,PUT操作,设置ACL
#方法2,设置http请求头
headers = {"x-amz-grant-read":"id=test1"} #这里切记要在id(type)后面加上"=",不然就踩坑了
key_.set_contents_from_string('aa', headers=headers) #一次http请求,PUT操作,上传数据同时设置Object的ACL
原理说明
方法2其实也是遵循了S3标准的操作,只需要在上传Object的HTTP请求header中加入"x-amz-acl"或“x-amz-grant-permission”,目前ceph已经实现了emailAddress、id、url三种类型认证方式,但是注意提交的value,一定要用 type=value的形式,下面介绍一下rgw中的源码具体实现。
S3中关于上传Object的具体标准请参考如下 http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUTacl.html
RGW的实现
第一步根据之前源码介绍的找入口方法,定位到以下函数
#src/rgw/rgw_rest_s3.cc
RGWOp *RGWHandler_REST_Obj_S3::op_put()
{
if (is_acl_op()) {
return new RGWPutACLs_ObjStore_S3;
}
if (s->init_state.src_bucket.empty())
return new RGWPutObj_ObjStore_S3; #这里对应的是PUT方式上传Object
else
return new RGWCopyObj_ObjStore_S3;
}
再看一下这个类的定义如下
#src/rgw/rgw_rest_s3.cc
class RGWPutObj_ObjStore_S3 : public RGWPutObj_ObjStore {
public:
RGWPutObj_ObjStore_S3() {}
~RGWPutObj_ObjStore_S3() {}
int get_params(); #这里实现HTTP请求头的数据处理
int get_data(bufferlist& bl);
void send_response();
int validate_aws4_single_chunk(char *chunk_str,
char *chunk_data_str,
unsigned int chunk_data_size,
string chunk_signature);
int validate_and_unwrap_available_aws4_chunked_data(bufferlist& bl_in,
bufferlist& bl_out);
};
在get_params的实现中通过create_s3_policy来实现对policy的构建
#src/rgw/rgw_rest_s3.cc
int RGWPutObj_ObjStore_S3::get_params()
{
RGWObjectCtx& obj_ctx = *static_cast<RGWObjectCtx *>(s->obj_ctx);
map<string, bufferlist> src_attrs;
size_t pos;
int ret;
RGWAccessControlPolicy_S3 s3policy(s->cct);
if (!s->length)
return -ERR_LENGTH_REQUIRED;
ret = create_s3_policy(s, store, s3policy, s->owner); #创建policy
if (ret < 0)
return ret;
policy = s3policy;
再看一下create_s3_policy有两类s3policy的构建方法,一类是create_from_headers,另外一类是create_canned,我们这里是通过http header设置,所以选create_from_headers
#src/rgw/rgw_rest_s3.cc
static int create_s3_policy(struct req_state *s, RGWRados *store,
RGWAccessControlPolicy_S3& s3policy,
ACLOwner& owner)
{
if (s->has_acl_header) {
if (!s->canned_acl.empty())
return -ERR_INVALID_REQUEST;
return s3policy.create_from_headers(store, s->info.env, owner);
}
return s3policy.create_canned(owner, s->bucket_owner, s->canned_acl);
}
再看一下create_from_headers的实现,其基本流程是遍历s3_acl_header获得对应HTTP的ACL类型,最后通过parse_grantee_str生成对应的规则
#src/rgw/rgw_acl_s3.cc
static const s3_acl_header acl_header_perms[] = {
{RGW_PERM_READ, "HTTP_X_AMZ_GRANT_READ"},
{RGW_PERM_WRITE, "HTTP_X_AMZ_GRANT_WRITE"},
{RGW_PERM_READ_ACP,"HTTP_X_AMZ_GRANT_READ_ACP"},
{RGW_PERM_WRITE_ACP, "HTTP_X_AMZ_GRANT_WRITE_ACP"},
{RGW_PERM_FULL_CONTROL, "HTTP_X_AMZ_GRANT_FULL_CONTROL"},
{0, NULL}
};
int RGWAccessControlPolicy_S3::create_from_headers(RGWRados *store, RGWEnv *env, ACLOwner& _owner)
{
std::list<ACLGrant> grants;
for (const struct s3_acl_header *p = acl_header_perms; p->rgw_perm; p++) {
if (parse_acl_header(store, env, p, grants) < 0) #根据匹配到的header生成对应acl
return false;
}
RGWAccessControlList_S3& _acl = static_cast<RGWAccessControlList_S3 &>(acl);
int r = _acl.create_from_grants(grants);
owner = _owner;
return r;
}
static int parse_acl_header(RGWRados *store, RGWEnv *env,
const struct s3_acl_header *perm, std::list<ACLGrant>& _grants)
{
std::list<string> grantees;
std::string hacl_str;
const char *hacl = get_acl_header(env, perm);
if (hacl == NULL)
return 0;
hacl_str = hacl;
get_str_list(hacl_str, ",", grantees);
for (list<string>::iterator it = grantees.begin(); it != grantees.end(); ++it) {
ACLGrant grant;
int ret = parse_grantee_str(store, *it, perm, grant); #最终实现acl的匹配在这里面
if (ret < 0)
return ret;
_grants.push_back(grant);
}
return 0;
}
static int parse_grantee_str(RGWRados *store, string& grantee_str,
const struct s3_acl_header *perm, ACLGrant& grant)
{
string id_type, id_val_quoted;
int rgw_perm = perm->rgw_perm;
int ret;
RGWUserInfo info;
ret = parse_key_value(grantee_str, id_type, id_val_quoted); #注意这里对header中传进来的type=value进行处理
if (ret < 0)
return ret;
string id_val = rgw_trim_quotes(id_val_quoted);
if (strcasecmp(id_type.c_str(), "emailAddress") == 0) {
ret = rgw_get_user_info_by_email(store, id_val, info);
if (ret < 0)
return ret;
grant.set_canon(info.user_id, info.display_name, rgw_perm);
} else if (strcasecmp(id_type.c_str(), "id") == 0) {
rgw_user user(id_val);
ret = rgw_get_user_info_by_uid(store, user, info);
if (ret < 0)
return ret;
grant.set_canon(info.user_id, info.display_name, rgw_perm);
} else if (strcasecmp(id_type.c_str(), "uri") == 0) {
ACLGroupTypeEnum gid = grant.uri_to_group(id_val);
if (gid == ACL_GROUP_NONE)
return -EINVAL;
grant.set_group(gid, rgw_perm);
} else {
return -EINVAL;
}
return 0;
}
这里需要重点提到的一点就是在parse_grantee_str中parse_key_value方法,如果你传进来的header字段的value不包含"=",那么你就悲剧了。
#src/rgw/rgw_common.cc
int parse_key_value(string& in_str, const char *delim, string& key, string& val)
{
if (delim == NULL)
return -EINVAL;
int pos = in_str.find(delim);
if (pos < 0)
return -EINVAL;
trim_whitespace(in_str.substr(0, pos), key);
pos++;
trim_whitespace(in_str.substr(pos), val);
return 0;
}
int parse_key_value(string& in_str, string& key, string& val)
{
return parse_key_value(in_str, "=", key,val); #将header里的value按"="进行拆分
}