Last updated on 8 months ago
前言
本文主题： 当我们用 s3cmd（s3，Swift等）上传文件时候，rgw是如何处理的（这次先单独讨论 整体上传）
从一个s3请求 到如何转换成 rgwRequest 再到 数据如何切割存储到rados中
整体架构
接受请求的 前端– beast
在N版中默认 是用 beast （可以异步处理http服务器 ）前端来解析HTTP请求，rgw 默认是使用beast，并用 RGWAsioFrontend 类封装，其关系如下图所示
RGWAsioFrontend 类关系
front 启动流程图
关键流程
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
| int AsioFrontend::init()
{
boost::system::error_code ec;
bool socket_bound = false;
for (auto& l : listeners) {
l.acceptor.listen(max_connection_backlog);
l.acceptor.async_accept(l.socket,
[this, &l] (boost::system::error_code ec) {
accept(l, ec);
});
}
}
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
| void AsioFrontend::accept(Listener& l, boost::system::error_code ec)
{
l.acceptor.async_accept(l.socket,
[this, &l] (boost::system::error_code ec) {
accept(l, ec);
});
#else
{
#endif
boost::asio::spawn(context,
[this, s=std::move(socket)] (boost::asio::yield_context yield) mutable {
Connection conn{s};
auto c = connections.add(conn);
auto buffer = std::make_unique<parse_buffer>();
boost::system::error_code ec;
handle_connection(context, env, s, *buffer, false, pause_mutex,
scheduler.get(), ec, yield);
s.shutdown(tcp::socket::shutdown_both, ec);
});
}
}
|
http 请求时如转换的 OP的
处理流程
Restful -> Handle -> OP
Restul 的注册
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
| rgw_main.cc
int mian(){
.....
RGWREST rest;
list<string> apis;
get_str_list(g_conf()->rgw_enable_apis, apis);
map<string, bool> apis_map;
for (list<string>::iterator li = apis.begin(); li != apis.end(); ++li) {
apis_map[*li] = true;
}
if (apis_map.count("s3") > 0 || s3website_enabled) {
if (! swift_at_root) {
rest.register_default_mgr(set_logging(rest_filter(store, RGW_REST_S3,
new RGWRESTMgr_S3(s3website_enabled, sts_enabled, iam_enabled, pubsub_enabled))));
} else {
return EINVAL;
}
}
}
......
.....
}
|
Handle 生成
相关类图
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
rgw_process.cc
process_request()
-->RGWHandler_REST *handler = rest->get_handler(store, s);
-->RGWHandler_REST* RGWRESTMgr_S3::get_handler(struct req_state* const s,
const rgw::auth::StrategyRegistry& auth_registry,
const std::string& frontend_prefix)
{ RGWHandler_REST* handler;
if (is_s3website) {
if (s->init_state.url_bucket.empty()) {
handler = new RGWHandler_REST_Service_S3Website(auth_registry);
} else if (s->object.empty()) {
handler = new RGWHandler_REST_Bucket_S3Website(auth_registry);
} else {
handler = new RGWHandler_REST_Obj_S3Website(auth_registry);
}
} else {
if (s->init_state.url_bucket.empty()) {
handler = new RGWHandler_REST_Service_S3(auth_registry, enable_sts, enable_iam, enable_pubsub);
} else if (s->object.empty()) {
handler = new RGWHandler_REST_Bucket_S3(auth_registry, enable_pubsub);
} else {
handler = new RGWHandler_REST_Obj_S3(auth_registry);
}
}
return handler;
}
|
OP生成
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
| rgw_process.cc
process_request()
-->RGWOp* op = nullptr;
op = handler->get_op(store);
RGWOp* RGWHandler_REST::get_op(RGWRados* store)
{
RGWOp *op;
switch (s->op) {
....
case OP_PUT:
op = op_put();
break;
.....
default:
return NULL;
}
if (op) {
op->init(store, s, this);
}
return op;
}
RGWOp *RGWHandler_REST_Obj_S3::op_put()
{
if (is_acl_op()) {
return new RGWPutACLs_ObjStore_S3;
} else if (is_tagging_op()) {
return new RGWPutObjTags_ObjStore_S3;
} else if (is_obj_retention_op()) {
return new RGWPutObjRetention_ObjStore_S3;
} else if (is_obj_legal_hold_op()) {
return new RGWPutObjLegalHold_ObjStore_S3;
}
if (s->init_state.src_bucket.empty())
return new RGWPutObj_ObjStore_S3;
else
return new ;
}
|
总结： 三者的联系（以 put obj 为例）
OP的执行
请求的验证
- verify_requester -> RGWHandler_REST_S3::authorize
根据不同 API类型来进行请求的权限验证，支持 STS 和 keystone 认证 
- postauth_init -> RGWHandler_REST_S3::postauth_init()
验证 Bucket 和 Object· 命名是否合法
- rgw_process_authenticated
- **RGWHandler_REST::init_permissions -> do_init_permissions()**
进行权限初始化 从持久化数据中获取ACL
- RGWHandler_REST::read_permissions ：
根据不同的OP来获取RGWOp的操作权限
- **RGWOp::init_processing -> RGWOp::init_quota()**
- 获取Bucket的quota
- 获取User的quota
- RGWOp::verify_op_mask：
- **通过位运算的方式来判断当前操作是否在支持的操作列表中**
- 
- RGWPutObj::verify_permission
验证ACL 是否匹配
- RGWOp::verify_params
对put 来说是 验证Obj的大小是否超过限制
1
2
3
4
5
6
7
8
9
| /third-bucket/15M
content-length:15728640
content-type:application/octet-stream
host:10.1.31.40:18021
x-amz-content-sha256:167b76d3a8d20df15c421d48877c330597f6309d6b55c7b5327df5d89a51423f
x-amz-date:20230320T040318Z
x-amz-meta-s3cmd-attrs:atime:1679131627/ctime:1679131611/gid:0/gname:root/md5:14b17234e237505421b6492b8d757507/mode:33188/mtime:1679131611/uid:0/uname:root
x-amz-storage-class:STANDARD
|