object-server用来接收用户发送的object请求(即文件的请求，比如上传，下载，删除等)。例如上传一个文件，proxy通过ring文件随机选择三个object-server，然后转发请求到object-server，然后object-server接收请求，做最终处理（例如存储文件的操作） 。
object-server的__call__方法主要用来判断请求的方法，然后执行。
def __call__(self, env, start_response):
"""WSGI Application entry point for the Swift Object Server."""
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):#不支持UTF8
res = HTTPPreconditionFailed(body='Invalid UTF8')
else:
try:
# disallow methods which have not been marked 'public'
try:#判断方法是否是共有的方法
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
except AttributeError:
res = HTTPMethodNotAllowed()
else:
res = method(req)#执行方法,例如PUT
except (Exception, Timeout):
self.logger.exception(_('ERROR __call__ error with %(method)s'
' %(path)s '), {'method': req.method, 'path': req.path})
res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = time.time() - start_time
if self.log_requests:
log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % (
req.remote_addr,
time.strftime('%d/%b/%Y:%H:%M:%S +0000',
time.gmtime()),
req.method, req.path, res.status.split()[0],
res.content_length or '-', req.referer or '-',
req.headers.get('x-trans-id', '-'),
req.user_agent or '-',
trans_time)
if req.method == 'REPLICATE':
self.logger.debug(log_line)
else:
self.logger.info(log_line)
if req.method in ('PUT', 'DELETE'):
slow = self.slow - trans_time
if slow > 0:
sleep(slow)
return res(env, start_response)
/swift/obj/server.py中的DiskFile类 用来初始化object，其中包括了一切对于一个object的操作,文件上传，会先存储在，tmp中（临时文件），等到文件上传完毕，存储在相应的路径中，关于路径，在DiskFile初始化实例的时候，会生成object的路径，通过哈希算法保证路径的唯一并且随机性。其中把生成的哈希值保存在hashes.pkl文件中。object 生成后，会通知对应的container，然后container会通知对应的account，这样account/container/object就上传成功了。
@public
def PUT(self, request):
"""Handle HTTP PUT requests for the Swift Object Server."""
start_time = time.time()
try:
device, partition, account, container, obj = \#得到device,partition,account,container,obj等相关信息
split_path(unquote(request.path), 5, 5, True)
validate_device_partition(device, partition)
except ValueError, err:
self.logger.increment('PUT.errors')
return HTTPBadRequest(body=str(err), request=request,
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):#是否检查挂载
self.logger.increment('PUT.errors')
return HTTPInsufficientStorage(drive=device, request=request)
if 'x-timestamp' not in request.headers or \#如果没有时间戳，或者时间戳不合法
not check_float(request.headers['x-timestamp']):
self.logger.increment('PUT.errors')
return HTTPBadRequest(body='Missing timestamp', request=request,
content_type='text/plain')
error_response = check_object_creation(request, obj)#检查要创建的obj是否正确，包括它的元数据。如果都正确返回None
if error_response:
self.logger.increment('PUT.errors')
return error_response
new_delete_at = int(request.headers.get('X-Delete-At') or 0)
if new_delete_at and new_delete_at < time.time():#判断dalete_at
self.logger.increment('PUT.errors')
return HTTPBadRequest(body='X-Delete-At in past', request=request,
content_type='text/plain')
file = DiskFile(self.devices, device, partition, account, container,#一切OK，实例化，文件的类
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
orig_timestamp = file.metadata.get('X-Timestamp')
upload_expiration = time.time() + self.max_upload_time
etag = md5()
upload_size = 0
last_sync = 0
with file.mkstemp() as (fd, tmppath):#创建临时文件
if 'content-length' in request.headers:
try:
fallocate(fd, int(request.headers['content-length']))#预留硬盘空间
except OSError:
return HTTPInsufficientStorage(drive=device,
request=request)
reader = request.environ['wsgi.input'].read
for chunk in iter(lambda: reader(self.network_chunk_size), ''):#写数据
upload_size += len(chunk)
if time.time() > upload_expiration:
self.logger.increment('PUT.timeouts')
return HTTPRequestTimeout(request=request)
etag.update(chunk)
while chunk:
written = os.write(fd, chunk)
chunk = chunk[written:]
# For large files sync every 512MB (by default) written
if upload_size - last_sync >= self.bytes_per_sync:
tpool.execute(os.fdatasync, fd)
drop_buffer_cache(fd, last_sync, upload_size - last_sync)
last_sync = upload_size
sleep()
if 'content-length' in request.headers and \
int(request.headers['content-length']) != upload_size:
return HTTPClientDisconnect(request=request)
etag = etag.hexdigest()
if 'etag' in request.headers and \
request.headers['etag'].lower() != etag:
return HTTPUnprocessableEntity(request=request)
metadata = {
'X-Timestamp': request.headers['x-timestamp'],
'Content-Type': request.headers['content-type'],
'ETag': etag,
'Content-Length': str(os.fstat(fd).st_size),
}
metadata.update(val for val in request.headers.iteritems()
if val[0].lower().startswith('x-object-meta-') and
len(val[0]) > 14)
for header_key in self.allowed_headers:
if header_key in request.headers:
header_caps = header_key.title()
metadata[header_caps] = request.headers[header_key]
old_delete_at = int(file.metadata.get('X-Delete-At') or 0)
if old_delete_at != new_delete_at:
if new_delete_at:
self.delete_at_update('PUT', new_delete_at, account,
container, obj, request.headers, device)
if old_delete_at:
self.delete_at_update('DELETE', old_delete_at, account,
container, obj, request.headers, device)
file.put(fd, tmppath, metadata)#把生成的临时文件，进行完善，然后rename到时间的位置上去。
file.unlinkold(metadata['X-Timestamp'])#删除旧版本的文件
if not orig_timestamp or \#更新container,
orig_timestamp < request.headers['x-timestamp']:
self.container_update('PUT', account, container, obj,
request.headers,
{'x-size': file.metadata['Content-Length'],
'x-content-type': file.metadata['Content-Type'],
'x-timestamp': file.metadata['X-Timestamp'],
'x-etag': file.metadata['ETag'],
'x-trans-id': request.headers.get('x-trans-id', '-')},
device)
resp = HTTPCreated(request=request, etag=etag)
self.logger.timing_since('PUT.timing', start_time)
return resp
DiskFile.put
def put(self, fd, tmppath, metadata, extension='.data'):
metadata['name'] = self.name
timestamp = normalize_timestamp(metadata['X-Timestamp'])
write_metadata(fd, metadata)#把元数据写入文件中，
if 'Content-Length' in metadata:
self.drop_cache(fd, 0, int(metadata['Content-Length']))
tpool.execute(os.fsync, fd)
invalidate_hash(os.path.dirname(self.datadir))#生成hashes.pkl
renamer(tmppath, os.path.join(self.datadir, timestamp + extension))#重命名文件
self.metadata = metadata
通知对应的container
def container_update(self, op, account, container, obj, headers_in,
headers_out, objdevice):
host = headers_in.get('X-Container-Host', None)
partition = headers_in.get('X-Container-Partition', None)
contdevice = headers_in.get('X-Container-Device', None)
if not all([host, partition, contdevice]):#判断数据的完整性
return
self.async_update(op, account, container, obj, host, partition,
contdevice, headers_out, objdevice)
aync方法，成功就返回，失败，进行处理，
aync方法，成功就返回，失败，进行处理，把数据写到aync_dir里，然后等待后台的更新器来更新。
def async_update(self, op, account, container, obj, host, partition,
contdevice, headers_out, objdevice):
full_path = '/%s/%s/%s' % (account, container, obj)
if all([host, partition, contdevice]):
try:
with ConnectionTimeout(self.conn_timeout):
ip, port = host.rsplit(':', 1)
conn = http_connect(ip, port, contdevice, partition, op,#建立连接
full_path, headers_out)
with Timeout(self.node_timeout):
response = conn.getresponse()
response.read()
if is_success(response.status):#成功就返回
return
else:
self.logger.error(_('ERROR Container update failed '
'(saving for async update later): %(status)d '
'response from %(ip)s:%(port)s/%(dev)s'),
{'status': response.status, 'ip': ip, 'port': port,
'dev': contdevice})
except (Exception, Timeout):
self.logger.exception(_('ERROR container update failed with '
'%(ip)s:%(port)s/%(dev)s (saving for async update later)'),
{'ip': ip, 'port': port, 'dev': contdevice})
async_dir = os.path.join(self.devices, objdevice, ASYNCDIR)#失败处理
ohash = hash_path(account, container, obj)
self.logger.increment('async_pendings')
write_pickle(
{'op': op, 'account': account, 'container': container,
'obj': obj, 'headers': headers_out},
os.path.join(async_dir, ohash[-3:], ohash + '-' +
normalize_timestamp(headers_out['x-timestamp'])),
os.path.join(self.devices, objdevice, 'tmp'))
扫一扫
350
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
