1. cloudkitty的计费算法
本文代码主要基于cloudkitty的ocata版本
1.1. 服务启动
在代码cloudkitty/cli/processor.py中的
def main():
service.prepare_service()
processor = orchestrator.Orchestrator()
try:
processor.process()
except KeyboardInterrupt:
processor.terminate()
if __name__ == '__main__':
main()
1.2. 周期性执行计费框架orchestrator执行process（）
def process(self):
while True:
#这里没有做任何事，应该是预留
self.process_messages()
#获取包含cloudkitty的并含有ratting角色的项目
self._load_tenant_list()
while len(self._tenants):
for tenant in self._tenants[:]:
#通过tooz获取项目锁，防止冲突
lock = self._lock(tenant)
if lock.acquire(blocking=False):
if not self._check_state(tenant):
self._tenants.remove(tenant)
else:
#处理计费统计任务
worker = Worker(self.collector,
self.storage,
tenant)
worker.run()
lock.release()
self.coord.heartbeat()
# NOTE(sheeprine): Slow down looping if all tenants are
# being processed
eventlet.sleep(1)
# FIXME(sheeprine): We may cause a drift here
eventlet.sleep(self._period)
1.3. _check_status说明
def _check_state(self, tenant_id):
timestamp = self.storage.get_state(tenant_id)
return ck_utils.check_time_state(timestamp,
self._period,
self._wait_time)
get_status位于cloudkitty/storage/sqlalchemy/__init__.py中
def get_state(self, tenant_id=None):
session = db.get_session()
#到数据库表rated_data_frames获取指定租户向的计费数据，并且根据时间记录的开始时间降序排列
q = utils.model_query(
self.frame_model,
session)
if tenant_id:
q = q.filter(
self.frame_model.tenant_id == tenant_id)
q = q.order_by(
self.frame_model.begin.desc())
#获取取到的第一个数据，如果初始情况下，可能获取不到数据
r = q.first()
if r:
return ck_utils.dt2ts(r.begin)
check_time_status位于cloudkitty/utils.py中
def check_time_state(timestamp=None, period=0, wait_time=0):
#获取不到时间，直接获取每个月的起始时间
if not timestamp:
month_start = get_month_start()
return dt2ts(month_start)
#获取到时间了，就看上次计费时间+统计周期是否已经到达预定值，如果达到预定值，则开始计费
now = utcnow_ts()
next_timestamp = timestamp + period
if next_timestamp + wait_time < now:
return next_timestamp
return 0
1.4. Worker run说明
该代码位于cloudkitty/orchestrator.py中
def run(self):
while True:
#判读是否到计费周期了，timestamp为起始周期，一般获取到的都是当前第一个计费周期
timestamp = self.check_state()
if not timestamp:
break
#服务包括：compute,image,volume,network.bw.in,network.bw.out,network.floating
for service in CONF.collect.services:
try:
try:
#根据stevestore的插件机制，获取指定服务在指定时间段的事件信息
data = self._collect(service, timestamp)
except collector.NoDataCollected:
raise
except Exception as e:
LOG.warning(
_LW('Error while collecting service '
'%(service)s: %(error)s'),
{'service': service, 'error': e})
raise collector.NoDataCollected('', service)
except collector.NoDataCollected:
begin = timestamp
end = begin + self._period
for processor in self._processors:
processor.obj.nodata(begin, end)
self._storage.nodata(begin, end, self._tenant_id)
else:
# Rating
# 根据stevestore的插件功能，查询’cloudkitty.rating.processors’提供的计费策略，并进行计费，当前主要讲hashmap的计费代码
for processor in self._processors:
processor.obj.process(data)
# Writing
self._storage.append(data, self._tenant_id)
# We're getting a full period so we directly commit
self._storage.commit(self._tenant_id)
# _collect是根据service的起始时间后端存储中获取对应的数据
def _collect(self, service, start_timestamp):
next_timestamp = start_timestamp + self._period
raw_data = self._collector.retrieve(service,
start_timestamp,
next_timestamp,
self._tenant_id)
# raw_data的格式为：instance = {'instance_id':xx, 'flavor': xxx}
raw_data = {'compute': [{'desc': instance,
'vol': {'unit': 'instance', 'qty': 1.0}}]}
if raw_data:
return [{'period': {'begin': start_timestamp,
'end': next_timestamp},
'usage': raw_data}]
# retrieve (本文以ceilometer， compute为例)，其中ceilometer的retrieve定义在基类中，最终走到ceilometer的get_compute方法中
代码路径：cloudkitty/collector/ceilometer.py
def get_compute(self, start, end=None, project_id=None, q_filter=None):
#这个方法是调用ceilometer的statistics的统计方法，统计所有云主机在指定时间的cpu统计值，在这段时间外创建/删除的云主机则没有统计值信息
active_instance_ids = self.active_resources('cpu', start, end,
project_id, q_filter)
compute_data = []
for instance_id in active_instance_ids:
if not self._cacher.has_resource_detail('compute', instance_id):
#调用ceilometer resource-show获取指定资源的信息
raw_resource = self._conn.resources.get(instance_id)
#组装instance的信息
instance = self.t_ceilometer.strip_resource_data('compute',
raw_resource)
self._cacher.add_resource_detail('compute',
instance_id,
instance)
instance = self._cacher.get_resource_detail('compute',
instance_id)
# 将数据转化成以下格式
data = {'desc': instance,
'vol': {'unit': 'instance', 'qty': 1.0}}
compute_data.append(
self.t_cloudkitty.format_item(instance, self.units_mappings[
"compute"], 1))
if not compute_data:
raise collector.NoDataCollected(self.collector_name, 'compute')
# 将数据转化成以下格式
data = {'compute': [{'desc': instance,
'vol': {'unit': 'instance', 'qty': 1.0}}]}
return self.t_cloudkitty.format_service('compute', compute_data)
#strip_resource_data就是根据从ceilometer中获取的资源，组装数据简体
def strip_resource_data(self, res_type, res_data):
res_type = res_type.replace('.', '_')
strip_func = getattr(self, '_strip_' + res_type, None)
if strip_func:
return strip_func(res_data)
return self.generic_strip(res_type, res_data) or res_data
#最终在cloudkitty/transformer/ceilometer.py 查询到_strip_computer的函数
def _strip_compute(self, data):
res_data = self.generic_strip('compute', data)
res_data['instance_id'] = data.resource_id
res_data['project_id'] = data.project_id
res_data['user_id'] = data.user_id
res_data['metadata'] = {}
for field in data.metadata:
if field.startswith('user_metadata'):
res_data['metadata'][field[14:]] = data.metadata[field]
return res_data
# generic_strip就是到对应的compute_map获取指定key值的数据，compute_map格式如下
compute_map = {
'name': ['display_name'],
'flavor': ['flavor.name', 'instance_type'],
'vcpus': ['vcpus'],
'memory': ['memory_mb'],
'image_id': ['image.id', 'image_meta.base_image_ref'],
'availability_zone': [
'availability_zone',
'OS-EXT-AZ.availability_zone'],
}
1.5. hashmap计费
process的计费代码该代码位于cloudkitty/rating/hash/__init__.py中
def process(self, data):
#data的格式为
instance = {'instance_id':xx, 'flavor': xxx}
raw_data = {'compute': [{'desc': instance, 'vol': {'unit': 'instance', 'qty': 1.0}}]}
data= [{'period': {'begin': start_timestamp,'end': next_timestamp}, 'usage': raw_data}]
for cur_data in data:
cur_usage = cur_data['usage']
for service_name, service_data in cur_usage.items():
for item in service_data:
self._res = {}
self.process_services(service_name, item)
self.process_fields(service_name, item)
self.add_rating_informations(item)
return data
1.6. process_services说明
process_services的计费代码该代码位于cloudkitty/rating/hash/__init__.py中
def process_services(self, service_name, data):
#self._entries策略的加载见，”加载hashmap计费策略”
“最终的格式如下：
{
'volume': {
'thresholds': {
u'volume_thresholds': {
Decimal('1.00000000'): {
'cost': Decimal('5.00000000'), 'type': u'flat'
}
}
},
'mappings': {}
},
'compute': {
'fields': {
u'flavor': {
'thresholds': {},
'mappings': {
u'instance_flavor': {
u'm1.tiny': {
'cost': Decimal('20.00000000'),
'type': u'flat'
}
}
}
}
},
'thresholds': {
u'instance_flavor': {
Decimal('2.00000000'): {
'cost': Decimal('2.00000000'),
'type': u'flat'
}
}
},
'mappings': {
u'instance_flavor': {
'cost': Decimal('20.00000000'),
'type': u'flat'
}
}
}
}
”
if service_name not in self._entries:
return
service_mappings = self._entries[service_name]['mappings']
"""
service_mappings = {'instance_flavor': {'cost': Decimal('20.00000000'),'type': u'flat'}
"""
for group_name, mapping in service_mappings.items():
#刷新self._res的值self.res = {"instance_flavor": {'flat': 20,
'rate': 1,
'threshold': {
'level': -1,
'cost': 0,
'type': 'flat',
'scope': 'field'}}}
self.update_result(group_name,
mapping['type'],
mapping['cost'])
service_thresholds = self._entries[service_name]['thresholds']
"""
service_thresholds = {'instance_flavor': {Decimal('2.00000000'): {'cost': Decimal('2.00000000'),'type': u'flat'}}
原始数据 qty的值为 1.0
"""
self.process_thresholds(service_thresholds,
data['vol']['qty'],
'service')
#self.update_result说明, 这个时间is_threshold为false，该函数主要在刷新资源计算值
def update_result(self,
group,
map_type,
cost,
level=0,
is_threshold=False,
threshold_scope='field'):
if group not in self._res:
self._res[group] = {'flat': 0,
'rate': 1,
'threshold': {
'level': -1,
'cost': 0,
'type': 'flat',
'scope': 'field'}}
if is_threshold:
best = self._res[group]['threshold']['level']
if level > best:
self._res[group]['threshold']['level'] = level
self._res[group]['threshold']['cost'] = cost
self._res[group]['threshold']['type'] = map_type
self._res[group]['threshold']['scope'] = threshold_scope
else:
if map_type == 'rate':
self._res[group]['rate'] *= cost
elif map_type == 'flat':
new_flat = cost
cur_flat = self._res[group]['flat']
if new_flat > cur_flat:
self._res[group]['flat'] = new_flat
1.7. Process_thrssholds说明
def process_thresholds(self,threshold_groups, cmp_level, threshold_type):
#service_thresholds = {'instance_flavor': {Decimal('2.00000000'): {'cost': Decimal('2.00000000'),'type': u'flat'}}
# cmp_level 假设为 1.0
# threshold_type 为service
for group_name, thresholds in threshold_groups.items():
for threshold_level, threshold in thresholds.items():
if cmp_level >= threshold_level:
self.update_result(
group_name,
threshold['type'],
threshold['cost'],
threshold_level,
True,
threshold_type)
#刷新self._res的值self.res = {"instance_flavor": {'flat': 20,
'rate': 1,
'threshold': {
'level': 2,
'cost': 2,
'type': 'flat',
'scope': 'field'}}}
1.8. process_fields说明
def process_fields(self, service_name, data):
if service_name not in self._entries:
return
if 'fields' not in self._entries[service_name]:
return
desc_data = data['desc']
field_mappings = self._entries[service_name]['fields']
#field_mappings = {
'flavor': {'thresholds': {},
'mappings': {'instance_flavor': {'m1.tiny': {'cost': Decimal('20.00000000'),'type': u'flat'}}}
}
for field_name, group_mappings in field_mappings.items():
if field_name not in desc_data:
continue
cmp_value = desc_data[field_name]
#这个只是简单的根据flavor的值，修改下self._res中的cost值
self.process_mappings(group_mappings['mappings'],
cmp_value)
if group_mappings['thresholds']:
self.process_thresholds(group_mappings['thresholds'],
decimal.Decimal(cmp_value),
'field')
#process_mapping的流程
def process_mappings(self,
mapping_groups,
cmp_value):
for group_name, mappings in mapping_groups.items():
for mapping_value, mapping in mappings.items():
if cmp_value == mapping_value:
self.update_result(
group_name,
mapping['type'],
mapping['cost'])
1.9. add_rating_informations说明
def add_rating_informations(self, data):
self.res = {"instance_flavor": {'flat': 20,
'rate': 1,
'threshold': {
'level': 2,
'cost': 2,
'type': 'flat',
'scope': 'field'}}}
if 'rating' not in data:
data['rating'] = {'price': 0}
#最终的计费规则则是再最外层的flat的基础价格上加上threshold的价格数据
#并将该数据保存到数据库中，实现周期计费的功能
for entry in self._res.values():
rate = entry['rate']
flat = entry['flat']
if entry['threshold']['scope'] == 'field':
if entry['threshold']['type'] == 'flat':
flat += entry['threshold']['cost']
else:
rate *= entry['threshold']['cost']
res = rate * flat
# FIXME(sheeprine): Added here to ensure that qty is decimal
res *= decimal.Decimal(data['vol']['qty'])
if entry['threshold']['scope'] == 'service':
if entry['threshold']['type'] == 'flat':
res += entry['threshold']['cost']
else:
res *= entry['threshold']['cost']
data['rating']['price'] += res
2. 加载hashmap计费策略_load_rates
改代码位于cloudkitty/rating/hash/__init__.py中
class HashMap(rating.RatingProcessorBase):
"""HashMap rating module.
HashMap can be used to map arbitrary fields of a resource to different
costs.
"""
module_name = 'hashmap'
description = 'HashMap rating module.'
hot_config = True
config_controller = root_api.HashMapConfigController
db_api = hash_db_api.get_instance()
def __init__(self, tenant_id=None):
super(HashMap, self).__init__(tenant_id)
self._entries = {}
self._res = {}
#加载计费策略
self._load_rates()
在计费算中的process.process_services代码中，会用到self._entries的全局变量，这个是计费策略，加载计费策略的代码如下：
def _load_rates(self):
self._entries = {}
hashmap = hash_db_api.get_instance()
# 从hashmap_services的表中获取当前的service列表
services_uuid_list = hashmap.list_services()
for service_uuid in services_uuid_list:
service_db = hashmap.get_service(uuid=service_uuid)
service_name = service_db.name
#加载mappings与thresholds的配置
self._load_service_entries(service_name, service_uuid)
fields_uuid_list = hashmap.list_fields(service_uuid)
for field_uuid in fields_uuid_list:
field_db = hashmap.get_field(uuid=field_uuid)
field_name = field_db.name
self._load_field_entries(service_name, field_name, field_uuid)
#经过上面转化，最终的self._entries格式为：
{
'volume': {
'thresholds': {
u'volume_thresholds': {
Decimal('1.00000000'): {
'cost': Decimal('5.00000000'), 'type': u'flat'
}
}
},
'mappings': {}
},
'compute': {
'fields': {
u'flavor': {
'thresholds': {},
'mappings': {
u'instance_flavor': {
u'm1.tiny': {
'cost': Decimal('20.00000000'),
'type': u'flat'
}
}
}
}
},
'thresholds': {
u'instance_flavor': {
Decimal('2.00000000'): {
'cost': Decimal('2.00000000'),
'type': u'flat'
}
}
},
'mappings': {
u'instance_flavor': {
'cost': Decimal('20.00000000'),
'type': u'flat'
}
}
}
}
2.1. _load_service_entries代码分析
#代码_load_service_entries()的逻辑，加载mappings与thresholds的配置
def _load_service_entries(self, service_name, service_uuid):
self._entries[service_name] = dict()
for entry_type in ('mappings', 'thresholds'):
for tenant in (None, self._tenant_id):
self._update_entries(entry_type,
self._entries[service_name],
service_uuid=service_uuid,
tenant_uuid=tenant)
#代码最终的entries的结构如下：
#界面创建的hashmap数据见hashmap截图
#其实compute中的service mappings 与 service thresholds可以不设置的
#由于本文为了说明其算法，简单的进行了设置
# 并且以下的 mappings中的"m1.tiny" :{"type":"flat", "cost":20},实际上是没有的,这个是在关联filed的时候，才会获取到的
mappings = {
"instance_flavor": { "m1.tiny" :{"type":"flat", "cost":20}, # 有value
"type" :"flat",
"cost":20
}
"_DEFAULT_": {}
}
}
thresholds = {
"instance_flavor": {
"threshold_level":{"type": "flat", "cost": 5},
"2":{"type": "flat", "cost": 2}
},
"_DEFAULT_": {}
}
self._entries = {
"compute": {
"thresholds":thresholds,
"mappings": mappings
}
}
#代码_update_entries的逻辑
def _update_entries(self,
entry_type,
root,
service_uuid=None,
field_uuid=None,
tenant_uuid=None):
hashmap = hash_db_api.get_instance()
#分别调用list_mappings, 以及 list_threshoulds方法，获取相关计费数据
list_func = getattr(hashmap, 'list_{}'.format(entry_type))
entries_uuid_list = list_func(
service_uuid=service_uuid,
field_uuid=field_uuid,
tenant_uuid=tenant_uuid)
#分别调用_load_mappings, 以及 _load_threshoulds方法，组装计费数据
load_func = getattr(self, '_load_{}'.format(entry_type))
entries = load_func(entries_uuid_list)
#将值保持到self._entries[service_name]中
if entry_type in root:
res = root[entry_type]
for group, values in entries.items():
if group in res:
res[group].update(values)
else:
res[group] = values
else:
root[entry_type] = entries
#_load_mappings的逻辑
def _load_mappings(self, mappings_uuid_list):
#加载后，返回的数据格式如下
"""
mappings = {
"instance_flavor": { "m1.tiny" :{"type":"flat", "cost":20}, # 有value
"type" :"flat", # 没有value
"cost":20 # 没有value
},
"_DEFAULT_": {}
}
}
"""
hashmap = hash_db_api.get_instance()
mappings = {}
for mapping_uuid in mappings_uuid_list:
mapping_db = hashmap.get_mapping(uuid=mapping_uuid)
if mapping_db.group_id:
group_name = mapping_db.group.name
else:
group_name = '_DEFAULT_'
if group_name not in mappings:
mappings[group_name] = {}
current_scope = mappings[group_name]
mapping_value = mapping_db.value
#含有value的情况是在flavor创建该flavor的价格的时候才会输入的，数据库表中没有对应的service_id
#在创建的最外层创建的时候，数据库表中有关联到具体的service_id
if mapping_value:
current_scope[mapping_value] = {}
current_scope = current_scope[mapping_value]
current_scope['type'] = mapping_db.map_type
current_scope['cost'] = mapping_db.cost
return mappings
#_load_thresholds的逻辑
def _load_thresholds(self, thresholds_uuid_list):
#加载后，返回的数据格式如下
"""
thresholds = {
"instance_flavor": {
"threshold_level":{"type": "flat", "cost": 5},
"2":{"type": "flat", "cost": 2}
},
"_DEFAULT_": {}
}
"""
hashmap = hash_db_api.get_instance()
thresholds = {}
for threshold_uuid in thresholds_uuid_list:
threshold_db = hashmap.get_threshold(uuid=threshold_uuid)
if threshold_db.group_id:
group_name = threshold_db.group.name
else:
group_name = '_DEFAULT_'
if group_name not in thresholds:
thresholds[group_name] = {}
current_scope = thresholds[group_name]
threshold_level = threshold_db.level
current_scope[threshold_level] = {}
current_scope = current_scope[threshold_level]
current_scope['type'] = threshold_db.map_type
current_scope['cost'] = threshold_db.cost
return thresholds
2.2. _load_field_entries代码分析
def _load_field_entries(self, service_name, field_name, field_uuid):
#参数例子：compute, flavor, 0b250b5d-78f5-4bb3-835f-2f9b5be67b05
if service_name not in self._entries:
self._entries[service_name] = {}
if 'fields' not in self._entries[service_name]:
self._entries[service_name]['fields'] = {}
#self._entries变成
{
"compute": {
"thresholds":thresholds,
"mappings": mappings,
" fields ": {“flavor”: "m1.tiny" :{"type":"flat", "cost":20},},
}
}
scope = self._entries[service_name]['fields'][field_name] = {}
for entry_type in ('mappings', 'thresholds'):
for tenant in (None, self._tenant_id):
self._update_entries(
entry_type,
scope,
field_uuid=field_uuid,
tenant_uuid=tenant)
91
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
