Python之数据结构改造（二）

最新推荐文章于 2021-01-21 22:52:14 发布

Quincy379

最新推荐文章于 2021-01-21 22:52:14 发布

阅读量189

点赞数

分类专栏： Python

本文链接：https://blog.csdn.net/qq_33733970/article/details/90716563

版权

Python 专栏收录该内容

271 篇文章 9 订阅

订阅专栏

背景：

因为日志易处理的数据和听云吐出的数据不一致，无法解析画图，所以需要进行数据结构改造。

思路：

根据日志易解析要求，需要的数据结构如下：

{'aggregateBy': 1, 'appIns_id': 66, 'yAxis': {'gridLineColor': '#c0c0c0', 'gridLineDashStyle': 'ShortDot', 'min': 0.0, 'title': {'text': None, 'style': {'color': '#000000'}}, 'labels': {'color': '#000000'}, 'gridLineWidth': 1, 'tickColor': '#000000', 'tickUnit': '%'}, 'series': {'name': 'Error rate', 'color': '#3598db', 'type': 'area', 'params': '7', 'marker': {'symbol': 'circle', 'radius': 4}, 'data': {'single': {'y': 0.0, 'x': 1559286660000, 'tooltip': '{"title":"05-31 15:11--05-31 15:12","data":[{"title":"Error rate","value":"0.0","unit":"%"},{"title":"Errors","value":"0"},{"title":"Traffic","value":"2"}]}'}}}, 'chart': {'spacingTop': 15, 'height': 300, 'spacingRight': 10, 'width': 375, 'plotBackgroundColor': '#ffffff', 'backgroundColor': '#ffffff', 'spacingBottom': 10, 'plotBorderColor': '#575757', 'type': 'area', 'plotBorderWidth': 1, 'spacingLeft': 0}, 'xAxis': {'gridLineColor': '#c0c0c0', 'gridLineDashStyle': 'ShortDot', 'title': {'text': None, 'style': {'color': '#000000'}}, 'labels': {'color': '#000000'}, 'dateTimeLabelFormats': {'millisecond': '%H:%M', 'week': '%H:%M', 'hour': '%H:%M', 'month': '%H:%M', 'second': '%H:%M', 'year': '%H:%M', 'day': '%H:%M', 'minute': '%H:%M'}, 'gridLineWidth': 1, 'tickColor': '#000000', 'minTickInterval': 60000, 'type': 'datetime'}, 'appId': 14, 'legend': {'align': 'center', 'verticalAlign': 'bottom'}, 'aggregateValue': 12.300000000000001}

鉴于上述数据结构都是json格式，可以利用Py字典键不可重复的特性对数据进行改造，程序见下面：

要改造的数据：

{"chart":{"backgroundColor":"#ffffff","width":375,"height":300,"plotBackgroundColor":"#ffffff","plotBorderColor":"#575757","plotBorderWidth":1,"spacingTop":15,"spacingBottom":10,"spacingLeft":0,"spacingRight":10,"type":"area"},"legend":{"align":"center","verticalAlign":"bottom"},"series":[{"name":"错误百分比","type":"area","color":"#3598db","data":[{"x":1559282160000,"y":1.0,"tooltip":"{\"title\":\"05-31 13:56--05-31 13:57\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"1.0\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"4\"},{\"title\":\"访问量\",\"value\":\"407\"}]}"},{"x":1559282220000,"y":5.0,"tooltip":"{\"title\":\"05-31 13:57--05-31 13:58\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"5.0\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"4\"},{\"title\":\"访问量\",\"value\":\"80\"}]}"},{"x":1559282280000,"y":1.2,"tooltip":"{\"title\":\"05-31 13:58--05-31 13:59\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"1.2\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"2\"},{\"title\":\"访问量\",\"value\":\"167\"}]}"},{"x":1559282340000,"y":2.2,"tooltip":"{\"title\":\"05-31 13:59--05-31 14:00\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"2.2\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"4\"},{\"title\":\"访问量\",\"value\":\"183\"}]}"},{"x":1559282400000,"y":1.5,"tooltip":"{\"title\":\"05-31 14:00--05-31 14:01\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"1.5\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"2\"},{\"title\":\"访问量\",\"value\":\"138\"}]}"},{"x":1559282460000,"y":0.8,"tooltip":"{\"title\":\"05-31 14:01--05-31 14:02\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"0.8\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"3\"},{\"title\":\"访问量\",\"value\":\"366\"}]}"},{"x":1559282520000,"y":2.7,"tooltip":"{\"title\":\"05-31 14:02--05-31 14:03\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"2.7\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"4\"},{\"title\":\"访问量\",\"value\":\"149\"}]}"},{"x":1559282580000,"y":1.3,"tooltip":"{\"title\":\"05-31 14:03--05-31 14:04\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"1.3\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"2\"},{\"title\":\"访问量\",\"value\":\"154\"}]}"},{"x":1559282640000,"y":1.3,"tooltip":"{\"title\":\"05-31 14:04--05-31 14:05\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"1.3\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"2\"},{\"title\":\"访问量\",\"value\":\"150\"}]}"},{"x":1559282700000,"y":0.8,"tooltip":"{\"title\":\"05-31 14:05--05-31 14:06\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"0.8\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"1\"},{\"title\":\"访问量\",\"value\":\"125\"}]}"},{"x":1559282760000,"y":1.6,"tooltip":"{\"title\":\"05-31 14:06--05-31 14:07\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"1.6\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"4\"},{\"title\":\"访问量\",\"value\":\"248\"}]}"},{"x":1559282820000,"y":0.7,"tooltip":"{\"title\":\"05-31 14:07--05-31 14:08\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"0.7\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"1\"},{\"title\":\"访问量\",\"value\":\"150\"}]}"},{"x":1559282880000,"y":0.7,"tooltip":"{\"title\":\"05-31 14:08--05-31 14:09\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"0.7\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"2\"},{\"title\":\"访问量\",\"value\":\"276\"}]}"},{"x":1559282940000,"y":3.2,"tooltip":"{\"title\":\"05-31 14:09--05-31 14:10\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"3.2\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"4\"},{\"title\":\"访问量\",\"value\":\"126\"}]}"},{"x":1559283000000,"y":2.8,"tooltip":"{\"title\":\"05-31 14:10--05-31 14:11\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"2.8\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"3\"},{\"title\":\"访问量\",\"value\":\"107\"}]}"},{"x":1559283060000,"y":1.3,"tooltip":"{\"title\":\"05-31 14:11--05-31 14:12\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"1.3\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"2\"},{\"title\":\"访问量\",\"value\":\"158\"}]}"},{"x":1559283120000,"y":3.3,"tooltip":"{\"title\":\"05-31 14:12--05-31 14:13\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"3.3\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"6\"},{\"title\":\"访问量\",\"value\":\"183\"}]}"},{"x":1559283180000,"y":2.3,"tooltip":"{\"title\":\"05-31 14:13--05-31 14:14\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"2.3\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"3\"},{\"title\":\"访问量\",\"value\":\"129\"}]}"},{"x":1559283240000,"y":0.0,"tooltip":"{\"title\":\"05-31 14:14--05-31 14:15\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"0.0\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"0\"},{\"title\":\"访问量\",\"value\":\"95\"}]}"},{"x":1559283300000,"y":4.7,"tooltip":"{\"title\":\"05-31 14:15--05-31 14:16\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"4.7\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"9\"},{\"title\":\"访问量\",\"value\":\"192\"}]}"},{"x":1559283360000,"y":0.7,"tooltip":"{\"title\":\"05-31 14:16--05-31 14:17\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"0.7\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"1\"},{\"title\":\"访问量\",\"value\":\"137\"}]}"},{"x":1559283420000,"y":2.9,"tooltip":"{\"title\":\"05-31 14:17--05-31 14:18\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"2.9\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"4\"},{\"title\":\"访问量\",\"value\":\"139\"}]}"},{"x":1559283480000,"y":1.8,"tooltip":"{\"title\":\"05-31 14:18--05-31 14:19\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"1.8\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"4\"},{\"title\":\"访问量\",\"value\":\"228\"}]}"},{"x":1559283540000,"y":0.0,"tooltip":"{\"title\":\"05-31 14:19--05-31 14:20\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"0.0\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"0\"},{\"title\":\"访问量\",\"value\":\"91\"}]}"},{"x":1559283600000,"y":0.7,"tooltip":"{\"title\":\"05-31 14:20--05-31 14:21\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"0.7\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"2\"},{\"title\":\"访问量\",\"value\":\"269\"}]}"},{"x":1559283660000,"y":0.9,"tooltip":"{\"title\":\"05-31 14:21--05-31 14:22\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"0.9\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"2\"},{\"title\":\"访问量\",\"value\":\"227\"}]}"},{"x":1559283720000,"y":1.4,"tooltip":"{\"title\":\"05-31 14:22--05-31 14:23\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"1.4\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"2\"},{\"title\":\"访问量\",\"value\":\"141\"}]}"},{"x":1559283780000,"y":1.6,"tooltip":"{\"title\":\"05-31 14:23--05-31 14:24\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"1.6\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"2\"},{\"title\":\"访问量\",\"value\":\"125\"}]}"},{"x":1559283840000,"y":1.0,"tooltip":"{\"title\":\"05-31 14:24--05-31 14:25\",\"data\":[{\"title\":\"错误百分比\",\"value\":\"1.0\",\"unit\":\"%\"},{\"title\":\"错误量\",\"value\":\"5\"},{\"title\":\"访问量\",\"value\":\"493\"}]}"}],"params":"7","marker":{"symbol":"circle","radius":4}}],"aggregateBy":1,"aggregateValue":1.6,"xAxis":[{"dateTimeLabelFormats":{"millisecond":"%H:%M","second":"%H:%M","minute":"%H:%M","hour":"%H:%M","day":"%H:%M","week":"%H:%M","month":"%H:%M","year":"%H:%M"},"gridLineColor":"#c0c0c0","gridLineDashStyle":"ShortDot","gridLineWidth":1,"labels":{"color":"#000000"},"minTickInterval":60000,"tickColor":"#000000","title":{"style":{"color":"#000000"},"text":null},"type":"datetime"}],"yAxis":[{"gridLineColor":"#c0c0c0","gridLineDashStyle":"ShortDot","gridLineWidth":1,"labels":{"color":"#000000"},"min":0.0,"tickColor":"#000000","title":{"style":{"color":"#000000"},"text":null},"tickUnit":"%"}]}

处理程序：

#!/usr/bin/env python
# -*- coding:utf-8 -*-
import json
import sys
import time
import requests
from kafka import KafkaProducer


reload(sys)
sys.setdefaultencoding('utf8')


name = '账号'
pawd = '密码'
doLoginurl = 'http://xxx.xxx.xxx/lens/manager/doLogin'
toserver = 'http://xxx.xxx.xxx/server/manager/platformLogin?target=/manager/index&product=server'
instancesxurl = 'http://xxx.xxx.xxx/server/application/instances'
errors_url = 'http://xxx.xxx.xxx/server/highcharts-chart-data/LATEST/accounts/applications/charts/application-errors.json'
resp_time_url = 'http://xxx.xxx.xxx/server/highcharts-chart-data/LATEST/accounts/applications/charts/application-application.json'


p=KafkaProducer(bootstrap_servers=['xxx:9092','xxx:9092','xxx:9092'])
topic='tingyun4rzy'


def do_login():
    """
    登录听云
    :return:
    """
    sess = requests.session()
    sess.post(doLoginurl, data={'name': name, 'password': pawd})
    # 获取csrftoken
    res = sess.get(toserver)
    csrftoken = res.text.split('name="csrftoken"')[1].split('"')[3]
    res.request.headers['csrftoken'] = csrftoken
    headers = res.request.headers

    return sess, headers


def get_app_ids():
    # 发送请求获取cookie
    s = requests.Session()
    pre_login = 'http://xxx.xxx.xxx/lens/login'
    s.get(pre_login)

    login_url = 'http://xxxx.xxx.xxx/server/manager/doLogin?name=账号&password=密码'
    s.get(login_url)

    post_login = 'http://xxx.xxx.xxx/server/manager/platformLogin?target=/manager/index&product=server'
    ret = s.get(login_url)
    # print ret
    header = {"X-Auth-Key": "xxx-xxx-xxx"}

    # 第四步：带上X-Auth-Key请求某个具体的API接口
    url_app_list = 'http://xxx.xxx.xxx/server/data/application/allApps?X-Auth-Key=xxx-xxx-xxx'
    ret = s.get(url_app_list)
    # 将所有的app信息加入kafka
    # x=p.send(topic,json.dumps(ret.json(),))

    ret = json.loads(ret.text)
    ids = []
    for i in ret['appList']:
        ids.append(i['id'])
    return ids


def get_app_instances(id):
    sess, headers = do_login()
    data = {'applicationId': id, 'timePeriod': '30', 'timeZone': 'GMT+0800'}
    instance_res = sess.post(instancesxurl, data=data, headers=headers).text
    #index_res = sess.post('http://xxx.xxx.xxx/server/overview/newAppList/' + str(id) + '/list',data={'timePeriod': '30'}, headers=headers).text
    #####print('*=*instance_res*=*======================================================', instance_res)
    #if index_res:
    #    data = json.loads(index_res)
    #    x = p.send(topic, json.dumps(data, ))
    if instance_res:
        data = json.loads(instance_res)
        appIns_ids = []
        for i in data:
            appIns_ids.append(i['id'])
        #print(appIns_ids)
        #if appIns_ids:
        #    print('==>>>',appIns_ids)
        return appIns_ids 

def get_app_errors(app_id):
    """
    获取app错误率
    """
    sess, headers = do_login()
    data = {'applicationId': app_id, 'timePeriod': '30', 'timeZone': 'GMT+0800', 'timeType': 1}
    errors_res = sess.post(errors_url, data=data, headers=headers).text
    #print('*=*errors_res*=*=====================================================', errors_res)
    if errors_res:
        data = json.loads(errors_res)
        data['appId'] = app_id
        data['xAxis'] = data.get('xAxis')[0]
        data['yAxis'] = data.get('yAxis')[0]
        #print(data)
        if data.get('series'):
            data['series'] = data.get('series')[0]
            #print(data.get('series').get('data'))
            data_list = data.get('series').get('data')
            dt_dict = {}
            for idx,val in enumerate(data_list):
                dt_dict.update({'single':val})
                #dt_dict.update({val.get('x'):val})
                 
                data['series']['data'] = dt_dict
                #print(data)
                #处理获取的数据
                x = p.send(topic, json.dumps(data, ))


def get_appIns_errors(app_id,appIns_id):
    """
    获取实例错误率
    """
    sess, headers = do_login()
    data = {'applicationId': app_id,'instanceId':appIns_id, 'timePeriod': '30', 'timeZone': 'GMT+0800', 'timeType': 1}
    errors_res = sess.post(errors_url, data=data, headers=headers).text
    if errors_res:
        data = json.loads(errors_res)
        data['appId'] = app_id
        data['appIns_id'] = appIns_id
        data['xAxis'] = data.get('xAxis')[0]
        data['yAxis'] = data.get('yAxis')[0]
        if data.get('series'):
            data['series'] = data.get('series')[0]
            data_list = data.get('series').get('data')
            dt_dict = {}
            for idx,val in enumerate(data_list):
                dt_dict.update({'single':val})
                data['series']['data'] = dt_dict
                x = p.send(topic, json.dumps(data, ))
                print('dooooooooooooooneeeeeeeeeeee',data)


def get_app_resp_time(app_id,appIns_id):
    """
    获取应用服务器响应时间
    """
    sess, headers = do_login()
    data = {'applicationId': app_id,'instanceId':appIns_id, 'timePeriod': '30', 'timeZone': 'GMT+0800', 'timeType': 1}
    resp_res = sess.post(resp_time_url, data=data, headers=headers).text
    if resp_res:
        data = json.loads(resp_res)
        data['appId'] = app_id
        data['appIns_id'] = appIns_id
        data['xAxis'] = data.get('xAxis')[0]
        data['yAxis'] = data.get('yAxis')[0]
        if data.get('series'):
            #data['series'] = data.get('series')[0]
            for i in data['series']:
                data['series'] = i
                data['factor'] = data['series']['name']
                data_list = data.get('series').get('data')
                dt_dict = {}
                for idx,val in enumerate(data_list):
                    dt_dict.update({'single':val})
                    data['series']['data'] = dt_dict
                    x = p.send(topic, json.dumps(data, ))
                    print(data['series']['name'],'=====>>>',data)
                    
    
if __name__ == '__main__':
    app_ids = get_app_ids()
    while 1:
        for app_id in app_ids:
            appIns_ids = get_app_instances(app_id)
            #print(appIns_ids)
            if appIns_ids:
                for appIns_id in appIns_ids:
                    get_appIns_errors(app_id,appIns_id)
                    #get_app_resp_time(app_id,appIns_id)
            #get_app_errors(app_id)

        time.sleep(120)

Quincy379

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
Python之数据结构改造（二）

在这里插入代码片#!/usr/bin/env python# -*- coding:utf-8 -*-import jsonimport sysimport timeimport requestsfrom kafka import KafkaProducerreload(sys)sys.setdefaultencoding('utf8')name = '账号'paw...
复制链接

扫一扫

专栏目录