python日志解析生成excel表

需求

        实时流媒体性能指标分析，提升问题定位效率

方案

        日志文件读取解析：python

        excel表生成：openpyxl

实现

代码

函数入口

# -*- coding: utf-8 -*-

from openpyxl import Workbook

from utils.common import get_parse_path, get_parse_files

from RtcParse import RtcParse

from WorkbookManager import WorkbookManager

if __name__ == '__main__':

    parse_path = get_parse_path()

    parse = RtcParse(parse_path)

    parse.run()

    wbm = WorkbookManager(parse)

    wbm.run()

   

解析文件过滤日志获取数据RtcParse.py

# -*- coding: utf-8 -*-

import os

import glob

import datetime

class RtcParse:

    def __init__(self, parse_path):

        self._parse_path = parse_path

        self.ts_records = []

        self.ts_records_title = None

        self.COMMANDS = {"ts:": self._parse_ts}

    def run(self):

        self._filter_files("rtc_mediatranse")

        self._organize_data()

        self._sort_data()

    def _filter_files(self, filter):

        self._parse_files = glob.glob(os.path.join(self._parse_path, filter + "*.log"))

    def _organize_data(self):

        for item in self._parse_files:

            with open(item, "r") as f:

                for line in f:

                    line = line.strip("\n")

                    self._filter_data(line)

    def _sort_data(self):

         self.ts_records = sorted(self.ts_records, key=lambda x:x[0], reverse=False)

    def _filter_data(self, line):

        for item in self.COMMANDS:

            if item in line:

                self.COMMANDS[item](line)

    def _parse_ts(self, line):

        items = line.split(" ")

        cur_time = datetime.datetime.strptime(items[0] + " " + items[1], "%Y-%m-%d %H:%M:%S.%f")

        items = items[4:]

        self.ts_records.append((cur_time,) + tuple(x.split(":")[1] for x in items))

        if self.ts_records_title is None:

            self.ts_records_title = ("time",) + tuple(x.split(":")[0] for x in items)

if __name__ == '__main__':

    parse = RtcParse("d:\\1.project")

生成图表WorkbookManager.py

# -*- coding: utf-8 -*-

from openpyxl import Workbook

from openpyxl.chart import (LineChart, Reference)

import os

class WorkbookManager:

    def __init__(self, rtc_parse=None):

        self._rtc_parse = rtc_parse

        self._max_row = 0

        self._max_col = 0

    def run(self):

        self._wb = Workbook()

        self._ws = self._wb.active

        self._generate_ts()

        self._generate_ts_chart()

        self._wb.save("d:\\123.xlsx")

        os.system("start d:\\123.xlsx")

    def _generate_ts_chart(self):

        ct = LineChart()

        ct.title = "ts chart"

        ct.style = 2

       

        rows = Reference(self._ws, min_col=1, min_row=1, max_col=self._max_col, max_row=self._max_row)

        x_values = Reference(self._ws, min_col = 1, min_row=2, max_row=self._max_row)

        y_values = Reference(self._ws, min_col=2, max_col=self._max_col, min_row=1, max_row=self._max_row)

       

        ct.add_data(y_values, titles_from_data=True)

        ct.set_categories(x_values)

        self._ws.add_chart(ct, "A1")

    def _generate_ts(self):

        self._max_row = 0

        titles = self._rtc_parse.ts_records_title

        records = self._rtc_parse.ts_records

        row = 1

        col = 0

        for item in titles:

            self._ws[self._get_cell(row, col)] = item

            col += 1

        self._max_col = col

        for items in records:

            row += 1

            col = 0

            self._ws[self._get_cell(row, col)] = items[0]

            col += 1

            for item in items[1:]:

                self._ws[self._get_cell(row, col)] = float(item) if "." in item else int(item)

                col += 1

        self._max_row = row

    def _get_cell(self, row, col):

        cell = ""

        if col < 26:

            cell += str(chr(ord('A') + col))

        cell += (str(row))

        return cell

   

if __name__ == '__main__':

    wbm = WorkbookManager()

================未完待续=====================