量化交易之hft篇 - re dump log源文件

最新推荐文章于 2024-11-15 16:06:58 发布

_Erwin_

最新推荐文章于 2024-11-15 16:06:58 发布

阅读量2.1k

点赞数

分类专栏： HFT 文章标签： vnpy Python c++

本文链接：https://blog.csdn.net/Michael_234198652/article/details/121619012

版权

日志解析数据转换 CSV导出 HFT数据代码分割

关键词由CSDN通过智能技术生成

HFT 专栏收录该内容

41 篇文章 6 订阅

订阅专栏


import os

from datetime import datetime

from vnpy.trader.tqz_extern.tools.file_path_operator.file_path_operator import TQZFilePathOperator
from vnpy.trader.tqz_extern.tools.pandas_operator.pandas_operator import pandas

from vnpy.app.tqz_hft_parser_app.tqz_constant import TQZLogTitles


class TQZHftParserApp:

    # --- api part ---
    @classmethod
    def tqz_reDump_format_log(cls, source_log_all_path):
        """
        Parse source_log_file with all_path to code_csv.
        """

        log_dataframe = cls.__get_source_log_dataframe(source_log_all_path=source_log_all_path)

        code_parser_fold = f'{cls.__parser_result_fold()}/{"hft_" + datetime.now().strftime("%Y%m%d")}'
        if os.path.exists(code_parser_fold) is False:
            os.mkdir(code_parser_fold)

        [pandas.DataFrame.to_csv(
            cls.__get_code_log_dataframe(source_log_all_path=source_log_all_path, code=code),
            path_or_buf=f'{code_parser_fold}/{code.replace(".", "_")}.csv',
            index=False
        ) for code in cls.__get_all_codes(log_dataframe)]


    # --- private part ---
    @classmethod
    def __get_source_log_all_path(cls):
        """
        Get all_path of source log file.
        """

        source_log_filename = "hft_" + datetime.now().strftime("%Y%m%d") + ".log"
        source_log_all_path = f'{cls.__source_log_fold()}/{source_log_filename}'

        assert os.path.exists(source_log_all_path) is True, f'{source_log_all_path} not exist.'
        return source_log_all_path


    @classmethod
    def __get_source_log_dataframe(cls, source_log_all_path: str):
        """
        Get dataframe based on source log with source_log_all_path.
        """

        line_log_dictionary_list = []
        for line in open(source_log_all_path).readlines():
            line_log_dictionary = {}
            for line_log_item in line.rstrip().split(" ")[-1].replace('[', '').replace(']', '').split(","):
                line_log_dictionary[line_log_item.split(':')[0]] = line_log_item.split(':')[1]
            line_log_dictionary_list.append(line_log_dictionary)

        return pandas.DataFrame(line_log_dictionary_list)

    @classmethod
    def __get_code_log_dataframe(cls, source_log_all_path: str, code: str):
        """
        Get code dataframe based on source log with source_log_all_path and code_name.
        """

        source_log_dataframe = cls.__get_source_log_dataframe(source_log_all_path=source_log_all_path)

        return source_log_dataframe.loc[source_log_dataframe[TQZLogTitles.CODE.value]==code]

    @staticmethod
    def __get_all_codes(source_log_dataframe: pandas.DataFrame) -> list:
        """
        Get codes list based on source log.
        """
        return list(set(list(source_log_dataframe[TQZLogTitles.CODE.value])))

    @staticmethod
    def __parser_result_fold():
        return f'{TQZFilePathOperator.father_path(source_path=__file__)}/parser_result_fold'

    @staticmethod
    def __source_log_fold():
        return f'{TQZFilePathOperator.father_path(source_path=__file__)}/source_log_fold'


if __name__ == '__main__':
    source_log_all_path_test = f'{TQZFilePathOperator.father_path(source_path=__file__)}/source_log_fold/{"hft_" + "20211123" + ".log"}'

    TQZHftParserApp.tqz_reDump_format_log(source_log_all_path=source_log_all_path_test)