ETL项目工具类开发--学习笔记

1，json数据类型处理

import json

json_data_str1 = '{"discountRate": 1, "storeShopNo": "None", "dayOrderSeq": 19, "storeDistrict": "龙华区", "isSigned": 0, "storeProvince": "广东省", "origin": 0}'

json_data_str2 = '{"discountRate": 1, "storeShopNo": "None", "dayOrderSeq": 19, "storeDistrict": "龙华区", "isSigned": 0, "storeProvince": "广东省", "origin": 0, "products":[{"name": "小浣熊洗洁精", "count":1},{"name": "劲霸童装", "count":2}]}'
"""
在JSON数据中只有 object(对象类型), array(数组类型), number(数值类型), string(字符串类型), boolean(布尔类型)
"""

# 1. 将Json字符串转换为python中的数据类型
python_data1 = json.loads(json_data_str1)
print(python_data1)
print(type(python_data1))

python_data2 = json.loads(json_data_str2)
print(python_data2)
print(type(python_data2))
print(type(python_data2['products']))

# 2.将python中的数据类型转换为JSON字符串
json_str1 = json.dumps(python_data1, ensure_ascii=False, indent=4)
print(json_str1)
print(type(json_str1))

2，json订单模板文件

model/order_model.py

"""
订单业务数据模型：
1）订单数据模型：OrdersModel
2）单个售卖商品数据模型：SingleProductSoldModel
3）订单详情数据模型：OrdersDetailModel
4）原始数据模型（订单数据+订单详情数据组合模型）：RetailOriginJsonModel
"""
import json
from config import project_config as conf
from util import str_util, time_util


class OrdersModel(object):
    """订单数据模型"""
    def __init__(self, data):
        """
        利用传入的订单json数据构建订单数据模型对象
        """
        # 将 json 数据转换为字典
        data = json.loads(data)
        # 初始化订单数据模型对象
        self.discount_rate = data['discountRate']  # 折扣率
        self.store_shop_no = data['storeShopNo']  # 店铺店号（无用列）
        self.day_order_seq = data['dayOrderSeq']  # 本单为当日第几单
        self.store_district = data['storeDistrict']  # 店铺所在行政区
        self.is_signed = data['isSigned']  # 是否签约店铺（签约第三方支付体系）
        self.store_province = data['storeProvince']  # 店铺所在省份
        self.origin = data['origin']  # 原始信息（无用）
        self.store_gps_longitude = data['storeGPSLongitude']  # 店铺GPS经度
        self.discount = data['discount']  # 折扣金额
        self.store_id = data['storeID']  # 店铺ID
        self.product_count = data['productCount']  # 本单售卖商品数量
        self.operator_name = data['operatorName']  # 操作员姓名
        self.operator = data['operator']  # 操作员ID
        self.store_status = data['storeStatus']  # 店铺状态
        self.store_own_user_tel = data['storeOwnUserTel']  # 店铺店主电话
        self.pay_type = data['payType']  # 支付类型
        self.discount_type = data['discountType']  # 折扣类型
        self.store_name = data['storeName']  # 店铺名称
        self.store_own_user_name = data['storeOwnUserName']  # 店铺店主名称
        self.date_ts = data['dateTS']  # 订单时间
        self.small_change = data['smallChange']  # 找零金额
        self.store_gps_name = data['storeGPSName']  # 店铺GPS名称
        self.erase = data['erase']  # 是否抹零
        self.store_gps_address = data['storeGPSAddress']  # 店铺GPS地址
        self.order_id = data['orderID']  # 订单ID
        self.money_before_whole_discount = data['moneyBeforeWholeDiscount']  # 折扣前金额
        self.store_category = data['storeCategory']  # 店铺类别
        self.receivable = data['receivable']  # 应收金额
        self.face_id = data['faceID']  # 面部识别ID
        self.store_own_user_id = data['storeOwnUserId']  # 店铺店主ID
        self.payment_channel = data['paymentChannel']  # 付款通道
        self.payment_scenarios = data['paymentScenarios']  # 付款情况（无用）
        self.store_address = data['storeAddress']  # 店铺地址
        self.total_no_discount = data['totalNoDiscount']  # 整体价格（无折扣）
        self.payed_total = data['payedTotal']  # 已付款金额
        self.store_gps_latitude = data['storeGPSLatitude']  # 店铺GPS纬度
        self.store_create_date_ts = data['storeCreateDateTS']  # 店铺创建时间
        self.store_city = data['storeCity']  # 店铺所在城市
        self.member_id = data['memberID'] # 会员ID

    def generate_insert_sql(self):
        """
        生成添加表数据的SQL语句
        """
        sql = f"INSERT IGNORE INTO {conf.target_orders_table_name}(" \
              f"order_id,store_id,store_name,store_status,store_own_user_id," \
              f"store_own_user_name,store_own_user_tel,store_category," \
              f"store_address,store_shop_no,store_province,store_city," \
              f"store_district,store_gps_name,store_gps_address," \
              f"store_gps_longitude,store_gps_latitude,is_signed," \
              f"operator,operator_name,face_id,member_id,store_create_date_ts," \
              f"origin,day_order_seq,discount_rate,discount_type,discount," \
              f"money_before_whole_discount,receivable,erase,small_change," \
              f"total_no_discount,pay_total,pay_type,payment_channel," \
              f"payment_scenarios,product_count,date_ts" \
              f") VALUES(" \
              f"'{self.order_id}', " \
              f"{self.store_id}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_name)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_status)}, " \
              f"{self.store_own_user_id}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_own_user_name)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_own_user_tel)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_category)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_address)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_shop_no)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_province)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_city)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_district)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_gps_name)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_gps_address)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_gps_longitude)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.store_gps_latitude)}, " \
              f"{self.is_signed}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.operator)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.operator_name)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.face_id)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.member_id)}, " \
              f"'{time_util.ts13_to_date_str(self.store_create_date_ts)}', " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.origin)}, " \
              f"{self.day_order_seq}, " \
              f"{self.discount_rate}, " \
              f"{self.discount_type}, " \
              f"{self.discount}, " \
              f"{self.money_before_whole_discount}, " \
              f"{self.receivable}, " \
              f"{self.erase}, " \
              f"{self.small_change}, " \
              f"{self.total_no_discount}, " \
              f"{self.payed_total}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.pay_type)}, " \
              f"{self.payment_channel}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.payment_scenarios)}, " \
              f"{self.product_count}, " \
              f"'{time_util.ts13_to_date_str(self.date_ts)}')"

        return sql

    @staticmethod
    def get_csv_header(sep=','):
        """
        生成 csv 数据的标头内容
        """
        header = f"order_id{sep}" \
                 f"store_id{sep}" \
                 f"store_name{sep}" \
                 f"store_status{sep}" \
                 f"store_own_user_id{sep}" \
                 f"store_own_user_name{sep}" \
                 f"store_own_user_tel{sep}" \
                 f"store_category{sep}" \
                 f"store_address{sep}" \
                 f"store_shop_no{sep}" \
                 f"store_province{sep}" \
                 f"store_city{sep}" \
                 f"store_district{sep}" \
                 f"store_gps_name{sep}" \
                 f"store_gps_address{sep}" \
                 f"store_gps_longitude{sep}" \
                 f"store_gps_latitude{sep}" \
                 f"is_signed{sep}" \
                 f"operator{sep}" \
                 f"operator_name{sep}" \
                 f"face_id{sep}" \
                 f"member_id{sep}" \
                 f"store_create_date_ts{sep}" \
                 f"origin{sep}" \
                 f"day_order_seq{sep}" \
                 f"discount_rate{sep}" \
                 f"discount_type{sep}" \
                 f"discount{sep}" \
                 f"money_before_whole_discount{sep}" \
                 f"receivable{sep}" \
                 f"erase{sep}" \
                 f"small_change{sep}" \
                 f"total_no_discount{sep}" \
                 f"pay_total{sep}" \
                 f"pay_type{sep}" \
                 f"payment_channel{sep}" \
                 f"payment_scenarios{sep}" \
                 f"product_count{sep}" \
                 f"date_ts\n"

        return header

    def check_and_transform_area(self):
        """
        检查省市区内容，为空就转换为未知
        """
        if str_util.check_null(self.store_province):
            self.store_province = '未知省份'
        if str_util.check_null(self.store_city):
            self.store_city = '未知城市'
        if str_util.check_null(self.store_district):
            self.store_district = '未知行政区'

    def check_and_transform_all_column(self):
        """
        转换全部的列，如果是空内容，就将其设置为空字符串
        """
        self.discount_rate = str_util.check_null_and_transform(self.discount_rate)
        self.store_shop_no = str_util.check_null_and_transform(self.store_shop_no)
        self.day_order_seq = str_util.check_null_and_transform(self.day_order_seq)
        self.store_district = str_util.check_null_and_transform(self.store_district)
        self.is_signed = str_util.check_null_and_transform(self.is_signed)
        self.store_province = str_util.check_null_and_transform(self.store_province)
        self.origin = str_util.check_null_and_transform(self.origin)
        self.store_gps_longitude = str_util.check_null_and_transform(self.store_gps_longitude)
        self.discount = str_util.check_null_and_transform(self.discount)
        self.store_id = str_util.check_null_and_transform(self.store_id)
        self.product_count = str_util.check_null_and_transform(self.product_count)
        self.operator_name = str_util.check_null_and_transform(self.operator_name)
        self.operator = str_util.check_null_and_transform(self.operator)
        self.store_status = str_util.check_null_and_transform(self.store_status)
        self.store_own_user_tel = str_util.check_null_and_transform(self.store_own_user_tel)
        self.pay_type = str_util.check_null_and_transform(self.pay_type)
        self.discount_type = str_util.check_null_and_transform(self.discount_type)
        self.store_name = str_util.check_null_and_transform(self.store_name)
        self.store_own_user_name = str_util.check_null_and_transform(self.store_own_user_name)
        self.date_ts = str_util.check_null_and_transform(self.date_ts)
        self.small_change = str_util.check_null_and_transform(self.small_change)
        self.store_gps_name = str_util.check_null_and_transform(self.store_gps_name)
        self.erase = str_util.check_null_and_transform(self.erase)
        self.store_gps_address = str_util.check_null_and_transform(self.store_gps_address)
        self.order_id = str_util.check_null_and_transform(self.order_id)
        self.money_before_whole_discount = str_util.check_null_and_transform(self.money_before_whole_discount)
        self.store_category = str_util.check_null_and_transform(self.store_category)
        self.receivable = str_util.check_null_and_transform(self.receivable)
        self.face_id = str_util.check_null_and_transform(self.face_id)
        self.store_own_user_id = str_util.check_null_and_transform(self.store_own_user_id)
        self.payment_channel = str_util.check_null_and_transform(self.payment_channel)
        self.payment_scenarios = str_util.check_null_and_transform(self.payment_scenarios)
        self.store_address = str_util.check_null_and_transform(self.store_address)
        self.total_no_discount = str_util.check_null_and_transform(self.total_no_discount)
        self.payed_total = str_util.check_null_and_transform(self.payed_total)
        self.store_gps_latitude = str_util.check_null_and_transform(self.store_gps_latitude)
        self.store_create_date_ts = str_util.check_null_and_transform(self.store_create_date_ts)
        self.store_city = str_util.check_null_and_transform(self.store_city)
        self.member_id = str_util.check_null_and_transform(self.member_id)

    def to_csv(self, sep=','):
        """
        生成 csv 数据，分割符默认为逗号。
        Note: 生成的数据顺序和header是一一对应的，不要混乱了。
        """
        self.check_and_transform_area()
        self.check_and_transform_all_column()

        csv_line = \
            f"{self.order_id}{sep}" \
            f"{self.store_id}{sep}" \
            f"{self.store_name}{sep}" \
            f"{self.store_status}{sep}" \
            f"{self.store_own_user_id}{sep}" \
            f"{self.store_own_user_name}{sep}" \
            f"{self.store_own_user_tel}{sep}" \
            f"{self.store_category}{sep}" \
            f"{self.store_address}{sep}" \
            f"{self.store_shop_no}{sep}" \
            f"{self.store_province}{sep}" \
            f"{self.store_city}{sep}" \
            f"{self.store_district}{sep}" \
            f"{self.store_gps_name}{sep}" \
            f"{self.store_gps_address}{sep}" \
            f"{self.store_gps_longitude}{sep}" \
            f"{self.store_gps_latitude}{sep}" \
            f"{self.is_signed}{sep}" \
            f"{self.operator}{sep}" \
            f"{self.operator_name}{sep}" \
            f"{self.face_id}{sep}" \
            f"{self.member_id}{sep}" \
            f"{time_util.ts13_to_date_str(self.store_create_date_ts)}{sep}" \
            f"{self.origin}{sep}" \
            f"{self.day_order_seq}{sep}" \
            f"{self.discount_rate}{sep}" \
            f"{self.discount_type}{sep}" \
            f"{self.discount}{sep}" \
            f"{self.money_before_whole_discount}{sep}" \
            f"{self.receivable}{sep}" \
            f"{self.erase}{sep}" \
            f"{self.small_change}{sep}" \
            f"{self.total_no_discount}{sep}" \
            f"{self.payed_total}{sep}" \
            f"{self.pay_type}{sep}" \
            f"{self.payment_channel}{sep}" \
            f"{self.payment_scenarios}{sep}" \
            f"{self.product_count}{sep}" \
            f"{time_util.ts13_to_date_str(self.date_ts)}\n"

        return csv_line


class SingleProductSoldModel(object):
    """订单售卖商品数据模型"""
    def __init__(self, order_id, product_detail):
        self.order_id = order_id
        self.count = product_detail['count']
        self.name = product_detail['name']
        self.unit_id = product_detail['unitID']
        self.barcode = product_detail['barcode']
        self.price_per = product_detail['pricePer']
        self.retail_price = product_detail['retailPrice']
        self.trade_price = product_detail['tradePrice']
        self.category_id = product_detail['categoryID']

    def generate_value_segment_for_sql_insert(self):
        """
        生成添加表数据SQL语句的VALUE语句段
        """
        segment = f"(" \
                  f"'{self.order_id}', " \
                  f"{str_util.check_str_null_and_transform_to_sql_null(self.barcode)}, " \
                  f"{str_util.check_str_null_and_transform_to_sql_null(self.name)}, " \
                  f"{self.count}, " \
                  f"{self.price_per}, " \
                  f"{self.retail_price}, " \
                  f"{self.trade_price}, " \
                  f"{self.category_id}, " \
                  f"{self.unit_id}" \
                  f")"

        return segment

    def to_csv(self, sep=","):
        """
        生成一条csv数据，分隔符默认逗号
        """
        csv_line = \
            f"{self.order_id}{sep}" \
            f"{self.barcode}{sep}" \
            f"{self.name}{sep}" \
            f"{self.count}{sep}" \
            f"{self.price_per}{sep}" \
            f"{self.retail_price}{sep}" \
            f"{self.trade_price}{sep}" \
            f"{self.category_id}{sep}" \
            f"{self.unit_id}\n"

        return csv_line

class OrdersDetailModel(object):
    """订单详情数据模型"""
    def __init__(self, data):
        """
        利用传入的订单json数据构建订单详情数据模型对象
        """
        data = json.loads(data)
        order_products_list = data['product']
        self.order_id = data['orderID']
        self.products_detail = [] # 记录当前订单卖出的商品

        for sing_product in order_products_list:
            product = SingleProductSoldModel(self.order_id, sing_product)
            self.products_detail.append(product)

    def generate_insert_sql(self):
        """
        生成添加表数据的SQL语句
        """
        sql = f"INSERT IGNORE INTO {conf.target_orders_detail_table_name}(" \
              f"order_id,barcode,name,count,price_per,retail_price,trade_price,category_id,unit_id" \
              f") VALUES"

        for single_product_sold_model in self.products_detail:
            sql += single_product_sold_model.generate_value_segment_for_sql_insert() + ", "

        # 去除最后的逗号
        sql = sql[:-2]
        return sql

    @staticmethod
    def get_csv_header(sep=','):
        """
        生成 csv 数据的标头内容
        """
        return f"order_id{sep}" \
               f"barcode{sep}" \
               f"name{sep}" \
               f"count{sep}" \
               f"price_per{sep}" \
               f"retail_price{sep}" \
               f"trade_price{sep}" \
               f"category_id{sep}" \
               f"unit_id\n"

    def to_csv(self):
        """生成添加csv的数据行"""
        csv_lines = ''

        for single_product_sold_model in self.products_detail:
            csv_lines += single_product_sold_model.to_csv()

        return csv_lines


class RetailOriginJsonModel(object):
    """
    原始订单JSON数据模型
    """
    def __init__(self, data):
        self.order_model = OrdersModel(data)
        self.order_detail_model = OrdersDetailModel(data)

    def get_order_model(self):
        return self.order_model

    def get_order_detail_model(self):
        return self.order_detail_model

3，商品数据

model/barcode_model.py模型类

"""
条码商品信息模型类
"""
from util import str_util
from config import project_config as conf


class BarcodeModel(object):
    """条码商品信息模型类"""
    def __init__(self, data_tuple: tuple):
        """条码商品模型对象初始化"""
        self.code = data_tuple[0]
        self.name = str_util.clear_str(data_tuple[1])
        self.spec = str_util.clear_str(data_tuple[2])
        self.trademark = str_util.clear_str(data_tuple[3])
        self.addr = str_util.clear_str(data_tuple[4])
        self.units = str_util.clear_str(data_tuple[5])
        self.factory_name = str_util.clear_str(data_tuple[6])
        self.trade_price = data_tuple[7]
        self.retail_price = data_tuple[8]
        self.update_at = str(data_tuple[9])
        self.wholeunit = data_tuple[10]
        self.wholenum = data_tuple[11]
        self.img = data_tuple[12]
        self.src = data_tuple[13]

    def generate_insert_sql(self):
        """生成SQL的插入语句"""
        sql = f"REPLACE INTO {conf.target_barcode_table_name}(" \
              f"code,name,spec,trademark,addr,units,factory_name,trade_price," \
              f"retail_price,update_at,wholeunit,wholenum,img,src) VALUES(" \
              f"'{self.code}', " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.name)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.spec)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.trademark)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.addr)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.units)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.factory_name)}, " \
              f"{str_util.check_number_null_and_transform_to_sql_null(self.trade_price)}, " \
              f"{str_util.check_number_null_and_transform_to_sql_null(self.retail_price)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.update_at)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.wholeunit)}, " \
              f"{str_util.check_number_null_and_transform_to_sql_null(self.wholenum)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.img)}, " \
              f"{str_util.check_str_null_and_transform_to_sql_null(self.src)}" \
              f")"

        return sql

    @staticmethod
    def get_csv_header(sep=','):
        """
        生成 csv 数据的标头内容
        """
        return f"code{sep}" \
               f"name{sep}" \
               f"spec{sep}" \
               f"trademark{sep}" \
               f"addr{sep}" \
               f"units{sep}" \
               f"factory_name{sep}" \
               f"trade_price{sep}" \
               f"retail_price{sep}" \
               f"update_at{sep}" \
               f"wholeunit{sep}" \
               f"wholenum{sep}" \
               f"img{sep}" \
               f"src\n"

    def to_csv(self, sep=','):
        """生成csv数据行"""
        csv_line = \
            f"{self.code}{sep}" \
            f"{self.name}{sep}" \
            f"{self.spec}{sep}" \
            f"{self.trademark}{sep}" \
            f"{self.addr}{sep}" \
            f"{self.units}{sep}" \
            f"{self.factory_name}{sep}" \
            f"{self.trade_price}{sep}" \
            f"{self.retail_price}{sep}" \
            f"{self.update_at}{sep}" \
            f"{self.wholeunit}{sep}" \
            f"{self.wholenum}{sep}" \
            f"{self.img}{sep}" \
            f"{self.src}\n"

        return csv_line

4,日志数据

model/backend_logs_model.py

from config import project_config as conf


class BackendLogsModel(object):
    """创建一个模型类"""

    def __init__(self, log_data: str):
        """初始化方法"""
        # 1. 将log_data数据拆分为列表,分隔符是\t
        data = log_data.split('\t')
        # 2. 将列表中的每一个元素赋值给相应的属性
        self.log_time = data[0]
        self.log_level = data[1].strip('[]')
        self.log_module = data[2]
        self.response_time = data[3][5:-2]
        self.province = data[4]
        self.city = data[5]
        self.log_text = data[6]

    def generate_insert_sql(self):
        """生成插入数据的sql语句"""
        return f'insert into {conf.target_logs_table_name}(' \
               f'log_time, log_level, log_module, response_time, province, city, log_text)' \
               f' values("{self.log_time}",' \
               f'"{self.log_level}",' \
               f'"{self.log_module}",' \
               f'"{self.response_time}",' \
               f'"{self.province}",' \
               f'"{self.city}",' \
               f'"{self.log_text}");'

    @staticmethod
    def generate_csv_header(sep=','):
        """生成csv文件的标头"""
        return f'log_time{sep}' \
               f'log_level{sep}' \
               f'log_module{sep}' \
               f'response_time{sep}' \
               f'province{sep}' \
               f'city{sep}' \
               f'log_text\n'

    # 生成csv文件的行内容
    def gender_csv_str(self, sep=','):
        return f'{self.log_time}{sep}' \
               f'{self.log_level}{sep}' \
               f'{self.log_module}{sep}' \
               f'{self.response_time}{sep}' \
               f'{self.province}{sep}' \
               f'{self.city}{sep}' \
               f'{self.log_text}'

模拟日志数据生成

/simulator/backend_logs_simulator.py

"""
后端服务写出log日志的模拟数据生成器
"""
import datetime
import random
import time

single_log_lines = 1024  # 一个logs文件生成多少行数据
generate_files = 5  # 一次运行生成多少个文件

output_path = "D:/etl/logs/"
log_level_array = ['WARN', 'WARN', 'WARN', 'INFO', 'INFO', 'INFO', 'INFO', 'INFO', 'INFO', 'INFO', 'INFO',
                   'INFO', 'INFO', 'INFO', 'INFO', 'INFO', 'INFO', 'INFO', 'INFO', 'INFO', 'INFO', 'INFO', 'INFO',
                   'ERROR']

backend_files_name = ['barcode_service.py', 'barcode_service.py', 'barcode_service.py',
                      'orders_service.py', 'orders_service.py', 'orders_service.py', 'orders_service.py',
                      'orders_service.py', 'orders_service.py',
                      'shop_manager.py', 'shop_manager.py',
                      'user_manager.py', 'user_manager.py', 'user_manager.py',
                      'goods_manager.py', 'goods_manager.py', 'goods_manager.py', 'goods_manager.py',
                      'goods_manager.py', 'goods_manager.py',
                      'base_network.py', 'base_network.py',
                      'event.py', 'event.py', 'event.py', 'event.py', 'event.py', 'event.py', 'event.py']

visitor_areas = {
    '北京市': ['海淀区', '大兴区', '丰台区', '朝阳区', '昌平区', '海淀区', '怀柔区'],
    '上海市': ['静安区', '黄浦区', '徐汇区', '普陀区', '杨浦区', '宝山区', '浦东新区', '浦东新区'],
    '重庆市': ['万州区', '万州区', '涪陵区', '渝中区', '沙坪坝区', '九龙坡区', '南岸区'],
    '江苏省': ['南京市', '南京市', '南京市', '苏州市', '苏州市', '无锡市', '常州市', '宿迁市', '张家港市'],
    '安徽省': ['阜阳市', '阜阳市', '六安市', '合肥市', '合肥市', '合肥市', '池州市', '铜陵市', '芜湖市'],
    '山东省': ['济南市', '济南市', '青岛市', '青岛市', '青岛市', '菏泽市'],
    '湖北省': ['武汉市', '武汉市', '武汉市', '十堰市', '荆州市', '恩施土家族苗族自治州'],
    '广东省': ['广州市', '广州市', '广州市', '深圳市', '深圳市', '深圳市', '珠海市'],
    '天津市': ['和平区', '河东区', '河西区', '武清区', '宝坻区'],
    '湖南省': ['长沙市', '长沙市', '长沙市', '长沙市', '长沙市', '长沙市', '长沙市', '株洲市', '张家界市', '常德市', '益阳市'],
    '浙江省': ['杭州市', '杭州市', '湖州市', '绍兴市', '舟山市', '金华市', '嘉兴市', '丽水市']
}
visitor_province = ['北京市', '上海市', '重庆市', '江苏省', '安徽省', '山东省', '湖北省', '广东省', '天津市', '湖南省', '浙江省']

response_flag = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]
response_for_error_flag = [1, 1, 1, 1, 1, 0]

for j in range(0, generate_files):
    write_file_path = f'{output_path}{datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}.log'
    with open(write_file_path, 'w', encoding="UTF-8") as f:
        for i in range(single_log_lines):
            date_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
            log_level = log_level_array[random.randint(0, len(log_level_array) - 1)]
            file_name = backend_files_name[random.randint(0, len(backend_files_name) - 1)]
            if not log_level == "ERROR":
                if response_flag[random.randint(0, len(response_flag) - 1)] == 1:
                    response_time = random.randint(0, 1000)
                else:
                    response_time = random.randint(1000, 9999)
            else:
                if response_for_error_flag[random.randint(0, len(response_for_error_flag) - 1)] == 1:
                    response_time = random.randint(0, 1000)
                else:
                    response_time = random.randint(1000, 9999)

            province = visitor_province[random.randint(0, len(visitor_province) - 1)]
            city = visitor_areas[province][random.randint(0, len(visitor_areas[province]) - 1)]

            log_str = f"{date_str}\t[{log_level}]\t{file_name}\t响应时间:{response_time}ms\t{province}\t{city}\t" \
                      f"这里是日志信息......"

            f.write(log_str)
            f.write("\n")
    print(f"本次写出第: {j + 1}个文件完成, 文件为: {write_file_path}, 行数:{single_log_lines}")
    time.sleep(1)