Python多线程+队列调用接口

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author: 风过无言花易落
# @Date  : 2021/03/31 14:33
# @Desc  : 接口多线程调用

'''
 mvno_token模块样例

allmark = {
        'test':{
        "ABC": ("5hdoc0g", "WFykRKbDPMiTZRnU1pw7296Pl8zgrbM4hV7I16jGAAyNxEgu"),
        "DFG": ("6jj4HEs", "LCJaN1WeTNzbmsRPjWvNCEpbOryXe9eNOi9JC5JqP37WmdiK")
                }
}
'''

import requests
import json
import string
import timeit
import time
from time import sleep,strftime
import random
import sql
import os
import mvno_token
import threading
from queue import Queue
from log import logs

path = str(os.path.dirname(__file__))
base_dir = path.replace('\\', '/')
log = logs('ERROR',os.path.basename(__file__).split(".")[0])

requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'
requests.packages.urllib3.disable_warnings()
log.debug('当前路径：{}'.format(base_dir))

API_IP = "123.123.123.123"  # 云化环境

class Creat_Thread(threading.Thread):
    def __init__(self, mode_type , t_msg , data_queue):
        threading.Thread.__init__(self)
        self.data_queue = data_queue
        self.mode_type = mode_type
        self.t_msg = t_msg

    def run(self):
        start = timeit.default_timer()
        log.error(':{}:--------------------------run----------------------------'.format(self.t_msg))
        try:
            while True:
                if self.data_queue.empty():
                    log.error(':{}:All items have been taken off the queue'.format(self.t_msg))
                    break;
                else:
                    dataFromQueue = self.data_queue.get_nowait()
                    log.debug(':{}:run-dataFromQueue:'.format(self.t_msg,dataFromQueue))
                    url, data_service = modify_service_lsms(self.mode_type,
                                                    dataFromQueue['svc_number'],
                                                    dataFromQueue['mvno_business_mark'])
                    log.debug(':{}:run-url:{}'.format(self.t_msg,url))
                    data = json.dumps(data_service, indent=4) #字典转换json格式，并且带格式
                    # print(data)
                    log.debug(':{}:run-data:{}'.format(self.t_msg,data))
                    try:
                        rsp = requests.post(url=url, data=data, verify=False) # 请求
                        #由更新 改为直接插入
                        # update_ldnum = {
                        #     'FLAG' : 1,
                        #     'SVC_NUMBER' : dataFromQueue['svc_number']
                        # }
                        # msg = sql.update_modify_lsms_user(update_ldnum)
                        insert_data = [(
                            dataFromQueue['svc_number'], dataFromQueue['mvno_business_mark'],dataFromQueue['prov_code'],
                            dataFromQueue['eparchy_code'], time.strftime('%Y-%m-%d %H:%M:%S.0', time.localtime()), 1)]
                        log.debug('insert_data:{}'.format(insert_data))
                        modify_lsms_user = sql.insert_modify_lsms_user(insert_data)
                        # print('操作数据入表：', modify_lsms_user)
                        log.debug(':{}:->号码：{}->数据入表->modify_lsms_user:{}'.format(self.t_msg,
                                                                        dataFromQueue['svc_number'],modify_lsms_user))
                    except Exception as e:
                        print('接口调用失败：{}'.format(e))
                    rsp_json = rsp.json() #响应文本json化
                    print(rsp_json)
                    log.debug(':{}:Response:{}->{}'.format(self.t_msg,dataFromQueue['svc_number'],rsp.text))
                    try:
                        if "data" in rsp_json and rsp_json.get('data').get('message') == 'SUCCESS':
                            log.error(':{}:run-rsp_json:{}:-:{}'.format(self.t_msg,'SUCCESS',
                                                                               dataFromQueue['svc_number']))
                            with open("Seccess_number.txt", 'a') as file:
                                file.write(strftime("%Y-%m-%d %H:%M:%S") + ':' + dataFromQueue['mvno_business_mark'] +
                                           ',' + str(dataFromQueue['svc_number']) + ',' + str(data_service['data']['order_id']) + '\n')
                        else:
                            log.error(':{}:run-rsp_json:{}:-:{}'.format(self.t_msg, 'Fail',
                                                                        dataFromQueue['svc_number']))
                            with open("fail_number.txt", 'a') as file:
                                file.write(strftime("%Y-%m-%d %H:%M:%S") + ':' + dataFromQueue['mvno_business_mark'] +
                                           ',' + str(dataFromQueue['svc_number']) +  ',' + str(data_service['data']['order_id']) +'\n')
                    except Exception as e:
                        log.debug(':{}:Error：-->{}'.format(self.t_msg,e))
        except Exception as e:
            log.info(':{}:Creat_Thread-run-Exception:-->{}'.format(self.t_msg,e))
        end = timeit.default_timer()
        print('--Running time: %s Seconds' % (end - start))


def getRandomString(number):

    rule = string.ascii_letters + string.digits
    str = random.sample(rule, number)
    return "".join(str)

def modify_service_lsms(mode_type,svc_number, mvno_business_mark):

    mvnokey,token = mvno_token.mvno_token(mode_type,mvno_business_mark)
    serial_number = "LDBGAPI" + strftime("%Y%m%d%H%M%S") + getRandomString(9)
    timestamp = strftime("%Y-%m-%d %H:%M:%S")
    service = {
        "mvnokey": mvnokey,
        "serial_number": serial_number,
        "timestamp": timestamp,
        "service_type": "basic_service",
        "service_name": "requests_url",
        "api_name": "cu.vop.basic_service.requests_url",
        "data": {
            "order_id": serial_number + "0000",
            "phone_number": svc_number
        }
    }
    global API_IP
    url = '接口地址'
    return url, service


if __name__ == '__main__':
    import ctypes
    nowTime = time.strftime('%m%d_%H', time.localtime(time.time()))
    pid = os.getpid()
    log.error('Process-ID:-->{}'.format(pid))
    print('Process-ID:-->{}'.format(pid))
    #*********************************************************************************
    # p = psutil.Process(pid)
    # log.error( '\r' +
    #            '{}'.format('*'*150) + '\r'
    #             'Process-ID:-->{}'.format(pid) + '\r'
    #             'Process-Name:-->{}'.format(p.name()) + '\r'
    #             'CPU-Usage:-->{}'.format(p.cpu_times()) + '\r'
    #             'Memory-usage:-->{}'.format(p.memory_percent()) + '\r'
    #             'Disk-Read-Msg:-->{}'.format(p.io_counters()) + '\r'
    #             'Process-Thread-Num:-->{}'.format(p.num_threads()) + '\r'
    #              '{}'.format('*'*150) + '\r'
    #           )
    # **********************************************************************************
    data_queue = Queue()  # 创建队列
    #*********************************************************************************
    data_dict = {
        'PROV_CODE' : '86',
        'FLAG':'0'
    }
    mode_type = 'test'  #获取 什么环境 中得 mvnokey token （测试 = test 、 生产 = produce）
    thread_num = 50  # 线程数
    #*********************************************************************************
    print('备份&删除-数据操作状态表记录：modify_lsms_user 中....')
    create_msg = sql.create_modify_lsms_user(nowTime)
    print('备份：{}'.format(create_msg))
    truncate_msg = sql.truncate_modify_lsms_user()
    print('删除：{}'.format(truncate_msg))
    data = sql.modify_lsms(data_dict) #获取漏电要变更得手机号码。入参省份和地市
    log.debug('本次执行数据：{}'.format(data))

    if data is None or data == []:
        print('modify_lsms无可用数据')
        exit(1)

    insert_mlu = []
    for i in data:
        # insert_data = (i[0],i[1],i[2],i[3],time.strftime('%Y-%m-%d %H:%M:%S.0', time.localtime()),i[4])
        # insert_mlu.append(insert_data)
        ld_data = {
            'svc_number':int(i[0]),
            'mvno_business_mark':i[1],
            'prov_code':i[2],
            'eparchy_code':i[3]
        }
        data_queue.put_nowait(ld_data)

    # modify_lsms_user = sql.insert_modify_lsms_user(insert_mlu)
    # print('操作数据入表：',modify_lsms_user)
    # log.error('数据入表->modify_lsms_user:{}'.format(modify_lsms_user))

    # 创建新线程
    threads = []
    t_msg = 0 #自定义线程号 Python的太难获取
    for tName in range(thread_num):
        t_msg += 1
        # t_id = ctypes.CDLL('libc.so.6').syscall(186) # 获取线程ID，windows环境需要进行适配
        thread = Creat_Thread(mode_type,t_msg,data_queue)
        thread.start()
        threads.append(thread)
    # 等待所有线程完成
    for t in threads:
        t.join()
    print("Exiting Main Thread")

其中sql不做展示，需要可以私密提供