python 多进程

# -*- coding:utf-8 -*-

import time
import pymysql
from multiprocessing import Process
from datetime import date

month = date.today().strftime("%Y%m")

HOST, USER, PASSWD, DB, PORT = '192.168.1.1', 'admin', 'password', 'database', 3306
sql_info = [HOST, USER, PASSWD, DB, PORT]
table_task = "xxxx"


def get_circle_address():
    select_sql = f"SELECT id,name FROM {table_task} "
    connect, cursor = mysql_connect(sql_info)
    num = cursor.execute(select_sql)
    tasklist = cursor.fetchall()
    connect.close()

    print("There is [%s] task need to do !!!" % num)
    process_count, works_process = 8, []
    process_task_num = len(tasklist) / process_count
    if process_task_num == 0:
        process_count, process_task_num = len(tasklist), 1
    for i in range(process_count):
        if i != process_count - 1:
            source_list = tasklist[int(i * process_task_num): int((i + 1) * process_task_num)]
        else:
            source_list = tasklist[int(i * process_task_num):]
        work = Process(target=handle_circle_address, args=(source_list,))
        work.start()
        works_process.append(work)
    for work in works_process:
        work.join()
    del works_process


# MySQL数据库连接
def mysql_connect(args):
    host, user, passwd, db, port = args
    count = 10
    while True:
        try:
            connect = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=port, charset='utf8')
            cursor = connect.cursor()
            break
        except pymysql.Error as e:
            count -= 1
            print("MySQL Error %d: %s" % (e.args[0], e.args[1]))
            time.sleep(5)
            if not count:
                count = 10
                print("Too many connection errors! Please wait 60 seconds!")
                time.sleep(60)
    return connect, cursor


def handle_circle_address(tasklist):
    # 具体的处理逻辑
    pass


if __name__ == '__main__':
    print("The Program Start Time:", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
    start_time = time.time()
    get_circle_address()
    print("The Program End Time:", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
    print("The Program Use Time: [%s]" % (time.time() - start_time))

评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值