futures/asyncio/multiprocess

多进程/多线程

import concurrent.futures
import multiprocessing
import asyncio

# ##########################多进程1####################################
# async def insertAndUpdate(x):
def insertAndUpdate(x):
    sql = text('''
    INSERT INTO db_test.dw_org_addr_latest(
        `org_id`             , -- '企业ID',
        `addr_type`          , -- '地址用途（1-注册地 2-经营地-总部）',
        `address`            , -- '企业地址',
        `addr_md5`           , -- '地址MD5（逻辑外键）',
        `lng_lat_GCJ02`      , -- '经纬度',
        `lng_lat_GCJ02_md5`  , -- '经纬度(GCJ02)MD5',
        `UA0003`             , -- '地址所在国家/地区',
        `UA0004`             , -- '地址所在[省(自治区/直辖市)]',
        `UA0005`             , -- '地址所在[地(区/市/州/盟)]',
        `UA0006`             , -- '地址所在[县(区/市/旗)]',
        `UA0007`             , -- '地址所在[(街道办事处)]',
        `UA0008`             , -- '地址所在社区/社区居委会',
        `park_code`          , -- '地址所在园区',
        `district_code`       -- '地址所在片区',
        ) 
        VALUES(
        :org_id             ,
        :addr_type          ,
        :address            ,
        :addr_md5           ,
        :lng_lat_GCJ02      ,
        :lng_lat_GCJ02_md5  ,
        :UA0003             ,
        :UA0004             ,
        :UA0005             ,
        :UA0006             ,
        :UA0007             ,
        :UA0008             ,
        :park_code          ,
        :district_code      
        )
        ON DUPLICATE KEY
        UPDATE 
        address              =  :address           ,
        addr_md5             =  :addr_md5          ,
        lng_lat_GCJ02        =  :lng_lat_GCJ02     ,
        lng_lat_GCJ02_md5    =  :lng_lat_GCJ02_md5 ,
        UA0003               =  :UA0003            ,
        UA0004               =  :UA0004            ,
        UA0005               =  :UA0005            ,
        UA0006               =  :UA0006            ,
        UA0007               =  :UA0007            ,
        UA0008               =  :UA0008            ,
        park_code            =  :park_code         ,
        district_code        =  :district_code     
        ;
    ''').bindparams(
        org_id=x.org_id,
        addr_type=x.addr_type,
        address=x.address,
        addr_md5=x.addr_md5,
        lng_lat_GCJ02=x.lng_lat_GCJ02,
        lng_lat_GCJ02_md5=x.lng_lat_GCJ02_md5,
        UA0003=x.UA0003,
        UA0004=x.UA0004,
        UA0005=x.UA0005,
        UA0006=x.UA0006,
        UA0007=x.UA0007,
        UA0008=x.UA0008,
        park_code=x.park_code,
        district_code=x.district_code
    )
    with engine.connect() as conn:
        conn.execute(sql)
    # await engine.connect().execute(sql)

# 普通0:20:35.624958
# data.apply(lambda x:insertAndUpdate(x),axis=1)
# 异步0:19:41.543852
# loop = asyncio.get_event_loop()
# tasks = [insertAndUpdate(row) for index, row in data.iterrows()]
# loop.run_until_complete(asyncio.wait(tasks))
# 多线 cost: 0:02:01.558449
with concurrent.futures.ThreadPoolExecutor(workers) as executor:
    executor.map(insertAndUpdate, [row for index,row in data.iterrows()])

# ########################多进程2######################################

def dw_addr_admin_area_code(data,workers,return_dict):
    with concurrent.futures.ThreadPoolExecutor(workers) as executor:
        res = executor.map(getDwAddrAdminAreaCode, data["lng_lat_GCJ02_md5"])
    res = pd.DataFrame(res,columns=["UA0003","UA0004","UA0005","UA0006","UA0007","UA0008"])
    return_dict["dw_addr_admin_area_code"]=res

def dw_addr_park_code(data,workers,return_dict):
    with concurrent.futures.ThreadPoolExecutor(workers) as executor:
        res = executor.map(getDwAddrParkCode, data["lng_lat_GCJ02_md5"])
    res = pd.DataFrame(res,columns=["park_code"])
    return_dict["dw_addr_park_code"]=res

def dw_addr_district_code(data,workers,return_dict):
    with concurrent.futures.ThreadPoolExecutor(workers) as executor:
        res = executor.map(getDwAddrDistrictCode, data["lng_lat_GCJ02_md5"])
    res = pd.DataFrame(res,columns=["district_code"])
    return_dict["dw_addr_district_code"]=res

manager = multiprocessing.Manager()
return_dict = manager.dict()

process_jobs = [dw_addr_admin_area_code,dw_addr_park_code,dw_addr_district_code]
process_jobs = [
    multiprocessing.Process(target=_, args=(data, workers, return_dict))
    for _ in process_jobs
                ]
[_.start() for _ in process_jobs]
while 1:
    process_statue = [_.is_alive() for _ in process_jobs]
    if sum(process_statue) == 0:
        break
[_.join() for _ in process_jobs]
# ##############################################################

异步

# https://docs.python.org/zh-cn/3/library/asyncio-task.html
import asyncio
import time
from datetime import datetime

async def say_after(delay, what):
    await asyncio.sleep(delay)
    print(what)
    return what

# 原版，没有启动到异步
async def main():  # 0:00:03.002929
    print(f"started at {time.strftime('%X')}")

    await say_after(1, 'hello')
    await say_after(2, 'world')

    print(f"finished at {time.strftime('%X')}")

# 异步1
async def main():  # 0:00:02.002949
    task1 = asyncio.create_task(
        say_after(1, 'hello'))

    task2 = asyncio.create_task(
        say_after(2, 'world'))

    print(f"started at {time.strftime('%X')}")

    # Wait until both tasks are completed (should take
    # around 2 seconds.)
    res = []
    a = await task1
    res.append(a)
    a = await task2
    res.append(a)

    print(f"finished at {time.strftime('%X')}")
    print(res)

# 异步2
async def main():  # 0:00:02.002968
    print(f"started at {time.strftime('%X')}")

    res = await asyncio.gather(
        say_after(1, 'hello'),
        say_after(2, 'world')
    )

    print(f"finished at {time.strftime('%X')}")
    print(res)

a = datetime.now()
asyncio.run(main())
print(datetime.now() -a )