多线程和多进程
线程队列
from queue import Queue
if __name__ == '__main__':
q = Queue()
q.put(100)
q.put(200)
print('个数:', q.qsize())
print(q.get())
print('个数:', q.qsize())
print(q.get())
print('个数:', q.qsize())
队列在多线程中用法
from multiprocessing import Process, Queue, current_process
import time
from random import randint
from threading import Thread
"""
1)基本操作:
创建队列对象:Queue()
添加数据: 队列对象.put(数据)
获取数据: 队列对象.get() / 队列对象.get(timeout=超时时间)
2)注意事项:
如果想要使用一个队列对象获取不同进程中的数据,这个队列必须通过关联进程的函数的参数传递到进程中
"""
def download(name, q:Queue):
print(f'{name}开始下载')
time.sleep(randint(2, 9))
print(f'{name}下载结束')
q.put(f'{name}数据')
def del_data(q: Queue):
while True:
data = q.get()
if data == 'end':
break
print(f'--------------处理{data}----------------')
if __name__ == '__main__':
queue = Queue()
del_p = Process(target=del_data, args=(queue,))
del_p.start()
names = [f'电影{x}' for x in range(1, 11)]
ps = []
for name in names:
p = Process(target=download, args=(name, queue))
p.start()
ps.append(p)
for p in ps:
p.join()
queue.put('end')
进程队列
from multiprocessing import Process, Queue, current_process
import time
from random import randint
from threading import Thread
"""
1)基本操作
"""
def download(name, q: Queue):
global result
print(f'{name}开始下载')
print('下载:', current_process())
time.sleep(randint(2, 9))
print(f'{name}下载结束')
q.put(100)
if __name__ == '__main__':
queue = Queue
线程池
from concurrent.futures import ThreadPoolExecutor
import time
from random import randint
from threading import current_thread
from datetime import datetime
def download(name):
print(f'{name}开始下载')
print(current_thread())
time.sleep(1)
print(f'{name}下载结束')
pool = ThreadPoolExecutor(max_workers=50)
names = [f'电影{x}' for x in range(100)]
pool.map(download, names)
pool.submit(download, 'hello!')
pool.shutdown()
线程池使用的细节
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED, as_completed
import time
from random import randint
def download(name, x):
print(f'{name}-{x}开始下载')
time.sleep(randint(1, 4))
print(f'{name}-{x}下载结束')
return f'{name}数据'
if __name__ == '__main__':
pool = ThreadPoolExecutor(max_workers=30)
f = pool.submit(download, '电影1', 10)
all_task = [pool.submit(download, f'电影{x}', x*10) for x in range(100)]
for task in as_completed(all_task):
print(f'------------{task.result()}------------------')
print('--------------------电影下载完成--------------------')
进程池
from multiprocessing import Pool
import time
from random import randint
def download(name):
print(f'{name}开始下载')
time.sleep(randint(1, 4))
print(f'{name}下载结束')
return f'{name}数据'
if __name__ == '__main__':
pool = Pool(10)
"""
a. 进程池对象.apply(函数, 参数) - 同步(串行);进程池中的多个任务串行执行
b. 进程池对象.apply_async(函数, 参数) - 异步执行(并行);必须配合close和join一起用
函数 - 任务对应的函数的函数名
参数 - 元组;调用任务函数的时候的实参,需要多少个实参,元组中就给多少个元素
"""
"""
进程池.map(函数, 参数序列) - 序列中有多少个元素就添加多少个任务;
进程池中的任务并行, 进程池中的任务和主进程串行
进程池.map_async(函数, 参数序列) - 进程池中的任务和主进程并行执行
map和map_async的返回值是所有任务对应的函数的返回值
"""
result = pool.map_async(download, [(f'电影{x}', x * 10) for x in range(20)])
print('--------------主进程-------------------')
pool.close()
pool.join()
print('---------------下载完成--------------------')
print(result.get())
线程池
import requests
from re import findall
from json import loads
from concurrent.futures import ThreadPoolExecutor, as_completed, wait
import csv
def get_one_page(page):
headers = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36'
}
url = f'https://search.51job.com/list/000000,000000,0000,00,9,99,python,2,{page}.html?lang=c&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare='
response = requests.get(url, headers=headers)
json_data = findall(r'window.__SEARCH_RESULT__ =(.+?)</script>', response.text)[0]
dict_data = loads(json_data)
page_data = []
for job in dict_data['engine_search_result']:
page_data.append({
'工作名称': job['job_name'],
'工作详情': job['job_href'],
'薪资': job['providesalary_text'],
'工作地点': job['workarea_text'],
'工作年限': job['workyear'],
'福利待遇': job['jobwelf'],
'公司': job['company_name']
})
return page_data
if __name__ == '__main__':
pool = ThreadPoolExecutor(20)
all_task = [pool.submit(get_one_page, page) for page in range(1, 101)]
f = open('files/python.csv', 'a', encoding='utf-8')
writer = csv.DictWriter(f, ['工作名称', '工作详情', '薪资', '工作地点', '工作年限', '福利待遇', '公司'])
writer.writeheader()
for t in as_completed(all_task):
print('----------处理一页数据---------')
writer.writerows(t.result())