python哪里接爬虫任务-python写爬虫如何实现任务队列？

至少三种方法及相应的参考实现：

from multiprocessing.dummy import Pool as ThreadPool

def worker(n):

return n + 2

numbers = range(100)

pool = ThreadPool(processes=10)

result = pool.map(worker, numbers)

pool.close()

pool.join()

print(result)

import concurrent.futures

def worker(n):

return n + 2

numbers = range(100)

with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:

result = executor.map(worker, numbers)

print(list(result))

from collections import deque

import queue

import threading

def do_work(n):

return n + 2

def worker():

while True:

item = q.get()

if item is None:

break

result.append(do_work(item))

q.task_done()

q = queue.Queue()

result = deque()

num_worker_threads = 10

threads = []

for i in range(num_worker_threads):

t = threading.Thread(target=worker)

t.start()

threads.append(t)

for item in range(100):

q.put(item)

# block until all tasks are done

q.join()

# stop workers

for i in range(num_worker_threads):

q.put(None)

for t in threads:

t.join()

print(result)

官方文档：