import requests
from multiprocessing import Pool
import time
headers={
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36'
}
def get_url():
url_list=[]
for i in range(10):
url='https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start={}'.format(str((i+1)*20))
url_list.append(url)
return url_list
def get_data(url):
res=requests.get(url,headers=headers)
print(res)
time.sleep(1)
if __name__=='__main__':
urls=get_url()
pool=Pool(processes=4)#设置进程数目
pool.map(get_data,urls)
multiprocessing包可以使用pip install mutiprocess下载