#案例:所有电影都下载结束打印 '全部下载完成'defdownload(name):print(f'{name}开始下载:{datetime.now()}')
time.sleep(randint(2,7))print(f'{name}下载结束:{datetime.now()}')
names =['肖申克的救赎','触不可及','头脑特工队','恐怖游轮']
ts =[]for name in names:
t = Thread(target=download, args=(name,))
t.start()
ts.append(t)for t in ts:
t.join()print('=============全部下载完成=============')
#案例:多线程获取豆瓣top250电影数据import requests
from threading import Thread
from bs4 import BeautifulSoup
import csv
defget_one_page(url):
headers ={'user-agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36'}
res = requests.get(url, headers=headers)
soup = BeautifulSoup(res.text,'lxml')
all_li = soup.select('ol.grid_view>li')
page_data =[]for li in all_li:
name = li.select_one('.title').text
score = li.select_one('.rating_num').text
page_data.append([name, score])# 方案一:获取完一页数据就往文件中写一页数据# writer.writerows(page_data)# 方案二:获取完一页数据后将数据添加到保存所有数据的大列表中
all_data.extend(page_data)print('======一页数据获取完成=====')if __name__ =='__main__':# 1. 创建csv文件对应的writer
writer = csv.writer(open('files/豆瓣电影2.csv','w', encoding='utf-8', newline=''))
writer.writerow(['名字','评分'])# 方案二:
all_data =[]# 2. 多线程下载数据
urls =[f'https://movie.douban.com/top250?start={x}&filter='for x inrange(0,250,25)]
ts =[]for url in urls:
t = Thread(target=get_one_page, args=(url,))
t.start()
ts.append(t)for t in ts:
t.join()print('===================全部完成====================')# 方案二:
all_data.sort(key=lambda item:float(item[-1]), reverse=True)
writer.writerows(all_data)
#案例:多进程爬取东京数据import time
from selenium.webdriver import Chrome, ChromeOptions
from multiprocessing import Process
from selenium.webdriver.common.keys import Keys
options = ChromeOptions()
options.add_experimental_option('excludeSwitches',['enable-automation'])
options.add_experimental_option("prefs",{"profile.managed_default_content_settings.images":2})defget_one_goods(name):
b = Chrome(options=options)
b.get('https://www.jd.com')
search = b.find_element_by_id('key')
search.send_keys(name)
search.send_keys(Keys.ENTER)
time.sleep(30)if __name__ =='__main__':
goods =['运动鞋','笔记本电脑']for x in goods:
p = Process(target=get_one_goods, args=(x,))
p.start()