利用Selenium爬取猫眼电影榜单前100的电影图片
import os
import time
import bs4
import requests
from selenium import webdriver
driver = webdriver.Chrome()
url = 'https://maoyan.com/board/4'
head = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/80.0.3987.163 Safari/537.36'
}
driver.get(url)
for page in range(10):
if page > 0:
next_page = driver.find_element_by_link_text('下一页')
time.sleep(1)
next_page.click()
soup = driver.page_source
soup_html = bs4.BeautifulSoup(soup, 'html.parser')
list_img = []
if page == 0:
folder = 'image'
g = 1
try:
os.mkdir(folder)
except:
folder = 'image(' + str(g) + ')'
g += 1
os.mkdir(folder)
os.chdir(folder)
list_html = soup_html.find_all('img', class_='board-img')
for x in list_html:
if x.get('src') != None:
list_img.append(x['src'])
else:
list_img.append(x['data-src'])
for i in range(len(list_img)):
with open("./image/"+str(page)+str(i)+'.jpg', 'wb') as f:
img_html = requests.get(list_img[i], headers=head)
f.write(img_html.content)
i += 1
page += 1
driver.close()