Python学习--第十一天

利用Selenium爬取猫眼电影榜单前100的电影图片

import os
import time
import bs4
import requests
from selenium import webdriver
driver = webdriver.Chrome()
url = 'https://maoyan.com/board/4'
head = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                      'AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/80.0.3987.163 Safari/537.36'
    }
driver.get(url)
for page in range(10):
    if page > 0:
        #翻页
        next_page = driver.find_element_by_link_text('下一页')
        time.sleep(1) 
        next_page.click()
    soup = driver.page_source
    soup_html = bs4.BeautifulSoup(soup, 'html.parser')
    # print(soup_html)
    list_img = []
    #创建image文件夹
      if page == 0:
        folder = 'image'
        g = 1
        try:
            os.mkdir(folder)
        except:
            folder = 'image(' + str(g) + ')'
            g += 1
            os.mkdir(folder)
            os.chdir(folder)
    list_html = soup_html.find_all('img', class_='board-img')
    # print(list_html)
    for x in list_html:
        if x.get('src') != None:
           list_img.append(x['src'])
        else:
           list_img.append(x['data-src'])
    for i in range(len(list_img)):
        with open("./image/"+str(page)+str(i)+'.jpg', 'wb') as f:
            img_html = requests.get(list_img[i], headers=head)
            f.write(img_html.content)
            i += 1
    page += 1
driver.close()