Python爬取漫画之咒术回战

爬取漫画

• 一次性加载的网站：使用requests + bs4就足够了
• 带加载的网站：使用requests + selenium无界面操作

import requests, os, pprint, time, re
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup

# 无界面初始化
chrome_opt = Options()      # 创建参数设置对象.
chrome_opt.add_argument('--headless')   # 无界面化.
chrome_opt.add_argument('--disable-gpu')    # 配合上面的无界面化.
chrome_opt.add_argument('--window-size=1366,768')   # 设置窗口大小, 窗口大小会有影响.

# 创建Chrome对象并传入设置信息.
driver = webdriver.Chrome(chrome_options=chrome_opt)


# “咒术回战”的漫画主网站
manhuaWebMap = 'https://www.xxxxxxxxxxxxxxxxxxxxxxxx.com'
# 获取html文本形式
res = requests.get(manhuaWebMap).text
# 解析lxml
soup = BeautifulSoup(res, 'lxml')
# 选中Id属性为"chapter-list-1"下的所有<li>标签
elems = soup.find(id="chapter-list-1").find_all('li')
# 读取标签
for i in elems:

    # 选中<a>标签的位置
    mainTarge = i.find('a')
    # 漫画第几话地址
    manhuaOneMap = mainTarge.get('href')
    # 漫画第几话编号
    manhuaOneName = mainTarge.get('title')
    print(f'Downloading page {manhuaOneName}')
    manhuaFolderMap = fr'G:\xxxxx\咒术回战\{manhuaOneName}'
    # 创建存放路径，exist_ok设置为True，若路径存在不报错
    os.makedirs(manhuaFolderMap, exist_ok=True)
    
    driver.get(f'https://www.manhuabei.com/{manhuaOneMap}')
    #得到页数
    pageNumber = int((driver.find_element_by_xpath('//*[@id="images"]/p').text)[-3:-1])
    for j in range(1, pageNumber+1):
        driver.get(f'https://www.manhuabei.com/{manhuaOneMap}?p={j}')
        # 图片地址
        manhuaPageMap = driver.find_element_by_xpath('//*[@id="images"]/img').get_attribute('src')
        try:
            with open(fr'{manhuaFolderMap}\{str(j).rjust(2, '0')}.jpg', 'wb') as f:
            	# 写入get到的图片
                f.write(requests.get(manhuaPageMap).content)
        except Exception:
            pass
        finally:
            time.sleep(1)
        print('-------------------------------------------')
    time.sleep(3)
print(len(elems))