爬取鬼灭之刃漫画

import requests
import pandas as pd
from bs4 import BeautifulSoup
import os
import time



headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
}


def getHTMLText(url):
    try:
        r = requests.get(url, headers=headers, timeout=20)
        r.raise_for_status()
        r.encoding = 'unicode'
        return r.text
    except:
        print("爬取失败")


def getChapterList(url, chapter_name_list, chapter_link_list):
    html = getHTMLText(url)
    soup = BeautifulSoup(html, "html.parser")
    for tag in soup.find_all(class_='j-chapter-link'):
        if tag.string is not None:
            chapter_link = url + tag['data-hreflink'].split("/")[-1]
            chapter_name = tag.string.strip()
            chapter_link_list.append(chapter_link)
            chapter_name_list.append(chapter_name)
    return chapter_link_list.reverse(), chapter_name_list.reverse()


def saveChapterName(chapter_name_list, chapter_link_list):
    DataSet = list(zip(chapter_name_list, chapter_link_list))
    df = pd.DataFrame(data=DataSet, columns=['name', 'link'])
    print(df)
    try:
        df.to_csv("mk_chapter_name.csv", mode="a+", index=None, encoding="gb18030")
    except:
        print("数据写入失败")


def getImage(chapter_name_list, chapter_link_list, images, root1, root2):
    if not os.path.exists(root1):
        os.makedirs(root1)
    for chapter_name in chapter_name_list:
        root2 = root1+"/"+chapter_name
        if not os.path.exists(root2):
            os.makedirs(root2)
    for chapter_link in chapter_link_list:
        html = getHTMLText(chapter_link)
        soup = BeautifulSoup(html, "html.parser")
        for tag in soup.find_all(class_="lazy-read"):
            image = tag['data-src']
            images.append(image)
    for image in images:
        path = root2+"/" + image.split("/")[-1].split("!")[0]
        if not os.path.exists(path):
            with open(path, "wb") as f:
                f.write(requests.get(image).content)
                f.close()
                print(image.split("/")[-1].split("!")[0]+"图片保存成功")
                print("总进度：+str()


def main():
    url = "https://www.mkzhan.com/209827/"
    list1 = []
    list2 = []
    list3 = []
    root = "D://guimiezhiren"
    root2 = ""
    getHTMLText(url)
    getChapterList(url, list1, list2)
    saveChapterName(list1, list2)
    getImage(list1, list2, list3, root, root2)


main()