爬取博客文章保存到本地（失败）

最新推荐文章于 2024-09-10 17:48:39 发布

Xiaoweidumpb

最新推荐文章于 2024-09-10 17:48:39 发布

阅读量145

点赞数

分类专栏： python

本文链接：https://blog.csdn.net/qq_43751489/article/details/109405250

版权

python 专栏收录该内容

23 篇文章 0 订阅

订阅专栏

用 Selenium实现自动化比较好…

import requests
from bs4 import BeautifulSoup
import re

header = {
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36"
}
article_info = {}


def get_html(url):
    r = requests.get(url=url, headers=header)
    if (r.status_code == 200):
        r.encoding = 'utf-8'
        soup = BeautifulSoup(r.text, 'html.parser')
        # print(soup)
        return soup
    else:
        print("请求网页失败")


def get_article_url(soup):
    try:
        article_list_div = soup.find('div', attrs={'class': 'article-list'})
        items = article_list_div.find_all('a', attrs={'target': '_blank'})
        for item in items:
            article_url = item['href']
            # 文章标题和原创标签混在一起，直接使用正则清洗
            # strip 清洗空格
            article_name = item.get_text().replace('原创', '').replace('转载', '').strip()
            article_info[article_name] = article_url
    except Exception as e:
        print(e)
        get_article_url(soup)


def save_article():
    for name, url in article_info.items():
        soup=get_html(url)
        for script in soup(["script", "style"]):
            #print(script)
            script.extract()
        # for link in soup(['link']):
        #     print(link)
        #     print("---------------")
        #     # link.extract()
        soup=str(soup).replace("csdn","")
        try:
            with open('D:/python_save/web/'+name+'.html', 'w',encoding='utf-8') as f:
                f.write(soup)
                f.close()
        except Exception as e:
            print(name+'.html'+"保存错误")
            print(e)

    # for name, url in article_info.items():
    #     #print(name, url)
    #     r = requests.get(url=url, headers=header)
    #     if (r.status_code == 200):
    #         r.encoding = 'utf-8'
    #         html=r.text
    #     try :
    #         with open('D:/python_save/web/'+name+'.html', 'w',encoding='utf-8') as f:
    #             f.write(html)
    #             f.close()
    #     except Exception as e:
    #         print(name+'.html'+"保存错误")
    #         print(e)


if __name__ == '__main__':
    for i in range(1, 3):
        url = 'https://blog.csdn.net/qq_43751489/article/list/' + str(i)
        get_article_url(get_html(url))
    save_article()