今天学requests遇到的bug和解决~10.14

想爬点知乎图片结果抱一堆错

一、pip install bs4时报错：

Fatal error in launcher: Unable to create process using '"c:\users\administrator\desktop\1\scripts\python.exe"  "D:\BaiduSyncdisk\code\python\1\Scripts\pip.exe" install bs4'，

所以就python -m pip install -U pip了一下，再安装bs4时就成功了，

接着尝试爬取，报错bs4.FeatureNotFound: Couldn't find a tree builder with the features you requested: lxml. Do you need to install a parser library?然后安装lxml

在运行就好了

代码附上

import requests
from bs4 import BeautifulSoup
from requests.exceptions import RequestException
import os
import re

url = "----------------改成你的网址---------------------"
os.makedirs(r"-----------改成你的保存路径------------")


def get_one_page(url):
    try:
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.82"
        }
        # 请求头要添加，不然访问失败
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
            # 返回页面html文本
        return None
    except RequestException:
        return None


def get_image_url(html):
    soup = BeautifulSoup(html, "lxml")
    items = soup.select("figure")
    # 将figure节点选择出来
    """for i in range(1,4):
       print(items[i])"""
    return items


def save_image(item, i):
    try:
        os.chdir(r"-----------改成你的保存路径------------")
        # 改变工作目录
        pattern = re.compile('<noscript>.*?data-original="(.*?)".*?</noscript>', re.S)
        # 构建正则表达式
        image_url = re.search(pattern, str(item))
        # 因为item格式为<soup.tag>,要转换成string
        r = requests.get(image_url.group(1))
        if r.status_code == 200:
            file_path = str(i) + ".jpg"
            with open(file_path, "wb") as f:
                f.write(r.content)
    except requests.ConnectionError:
        print("Failed to save image")


def main():
    html = get_one_page(url)
    image_urls = get_image_url(html)
    i = 0
    for image_url in image_urls:
        i += 1
        save_image(image_url, i)
        print("page", i)


main()