简单爬取网页数据

首先定义一个获取网页文本的函数

import requests
def get_text(url):
    # 手动设置请求头中的 user-agent
    headers = {
        "user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0"
    }
    try:
        #记录访问信息
        print(f"正在访问{url}...")
        response = requests.get(url,headers=headers)
        #如果状态码不是2xx，则抛出异常
        response.raise_for_status()
        #根据适配编码响应
        response.encoding=response.apparent_encoding
        #返回网页信息
        return response.text
    except:
        #访问失败则返回空白
        print(f"访问{url}错误!")
        return ""

定义一个函数获取网页的bytes数据

def get_content(url):
    # 手动设置请求头中的 user-agent
    headers = {
        "user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0"
    }
    try:
        #记录访问信息
        print(f"正在访问{url}...")
        response = requests.get(url,headers=headers)
        #如果状态码不是2xx，则抛出异常
        response.raise_for_status()
        #根据适配编码响应
        response.encoding=response.apparent_encoding
        #返回网页信息
        return response.content
    except:
        #访问失败则返回空白
        print(f"访问{url}错误!")
        #返回一个二进制数据
        return b""

1.利用BeatifulSoup库获取对应信息

基本步骤

#BeautifulSoup => bs =>将html解析成BS对象

import bs4
import requests
import os
from download import get_text

# 创建存放图片的文件夹
img_path = "./douban_top_250_imgs"
if not os.path.exists(img_path):
    os.makedirs(img_path)

# 存放数据的位置
file = open("douban_top_250.csv","w",encoding="utf-8")
#访问网址
base_url = "https://book.douban.com/top250?start=0"

text = get_text(base_url)
#得到了一个html文本数据
#从html中提取需要的数据

#BeautifulSoup => bs =>将html解析成BS对象


if text:
    # 转化成bs4对象
    soup =bs4.BeautifulSoup(text,'lxml')
    #找到标签名为div，属性中id为content的对象，再寻找包括其中的table标签
    b_list = soup.find("div",attrs={"id":"content"}).find_all("table")
    print(b_list,len(b_list))


    for book in b_list:
        #书名数据
        book_title = book.find("div",attrs={"class":"pl2"}).a.get("title")
        #print(book_title)
        #书的具体信息数据
        b_info = book.find("p",attrs={"class":"pl"}).get_text()
        #print(b_info)
        #评分数据
        book_rate = book.find("span", attrs={"class":"rating_nums"}).get_text()
        #print(book_rate)
        #图片路径
        img_url =book.find("img").get("src")
        #print(img_url)
        # 数据存储
        file.write(f"{book_title},{b_info},{book_rate},\n")

        with open(f"{img_path}/{book_title}.jpg","wb") as f:
            f.write(download.get_content(img_url))

file.close()

2.利用lxml库将html解析成Element对象对元素进行查找

根据lxml查找的基本格式

# root_ele.xpath('//标签[@属性名="属性值"]')
# root_ele.xpath('//div[@class="mnav"]')
# root_ele.xpath('//div[@class]/li[@class="nav"]/@属性名')
# root_ele.xpath('//div[@class]/li[@class="nav"]/text()')

# /nodename : 从根节点开始选取子节点
# ./nodename  : 从当前节点选取子节点
# //nodename  :从根节点开始匹配子孙节点
# .//nodename :从当前节点开始匹配子孙节点

from lxml import html
import os
import download
from download import get_text

# 创建存放图片的文件夹
img_path = "./douban_top_250_xpath_imgs"
if not os.path.exists(img_path):
    os.makedirs(img_path)


file = open("douban_top_250_xpath.csv","w",encoding="utf-8")



# 获取10个网页数据
for i in range(10):
    base_url = f"https://book.douban.com/top250?start={i*25}"
    text = get_text(base_url)

    if text:
        root = html.fromstring(text)
        # 先找到所有书的table
        all_books = root.xpath("//table")

        for book in all_books:
            # 书名中的空格进行切割再重新拼接
            book_title1 = "".join(book.xpath(".//td[2]/div[@class='pl2']/a/text()")[0].split())
            book_title2 = "".join(book.xpath(".//td[2]/div[@class='pl2']/a/span/text()"))
            book_title =f"{book_title1}{book_title2}".replace(":","：")
            #print(book_title)
            book_info = book.xpath(".//td[2]/p[@class='pl']/text()")[0]
            #print(book_info)
            book_rating =book.xpath(".//td[2]/div[@class='star clearfix']/span[@class='rating_nums']/text()")[0]
            #print(book_rating)
            book_cover_url =book.xpath(".//img[@width='90']/@src")[0]
            #print(book_cover_url)
            path = os.path.join(img_path,book_title)
            #写入封面图片
            with open(path,"wb") as f:
                f.write(download.get_content(book_cover_url))
            #保存基本数据到csv文件中
            file.write(f"{book_title},{book_info},{book_rating}\n")

file.close()

结果部分展示：