爬取豆瓣网读书数据

# 获取250本书的数据，保存到csv文件中
# 封面链接地址，书名，书本信息，书本评分 => csv
# 扩展： 把书的封面保存到本地文件夹 douban_top_250_imgs 

主代码   其他自己设置模块附后

import bs4
import requests
# http协议的请求方法：GET获取数据 POST创建数据 PUT DELETE PATCH
# base_url = "https://www.baidu.com"
# response = requests.get(base_url)
# print(response,type(response))#响应对象（状态码 响应头 响应数据）
# print(response.content)#响应二进制内容 获取图片/文件
# response.encoding="utf-8"#手动设置响应编码
# print(response.text)#响应文本内容  获取文本
# print(response.apparent_encoding)#可能的编码方式
# print(response.headers)#响应头
# print(response.cookies)#响应cookie
# print(response.status_code)#状态码
# print(response.url)#请求的URL
# print(response.history)#重定向的响应
# print(response.request)#请求对象
# print(response.request.headers)#请求头
# print(response.request.body)#请求体
from download import get_text
file = open("douban_top_250.csv", "w", encoding="utf-8")
#base_url = "https://book.douban.com/top250"
base_urls = ["https://book.douban.com/top250","https://book.douban.com/top250?start=25","https://book.douban.com/top250?start=50","https://book.douban.com/top250?start=75","https://book.douban.com/top250?start=100","https://book.douban.com/top250?start=125","https://book.douban.com/top250?start=150","https://book.douban.com/top250?start=175","https://book.douban.com/top250?start=200","https://book.douban.com/top250?start=225"]
# 写入CSV文件的头部
file.write("书名,作者/出版社,评分,封面图片\n")
for url in base_urls:
    text = get_text(url)  # 获取网页数据
# 得到一个html文本数据  python不能解析html文本
# 从html数据中提取需要的数据
# 1.re模块 正则表达式 速度很快/难
# 2.BeautifulSoup模块 bs将html解析成BS对象
# 3.lxml.xpath模块
# print(text)
    if text:
    # 转换成bs4对象
        soup = bs4.BeautifulSoup(text, "lxml")
    # 获取class="indent"的div下所有的table
    # all_books = soup.find("div", attrs={"class":"indent"}).find_all("table")
    # print(all_books, len(all_books))
    # all_books = soup.find("div", attrs={"id": "content"}).find_all("table")
    # print(all_books, len(all_books))
    all_books = soup.find_all("table")
    # print(all_books, len(all_books))

    # 提取数据
    for book in all_books:
        # book_title = book.find_all("a")[1].text
        book_title = "".join(book.find("div", attrs={"class": "pl2"}).a.text.strip().split())
        # print(book_title)
        book_info = book.find("p", attrs={"class": "pl"}).get_text()
        # print(book_info)
        book_rate = book.find("span", attrs={"class": "rating_nums"}).text
        # print(book_rate)
        book_picture = book.find("img").attrs["src"]
        # print(book_picture)
        # 数据存储
        file.write(f"{book_title},{book_info},{book_rate},{book_picture}\n")
file.close()

 download模块

#download => 模块
# 封装一个函数，实现从获取网页数据
# get方式获取，返回text
# 自适应网页编码

import requests

def get_text(url):
    # 200 => 正常返回
    # 404 => 404的html页面/空白/报错
    # logging模块（记日志）
    headers = {
        "user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0"
    }
    # try 检测异常,如果遇到异常会跳转到except分支
    try:
        print(f"正在访问{url}...")
        response = requests.get(url, headers=headers)
        # 如果状态码不是2xx，就抛出异常
        response.raise_for_status()
        response.encoding = response.apparent_encoding
        return response.text
    except:
        print(f"访问{url}出错！")
        return ""


# 测试代码要像下面这样写：
# __name__ => 如果直接执行的当前文件，__name__ 的值为 "__main__"
#          => 当是被当成模块导入的时候，__name__的值为 "download"（模块名）
## 内置变量(类)
print(__file__)
print(__name__)

if __name__ == "__main__":
    print(get_text("https://www.baidu.com"))

 了解bs4

#先导入标准库 再导入三方库
import bs4

#最后导入自定义模块
file = open("../python/aa.html", encoding="utf-8")
text = file.read()
#print (text)

#使用bs4解析数据
#参数1：要解析的文本数据，参数2：用什么解析器（html.parser,lxml快,xml，html5lib）
#返回值是一个对象 BeautifulSoup对象
soup = bs4.BeautifulSoup(text, "lxml")
print(soup,type(soup))

#获取所有a标签
#<a class="mnav"  href="http://news.baidu.com" name="tj_trnews">新闻 </a>
#标签名a  属性class="mnav" 标签文本内容：新闻
#soup.tagname =>获取第一个匹配的Tag
print(soup.title,type(soup.title))
print(soup.meta,type(soup.meta))
print(soup.div,type(soup.div))
print(soup.div.div.a,type(soup.div.div.a))
print(soup.a,type(soup.a))

#获取Tag的其他信息
#获取文本 如果tag对象包含子tag string获取不到子tag数据
#一般如果获取文本，使用get_text()方法或text
print(soup.a.get_text(),type(soup.a.get_text()))
print(soup.a.text,type(soup.a.text))
print(soup.a.string,type(soup.a.string))
print(soup.a.name)#标签名
print(soup.a.attrs)#属性
print(soup.a.attrs['class'])#获取属性值
print(soup.a["class"])
print(soup.a.get("href"))
# #修改
# soup.a["class"] = "newclass"
# del soup.a["href"]
# print(soup.a)

#查找  返回值是列表 元素=>tag
print(soup.find("a"))#获取第一个匹配的Tag
print(soup.find_all("a"))#获取所有匹配的Tag
print(soup.find_all("a",limit=2))#获取所有匹配的Tag,限制数量
#.find_all(tagname,attrs={},limit=None,**kwargs)
a_list = soup.find_all("a")
#获取所有a标签的name属性
for item in a_list:
    print(item.get("name"))

#标签名包含a
import re
a_list = soup.find_all(re.compile(".*a.*"))#获取所有匹配的Tag .*a.*
print(a_list,len(a_list))

#查找指定tag
a_list = soup.find_all(["a","title"],string=["hao123","地图"])
print(a_list,len(a_list))

a_list = soup.find_all(["a","title"],string=re.compile("hao123\d"))
print(a_list,len(a_list))

a_list = soup.find_all("a",string="test")
print(a_list,len(a_list))

#通过属性来查找
a_list = soup.find_all(attrs={"href":"https://www.hao123.com"})
print(a_list,len(a_list))

a_list = soup.find_all(href="https://www.hao123.com")
print(a_list,len(a_list))

#注意attrs和可变长关键字参数
#符合标识符命名规范的属性 可直接作为关键字参数查找Tag
#不符合标识符命名规范的属性，可通过attrs参数查找
a_list= soup.find_all(["a"],attrs={"class":"","name":""})
print(a_list,len(a_list))

a_list = soup.find("div",attrs={"class":"foot"}).find_all("a")
print(a_list)

 把结果放入csv文件中