目标:爬取中国古诗文网的诗词
注意:
- 用正则的思路不一样:不存在树结构,在正则眼里只当作‘str’
- 关注zip的用法
中国古诗文网网页示例:
1、xpath方法
# -*- coding: utf-8 -*-
# 一、数据解析方法1:xpath
import requests
from lxml import etree
url = 'https://www.gushiwen.cn/default_1.aspx'
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36'
}
response = requests.get(url=url, headers=headers, verify=False)
html = etree.HTML(response.text)
sons = html.xpath("/html/body/div[2]/div[1]/div[@class='sons']")
# etree.tostring(sons[0],encoding='utf8').decode('utf8') # 查看sons的第一项情况
# =============================================================================
poems = []
for index,son in enumerate(sons):
if index in [5,11]: # 位于6、12的非一首完整的诗
continue
title = son.xpath(".//p[1]//b/text()")[0] # 注意:前面不加'.',则会在整个HTML中搜索
dynasty = son.xpath(".//p[2]/a[1]/text()")[0]
author = son.xpath(".//p[2]/a[2]/text()")[0]
content = son.xpath("string(.//div[@class='contson'])").strip() # 注意string()与text()的区别,strip()的使用(CSDN收藏)
poem ={
'title': title,
'dynasty': dynasty,
'author': author,
'content': content
}
poems.append(poem)
# 将列表转换成DataFrame格式
import pandas as pd
result = pd.DataFrame(poems)
# =============================================================================
'''
1、string()与text()的区别
文中有换行或者标签时,text()不能完全提取;此时用:string可以把标签中的内容提取
2、strip()
作用:去除字符串中的空格或指定字符
默认去除两端空字符。注:此处的空包含'\n', '\r', '\t', ' '
'''
2、正则方法
# -*- coding: utf-8 -*-
# 二、数据解析方法2:正则
import requests
import re
url = 'https://www.gushiwen.cn/default_1.aspx'
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36'
}
response = requests.get(url=url, headers=headers, verify=False)
text = response.text
titles = re.findall(r'<div class="cont">.*?<b>(.*?)</b>', text, re.DOTALL)
dynasties = re.findall(r'<p class="source">.*?<a.*?>(.*?)</a>', text, re.DOTALL)
authors = re.findall(r'<p class="source">.*?<a.*?>.*?<a.*?>(.*?)</a>', text, re.DOTALL)
content_tags = re.findall(r'<div class="contson".*?>(.*?)</div>', text, re.DOTALL)
contents = []
for content in content_tags:
# pring(content)
x = re.sub(r'<.*?>',"",content)
contents.append(x.strip())
# 组合
poems = []
for value in zip(titles, dynasties, authors, contents):
title, dynasty, author, content = value
poem ={
'title': title,
'dynasty': dynasty,
'author': author,
'content': content
}
poems.append(poem)
# 将列表转换成DataFrame格式
import pandas as pd
result = pd.DataFrame(poems)