使用requests爬取网页的四种解析方式（正则、bs4、Xpath、parsel）

部分内容转载自：https://blog.csdn.net/qiushuidongshi/article/details/81252838

0x01 解析方式一——正则

0x02 解析方式二——bs4

0x03 解析方式三——Xpath

0x04 解析方式四——parsel

#0x00 requests爬取网页

import requests
r = requests.get('https://book.douban.com/')
content = r.text

#0x01 解析方式一——正则

import re
pattern=re.compile('<h4.*?>(.*?)</h4>.*?<p>.*?author">.*?(.*?)</span>.*?year">.*?(.*?)</span>.*?publisher">.*?(.*?)</span>.*?</p>',re.S)
 
results = re.findall(pattern, content)
print(results)
for result in results:
    name,author,time,chuban=result
    name=re.sub("\s",'',name).replace(' ','').strip()
    author=re.sub('\s','',author).strip()
    time=re.sub("\s",'',time).strip()
    chuban=re.sub("\s",'',chuban).strip()
    print(name,author,time,chuban)

#0x02 解析方式二——bs4

from bs4 import BeautifulSoup
 
html = r.content
soup = BeautifulSoup(html,"lxml")                              
print(type(soup))                                                                                                                                                                 
name = soup.findAll(name='h4',class_='title',text=re.compile(".*?"))                                                                                                  
author=soup.findAll(name='span',class_='author',text=re.compile(".*?"))                                                                                          
time=soup.findAll(name='span',class_="year",text=re.compile(".*?"))                                                                                                  
chuban=soup.findAll(name="span",class_="publisher",text=re.compile((".*?")))  

#0x03 解析方式三——Xpath

from lxml import etree
 
html=r.content
tree = etree.HTML(html)
name=tree.xpath("//h4/text()")
author=tree.xpath("//span[@class='author']/text()")
time=tree.xpath("//span[@class='year']/text()")
chuban=tree.xpath("//span[@class='publisher']/text()")
print(name,author,time,chuban)

#0x04 解析方式四——parsel

import requests
import parsel
 
 
response = requests.get(url)
sel = parsel.Selector(response.text)  #注意这里的S要大写
 
# re正则
# print(sel.re('正则匹配格式'))
 
# xpath
# print(sel.xpath('xpath').getall()) #getall获取所有
 
# css选择器
# print(sel.css('css选择器 ::text').extract_first())#获取第一个