xpath 与BeautifulSoup的比较 xpath的基本使用

 

#xpath简单例子

import requests

from lxml import etree

headers = {
  'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'
} 

url='https://tieba.baidu.com/f?kw=lol&ie=utf-8&pn={}'

for i in range(3):
    urls=url.format(i*50)
    response=requests.get(urls,headers=headers)

    result=response.content.decode('utf-8')#将爬取数据类型解析出来result为字符串

    result=result.replace('<!--','').replace('-->','')#<!-- -->为绝大数html的注释符

    html=etree.HTML(result)#此方法将result转换成python可解析的html对象    与BeautifulSoup解析对象类型不同

    texts = html.xpath('//li[@class=" j_thread_list clearfix thread_item_box"]//div[@class="threadlist_title pull_left j_th_tit "]//a[@rel="noreferrer"]/text()') 
    print(texts) 
    links = html.xpath('//li[@class=" j_thread_list clearfix thread_item_box"]//div[@class="threadlist_title pull_left j_th_tit "]//a[@rel="noreferrer"]/@href') 
    links 

#BeautifulSoup解析部分如下

    from bs4 import BeautifulSoup
    response = requests.get(urls, headers=headers) 
    rb=response.text.replace('<!--','').replace('-->','') 
    bs=BeautifulSoup(rb,'html.parser')#解析对象为字符串
    tb=bs.find('div',class_='threadlist_title pull_left j_th_tit')
    tb.text