使用xpath,Beautifu Soup,re匹配爬取数据

使用xpath,Beautiful Soup,re匹配爬取数据

1、使用xpath匹配爬取数据

import requests
from lxml import etree

headers = {
'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)'
}
# 爬取一级页面，获取到所有的图片地址
# url = 'https://www.baidu.com'
url = 'http://www.baidu.com'
response = requests.get(url=url,headers=headers)
html_str = response.text
#建立一个供xpath匹配的对象
xpath_obj = etree.HTML(html_str)
#写xpath匹配的规则
text = xpath_obj.xpath('//a[@target="_blank"]/text()')

print(text)

2、使用Beautiful Soup匹配爬取数据

import requests
from bs4 import BeautifulSoup

headers = {
'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)'
}
# 爬取一级页面，获取到所有的图片地址
# url = 'https://www.baidu.com'
url = 'http://www.baidu.com'
response = requests.get(url=url,headers=headers)
html_str = response.text
#建立一个供BeautifulSoup匹配的对象
xpath_obj = BeautifulSoup(html_str)
#写xpath匹配的规则
text = xpath_obj.select('a[target="_blank"]')[0].get_text()

print(text)

3、使用re匹配爬取数据

import requests
import re

headers = {
'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)'
}
# 爬取一级页面，获取到所有的图片地址
url = 'http://www.baidu.com'
response = requests.get(url=url,headers=headers)
html_str = response.text

#写re匹配的规则
ret = '<a href=".*?" target="_blank" class="mnav">(.*?)</a>'

ret_2 = re.findall(pattern=ret,string=html_str)
print(ret_2)