以FreeBuf的首页为例爬取链接和标题
import requests
from lxml import etree
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:94.0) Gecko/20100101 Firefox/94.0'
}
def getTree(url):
res = requests.get(url,headers,timeout = 5)
# 自动分析网页内容的编码方式
res.encoding = res.apparent_encoding
res_text = res.text
# 解析网页
tree = etree.HTML(res_text)
return tree
url = "https://www.freebuf.com/"
xml = getTree(url)
#获取节点href内容,//表示任意位置定位,/表示从根结点定位,@后为属性名
url_links = xml.xpath('//div[@class="article-item"]//div[@class="title-view"]//div[@class="title-left"]//a[1]/@href')
#text()方法获取节点span所有内容
span = xml.xpath('//div[@class="article-item"]//div[@class="title-view"]//div[@class="title-left"]//a[1]/span/text()')
print(span)
print(url_links)
url_link = []
for i in url_links:
link = url+i
url_link.append(link)
print(url_link)