利用python中的正则爬虫

以FreeBuf的首页为例爬取链接和标题

# coding:utf-8
import requests
from bs4 import BeautifulSoup
import re

def get_data():
    url = "https://www.freebuf.com/"
    headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:94.0) Gecko/20100101 Firefox/94.0'
        }
    res = requests.get(url, headers=headers, timeout=5)
    print(res)
    # 自动分析网页内容的编码方式
    res.encoding = res.apparent_encoding
    #lxml解析网页
    soup = BeautifulSoup(res.text, 'lxml')
    # 正则匹配链接和标题
    findlink = re.compile(r'<a .* href="(/.*?)" .*>',re.S)
    findspan = re.compile(r'<span .*>(.*?)</span>')
    #遍历div，正则匹配
    for i in soup.find_all('div', class_='title-left'):
        i = str(i)
        link = findlink.findall(i)
        URL = url+link[0]
        span = findspan.findall(i)
        SPAN = span[0]
        print(SPAN, '\n', URL, '\n')


if __name__ == '__main__':
    get_data()