python 爬虫百度广告过滤系统

import re
import requests
import string
import json
from lxml import etree
from bs4 import BeautifulSoup


# 定制请求头
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36'}

#请求头获取页面
def get_page(url,headers):
    try:
        r=requests.get(url, headers=headers)
        r.raise_for_status()
        r.encoding=r.apparent_encoding
        return r.text
    except Exception as e:
        print(e)


# text = get_page('https://www.baidu.com/s?wd=Scala',headers=headers)
# text = get_page('https://www.baidu.com/s?wd=Go',headers=headers)
text = get_page('https://www.baidu.com/s?wd=Sql',headers=headers)
text_html = etree.HTML(text,etree.HTMLParser())

p = text_html.xpath('//div[@class="result c-container new-pmd"]')

for i in range(5):
    p2 = p[i].xpath('string(.)')
    #string(.)可以用于提取标签嵌套标签的内容。
    print(p2,end='\n')
    print('\n\n')