import re
import requests
import string
import json
from lxml import etree
from bs4 import BeautifulSoup
# 定制请求头
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36'}
#请求头获取页面
def get_page(url,headers):
try:
r=requests.get(url, headers=headers)
r.raise_for_status()
r.encoding=r.apparent_encoding
return r.text
except Exception as e:
print(e)
# text = get_page('https://www.baidu.com/s?wd=Scala',headers=headers)
# text = get_page('https://www.baidu.com/s?wd=Go',headers=headers)
text = get_page('https://www.baidu.com/s?wd=Sql',headers=headers)
text_html = etree.HTML(text,etree.HTMLParser())
p = text_html.xpath('//div[@class="result c-container new-pmd"]')
for i in range(5):
p2 = p[i].xpath('string(.)')
#string(.)可以用于提取标签嵌套标签的内容。
print(p2,end='\n')
print('\n\n')
python 爬虫百度广告过滤系统
最新推荐文章于 2023-06-20 06:40:51 发布