import requests
import xlwt
from bs4 import BeautifulSoup
def getCid():
hd = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36 SLBrowser/7.0.0.4071 SLBChan/21"}
url = "https://top.baidu.com/board"
page = requests.get(url,headers=hd).content.decode("utf-8")
soup = BeautifulSoup(page, 'html.parser')
dIv = soup.find_all('a', attrs={'class':"tab-item_17MbK"})
code = []
type_L = []
for i in dIv:
src = i['href']
code.append(src.strip('.'))
type_L.append(i.text)
return code,type_L
def getItem(page):
hd = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36 SLBrowser/7.0.0.4071 SLBChan/21"}
url = "https://top.baidu.com%s"
url = url%page
page = requests.get(url,headers=hd).content.decode("utf-8")
soup = BeautifulSoup(page, 'html.parser')
code = []
tBody = soup.find('div',attrs={'style':"margin-bottom:20px"})
name_M = tBody.find_all('a',attrs={'class':"title_dIF3B"})
text_M = tBody.select("div[class^='hot-desc_1m_jR large_nSuFU']")
top_M = tBody.find_all('a', attrs={'class': "img-wrapper_29V76"})
score_M = tBody.find_all('div', attrs={'class': "hot-index_1Bl1a"})
for i in range(0,len(name_M)):
# code.append(dict(排名=top_M[i].div.text,标题=name_M[i].text,分数=score_M[i].text))
code.append([top_M[i].div.text,name_M[i].text,score_M[i].text])
return code
def saveExcel(result,type_L):
wb = xlwt.Workbook() # 创建工作表
for i in range(0,len(type_L)):
sheet = wb.add_sheet(type_L[i])
list_h = ['排名','标题','分数']
for row,str in enumerate(list_h):
sheet.write(0, row, str)
for m in range(0,len(result[i])):
for n,inV in enumerate(list_h):
sheet.write(m+1,n,result[i][m][inV])
wb.save('热搜全爬.xlsx')
from openpyxl import Workbook
def OaveExcel(data):
workbook = Workbook()
sheet = workbook.create_sheet()
for x in data:
sheet.append(x)
workbook.save('热搜全爬.xlsx')
page,type_L=getCid()
all_S = []
del page[0]
del type_L[0]
for i in range(0,len(page)):
all_S.append(getItem(page[i]))
saveExcel(all_S,type_L)
Python 百度热搜 全页面爬取
最新推荐文章于 2024-03-20 13:11:09 发布