import re
import time
import pymysql
import requests
from bs4 import BeautifulSoup
# 获取热榜的HTML
def GetContends(url):
cookie = '这里填写你的cookie'
headers = {
'User-Agent': 'Mozilla/5.0',
'Cookie': cookie
}
html = requests.get(url, headers=headers).text
return html
def Create_HotTitles_Table():
db = pymysql.connect(host='localhost', user='root', password='admin', port=3306, database='zhihu_hot_topic')
cursor = db.cursor()
sql_create = 'create table if not exists Hot_titles(id int(100) auto_increment, question_id varchar(255), question_time varchar(100), question_rank varchar(100), question_topic varchar(255), question_url varchar(255), question_level varchar(255), primary key (id))'
cursor.execute(sql_create)
db.close()
def Insert_HotTitles_Data(question_id, question_time, question_rank, question_topic, question_url, question_level):
db = pymysql.connect(host='localhost', user='root', password='admin', port=3306, database='zhihu_hot_topic')
cursor = db.cursor()
sql_insert = "insert into hot_titles (question_id, question_time, question_rank, question_topic, question_url, question_level) values (%s,%s,%s,%s,%s,%s)"
val = (question_id, question_time, question_rank, question_topic, question_url, question_level)
try:
cursor.execute(sql_insert, val)
db.commit()
except:
db.rollback()
db.close()
if __name__ == '__main__':
url_hot = "https://www.zhihu.com/hot"
# 共50个热榜,每次都处理一条热榜
while True:
soup = BeautifulSoup(GetContends(url_hot), 'html.parser')
contend = list(soup.find_all('section', 'HotItem'))
Create_HotTitles_Table()
for i in range(50):
try:
if i <= 3:
question_rank = re.findall(r'HotItem-rank HotItem-hot">(.*?)</div>', str(contend[i]))[0]
else:
question_rank = re.findall(r'"HotItem-rank">(.*?)</div>', str(contend[i]))[0]
question_level = re.findall(r'</svg>(.*?) 万热度<span', str(contend[i]))[0] # 热榜热度
question_title = re.findall(r'target="_blank" title="(.*?)"><h2', str(contend[i]))[0] # 热榜标题
question_url = re.findall(r'href="(.+?)"', str(contend[i]))[0] # 热榜网址
question_id = question_url[31:] # 问题id
question_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
print(" ----------------------------------------------------------------------------------------------------------------------")
print(" 热榜时间戳:%s" % question_time)
print(" 热榜排名:%s" % question_rank)
print(" 热榜ID:%s" % question_id)
print(" 热榜主题:%s" % question_title)
print(" 热榜网址:%s" % question_url)
print(" 热榜热度(万):%s" % question_level)
print(" ----------------------------------------------------------------------------------------------------------------------")
Insert_HotTitles_Data(question_id, question_time, question_rank, question_title, question_url, question_level)
except:
continue
time.sleep(60)
Python + Selenium(知乎50条热榜以及热榜内答案信息存储)
最新推荐文章于 2021-12-25 12:01:54 发布