import re
import json
import requests
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'
}
a = []
def get_html(link):
print("\n===========正在获取数据的目标地址: ",link)
try:
resp = requests.get(link,headers)
if resp.status_code == 200:
return resp.text
except requests.RequestException as b:
print(b)
def parser_html(resp):
patt = re.compile(' <h2><a target="_blank" href="(.*)">(.*\n?)</a></h2>') # 加了 ? 为非贪婪
mess_list = patt.findall(resp)
global a # 调用 全局变量(空列表)
for mess in mess_list:
mess_dict = {}
mess_dict['title'] = mess[1]
mess_dict['link'] = mess[0]
a.append(mess_dict) # 列表里的字典
conten = json.dumps(a,ensure_ascii=False) # 数据转换 此时类型是 str 也是需要保存 json的内容
print("\n==========正在保存数据============")
with open(r'C:\Users\DELL\Desktop\python_wd\文本信息\\'+ '果壳问答'+ '.json','w',encoding='utf-8')as f:
f.write(conten)
print("\n\t\t数据保存完毕。。。。。。")
def main():
for page in range(1,101):
link = 'https://www.guokr.com/ask/highlight/?page={}'.format(page)
resp = get_html(link)
parser_html(resp)
if __name__ == '__main__':
main()
果壳问答——函数
最新推荐文章于 2020-05-08 14:09:48 发布