import requests
import re
#开始要访问的,获取全部内容
def parse_page(url):
headers={"user-agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6776.400 QQBrowser/10.3.2577.400"}
response =requests.get(url,headers)
response.encoding='utf-8'
text=response.text
#爬取标题['a','b']
titles=re.findall(r'<div\sclass="cont">.*?<b>(.*?)</b>',text,re.DOTALL)
#爬取朝代\s=空格
chaodais =re.findall(r'<p\sclass="source"><a.*?>(.*?)</a>',text)
#作者
authors =re.findall(r'<p\sclass="source">.*?<a.*>.*?<a.*?>(.*?)</a>',text)
#内容
neirs =re.findall(r'<div\sclass="contson".*?>(.*?)</div>',text,re.DOTALL)
for nei in neirs:
#sub(内容,替换) strip()空格去掉
x=re.sub(r'<.*?>',"",nei)
# print(x.strip())
list_pomes=[]
#循环1:1合并字符串,{}
for value in zip(titles,chaodais,authors,neirs):
#a=[1,2,3] 和1,2,3=a
title,chaodai,author,neir =value
poem ={
'title':titles,
'chaodai':chaodais,
'author':authors,
'neir':neirs,
}
list_pomes.append(poem)
for poem in list_pomes:
print(poem)
print('---'*40)
#网址
def main():
url ='https://www.gushiwen.org/shiwen/'
for x in range(1,10):
url='https://www.gushiwen.org/shiwen/' %x
#获取要访问
parse_page(url)
if __name__ == '__main__':
main()