python正则表达式 爬取丑事百科

import re
import requests

def parse_page(url):
headers ={
‘User-Agent’:‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36’
}

response = requests.get(url,headers)
text= response.text
# titles =['a','b']
# dynasties =['唐朝','清代']
# authors = ['伪装','纳兰']
# print(text)
titles = re.findall(r'<div\sclass="cont">.*?<b>(.*?)</b>',text,re.DOTALL)
dynasties = re.findall(r'<p\sclass="source">.*?<a.*?>(.*?)</a>', text, re.DOTALL)
authors = re.findall(r'<p\sclass="source">.*?<a.*?>.*?<a.*?>(.*?)</a>', text, re.DOTALL)
content_tags = re.findall(r'<div\sclass="contson" .*?>(.*?)</div>', text, re.DOTALL)
# print(authors)
contents =[]
for content in content_tags:
    x = re.sub(r'<.*?>',"",content)
    contents.append(x.strip())
    # print(x.strip())
poems=[]
for value in zip(titles,dynasties,authors,contents):
    title, dynastie, author, content = value
    poem={
        'title':title,
        'dynasty':dynastie,
        'author':author,
        'content':content
    }
    poems.append(poem)

for poem in poems:
    print(poem)
    print('='*40)

def main():
url=“https://www.gushiwen.org/default_2.aspx”
for x in range(1,11):
url =“https://www.gushiwen.org/default_%s.aspx” % x
parse_page(url)

if name == ‘main’:
main()