爬虫实战——利用正则表达式爬取古诗文网

import re
import requests

def parse_url(url):

    headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36"}
    response=requests.get(url,headers=headers)
    text=response.text
    end=[]
    #titles=re.findall(r'<div\sclass="ycd">.*?<h2>.*?<a.*?>(.*?)</a>',text,re.DOTALL)
    titles=re.findall(r'<div\sclass="cont">.*?<b>(.*?)</b>',text,re.DOTALL)
    dynesties=re.findall(r'<p\sclass="source">.*?<a.*?>(.*?)</a>',text,re.DOTALL)
    authors=re.findall(r'<p\sclass="source">.*?<a.*?><a.*?>(.*?)</a>',text,re.DOTALL)
    poems=re.findall(r'<div class="contson" .*?>(.*?)</div>',text,re.DOTALL)
    content=[]
    for poem in poems:
        x=re.sub(r'<.*?>'," ",poem)
        content.append(x.strip())
    ends=[]
    for value in zip(titles,dynesties,authors,content):
        titles, dynesties, authors, content=value
        poe={
            "title":titles,
            "dynasty":dynesties,
            "author":authors,
            "content":content
        }
        ends.append(poe)
    for end in ends:
        print(end)


    #print(authors)
def main():
    for i in range(10):
        url="https://www.gushiwen.org/default_%s.aspx" %i
    #url = "https://gushiwen.com/type/n/xianqin/n/1.html"
        parse_url(url)



main()