爬某博客

import re
import requests
from bs4 import BeautifulSoup
import time


requests.packages.urllib3.disable_warnings()
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 \
                                    (KHTML, like Gecko) Chrome/51.0.    2704.103 Safari/537.36",
                                    }


url="https://3gstudent.github.io/"
i=0
s=requests.get(url=url,headers=headers).text
url2=[]
text=re.findall(r'<h1><a href="(.*?)">',s)
for i in range(len(text)):
    #Sprint(text[i])
    count = 0

    z = []
    try:
        url2.append("url"+text[i])

        index=requests.get(headers=headers,url=url2[i]).text
        html = BeautifulSoup(index, 'html.parser')

        title = re.findall(r"<title>(.*?)</title>", str(html.title))

        title=title[0]+".md"

        s=open("text.txt",'w')
        xz=open(title,'w')
        s.write(str(html.body))
        s.close()
        print(111)
        s = open("text.txt", 'r')
        for line in s.readlines():
            r = re.findall(r"<img alt=\"Alt text\" src=\"(.*?)\"/>", line)

            if (len(r) > 0):

                ca=r[0]+".png"
                inurl2 = requests.get(headers=headers, url=r[0]).content
                print(111)
                t = time.time()
                zxc = "./images/" + str(int(t)) + ".png"
                cx = open("./images/" + str(int(t))+ ".png", 'wb+')
                cx.write(inurl2)
                line = line.replace(r[0], zxc)
                print(line)
                time.sleep(1)
            z.append(line)



    
        s.close()
        z[1:13] = []
        z[-17:-3] = []

        for i in range(len(z)):
            xz.write(z[i])
        xz.close()



    except:
        print(i)