爬虫学习笔记（一）requests-bs4-re

准备

1. python IDLE
2. robots的使用：域名后追加robots.txt可查看
3. 安装requests库：管理员启动cmd命令pip install requests
4. 安装beautifulsoup4库：管理员启动cmd命令pip install beautifulsoup4

实例

1. 爬取单张图片

import requests
import os

url = "https://i0.hippopx.com/photos/320/918/427/sky-clouds-sunlight-dark-thumb.jpg"
root = "D://pics//"

def getPics(url,root):
    path = root + url.split('/')[-1]    #以'/'为分割符保留最后一段
    try:
        #若跟路径不存在，则创建
        if not os.path.exists(root):
            os.mkdir(root)
        #若文件路径不存在，则创建，否则提示已保存
        if not os.path.exists(path):
            r = requests.get(url)
            with open(path,'wb') as f:
                f.write(r.content)
                f.close()
                print("文件保存成功")
        else:
            print("文件已存在")
    except:
        print("爬取失败")

2. 爬取大学排名

import requests
import bs4
from bs4 import BeautifulSoup

#获取网页文本
def getHTMLText(url):   
    try:
        r = requests.get( url,timeout = 30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""

#提取排名信息
def fillUnivList(ulist,html):   
    soup = BeautifulSoup(html,"html.parser")
    for tr in soup.find('tbody').children:  #查找'tbody'的子孙节点
        if isinstance(tr,bs4.element.Tag): #判断'tr'是否为标签类型
            tds = tr('td')
            ulist.append([tds[0].string, tds[1].string, tds[3].string])

#显示前num个排名
def printUnivList(ulist,num):
    #格式输出
    print("{:^6}\t{:<10}\t{:^6}".format("排名","学校名称","总分"))
    for i in range(num):
        u = ulist[i]
        print("{:^6}\t{:<10}\t{:^6}".format(u[0],u[1],u[2]))

def main():
    uinfo = []
    url = "http://www.zuihaodaxue.com/Greater_China_Ranking2019_0.html"
    html = getHTMLText(url)
    fillUnivList(uinfo,html)
    printUnivList(uinfo,50)

main()

3. 爬取豆瓣电压Top250

import requests
import bs4
import random
from bs4 import BeautifulSoup

tplt =