python-爬虫

爬取网页标题

 

import requests
from bs4 import BeautifulSoup

headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Connection": "keep-alive"
           }
url = "https://movie.douban.com/top250?start=101"
res = requests.get(url=url, headers=headers)   #res获得请求到的结果
res.encoding = 'UTF-8'

soup = BeautifulSoup(res.text,"html.parser")  #使用指定解析器解析获得res文本

head = soup.select("#content > div > div.article > ol > li:nth-child(1) > div > div.info > div.hd > a > span:nth-child(1)")
print(head)
for i in head:
    print(i.get_text())

select里面的参数，通过下面的方式获取：

获取网页标题的列表

import requests
from bs4 import BeautifulSoup

headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Connection": "keep-alive"
           }
url = "https://movie.douban.com/top250?start=101"
res = requests.get(url=url, headers=headers)   #res获得请求到的结果
res.encoding = 'UTF-8'

soup = BeautifulSoup(res.text,"html.parser")  #使用指定解析器解析获得res文本

head = soup.select("#content > div > div.article > ol > li > div > div.info > div.hd > a > span:nth-child(1)")

for i in head:
    print(i.get_text())

和上面的那行代码相比，这里将上一级目录的li:nth-child(1)来改为了li，也就是删除了冒号后面的特定选择的内容，这样获得内容就是所有的标题了。

爬取豆瓣Top250的电影

import requests
from bs4 import BeautifulSoup

headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Connection": "keep-alive",
            "cookie": "p_h5_u=F0E2CDDF-339E-4DB7-A832-3EB9CF0574C7; selectedStreamLevel=LD; Hm_lvt_be772be263fd7bf6cdd60433bd714862=1581386367,1581515417,1581674685,1581675681; token=9QCm_jwuffebIWTReplyU; Hm_lpvt_be772be263fd7bf6cdd60433bd714862=1581699602"
           }

for i in range(10):
    url = "https://movie.douban.com/top250?start={}".format(i*25)
    res = requests.get(url=url, headers=headers)   #res获得请求到的结果
    res.encoding = 'UTF-8'
    soup = BeautifulSoup(res.text,"html.parser")  #使用指定解析器解析获得res文本
    selecter ="#content > div > div.article > ol > li > div > div.info > div.hd > a > span:nth-child(1)"
    head = soup.select(selecter)

    for j in head:
        print(j.get_text())

读取伯禹平台的课程列表

import requests
from bs4 import BeautifulSoup
filename = "E:\Documents\Desktop\伯禹学习平台.html"
f = open(filename, "r", encoding='UTF-8')
data = f.read()
print(data)
f.close()


soup = BeautifulSoup(data)
selecter = "#root > div > div.page-course > div > div.sidebar > div > div.ui-card-body > div > ul > li > div > div.title-wrapper > div > a"

head = soup.select(selecter)

for j in head:
    print(j.get_text())

这个其实是采用的撒狗血的方式，先将网页来手动保存下来，然后在调用beautifulSoup进行解析。