爬虫学得好，牢饭少不了

爬虫笔记

一.基本流程
1 获取网页的URL地址
2 分辨请求网页的办法
3 查看网页的源代码
4 撰写爬虫请求头
5 解析页面内容
6 存储信息到本地

二. 正则表达式
match 从头匹配
search 全局匹配
group 获取内容
findall 匹配全部
re.S 匹配换行符
re.I 不区分大小写
实战
1 爬取知乎热榜

import requests   #导入requests 
from bs4 import BeautifulSoup  #导入bs4
import re         #导入re
headers = {
"user-agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
}         #请求头，防止报错
url = "https://www.zhihu.com/billboard"  
html = requests.get(url,headers = headers)
soup = BeautifulSoup(html.text,"lxml")
divs = soup.select(".HotList-item")
for div in divs:
    title = div.select(" .HotList-itemBody .HotList-itemMetrics")[0].text
    redu = div.select(".HotList-itemBody .HotList-itemTitle")[0].text
    print(f"标题是{redu},热度是{title}")
    #图片，防止报错
    try:

        img = div.select(".HotList-itemImgContainer  img")[0]["src"]
        print(img)

    except:
        pass
re_introd = re.compile('"excerptArea":{"text":"(.*?)"}') #正则从源代码获取简介
result = re_introd.findall(html.text)
for r in result:
    print(f"简介{r}")