python爬虫入门

一、顺序写法

import requests
import re
#数据爬取
url = "爬取页面"
header = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36"
}
response = requests.get(url,headers = header)
print(response.status_code)#输出状态码
html_str = response.text#网页正文

#数据解析
tre = re.compile(正则表达式)#正则规则
sre = re.findall(tre,html_str)#正则过滤后的正文
print(sre)

二、封装写法

import requests
import re
#数据爬取
def get_html(url):
    header = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36"
    }
    response = requests.get(url,headers=header)
    html_str = response.text
    return  html_str

#数据解析
def get_re(html_str):
    pre = re.compile('正则规则')
    rre = re.findall(pre,html_str)
    print(rre)

if __name__ == '__main__':
    for i in range(10):
        url = ”地址“
        a = get_html(url)
        get_re(a)