网页信息爬取脚本实现

网页信息爬取脚本实现

requests模块处理网页基本逻辑
	定义一个url
	发送http请求
	处理http响应

获取整个网页所有源码

import requests

url = "http://10.9.75.133/python-spider/index.html"

#res = requests.get(url = url)
#print(res.status_code)#响应状态码
#print(res.url)
#print(res.encoding)#编码
#print(res.headers)#响应头部
#print(res.text)#响应正文


def get_html(url):
    res = requests.get(url = url)
    return res.content #输出二进制数据

html = get_html(url = url)#传参

print(html.decode("utf8"))

筛选出源码中图片

import requests
import re

url = "http://10.9.75.133/python-spider/index.html"

#res = requests.get(url = url)
#print(res.status_code)#响应状态码
#print(res.url)
#print(res.encoding)#编码
#print(res.headers)#响应头部
#print(res.text)#响应正文


def get_html(url):
    res = requests.get(url = url)
    return res.content #输出二进制数据

html = get_html(url = url)#传参

img_path_list = re.findall(r"style/\w*\.jpg",html.decode("utf8"))
for img_path in img_path_list:
    print(img_path)

#print(html.decode("utf8"))

将图片下载到本地

import requests

url = "http://10.9.75.133/python-spider/index.html"

img_path = "style/u16574389331092189023fm170sE3009908E2E32EA69DDCA09C0100C0E3w.jpg"

def download_img(img_path):

    img_url = url[0:url.rfind('/') +1] + img_path

    res = requests.get(url = img_url)

    img_save_path = "./img/1.img"

    with open(img_save_path, "wb") as f:
        f.write(res.content)

download_img(img_path)

##网页信息爬取

import requests
import re
import time

url = "http://10.9.75.133/python-spider/index.html"

#获取网页图片地址
def get_html(url):
    res = requests.get(url = url)
    return res.content #输出二进制数据

def get_img_path_list(html):
    img_path_list = re.findall(r"style/\w*\.jpg",html.decode("utf8"))
    return img_path_list

#下载图片方法
def download_img(img_path):

    img_url = url[0:url.rfind('/') +1] + img_path

    res = requests.get(url = img_url)

    img_save_path = f"./img/{time.time()}.img"

    with open(img_save_path, "wb") as f:
        f.write(res.content)


html = get_html(url = url)#传参

img_path_list = get_img_path_list(html = html)
for img_path in img_path_list:
    #print(img_path)
    download_img(img_path= img_path)