python爬虫网页分页_python爬虫 下载一个网页内的图片解决分页以及图片懒加载的问题...

话不都说,先上代码:

import urllib.parse

import urllib.request

from lxml import etree

import time

import os

def handle_request(url, page):

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36',

}

# 由于第一页和后面页码规律不一致，所以要进行判断

if page == 1:

url = url.format('')

else:

url = url.format('_' + str(page))

# print(url)

request = urllib.request.Request(url=url,headers=headers)

return request

def download_image(image_src,i):

dirpath = 'gudian'

# 创建一个文件夹

print('正在下载第'+str(i)+'张图片')

if not os.path.exists(dirpath):

os.mkdir(dirpath)

# 创建文件名

filename = os.path.basename(str(i)+'.png')

# 图片路径

filepath = os.path.join(dirpath, filename)

print(filepath)

# 发送请求，保存图片

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36',

}

request = urllib.request.Request(url=image_src, headers=headers)

response = urllib.request.urlopen(request)

with open(filepath, 'wb') as fp:

fp.write(response.read())

def parse_content(content):

# 生成对象

tree = etree.HTML(content)

image_list = tree.xpath('//div[@id="f-content"]/div/div/a/img/@data-src') # //div[@id="container"]/div/div/a/img/@src

# 懒加载

print(image_list)

# print(len(image_list))

# 遍历列表，依次下载图片

i = 1

for image_src in image_list:

i += 1

download_image(image_src,i)

def main():

url = 'http://www.51yuansu.com/search/canyinkatong.html'

# http://sc.chinaz.com/tupian/gudianmeinvtupian_2.html

start_page = int(input("请输入开始页码："))

end_page = int(input("请输入结束页码："))

for page in range(start_page,end_page+1):

print("第%s页开始下载······" % page)

request = handle_request(url, page)

content = urllib.request.urlopen(request).read().decode()

# 解析内容

parse_content(content)

time.sleep(2)

print("第%s页下载完毕······" % page)

print("下载完成！")

if __name__ == "__main__":

main()

复制代码

运行结果如图:

我们来看一下网站的目录结构

然后查看网页源代码发现这个懒加载图片的链接地址是存放在data-src当中的 写对应代码 image_list = tree.xpath('//div[@id="f-content"]/div/div/a/img/@data-src') 一个简单的爬虫就实现了 各位想要爬取自己想爬取网页的图片的话 理清目录结构 根据情况修改一下就可以了