天堂图片全部图片爬取

#1.导包
import requests,os
from lxml import etree
from urllib.request import urlretrieve
from urllib.parse import urljoin
import zmail
import zipfile
headers = {
          'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}

#2.获取网页源码
url = 'http://www.ivsky.com/tupian/'
response = requests.get(url,headers = headers)
html_content = response.content
#3.解析大分类标题和地址
root = etree.HTML(html_content)
big_hrefs = root.xpath('//ul[@class="tpmenu"]/li/a/@href')#标签后面属性的值
big_titles = root.xpath('//ul[@class="tpmenu"]/li/a/text()')

for big_title,big_href in zip(big_titles,big_hrefs):
   os.makedirs(f'天堂图片/{big_title}',exist_ok=True)
   big_href = urljoin(url,big_href)

   response = requests.get(big_href,headers = headers)
   html_content = response.content
   root = etree.HTML(html_content)
   small_hrefs = root.xpath('//div[@class="sline"]/div/a/@href')
   small_titles = root.xpath('//div[@class="sline"]/div/a/text()')

   for small_href,small_title in zip(small_hrefs,small_titles):
      os.makedirs(f'天堂图片/{big_title}/{small_title}',exist_ok=True)
      small_href = urljoin(url,small_href)
      response = requests.get(small_href,headers = headers)
      html_content = response.content
      root = etree.HTML(html_content)
      img_srcs = root.xpath("//ul[@class = 'pli']/li/div/a/img/@src")

      for src in img_srcs:
         urlretrieve(src,f'天堂图片/{big_title}/{small_title}/'+src.split('/')[-1])
         # high_src = src.replace('/t/', '/pre/')
         # urlretrieve(high_src, f'天堂图片网/{big_title}/{small_title}/' + 'big_' + high_src.split('/')[-1])