爬虫—图片下载

图片下载

代码附上

import random

import requests
from lxml import etree


def getHtml(url):
    # proxies = {'http': '123.160.68.82:9999'}
    try:
        head = ['Mozilla/5.0', 'Chrome/78.0.3904.97', 'Safari/537.36']
        headers = {
            'user-agent': head[random.randint(0, 2)]
        }
        response = requests.get(url=url, headers=headers, timeout=5)  # , proxies=proxies)
        if response.status_code == 200:
            content = response.content
            return content
        else:
            return response.status_code
    except Exception as e:
        return e


def htmlToTree(html):
    tree = etree.HTML(html)
    return tree


def downloadImg(url, desc):
    response = getHtml(url)
    path = "source/img/{}.gif".format(desc)
    with open(path, 'wb') as f:
        f.write(response)


def parseHtml(html):
    tree = htmlToTree(html=html)
    data = tree.xpath('//div[@class="main"]//div[@class="Left_bar"]//div[@class="tab_tj"]/'
                      'div[@class="tab_box"]/div/ul[@class="clearfix"]/li/a')
    for item in data:
        a = etree.tostring(item, pretty_print=True, encoding='utf-8').decode('utf-8')
        a_tree = htmlToTree(a)
        img_url = a_tree.xpath("//a/img/@data-src")[0]
        img_desc = a_tree.xpath("//a/p/text()")[0]
        downloadImg(img_url, img_desc)
        print(img_url, img_desc)


if __name__ == '__main__':
    url = 'http://www.win4000.com/meinvtag12327.html'
    html = getHtml(url)
    parseHtml(html)

原本想按照类别爬取一下图片，这个网站有点乱，没看懂他的分类，就随便分析了一个url，源码其实都一样。