python简单爬取图片的一点总结

折腾了好几天，终于开发了一个能够爬取mzitu的单进程程序，虽然只有短短的几十行code，

但由于没怎么搞过爬虫，有很多的坑都是费了很大劲儿才爬出来，不过不断的查询、实验等学到的东西还真挺受用的：

学习了：

1、requests，urllib2，BeautifulSourp，selenium+webdriver(mzitu没涉及到，但还是学了下）

2、每级URL的变化分析和提取

3、路径和字符串的处理

4、防盗链  'Referer'

终于能够完整爬取主页上第一页的24个连接的所有图片，在此记录下！

       

#coding=utf-8

import requests
from bs4 import BeautifulSoup
import urllib2
import re
import time
import os

'''
运行平台：Mac OS python 2.7
'''

url = 'http://www.mzitu.com'
localDir = os.path.expanduser('~/Desktop/mzitu')
header = {
            'Host':'www.mzitu.com',
            'Accept-Language': 'en-us',
            'Connection': 'close',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15',
            "Cookie":"Hm_lpvt_dbc355aef238b6c32b43eacbbf161c3c=1586961981; Hm_lvt_dbc355aef238b6c32b43eacbbf161c3c=1586939099"
            }

def get_url_content(aURL):
    request = urllib2.Request(aURL,None,header)
    html = urllib2.urlopen(request, timeout=10)
    data = html.read()
    return data


def get_all_link_from_main_url():
    data = get_url_content(url)
    pan = r'<li><a href="(.*)" target="_blank.* target="_blank.*'
    https = re.compile(pan).findall(data)
    return https


def get_max_page(pin):
    data = get_url_content(pin)
    bs = BeautifulSoup(data, 'lxml')
    mp = bs.find_all('div', class_='pagenavi')
    aList = mp[0].find_all('span')[-2].string
    return aList

def imageHref(pageURL):
    data = get_url_content(pageURL)
    bs = BeautifulSoup(data, 'lxml')
    mp = bs.find_all('div', class_='main-image')
    src = mp[0].img['src']
    return src

def downloadImage(pageURL,imgURL):
    subdir = pageURL.split('/')[-2]
    mkDir = localDir+'/'+subdir
    if not os.path.exists(mkDir):
        os.mkdir(mkDir)
    filename = imgURL.split('/')[-1]
    localURL = mkDir+'/'+filename
    with open(localURL,'wb') as f:
        headersURL = {
            'Referer': 'https://www.mzitu.com',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36',
        }
        rst = urllib2.Request(imgURL,None,headersURL)
        rsp = urllib2.urlopen(rst)
        f.write(rsp.read())
        f.close()


def download_one_set(pOne):
    print pOne
    max_page = int(get_max_page(pOne))
    for m in range(max_page):
        each_url = pOne+'/'+str(m+1)
        print each_url
        img_url = imageHref(each_url)
        print img_url
        downloadImage(each_url,img_url)
        print '\t--',img_url,' -- ok'
        time.sleep(1)


if __name__ == '__main__':
    p_list = get_all_link_from_main_url()
    for page in p_list:
        download_one_set(page)
    print '下载 ok'