菜鸟爬虫-爬妹子图

转自：http://cuiqingcai.com/3179.html 大神

from bs4 import BeautifulSoup
import requests
import os
headers ={'User-Agent':"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}
all_url =('http://www.mzitu.com/all')
stare_html=requests.get(all_url ,headers=headers )
Soup = BeautifulSoup(stare_html.text ,'lxml')
a_all =Soup.find('div',class_='all').find_all('a')
for a in a_all:
    title =a.get_text()
    path =str(title).strip()
    os.makedirs(os.path.join("D:\meizitu",path))
    os.chdir("D:\meizitu\\"+path)
    href =a['href']
    html =requests.get(href,headers=headers )
    Soup_html =BeautifulSoup(html.text,'lxml')
    max_span =Soup_html.find('div',class_='pagenavi').find_all('span')[-2].get_text()
    for page in range(1,int(max_span)+1):
        page_url=href + '/'+str(page)
        img_html=requests.get(page_url,headers=headers )
        img_soup=BeautifulSoup(img_html.text ,'lxml')
        img_url=img_soup.find('div',class_='main-image').find('img')['src']
        name=img_url[-9:-4]
        img =requests.get(img_url,headers=headers)
        f =open(name+'.jpg','ab')
        f.write(img.content )
        f.close()