学习写爬虫

最近学习爬虫，简单记录下。

#!/usr/bin/env python
# -*- coding:utf-8 -*-

from bs4 import BeautifulSoup
from urllib import request
import requests
import os,time


web_url = 'http://www.haopic.me'
urls = []
img_urls = []
save_path = 'E:\\haopic\\'

def get_urls(url):
    web_data = requests.get(url)
    soup = BeautifulSoup(web_data.text,'lxml')
    ur = soup.select('ul.item_list div.inner_item_box  h2 > a')
    for url in ur:
        urls.append(url.get('href'))
    return urls


def get_img_urls(img_url):
    web_data = requests.get(img_url)
    soup = BeautifulSoup(web_data.text,'lxml')
    title = soup.title.text.strip()
    image_urls = soup.select('div.content-c img')
    if os.path.exists('E:\\haopic\\{}'.format(title)):
        print('{}目录已经存在'.format(title))
        return
    else:
        os.mkdir('E:\\haopic\\{}'.format(title))
        print(title)
    i = 0
    for img in image_urls:
        imgurl = img.get('src')
        try:
            pic = requests.get(imgurl, timeout=5)  # 超时异常判断 5秒超时
        except requests.exceptions.ConnectionError:
            print('当前图片无法下载')
            continue
        img_name = "image" + str(i) + ".jpg"  # 拼接图片名
        print(img_name)
        # 将图片存入本地
        with open('{}\\{}\\{}'.format(save_path,title,img_name), 'wb') as fp:
            fp.write(pic.content)  # 写入图片

        i += 1

if __name__ == '__main__':
    for img_url in get_urls(web_url):
        get_img_urls(img_url)
        time.sleep(5)

 

转载于:https://my.oschina.net/u/3183099/blog/1617336