之前用的一个多进程python爬虫

#!C:\Python27\python.exe
#coding=utf8

import os
import urllib
import urllib2
from bs4 import BeautifulSoup
from multiprocessing import Pool

import socket
socket.setdefaulttimeout(60)


def url_open(url):
    user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36'
    headers = {'User-Agent': user_agent}
    request = urllib2.Request(url=url, headers=headers)
    try:
        page = urllib2.urlopen(request, timeout=60)
    except urllib2.HTTPError as e:
        return 1
    contents = page.read()
    # print contents
    soup = BeautifulSoup(contents.decode('gb2312','ignore'), "lxml")
    return soup

def retrieve_img(dir, link):
    if url_open(link) != 1:
        soup = url_open(link)
        for line in soup.find_all("div", "ContentBox"):
            img_list = line.find_all('img')
            for img_item in img_list:
                raw_img = img_item.get('src')
                img_url = 'http:' + raw_img
                img_name = raw_img.split('/')[-1]
                print "Download: %s" % img_url
                # print img_name
                urllib.urlretrieve(img_url, dir + '\\' + img_name)

def crawler(root, url):
        # print url
        if url_open(url) != 1:
            soup = url_open(url)
            # print soup
            for line in soup.find_all("div", "box_con newslist"):
                li = line.find_all('li')
                for a in li:
                    href = a.find_all('a')
                    link = root + href[0].get('href')
                    text = href[0].get_text()
                    # print link
                    print "Creating directory %s..." % text
                    if not os.path.isdir(text):
                        try:
                            os.mkdir(text)
                            if os.path.isdir(text):
                                print "Directory %s successfully created!" % text
                                print "Crawling image page %s." % link
                                retrieve_img(text, link)
                        except IOError, e:
                            print e
                    elif os.path.isdir(text):
                        print "Directory %s already exists!" % text
                        print "Crawling image page %s." % link
                        retrieve_img(text, link)

                    # break #func test


def single_func(num):
    root = 'http://xxx.com'
    url = "http://xxx.com/articlelist/?20-" + str(num) + '.html'
    crawler(root, url)

if __name__ == '__main__':
    # single_func(1) #func test
    pool = Pool(processes=8)
    for i in range(1, 187):
        result = pool.apply_async(single_func, (i,))
    pool.close()
    pool.join()
    if result.successful():
        print 'Successful!'