Python之网络爬虫

#-*-coding:utf-8 -*-
import DataBaseHelper
import ThreadPool
import http.cookiejar
import urllib




#cookie登陆
cj = http.cookiejar.LWPCookieJar()
cookie_support = urllib.request.HTTPCookieProcessor(cj)
opener = urllib.request.build_opener(cookie_support, urllib.request.HTTPHandler)




db=DataBaseHelper.DbHelper(1,50000)
pool=ThreadPool.ThreadPool(20,500)
pool.start()


def crab(i):
    url="http://product.pconline.com.cn/notebook/c1113/"+str(i*25)+"s1.shtml"
    
    try:
        data=opener.open(url, timeout=30).read()
        print(data)
        db.add("insert into craw(information) values(%s)", [data.decode("UTF-8")])
    finally:
        opener.close()
        
try:
    for i in range(0,1):
        pool.add(crab, [i])
finally:
    print("runOutAndJoin")
    pool.runOutAndJoin()
    print("pool quit")
    pool.syncQuit()
    print("Database quit")
    db.quit()