python多线程队列爬虫流程图_Python+多线程+队列爬虫

Python+多线程+队列，爬虫例子

# -*- coding: utf-8-*-

import urllib2

import urllib

import json

import time

import datetime

import threading

import Queue

import sys

reload(sys)

sys.setdefaultencoding( "utf-8" )

def get_response(url):

for a in range(3):

try:

request = urllib2.Request(url)

response = urllib2.urlopen(request)

result= response.read()

return result

except Exception,e:

print e

time.sleep(2)

continue

class ThreadCity(threading.Thread):

def __init__(self,queue_zq_citys):

threading.Thread.__init__(self)

self.queue_zq_citys=queue_zq_citys

def run(self):

sql = 'select cityid,cname from table '

citylist=DBHelper.SqlHelper.ms.ExecQuery(sql)

for c in citylist:

try:

#根据搜索城市名称获取城市

qm_cname=c[1]

data=urllib.urlencode({'q':qm_cname})

url='xxx.xxx.com/ajax.do?' % data

result=get_response(url)

cjson=json.loads(result.decode('gb2312', 'ignore')) #json格式字符串转换为python对象

cityId=cjson["id"]

cityname=cjson["cName"]

#加入队列

self.queue_zq_citys.put({'cityid':cityId,'cityname':cityname})

time.sleep(1)

except Exception,e:

pass

class ThreadCityDB(threading.Thread):

def __init__(self, queue_zq_citys):

threading.Thread.__init__(self)

self.queue_zq_citys = queue_zq_citys

def run(self):

while True:

try:

if self.queue_zq_citys.empty(): #队列为空

pass

else:

citys=self.queue_zq_citys.get() #从队列中取出数据

if citys is not None:

sql = "insert into Table(cityid,cityname) values(%s,'%s')" % (

citys['cityid'], citys['cityname'])

#print sql

DBHelper.SqlHelper.ms.ExecNonQuery(sql.encode('utf-8'))

self.queue_zq_citys.task_done() #告诉线程我完成了这个任务 是否继续join阻塞 让线程向前执行或者退出

else:

pass

except Exception,e:

pass

def main():

try:

queue_zq_citys=Queue.Queue() # 实例化存放抓取到的城市队列

#创建线程

city=ThreadCity(queue_zq_citys) #抓取线程 入队操作

cityDB=ThreadCityDB(queue_zq_citys) #出队操作 存入数据库

#启动线程

city.start()

cityDB.start()

#阻塞等待子线程执行完毕后再执行主线程

city.join()

cityDB.join()

except Exception,e:

pass

if __name__ == '__main__':

main()