本文探讨了Python爬虫在抓取百度数据时遇到的多线程假死问题。通过分析代码，可以看到作者使用了多线程和代理IP来尝试解决线程卡死的问题，但仍然遇到程序无响应且不报错的情况。代码中包含了获取百度地域节点、构造搜索URL、使用代理IP、设置User-Agent等方法，以及一个Fetcher类用于管理线程。虽然代码试图通过异常处理和代理IP更换来避免线程卡死，但可能需要进一步优化线程管理和错误处理策略以提高爬虫的稳定性。
抓baidu数据，但跑不了多久就假死，无反应也不报错，初学python，搞了两礼拜没搞明白，望各位前辈指点下
#coding:utf-8
'''
百度排名查询，代理多线程版本
'''
import StringIO,pycurl,time,random,re,os,csv
from threading import Thread,Lock
from Queue import Queue
from bs4 import BeautifulSoup as bs
csvfile = open('serp_html.csv','wb') #存放关键词和搜索结果页源码的文件
bdjd_dict = {}
#bdjd_list = ["www.baidu.com","180.97.33.107","115.239.210.27","180.97.33.108","180.97.33.107","180.97.33.107","180.97.33.108","220.181.111.188","220.181.111.188","180.97.33.107","180.97.33.107","115.239.211.112","180.97.33.108","180.97.33.108","180.97.33.108","180.97.33.108","180.97.33.108","115.239.211.112","180.97.33.108","115.239.211.112","115.239.210.27","180.97.33.108","115.239.211.112","115.239.210.27","180.97.33.108","115.239.210.27","61.135.169.125","115.239.211.112","115.239.210.27","180.97.33.107","180.97.33.107","180.97.33.108","115.239.210.27","180.97.33.107","61.135.169.121","115.239.210.27","61.135.169.121","61.135.169.125","115.239.211.112","115.239.210.27","61.135.169.125","112.80.248.73","61.135.169.121","112.80.248.74","112.80.248.73","61.135.169.125","180.97.33.108","115.239.210.27","61.135.169.125","61.135.169.125","112.80.248.74","112.80.248.74","61.135.169.121","115.239.210.27","61.135.169.125","111.13.100.92","111.13.100.92","111.13.100.91","111.13.100.91","115.239.211.112","111.13.100.92","111.13.100.91","111.13.100.92","115.239.211.112","115.239.210.27","115.239.211.112","115.239.210.27","115.239.210.27","115.239.210.27","115.239.210.27"]
bdjd_list = ["www.baidu.com"]
#提取百度地域节点
def getBDJD(bdjd_str):
bdjd_list = bdjd_str.split(',')
bdjd = random.choice(bdjd_list)
return bdjd
def baidu_url(word): #百度搜索url
return 'http://www.baidu.com/s?wd=%s' % (bdjd,word)
daili_list = [] #存储代理ip
#读取代理文件，随机提取1个代理
def ip():
for x in open('hege_daili.txt'):
x = x.strip()
daili_list.append(x)
newip = random.choice(daili_list)
return newip
#如果代理不可用，则从代理文件中删除，此函数在baidu_cout中应用
def daili_delete(ip):
dailifile = open('daili_beifen.txt','w')
for line in open('hege_daili.txt'):
line = line.strip()
if ip not in line:
dailifile.write(line+"\n")
os.system("mv daili_beifen.txt hege_daili.txt")
def baidu_url(word): #百度搜索url
return 'http://www.baidu.com/s?wd=%s' % word
def getUA():
uaList = [
'Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1;+.NET+CLR+1.1.4322;+TencentTraveler)',
'Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1;+.NET+CLR+2.0.50727;+.NET+CLR+3.0.4506.2152;+.NET+CLR+3.5.30729)',
'Mozilla/5.0+(Windows+NT+5.1)+AppleWebKit/537.1+(KHTML,+like+Gecko)+Chrome/21.0.1180.89+Safari/537.1',
'Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1)',
'Mozilla/5.0+(Windows+NT+6.1;+rv:11.0)+Gecko/20100101+Firefox/11.0',
'Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0;+SV1)',
'Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0;+GTB7.1;+.NET+CLR+2.0.50727)',
'Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0;+KB974489)',
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36'
]
ua = random.choice(uaList)
return ua
def baidu_cont(url,headers,ip): #百度搜索结果页源码
while 1:
try:
c = pycurl.Curl()
c.setopt(pycurl.MAXREDIRS,5)
c.setopt(pycurl.REFERER, url)
c.setopt(pycurl.FOLLOWLOCATION, True)
c.setopt(pycurl.CONNECTTIMEOUT, 60)
c.setopt(pycurl.TIMEOUT,120)
c.setopt(pycurl.ENCODING,'gzip,deflate')
c.setopt(c.PROXY,ip)
c.fp = StringIO.StringIO()
c.setopt(pycurl.URL, url)
c.setopt(pycurl.HTTPHEADER,headers)
c.setopt(c.WRITEFUNCTION, c.fp.write)
c.perform()
#code = c.getinfo(c.HTTP_CODE) 返回状态码
html = c.fp.getvalue()
if '="http://verify.baidu.com' in html:
time.sleep(1200)
print '重启'
continue
return html
except Exception, what:
information = '错误信息：%s' % what
return str(information)
continue
def search(req,line):
text = re.search(req,line)
if text:
data = text.group(1)
else:
data = 'no'
return data
url_list = []
for word in open('word'):
word = word.strip()
url_list.append(word)
class Fetcher:
def __init__(self,threads):
self.lock = Lock() #线程锁
self.q_req = Queue() #任务队列
self.q_ans = Queue() #完成队列
self.threads = threads
for i in range(threads):
t = Thread(target=self.threadget) #括号中的是每次线程要执行的任务
t.setDaemon(True) #设置子线程是否随主线程一起结束，必须在start()
#之前调用。默认为False
t.start() #启动线程
self.running = 0 #设置运行中的线程个数
def __del__(self): #解构时需等待两个队列完成
time.sleep(0.5)
self.q_req.join() #Queue等待队列为空后再执行其他操作
self.q_ans.join()
#返回还在运行线程的个数，为0时表示全部运行完毕
def taskleft(self):
return self.q_req.qsize()+self.q_ans.qsize()+self.running
def push(self,req):
self.q_req.put(req)
def pop(self):
return self.q_ans.get()
#线程执行的任务，根据req来区分
def threadget(self):
while True:
line = self.q_req.get()
word = line.strip()
'''
Lock.lock()操作，使用with可以不用显示调用acquire和release，
这里锁住线程，使得self.running加1表示运行中的线程加1，
如此做防止其他线程修改该值，造成混乱。
with下的语句结束后自动解锁。
'''
with self.lock:
self.running += 1
bdjd_str = ','.join(bdjd_list)
newip = ip()
bdjd = getBDJD(bdjd_str)
url = baidu_url(word)
headers = [
"Accept:*/*",
"Accept-Encoding:gzip, deflate, sdch",
"Accept-Language:zh-CN,zh;q=0.8,en;q=0.6",
"Connection:keep-alive",
#"Cookie:BIDUPSID=4DEE9B78AA3A97A51CFC916C43C30EC6; BAIDUID=380B58B009DF49761A4D9C30E19B34D0:FG=1; B64_BOT=1; PSTM=1432281826; BDRCVFR[G4oNZs7I7B3]=duGuzxHDPeTmy-lpA78QhPEUf; BDUSS=X5PYndScmJYc3lPSm1yQy15Vlg2YW45cWl3ZVFEalFjckFSNk16NHFIZTZkWVpWQVFBQUFBJCQAAAAAAAAAAAEAAAAJkstJv7TXvMTj1NnM-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALroXlW66F5Vc; SFSSID=ap4nu9shvsoc52f5nvnq6nev42; uc_login_unique=95684e36f6d0a03b984187546ec66f06; SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a01823607022; BDRCVFR[ltbVPlNi2ac]=mk3SLVN4HKm; BD_HOME=1; BD_UPN=32; sug=3; sugstore=1; ORIGIN=0; bdime=0; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; H_PS_645EC=5cdfPyG9PHeV%2BYlX03NQsmg4%2BZ0wlUr4XqxnORGloYCIlW8tFgZVbm9AliNr1w2Ls75g; BD_CK_SAM=1; BDSVRTM=106; H_PS_PSSID=13783_1454_13519_13075_12824_12867_14167_10562_12722_14155_14172_13202_14329_11951_13936_13741_14369_14182_8498_14195; WWW_ST=1432372813391",
"Host:www.baidu.com",
"RA-Sid:7739A016-20140918-030243-3adabf-48f828",
"RA-Ver:2.10.3",
"Referer:http://www.baidu.com/",
"X-Requested-With:XMLHttpRequest",
"User-Agent:%s" % getUA()
]
html = baidu_cont(url, headers, newip)
soup = bs(html)
b_tags = soup.find_all('div', {'class': 'result c-container '})
for line in b_tags:
newline = str(line)
number = search(r'id="(\d+)"',newline)
urldiv = search(r'(.*?)',newline) #获取源码中domain所在的
data = []
data.append(word)
data.append(newline)
writer = csv.writer(csvfile,dialect='excel')
writer.writerow(data)
if len(b_tags) == 0:
print html
else:
print '》》当前IP：%s,已抓取：%s，返回%s条结果' % (newip,word,len(b_tags))
#self.q_ans.put((req,ans)) # 将完成的任务压入完成队列，在主程序中返回
self.q_ans.put(word)
with self.lock:
self.running -= 1
self.q_req.task_done() # 在完成一项工作之后，Queue.task_done()
# 函数向任务已经完成的队列发送一个信号
time.sleep(0.1) # don't spam
if __name__ == "__main__":
#links = [ 'http://www.verycd.com/topics/%d/'%i for i in range(5420,5450) ]
f = Fetcher(threads=10) #设置线程数为10
for url in url_list:
f.push(url) #所有url推入下载队列
while f.taskleft(): #若还有未完成的的线程
f.pop() #从下载完成的队列中取出结果
'''
# 如果百度节点超时次数》10，则从百度节点列表中删除
# if '错误信息' in html:
# print html
# if 'Connection refused' in html:
# #判断访问超时的节点存入字典，若该节点已超过10次链接超时，则从节点列表中删除
# if bdjd_dict.has_key(bdjd):
# bdjd_dict[bdjd] += 1
# print '节点：%s，已%s次超时' % (bdjd,bdjd_dict[bdjd])
# if int(bdjd_dict[bdjd]) >= 10:
# bdjd_list.remove(bdjd)
# print "节点：%s 已删除" % bdjd
# else:
# bdjd_dict[bdjd] = 1
# continue
'''
扫一扫
1万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
