pool=eventlet.GreenPool(20)

pool=eventlet.GreenPool(20)

默认是1000,可能是我在 ubuntu下unlimit -n没有调 ，或是python的httplib有问题，抓同一个站就抱错


    r = h.getresponse()
  File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
    response.begin()
  File "/usr/lib/python2.6/httplib.py", line 391, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
    raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
    func(*args, **kwargs)
  File "batch_download.py", line 17, in fetch
    html=urllib2.urlopen(url).read()
  File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib/python2.6/urllib2.py", line 391, in open
    response = self._open(req, data)
  File "/usr/lib/python2.6/urllib2.py", line 409, in _open
    '_open', req)
  File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
    r = h.getresponse()
  File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
    response.begin()
  File "/usr/lib/python2.6/httplib.py", line 391, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
    raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
    func(*args, **kwargs)
  File "batch_download.py", line 17, in fetch
    html=urllib2.urlopen(url).read()
  File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib/python2.6/urllib2.py", line 391, in open
    response = self._open(req, data)
  File "/usr/lib/python2.6/urllib2.py", line 409, in _open
    '_open', req)
  File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
    r = h.getresponse()
  File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
    response.begin()
  File "/usr/lib/python2.6/httplib.py", line 391, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
    raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
    func(*args, **kwargs)
  File "batch_download.py", line 17, in fetch
    html=urllib2.urlopen(url).read()
  File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib/python2.6/urllib2.py", line 391, in open
    response = self._open(req, data)
  File "/usr/lib/python2.6/urllib2.py", line 409, in _open
    '_open', req)
  File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
    r = h.getresponse()
  File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
    response.begin()
  File "/usr/lib/python2.6/httplib.py", line 391, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
    raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
    func(*args, **kwargs)
  File "batch_download.py", line 17, in fetch
    html=urllib2.urlopen(url).read()
  File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib/python2.6/urllib2.py", line 391, in open
    response = self._open(req, data)
  File "/usr/lib/python2.6/urllib2.py", line 409, in _open
    '_open', req)
  File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
    r = h.getresponse()
  File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
    response.begin()
  File "/usr/lib/python2.6/httplib.py", line 391, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
    raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
    func(*args, **kwargs)
  File "batch_download.py", line 17, in fetch
    html=urllib2.urlopen(url).read()
  File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib/python2.6/urllib2.py", line 391, in open
    response = self._open(req, data)
  File "/usr/lib/python2.6/urllib2.py", line 409, in _open
    '_open', req)
  File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
    r = h.getresponse()
  File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
    response.begin()
  File "/usr/lib/python2.6/httplib.py", line 391, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
    raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
    func(*args, **kwargs)
  File "batch_download.py", line 17, in fetch
    html=urllib2.urlopen(url).read()
  File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib/python2.6/urllib2.py", line 391, in open
    response = self._open(req, data)
  File "/usr/lib/python2.6/urllib2.py", line 409, in _open
    '_open', req)
  File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
    r = h.getresponse()
  File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
    response.begin()
  File "/usr/lib/python2.6/httplib.py", line 391, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
    raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
    func(*args, **kwargs)
  File "batch_download.py", line 17, in fetch
    html=urllib2.urlopen(url).read()
  File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib/python2.6/urllib2.py", line 391, in open
    response = self._open(req, data)
  File "/usr/lib/python2.6/urllib2.py", line 409, in _open
    '_open', req)
  File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
    r = h.getresponse()
  File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
    response.begin()
  File "/usr/lib/python2.6/httplib.py", line 391, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
    raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
    func(*args, **kwargs)
  File "batch_download.py", line 17, in fetch
    html=urllib2.urlopen(url).read()
  File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib/python2.6/urllib2.py", line 391, in open
    response = self._open(req, data)
  File "/usr/lib/python2.6/urllib2.py", line 409, in _open
    '_open', req)
  File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
    r = h.getresponse()
  File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
    response.begin()
  File "/usr/lib/python2.6/httplib.py", line 391, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
    raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
    func(*args, **kwargs)
  File "batch_download.py", line 17, in fetch
    html=urllib2.urlopen(url).read()
  File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib/python2.6/urllib2.py", line 391, in open
    response = self._open(req, data)
  File "/usr/lib/python2.6/urllib2.py", line 409, in _open
    '_open', req)
  File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
    r = h.getresponse()
  File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
    response.begin()
  File "/usr/lib/python2.6/httplib.py", line 391, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
    raise BadStatusLine(line)
BadStatusLine
^CTraceback (most recent call last):
  File "batch_download.py", line 29, in <module>
    pool.spawn_n(fetch, url)
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 104, in spawn_n
    self.sem.acquire()
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/semaphore.py", line 71, in acquire
    hubs.get_hub().switch()
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/hubs/hub.py", line 135, in switch
    return self.greenlet.switch()
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/hubs/hub.py", line 169, in run
    self.fire_timers(self.clock())
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/hubs/hub.py", line 285, in fire_timers
    timer()
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/hubs/timer.py", line 56, in __call__
    cb(*args, **kw)
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
    func(*args, **kwargs)
  File "batch_download.py", line 17, in fetch
    html=urllib2.urlopen(url).read()
  File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib/python2.6/urllib2.py", line 391, in open
    response = self._open(req, data)
  File "/usr/lib/python2.6/urllib2.py", line 409, in _open
    '_open', req)
  File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/usr/lib/python2.6/urllib2.py", line 1133, in do_open
    h.request(req.get_method(), req.get_selector(), req.data, headers)
  File "/usr/lib/python2.6/httplib.py", line 910, in request
^[[A    self._send_request(method, url, body, headers)
  File "/usr/lib/python2.6/httplib.py", line 947, in _send_request
    self.endheaders()
  File "/usr/lib/python2.6/httplib.py", line 904, in endheaders
    self._send_output()
  File "/usr/lib/python2.6/httplib.py", line 776, in _send_output
    self.send(msg)
  File "/usr/lib/python2.6/httplib.py", line 735, in send
    self.connect()
  File "/usr/lib/python2.6/httplib.py", line 716, in connect
    self.timeout)
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/green/socket.py", line 78, in create_connection
    for res in getaddrinfo(host, port, 0, SOCK_STREAM):
KeyboardInterrupt
mlzboy@mlzboy-mac:~/my/idea2/download/hqs$ python batch_download.py ^C
mlzboy@mlzboy-mac:~/my/idea2/download/hqs$ python batch_download.py
Traceback (most recent call last):
  File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
    func(*args, **kwargs)
  File "batch_download.py", line 17, in fetch
    html=urllib2.urlopen(url).read()
  File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib/python2.6/urllib2.py", line 391, in open
    response = self._open(req, data)
  File "/usr/lib/python2.6/urllib2.py", line 409, in _open
    '_open', req)
  File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/usr/lib/python2.6/urllib2.py", line 1136, in do_open
    raise URLError(err)
URLError: <urlopen error [Errno 110] ETIMEDOUT>

刚发现设pool的一个好处是一个坏了，不影响其它的，只是速度慢一点，对我而言，少一两个页面也没事，

posted on 2010-07-28 15:04  lexus 阅读( ...) 评论( ...) 编辑 收藏

转载于:https://www.cnblogs.com/lexus/archive/2010/07/28/1787041.html