在使用selenium的过程中我们经常遇到对网页操作的报错,常常程序会停留在这一步,没有办法继续下去,这时候我们就需要对程序做一个异常处理。
添加异常处理
def get_url(driver,url,num_retries = 5):
try:
driver.get(url)
# 切换成frame
driver.switch_to_frame("g_iframe")
# 休眠3秒,等待加载完成!
time.sleep(3)
html = driver.page_source
driver.close()
return html
except Exception as e:
if num_retries > 0:
time.sleep(5)
print(url)
print('Requests fail, retry!')
return get_url(driver = driver,url = url,num_retries = num_retries-1)
else:
print("Error:%s"%e)
return
结果还是出现错误:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/urllib3/connectionpool.py", line 601, in urlopen
chunked=chunked)
File "/usr/local/lib/python3.6/dist-packages/urllib3/connectionpool.py", line 387, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/usr/local/lib/python3.6/dist-packages/urllib3/connectionpool.py", line 383, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.6/http/client.py", line 1331, in getresponse
response.begin()
File "/usr/lib/python3.6/http/client.py", line 297, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.6/http/client.py", line 258, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib/python3.6/socket.py", line 586, in readinto
return self._sock.recv_into(b)
ConnectionResetError: [Errno 104] Connection reset by peer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "get_hot_songs.py", line 73, in <module>
html = get_html_src(url)
File "get_hot_songs.py", line 50, in get_html_src
driver= webdriver.Chrome(chrome_options=chrome_options)
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/chrome/webdriver.py", line 81, in __init__
desired_capabilities=desired_capabilities)
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/webdriver.py", line 157, in __init__
self.start_session(capabilities, browser_profile)
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/webdriver.py", line 252, in start_session
response = self.execute(Command.NEW_SESSION, parameters)
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/webdriver.py", line 319, in execute
response = self.command_executor.execute(driver_command, params)
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/remote_connection.py", line 374, in execute
return self._request(command_info[0], url, body=data)
File "/usr/local/lib/python3.6/dist-packages/selenium/webdriver/remote/remote_connection.py", line 397, in _request
resp = self._conn.request(method, url, body=body, headers=headers)
File "/usr/local/lib/python3.6/dist-packages/urllib3/request.py", line 70, in request
**urlopen_kw)
File "/usr/local/lib/python3.6/dist-packages/urllib3/request.py", line 148, in request_encode_body
return self.urlopen(method, url, **extra_kw)
File "/usr/local/lib/python3.6/dist-packages/urllib3/poolmanager.py", line 321, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
File "/usr/local/lib/python3.6/dist-packages/urllib3/connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/local/lib/python3.6/dist-packages/urllib3/util/retry.py", line 357, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/local/lib/python3.6/dist-packages/urllib3/packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/usr/local/lib/python3.6/dist-packages/urllib3/connectionpool.py", line 601, in urlopen
chunked=chunked)
File "/usr/local/lib/python3.6/dist-packages/urllib3/connectionpool.py", line 387, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/usr/local/lib/python3.6/dist-packages/urllib3/connectionpool.py", line 383, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.6/http/client.py", line 1331, in getresponse
response.begin()
File "/usr/lib/python3.6/http/client.py", line 297, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.6/http/client.py", line 258, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib/python3.6/socket.py", line 586, in readinto
return self._sock.recv_into(b)
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))
而且输出中没有看到Requests fail, retry!
,说明程序并没有进入到except
。下面我对代码进行了修改,将初始化实例放到这个请求函数里面。
def get_url(chrome_options,url,num_retries = 5):
try:
# 初始化实例
driver= webdriver.Chrome(chrome_options=chrome_options)
driver.get(url)
# 切换成frame
driver.switch_to_frame("g_iframe")
# 休眠3秒,等待加载完成!
time.sleep(3)
html = driver.page_source
driver.close()
return html
except Exception as e:
if num_retries > 0:
time.sleep(5)
print(url)
print('Requests fail, retry!')
return get_url(chrome_options = chrome_options,url = url,num_retries = num_retries-1)
else:
print("Error:%s"%e)
return
后面程序运行正常,请求失败会按照我的设想进行重复请求!