python爬虫报错:
[E 181215 13:24:58 base_handler:203] Host xn--jsrz53d not found
Traceback (most recent call last):
File "D:\Python27\lib\site-packages\pyspider\libs\base_handler.py", line 196, in run_task
result = self._run_task(task, response)
File "D:\Python27\lib\site-packages\pyspider\libs\base_handler.py", line 175, in _run_task
response.raise_for_status()
File "D:\Python27\lib\site-packages\pyspider\libs\response.py", line 184, in raise_for_status
raise http_error
HTTPError: Host xn--jsrz53d not found
代码:
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-12-15 10:52:00
# Project: music163
from pyspider.libs.base_handler import *
class Handler(BaseHandler):
crawl_config = {
}
@every(minutes=24 * 60)
def on_start(self):
self.crawl('https://www.kugou.com/yy/html/singer.html',callback=self.index_page,validate_cert=False,fetch_type='js')
@config(age=10 * 24 * 60 * 60)
def index_page(self, response):
for each in response.doc('strong > a').items():
self.crawl(each.attr.href, callback=self.detail_page,validate_cert=False,fetch_type='js')
@config(priority=2)
def detail_page(self, response):
for each in response.doc('.text > i').items():
self.crawl("https:"+each.text(),callback=self.song_page,validate_cert=False,fetch_type='js')
@config(priority=2)
def song_page(self,response):
pass
解决方法:
重启一下pyspider,之前手误启动是用的命令:
pyspider
应该用:
pyspider all
解决
我也不知道两者有什么区别,可能是加了all就是启动全部组件吧