原因:原因有很多,这里是爬虫没有代理IP,配置一下即可。
setting.py
DOWNLOADER_MIDDLEWARES = {
# 'myproject.middlewares.myprojectDownloaderMiddleware': 543,
'myproject.middlewares.myprojectProxy': 541
}
middlewares.py
import random
class MyprojectProxy(object):
def __init__(self):
self.proxy_list = ["127.0.0.1:126"] # 代理IP需改成自己的,此IP不能用
def process_request(self, request, spider):
proxy = random.choice(self.proxy_list)
request.meta["proxy"] = "http://" + proxy