spiders中爬虫.py
# -*- coding: utf-8 -*-
import scrapy
import re
class RenrenSpider(scrapy.Spider):
name = '爬虫名'
allowed_domains = ['renren.com']
start_urls = ['http://www.renren.com/327550029/profile'] # 需要登陆后才能正常访问的url
# start_urls都是默认交给start_requests函数处理的,可以重写该方法添加请求头、cookie等。
def start_requests(self):
# 该cookie是登陆后获取的cookie字符串。
cookies = "anonymid=jcokuqturos8ql; depovince=GW; jebecookies=f90c9e96-78d7-4f74-b1c8-b6448492995b|||||; _r01_=1; JSESSIONID=abcx4tkKLbB1-hVwvcyew; ick_login=ff436c18-ec61-4d65-8c56-a7962af397f4; _de=BF09EE3A28DED52E6B65F6A4705D973F1383380866D39FF5; p=90dea4bfc79ef80402417810c0de60989; first_login_flag=1; ln_uact=mr_mao_hacker@163.com; ln_hurl=http://hdn.xnimg.cn/photos/hdn421/20171230/1635/main_JQzq_ae7b0000a8791986.jpg; t=24ee96e2e2301bf2c350d7102956540a9; societyguester=24ee96e2e2301bf2c350d7102956540a9; id=327550029; xnsid=e7f66e0b; loginfrom=syshome; ch_id=10016"
# 转换成字典
cookies = {i.split("=")[0]:i.split("=")[1] for i in cookies.split("; ")}
# headers = {"Cookie":cookies} # cookie放到headers中无效
yield scrapy.Request(
self.start_urls[0],
callback=self.parse,
cookies=cookies # 携带cookie进行请求
# headers = headers # cookie放到headers中无效
)
def parse(self, response):
......
# 后续的请求会自动携带之前的cookie (setting.py配置文件中不要禁用cookie)
yield scrapy.Request(
"http://www.renren.com/327550029/profile?v=info_timeline",
callback=self.parse_detial
)
def parse_detial(self,response):
.....
settings.py(配置文件,不要禁用cookie)
# 禁用cookie (默认开启cookie)
# COOKIES_ENABLED = False
COOKIES_DEBUG = True # 开启cookie的调试信息