1. scrapy 接受命令行参数
def __init__(self,*args, **kwargs): super(SinopecsalesSpider, self).__init__(*args, **kwargs)
启动spider 时 传 -a testkey=testval 通过 kwargs['testkey']获取
2. cookie传递
def start_requests(self): self.logger.info('Parse function called on start_requests') return [Request(url=url meta={'cookiejar': 1}, callback=self.post_login)]
在回调函数里面通过 response.meta['cookiejar'] 里面获取cookie,请求的时候就会发送cookie
return [FormRequest(url=self.login_submit, meta={'cookiejar': response.meta['cookiejar'] }, headers=reqheaders, formdata=data, callback=self.after_login, errback=self.errback_httpbin, dont_filter = True )]
3. xpath 和 css 选择器
提取html里面的数据时尽量用xpath,css , 也可以结合正则使用
tag=sel.xpath('//a/text()').extract() total=sel.xpath('//b/text()').extract()
response.xpath('*').re(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}')
4.设置请求头
DEFAULT_REQUEST_HEADERS = {"Accept": "application/json, text/javascript, */*; q=0.01", "Accept-Encoding": "gzip,deflate", "Accept-Language": "en-US,en;q=0.8,zh-TW;q=0.6,zh;q=0.4", "Connection": "keep-alive", 'Cache-Control': 'no-cache', 'X-Requested-With': 'XMLHttpRequest', 'Origin': 'http://xx.com',"Content-Type": " application/x-www-form-urlencoded; charset=UTF-8", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Referer":"http://xx.com/"}
5. unicode字符处理
比较相等加
item['traName'] == u'相等字符'
douban_item['info'] = str[0].strip('()')
6.gbk to utf-8
response.body.decode('gbk').encode('utf-8')
7. 随机数和sleep
time.sleep(5)
random.random()
8.破解验证码
用pytesser,可能需要二值化
def process_image(self, response): image = Image.open(StringIO(response.body)) # image.show() image.filter(ImageFilter.SHARPEN) box = (2, 2, 88, 23) # 设定裁剪区域 region = image.crop(box) # 裁剪图片,并获取句柄region return self.getverify(region) # region.save( "crop.jpg" ) # 保存图片
def getverify(self, im): # 二值化 threshold = 140 table = [] for i in range(256): if i < threshold: table.append(0) else: table.append(1) # 由于都是数字对于识别成字母的 采用该表进行修正 rep = {'O': '0', 'I': '1', 'L': '1', 'Z': '2', 'S': '8', 'X': '*'}; # 转化到灰度图 imgry = im.convert('L') # 二值化,采用阈值分割法,threshold为分割点 out = imgry.point(table, '1') # 识别 text = image_to_string(out) # 识别对吗 text = text.upper(); # print text for r in rep: text = text.replace(r, rep[r]) # out.save(text+'.jpg') text = text.strip() str = eval(text) return str