scrapy 总结

1. scrapy 接受命令行参数

def __init__(self,*args, **kwargs):
    super(SinopecsalesSpider, self).__init__(*args, **kwargs)

启动spider 时 传 -a testkey=testval 通过 kwargs['testkey']获取

2. cookie传递

def start_requests(self):
    self.logger.info('Parse function called on start_requests')
    return [Request(url=url
                    meta={'cookiejar': 1},
                    callback=self.post_login)]

在回调函数里面通过 response.meta['cookiejar'] 里面获取cookie,请求的时候就会发送cookie

return [FormRequest(url=self.login_submit,
                                meta={'cookiejar': response.meta['cookiejar'] },
                                headers=reqheaders,
                                formdata=data,
                                callback=self.after_login,
                                errback=self.errback_httpbin,
                                dont_filter = True
                                  )]

3. xpath 和 css 选择器

 提取html里面的数据时尽量用xpath,css ， 也可以结合正则使用

tag=sel.xpath('//a/text()').extract()
total=sel.xpath('//b/text()').extract()

response.xpath('*').re(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}')

4.设置请求头

DEFAULT_REQUEST_HEADERS = {
        
	"Accept": "application/json, text/javascript, */*; q=0.01",
	"Accept-Encoding": "gzip,deflate",
	"Accept-Language": "en-US,en;q=0.8,zh-TW;q=0.6,zh;q=0.4",
	"Connection": "keep-alive",
	'Cache-Control': 'no-cache',
	'X-Requested-With': 'XMLHttpRequest',
	'Origin': 'http://xx.com',
        "Content-Type": " application/x-www-form-urlencoded; charset=UTF-8",        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",        "Referer":"http://xx.com/"}

5. unicode字符处理

比较相等加 

item['traName'] == u'相等字符'

douban_item['info'] = str[0].strip('()')

6.gbk to utf-8

response.body.decode('gbk').encode('utf-8')

7. 随机数和sleep

time.sleep(5)
random.random()

8.破解验证码

用pytesser,可能需要二值化

def process_image(self, response):
    image = Image.open(StringIO(response.body))
    # image.show()
    image.filter(ImageFilter.SHARPEN)
    box = (2, 2, 88, 23)  # 设定裁剪区域
    region = image.crop(box)  # 裁剪图片，并获取句柄region
    return self.getverify(region)
    # region.save( "crop.jpg" )  # 保存图片

def getverify(self, im):
    # 二值化
    threshold = 140
    table = []
    for i in range(256):
        if i < threshold:
            table.append(0)
        else:
            table.append(1)
    # 由于都是数字对于识别成字母的 采用该表进行修正
    rep = {'O': '0', 'I': '1', 'L': '1', 'Z': '2', 'S': '8', 'X': '*'};
    # 转化到灰度图
    imgry = im.convert('L')
    # 二值化，采用阈值分割法，threshold为分割点
    out = imgry.point(table, '1')
    # 识别
    text = image_to_string(out)
    # 识别对吗
    text = text.upper();
    # print text
    for r in rep:
        text = text.replace(r, rep[r])
        # out.save(text+'.jpg')
    text = text.strip()
    str = eval(text)
    return str