pyspider Ajax采集,pyspider采集例子(js)

#!/usr/bin/env python

# -*- encoding: utf-8 -*-

# Created on 2015-01-04 03:35:01

# Project: tutorial_pinterest

from pyspider.libs.base_handler import *

class Handler(BaseHandler):

"""

This is a sample scrape script for: Level 3: Render with PhantomJS

http://docs.pyspider.org/en/latest/tutorial/Render-with-PhantomJS/#running-javascript-on-page

"""

def on_start(self):

self.crawl('http://www.pinterest.com/categories/popular/',

fetch_type='js', js_script="""

function() {

window.scrollTo(0,document.body.scrollHeight);

}

""", callback=self.index_page)

def index_page(self, response):

return {

"url": response.url,

"p_w_picpaths": [{

"title": x('.richPinGridTitle').text(),

"img": x('.pinImg').attr('src'),

"author": x('.creditName').text(),

} for x in response.doc('.item').items() if x('.pinImg')]

}