用scrapy 大规模 无登陆 爬取 58同城 房产信息

快过年了，早上没什么事情就把之前自己写的一个小demo 拿出来记录在博客上

import scrapy

class CourseItem(scrapy.Item):
    title = scrapy.Field()   
    
    number = scrapy.Field()
    
    area = scrapy.Field()
    
    huxing = scrapy.Field()
    
    nature = scrapy.Field()

class MySpider(scrapy.Spider):
    
    name = "MySpiders"
    
    allowed_domains = ["xa.58.com"]
    
    start_urls = ["http://xa.58.com/chuzu/?PGTID=0d200001-001e-3800-de6e-943e5224b994&ClickID=1"]
    encoding='gb18030'
    def parse(self, response):
        item = CourseItem()
        for box  in response.xpath('//ul[@class="listUl"]/li'):
        
         item['title'] = box.xpath('.//div[@class="des"]/h2/a/text()').extract()
         try:
             item['title'] = item['title'][0].strip()
         except IndexError:
             item['title'] = ''    
         item['number'] = box.xpath('.//div[@class="listliright"]/div[@class="money"]/b/text()').extract()
         try:
             item['number'] = item['number'][0]
         except IndexError:
             item['number'] = ''
         item['area'] = box.xpath('.//div[@class="des"]/p[@class="add"]/a/text()').extract()
         try:
             item['area'] = item['area'][0]
         except IndexError:
             item['area'] = ''
         item['huxing'] = box.xpath('.//div[@class="des"]/p[@class="room"]/text()').extract()
         try:
             item['huxing'] = item['huxing'][0]
         except IndexError:
             item['huxing'] = ''
         item['nature'] = box.xpath('.//div[@class="des"]/div[@class="jjr"]/text()').extract()
         try:
             item['nature'] = item['nature'][0].strip()
         except IndexError:
             item['nature'] = ''
        
         yield item
         
         
         next_page = response.xpath('.//li[@id="bottom_ad_li"]/div[@class="pager"]/a[@class="next"]/@href').extract_first()
         
         
         
         if next_page:
            next_page = response.urljoin(str(next_page))
            
            yield scrapy.Request(next_page, callback=self.parse)
         else:
             print(None)
            

这个呢，比较简单，直接进入你想要抓取的那个页面，然后用xpath 提取你想要的内容，

这里稍微注意一下 for 循环，提取的标签是你即将循环的那些内容的外层标签例如。随便找了段代码

		<ul class="ul-form">
			<li><label>默认语句：</label>
				<form:input path="statement" htmlEscape="false" class="input-medium"/>
			</li>
			<li><label>类型：</label>
				<form:input path="type" htmlEscape="false" maxlength="4" class="input-medium"/>
			</li>
			<li class="btns"><input id="btnSubmit" class="btn btn-primary" type="submit" value="查询"/></li>
			<li class="clearfix"></li>
		</ul>

你要循环的是ul 那个标签，然后在网页找到下一页的那个标签，发送请求 然后回调上面的函数

展示一下