python爬去搜狐论坛笔记

搜狐论坛搜索页面：http://s.club.sohu.com/?action=search

——————————

改于：2016.09.08

可是！论坛搜索自己跪了！都一个星期了还没修好==！是不是搜狐自己还没发现搜索自己跪了？！==

——————————

今天搞了半天才搞定搜狐的爬虫，坑还蛮多的。特意记下来，方便自己下次捡起来。

首先是搜狐论坛每个板块的url不一样，不能直接在得到的  【a/@href  】 加前缀。

后来找到了介个，是一个list，写着不同的板块对应的url前缀

我用的是python的scrapy框架，搜狐的论坛上，每一层楼的数据是用get的方式得到的，所以response.xpath('//table[@class="viewpost"]').extract()为空！

没办法了，直接从源码上用正则截取数据

除了正则不知道还有啥方法

table = re.findall('<table class.*table>',response.body)[0].decode('unicode-escape')

Selector(text=table).xpath('//div[@class="wrap"]/p/text()')

# -*- encoding:utf-8 -*-
import scrapy
import re
import time
from spider.items import spiderItem
#import urllib2,urllib
import json
from scrapy.selector import Selector
from w3lib.html import remove_tags
from w3lib.html import replace_tags

url_search = ...#这里是用url编码的搜索内容
f=open('sohu_.json','r')
sohu_url=json.load(f)
f.close()

class MySpider(scrapy.Spider):
    name = 'sohu'
    allowed_domains = ['sohu.com']
    start_urls = [ 'http://club.sohu.com/']
    time_list = {u"昨天": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 60 * 24)),
                 u"前天": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 60 * 24 * 2)),
                 u"三天前": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 60 * 24 * 3)),
                 u"四天前": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 60 * 24 * 4)),
                 u"五天前": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 60 * 24 * 5)),
                 u"六天前": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 60 * 24 * 6)),
                 u"七天前": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 60 * 24 * 7)),
                 u"3天前": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 60 * 24 * 3)),
                 u"4天前": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 60 * 24 * 4)),
                 u"5天前": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 60 * 24 * 5)),
                 u"6天前": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 60 * 24 * 6)),
                 u"7天前": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 60 * 24 * 7))
                 }

    def parse(self, response):
        urls_1 = ['http://s.club.sohu.com/?action=search&type=0&keyword=%s' % i for i in url_search]#搜索 
        urls_2 =['&timeauto=1&thread=1&page=%s' % i for i in range(1, 30)]
        urls = [i+j for i in urls_1 for j in urls_2]#拼接
        for url in urls:
            yield scrapy.Request(url, callback=self.parse_url)

    def parse_url(self, response):
        urls = response.xpath('//h1/a/@href').extract()
        bankuai=[i.replace('\n','').strip() for i in response.xpath('//div[@class="resultItem"]/a[2]/text()').extract()]
        url_dict = zip(urls,bankuai)
        url_dict = dict((urls,bankuai)for urls,bankuai in url_dict)
        for url,bankuai in url_dict.items():
            new_url = 'http://'+sohu_url[bankuai]+url
            yield scrapy.Request(new_url, callback=self.parse_page)

    def parse_page(self, response):
        item = spiderItem()
        table = re.findall('<table class.*table>',response.body)[0].decode('unicode-escape')

        item['content'] = ''.join(Selector(text=table).xpath('//div[@class="wrap"]/p/text()').extract()).strip().replace('\r','').replace('\t','').replace(' ','').replace(u'\u3000','')
        try:
            time_str = ''.join(Selector(text=table).xpath('//div[@class="grey"]//div//text()').extract())
            try:
                item['time'] = re.findall('\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}',time_str)[0]
                item['time'] = time.strptime(item['time'], "%Y-%m-%d %H:%M:%S")
                item['time'] = time.strftime("%Y-%m-%d %H:%M:%S", item['time'])
            except:
                for t in self.time_list.keys():
                    if t in time_str:
                        item['time'] = self.time_list[t]
        except:
            item['time'] = '1999-09-09 09:09:09'
            item['time'] = time.strptime(item['time'], "%Y-%m-%d %H:%M:%S")
            item['time'] = time.strftime("%Y-%m-%d %H:%M:%S", item['time'])
        yield item