舆情分析 - JD数据抓取优化

JD数据抓取优化

我们期望是在文件中指定制定产品信息和表名，程序可以依据这些信息自动抓取到对应的表中。

方案

我们的想法是：

• 定义config.json文件存放产品信息和表名
• PySpider Project文件负责定时抓取
• 封装Operation基类负责解析网页和存储

1. config.json

config文件信息如下：

{
  "configs": [
    {
      "index": 0,
      "search_word": "小米air13.3",
      "price": "4000",
      "productName": "小米(MI)Air 13.3英寸全金属轻薄笔记本",
      "keyword": "13.3",
      "tables": [
        "compete_info",
        "compete_comments",
        "compete_comments_sub",
        "compete_comments_words"
      ]
    }
  ]
}

2. Operation基类

核心Code如下：

def index_page(self, client, response):
    page = response.doc('.fp-text > i').text()
    print "search total page:", page
    for i in range(1, int(page) + 1):
        num = i * 2 - 1
        turn_page = self.turnpage_url + str(i)
        print "search page index:", i, ",", turn_page
        client.crawl(turn_page, callback=client.turn_page, validate_cert=False, fetch_type='js')

def turn_page(self, client, response):
    plist = response.doc('.J-goods-list > .clearfix >li')
    index = 1
    for each in plist.items():
        name = each.find('.p-name-type-2 em').text()
        if self.keyword in name:
            detail_url = each.find('.p-commit > strong > a').attr("href")
            print "product index:", index, ",url", detail_url
            index = index + 1
            client.crawl(detail_url, callback=client.detail_page, validate_cert=False, fetch_type='js')

def detail_page(self, client, response):
    # 获取评论的页数
    pages = response.doc('div.com-table-footer > div > div >a')
    productId = re.findall("/\d+.html", response.url)[0]
    productId = re.findall("\d+", productId)[0]
    if pages:
        # check if there are some pages and set max page number 100
        page = self.getVisiableNumber(pages.items())
        print "productId:", productId, ",visible comment page:", page
        # write part product info to db
        dict = {}
        dict['productName'] = self.productName
        dict['productId'] = productId
        dict['displayName'] = response.doc('.sku-name').text()
        dict['detailUrl'] = response.url
        dict['price'] = response.doc('.p-price > span').eq(1).text()
        self.add_product(dict)
        if page >= 6:  # 超过6页会有...无法计算准确页面，采用递归方式获取
            # set first comment page url
            multi_url = self.comment_url + productId + "&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&rid=0&fold=1"
            client.crawl(multi_url, callback=client.comment_circle_page, validate_cert=False, fetch_type='js',
                         save={'page': 0, 'productId': productId})
        elif page > 0:  # 小于6页，依据页数，采用递归方式获取
            multi_url = self.comment_url + productId + "&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&rid=0&fold=1"
            client.crawl(multi_url, callback=client.comment_page, validate_cert=False, fetch_type='js',
                         save={'page': page, 'pos': 0, 'productId': productId})
    else:
        print "productId:", productId, ",no comments"

def comment_page(self, client, response):
    try:
        # get page position information
        page = response.save.get('page')
        productId = response.save.get('productId')
        pos = int(response.save.get('pos'))
        print "total page:", page, "current page index:", pos, ",", response.url
    except Exception, e:
        print "comment_page params ex ", e
        return
    try:
        # json parse
        jsonstr = response.doc("body").text()
        hjson = json.loads(jsonstr)
        comments = hjson["comments"]
        summary = hjson["productCommentSummary"]
        # comments list infomation
        commentslist = []
        for each in comments:
            time_int = tool.dateTimeToInt(each["creationTime"])
            data = (productId, each["id"], each["nickname"], time_int, each["score"],
                    each["userLevelName"], each['userImgFlag'], each["content"])
            commentslist.append(data)
        # write comments to db
        self.add_comments(commentslist, productId)

        # if last page, write product info summary to db
        if len(commentslist) == 0 or pos >= page - 1:
            print "is last page, supplement product info"
            # summary infomation
            dict_sum = {}
            dict_sum['productId'] = productId
            dict_sum['goodCount'] = summary['goodCount']
            dict_sum['generalCount'] = summary['generalCount']
            dict_sum['poorCount'] = summary['poorCount']
            self.supplement_product(dict_sum)
        else:
            multi_url = self.comment_url + str(productId) + "&score=0&sortType=5&page=" + str(
                pos + 1) + "&pageSize=10&isShadowSku=0&rid=0&fold=1"
            client.crawl(multi_url, callback=client.comment_page, validate_cert=False, fetch_type='js',
                         save={'page': page, 'pos': pos + 1, 'productId': productId})

    except Exception, e:
        print "comment_page ex ", e
        # goto next page
        multi_url = self.comment_url + str(productId) + "&score=0&sortType=5&page=" + str(
            pos + 1) + "&pageSize=10&isShadowSku=0&rid=0&fold=1"
        client.crawl(multi_url, callback=client.comment_page, validate_cert=False, fetch_type='js',
                     save={'page': page, 'pos': pos + 1, 'productId': productId})

def comment_circle_page(self, client, response):
    try:
        # get page position information
        page = response.save.get('page')
        productId = response.save.get('productId')
        print "current page index:", page, ",productId", productId, ",", response.url

        # json parse
        jsonstr = response.doc("body").text()
        hjson = json.loads(jsonstr)
        comments = hjson["comments"]
        summary = hjson["productCommentSummary"]
        # comments list infomation
        commentslist = []
        commentsIds = []
        for each in comments:
            time_int = tool.dateTimeToInt(each["creationTime"])  # convert comment time to timestamp
            data = (productId, each["id"], each["nickname"], time_int, each["score"],
                    each["userLevelName"], each['userImgFlag'], each["content"])
            commentslist.append(data)
            commentsIds.append(each["id"])

        # save comments
        if len(commentsIds) == 0 or page == 100:  # it's the last page, write product in db
            print "it's the last page, write product in db"
            dict_sum = {}
            dict_sum['productId'] = productId
            dict_sum['goodCount'] = summary['goodCount']
            dict_sum['generalCount'] = summary['generalCount']
            dict_sum['poorCount'] = summary['poorCount']
            self.supplement_product(dict_sum)
        else:
            # write comments to db
            self.add_comments(commentslist, productId)
            # goto next page
            page = page + 1
            multi_url = self.comment_url + str(productId) + "&score=0&sortType=5&page=" + str(
                page) + "&pageSize=10&isShadowSku=0&rid=0&fold=1"
            client.crawl(multi_url, callback=client.comment_circle_page, validate_cert=False, fetch_type='js',
                         save={'page': page, 'productId': productId})
    except Exception, e:
        print "comment_circle_page ex ", e
        # goto next page
        page = page + 1
        multi_url = self.comment_url + str(productId) + "&score=0&sortType=5&page=" + str(
            page) + "&pageSize=10&isShadowSku=0&rid=0&fold=1"
        client.crawl(multi_url, callback=client.comment_circle_page, validate_cert=False, fetch_type='js',
                     save={'page': page, 'productId': productId})

3. PySpider项目

核心Code：

class Handler(BaseHandler):
    crawl_config = BaseOperation.crawl_config

    def __init__(self):
        configJson = tool.readConfigFromFile(CONFIG_PATH)
        cfg = configJson[cm.CONFIGS][POS]
        self.baseOperation = BaseOperation()
        self.baseOperation.initConfig(self, cfg, DEBUG)

    @every(minutes=TIME)  # 每小时
    def on_start(self):
        self.crawl(self.baseOperation.url, callback=self.index_page, validate_cert=False, fetch_type='js')

    # 搜索出的列表页面
    @config(age=TIME * 60)
    @config(priority=4)
    def index_page(self, response):
        self.baseOperation.index_page(self, response)

    # 搜索翻页页面
    @config(age=TIME * 60)
    @config(priority=3)
    def turn_page(self, response):
        self.baseOperation.turn_page(self, response)

    @config(age=TIME * 60)
    @config(priority=2)
    def detail_page(self, response):
        self.baseOperation.detail_page(self, response)
        return {
            "url": response.url,
            "title": response.doc('title').text(),
        }

    # we get comments through json api
    @config(age=TIME * 60)
    @config(priority=1)
    def comment_page(self, response):
        self.baseOperation.comment_page(self, response)

    # we get comments through json api
    @config(age=TIME * 60)
    @config(priority=1)
    def comment_circle_page(self, response):
        self.baseOperation.comment_circle_page(self, response)

4. flow说明

Github源码下载地址，点击这里