开发记录---python爬虫之gooseeker尝试

首先下载gooseeker抓取环境，到官网http://www.gooseeker.com/下载相关环境，根据教程文档http://www.gooseeker.com/tuto/tutorial.html创建抓取规则，并保存。这里尝试爬取的是某猫的搜索“笔记本”的网页https://list.tmall.com/search_product.htm?type=pc&q=%B1%CA%BC%C7%B1%BE&totalPage=1&sort=s&style=g&from=mallfp..pc_1_searchbutton

参数totalPage是页数，总共100页，我这是利用一个循环爬取每页的数据，亦可通过 教程中的翻页和层级 爬取所有数据

我的python环境是2.7的，使用下面2.7的的gooseeker的接口，返回出来的是定义规则的xml列表

#!/usr/bin/python
# -*- coding: utf-8 -*-
# 模块名: gooseeker_py2
# 类名: GsExtractor
# Version: 2.0
# 适配Python版本: 2.7
# 说明: html内容提取器
# 功能: 使用xslt作为模板，快速提取HTML DOM中的内容。
# released by 集搜客(http://www.gooseeker.com) on May 18, 2016
# github: https://github.com/FullerHua/jisou/core/gooseeker_py2.py

from urllib2 import urlopen
from urllib import quote
from lxml import etree

class GsExtractor(object):
    def _init_(self):
        self.xslt = ""
    # 从文件读取xslt
    def setXsltFromFile(self , xsltFilePath):
        file = open(xsltFilePath , 'r')
        try:
            self.xslt = file.read()
        finally:
            file.close()
    # 从字符串获得xslt
    def setXsltFromMem(self , xsltStr):
        self.xslt = xsltStr
    # 通过GooSeeker API接口获得xslt
    def setXsltFromAPI(self , APIKey , theme, middle=None, bname=None):
        apiurl = "http://www.gooseeker.com/api/getextractor?key="+ APIKey +"&theme="+quote(theme)
        if (middle):
            apiurl = apiurl + "&middle="+quote(middle)
        if (bname):
            apiurl = apiurl + "&bname="+quote(bname)
        apiconn = urlopen(apiurl)
        self.xslt = apiconn.read()
    # 返回当前xslt
    def getXslt(self):
        return self.xslt
    # 提取方法，入参是一个HTML DOM对象，返回是提取结果
    def extract(self , html):
        xslt_root = etree.XML(self.xslt)
        transform = etree.XSLT(xslt_root)
        result_tree = transform(html)
        return result_tree

通过上述接口获取xml数据，使用 import xml.dom.minidom解析xml数据，如下代码

# -*- encoding=utf-8 -*-
# crawler_gooseeker_bbs.py
# 版本: V1.0
## 设置xslt抓取规则，第一个参数是app key  817b2794557252f2f46fb88733079c07，请到会员中心申请  规则名
## xml解析教程  http://www.runoob.com/python/python-xml.html
from urllib2 import urlopen
from lxml import etree

import tmallInfo
import GsExtractor
import xml.dom.minidom
import MySQLdb

conn = MySQLdb.connect(
    host='localhost',
    port=3306,
    user='root',
    passwd='*****',    #mysql的密码
    db='*****',    #mysql的数据库名
    charset="utf8"  #这里设置格式，报这个错mySql默认设置 latin-1 ,UnicodeEncodeError: 'latin-1' codec can't encode characters in position 0-3: ordinal not in range(256)
)
cur = conn.cursor()
cur.execute(
    "create table if not exists tmallInfo(id int unsigned not null auto_increment primary key,pcName varchar(100),pcPrice varchar(30),storeName varchar(30),shopLink varchar(250),monthlyTurnover varchar(30),imgSrc varchar(150))")
tmallInfo.conn = conn

#这里可用线程优化 爬虫速度 http://www.runoob.com/python/python-multithreading.html
for n in range(1,101,1):
    url = "https://list.tmall.com/search_product.htm?type=pc&q=%B1%CA%BC%C7%B1%BE&totalPage=" + str(n) + "&sort=s&style=g&from=mallfp..pc_1_searchbutton"
    conn = urlopen(url)
    doc = etree.HTML(unicode(conn.read(), "gbk"))  # 乱码

    bbsExtra = GsExtractor()
    bbsExtra.setXsltFromAPI("817b2794557252f2f46fb88733079c07", "lyTmall")  # 设置xslt抓取规则，第一个参数是app key，请到会员中心申请
    result = bbsExtra.extract(doc)  # 调用extract方法提取所需内容

    DOMTREE = xml.dom.minidom.parseString(str(result))
    pcList = DOMTREE.documentElement
    items = pcList.getElementsByTagName("item")
    for item in items:
        pcName = item.getElementsByTagName('pcName')[0].childNodes[0].data
        pcPrice = item.getElementsByTagName('pcPrice')[0].childNodes[0].data
        storeName = item.getElementsByTagName('StoreName')[0].childNodes[0].data
        shopLink = item.getElementsByTagName('ShopLink')[0].childNodes[0].data
        monthlyTurnover = item.getElementsByTagName('monthlyTurnover')[0].childNodes[0].data
        imgSrcs = item.getElementsByTagName('imgSrc')[0]
        if len(imgSrcs.childNodes) > 0:
            imgSrc = imgSrcs.childNodes[0].data
        else:
            imgSrc = "null"

        info = tmallInfo(pcName, pcPrice, storeName, shopLink, monthlyTurnover, imgSrc)
        info.save()

    tmallInfo.conn.commit()
    print n

tmallInfo.conn.close()

将爬取的数据保存到mysql数据库，总共5500多条数据，在保存时创建一个bean对象，方便保存，如下：

# -*- coding:utf-8 -*-
#myspl环境下载   https://pypi.python.org/pypi/MySQL-python#downloads
import MySQLdb

class tmallInfo:
    conn = None;
    def __init__(self,pcName,pcPrice,storeName,shopLink,monthlyTurnover,imgSrc):
        self.pcName = pcName
        self.pcPrice = pcPrice
        self.storeName = storeName
        self.shopLink = shopLink
        self.monthlyTurnover = monthlyTurnover
        self.imgSrc = imgSrc

    def save(self):
        cur = tmallInfo.conn.cursor()
        #  %s 为字符串占位符   name age为表中的属性字段，写死
        sql = "insert into tmallInfo(pcName,pcPrice,storeName,shopLink,monthlyTurnover,imgSrc) values(%s,%s,%s,%s,%s,%s)"
        param = (self.pcName,self.pcPrice,self.storeName,self.shopLink,self.monthlyTurnover,self.imgSrc)
        n = cur.execute(sql, param)

先新建一个对象，再调用save将其保存到数据库中

info = tmallInfo(pcName, pcPrice, storeName, shopLink, monthlyTurnover, imgSrc)
info.save()

将上述三个py文件放到一个目录中即可使用

本次爬虫使用到的基本技术：

• python操作数据库
• python xml解析
• gooseeker的网页采集器的使用