爬虫技术

python爬虫其实并没有什么太高深的技术，耗费精力的地方在于对网站返回结果的解析，以及对一些反爬机制的研究。

---

爬虫demo

下面这个例子可以爬取起点免费小说，直接看代码（本例来源于参考资料1）：

# coding=utf-8
import urllib2
import sys
from bs4 import BeautifulSoup

# 设置编码
reload(sys)
sys.setdefaultencoding('utf-8')

class YuewenFreeSpider:

    def __init__(self):
        pass

    # 获取一个章节的内容
    def get_chapter_content(self, file, url):
        try:
            book_content_res = urllib2.urlopen(url)
            book_content_soup = BeautifulSoup(book_content_res.read(), "html.parser")
            file.write(book_content_soup.select("h3[class='j_chapterName'] span")[0].string + '\n')
            for p in book_content_soup.select(".j_readContent p"):
                file.write(p.next + '\n')
        except Exception, e:
            # 如果出错了，就重新运行一遍
            print(e)
            self.get_chapter_content(file, url)
        else:
            chapter_next = book_content_soup.select("a#j_chapterNext")[0]
            if chapter_next.string != "书末页":
                next_url = "https:" + chapter_next["href"]
                self.get_chapter_content(file, next_url)

    # 获取当前页所有书的内容
    def get_current_url_books(self, url):
        response = urllib2.urlopen(url)
        the_page = response.read()
        soup = BeautifulSoup(the_page, "html.parser")
        book_arr = soup.select("ul[class='all-img-list cf'] > li")
        global start_index
        if start_index > 0:
            book_arr = book_arr[start_index:]
            start_index = 0
        for book in book_arr:
            book_cover = book.select("div[class='book-mid-info'] h4 > a")[0]
            print "书名：" + book_cover.string
            # 先创建.txt文件，然后获取文本内容写入
            book_file = open("/home/username/crawler/books/" + book_cover.string + ".txt", "a+")
            bres = urllib2.urlopen("https:" + book_cover['href'])
            bsoup = BeautifulSoup(bres.read(), "html.parser")
            book_content_href = bsoup.select("a[class='red-btn J-getJumpUrl']")[0]["href"]
            self.get_chapter_content(book_file, "https:" + book_content_href)
            book_file.close()
        next_page = soup.select("a.lbf-pagination-next")[0]
        return next_page["href"]


# 根据传入参数设置从哪里开始下载
url = "//www.qidian.com/free/all?orderId=&vip=hidden&style=1&pageSize=20&siteid=1&pubflag=0&hiddenField=1&page=1"


if __name__ == '__main__':
    start_index = 0
    # 死循环 直到没有下一页
    while True:
        if url.startswith("//"):
            url = YuewenFreeSpider().get_current_url_books("https:" + url)
        else:
            break

这里主要用到了两个库，分别是urllib2和BeautifulSoup。urllib2是http请求库，BeautifulSoup的官方解释是：

• Beautiful Soup提供一些简单的、python式的函数用来处理导航、搜索、修改分析树等功能。它是一个工具箱，通过解析文档为用户提供需要抓取的数据，因为简单，所以不需要多少代码就可以写出一个完整的应用程序。

• Beautiful Soup自动将输入文档转换为Unicode编码，输出文档转换为utf-8编码。你不需要考虑编码方式，除非文档没有指定一个编码方式，这时，Beautiful Soup就不能自动识别编码方式了。然后，你仅仅需要说明一下原始编码方式就可以了。

Beautiful Soup已成为和lxml、html6lib一样出色的python解释器，为用户灵活地提供不同的解析策略或强劲的速度。

签约小说爬取

上节的例子中展示的是免费小说的爬取过程。如果要爬取签约小说的具体信息呢？示例如下：

# coding=utf-8
import urllib2
import sys
import time
import logging
import requests
from bs4 import BeautifulSoup
from lxml import etree
import json


class YuewenSpider:

    def __init__(self):
        self.session = requests.session()

    @staticmethod
    def get_url(url_key):
        url = {
            'search': 'https://www.qidian.com/search?kw=',
            'category': 'https://book.qidian.com/ajax/book/category?',
            'vipreader': 'https://vipreader.qidian.com/chapter/',
            'subscribe': 'https://vipreader.qidian.com/ajax/subscribe/subscribe?',
            'getSubscribe': 'https://vipreader.qidian.com/ajax/subscribe/getSubscribe?'
        }
        return url[url_key]

    @staticmethod
    def get_header_and_token():
        header = dict()
        header['Accept'] = 'text/html,application/xhtml+xml,application/xml;' \
                           'q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3'
        header['Accept-Encoding'] = 'gzip, deflate, br'
        header['Accept-Language'] = 'zh-CN,zh;q=0.9'
        header['Cache-Control'] = 'max-age=0'
        header['Connection'] = 'keep-alive'
        header['cookie'] = '_csrfToken=AYnkW4AoqnnkKTIjgbgbndUM8qQ2rikJA2gqBWvt; newstatisticUUID=1564129077_1845515318; qdrs=0%7C3%7C0%7C0%7C1; showSectionCommentGuide=1; qdgd=1; se_ref=baidu; se_ref_bid=1015221208; gender=male; e1=%7B%22pid%22%3A%22qd_P_Searchresult%22%2C%22eid%22%3A%22qd_S05%22%2C%22l1%22%3A3%7D; e2=%7B%22pid%22%3A%22qd_P_Searchresult%22%2C%22eid%22%3A%22qd_S05%22%2C%22l1%22%3A3%7D; rcr=1013432302%2C1015221208%2C1015741318%2C1014139104%2C1015129326%2C1015055967%2C1015235392%2C1015336641%2C1015835395%2C1010868264%2C1015444718%2C1010144088; lrbc=1013432302%7C432810477%7C0; pageOps=1'
        header['Host'] = 'www.qidian.com'
        header['Upgrade-Insecure-Requests'] = '1'
        header['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) ' \
                               'Chrome/75.0.3770.100 Safari/537.36'
        token = ''
        cookie = header['cookie']
        if cookie:
            cookie_pair = cookie.split(";")
            for p in cookie_pair:
                item = p.split("=")
                if item[0].strip() == "_csrfToken":
                    token = item[1]
        return header, token

    # 根据更新时间获取原创id和需要更新的章节列表
    def spider_chapter_list(self, book_title, book_author, updated_time):
        try:
            book = self.search_book(book_title, book_author)
            if not book:
                return None

            data = self.get_header_and_token()
            header, token = data[0], data[1]

            spider_book_chapters = dict()

            chapter_list = []

            # 爬取章节列表信息，判断是否存在更新章节book_chapter
            url = self.get_url(url_key='category') + '_csrfToken=' + token + '&bookId=' + book['id']
            r = self.session.get(url, headers=header)
            detail = json.loads(r.content)
            if detail['code'] == 1:
                raise Exception("get chapter list info failed")
            else:
                # 遍历卷信息
                for d in detail['data']['vs'][::-1]:
                    # 遍历章节信息
                    for c in d['cs'][::-1]:
                        t = int(time.mktime(time.strptime(c['uT'], '%Y-%m-%d %H:%M:%S')))
                        if t > updated_time:
                            chapter = dict()
                            chapter['chapter_title'] = c['cN']
                            chapter['chapter_url'] = str(book['id']) + "/" + str(c['id'])
                            chapter['chapter_cnt'] = c['cnt']  # 章节字数
                            chapter['chapter_id'] = c['id']
                            chapter['free'] = c['sS']
                            chapter['updated_time'] = t
                            chapter['uuid'] = c['uuid']
                            chapter_list.append(chapter)
                        else:
                            break
                    else:
                        continue
                    break
            if chapter_list:
                spider_book_chapters['book_id'] = int(book['id'])
                spider_book_chapters['chapters'] = chapter_list[::-1]
            return spider_book_chapters

        except Exception, e:
            logging.error("YueWenSpider-spider_chapter_list except: %s", e.message)
            return None


    # 根据书名和作者搜书
    def search_book(self, book_title, book_author):
        url = self.get_url(url_key='search') + book_title
        r = self.session.get(url, headers=self.get_header_and_token()[0])
        selector = etree.HTML(r.content)
        # 获取图书id
        book_ids = selector.xpath('//li[@class="res-book-item"]/@data-bid')
        # 获取图书名称和作者信息
        book_titles = selector.xpath('//li[@class="res-book-item"]//div[@class="book-mid-info"]/h4/a//text()')
        book_authors = selector.xpath('//div[@class="book-mid-info"]/p[@class="author"]/a[1]/text()')
        if not book_ids or not book_titles or not book_authors:
            return None
        for i in range(len(book_ids)):
            book = dict()
            book['id'] = book_ids[i] if book_ids[i] else ''
            book['author'] = book_authors[i] if book_authors[i] else ''
            book['title'] = book_titles[i] if book_titles[i] else ''
            if book['title'] == book_title and book['author'] == book_author:
                return book
        return None

if __name__ == '__main__':
    logging.basicConfig(stream=sys.stdout, level=logging.INFO,
                        format='%(asctime)s %(levelno)s %(message)s',
                        datefmt='%Y-%m-%d %H:%M:%S')
    start_time = int(time.time()) - 3600 * 24
    new_chapter_list = YuewenSpider().spider_chapter_list(u"九星毒奶", u"育", start_time)
    print new_chapter_list

爬取到需要更新的章节列表后，就需要订阅具体章节内容了，具体代码就不贴了。说一下主要过程：首先先获取章节价格，查询账户余额，然后发起订阅。

获取价格：

url = self.get_url('vipreader') + spider_chapter['chapter_url']
header['Host'] = 'vipreader.qidian.com'
 r = self.session.get(url, headers=header, verify=False)
 selector = etree.HTML(r.text)
 price = selector.xpath('//a[@class="single j_subscribeBtn"]/span/i/text()')

查询账户余额：

price = int(price[0])
# 准备参数
 data = dict()
 data['bookId'] = book_id
 data['chapterPrice'] = price
 chapters = dict()
 chapters['chapterId'] = spider_chapter['chapter_id']
 chapters['chapterCnt'] = spider_chapter['chapter_cnt']
 chapters['price'] = price
 chapters['uuid'] = spider_chapter['uuid']
 data['chapters'] = [chapters]
 data['isRenew'] = 0
 data['chapterCnt'] = 1
 data['isBuyAll'] = 0
 # 查询余额信息
 url = self.get_url(url_key='getSubscribe') + token
 r = self.session.post(url, headers=header, data=json.dumps(data))

最后订阅具体章节：

# 发起订阅
url = self.get_url(url_key='subscribe') + token
r = self.session.post(url, headers=header, data=json.dumps(data))

bs和lxml

在本节中解析html时用的是lxml。
BeautifulSoup是一个库，而XPath是一种技术，python中最常用的XPath库是lxml。

两者比较起来，性能方面lxml高于bs，而易用性方面则是bs好一些。

从性能方面看，BeautifulSoup和lxml的原理不一样，BeautifulSoup是基于DOM的，会载入整个文档，解析整个DOM树，因此时间和内存开销都会大很多。而lxml只会局部遍历，另外lxml是用c写的，而BeautifulSoup是用python写的，因此性能方面自然会差很多。

从易用性方面看，BeautifulSoup用起来比较简单，API非常人性化，支持css选择器。lxml的XPath写起来麻烦，开发效率不如BeautifulSoup。

举个例子：

title = soup.select('.content div.title h3')

同样的代码用Xpath写起来会很麻烦：

title = tree.xpath("//*[@class='content']/div[@class='content']/h3")

本文介绍了爬虫的实际使用实例。但很多情况下，内容网站都有一些反爬机制，比如说禁止相同ip在短时间内大量访问，这时候我们就要用到代理了。代理将在下一篇blog中介绍。

参考资料

[1]https://www.jianshu.com/p/1d658f67fbdf