爬虫笔记

代码部分（Python3，Pycharm）：
main.py

import manager
import download
import html_parser
import output

class SpiderMain(object):
    def __init__(self):
        self.urls = manager.Manager() #管理器
        self.download = download.Download() #下载器
        self.parser = html_parser.Parser() #解析器
        self.output = output.Output() #输出

    def craw(self, root_url):  #爬虫函数
        count = 1
        self.urls.add_new_url(root_url)  #将URL添加进
        while self.urls.has_new_url():

            try:
                new_url = self.urls.get_new_url()  #取出一个新的待爬URL
                html_cont = self.download.download(new_url)  #将网页下载下来
                #print(html_cont)
                new_urls, new_data = self.parser.parse(new_url, html_cont)  #解析所需数据

                self.urls.add_new_urls(new_urls) #将网页中解析出来的新URL添加进库中
                self.output.collect_data(new_data)

                if count == 1000:
                    break
                count = count + 1
            except:
                print("failed")
        self.output.output_html()

if __name__=="__main__":
    root_url = "https://baike.baidu.com/item/Python/407313.htm"
    obj_spider = SpiderMain()
    obj_spider.craw(root_url)

manager.py

class Manager(object):
    def __init__(self):
        self.new_urls = set()  #待爬取URL
        self.old_urls = set()  #已爬取URL


    def add_new_url(self, url):  #添加新的URL进库
        if url is None:  #如果URL为空，返回
            return
        if url not in self.new_urls and url not in self.old_urls: #如果URL既不在待爬取库中也不在已爬取库中，添加进待爬取库
            self.new_urls.add(url)

    def add_new_urls(self, urls):
        if urls is None or len(urls) == 0:
            return
        for url in urls:
            self.add_new_url(url)

    def has_new_url(self):  #如果待爬取库还有未爬取URL，返回1
        return len(self.new_urls) != 0

    def get_new_url(self):  #取出一个待爬取URL
        new_url = self.new_urls.pop()
        self.old_urls.add(new_url)  #将该URL添加进已爬取库中
        return new_url  #返回这个URL

download.py

import urllib.request

class Download(object):
    def download(self, url):
        if url is None:
            return None

        response =  urllib.request.urlopen(url)  #调用urllib库，请求页面

        if response.getcode() != 200:
            return None
        #print(response.read().decode('utf-8'))
        return response.read().decode('utf-8')

下载的页面（部分）：

html_parser.py

from bs4 import BeautifulSoup
import re
from urllib.parse import urlparse

class Parser(object):
    def _get_new_urls(self, page_url, soup): : #解析出需要的URL
        links = soup.find_all('a', href=re.compile(r"/item/"))  #利用正则表达式找到页面中所有符合要求的URL
        for link in links:
            new_url = link['href']
            #new_full_url = urlparse.urljoin(page_url, new_url)
            new_full_url = "https://baike.baidu.com" + new_url  #拼合URL
            print(new_full_url)
            new_urls.add(new_full_url) #将URL添加进库中
        return new_urls

    def _get_new_data(self, page_url, soup): #解析出需要的数据
        res_data = {}
        res_data['url'] = page_url
        #<dd class="lemmaWgt-lemmaTitle-title"><h1>Python</h1>
        title_node = soup.find('dd', class_="lemmaWgt-lemmaTitle-title").find("h1")
        res_data['title'] = title_node.get_text()
        #<div class="lemma-summary"
        summary_node = soup.find('div', class_="lemma-summary")
        res_data['summary'] = summary_node.get_text()
        return res_data



    def parse(self, page_url, html_cont): #解析函数
        if page_url is None or html_cont is None:  #如果网页URL为空或者下载的页面为空，返回
            return

        soup = BeautifulSoup(html_cont, 'html.parser')  #调用Beautiful Soup解析页面
        new_urls = self._get_new_urls(page_url, soup)
        new_data = self._get_new_data(page_url, soup)
        return new_urls, new_data

解析出的部分URL：

output.py(实际作用可有可无)

class Output(object):
    def __init__(self):
        self.datas = []

    def collect_data(self, data):
        if data is None:
            return
        self.datas.append(data)


    def output_html(self):
        fout = open('output.html', 'w', encoding='utf-8')

        fout.write("<html>")
        fout.write("<body>")
        fout.write("<table>")

        for data in self.datas:
            fout.write("<tr>")
            fout.write("<td>%s</td>" % data["url"])
            fout.write("<td>%s</td>" % data["title"].encode("utf-8"))
            fout.write("<td>%s</td>" % data['summary'].encode("utf-8"))
            fout.write("</tr>")

        fout.write("</table>")
        fout.write("</body>")
        fout.write("</html>")

        fout.close()

简易版：
按照下载页面->提取内容的简易流程

import urllib.request
import re
from bs4 import BeautifulSoup

root_url = "https://baike.baidu.com/item/Python/407313.htm"
response = urllib.request.urlopen(root_url)  # 调用urllib库，请求页面
#print(response.read().decode('utf-8'))
html_cont = response.read().decode('utf-8')  #读取页面
soup = BeautifulSoup(html_cont, 'html.parser')  #创建Beautiful Soup对象
links = soup.find_all('a', href=re.compile(r"/item/"))
for link in links:
    new_url = link['href']
    new_full_url = "https://baike.baidu.com" + new_url  # 拼合URL
    print(new_full_url)