基础爬虫框架

最新推荐文章于 2024-06-27 09:50:35 发布

DMU_lzq1996

最新推荐文章于 2024-06-27 09:50:35 发布

阅读量179

点赞数

分类专栏：爬虫

本文链接：https://blog.csdn.net/DMU_lzq1996/article/details/84334415

版权

爬虫专栏收录该内容

17 篇文章 0 订阅

订阅专栏

在这里插入图片描述

URL管理器：

class UrlManager (object):
    def __init__(self):
        self.new_urls = set() #未爬取URL集合
        self.old_urls = set()#已爬取URL集合
    def has_new_url(self):
        #判断是否有未爬取的URL
        return self.new_url_size()!=0
    def get_new_url(self):
        #获取一个未爬取的URL
        new_url = self.new_urls.pop()
        self.old_urls.add(new_url)
        return new_url
    def add_new_url(self,url):
        #将新的URL添加到未爬取的URL集合
        if url is None:
            return
        if url not in self.new_urls and url not in self.old_urls:
            self.new_urls.add(url)
    def add_new_urls(self,urls):
        #将新的url集合添加到未爬取的url集合中
        if urls is None or len(urls)==0:
            return
        for url in urls:
            self.add_new_url(url)
    def new_url_size(self):
        #获取未爬取URL集合大小
        return len(self.new_urls)
    def old_url_size(self):
        return len(self.old_urls)

HTML下载器：

import requests

class HtmlDownloader(object):
    def download(self,url):
        if url is None:
            return None
        user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:64.0) Gecko/20100101 Firefox/64.0'
        headers = {'User-Agent':user_agent}
        r = requests.get(url,headers=headers)
        if r.status_code ==200:
            r.encoding = 'utf-8'
            return r.text
        return None

HTML解析器：

import re
from urllib import parse
from bs4 import BeautifulSoup

class HtmlParser(object):
    
    def parser(self,page_url,html_cont):
        #用于解析网页内容，抽取URL和数据
        if page_url is None or html_cont is None:
            return
        soup = BeautifulSoup(html_cont,'html.parser')
        new_urls = self._get_new_urls(page_url,soup)
        new_data = self._get_new_data(page_url,soup)
        return new_urls,new_data
    
    def _get_new_urls(self,page_url,soup):
        #抽取新的URL集合
        new_urls = set()
        #抽取符合要求的a标记
        links = soup.find_all('a',href=re.compile("/item/.*"))
        for link in links:
            #提取href属性
            new_url = link['href']
            #拼接成完整网址
            new_full_url = parse.urljoin('https://baike.baidu.com',new_url)
            new_urls.add(new_full_url)
        return new_urls
    
    def _get_new_data(self,page_url,soup):
        #抽取有效数据
        data = {}
        data['url']=page_url
        title = soup.find('dd',class_='lemmaWgt-lemmaTitle-title').find('h1')
        data['title']=title.get_text()
        summary = soup.find('div',class_='lemma-summary')
        try:
            data['summary']=summary.get_text()
        except Exception:
            data['summary']="暂无说明"
        return data

数据存储器：

import codecs

class DataOutput(object):
    
    def __init__(self):
        self.datas=[]
    def store_data(self,data):
        if data is None:
            return
        self.datas.append(data)
        
    def output_html(self):
        fout=codecs.open('baike.html','w')
        fout.write("<html>")
        fout.write("<body>")
        fout.write("<table>")
        for data in self.datas:
            fout.write("<tr>")
            fout.write("<td>%s</td>"%data['url'])
            fout.write("<td>%s</td>"%data['title'])
            fout.write("<td>%s</td>"%data['summary'])
            fout.write("</tr>")
            self.datas.remove(data)
        fout.write("</table>")
        fout.write("</body>")
        fout.write("</html>")
        fout.close()

爬虫调度器：

from DataOutput import DataOutput
from HtmlDownloader import HtmlDownloader
from HtmlParser import HtmlParser
from URLManager import UrlManager

class SpiderMan(object):
    def __init__(self):
        self.manager = UrlManager()
        self.downloader = HtmlDownloader()
        self.parser = HtmlParser()
        self.output = DataOutput()
    def crawl(self,root_url):
        #添加入口URL
        self.manager.add_new_url(root_url)
        #判断url管理器中是否有新的url，同时判断抓取了多少url
        while(self.manager.has_new_url() and self.manager.old_url_size()<100):
            try:
                new_url = self.manager.get_new_url()
                html = self.downloader.download(new_url)
                new_urls,data = self.parser.parser(new_url,html)
                self.manager.add_new_urls(new_urls)
                self.output.store_data(data)
                #print(data)
                print("已经抓取%s个链接"%self.manager.old_url_size())
            except Exception:
                print("crawl failed")
        self.output.output_html()
        

if __name__=="__main__":
    spider_man = SpiderMan()
    spider_man.crawl("https://baike.baidu.com/item/网络爬虫")

运行结果：
在这里插入图片描述

DMU_lzq1996

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
基础爬虫框架

URL管理器：class UrlManager (object): def __init__(self): self.new_urls = set() #未爬取URL集合 self.old_urls = set()#已爬取URL集合 def has_new_url(self): #判断是否有未爬取的URL ret...
复制链接

扫一扫