spiderman.py 主服务 # -*- encoding=UTF-8 -*- ''' author:vfast name:spiderman data:2021/6/28 ''' from dataoutput import DataOutput from downloadhtml import HtmlDownload from exparehtml import Exparsehtml from urlmanager import UrlManager class SpiderMan(object): def __init__(self): self.manager = UrlManager() self.downloader = HtmlDownload() self.parser = Exparsehtml() self.output = DataOutput() def crawl(self, root_url): # 添加入口url self.manager.add_new_url(root_url) # 判断url管理器中是否有新的url,同时判断抓取了多少个url while (self.manager.has_new_url() and self.manager.old_url_size() < 100): try: # 从url管理器获取新的url new_url = self.manager.get_new_url() # html下载器下载网页 html = self.downloader.download(new_url) # HTML解析器抽取网页数据 new_urls, data = self.parser.parser(new_url, html) # 将抽取的url添加到url管理器中 self.manager.add_new_urls(new_urls) # 数据存储器存储文件 self.output.store_data(data) print "已经抓取%s个链接" % self.manager.old_url_size() except Exception, e: print "crawl failed" # 数据存储器将文件输出成指定格式 self.output.output_html() if __name__ == '__main__': spider_man = SpiderMan() spider_man.crawl("http://baike.baidu.com/view/284853.htm")
urlmanager.py url管理服务
# -*- encoding=UTF-8 -*- ''' author:vfast name:spider data:2021/6/24 ''' class UrlManager(object): def __init__(self): self.new_urls = set() # 没有爬取的url集合 self.old_urls = set() # 已经爬取的url集合 def has_new_url(self): ''' 判断是否有没有爬取的url :return: ''' return self.new_url_size() != 0 def get_new_url(self): ''' 获取一个未爬取的url :return: ''' new_url = self.new_urls.pop() self.old_urls.add(new_url) return new_url def add_new_url(self, url): ''' 添加新的url到未爬取的url集合中 :param url: url地址 :return: ''' if url is None: return if url not in self.new_urls and url not in self.old_urls: self.new_urls.add(url) def add_new_urls(self, urls): ''' 添加新的url到未爬取的url集合中 :param urls: url集合 :return: ''' if urls is None or len(urls) == 0: return for url in urls: self.add_new_url(url) def new_url_size(self): ''' 获取未爬取的url集合大小 :return: ''' return len(self.new_urls) def old_url_size(self): ''' 获取已经爬取的url集合大小 :return: ''' return len(self.old_urls)
downloadhtml.py 下载html网页内容
# -*- encoding=UTF-8 -*- ''' author:vfast name:downloadhtml data:2021/6/25 ''' import requests class HtmlDownload(object): def download(self, url): if url is None: return None user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0' headers = {'User-Agent': user_agent} r = requests.get(url, headers=headers) if r.status_code == 200: r.encoding = 'utf-8' return r.text return None
exparehtml.py 解析HTML网页
# -*- encoding=UTF-8 -*- ''' author:vfast name:exparehtml data:2021/6/25 ''' import re import urlparse from bs4 import BeautifulSoup class Exparsehtml(object): def parser(self, page_url, html_cont): ''' 用于解析网页内容,抽取url和数据 :param page_url: 下载页面的url :param html_cont: 下载的网页内容 :return: 返回url和数据 ''' if page_url is None or html_cont is None: return soup = BeautifulSoup(html_cont, 'html.parser') new_urls = self._get_new_urls(page_url, soup) new_data = self._get_new_data(page_url, soup) return new_urls, new_data def _get_new_urls(self, page_url, soup): ''' 抽取新的url集合 :param page_url: 下载页面的url :param soup: :return: 返回新的集合 ''' new_urls = set() # 抽取符合要求的a标签 links = soup.find_all('a', href=re.compile(r'/view/\d+\.htm')) for link in links: new_url = link['href'] new_full_url = urlparse.urljoin(page_url, new_url) new_urls.add(new_full_url) return new_urls def _get_new_data(self, page_url, soup): ''' 抽取有效的数据 :param page_url: 下载页面的url :param soup: :return: 返回有效数据 ''' data = {} data['url'] = page_url title = soup.find('dd', class_='lemmaWgt-lemmaTitle-title').find('h1') data['title'] = title.get_text() summary = soup.find('div', class_='lemma-summary') data['summary'] = summary.get_text() return data
dataoutput.py 保存数据到文件
# -*- encoding=UTF-8 -*- ''' author:vfast name:dataoutput data:2021/6/28 ''' import codecs class DataOutput(object): def __init__(self): self.datas = [] def store_data(self, data): if data is None: return self.datas.append(data) def output_html(self): fout = codecs.open('baike.html', 'w', encoding='utf-8') fout.write("<html>") fout.write("<body>") fout.write("<table>") for data in self.datas: fout.write("<tr>") fout.write("<td>%s</td>" % data['url']) fout.write("<td>%s</td>" % data['title']) fout.write("<td>%s</td>" % data['summary']) fout.write("</tr>") self.datas.remove(data) fout.write("</table>") fout.write("</body>") fout.write("</html>") fout.close()
来源:Python爬虫开发与项目实战,没有地址记录一下