# HTML解析器
import re
from bs4 import BeautifulSoup
import urllib.parse
class HtmlParser(object):
def parse(self, page_url, html_cont):
if page_url is None or html_cont is None:
return
soup = BeautifulSoup(html_cont, 'html.parser', from_encoding='utf-8')
new_urls = self._get_new_urls(page_url, soup)
return new_urls
def _get_new_urls(self, page_url, soup): # 解析URL列表
new_urls = set()
links = soup.find_all('span', class_="link_title")
for link in links:
new_url = link.a['href']
new_full_url = urllib.parse.urljoin(page_url, new_url) # 以page_url的格式完整new_url
new_urls.add(new_full_url)
return new_urls
def _get_new_data(self, page_url, soup): # 解析数据
res_data = {}
# url
res_data['url'] = page_url
# <dd class="lemmaWgt-lemmaTitle-title"><h1>Python</h1>
title_node = soup.find('span', class_="link_title") # 标题
res_data['title'] = title_node.a.get_text()
summary_node = soup.find('div', class_="markdown_views") # 正文
if(len(summary_node.get_text()) ==0 ):
return None
res_data['summary'] = summary_node.get_text()
return res_data
def parse_data(self, page_url, html_cont):
if page_url is None or html_cont is None:
return
soup = BeautifulSoup(html_cont, 'html.parser', from_encoding='utf-8')
new_data = self._get_new_data(page_url, soup)
return new_data
一个最简单的爬虫-HTML解析器
最新推荐文章于 2024-07-03 20:27:38 发布
本文将引导您逐步创建一个简单的HTML爬虫,学习如何解析网页内容,提取关键信息。通过这个过程,您将理解HTTP请求、DOM遍历以及数据提取的基本原理。
摘要由CSDN通过智能技术生成