爬虫笔记(仅作为学习记录,源来源慕课网)
![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/a46953d1b01a501c836e21349f7acf88.png)
![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/ffae376b1d15bcf08116ded192e65b80.png)
爬虫架构中,最为重要的就是图一画框部分,首先准备待爬取网页URL,然后通过网页下载器将目标页面下载下来,之后再将下载页面中自己需要的东西通过网页解析器提取出来。
![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/27574933c8f8975c299d6d1d5070905a.png)
![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/5d8ca4641f7d7ddf0b9a4515bd7ec051.png)
![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/516bb5399983832fd2f42f6709ac690e.png)
![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/4bcad7bee2651d514d6cd3cbaaed3e7a.png)
![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/a647aec33945bed56f64b94ecab40f25.png)
创建Beautiful Soup对象->find/find_all->访问节点名称,属性,文字。
< a href=‘123.html’ class=‘article_link’>python< /a >
节点名称:a
节点属性:href=‘123.html’ class=‘article_link’
节点内容:python
代码部分(Python3,Pycharm):
main.py
import manager
import download
import html_parser
import output
class SpiderMain(object):
def __init__(self):
self.urls = manager.Manager() #管理器
self.download = download.Download() #下载器
self.parser = html_parser.Parser() #解析器
self.output = output.Output() #输出
def craw(self, root_url): #爬虫函数
count = 1
self.urls.add_new_url(root_url) #将URL添加进
while self.urls.has_new_url():
try:
new_url = self.urls.get_new_url() #取出一个新的待爬URL
html_cont = self.download.download(new_url) #将网页下载下来
#print(html_cont)
new_urls, new_data = self.parser.parse(new_url, html_cont) #解析所需数据
self.urls.add_new_urls(new_urls) #将网页中解析出来的新URL添加进库中
self.output.collect_data(new_data)
if count == 1000:
break
count = count + 1
except:
print("failed")
self.output.output_html()
if __name__=="__main__":
root_url = "https://baike.baidu.com/item/Python/407313.htm"
obj_spider = SpiderMain()
obj_spider.craw(root_url)
class Manager(object):
def __init__(self):
self.new_urls = set() #待爬取URL
self.old_urls = set() #已爬取URL
def add_new_url(self, url): #添加新的URL进库
if url is None: #如果URL为空,返回
return
if url not in self.new_urls and url not in self.old_urls: #如果URL既不在待爬取库中也不在已爬取库中,添加进待爬取库
self.new_urls.add(url)
def add_new_urls(self, urls):
if urls is None or len(urls) == 0:
return
for url in urls:
self.add_new_url(url)
def has_new_url(self): #如果待爬取库还有未爬取URL,返回1
return len(self.new_urls) != 0
def get_new_url(self): #取出一个待爬取URL
new_url = self.new_urls.pop()
self.old_urls.add(new_url) #将该URL添加进已爬取库中
return new_url #返回这个URL
import urllib.request
class Download(object):
def download(self, url):
if url is None:
return None
response = urllib.request.urlopen(url) #调用urllib库,请求页面
if response.getcode() != 200:
return None
#print(response.read().decode('utf-8'))
return response.read().decode('utf-8')
下载的页面(部分):
html_parser.py
from bs4 import BeautifulSoup
import re
from urllib.parse import urlparse
class Parser(object):
def _get_new_urls(self, page_url, soup): : #解析出需要的URL
links = soup.find_all('a', href=re.compile(r"/item/")) #利用正则表达式找到页面中所有符合要求的URL
for link in links:
new_url = link['href']
#new_full_url = urlparse.urljoin(page_url, new_url)
new_full_url = "https://baike.baidu.com" + new_url #拼合URL
print(new_full_url)
new_urls.add(new_full_url) #将URL添加进库中
return new_urls
def _get_new_data(self, page_url, soup): #解析出需要的数据
res_data = {}
res_data['url'] = page_url
#<dd class="lemmaWgt-lemmaTitle-title"><h1>Python</h1>
title_node = soup.find('dd', class_="lemmaWgt-lemmaTitle-title").find("h1")
res_data['title'] = title_node.get_text()
#<div class="lemma-summary"
summary_node = soup.find('div', class_="lemma-summary")
res_data['summary'] = summary_node.get_text()
return res_data
def parse(self, page_url, html_cont): #解析函数
if page_url is None or html_cont is None: #如果网页URL为空或者下载的页面为空,返回
return
soup = BeautifulSoup(html_cont, 'html.parser') #调用Beautiful Soup解析页面
new_urls = self._get_new_urls(page_url, soup)
new_data = self._get_new_data(page_url, soup)
return new_urls, new_data
解析出的部分URL:
output.py(实际作用可有可无)
class Output(object):
def __init__(self):
self.datas = []
def collect_data(self, data):
if data is None:
return
self.datas.append(data)
def output_html(self):
fout = open('output.html', 'w', encoding='utf-8')
fout.write("<html>")
fout.write("<body>")
fout.write("<table>")
for data in self.datas:
fout.write("<tr>")
fout.write("<td>%s</td>" % data["url"])
fout.write("<td>%s</td>" % data["title"].encode("utf-8"))
fout.write("<td>%s</td>" % data['summary'].encode("utf-8"))
fout.write("</tr>")
fout.write("</table>")
fout.write("</body>")
fout.write("</html>")
fout.close()
简易版:
按照下载页面->提取内容的简易流程
import urllib.request
import re
from bs4 import BeautifulSoup
root_url = "https://baike.baidu.com/item/Python/407313.htm"
response = urllib.request.urlopen(root_url) # 调用urllib库,请求页面
#print(response.read().decode('utf-8'))
html_cont = response.read().decode('utf-8') #读取页面
soup = BeautifulSoup(html_cont, 'html.parser') #创建Beautiful Soup对象
links = soup.find_all('a', href=re.compile(r"/item/"))
for link in links:
new_url = link['href']
new_full_url = "https://baike.baidu.com" + new_url # 拼合URL
print(new_full_url)