基本步骤:
1.明确目的
2.找到数据对应的网页,分析网页结构找到数据所在的标签位置
3.模拟HTTP请求,向服务器发送这个请求,获取服务器返回的HYML
4.用正则表达式提取 所需数据(主播名、人气)
5.精炼数据、处理数据、存储数据
目的:
爬取熊猫直播英雄联盟主播人气排行
原理:
对HTML文件进行文本分析,从而提取出所需数据
分析网页结构:
数据的获取与处理
import re
from urllib import request
class Spider():
url = 'https://www.panda.tv/cate/lol?pdt=1.24.s1.3.7o5937s6suv'
root_pattern = '<div class="video-info">([\s\S]*?)</div>'
name_pattern = '</i>([\s\S]*?)</span>'
number_pattern = '<span class="video-number">([\s\S]*?)</span>'
# 私有方法,获取数据
def __fetch_content(self):
r = request.urlopen(Spider.url)
htmls = r.read()
htmls = str(htmls, encoding='utf-8')
return htmls
# 正则分析html
def __analysis(self,htmls):
root_html=re.findall(Spider.root_pattern,htmls)
anchors = []
for html in root_html:
name = re.findall(Spider.name_pattern, html)
number = re.findall(Spider.number_pattern, html)
anchor = {'name':name, 'number':number}
anchors.append(anchor)
return anchors
# 精炼数据
def __refine(self,anchors):
l = lambda anchor:{'name':anchor['name'][0].strip(),'number':anchor['number'][0]}
return map(l, anchors)
# 数据排序
def __sort(self,anchors):
anchors = sorted(anchors,key = self.__sort_seed,reverse=True)
return anchors
def __sort_seed(self,anchor):
r = re.findall('\d*',anchor['number'])
number = float(r[0])
if '万' in anchor['number']:
number *=10000
return number
# 显示函数
def __show(self,anchors):
for rank in range(0,len(anchors)):
print('rank'+str(rank+1)+' : '+anchors[rank]['name'] + ' '+anchors[rank]['number'])
# 入口方法(主方法)
def go(self):
htmls = self.__fetch_content()
anchors = self.__analysis(htmls)
result = list(self.__refine(anchors))
sorted_result = self.__sort(result)
self.__show(sorted_result)
spider = Spider()
spider.go()
结果: