Python3原生爬虫获取熊猫直播某一分类下的主播人气并保存到Excel

import re
import openpyxl
from urllib import request

# 断点调试

class Spider:
    url = 'https://www.panda.tv/cate/lol'
    root_pattern = '<div class="video-info">([\s\S]*?)</div>'
    name_pattern = '<span class="video-nickname"[\w\W]*?</i>([\s\S]*?)</span>'
    number_pattern = '<span class="video-number">([\s\S]*?)</span>'
#获取页面
    def __fetch_content(self):
        r = request.urlopen(Spider.url)
        # bytes 将其转成字符串
        htmls = r.read()
        htmls = str(htmls,encoding='utf-8')
        return htmls
#分析
    def __analysis(self, htmls):
        root_html = re.findall(Spider.root_pattern,htmls)
        anchors = []
        for html in root_html:
            name = re.findall(Spider.name_pattern,html)
            number = re.findall(Spider.number_pattern,html)
            anchor = {'name':name,'number':number}
            anchors.append(anchor)
        # print(anchors[0])
        return anchors
#精炼/清洗
    def __refine(self, anchors):
        l = lambda anchor: {
            'name':anchor['name'][0].strip(),
            'number':anchor['number'][0]
        }
        return map(l,anchors)
#排序
    def __sort(self, anchors):
        #py默认排序方法
        anchors = sorted(anchors,key=self.__sort__seed,reverse=True)
        return anchors
#按xx排序
    def __sort__seed(self, anchor):
        r = re.findall('\d*',anchor['number'])
        number = float(r[0])
        if '万' in anchor['number']:
            number *= 10000
        return number
#打印排序完的数据
    def __show(self, anchors):
        for rank in range(0,len(anchors)):
            print('rank ' + str(rank + 1)
                + ' : ' + anchors[rank]['name']
                + '  ' +  anchors[rank]['number']
            )
#导入excel表
    def __import_excel(self,anchors):
        #新建
        info = openpyxl.Workbook()
        sheet = info.active
        sheet.title = 'pandatv_sheet'
        title = ['排名','主播','人气']
        col = 1
        for t in title:
            sheet.cell(1,col,t)
            col += 1
        row = 2
        for anchor in anchors:
            col = 2
            for key,value in anchor.items():
                sheet.cell(row,1,row-1)
                sheet.cell(row,col,value)
                col += 1
            row += 1
        info.save('pandatv_sheet.xlsx')
#入口/总控方法
    def go(self):
        htmls = self.__fetch_content()
        anchors = self.__analysis(htmls)
        anchors = list(self.__refine(anchors))
        anchors = self.__sort(anchors)
        # print(anchors)
        self.__import_excel(anchors)
        # self.__show(anchors)

spider = Spider()
spider.go()

在这里插入图片描述

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值