python爬虫一

爬取斗鱼直播（博主姓名和热度）

from urllib import request
import requests
from bs4 import BeautifulSoup
import re
class Spider():
    url='https://www.douyu.com/g_LOL'
    headers={
    'User-Agent': 'Mozilla/5.0 (Windows NT'
                 ' 10.0; Win64; x64) AppleWebKi'
                 't/537.36 (KHTML, like Gecko) '
                 'Chrome/64.0.3282.140 Safari/537.3'
                 '6 Edge/18.17763'}
    root_pattern='<div class="DyListCover' \
                 '-info">[\s\S]*?</h2></div>'
    name_pattern='<use xlink:href="#icon-user_c95a' \
                 'cf8"></use></svg>(.*?)</h2></div>'
    number_pattern='<use xlink:href="#icon-hot_8a57f0b"></' \
                   'use></svg>(.*?)</span>'

    def __fetch_content(self):
        htmls=requests.get(Spider.url,headers=Spider.headers)
        htmls=htmls.text
        return htmls

    def _analysis(self,htmls):
        root_html=re.findall(Spider.root_pattern,htmls)

        anchors=[]
        for html in root_html:
            name=re.findall(Spider.name_pattern,html)
            number=re.findall(Spider.number_pattern,html)
            anchor={'name':name,'number':number}
            anchors.append(anchor)

        return anchors

    def __refine(self,anchors):
        l=lambda anchor:{'name':anchor['name'][0].strip(),
                         'number':anchor['number'][0]
                         }
        return map(l,anchors)

    def __sort(self,anchors):
         anchors=sorted(anchors,key=self.__sort_seed,reverse=True)
         return anchors

    def __sort_seed(self,anchor):
        r=re.findall('\d*',anchor['number'])
        number=float(r[0])
        if '万' in anchor['number']:
            number*=10000
        return number

    def __show(self,anchors):
        for rank in range(0,len(anchors)):
            print('rank '+str(rank+1)+' : '+
                  anchors[rank]['name']+'~~~~~~'
                  +anchors[rank]['number'])

    def go(self):
        htmls=self.__fetch_content()
        anchors=self._analysis(htmls)
        anchors=self.__refine(anchors)
        anchors=self.__sort(anchors)
        self.__show(anchors)

spider=Spider()
spider.go()