python爬虫笔记四：大众点评店铺信息（字体反爬-静态映射）

https://jia666666.blog.csdn.net/article/details/108885263

里面讲的非常详细了，不过点评有改动，里面的代码也要相应的改动一下

#coding:utf-8
import re
import requests
from fontTools.ttLib import TTFont  #coding:utf-8

# font = TTFont('01.woff')  # 打开本地字体文件01.ttf
# font.saveXML('01.xml')  # 将ttf文件转化成xml格式并保存到本地，主要是方便我们查看内部数据结构

class Public_Spider():
    def __init__(self):
        self.url = "https://www.dianping.com/wuhan/ch10"  #目标URL
        self.head = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Cache-Control': 'max-age=0',
            'Connection': 'keep-alive',
            #cookie需要更换为你登录的Cookie
            'Cookie': 's_ViewType=10; _lxsdk_cuid=17b5c1c509cc8-02cf378855e65a-4373266-240000-17b5c1c509cc8; _lxsdk=17b5c1c509cc8-02cf378855e65a-4373266-240000-17b5c1c509cc8; _hc.v=236ecca1-8314-7192-b727-b6fa4cbed455.1629337965; Hm_lvt_602b80cf8079ae6591966cc70a3940e7=1629337965; fspop=test; cy=16; cye=wuhan; Hm_lpvt_602b80cf8079ae6591966cc70a3940e7=1629342377; _lxsdk_s=17b5c1c509d-0b-434-d5%7C%7C333',
            'Host': 'www.dianping.com',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36',
        }

    def Get_html(self):
        response = requests.get(self.url, headers=self.head).content.decode('utf-8')
        print(response)
        #去掉空格
        self.html = re.sub('\s', '', response)

    #获取字体
    def Get_TTF(self):
        #获取所有的CSS
        #<!-- 图文混排css -->
        #<link rel="stylesheet" type="text/css" href="//s3plus.meituan.net/v1/mss_0a06a471f9514fc79c981b5466f56b91/svgtextcss/cb675fe86e75080961c3c43fedc513ab.css">
        TTF = re.findall('<linkrel="stylesheet"type="text\/css"href="(.*?)">', self.html, re.S)[1]  # 缩小范围，店铺信息
        header = {
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36", }
        res=requests.get('http:'+TTF,headers=header).text                                #请求css

        TTF_dict={}                     #字体字典
        fontlist=re.findall('@font-face{(.*?)}',res,re.S)                #缩小范围

        for font in fontlist:
            TTF_name=re.findall('font-family: "PingFangSC-Regular-(.*?)"',font,re.S)[0] #TTF的类别
            TTF_link=re.findall(',url\("(.*?)"\);',font,re.S)[0]
            TTF_dict.update({TTF_name:TTF_link})
        shopNum=TTF_dict.get('shopNum')         #店铺ttf
        tagName=TTF_dict.get('tagName')         #标签ttf


        b = requests.get('http:'+shopNum, headers=header).content
        with open('大众点评shopNum.ttf', 'wb')as f:
            f.write(b)

        b = requests.get('http:'+tagName, headers=header).content
        with open('大众点评tagName.ttf', 'wb')as f:
            f.write(b)

        font1 = TTFont('大众点评shopNum.ttf')  # 打开本地字体文件
        uni_list1 = font1.getGlyphOrder()[2:]  # 获取所有编码，去除前2个
        self.utf8List = ['&#x' + uni[3:] for uni in uni_list1]

        font2 = TTFont('大众点评tagName.ttf')  # 打开本地字体文件01.ttf
        uni_list2 = font2.getGlyphOrder()[2:]  # 获取所有编码，去除前2个
        self.utf8List2 = ['&#x' + uni[3:] for uni in uni_list2]

        self.wordlist = '1234567890店中美家馆小车大市公酒行国品发电金心业商司超生装园场食有新限天面工服海华水房饰城乐汽香部利子老艺花专东肉菜学福饭人百餐茶务通味所山区门药银农龙停尚安广鑫一容动南具源兴鲜记时机烤文康信果阳理锅宝达地儿衣特产西批坊州牛佳化五米修爱北养卖建材三会鸡室红站德王光名丽油院堂烧江社合星货型村自科快便日民营和活童明器烟育宾精屋经居庄石顺林尔县手厅销用好客火雅盛体旅之鞋辣作粉包楼校鱼平彩上吧保永万物教吃设医正造丰健点汤网庆技斯洗料配汇木缘加麻联卫川泰色世方寓风幼羊烫来高厂兰阿贝皮全女拉成云维贸道术运都口博河瑞宏京际路祥青镇厨培力惠连马鸿钢训影甲助窗布富牌头四多妆吉苑沙恒隆春干饼氏里二管诚制售嘉长轩杂副清计黄讯太鸭号街交与叉附近层旁对巷栋环省桥湖段乡厦府铺内侧元购前幢滨处向座下澩凤港开关景泉塘放昌线湾政步宁解白田町溪十八古双胜本单同九迎第台玉锦底后七斜期武岭松角纪朝峰六振珠局岗洲横边济井办汉代临弄团外塔杨铁浦字年岛陵原梅进荣友虹央桂沿事津凯莲丁秀柳集紫旗张谷的是不了很还个也这我就在以可到错没去过感次要比觉看得说常真们但最喜哈么别位能较境非为欢然他挺着价那意种想出员两推做排实分间甜度起满给热完格荐喝等其再几只现朋候样直而买于般豆量选奶打每评少算又因情找些份置适什蛋师气你姐棒试总定啊足级整带虾如态且尝主话强当更板知己无酸让入啦式笑赞片酱差像提队走嫩才刚午接重串回晚微周值费性桌拍跟块调糕'


    def RE_html(self):
        shop_info = re.findall('<liclass="">(.*?)<\/li>', self.html, re.S)  # 缩小范围，店铺信息

        for i in range(len(shop_info)):
            shop_name = re.findall('data-name="(.*?)"', shop_info[i], re.S)  # 店铺名称
            shop_goods = re.findall('target="_blank">(.*?)<\/a>', shop_info[i], re.S)  # 店铺推荐菜
            coment = re.findall('<b>(.*?)<\/b>条评价', shop_info[i], re.S)[0]  # 店铺评价数目
            per_capita = re.findall('人均<b>(.*?)<\/b', shop_info[i], re.S)[0]  # 人均价格

            tag = '|'.join(re.findall('<spanclass="tag">(.*?)<\/span>', shop_info[i], re.S))  # 标签

            for i in range(10):
                coment = coment.replace(self.utf8List[i], self.wordlist[i])  # 字体映射
                per_capita = per_capita.replace(self.utf8List[i], self.wordlist[i])  # 字体映射
            shop_comment = (''.join(re.findall('\d', coment, re.S)))  # 评价数目
            per_capita = (''.join(re.findall('\d', per_capita, re.S)))  # 人均消费


            for i in range(len(self.utf8List2)):
                if self.utf8List2[i] in tag:
                    tag = tag.replace(self.utf8List2[i], self.wordlist[i])
            tag = tag.split('|')

            Tag = []  # 标签列表
            for tag in tag:
                Tag.append(''.join(re.findall('[\u4e00-\u9fa5]', tag, re.S)))#确实只匹配中文
            Tag = ' | '.join(Tag)
            print(
                '*' * 50 + '\n' +
                str(shop_name[0]) + '\n'+
                shop_comment + '条评价\t' + '人均￥' + str(per_capita) + '\n'+
                str(Tag) + '\n' + #str(shop_address[0])+
                '推荐菜:' + str(','.join(shop_goods))
            )

    def run(self):
        self.Get_html()
        self.Get_TTF()
        self.RE_html()

if __name__ == '__main__':
    example=Public_Spider()
    example.run()

结果

 对应的转换后的字体xml: