爬虫---爬取大众点评

使用命令行创建scrapy项目
1、####ping.py

# -*- coding: utf-8 -*-
import os
import random
import re

import requests
import scrapy
# from fake_useragent import UserAgent
# ua = UserAgent()
from lxml import etree

from dianping.items import shop
from dianping.settings import user_agent
from fontTools.ttLib import TTFont

class PingSpider(scrapy.Spider):
    name = 'ping'
    allowed_domains = ['https://www.dianping.com/']
    start_urls = ['https://www.dianping.com/beijing/ch10']
    custom_settings = {
   
        'ITEM_PIPELINES': {
   'dianping.pipelines.shopPipelines': 301},
    }
    headers = {
   
        'lgtoken': '0897479c0-088d-4343-a98f-82bff8f7bf23',
        'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        'Connection': 'keep-alive',
        'Cookie': '_lxsdk_cuid=170b78cc76fc8-0a5b2ec62d67f4-4313f6a-144000-170b78cc76fc8; _lxsdk=170b78cc76fc8-0a5b2ec62d67f4-4313f6a-144000-170b78cc76fc8; _hc.v=32cef2f6-b6e1-4f3f-52cc-f7f971670e42.1583627422; s_ViewType=10; ua=dpuser_8153713552; ctu=307ee90b2640a3fdfb7d5160e160f22434ac9fe3475d11d27394f1c12f34fba0; cy=3592; cye=zhongningxian; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; _lxsdk_s=170c46808f8-d46-74c-84b%7C%7C65',
        'Host': 'www.dianping.com',
        'Referer': 'http://www.dianping.com/zhongningxian/ch10',
        'User-Agent': random.choice(user_agent),
        'X-Requested-With': 'XMLHttpRequest'
    }
    cookies = {
   
        "s_ViewType": "10",
        " _lxsdk_cuid": "16e48a565e1c8-04659e09b895fe-e343166-144000-16e48a565e373",
        " _lxsdk": "170c3969300-358-fb-b16%7C%7C140",
        " _hc.v": "43db2bd6-f5d5-fee9-59df-981a1d70742c.1573176830"
    }
    fontDir={
   }
    def get_font(self,filePath):
        # 处理数字
        font = TTFont(filePath)
        font_names = font.getGlyphOrder()
        # 这些文字就是在FontEditor软件打开字体文件后看到的文字名字
        texts = ['', '', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0']
        font_name = {
   }
        # 将字体名字和它们所对应的乱码构成一个字典
        for index, value in enumerate(texts):
            a = font_names[index].replace('uni'