美团app人气榜切片数据逐段提取

# 仅需修改这个地方https://jn.lianjia.com/ershoufang/pg{}rs/   将jn换成你所在城市的拼写首字母小写
import requests
from lxml import etree
import time
import random
import csv
import requests
import json

class LianjiaSpider(object):
    def __init__(self):
        self.url = "https://mobilenext-web.meituan.com/api/newSalesBoard/getSaleBoardDetail?cityId=96&boardType=9&districtId=0&cateId=1&offset=0&limit=15&lat=36.526046191159445&lng=122.062217811"

        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1"}

    def get_page(self, url):
        res = requests.get(url=url, headers=self.headers)
        res.encoding = "utf-8"
        html = res.text
        print(html)


        results = html.replace('{"totalSize":50,"saleBoardDealList":', "").replace("}", "")

        # {"status":0,"data":[["\u4e0a\u6d77","205293","0.1090","0.5045","0.3865"],["\u4fdd\u5b9a","178489","0.3649","0.6351","0.0000"],["\u91cd\u5e86","165672","0.0082","0.2715","0.7204"],["\u5eca\u574a","138381","0.5009","0.4991","0.0000"],["\u957f\u6c99","136803","0.0878","0.5661","0.3461"],["\u5929\u6d25","129458","0.2654","0.7346","0.0000"],["\u676d\u5dde","98526","0.0968","0.5175","0.3857"],["\u6b66\u6c49","89813","0.0912","0.7516","0.1572"],["\u5f20\u5bb6\u53e3","77457","0.3326","0.6674","0.0000"],["\u77f3\u5bb6\u5e84","73055","0.2583","0.7417","0.0000"]]}

        # print(results)
        for item in json.loads(results):
            print(item)

        # 调用解析函数
     #   self.parse_page(html)


    def main(self):
        try:
            for i in range(1, 2):
                time.sleep(random.randint(3, 5))
                url = self.url.format(i)
                self.get_page(url)
        except:
            self.main()


if __name__ == '__main__':
    start = time.time()
    spider = LianjiaSpider()
    spider.main()
    end = time.time()
    print("执行时间：%.2f" % (end - start))





'''


# 通传入日期和身份号编码获取数据
def get_data():
    url = "https://mobilenext-web.meituan.com/api/newSalesBoard/getSaleBoardDetail?cityId=96&boardType=9&districtId=0&cateId=1&offset=0&limit=15&lat=36.526046191159445&lng=122.062217811"
    res = requests.get(url)
    return res.text

#date_list=["2019-10-01","2019-10-02","2019-10-03","2019-10-04","2019-10-05","2019-10-06","2019-10-07"]

#for date in date_list:
res_text = get_data()
 #   print(res_text)
results = res_text.replace('{"totalSize":50,"saleBoardDealList":', "").replace("}", "")



#{"status":0,"data":[["\u4e0a\u6d77","205293","0.1090","0.5045","0.3865"],["\u4fdd\u5b9a","178489","0.3649","0.6351","0.0000"],["\u91cd\u5e86","165672","0.0082","0.2715","0.7204"],["\u5eca\u574a","138381","0.5009","0.4991","0.0000"],["\u957f\u6c99","136803","0.0878","0.5661","0.3461"],["\u5929\u6d25","129458","0.2654","0.7346","0.0000"],["\u676d\u5dde","98526","0.0968","0.5175","0.3857"],["\u6b66\u6c49","89813","0.0912","0.7516","0.1572"],["\u5f20\u5bb6\u53e3","77457","0.3326","0.6674","0.0000"],["\u77f3\u5bb6\u5e84","73055","0.2583","0.7417","0.0000"]]}

# print(results)
for item in json.loads(results):
    print(item)
    
    
'''