fiddler移动端抓取

夜神模拟器配置

fiddler设置

• 选择这个选项

• 打开这个浏览器

• 直接点击继续
• 提示有证书问题，我们来安装证书，点击继续

• 输入我们的ip地址

总体过程

代码爬取豆果美食

• 所要携带的信息

header = {
    "client": "4",
    "version": "6922.2",
    "device": "MI 6",
    "sdk": "19,4.4.2",
    "imei": "863254010448503",
    "channel": "qqkp",
    "mac":"44:85:00:5E:5B:28",
    "resolution": "720*1280",
    "dpi": "1.5",
    "android-id":"4485005e5b281516",
    "pseudo-id":"05e5b28151644850",
    "brand": "Xiaomi",
    "scale": "1.5",
    "timezone": "28800",
    "language": "zh",
    "cns": "3",
    "carrier": "CMCC",
    "imsi":"460074485009491",
    "user-agent": "Mozilla/5.0 (Linux; Android 4.4.2; MI 6  Build/NMF26X) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/30.0.0.0 Mobile Safari/537.36",
    "reach": "1",
    "newbie": "1",
    "lon":"116.568176",
    "lat":"26.997867",
    "cid":"361000",
    "Content-Type": "application/x-www-form-urlencoded; charset=utf-8",
    "Accept-Encoding": "gzip, deflate",
    "Connection": "Keep-Alive",
    "Cookie":"duid=57158696",
    "Host": "api.douguo.net",
    "Content-Length":"68",
}

data = {
    "client": "4",
    "_session": "1537295931652863254010448503",
    "v": "1503650468",
    "_vs": "2305",
}

data_2 = {
    "client": "4",
    "_session": "1537295931652863254010448503",
    "keyword": item['name'],
    "order": "3",
    "_vs": "400",
}

detail_data = {
    "client": "4",
    # "_session": "1537295931652863254010448503",
    "author_id": "0",
    "_vs": "2803",
    "_ext": '{"query":{"id":' + str(caipu_info['shicai_id']) + ',"kw":' + caipu_info[
        'shicai'] + ',"idx":"4","src":"2803","type":"13"}}',
}

• 爬取代码

# @Time    : 2021/3/24 20:46
# @Author  : Jerry
# @File    : 豆果美食.py
# @Software: PyCharm

import requests
import json
from multiprocessing import Queue
from handle_mongo import mongo_info

queue_list = Queue()

def handel_request(url,data):
	#移动端请求头的信息
    header = {
        "client": "4",
        "version": "6922.2",
        "device": "MI 6",
        "sdk": "19,4.4.2",
        "imei": "863254010448503",
        "channel": "qqkp",
        "mac": "44:85:00:5E:5B:28",
        "resolution": "720*1280",
        "dpi": "1.5",
        # "android-id": "4485005e5b281516",
        # "pseudo-id": "05e5b28151644850",
        "brand": "Xiaomi",
        "scale": "1.5",
        "timezone": "28800",
        "language": "zh",
        "cns": "3",
        "carrier": "CMCC",
        # "imsi": "460074485009491",
        "user-agent": "Mozilla/5.0 (Linux; Android 4.4.2; MI 6  Build/NMF26X) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/30.0.0.0 Mobile Safari/537.36",
        "reach": "1",
        "newbie": "1",
        "lon": "116.568176",
        "lat": "26.997867",
        "cid": "361000",
        "Content-Type": "application/x-www-form-urlencoded; charset=utf-8",
        "Accept-Encoding": "gzip, deflate",
        "Connection": "Keep-Alive",
        # "Cookie": "duid=57158696",
        "Host": "api.douguo.net",
        # "Content-Length": "68",
    }
    response = requests.post(url=url,headers=header,data=data)
    return response

def handle_index():
    url = 'http://api.douguo.net/recipe/flatcatalogs'

    data = {
        "client": "4",
        # "_session": "1537295931652863254010448503",
        # "v": "1503650468",
        "_vs": "2305",
    }
    response = handel_request(url=url,data=data)
    index_response_dict = json.loads(response.text) # 数据转换
    for index_item in index_response_dict['result']['cs']:
        for index_item_1 in index_item['cs']:
            for item in index_item_1['cs']:
                data_2 = {
                    "client": "4",
                    # "_session": "1537295931652863254010448503",
                    "keyword": item['name'],
                    "order": "3",
                    "_vs": "400",
                }
                queue_list.put(data_2)

def handle_caipu_list(data):
    print('当前处理的食材:',data['keyword'])
    caipu_list_url = 'http://api.douguo.net/recipe/v2/search/0/20'
    caipu_list_response = handel_request(url=caipu_list_url, data=data)
    caipu_list_response_dict = json.loads(caipu_list_response.text)
    for item in caipu_list_response_dict['result']['list']:
        caipu_info = {}
        caipu_info['shicai'] = data['keyword']
        if item['type'] == 13:
            caipu_info['user_name'] = item['r']['an']
            caipu_info['shicai_id'] = item['r']['id']
            caipu_info['describe'] = item['r']['cookstory']
            caipu_info['shicai_name'] = item['r']['n']
            caipu_info['zuoliao_list'] = item['r']['major']
            detail_url = 'http://api.douguo.net/recipe/detail/'+str(caipu_info['shicai_id'])
            detail_data = {
                "client": "4",
                # "_session": "1537295931652863254010448503",
                "author_id": "0",
                "_vs": "2803",
                "_ext": '{"query":{"id":' + str(caipu_info['shicai_id']) + ',"kw":' + caipu_info[
                    'shicai'] + ',"idx":"4","src":"2803","type":"13"}}',
            }
            detail_response = handel_request(url=detail_url,data=detail_data)
            detail_response_dict = json.loads(detail_response.text)
            caipu_info['tips'] = detail_response_dict['result']['recipe']['tips']
            caipu_info['cook_step'] = detail_response_dict['result']['recipe']['cookstep']
            print('当前入库的菜谱是:',caipu_info['shicai_name'])
            mongo_info.insert_item(caipu_info)

        else:
            continue
        print(item)



handle_index()
handle_caipu_list(queue_list.get())

• 存入mongo数据库

# @Time    : 2021/3/24 21:47
# @Author  : Jerry
# @File    : handle_mongo.py
# @Software: PyCharm

import pymongo
from pymongo.collection import Collection

class Connect_mongo():

    def __init__(self):
        self.client = pymongo.MongoClient(host='127.0.0.1',port=27017)
        # 定义数据库的名字
        self.db_data = self.client['douguo']

    def insert_item(self,item):
        # 定义表名
        db_collection = Collection(self.db_data,'douguo_item')
        db_collection.insert(item)

mongo_info = Connect_mongo()