03爬虫-jsonpath模块的使用

jsonpath模块的使用场景

如果有一个多层嵌套的复杂字典，想要根据key和下标来批量提取value，这是比较困难的。jsonpath模块就能解决这个痛点

from jsonpath import jsonpath

data={'key1':{'key2':{'key3':{'key4':{'key5':{'key6':'python'}}}}}}

#普通通过字典索引
#print(data['key1']['key2']['key3']['key4']['key5']['key6'])

#通过jsonpath索引
# $：根节点   .：取子节点     ..：不管位置，选择所有符合条件的条件
print(jsonpath(data,'$..key6')[0])
#在根节点的{}下，直接找名字为key6的子节点

例子

#coding:utf-8
import jsonpath
import json

#将一个json形式的字符串赋值为book_dict变量
book_dict = '''{ 
  "store": {
    "book": [ 
      { "category": "reference",
        "author": "Nigel Rees",
        "title": "Sayings of the Century",
        "price": 8.95
      },
      { "category": "fiction",
        "author": "Evelyn Waugh",
        "title": "Sword of Honour",
        "price": 12.99
      },
      { "category": "fiction",
        "author": "Herman Melville",
        "title": "Moby Dick",
        "isbn": "0-553-21311-3",
        "price": 8.99
      },
      { "category": "fiction",
        "author": "J. R. R. Tolkien",
        "title": "The Lord of the Rings",
        "isbn": "0-395-19395-8",
        "price": 22.99
      }
    ],
    "bicycle": {
      "color": "red",
      "price": 19.95
    }
  }
}'''

#将变量book_dict中的字符串通过json.loads()转换成字典的格式赋值给data
data = json.loads(book_dict)

#通过jsonpath模块中的jsonpath方法将字典中的color对应的值打印出来
print(jsonpath.jsonpath(data, "$..color"))
print(jsonpath.jsonpath(data, "$..price"))

jsonpath模块在多级节点下例子

#coding:utf-8
import requests
import jsonpath
import json

headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
}

#访问url并返回json格式的字符串
response = requests.get('https://www.lagou.com/lbs/getAllCitySearchLabels.json', headers=headers)

#将返回的json格式字符串转换为字典格式
dict_data = json.loads(response.content)

#通过jsonpath查找根节点下的A节点下的参数name对应的值
print(jsonpath.jsonpath(dict_data,'$..A..name'))