爬虫--jsonpath学习

jsonpath使用示例

book_dict = { 
  "store": {
    "book": [ 
      { "category": "reference",
        "author": "Nigel Rees",
        "title": "Sayings of the Century",
        "price": 8.95
      },
      { "category": "fiction",
        "author": "Evelyn Waugh",
        "title": "Sword of Honour",
        "price": 12.99
      },
      { "category": "fiction",
        "author": "Herman Melville",
        "title": "Moby Dick",
        "isbn": "0-553-21311-3",
        "price": 8.99
      },
      { "category": "fiction",
        "author": "J. R. R. Tolkien",
        "title": "The Lord of the Rings",
        "isbn": "0-395-19395-8",
        "price": 22.99
      }
    ],
    "bicycle": {
      "color": "red",
      "price": 19.95
    }
  }
}

from jsonpath import jsonpath

print(jsonpath(book_dict, '$..author')) # 如果取不到将返回False # 返回列表，如果取不到将返回False

我们以拉勾网城市JSON文件 http://www.lagou.com/lbs/getAllCitySearchLabels.json 为例，做一些简单的数据查询，并获取所有城市的名字的列表，并写入文件。

参考代码：

#导入jsonpath
from jsonpath import jsonpath
import json
import jsonpath
import requests
from jsonpath_ng import jsonpath, parse
import jsonpath
import parse

header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0'
}
# 获取JSON数据
url = "https://www.lagou.com/lbs/getAllCitySearchLabels.json"
response = requests.get(url,headers=header,verify = False)
# 检查响应状态码
if response.status_code == 200:
    # 如果请求成功，打印响应内容
    print(response.text)
else:
    # 如果请求失败，打印错误信息
    print(f"请求失败，状态码：{response.status_code}")

response =requests.get(url, headers=header)

html_str = response.content.decode()

# 把json格式字符串转换成python对象
jsonobj = json.loads(html_str)

# 从根节点开始，获取所有key为name的值
citylist = jsonpath.jsonpath(jsonobj,'$..name')

print(citylist)

#找到所有以G开头的城市名
print(jsonpath.jsonpath(jsonobj,'$.content.data.allCitySearchLabels.G[*].name'))

#打印所有的城市名
print(jsonpath.jsonpath(jsonobj,'$..name'))

#找到id等于666的城市名
#print(jsonpath.jsonpath(jsonobj,'$..[?(@.id=="666")].name'))
print(jsonpath.jsonpath(jsonobj,'$.content.data.allCitySearchLabels.G[*].name'))

#写入文件
import requests
import jsonpath
import json

# 获取拉勾网城市json字符串
url = 'http://www.lagou.com/lbs/getAllCitySearchLabels.json'
headers = {"User-Agent": "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"}
response =requests.get(url, headers=headers)
html_str = response.content.decode()

# 把json格式字符串转换成python对象
jsonobj = json.loads(html_str)

# 从根节点开始，获取所有key为name的值
citylist = jsonpath.jsonpath(jsonobj,'$..name')

# 写入文件
with open('city_name.txt','w') as f:
    content = json.dumps(citylist, ensure_ascii=False)
    f.write(content)