json模块
json模块是python内置的库,在使用的时候直接导入就可以。
json模块只有四个方法dump()、load()、dumps()、loads()。
json模块的主要功能是将序列化数据从文件里读取出来或者存入文件。
- dump()是将数据存入文件中
- load()是用于读取文件。
- dumps()是将python对象编码成json字符串
- loads()是将json字符串解码成python对象。
json.dumps()和json.dump()的区别
json.dumps() 是把python对象转换成json对象的一个过程,生成的是字符串。
json.dump() 是把python对象转换成json对象生成一个fp的文件流,和文件相关。
json.loads()和json.load()的区别同上类似,加上s的方法是用来处理字符串类型的,而不加s是用来处理文件类型的。
练手:利用函数爬取网易云音乐评论
import requests
import json
def get_hot_comments(res):
comments_json=json.loads(res.text)
hot_comments=comments_json['hotComments']
with open('hot_comments.txt','w',encoding='utf-8') as file:
for each in hot_comments:
file.write(each['user']['nickname']+':\n\n')
file.write(each['content']+'\n')
file.write('--------------------------------------\n')
def get_comments(url):
headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.125 Safari/537.36',
'referer':'http://music.163.com/song?id=531051217'}
params="GYQgMlbQaJ8ASN378SmMrKAH5bQ0KEgMclFN7wB5HIOiKjrlFOsymderYVvaixDpZCgsPQJ/T1r2LZuyS7RKPSBJhZueXZyUQCRUp5zY5bsPXGsrA4Gu4dVKEpwdZ5EavnDHMNxtGD9t34JVUtnUu0oJ4Qq2QedbMeLJ28f7c0M64uYpkCdiVwa9Qh67BWuy"
encSecKey = "48e73058832c006ee54acb66fecbdab52ac9b47ae9f956003f99082c0cdf44de57630417119cc32e7940f31aed4d189b3d2f5780bd497ee3d392a50b66867c48eb87a8ef5389379a0cb9b8cfe6f25ef2445143b70eabc831eb9918efa5d38c5d92e1386cc114b549f1fa88d21a06de3e556955d506434be440e2c6485991656b"
data={"params":params,"encSecKey":encSecKey}
name_id = url.split('=')[1]
target_url= "http://music.163.com/weapi/v1/resource/comments/R_SO_4_{}?csrf_token=".format(name_id)
res=requests.post(target_url,headers=headers,data=data)
return res
def main():
url=input("请输入链接地址:")
res=get_comments(url)
get_hot_comments(res)
if __name__=="__main__":
main()
小项目3:爬取网易云音乐评论
import requests
import json
__name__=="__main__"
headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.125 Safari/537.36',
'referer':'http://music.163.com/song?id=531051217'}
params = "GYQgMlbQaJ8ASN378SmMrKAH5bQ0KEgMclFN7wB5HIOiKjrlFOsymderYVvaixDpZCgsPQJ/T1r2LZuyS7RKPSBJhZueXZyUQCRUp5zY5bsPXGsrA4Gu4dVKEpwdZ5EavnDHMNxtGD9t34JVUtnUu0oJ4Qq2QedbMeLJ28f7c0M64uYpkCdiVwa9Qh67BWuy"
encSecKey = "48e73058832c006ee54acb66fecbdab52ac9b47ae9f956003f99082c0cdf44de57630417119cc32e7940f31aed4d189b3d2f5780bd497ee3d392a50b66867c48eb87a8ef5389379a0cb9b8cfe6f25ef2445143b70eabc831eb9918efa5d38c5d92e1386cc114b549f1fa88d21a06de3e556955d506434be440e2c6485991656b"
data = {"params": params, "encSecKey": encSecKey}
url=input("请输入链接地址:")
name_id = url.split('=')[1]
target_url = "http://music.163.com/weapi/v1/resource/comments/R_SO_4_{}?csrf_token=".format(name_id)
res = requests.post(target_url, headers=headers, data=data)
comments_json=json.loads(res.text)
hot_comments=comments_json['hotComments']
with open('hot_comments.txt','w',encoding='utf-8') as file:
for each in hot_comments:
file.write(each['user']['nickname']+':\n\n')
file.write(each['content']+'\n')
file.write('--------------------------------------\n')