爬虫小操作

Pert-

已于 2022-08-18 13:24:45 修改

阅读量84

点赞数

文章标签：爬虫 python json

于 2022-07-19 11:23:54 首次发布

本文链接：https://blog.csdn.net/s_frozen/article/details/125868399

版权

将reponse内容显示为中文，再将其str转为dict

res=response.content.decode("unicode_escape")

res_dict=json.loads(res)

获取token(看是localStorage还是SessionStorage)，需要sleep一会等待token传过来才可以获取到

token = self.driver.execute_script('return localStorage.getItem("token");')
print('\ntoken为：%s\n'%token)

获取cookie

#获取cookie
cookie_tag = self.driver.get_cookies()
# print(cookie_tag)
#处理cookie成需要的格式
cookie = cookie_tag[0]['name'] + '=' + cookie_tag[0]['value']
print('\ncookie为：%s\n'%cookie)

读取、写入文件乱码

###读取
f=open('{}'.format(file_catalogue+'/'+file_list[0]),mode='r',encoding='utf-8',errors='ignore')
contentJson=json.load(f)
print(contentJson)


###写入
with open('student.json',mode='w',encoding='utf8') as f:
	json.dump(jsonA,f,ensure_ascii=False,index=2)  #ensure_ascii=False不让汉字转为ASCII
	#f.write(jsonA)  #不允许用普通文件的方式写入  因为后缀是json

将字典更方便查看

print(json.dumps(dict_data,indent=1,ensure_ascii=False))

将爬下来的数据转为python类型

json.loads(str(response.text),strict=False)

requests.session

session=requests.session()

headers={
	'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36'
}

data={
	'TxtUserName': '宜康养老',
	'TxtUserPassword': '888888'
}
url='https://www.miwitracker.com/Login/SignIn'
response=session.post(url=url,data=data,headers=headers)



data={
	'PageIndex': 1,
	'PageSize': 100000,
}
url='https://www.miwitracker.com/Exception/ExceptionMessage'
res=session.post(url=url,data=data,headers=headers)

print(res.text)

selenium获取cookie 使用requests实现登录

from selenium import webdriver
import time

# 初始化驱动
driver = webdriver.Chrome()

##窗口最小化
driver.maximize_window()

##请求页面
driver.get(url='https://www.miwitracker.com/zh-cn')


#输入账号
driver.find_element_by_xpath('//*[@id="TxtUserName"]').send_keys('宜康养老')
#输入密码
driver.find_element_by_xpath('//*[@id="txtUserPassword"]').send_keys('888888')

##点击登录
driver.find_element_by_xpath('//*[@id="btn-login"]').click()

time.sleep(1)


get_cookies = driver.get_cookies()[0]

cookies=get_cookies['name']+'='+get_cookies['value']


import requests

url='https://www.miwitracker.com/Exception/ExceptionMessage'

headers={
	'cookie':cookies,
	'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36'
}

data={
	'PageIndex': 1,
	'PageSize': 100000,
}
res=requests.post(url=url,data=data,headers=headers)
print(res.text)