Python：爬虫常用模块：requests（get、post、代理、伪装headers和session）、json、selenium

1、requests

（1）get

import requests
url = "http://www.baidu.com"
response = requests.get(url)
if response.status_code == 200:
	print(response.text)

（2）post

import requests
url = 'http://www.renren.com/PLogin.do'
data = {'email': '18500344808', 'password': '196833'}
response = requests.post(url, data = data)
if response.status_code == 200:
	print(response.text)

（3）代理

import requests
url = 'http://www.baidu.com'
proxies = {'http':'http://117.191.11.107:8080'}
response = requests.get(url, proxies = proxies)
if response.status_code == 200:
    print(response.text)

（4）伪装----请求头headers和session登录

import requests
url = 'http://www.renren.com/PLogin.do'
session = requests.session()
headers ={'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'}
data = {'email': '15811155899', 'password': '*******'}
session.post(url, data = data, headers = headers)
response = session.get('http://www.renren.com/304428818', headers = headers)
if response.status_code == 200:
    print(response.text)

2、json----loads和dumps

# 从对象到文件
import json
s = {'name': '张三', 'age': 22}
f = open('student.json', 'w')
json.dump(s,f)
f.close()
# 从文件到内存对象
import json
f = open('student.json', 'r')
print(json.load(f))
f.close()

3、selenium

from selenium import webdriver
# driver = webdriver.Firefox(executable_path = 'bin\geckodriver.exe')
driver = webdriver.Chrome(executable_path = 'bin\chromedriver.exe')
driver.get("http://www.baidu.com/")
# driver.save_screenshot("a.png")
# driver.find_element_by_id('kw').send_keys('长城')
driver.find_element_by_xpath('//*[@id="kw"]').send_keys('aaa')
driver.find_element_by_id("su").click()
# print(driver.page_source)
print(driver.get_cookies())