python爬虫_2

requests模块

requests. get()

import requests
data = {'wd':'python'}
url = 'https://www.baidu.com/s'
get_1 = requests.get(url,params=data)#多了 params参数

print(get_1.url)
#https://www.baidu.com/s?wd=python

requests. post()

data = {'firstname':'5','lastname':'5'}
post_url = 'http://pythonscraping.com/pages/files/processing.php'
post_ = requests.post(post_url,data=data)

print(post_.text)

#cookies 模拟登陆
#1
form_data = {'username':'anything','password':'password'}
url_post = 'http://pythonscraping.com/pages/cookies/welcome.php'
r = requests.post(url_post,data=form_data)
print(r.cookies.get_dict())

r1 = requests.get(url_post,cookies=r.cookies)
print(r1.text)

#2
session = requests.Session()
data_form = {'username':'anything','password':'password'}
r2 = session.post(url_post,data=data_form)
print(r2.cookies.get_dict())

r3 = session.get(url_post)
print(r3.text)

下载文件

#1
from urllib.request import urlretrieve
urlretrieve(image_url,'img/img1.png')

#2
r = requests.get(image_url)
with open('img/img2.png','wb') as f:
  f.write(r.content)

#3
r = requests.get(image_url, stream=True)    # stream loading

with open('img/img3.png', 'wb') as f:
    for chunk in r.iter_content(chunk_size=32):
        f.write(chunk)