爬虫常用模块：requests ，cookie处理，xlwt(写入excel), lxml

1. requests

import requests

url = 'http://www.baidu.com'
response =requests.get(url)
print(response)      # 请求成功

print(response.text)    # 返回网页的文本内容

print(response.reason)   # reason 原因 对请求状态的解释

print(response.links)   # link 跳转的地址

print(response.history) # 请求历史

print(response.apparent_encoding    # 获取网页的编码

response.encoding = response.apparent_encoding) # 设置响应的编码格式为网页的编码格式

print(response.content) # 获取网页内容(bytes 二进制 形式的)

print(response.cookies) # 获取网页cookies
# <RequestsCookieJar[<Cookie BDORZ=27315 for .baidu.com/>]>

print(response.headers)     # 获取响应头信息

print(response.request.url) # 获取请求的网址

2.0 ip代理

from urllib.request import urlopen, Request, build_opener, ProxyHandler
import random

user_agent_list = [
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
    # 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
    # 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Mobile Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0'
]
headers ={
    'User-Agent': 'random.choice(user_agent_list)'
}
ip_list = [
'124.193.85.88:8080',