Python学习——10.爬虫练习

代码函数调用规范：

def main():
    print("hello")

if __name__ == "__main__":                   #当程序执行时
    #调用函数
    main()

函数调用都放在main里面

引入自定义模块： 

Get请求：

import  urllib.request

#获取一个get请求
response = urllib.request.urlopen("http://www.baidu.com")
#对获取的远吗进行utf-8解码
print(response.read().decode('utf-8'))

Post请求：

import  urllib.request
import  urllib.parse

#获取一个post请求
data = bytes(urllib.parse.urlencode({"hello":"feng"}),encoding="utf-8")
response = urllib.request.urlopen("http://httpbin.org/post",data=data)
print(response.read().decode("utf-8"))

超时处理：

#Try处理超时
try:
    response = urllib.request.urlopen("http://www.baidu.com",timeout=0.01)  ###0.01时间太短，用于测试所以会超时
    print(response.read().decode('utf-8'))
except urllib.error.URLError as e:
    print("time out")

爬虫伪装：

import  urllib.request
import  urllib.parse

#爬虫伪装
url = "http://httpbin.org/post"

#将浏览器User-Agent信息赋给header，如果还是不行再把其他字段也加进去
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"}
data = bytes(urllib.parse.urlencode({"te":"hello"}),encoding="utf-8")

req = urllib.request.Request(url=url,data=data,headers=headers,method="POST")
response = urllib.request.urlopen(req)
print(response.read().decode("utf-8"))

Post方法伪装

import  urllib.request
import  urllib.parse

url = "https://www.douban.com"

headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"}
req = urllib.request.Request(url=url,headers=headers)
response = urllib.request.urlopen(req)

print(response.read().decode("utf-8"))

Get方法伪装