urllib

import urllib.request
import urllib.parse
#获取一个get请求
response=urllib.request.urlopen("http://www.baidu.com")
print(response.read().decode('utf-8'))

常用get or POST

import urllib.request
import urllib.parse

#获取post请求
#封装一个data
data=bytes(urllib.parse.urlencode({"hello":"world"}),encoding="utf-8")
response=urllib.request.urlopen("https://httpbin.org/post",data=data)
print(response.read().decode("utf-8"))

#超时处理
 try:
     response = urllib.request.urlopen("https://httpbin.org/get",timeout=0.01)
     print(response.read().decode("utf-8"))
 except urllib.error.URLError as e:
     print("time out!")

测试网站：

https://httpbin.org/post

一般只需要headers & url

 url="https://httpbin.org/post"

 headers={
 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"
 }

 data=bytes(urllib.parse.urlencode({'name':'Erica'}),encoding="utf-8")

#用一个函数存储数据：
 req=urllib.request.Request(url=url,data=data,headers=headers,method="POST")

 response=urllib.request.urlopen(req)

 print(response.read().decode("utf-8"))

用lofter 首页测试

爬取成功

url="http://www.lofter.com/view?act=qbview_20130930_01"

headers={
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"
}
req=urllib.request.Request(url=url,headers=headers)
response=urllib.request.urlopen(req)
print(response.read().decode("utf-8"))

任务中，

写一个爬取豆瓣电影TOP250的第一页的函数

def main():
    
    askURL("https://movie.douban.com/top250?start=0")


def askURL(url):
    head={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"
        #模拟浏览器,向豆瓣服务器发送消息
    }
    request=urllib.request.Request(url,headers=head)
    html=""
    try:
        response=urllib.request.urlopen(request)
        html=response.read().decode("utf-8")
        print(html)
    except urllib.error.URLError as e:
        if hasattr(e,"code"):
            print(e.code)
        if hasattr(e,"reason"):
            print(e.reason)

if __name__=="__main__":
    main()