import urllib.request import urllib.error # response=urllib.request.urlopen("http://www.baidu.com") # print(response.read().decode('utf-8')) import urllib.parse # try: # data = bytes(urllib.parse.urlencode({"hello":"world"}),encoding="utf-8") # response=urllib.request.urlopen("http://httpbin.org/post",data,timeout=0.01) # # print(response.read().decode("utf-8")) # except urllib.error.URLError as e: # print("time out") # try: # response = urllib.request.urlopen("http://httpbin.org/get",timeout=0.01) # print(response.read().decode("utf-8")) # except urllib.error.URLError as e: # print("time out") # response=urllib.request.urlopen("http://httpbin.org/get") # print(response.status) # response=urllib.request.urlopen("http://www.baidu.com") # print(response.status) # print(response.getheaders()) # print(response.getheader("server")) # url = "https://douban.com" # headers = { # "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36" # } # data = bytes(urllib.parse.urlencode({'name':'eric'}),encoding="utf-8") # req = urllib.request.Request(url=url, data=data, headers=headers, method="post") # response = urllib.request.urlopen(req) # print(response.read().decode("utf-8"))
from bs4 import BeautifulSoup import re import urllib.request, urllib.error import xlwt import sqlite3 def main(): basel = "https://movie.douban.com/top250?start=" askurl(basel) # 1 抓取网页 sabbath = ".\\top250.xls" saveData(sabbath) # 2 解析数据 print("hello") # 爬取网页 def getData(baseurl): datalist = [] return datalist # 3 保存数据 def saveData(savepath): print("save....") def askurl(url): head={ "user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36" } request=urllib.request.Request(url,headers=head) html ="" try: response=urllib.request.urlopen(request) html=response.read().decode("utf-8") print(html) except urllib.error.URLError as e: if hasattr(e,"code"): print(e.code) print("error") if __name__ == '__main__': main()