import select,socket
class HttpRequest:#HTTP请求类
def __init__(self, sk, host, callback):#初始化
self.host = host
self.socket = sk
self.callback = callback
def fileno(self):
return self.socket.fileno()#返回文件描述符
class AsyncRequest:#异步请求类
def __init__(self):
self.conn = []#接收数据对象的列表
self.connection = []#连接成功对象的列表
def add_request(self, host, callback):
try:
sk = socket.socket()#生成socket对象
sk.setblocking(0)#默认为阻塞修改成非阻塞
sk.connect((host, 80,))#连接socket
except BlockingIOError as e:#抓住报错
pass
request = HttpRequest(sk,host,callback)
self.conn.append(request)
self.connection.append(request)
def run(self):
while True:
# select:对象必须有fileno方法 返回一个文件描述符
rlist,wlist,elist = select.select(self.conn, self.connection, self.conn, 0.05)
#连接完成
for w in wlist:#连接成功则会在wlist生成对象
tpl = "GET / HTTP/1.0\r\nHost:%s\r\n\r\n" % (w.host)
print(w.host, "连接成功!")
w.socket.send(bytes(tpl, encoding="utf-8"))
self.connection.remove(w)
for r in rlist:#传输数据
recv_data = bytes()
#接受数据
while True:
try:
data = r.socket.recv(8096)
recv_data += data
except Exception as e:#接受完成会报错,catch这个错误 break
break
print(r.host, "有数据返回:",recv_data)
r.callback(recv_data)#数据接收完成后执行回调函数
r.socket.close()
self.conn.remove(r)
if len(self.conn) == 0:#说明所有连接的数据都接收完成
break
def f1(data):#回调函数
print("保存数据")
def f2(data):#回调函数
print("导入数据")
url_lists = [
{"host":"www.baidu.com", "callback": f1},
{"host":"www.cnblogs.com", "callback": f2},
{"host":"www.taobao.com", "callback": f2}
]
req = AsyncRequest()
for item in url_lists:
req.add_request(item["host"], item["callback"])
req.run()
www.baidu.com 连接成功!
www.cnblogs.com 连接成功!
www.baidu.com 有数据返回: b'HTTP/1.0 200 OK\r\nAccept-Ranges: bytes\r\nCache-Control: no-cache\r\nContent-Length: 14615\r\nContent-Type: text/html\r\nDate: Fri, 12 Oct 2018 13:38:30 GMT\r\nEtag: "5ba38008-3917"\r\nLast-Modified: Thu, 20 Sep 2018 11:10:00 GMT\r\nP3p: CP=" OTI DSP COR IVA OUR IND COM "\r\nPragma: no-cache\r\nServer: BWS/1.1\r\nSet-Cookie: BAIDUID=6D2C020C5C49FF78F60F0701B32F7535:FG=1; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com\r\nSet-Cookie: BIDUPSID=6D2C020C5C49FF78F60F0701B32F7535; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com\r\nSet-Cookie: PSTM=1539351510; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com\r\nVary: Accept-Encoding\r\nX-Ua-Compatible: IE=Edge,chrome=1\r\n\r\n<!DOCTYPE html><!--STATUS OK-->\r\n<html>\r\n<head>\r\n\t<meta http-equiv="content-type" content="text/html;charset=utf-8">\r\n\t<meta http-equiv="X-UA-Compatible" content="IE=Edge">\r\n\t<link rel="dns-prefetch" href="//s1.bdstatic.com"/>\r\n\t<link rel="dns-prefetch" href="//t1.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t2.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t3.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t10.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t11.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t12.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//b1.bdstatic.com"/>\r\n\t<title>\xe7\x99\xbe\xe5\xba\xa6\xe4\xb8\x80\xe4\xb8\x8b\xef\xbc\x8c\xe4\xbd\xa0\xe5\xb0\xb1\xe7\x9f\xa5\xe9\x81\x93</title>\r\n\t<link href="http://s1.bdstatic.com/r/www/cache/static/home/css/index.css" rel="stylesheet" type="text/css" />\r\n\t<!--[if lte IE 8]><style index="index" >#content{height:480px\\9}#m{top:260px\\9}</style><![endif]-->\r\n\t<!--[if IE 8]><style index="index" >#u1 a.mnav,#u1 a.mnav:visited{font-family:simsun}</style><![endif]-->\r\n\t<script>var hashMatch = document.location.href.match(/#+(.*wd=[^&].+)/);if (hashMatch && hashMatch[0] && hashMatch[1]) {document.location.replace("http://"+location.host+"/s?"+hashMatch[1]);}var ns_c = function(){};</script>\r\n\t<script>function h(obj){obj.style.behavior=\'url(#default#homepage)\';var a = obj.setHomePage(\'//www.baidu.com/\');}</script>\r\n\t<noscript><meta http-equiv="refresh" content="0; url=/baidu.html?from=noscript"/></noscript>\r\n\t<script>window._ASYNC_START=new Da'
保存数据
www.taobao.com 连接成功!
www.taobao.com 有数据返回: b'HTTP/1.1 302 Found\r\nServer: Tengine\r\nDate: Fri, 12 Oct 2018 13:38:30 GMT\r\nContent-Type: text/html\r\nContent-Length: 258\r\nConnection: close\r\nLocation: https://www.taobao.com/\r\nSet-Cookie: thw=cn; Path=/; Domain=.taobao.com; Expires=Sat, 12-Oct-19 13:38:30 GMT;\r\nStrict-Transport-Security: max-age=31536000\r\n\r\n<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">\r\n<html>\r\n<head><title>302 Found</title></head>\r\n<body bgcolor="white">\r\n<h1>302 Found</h1>\r\n<p>The requested resource resides temporarily under a different URI.</p>\r\n<hr/>Powered by Tengine</body>\r\n</html>\r\n'
导入数据
www.cnblogs.com 有数据返回: b'HTTP/1.1 301 Moved Permanently\r\nDate: Fri, 12 Oct 2018 13:38:30 GMT\r\nContent-Type: text/html; charset=UTF-8\r\nContent-Length: 147\r\nConnection: close\r\nLocation: https://www.cnblogs.com/\r\nX-UA-Compatible: IE=10\r\nX-Frame-Options: SAMEORIGIN\r\n\r\n<head><title>Document Moved</title></head>\n<body><h1>Object Moved</h1>This document may be found <a HREF="https://www.cnblogs.com/">here</a></body>'
导入数据