通過IO多路複用自定義非同步IO模組
阿新 • • 發佈:2018-11-05
import select,socket class HttpRequest:#HTTP請求類 def __init__(self, sk, host, callback):#初始化 self.host = host self.socket = sk self.callback = callback def fileno(self): return self.socket.fileno()#返回檔案描述符 class AsyncRequest:#非同步請求類 def __init__(self): self.conn = []#接收資料物件的列表 self.connection = []#連線成功物件的列表 def add_request(self, host, callback): try: sk = socket.socket()#生成socket物件 sk.setblocking(0)#預設為阻塞修改成非阻塞 sk.connect((host, 80,))#連線socket except BlockingIOError as e:#抓住報錯 pass request = HttpRequest(sk,host,callback) self.conn.append(request) self.connection.append(request) def run(self): while True: # select:物件必須有fileno方法 返回一個檔案描述符 rlist,wlist,elist = select.select(self.conn, self.connection, self.conn, 0.05) #連線完成 for w in wlist:#連線成功則會在wlist生成物件 tpl = "GET / HTTP/1.0\r\nHost:%s\r\n\r\n" % (w.host) print(w.host, "連線成功!") w.socket.send(bytes(tpl, encoding="utf-8")) self.connection.remove(w) for r in rlist:#傳輸資料 recv_data = bytes() #接受資料 while True: try: data = r.socket.recv(8096) recv_data += data except Exception as e:#接受完成會報錯,catch這個錯誤 break break print(r.host, "有資料返回:",recv_data) r.callback(recv_data)#資料接收完成後執行回撥函式 r.socket.close() self.conn.remove(r) if len(self.conn) == 0:#說明所有連線的資料都接收完成 break def f1(data):#回撥函式 print("儲存資料") def f2(data):#回撥函式 print("匯入資料") url_lists = [ {"host":"www.baidu.com", "callback": f1}, {"host":"www.cnblogs.com", "callback": f2}, {"host":"www.taobao.com", "callback": f2} ] req = AsyncRequest() for item in url_lists: req.add_request(item["host"], item["callback"]) req.run()
www.baidu.com 連線成功! www.cnblogs.com 連線成功! www.baidu.com 有資料返回: b'HTTP/1.0 200 OK\r\nAccept-Ranges: bytes\r\nCache-Control: no-cache\r\nContent-Length: 14615\r\nContent-Type: text/html\r\nDate: Fri, 12 Oct 2018 13:38:30 GMT\r\nEtag: "5ba38008-3917"\r\nLast-Modified: Thu, 20 Sep 2018 11:10:00 GMT\r\nP3p: CP=" OTI DSP COR IVA OUR IND COM "\r\nPragma: no-cache\r\nServer: BWS/1.1\r\nSet-Cookie: BAIDUID=6D2C020C5C49FF78F60F0701B32F7535:FG=1; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com\r\nSet-Cookie: BIDUPSID=6D2C020C5C49FF78F60F0701B32F7535; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com\r\nSet-Cookie: PSTM=1539351510; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com\r\nVary: Accept-Encoding\r\nX-Ua-Compatible: IE=Edge,chrome=1\r\n\r\n<!DOCTYPE html><!--STATUS OK-->\r\n<html>\r\n<head>\r\n\t<meta http-equiv="content-type" content="text/html;charset=utf-8">\r\n\t<meta http-equiv="X-UA-Compatible" content="IE=Edge">\r\n\t<link rel="dns-prefetch" href="//s1.bdstatic.com"/>\r\n\t<link rel="dns-prefetch" href="//t1.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t2.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t3.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t10.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t11.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t12.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//b1.bdstatic.com"/>\r\n\t<title>\xe7\x99\xbe\xe5\xba\xa6\xe4\xb8\x80\xe4\xb8\x8b\xef\xbc\x8c\xe4\xbd\xa0\xe5\xb0\xb1\xe7\x9f\xa5\xe9\x81\x93</title>\r\n\t<link href="http://s1.bdstatic.com/r/www/cache/static/home/css/index.css" rel="stylesheet" type="text/css" />\r\n\t<!--[if lte IE 8]><style index="index" >#content{height:480px\\9}#m{top:260px\\9}</style><![endif]-->\r\n\t<!--[if IE 8]><style index="index" >#u1 a.mnav,#u1 a.mnav:visited{font-family:simsun}</style><![endif]-->\r\n\t<script>var hashMatch = document.location.href.match(/#+(.*wd=[^&].+)/);if (hashMatch && hashMatch[0] && hashMatch[1]) {document.location.replace("http://"+location.host+"/s?"+hashMatch[1]);}var ns_c = function(){};</script>\r\n\t<script>function h(obj){obj.style.behavior=\'url(#default#homepage)\';var a = obj.setHomePage(\'//www.baidu.com/\');}</script>\r\n\t<noscript><meta http-equiv="refresh" content="0; url=/baidu.html?from=noscript"/></noscript>\r\n\t<script>window._ASYNC_START=new Da' 儲存資料 www.taobao.com 連線成功! www.taobao.com 有資料返回: b'HTTP/1.1 302 Found\r\nServer: Tengine\r\nDate: Fri, 12 Oct 2018 13:38:30 GMT\r\nContent-Type: text/html\r\nContent-Length: 258\r\nConnection: close\r\nLocation: https://www.taobao.com/\r\nSet-Cookie: thw=cn; Path=/; Domain=.taobao.com; Expires=Sat, 12-Oct-19 13:38:30 GMT;\r\nStrict-Transport-Security: max-age=31536000\r\n\r\n<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">\r\n<html>\r\n<head><title>302 Found</title></head>\r\n<body bgcolor="white">\r\n<h1>302 Found</h1>\r\n<p>The requested resource resides temporarily under a different URI.</p>\r\n<hr/>Powered by Tengine</body>\r\n</html>\r\n' 匯入資料 www.cnblogs.com 有資料返回: b'HTTP/1.1 301 Moved Permanently\r\nDate: Fri, 12 Oct 2018 13:38:30 GMT\r\nContent-Type: text/html; charset=UTF-8\r\nContent-Length: 147\r\nConnection: close\r\nLocation: https://www.cnblogs.com/\r\nX-UA-Compatible: IE=10\r\nX-Frame-Options: SAMEORIGIN\r\n\r\n<head><title>Document Moved</title></head>\n<body><h1>Object Moved</h1>This document may be found <a HREF="https://www.cnblogs.com/">here</a></body>' 匯入資料