asyncio的as_completed和socket ——》收集http响应实例

最新推荐文章于 2023-09-19 14:10:23 发布

迷心兔

最新推荐文章于 2023-09-19 14:10:23 发布

阅读量392

点赞数

原文链接：https://blog.csdn.net/kisslotus/article/details/85216745

版权

Python asyncio 专栏收录该内容

29 篇文章 2 订阅

订阅专栏

原因：as_completed：逐个返回结果

# asyncio 没有提供http协议的接口
import time
import asyncio
from urllib.parse import urlparse


# 改成使用协程
async def get_url (url):
    all_lines = []
    
    # 通过socket请求html----------------------------------------------------------------------------------
    url = urlparse(url)
    host = url.netloc
    path = url.path
    if path == "":
        path = "/"
    
    # 建立socket连接---------------------------------------------------------------------------------------
    # open_connection 源码中调用了 yield from loop.create_connection 。
    # 这里实现的功能完成了 register 和 unregister
    reader, writer = await asyncio.open_connection(host, 80)
    # write 方法完成 register 和 unregister
    writer.write("GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n".format(path, host).encode("utf8"))
    
    # async for 语法，可以将读数据的方式异步化---------------------------------------------------------------
    async for raw_line in reader:
        all_lines.append(raw_line.decode("utf8"))
    return all_lines


async def main ():
    # 使用 tasks 放置 future 对象
    tasks = []
    for number in range(20):
        url = "http://shop.projectsedu.com/goods/{}/".format(number)
        # 使用 asyncio.ensure_future 包装协程，变成 Future 对象，从而获得协程的结果
        tasks.append(asyncio.ensure_future(get_url(url)))
    
    # 逐个返回结果
    for task in asyncio.as_completed(tasks):
        result = await task
        print(result)


if __name__ == "__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()
    # 这儿重新定义了一个函数 main，从而实现 as_completed，即完成一个 task，打印一个 task
    # 在 main 中使用了 asyncio.as_completed，这和线程池中的 as_completed 效果一样
    loop.run_until_complete(main())
    print('last time:{}'.format(time.time() - start_time))

输出：

['HTTP/1.1 404 Not Found\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:24 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 25\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"detail":"未找到。"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:24 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 984\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":4,"category":{"id":129,"sub_cat":[],"name":"根茎类","code":"gjl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/47_P_1448946213263.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/47_P_1448946213157.jpg"}],"goods_sn":"","name":"日本蒜蓉粉丝扇贝270克6只装","click_num":2866,"sold_num":0,"fav_num":-1,"goods_num":0,"market_price":156.0,"shop_price":108.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/47_P_1448946213263.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:24 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 1497\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":1,"category":{"id":129,"sub_cat":[],"name":"根茎类","code":"gjl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/1_P_1449024889889.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/1_P_1449024889264.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/1_P_1449024889726.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/1_P_1449024889018.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/1_P_1449024889287.jpg"}],"goods_sn":"","name":"新鲜水果甜蜜香脆单果约800克","click_num":4244,"sold_num":0,"fav_num":0,"goods_num":-16,"market_price":232.0,"shop_price":156.0,"goods_brief":"食用百香果可以增加胃部饱腹感，减少余热量的摄入，还可以吸附胆固醇和胆汁之类有机分子，抑制人体对脂肪的吸收。因此，长期食用有利于改善人体营养吸收结构，降低体内脂肪，塑造健康优美体态。","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/1_P_1449024889889.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:52"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:24 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 1002\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":7,"category":{"id":132,"sub_cat":[],"name":"进口生鲜","code":"jksx","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/8_P_1448945032810.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/8_P_1448945032646.jpg"}],"goods_sn":"","name":"五星眼肉牛排套餐8片装原味原切生鲜牛肉","click_num":2660,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":150.0,"shop_price":125.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/8_P_1448945032810.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:24 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 1165\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":10,"category":{"id":129,"sub_cat":[],"name":"根茎类","code":"gjl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/9_P_1448944791617.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/9_P_1448944791129.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/9_P_1448944791077.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/9_P_1448944791229.jpg"}],"goods_sn":"","name":"爱食派内蒙古呼伦贝尔冷冻生鲜牛腱子肉1000g","click_num":2400,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":202.0,"shop_price":168.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/9_P_1448944791617.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:24 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 1964\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":2,"category":{"id":116,"sub_cat":[{"id":117,"sub_cat":[],"name":"参鲍","code":"cb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":118,"sub_cat":[],"name":"鱼","code":"yu","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":119,"sub_cat":[],"name":"虾","code":"xia","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":120,"sub_cat":[],"name":"蟹/贝","code":"xb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116}],"name":"海鲜水产","code":"hxsc","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/2_P_1448945810202.jpg"},{"image":"http://shop.projectsedu.com/media/15_P_1448947257324.jpg"},{"image":"http://shop.projectsedu.com/media/2_20170719161405_249.jpg"},{"image":"http://shop.projectsedu.com/media/9_P_1448944791617.jpg"}],"goods_sn":"sssss","name":"田然牛肉大黄瓜条生鲜牛肉冷冻真空黄牛","click_num":3513,"sold_num":100,"fav_num":0,"goods_num":-5,"market_price":106.0,"shop_price":88.0,"goods_brief":"前腿+后腿+羊排共8斤，原生态大山放牧羊羔，曾经的皇室贡品，央视推荐，2005年北京招待全球财金首脑。五层专用包装箱+真空包装+冰袋+保鲜箱+顺丰冷链发货，路途保质期8天","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/> </p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/> </p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/> </p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/2_P_1448945810202.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:00"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:24 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 1605\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":3,"category":{"id":124,"sub_cat":[{"id":125,"sub_cat":[],"name":"生菜","code":"sc","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":124},{"id":126,"sub_cat":[],"name":"菠菜","code":"bc","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":124},{"id":127,"sub_cat":[],"name":"圆椒","code":"yj","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":124},{"id":128,"sub_cat":[],"name":"西兰花","code":"xlh","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":124}],"name":"叶菜类","code":"ycl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/7_P_1448945104883.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/7_P_1448945104734.jpg"}],"goods_sn":"","name":"酣畅家庭菲力牛排10片澳洲生鲜牛肉团购套餐","click_num":2754,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":286.0,"shop_price":238.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/7_P_1448945104883.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:24 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 1678\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":5,"category":{"id":116,"sub_cat":[{"id":117,"sub_cat":[],"name":"参鲍","code":"cb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":118,"sub_cat":[],"name":"鱼","code":"yu","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":119,"sub_cat":[],"name":"虾","code":"xia","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":120,"sub_cat":[],"name":"蟹/贝","code":"xb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116}],"name":"海鲜水产","code":"hxsc","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/10_P_1448944572085.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/10_P_1448944572532.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/10_P_1448944572872.jpg"}],"goods_sn":"","name":"内蒙新鲜牛肉1斤清真生鲜牛肉火锅食材","click_num":2767,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":106.0,"shop_price":88.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/10_P_1448944572085.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:24 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 1674\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":8,"category":{"id":116,"sub_cat":[{"id":117,"sub_cat":[],"name":"参鲍","code":"cb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":118,"sub_cat":[],"name":"鱼","code":"yu","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":119,"sub_cat":[],"name":"虾","code":"xia","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":120,"sub_cat":[],"name":"蟹/贝","code":"xb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116}],"name":"海鲜水产","code":"hxsc","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/11_P_1448944388277.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/11_P_1448944388034.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/11_P_1448944388201.jpg"}],"goods_sn":"","name":"澳洲进口120天谷饲牛仔骨4份原味生鲜","click_num":2625,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":31.0,"shop_price":26.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/11_P_1448944388277.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:25 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 1674\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":12,"category":{"id":116,"sub_cat":[{"id":117,"sub_cat":[],"name":"参鲍","code":"cb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":118,"sub_cat":[],"name":"鱼","code":"yu","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":119,"sub_cat":[],"name":"虾","code":"xia","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116},{"id":120,"sub_cat":[],"name":"蟹/贝","code":"xb","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":116}],"name":"海鲜水产","code":"hxsc","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/48_P_1448943988970.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/48_P_1448943988898.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/48_P_1448943988439.jpg"}],"goods_sn":"","name":"新疆巴尔鲁克生鲜牛排眼肉牛扒1200g","click_num":2380,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":126.0,"shop_price":88.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/48_P_1448943988970.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:25 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 980\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":6,"category":{"id":130,"sub_cat":[],"name":"茄果类","code":"qgl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/4_P_1448945381985.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/4_P_1448945381013.jpg"}],"goods_sn":"","name":"乌拉圭进口牛肉卷特级肥牛卷","click_num":2688,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":90.0,"shop_price":75.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/4_P_1448945381985.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:25 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 993\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":9,"category":{"id":131,"sub_cat":[],"name":"菌菇类","code":"jgl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/6_P_1448945167279.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/6_P_1448945167015.jpg"}],"goods_sn":"","name":"潮香村澳洲进口牛排家庭团购套餐20片","click_num":2634,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":239.0,"shop_price":199.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/6_P_1448945167279.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:25 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 888\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":14,"category":{"id":130,"sub_cat":[],"name":"茄果类","code":"qgl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/images/201705/goods_img/53_P_1495068879687.jpg"}],"goods_sn":"","name":"帐篷出租","click_num":2427,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":120.0,"shop_price":100.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/53_P_1495068879687.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:54"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:25 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 1639\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":13,"category":{"id":121,"sub_cat":[{"id":122,"sub_cat":[],"name":"松花蛋/咸鸭蛋","code":"xhd_xyd","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":121},{"id":123,"sub_cat":[],"name":"鸡蛋","code":"jd","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":121}],"name":"蛋制品","code":"dzp","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/5_P_1448945270390.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/5_P_1448945270067.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/5_P_1448945270432.jpg"}],"goods_sn":"","name":"澳洲进口安格斯牛切片上脑牛排1000g","click_num":2409,"sold_num":0,"fav_num":0,"goods_num":-1,"market_price":144.0,"shop_price":120.0,"goods_brief":"澳大利亚是国际公认的没有疯牛病和口蹄疫的国家。为了保持澳大利亚产品的高标准，澳大利亚牛肉业和各级政府共同努力简历了严格的标准和体系，以保证生产的整体化和产品的可追溯性","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/5_P_1448945270390.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:54"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:25 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 978\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":16,"category":{"id":145,"sub_cat":[],"name":"饮料/水","code":"yls","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":133},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/14_P_1448947354031.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/14_P_1448947354433.jpg"}],"goods_sn":"","name":"52度水井坊臻酿八號500ml","click_num":2419,"sold_num":0,"fav_num":0,"goods_num":-3,"market_price":43.0,"shop_price":36.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/14_P_1448947354031.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:54"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:25 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 1820\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":11,"category":{"id":111,"sub_cat":[{"id":112,"sub_cat":[],"name":"羊肉","code":"yr","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":111},{"id":113,"sub_cat":[],"name":"禽类","code":"ql","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":111},{"id":114,"sub_cat":[],"name":"猪肉","code":"zr","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":111},{"id":115,"sub_cat":[],"name":"牛肉","code":"nr","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":111}],"name":"精品肉类","code":"jprl","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:33","parent_category":110},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/3_P_1448945490837.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/3_P_1448945490084.jpg"}],"goods_sn":"","name":"澳洲进口牛尾巴300g新鲜肥牛肉","click_num":2471,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":306.0,"shop_price":255.0,"goods_brief":"新鲜羊羔肉整只共15斤，原生态大山放牧羊羔，曾经的皇室贡品，央视推荐，2005年北京招待全球财金首脑。五层专用包装箱+真空包装+冰袋+保鲜箱+顺丰冷链发货，路途保质期8天","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/3_P_1448945490837.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:53"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:25 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 979\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":19,"category":{"id":145,"sub_cat":[],"name":"饮料/水","code":"yls","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":133},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/21_P_1448946793276.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/21_P_1448946793153.jpg"}],"goods_sn":"","name":"西夫拉姆进口洋酒小酒版","click_num":2376,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":55.0,"shop_price":46.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/21_P_1448946793276.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:54"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:25 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 1429\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":15,"category":{"id":146,"sub_cat":[{"id":147,"sub_cat":[],"name":"白兰地","code":"bld","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":146},{"id":148,"sub_cat":[],"name":"威士忌","code":"wsj","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":146}],"name":"红酒","code":"hj","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":133},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/16_P_1448947194687.jpg"}],"goods_sn":"","name":"52度茅台集团国隆双喜酒500mlx6","click_num":2477,"sold_num":0,"fav_num":0,"goods_num":-5,"market_price":23.0,"shop_price":19.0,"goods_brief":"贵州茅台酒厂（集团）保健酒业有限公司生产，是以“龙”字打头的酒水。中国龙文化上下8000年，源远而流长，龙的形象是一种符号、一种意绪、一种血肉相联的情感。","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/16_P_1448947194687.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:54"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:25 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 989\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":18,"category":{"id":138,"sub_cat":[],"name":"葡萄酒","code":"ptj","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":133},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/46_P_1448946598711.jpg"},{"image":"http://shop.projectsedu.com/media/goods/images/46_P_1448946598301.jpg"}],"goods_sn":"","name":"双响炮洋酒JimBeamwhiskey美国白占边","click_num":2392,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":38.0,"shop_price":28.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/46_P_1448946598711.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:54"}']
['HTTP/1.1 200 OK\r\n', 'Server: nginx/1.12.2\r\n', 'Date: Thu, 10 Oct 2019 02:54:25 GMT\r\n', 'Content-Type: application/json\r\n', 'Content-Length: 1352\r\n', 'Connection: close\r\n', 'Vary: Accept, Cookie\r\n', 'Allow: GET, HEAD, OPTIONS\r\n', 'X-Frame-Options: SAMEORIGIN\r\n', '\r\n', '{"id":17,"category":{"id":141,"sub_cat":[{"id":142,"sub_cat":[],"name":"其他品牌","code":"qtpp","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":141},{"id":143,"sub_cat":[],"name":"黄酒","code":"hj","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":141},{"id":144,"sub_cat":[],"name":"养生酒","code":"ysj","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":141}],"name":"其他酒品","code":"qtjp","desc":"","category_type":2,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":133},"images":[{"image":"http://shop.projectsedu.com/media/goods/images/12_P_1448947547989.jpg"}],"goods_sn":"","name":"53度茅台仁酒500ml","click_num":2361,"sold_num":0,"fav_num":0,"goods_num":0,"market_price":190.0,"shop_price":158.0,"goods_brief":"","goods_desc":"<p><img src=\\"/media/goods/images/2_20170719161405_249.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161414_628.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p><p><img src=\\"/media/goods/images/2_20170719161435_381.jpg\\" title=\\"\\" alt=\\"2.jpg\\"/></p>","ship_free":true,"goods_front_image":"http://shop.projectsedu.com/media/goods/images/12_P_1448947547989.jpg","is_new":false,"is_hot":false,"add_time":"2017-07-31T23:53:54"}']
last time:2.3423449993133545

迷心兔

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
asyncio的as_completed和socket ——》收集http响应实例

原因：as_completed：逐个返回结果# asyncio 没有提供http协议的接口import timeimport asynciofrom urllib.parse import urlparse# 改成使用协程async def get_url (url): all_lines = [] # 通过socket请求html------------...
复制链接

扫一扫