服务器 python cant open file_新手尝试用 aiohttp 写了个爬虫，但是目前因为 task 过多(超过 1000 个)，报错 Too many open files，请问如何解...

完整代码

import time

import asyncio

import aiohttp

from bs4 import BeautifulSoup as bs

BASE_URL = "http://www.biqudu.com"

TITLE2URL = dict()

CONTENT = list()

async def fetch(url, callback=None, **kwarags):

headers = {'User-Agent':'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}

sem = asyncio.Semaphore(5)

with (await sem):

async with aiohttp.ClientSession() as session:

async with session.get(url, headers=headers) as res:

page = await res.text()

if callback:

callback(page, **kwarags)

else:

return page

def parse_url(page):

soup = bs(page, "lxml")

dd_a_doc = soup.select("dd > a")

for a_doc in dd_a_doc:

article_page_url = a_doc['href']

article_title = a_doc.get_text()

if article_page_url:

TITLE2URL[article_title] = article_page_url

def parse_body(page, **kwarags):

title = kwarags.get('title', '')

print("{}".format(title))

soup = bs(page, "lxml")

content_doc = soup.find("div", id="content")

content_text = content_doc.get_text().replace('readx();', '').replace(' ', "\r\n")

content = "%s\n%s\n\n" % (title, content_text)

CONTENT.append(content)

def main():

t0 = time.time()

loop = asyncio.get_event_loop()

loop.run_until_complete(fetch(BASE_URL+"/43_43074/", callback=parse_url))

tasks = [fetch(BASE_URL + page_url, callback=parse_body, title=title) for title, page_url in TITLE2URL.items()]

loop.run_until_complete(asyncio.gather(*tasks[:500]))

loop.close()

elapsed = time.time() - t0

print("cost {}".format(elapsed))

if __name__ == "__main__":

main()

错误信息

Traceback (most recent call last):

File "/usr/local/lib/python3.5/dist-packages/aiohttp/connector.py", line 797, in _wrap_create_connection

return (yield from self._loop.create_connection(*args, **kwargs))

File "/usr/lib/python3.5/asyncio/base_events.py", line 695, in create_connection

raise exceptions[0]

File "/usr/lib/python3.5/asyncio/base_events.py", line 662, in create_connection

sock = socket.socket(family=family, type=type, proto=proto)

File "/usr/lib/python3.5/socket.py", line 134, in __init__

_socket.socket.__init__(self, family, type, proto, fileno)

OSError: [Errno 24] Too many open files

The above exception was the direct cause of the following exception:

Traceback (most recent call last):

File "/home/xsy/Workspace/Self/aiotest/aiotest.py", line 58, in

main()

File "/home/xsy/Workspace/Self/aiotest/aiotest.py", line 52, in main

loop.run_until_complete(asyncio.gather(*tasks[:500]))

File "/usr/lib/python3.5/asyncio/base_events.py", line 387, in run_until_complete

return future.result()

File "/usr/lib/python3.5/asyncio/futures.py", line 274, in result

raise self._exception

File "/usr/lib/python3.5/asyncio/tasks.py", line 239, in _step

result = coro.send(None)

File "/home/xsy/Workspace/Self/aiotest/aiotest.py", line 18, in fetch

async with session.get(url, headers=headers) as res:

File "/usr/local/lib/python3.5/dist-packages/aiohttp/client.py", line 690, in __aenter__

self._resp = yield from self._coro

File "/usr/local/lib/python3.5/dist-packages/aiohttp/client.py", line 267, in _request

conn = yield from self._connector.connect(req)

File "/usr/local/lib/python3.5/dist-packages/aiohttp/connector.py", line 402, in connect

proto = yield from self._create_connection(req)

File "/usr/local/lib/python3.5/dist-packages/aiohttp/connector.py", line 749, in _create_connection

_, proto = yield from self._create_direct_connection(req)

File "/usr/local/lib/python3.5/dist-packages/aiohttp/connector.py", line 860, in _create_direct_connection

raise last_exc

File "/usr/local/lib/python3.5/dist-packages/aiohttp/connector.py", line 832, in _create_direct_connection

req=req, client_error=client_error)

File "/usr/local/lib/python3.5/dist-packages/aiohttp/connector.py", line 804, in _wrap_create_connection

raise client_error(req.connection_key, exc) from exc

aiohttp.client_exceptions.ClientConnectorError: Cannot connect to host www.biqudu.com:443 ssl:True [Too many open files]

目前我能想到的方法是

修改 linux 的最大文件打开数量限制

对 task 切片多次运行

但是这样解决都感觉太蠢了，请问有什么更好的方式吗？