How i can clear memory in this python loop ?
import concurrent.futures as futures
with futures.ThreadPoolExecutor(max_workers=100) as executor:
fs = [executor.submit(get_data, url) for url in link]
for i, f in enumerate(futures.as_completed(fs)):
x= (f.result())
results.append(x)
del x
del f
get_data - simple function wich using requests
解决方案
My solution would be as such:
import concurrent.futures as futures
#split the original grand list into smaller batches
batchurlList = [grandUrlList[x:x+batchSize] for x in range(0, len(grandUrlList), batchSize)]
for tmpurlList in batchurlList:
with futures.ThreadPoolExecutor(max_workers=100) as executor:
myfuture = {executor.submit(myFunction, url): url for url in tmpurlList}
for future in futures.as_completed(myfuture, timeout=60):
originalUrl = myfuture[future]
results.append(future.result())