Python 3.4包括一个新模块：tracemalloc..它提供了关于哪些代码分配内存最多的详细统计信息。下面是一个显示分配内存的前三行的示例。
from collections import Counterimport linecacheimport osimport tracemallocdef display_top(snapshot, key_type='lineno', limit=3):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, ""),
tracemalloc.Filter(False, ""),
))
top_stats = snapshot.statistics(key_type)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#%s: %s:%s: %.1f KiB"
% (index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))tracemalloc.start()counts = Counter()fname = '/usr/share/dict/american-english'with open(fname) as words:
words = list(words)
for word in words:
prefix = word[:3]
counts[prefix] += 1print('Top prefixes:', counts.most_common(3))snapshot = tracemalloc.take_snapshot()display_top(snapshot)
结果如下：Top prefixes: [('con', 1220), ('dis', 1002), ('pro', 809)]Top 3 lines#1: scratches/memory_test.py:37: 6527.1 KiB
words = list(words)#2: scratches/memory_test.py:39: 247.7 KiB
prefix = word[:3]#3: scratches/memory_test.py:40: 193.0 KiB
counts[prefix] += 14 other: 4.3 KiBTotal allocated size: 6972.1 KiB
什么时候是内存泄漏而不是泄漏？
当内存仍然保存在计算的末尾时，这个例子是很棒的，但是有时您的代码分配了大量内存，然后全部释放出来。从技术上讲，这不是内存泄漏，但它使用的内存比您认为的要多。当所有内存都被释放时，如何跟踪内存的使用情况？如果是您的代码，您可能可以在运行时添加一些调试代码来获取快照。如果没有，则可以在主线程运行时启动后台线程来监视内存使用情况。
下面是前面的示例，代码都已移到count_prefixes()功能。当该函数返回时，将释放所有内存。我也加了一些sleep()调用来模拟长期运行的计算。from collections import Counterimport linecacheimport osimport tracemallocfrom time import sleepdef count_prefixes():
sleep(2) # Start up time.
counts = Counter()
fname = '/usr/share/dict/american-english'
with open(fname) as words:
words = list(words)
for word in words:
prefix = word[:3]
counts[prefix] += 1
sleep(0.0001)
most_common = counts.most_common(3)
sleep(3) # Shut down time.
return most_commondef main():
tracemalloc.start()
most_common = count_prefixes()
print('Top prefixes:', most_common)
snapshot = tracemalloc.take_snapshot()
display_top(snapshot)def display_top(snapshot, key_type='lineno', limit=3):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, ""),
tracemalloc.Filter(False, ""),
))
top_stats = snapshot.statistics(key_type)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#%s: %s:%s: %.1f KiB"
% (index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))main()
当我运行该版本时，内存使用量从6MB降到了4KB，因为函数完成时释放了它的所有内存。Top prefixes: [('con', 1220), ('dis', 1002), ('pro', 809)]Top 3 lines#1: collections/__init__.py:537: 0.7 KiB
self.update(*args, **kwds)#2: collections/__init__.py:555: 0.6 KiB
return _heapq.nlargest(n, self.items(), key=_itemgetter(1))#3: python3.6/heapq.py:569: 0.5 KiB
result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)]10 other: 2.2 KiBTotal allocated size: 4.0 KiB
下面是一个灵感来自另一个答案这将启动第二个线程来监视内存使用情况。from collections import Counterimport linecacheimport osimport tracemallocfrom datetime import datetimefrom queue import Queue, Emptyfrom resource import getrusage, RUSAGE_SELFfrom threading import Threadfrom time import sleepdef memory_monitor(command_queue: Queue, poll_interval=1):
tracemalloc.start()
old_max = 0
snapshot = None
while True:
try:
command_queue.get(timeout=poll_interval)
if snapshot is not None:
print(datetime.now())
display_top(snapshot)
return
except Empty:
max_rss = getrusage(RUSAGE_SELF).ru_maxrss if max_rss > old_max:
old_max = max_rss
snapshot = tracemalloc.take_snapshot()
print(datetime.now(), 'max RSS', max_rss)def count_prefixes():
sleep(2) # Start up time.
counts = Counter()
fname = '/usr/share/dict/american-english'
with open(fname) as words:
words = list(words)
for word in words:
prefix = word[:3]
counts[prefix] += 1
sleep(0.0001)
most_common = counts.most_common(3)
sleep(3) # Shut down time.
return most_commondef main():
queue = Queue()
poll_interval = 0.1
monitor_thread = Thread(target=memory_monitor, args=(queue, poll_interval))
monitor_thread.start()
try:
most_common = count_prefixes()
print('Top prefixes:', most_common)
finally:
queue.put('stop')
monitor_thread.join()def display_top(snapshot, key_type='lineno', limit=3):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, ""),
tracemalloc.Filter(False, ""),
))
top_stats = snapshot.statistics(key_type)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#%s: %s:%s: %.1f KiB"
% (index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))main()
这个resource模块允许您检查当前内存使用情况，并从峰值内存使用量中保存快照。队列允许主线程告诉内存监视器线程何时打印其报表并关闭。运行时，它将显示list()呼叫：2018-05-29 10:34:34.441334 max RSS 101882018-05-29 10:34:36.475707 max RSS 235882018-05-29 10:34:36.616524 max RSS 381042018-05-29 10:34:36.772978 max RSS 459242018-05-29 10:34:36.929688 max RSS 468242018-05-29 10:34:37.087554 max RSS 46852Top prefixes: [('con', 1220), ('dis', 1002), ('pro', 809)]2018-05-29 10:34:56.281262Top 3 lines#1: scratches/scratch.py:36: 6527.0 KiB
words = list(words)#2: scratches/scratch.py:38: 16.4 KiB
prefix = word[:3]#3: scratches/scratch.py:39: 10.1 KiB
counts[prefix] += 119 other: 10.8 KiBTotal allocated size: 6564.3 KiB
如果你在Linux上，你会发现/proc/self/statm比resource模块。
扫一扫
727
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
