from __future__ import print_function
def dedupe(items, key=None):
seen = set()
for item in items:
val = item if key is None else key(item)
if val not in seen:
yield item
seen.add(val)
with open("somefile",'r') as f:
for line in dedupe(f):
print(line, end='')
转载于:https://my.oschina.net/itfanr/blog/1619867