文件行去重

为什么80%的码农都做不了架构师？>>>   

from __future__ import print_function

def dedupe(items, key=None):
        seen = set()
        for item in items:
                val = item if key is None else key(item)
                if val not in seen:
                        yield item
                        seen.add(val)


with open("somefile",'r') as f:
        for line in dedupe(f):
                print(line, end='')

转载于:https://my.oschina.net/itfanr/blog/1619867