文本去重（python&shell）

python:

def TextDR(sourcepath,destpath):
    sum = 0
    sum_pre = 0
    addrs = set()
    with open(sourcepath, 'r',encoding='utf8') as scan_file:
        for line in scan_file.readlines():
            sum_pre += 1
            # addr = get_addr(line)
            # line.decode('utf8')
            addrs.add(line)
    scan_file.close()
    with open(destpath, 'w',encoding='utf8') as infile:
        while len(addrs) > 0:
            sum += 1
            infile.write(addrs.pop())
    infile.close()
    #print(addrs)
    print("去重之前文本条数: "+str(sum_pre))
    print("去重之后文本条数: "+str(sum))
    return sum_pre,sum

#example
sourcepath = r'D:/zz/source.txt'
destpath = r'D:/zz/destpath.txt'
TextDR(sourcepath,destpath)

shell:

#去除重复行（去重）
sort file.txt | uniq -d
#例：
sort t1.txt |uniq -d > newfile1.txt