#coding=utf-8
import sys, re, os
def getDictList(dict):
regx = '''[\w\~`\!\@\#\$\%\^\&\*\(\)\_\-\+\=\[\]\{\}\:\;\,\.\/\<\>\?]+'''
with open(dict) as f:
data = f.read()
return re.findall(regx, data)
def rmdp(dictList):
return list(set(dictList))
def fileSave(dictRmdp, out):
with open(out, 'a') as f:
for line in dictRmdp:
f.write(line + '\n')
def main():
try:
dict = sys.argv[1].strip()
out = sys.argv[2].strip()
except Exception, e:
print 'error:', e
me = os.path.basename(__file__)
print 'usage: %s <input> <output>' %me
print 'example: %s dict.txt dict_rmdp.txt' %me
exit()
dictList = getDictList(dict)
dictRmdp = rmdp(dictList)
fileSave(dictRmdp, out)
if __name__ == '__main__':
main()
python 高效去重复 支持GB级别大文件
最新推荐文章于 2024-06-27 16:41:53 发布