去除python list中的重复值

最新推荐文章于 2021-07-20 16:44:02 发布

雨落

最新推荐文章于 2021-07-20 16:44:02 发布

阅读量846

点赞数

分类专栏： Python 文章标签： list python import random string class

本文链接：https://blog.csdn.net/anbo724/article/details/6535557

版权

Python 专栏收录该内容

14 篇文章 0 订阅

订阅专栏

在一篇很牛的介绍python中list去重的博客（http://www.peterbe.com/plog/uniqifiers-benchmark ）中收集到的：

view plaincopy to clipboardprint?
   1. from random import shuffle, randint  
   2. import re  
   3. from sets import Set  
   4. def f1(seq): # Raymond Hettinger  
   5.     # not order preserving  
   6.     set = {}  
   7.     map(set.__setitem__, seq, [])  
   8.     return set.keys()  
   9.       
  10. def f2(seq):   # *********  
  11.     # order preserving  
  12.     checked = []  
  13.     for e in seq:  
  14.         if e not in checked:  
  15.             checked.append(e)  
  16.     return checked  
  17. def f3(seq):  
  18.     # Not order preserving  
  19.     keys = {}  
  20.     for e in seq:  
  21.         keys[e] = 1  
  22.     return keys.keys()  
  23. def f4(seq): # ********** order preserving  
  24.     noDupes = []  
  25.     [noDupes.append(i) for i in seq if not noDupes.count(i)]  
  26.     return noDupes  
  27. def f5(seq, idfun=None): # Alex Martelli ******* order preserving  
  28.     if idfun is None:  
  29.         def idfun(x): return x  
  30.     seen = {}  
  31.     result = []  
  32.     for item in seq:  
  33.         marker = idfun(item)  
  34.         # in old Python versions:  
  35.         # if seen.has_key(marker)  
  36.         # but in new ones:  
  37.         if marker in seen: continue  
  38.         seen[marker] = 1  
  39.         result.append(item)  
  40.     return result  
  41. def f5b(seq, idfun=None): # Alex Martelli ******* order preserving  
  42.     if idfun is None:  
  43.         def idfun(x): return x  
  44.     seen = {}  
  45.     result = []  
  46.     for item in seq:  
  47.         marker = idfun(item)  
  48.         # in old Python versions:  
  49.         # if seen.has_key(marker)  
  50.         # but in new ones:  
  51.         if marker not in seen:  
  52.             seen[marker] = 1  
  53.             result.append(item)  
  54.               
  55.     return result  
  56. def f6(seq):  
  57.     # Not order preserving  
  58.     return list(Set(seq))  
  59. def f7(seq):  
  60.     # Not order preserving  
  61.     return list(set(seq))  
  62. def f8(seq): # Dave Kirby  
  63.     # Order preserving  
  64.     seen = set()  
  65.     return [x for x in seq if x not in seen and not seen.add(x)]  
  66. def f9(seq):  
  67.     # Not order preserving  
  68.     return {}.fromkeys(seq).keys()  
  69. def f10(seq, idfun=None): # Andrew Dalke  
  70.     # Order preserving  
  71.     return list(_f10(seq, idfun))  
  72. def _f10(seq, idfun=None):  
  73.     seen = set()  
  74.     if idfun is None:  
  75.         for x in seq:  
  76.             if x in seen:  
  77.                 continue  
  78.             seen.add(x)  
  79.             yield x  
  80.     else:  
  81.         for x in seq:  
  82.             x = idfun(x)  
  83.             if x in seen:  
  84.                 continue  
  85.             seen.add(x)  
  86.             yield x  
  87.               
  88.               
  89. def f11(seq): # f10 but simpler  
  90.     # Order preserving  
  91.     return list(_f10(seq))  
  92. def _f11(seq):  
  93.     seen = set()  
  94.     for x in seq:  
  95.         if x in seen:  
  96.             continue  
  97.         seen.add(x)  
  98.         yield x  
  99.               
 100. import time  
 101. def timing(f, n, a):  
 102.     print f.__name__,  
 103.     r = range(n)  
 104.     t1 = time.clock()  
 105.     for i in r:  
 106.         f(a); f(a); f(a); f(a); f(a); f(a); f(a); f(a); f(a); f(a)  
 107.     t2 = time.clock()  
 108.     print round(t2-t1, 3)  
 109.       
 110. def getRandomString(length=10, loweronly=1, numbersonly=0,  
 111.                     lettersonly=0):  
 112.     """ return a very random string """  
 113.     _letters = 'abcdefghijklmnopqrstuvwxyz'  
 114.     if numbersonly:  
 115.         l = list('0123456789')  
 116.     elif lettersonly:  
 117.         l = list(_letters + _letters.upper())  
 118.     else:  
 119.         lowercase = _letters+'0123456789'*2  
 120.         l = list(lowercase + lowercase.upper())  
 121.     shuffle(l)  
 122.     s = ''.join(l)  
 123.     if len(s) < length:  
 124.         s = s + getRandomString(loweronly=1)  
 125.     s = s[:length]  
 126.     if loweronly:  
 127.         return s.lower()  
 128.     else:  
 129.         return s  
 130. testdata = {}  
 131. for i in range(35):  
 132.     k = getRandomString(5, lettersonly=1)  
 133.     v = getRandomString(100 )  
 134.     testdata[k] = v  
 135.       
 136. testdata = [int(x) for x in list('21354612')]  
 137. testdata += list('abcceeaa5efm')  
 138. class X:  
 139.     def __init__(self, n):  
 140.         self.foo = n  
 141.     def __repr__(self):  
 142.         return "<foo %r>"%self.foo  
 143.     def __cmp__(self, e):  
 144.         return cmp(self.foo, e.foo)  
 145.           
 146. testdata = []  
 147. for i in range(10000):  
 148.     testdata.append(getRandomString(3, loweronly=True))  
 149. #testdata = ['f','g','c','d','b','a','a']  
 150. order_preserving = f2, f4, f5, f5b, f8, f10, f11  
 151. order_preserving = f5, f5b, f8, f10, f11  
 152. not_order_preserving = f1, f3, f6, f7, f9  
 153. testfuncs = order_preserving + not_order_preserving  
 154. for f in testfuncs:  
 155.     if f in order_preserving:  
 156.         print "*",  
 157.     timing(f, 100, testdata)

作者的实验结果为：