from itertools import repeat
from operator import attrgetter
class _ItemWrapper(object):
def __init__(self, item):
self.item = item
self.item_id = id(item)
def __hash__(self):
return self.item_id
def __eq__(self, other):
self.item is other.item
def __ne__(self, other):
self.item is not other.item
class _KeyedItemWrapper(_ItemWrapper):
def __init__(self, actual_item, key):
self.actual_item = actual_item
self.item = item = key(actual_item)
self.item_id = id(item)
def iteruniq(iterable, key=None, preserve_order=True, hashable_only=True):
if key is None and not preserve_order and hashable_only:
return iter(set(iterable))
elif preserve_order or hashable_only:
if hashable_only:
memo = set()
add = memo.add
else:
memo = list()
add = memo.append
if key is None:
def predicate(item):
if item in memo:
return False
add(item)
return True
else:
def predicate(actual_item):
item = key(actual_item)
if item in memo:
return False
add(item)
return True
return filter(predicate, iterable)
elif key is None:
return map(attrgetter('item'),
set(map(_ItemWrapper, iterable)))
else:
return map(attrgetter('actual_item'),
set(map(_KeyedItemWrapper, iterable, repeat(key))))
def prep_test():
from random import randint
return [randint(0, 500) for i in range(2000)]
def test1_nokey_noorder_hashonly(seq):
list(iteruniq(seq, preserve_order=False, hashable_only=True))
def test2_nokey_noorder_universal(seq):
list(iteruniq(seq, preserve_order=False, hashable_only=False))
def test3_nokey_withorder_hashonly(seq):
list(iteruniq(seq, preserve_order=True, hashable_only=True))
def test4_nokey_withorder_universal(seq):
list(iteruniq(seq, preserve_order=True, hashable_only=False))
def test5_withkey_noorder_hashonly(seq):
list(iteruniq(seq, str, preserve_order=False, hashable_only=True))
def test6_withkey_noorder_universal(seq):
list(iteruniq(seq, str, preserve_order=False, hashable_only=False))
def test7_withkey_withorder_hashonly(seq):
list(iteruniq(seq, str, preserve_order=True, hashable_only=True))
def test8_withkey_withorder_universal(seq):
list(iteruniq(seq, str, preserve_order=True, hashable_only=False))
if __name__ == '__main__':
import timeit
for name in sorted(globals()):
if name.startswith('test'):
print(name,
timeit.repeat('{}(seq)'.format(name),
setup='from __main__ import prep_test, {}; '
'seq = prep_test()'
.format(name),
number=1000))