From 8698e33a77f20109635a894cf38bb2bcfeb4e028 Mon Sep 17 00:00:00 2001 From: rkompass Date: Thu, 1 Dec 2022 20:19:08 +0100 Subject: [PATCH 1/2] updates:4 additions: 13 tests updated: accumulate, chain, islice, tee; added: chain.from_iterable, combinations, combinations_with_replacement,compress, dropwhile, filterfalse, groupby, pairwise, permutations, product,takewhile, zip_longest; also added tests for all functions --- python-stdlib/itertools/itertools.py | 341 ++++++++++++++++++---- python-stdlib/itertools/manifest.py | 2 +- python-stdlib/itertools/test_itertools.py | 137 +++++++-- 3 files changed, 413 insertions(+), 67 deletions(-) diff --git a/python-stdlib/itertools/itertools.py b/python-stdlib/itertools/itertools.py index 9bf1b215..df868ee5 100644 --- a/python-stdlib/itertools/itertools.py +++ b/python-stdlib/itertools/itertools.py @@ -1,74 +1,317 @@ +# Python itertools adapted for Micropython by rkompass (2022) +# Largely, but not exclusively based on code from the offical Python documentation +# (https://docs.python.org/3/library/itertools.html) +# Copyright 2001-2019 Python Software Foundation; All Rights Reserved + +# consumes about 5kB if imported + +# accumulate([1,2,3,4,5]) --> 1 3 6 10 15 +# accumulate([1,2,3,4,5], initial=100) --> 100 101 103 106 110 115 +# accumulate([1,2,3,4,5], lambda x, y: x * y) --> 1 2 6 24 120 +def accumulate(iterable, func=lambda x, y: x + y, initial=None): + it = iter(iterable) + total = initial + if initial is None: + try: + total = next(it) + except StopIteration: + return + yield total + for element in it: + total = func(total, element) + yield total + +# chain('abcd',[],range(5))) --> 'a' 'b' 'c' 'd' 0 1 2 3 4 +class chain: + def __init__(self, *iterables): + self.iterables = list(iterables) + self.it = iter([]) + def __iter__(self): + return self + def __next__(self): + while True: + try: + return next(self.it) + except StopIteration: + try: + self.it = iter(self.iterables.pop(0)) + continue + except IndexError: + raise StopIteration + # chain.from_iterable(['ABC', 'DEF']) --> 'A' 'B' 'C' 'D' 'E' 'F' + @staticmethod + def from_iterable(iterables): + for it in iterables: + yield from it + +# combinations('ABCD', 2) --> ('A','B') ('A','C') ('A','D') ('B','C') ('B','D') ('C','D') +def combinations(iterable, r): + pool = tuple(iterable) + n = len(pool) + if r > n: + return + indices = list(range(r)) + yield tuple(pool[i] for i in indices) + while True: + index = 0 + for i in reversed(range(r)): + if indices[i] != i + n - r: + index = i + break + else: + return + indices[index] += 1 + for j in range(index + 1, r): + indices[j] = indices[j - 1] + 1 + yield tuple(pool[i] for i in indices) + +# combinations_with_replacement('ABC', 2) --> ('A','A') ('A','B') ('A','C') ('B','B') ('B','C') ('C','C') +def combinations_with_replacement(iterable, r): + pool = tuple(iterable) + n = len(pool) + if not n and r: + return + indices = [0] * r + yield tuple(pool[i] for i in indices) + while True: + index = 0 + for i in reversed(range(r)): + if indices[i] != n - 1: + index = i + break + else: + return + indices[index:] = [indices[index] + 1] * (r - index) + yield tuple(pool[i] for i in indices) + +# compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F +def compress(data, selectors): + return (d for d, s in zip(data, selectors) if s) + +# count(4, 3) --> 4 7 10 13 16 19 .... def count(start=0, step=1): while True: yield start start += step - -def cycle(p): +# cycle('abc') --> a b c a b c a b c a .... +def cycle(iterable): try: - len(p) - except TypeError: - # len() is not defined for this type. Assume it is - # a finite iterable so we must cache the elements. + len(iterable) + except TypeError: # len() not defined: Assume p is a finite iterable: We cache the elements. cache = [] - for i in p: + for i in iterable: yield i cache.append(i) - p = cache - while p: - yield from p + iterable = cache + while iterable: + yield from iterable +# # dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1 +def dropwhile(predicate, iterable): + it = iter(iterable) + for x in it: + if not predicate(x): + yield x + break + for x in it: + yield x -def repeat(el, n=None): - if n is None: - while True: - yield el - else: - for i in range(n): - yield el - - -def chain(*p): - for i in p: - yield from i - - -def islice(p, start, stop=(), step=1): - if stop == (): - stop = start - start = 0 - # TODO: optimizing or breaking semantics? - if start >= stop: - return - it = iter(p) - for i in range(start): - next(it) +# filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8 +def filterfalse(predicate, iterable): + if predicate is None: + predicate = bool + for x in iterable: + if not predicate(x): + yield x +# groupby('aaaabbbccdaa'))) --> ('a', gen1) ('b', gen2) ('c', gen3) ('d', gen4) ('a', gen5) +# where gen1 --> a a a a, gen2 --> b b b, gen3 --> c c, gen4 --> d, gen5 --> a a +def groupby(iterable, key=None): + it = iter(iterable) + keyf = key if key is not None else lambda x: x + def ggen(ktgt): + nonlocal cur, kcur + while kcur == ktgt: + yield cur + try: + cur = next(it); kcur = keyf(cur) + except StopIteration: + break + kcur = kold = object() # need an object that never can be a returned from key function while True: - yield next(it) - for i in range(step - 1): + while kcur == kold: # not all iterables with the same (old) key were used up by ggen, so use them up here + try: + cur = next(it); kcur = keyf(cur) + except StopIteration: + return + kold = kcur + yield (kcur, ggen(kcur)) + +# islice('abcdefghij', 2, None, 3)) --> c f i +# islice(range(10), 2, 6, 2)) --> 2 4 +def islice(iterable, *sargs): + if len(sargs) < 1 or len(sargs) > 3: + raise TypeError('islice expected at least 2, at most 4 arguments, got {:d}'.format(len(sargs)+1)) + step = 1 if len(sargs) < 3 else sargs[2] + step = 1 if step is None else step + if step <= 0: + raise ValueError('step for islice() must be a positive integer or None') + start = 0 if len(sargs) < 2 else sargs[0] + stop = sargs[0] if len(sargs) == 1 else sargs[1] + it = iter(iterable) + try: + for i in range(start): next(it) - start += step - if start >= stop: + while True: + if stop is not None and start >= stop: + return + yield next(it) + for i in range(step - 1): + next(it) + start += step + except StopIteration: + return + +# pairwise(range(5)) --> (0,1) (1,2) (2,3) (3,4) +# pairwise('abcdefg') --> ('a','b') ('b','c') ('c','d') ('d','e') ('e','f') ('f','g') +def pairwise(iterable): + it = iter(iterable) + try: + l = next(it) + while True: + c = next(it) + yield l, c + l = c + except StopIteration: + return + +# permutations('ABCD', 2) --> AB AC AD BA BC BD CA CB CD DA DB DC +# permutations(range(3)) --> 012 021 102 120 201 210 +def permutations(iterable, r=None): + pool = tuple(iterable) + n = len(pool) + r = n if r is None else r + if r > n: + return + indices = list(range(n)) + cycles = list(range(n, n - r, -1)) + yield tuple(pool[i] for i in indices[:r]) + while n: + for i in reversed(range(r)): + cycles[i] -= 1 + if cycles[i] == 0: + indices[i:] = indices[i + 1 :] + indices[i : i + 1] + cycles[i] = n - i + else: + j = cycles[i] + indices[i], indices[-j] = indices[-j], indices[i] + yield tuple(pool[i] for i in indices[:r]) + break + else: return +# product('ABCD', 'xy') --> ('A','x') ('A','y') ('B','x') ('B','y') ('C','x') ('C','y') ('D','x') ('D','y') +# product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 # but in tuples, of course +def product(*args, repeat=1): + pools = [tuple(pool) for pool in args] * repeat + result = [[]] + for pool in pools: + result = [x + [y] for x in result for y in pool] + for prod in result: + yield tuple(prod) -def tee(iterable, n=2): - return [iter(iterable)] * n - +# repeat(10, 3) --> 10 10 10 +def repeat(obj, times=None): + if times is None: + while True: + yield obj + else: + for _ in range(times): + yield obj +# starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000 def starmap(function, iterable): for args in iterable: yield function(*args) +# takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4 +def takewhile(predicate, iterable): + for x in iterable: + if predicate(x): + yield x + else: + break -def accumulate(iterable, func=lambda x, y: x + y): +# tee(range(2,10), 3) --> (it1, it2, it3) all parallel generators, but dependent on original generator (e.g. range(2,10)) +# --> (min(it1), max(it2), sum(it3)) --> (2, 9, 44) +def tee(iterable, n=2): + if iter(iterable) is not iter(iterable): # save buffer for special cases that iterable is range, tuple, list ... + return [iter(iterable) for _ in range(n)] # that have independent iterators it = iter(iterable) - try: - acc = next(it) - except StopIteration: + if n < 1: + return () + elif n == 1: + return (it,) + buf = [] # Buffer, contains stored values from itr + ibuf = [0]*n # Indices of the individual generators, could be array('H', [0]*n) + def gen(k): # but we have no 0 in ibuf in MP + nonlocal buf, ibuf # These are bound to the generators as closures + while True: + if ibuf[k] < len(buf): # We get an object stored in the buffer. + r = buf[ibuf[k]] + ibuf[k] += 1 + if ibuf[k] == 1: # If we got the first object in the buffer, + if 0 not in ibuf: # then check if other generators do not wait anymore on it + buf.pop(0) # so it may be popped left. Afterwards decrease all indices by 1. + for i in range(n): + ibuf[i] -= 1 + elif ibuf[k] == len(buf): + try: + r = next(it) + buf.append(r) + ibuf[k] += 1 + except StopIteration: + return + yield r # The returned generators are not thread-safe. For that the access to the + return tuple(gen(i) for i in range(n)) # shared buf and ibuf should be protected by locks. + +# zip_longest('ABCD', 'xy', fillvalue='-') --> ('A','x') ('B','y') ('C','-') ('D','-') +def zip_longest(*args, fillvalue=None): + iterators = [iter(it) for it in args] + num_active = len(iterators) + if not num_active: return - yield acc - for element in it: - acc = func(acc, element) - yield acc + while True: + values = [] + for i, it in enumerate(iterators): + try: + value = next(it) + except StopIteration: + num_active -= 1 + if not num_active: + return + iterators[i] = repeat(fillvalue) + value = fillvalue + values.append(value) + yield tuple(values) + + +# # Full analog of CPython builtin iter with 2 arguments +# def iter(*args): +# +# if len(args) == 1: +# return builtins.iter(args[0]) +# +# class _iter: +# +# def __init__(self, args): +# self.f, self.sentinel = args +# def __next__(self): +# v = self.f() +# if v == self.sentinel: +# raise StopIteration +# return v +# +# return _iter(args) + diff --git a/python-stdlib/itertools/manifest.py b/python-stdlib/itertools/manifest.py index 80ebd502..5ef1d675 100644 --- a/python-stdlib/itertools/manifest.py +++ b/python-stdlib/itertools/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.2.3") +metadata(version="0.5.0") module("itertools.py") diff --git a/python-stdlib/itertools/test_itertools.py b/python-stdlib/itertools/test_itertools.py index bd16ce0b..bbd4bafa 100644 --- a/python-stdlib/itertools/test_itertools.py +++ b/python-stdlib/itertools/test_itertools.py @@ -1,24 +1,127 @@ import itertools -assert list(itertools.islice(list(range(10)), 4)) == [0, 1, 2, 3] -assert list(itertools.islice(list(range(10)), 2, 6)) == [2, 3, 4, 5] -assert list(itertools.islice(list(range(10)), 2, 6, 2)) == [2, 4] - - -def g(): - while True: - yield 123 - - -assert list(itertools.islice(g(), 5)) == [123, 123, 123, 123, 123] - -assert list(itertools.islice(itertools.cycle([1, 2, 3]), 10)) == [1, 2, 3, 1, 2, 3, 1, 2, 3, 1] -assert list(itertools.islice(itertools.cycle(reversed([1, 2, 3])), 7)) == [3, 2, 1, 3, 2, 1, 3] - -assert list(itertools.starmap(lambda x, y: x * y, [[1, 2], [2, 3], [3, 4]])) == [2, 6, 12] - +# accumulate assert list(itertools.accumulate([])) == [] assert list(itertools.accumulate([0])) == [0] assert list(itertools.accumulate([0, 2, 3])) == [0, 2, 5] assert list(itertools.accumulate(reversed([0, 2, 3]))) == [3, 5, 5] assert list(itertools.accumulate([1, 2, 3], lambda x, y: x * y)) == [1, 2, 6] +assert list(itertools.accumulate([1,2,3,4,5], func=lambda x, y: x - y, initial=10)) == [10, 9, 7, 4, 0, -5] + +# chain +assert list(itertools.chain()) == [] +assert list(itertools.chain([],[],[])) == [] +assert list(itertools.chain(range(3),[2*(i+1) for i in range(4)])) == [0, 1, 2, 2, 4, 6, 8] +assert list(itertools.chain('abcd',[],range(5))) == ['a', 'b', 'c', 'd', 0, 1, 2, 3, 4] + +assert list(itertools.chain.from_iterable([])) == [] +assert list(itertools.chain.from_iterable(['ABC', 'DEF'])) == ['A', 'B', 'C', 'D', 'E', 'F'] + +# combinations +assert list(itertools.combinations('', 1)) == [] +assert list(itertools.combinations('ABCD', 0)) == [()] +assert list(itertools.combinations('ABCD', 1)) == [('A',), ('B',), ('C',), ('D',)] +assert list(itertools.combinations('ABCD', 3)) == [('A', 'B', 'C'), ('A', 'B', 'D'), ('A', 'C', 'D'), ('B', 'C', 'D')] +assert list(itertools.combinations('ABCD', 4)) == [('A', 'B', 'C', 'D')] +assert list(itertools.combinations('ABCD', 5)) == [] +assert len(list(itertools.combinations(range(7), 4))) == 35 +assert len(set(itertools.combinations(range(7), 4))) == 35 + +# combinations with replacement +assert list(itertools.combinations_with_replacement('ABCD', 0)) == [()] +assert list(itertools.combinations_with_replacement('ABCD', 1)) == [('A',), ('B',), ('C',), ('D',)] +assert list(itertools.combinations_with_replacement('ABC', 2)) == [('A', 'A'), ('A', 'B'), ('A', 'C'), ('B', 'B'), ('B', 'C'), ('C', 'C')] +assert list(itertools.combinations_with_replacement('ABC', 3)) == [('A', 'A', 'A'), ('A', 'A', 'B'), ('A', 'A', 'C'), ('A', 'B', 'B'), ('A', 'B', 'C'), ('A', 'C', 'C'), ('B', 'B', 'B'), ('B', 'B', 'C'), ('B', 'C', 'C'), ('C', 'C', 'C')] + +# compress +assert tuple(itertools.compress('ABCDEF', (1,0,1,0,1,1))) == ('A', 'C', 'E', 'F') +assert tuple(itertools.compress('ABCDEF', (1,0,1,1,0))) == ('A', 'C', 'D') + +# count +it = itertools.count(4, 3) +for _ in range(200): + n = next(it) +assert list(next(it) for _ in range(5)) == [604, 607, 610, 613, 616] + +# cycle +it = itertools.cycle(iter('abcde')) +assert list(next(it) for _ in range(12)) == ['a', 'b', 'c', 'd', 'e', 'a', 'b', 'c', 'd', 'e', 'a', 'b'] +it = itertools.cycle([2, 4, 'x']) +assert list(next(it) for _ in range(7)) == [2, 4, 'x', 2, 4, 'x', 2] + +# dropwhile +assert list(itertools.dropwhile(lambda x: x<5, [1,4,6,4,1])) == [6, 4, 1] +assert list(itertools.dropwhile(lambda x: ord(x)<118, '')) == [] +assert list(itertools.dropwhile(lambda x: ord(x)<118, 'dropwhile')) == ['w', 'h', 'i', 'l', 'e'] + +# filterfalse +assert list(itertools.filterfalse(lambda x: ord(x)<110, 'dropwhile')) == ['r', 'o', 'p', 'w'] + +# groupby +assert list(((k,''.join(g)) for k,g in itertools.groupby('aaaabbbccdaa'))) == [('a', 'aaaa'), ('b', 'bbb'), ('c', 'cc'), ('d', 'd'), ('a', 'aa')] + +# islice +assert ''.join(itertools.islice('', 2, 5)) == '' +assert ''.join(itertools.islice('abcdefgh', 2, 5)) == 'cde' +assert ''.join(itertools.islice('abcdefghij', 2, None, 3)) == 'cfi' +assert ''.join(itertools.islice('abcdefghij', 2, None)) == 'cdefghij' +assert ''.join(itertools.islice('abcdefghij', 6)) == 'abcdef' +assert ''.join(itertools.islice('abcdefghij', 6, 6)) == '' +assert list(itertools.islice(range(10), 2, 6, 2)) == [2, 4] +assert list(itertools.islice(itertools.cycle([1, 2, 3]), 10)) == [1, 2, 3, 1, 2, 3, 1, 2, 3, 1] + +# pairwise +assert list(itertools.pairwise(range(5))) == [(0, 1), (1, 2), (2, 3), (3, 4)] +assert list((''.join(t) for t in itertools.pairwise('abcdefg'))) == ['ab', 'bc', 'cd', 'de', 'ef', 'fg'] +assert list((''.join(t) for t in itertools.pairwise('ab'))) == ['ab'] +assert list((''.join(t) for t in itertools.pairwise('a'))) == [] +assert list((''.join(t) for t in itertools.pairwise(''))) == [] + +# permutations +assert list(itertools.permutations('', 1)) == [] +assert list((''.join(t) for t in itertools.permutations('a', 2))) == [] +assert list((''.join(t) for t in itertools.permutations('abcd', 0))) == [''] +assert list((''.join(t) for t in itertools.permutations('ab', 2))) == ['ab', 'ba'] +assert list((''.join(t) for t in itertools.permutations('abcd', 2))) == ['ab', 'ac', 'ad', 'ba', 'bc', 'bd', 'ca', 'cb', 'cd', 'da', 'db', 'dc'] +assert list(itertools.permutations(range(3))) == [(0, 1, 2), (0, 2, 1), (1, 0, 2), (1, 2, 0), (2, 0, 1), (2, 1, 0)] + +# product +assert list(itertools.product()) == [()] +assert list(itertools.product(range(2), repeat=0)) == [()] +assert list((''.join(t) for t in itertools.product('ABC', 'xy'))) == ['Ax', 'Ay', 'Bx', 'By', 'Cx', 'Cy'] +assert list((''.join(t) for t in itertools.product('A', 'xy', repeat=2))) == ['AxAx', 'AxAy', 'AyAx', 'AyAy'] +assert list((''.join(map(str,t)) for t in itertools.product(range(2), repeat=3))) == ['000','001','010','011','100','101','110','111'] + +# repeat +assert list(itertools.repeat(10, 0)) == [] +assert list(itertools.repeat(10, 1)) == [10] +assert list(itertools.repeat(10, 3)) == [10, 10, 10] + +# starmap +assert list(itertools.starmap(pow, [])) == [] +assert list(itertools.starmap(pow, [(2,5), (3,2), (10,3)])) == [32, 9, 1000] +assert list(itertools.starmap(lambda x, y: x * y, [[1, 2], [2, 3], [3, 4]])) == [2, 6, 12] + +# takewhile +assert list(itertools.takewhile(lambda x: x<5, [1,4,6,4,1])) == [1, 4] +assert list(itertools.takewhile(lambda x: ord(x)<118, 'dropwhile')) == ['d', 'r', 'o', 'p'] + +# tee +def genx(n): + i=1; + while True: + yield i; i+=1 + if i >n: + return +it1, it2, it3 = itertools.tee(genx(1000), 3); _ = next(it1) # case of iterable that is unique; iterate once +assert [min(it1), max(it2), sum(it3)] == [2, 1000, 500500] +it1, it2, it3 = itertools.tee(range(2,10), 3); _ = next(it1) # iterable that is not unique; iterate once +assert [min(it1), max(it2), sum(it3)] == [3, 9, 44] # the min is increased, other iterators remained full + +# zip_longest +assert list(itertools.zip_longest('', '')) == [] +assert list(itertools.zip_longest('', '', fillvalue='-')) == [] +assert list(itertools.zip_longest('', 'xy')) == [(None, 'x'), (None, 'y')] +assert list(itertools.zip_longest('', 'xy', fillvalue='-')) == [('-', 'x'), ('-', 'y')] +assert list(itertools.zip_longest('ABCD', 'xy', fillvalue='-')) == [('A','x'),('B','y'),('C','-'),('D','-')] + From c8b81047e86eb79eda8097a75ca69f55d80e8b31 Mon Sep 17 00:00:00 2001 From: rkompass Date: Fri, 2 Dec 2022 01:54:15 +0100 Subject: [PATCH 2/2] applied codeformat --- python-stdlib/itertools/itertools.py | 88 +++++--- python-stdlib/itertools/test_itertools.py | 232 ++++++++++++++++------ 2 files changed, 232 insertions(+), 88 deletions(-) diff --git a/python-stdlib/itertools/itertools.py b/python-stdlib/itertools/itertools.py index df868ee5..b019472f 100644 --- a/python-stdlib/itertools/itertools.py +++ b/python-stdlib/itertools/itertools.py @@ -21,13 +21,16 @@ def accumulate(iterable, func=lambda x, y: x + y, initial=None): total = func(total, element) yield total + # chain('abcd',[],range(5))) --> 'a' 'b' 'c' 'd' 0 1 2 3 4 class chain: def __init__(self, *iterables): self.iterables = list(iterables) self.it = iter([]) + def __iter__(self): return self + def __next__(self): while True: try: @@ -38,12 +41,14 @@ class chain: continue except IndexError: raise StopIteration + # chain.from_iterable(['ABC', 'DEF']) --> 'A' 'B' 'C' 'D' 'E' 'F' @staticmethod - def from_iterable(iterables): + def from_iterable(iterables): for it in iterables: yield from it + # combinations('ABCD', 2) --> ('A','B') ('A','C') ('A','D') ('B','C') ('B','D') ('C','D') def combinations(iterable, r): pool = tuple(iterable) @@ -65,6 +70,7 @@ def combinations(iterable, r): indices[j] = indices[j - 1] + 1 yield tuple(pool[i] for i in indices) + # combinations_with_replacement('ABC', 2) --> ('A','A') ('A','B') ('A','C') ('B','B') ('B','C') ('C','C') def combinations_with_replacement(iterable, r): pool = tuple(iterable) @@ -84,21 +90,24 @@ def combinations_with_replacement(iterable, r): indices[index:] = [indices[index] + 1] * (r - index) yield tuple(pool[i] for i in indices) + # compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F def compress(data, selectors): return (d for d, s in zip(data, selectors) if s) + # count(4, 3) --> 4 7 10 13 16 19 .... def count(start=0, step=1): while True: yield start start += step + # cycle('abc') --> a b c a b c a b c a .... def cycle(iterable): try: len(iterable) - except TypeError: # len() not defined: Assume p is a finite iterable: We cache the elements. + except TypeError: # len() not defined: Assume p is a finite iterable: We cache the elements. cache = [] for i in iterable: yield i @@ -107,6 +116,7 @@ def cycle(iterable): while iterable: yield from iterable + # # dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1 def dropwhile(predicate, iterable): it = iter(iterable) @@ -117,6 +127,7 @@ def dropwhile(predicate, iterable): for x in it: yield x + # filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8 def filterfalse(predicate, iterable): if predicate is None: @@ -125,38 +136,48 @@ def filterfalse(predicate, iterable): if not predicate(x): yield x + # groupby('aaaabbbccdaa'))) --> ('a', gen1) ('b', gen2) ('c', gen3) ('d', gen4) ('a', gen5) # where gen1 --> a a a a, gen2 --> b b b, gen3 --> c c, gen4 --> d, gen5 --> a a def groupby(iterable, key=None): it = iter(iterable) keyf = key if key is not None else lambda x: x + def ggen(ktgt): nonlocal cur, kcur while kcur == ktgt: yield cur try: - cur = next(it); kcur = keyf(cur) + cur = next(it) + kcur = keyf(cur) except StopIteration: - break - kcur = kold = object() # need an object that never can be a returned from key function + break + + kcur = kold = object() # need an object that never can be a returned from key function while True: - while kcur == kold: # not all iterables with the same (old) key were used up by ggen, so use them up here + while ( + kcur == kold + ): # not all iterables with the same (old) key were used up by ggen, so use them up here try: - cur = next(it); kcur = keyf(cur) + cur = next(it) + kcur = keyf(cur) except StopIteration: return kold = kcur yield (kcur, ggen(kcur)) + # islice('abcdefghij', 2, None, 3)) --> c f i # islice(range(10), 2, 6, 2)) --> 2 4 def islice(iterable, *sargs): if len(sargs) < 1 or len(sargs) > 3: - raise TypeError('islice expected at least 2, at most 4 arguments, got {:d}'.format(len(sargs)+1)) + raise TypeError( + "islice expected at least 2, at most 4 arguments, got {:d}".format(len(sargs) + 1) + ) step = 1 if len(sargs) < 3 else sargs[2] step = 1 if step is None else step if step <= 0: - raise ValueError('step for islice() must be a positive integer or None') + raise ValueError("step for islice() must be a positive integer or None") start = 0 if len(sargs) < 2 else sargs[0] stop = sargs[0] if len(sargs) == 1 else sargs[1] it = iter(iterable) @@ -173,10 +194,11 @@ def islice(iterable, *sargs): except StopIteration: return + # pairwise(range(5)) --> (0,1) (1,2) (2,3) (3,4) # pairwise('abcdefg') --> ('a','b') ('b','c') ('c','d') ('d','e') ('e','f') ('f','g') def pairwise(iterable): - it = iter(iterable) + it = iter(iterable) try: l = next(it) while True: @@ -184,7 +206,8 @@ def pairwise(iterable): yield l, c l = c except StopIteration: - return + return + # permutations('ABCD', 2) --> AB AC AD BA BC BD CA CB CD DA DB DC # permutations(range(3)) --> 012 021 102 120 201 210 @@ -211,6 +234,7 @@ def permutations(iterable, r=None): else: return + # product('ABCD', 'xy') --> ('A','x') ('A','y') ('B','x') ('B','y') ('C','x') ('C','y') ('D','x') ('D','y') # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 # but in tuples, of course def product(*args, repeat=1): @@ -221,6 +245,7 @@ def product(*args, repeat=1): for prod in result: yield tuple(prod) + # repeat(10, 3) --> 10 10 10 def repeat(obj, times=None): if times is None: @@ -230,11 +255,13 @@ def repeat(obj, times=None): for _ in range(times): yield obj + # starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000 def starmap(function, iterable): for args in iterable: yield function(*args) + # takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4 def takewhile(predicate, iterable): for x in iterable: @@ -243,27 +270,33 @@ def takewhile(predicate, iterable): else: break + # tee(range(2,10), 3) --> (it1, it2, it3) all parallel generators, but dependent on original generator (e.g. range(2,10)) # --> (min(it1), max(it2), sum(it3)) --> (2, 9, 44) def tee(iterable, n=2): - if iter(iterable) is not iter(iterable): # save buffer for special cases that iterable is range, tuple, list ... + if iter(iterable) is not iter( + iterable + ): # save buffer for special cases that iterable is range, tuple, list ... return [iter(iterable) for _ in range(n)] # that have independent iterators it = iter(iterable) if n < 1: return () elif n == 1: return (it,) - buf = [] # Buffer, contains stored values from itr - ibuf = [0]*n # Indices of the individual generators, could be array('H', [0]*n) - def gen(k): # but we have no 0 in ibuf in MP - nonlocal buf, ibuf # These are bound to the generators as closures + buf = [] # Buffer, contains stored values from itr + ibuf = [0] * n # Indices of the individual generators, could be array('H', [0]*n) + + def gen(k): # but we have no 0 in ibuf in MP + nonlocal buf, ibuf # These are bound to the generators as closures while True: - if ibuf[k] < len(buf): # We get an object stored in the buffer. + if ibuf[k] < len(buf): # We get an object stored in the buffer. r = buf[ibuf[k]] ibuf[k] += 1 - if ibuf[k] == 1: # If we got the first object in the buffer, - if 0 not in ibuf: # then check if other generators do not wait anymore on it - buf.pop(0) # so it may be popped left. Afterwards decrease all indices by 1. + if ibuf[k] == 1: # If we got the first object in the buffer, + if 0 not in ibuf: # then check if other generators do not wait anymore on it + buf.pop( + 0 + ) # so it may be popped left. Afterwards decrease all indices by 1. for i in range(n): ibuf[i] -= 1 elif ibuf[k] == len(buf): @@ -273,8 +306,10 @@ def tee(iterable, n=2): ibuf[k] += 1 except StopIteration: return - yield r # The returned generators are not thread-safe. For that the access to the - return tuple(gen(i) for i in range(n)) # shared buf and ibuf should be protected by locks. + yield r # The returned generators are not thread-safe. For that the access to the + + return tuple(gen(i) for i in range(n)) # shared buf and ibuf should be protected by locks. + # zip_longest('ABCD', 'xy', fillvalue='-') --> ('A','x') ('B','y') ('C','-') ('D','-') def zip_longest(*args, fillvalue=None): @@ -299,12 +334,12 @@ def zip_longest(*args, fillvalue=None): # # Full analog of CPython builtin iter with 2 arguments # def iter(*args): -# +# # if len(args) == 1: # return builtins.iter(args[0]) -# +# # class _iter: -# +# # def __init__(self, args): # self.f, self.sentinel = args # def __next__(self): @@ -312,6 +347,5 @@ def zip_longest(*args, fillvalue=None): # if v == self.sentinel: # raise StopIteration # return v -# +# # return _iter(args) - diff --git a/python-stdlib/itertools/test_itertools.py b/python-stdlib/itertools/test_itertools.py index bbd4bafa..75069c67 100644 --- a/python-stdlib/itertools/test_itertools.py +++ b/python-stdlib/itertools/test_itertools.py @@ -6,36 +6,66 @@ assert list(itertools.accumulate([0])) == [0] assert list(itertools.accumulate([0, 2, 3])) == [0, 2, 5] assert list(itertools.accumulate(reversed([0, 2, 3]))) == [3, 5, 5] assert list(itertools.accumulate([1, 2, 3], lambda x, y: x * y)) == [1, 2, 6] -assert list(itertools.accumulate([1,2,3,4,5], func=lambda x, y: x - y, initial=10)) == [10, 9, 7, 4, 0, -5] +assert list(itertools.accumulate([1, 2, 3, 4, 5], func=lambda x, y: x - y, initial=10)) == [ + 10, + 9, + 7, + 4, + 0, + -5, +] # chain assert list(itertools.chain()) == [] -assert list(itertools.chain([],[],[])) == [] -assert list(itertools.chain(range(3),[2*(i+1) for i in range(4)])) == [0, 1, 2, 2, 4, 6, 8] -assert list(itertools.chain('abcd',[],range(5))) == ['a', 'b', 'c', 'd', 0, 1, 2, 3, 4] +assert list(itertools.chain([], [], [])) == [] +assert list(itertools.chain(range(3), [2 * (i + 1) for i in range(4)])) == [0, 1, 2, 2, 4, 6, 8] +assert list(itertools.chain("abcd", [], range(5))) == ["a", "b", "c", "d", 0, 1, 2, 3, 4] assert list(itertools.chain.from_iterable([])) == [] -assert list(itertools.chain.from_iterable(['ABC', 'DEF'])) == ['A', 'B', 'C', 'D', 'E', 'F'] +assert list(itertools.chain.from_iterable(["ABC", "DEF"])) == ["A", "B", "C", "D", "E", "F"] # combinations -assert list(itertools.combinations('', 1)) == [] -assert list(itertools.combinations('ABCD', 0)) == [()] -assert list(itertools.combinations('ABCD', 1)) == [('A',), ('B',), ('C',), ('D',)] -assert list(itertools.combinations('ABCD', 3)) == [('A', 'B', 'C'), ('A', 'B', 'D'), ('A', 'C', 'D'), ('B', 'C', 'D')] -assert list(itertools.combinations('ABCD', 4)) == [('A', 'B', 'C', 'D')] -assert list(itertools.combinations('ABCD', 5)) == [] +assert list(itertools.combinations("", 1)) == [] +assert list(itertools.combinations("ABCD", 0)) == [()] +assert list(itertools.combinations("ABCD", 1)) == [("A",), ("B",), ("C",), ("D",)] +assert list(itertools.combinations("ABCD", 3)) == [ + ("A", "B", "C"), + ("A", "B", "D"), + ("A", "C", "D"), + ("B", "C", "D"), +] +assert list(itertools.combinations("ABCD", 4)) == [("A", "B", "C", "D")] +assert list(itertools.combinations("ABCD", 5)) == [] assert len(list(itertools.combinations(range(7), 4))) == 35 assert len(set(itertools.combinations(range(7), 4))) == 35 # combinations with replacement -assert list(itertools.combinations_with_replacement('ABCD', 0)) == [()] -assert list(itertools.combinations_with_replacement('ABCD', 1)) == [('A',), ('B',), ('C',), ('D',)] -assert list(itertools.combinations_with_replacement('ABC', 2)) == [('A', 'A'), ('A', 'B'), ('A', 'C'), ('B', 'B'), ('B', 'C'), ('C', 'C')] -assert list(itertools.combinations_with_replacement('ABC', 3)) == [('A', 'A', 'A'), ('A', 'A', 'B'), ('A', 'A', 'C'), ('A', 'B', 'B'), ('A', 'B', 'C'), ('A', 'C', 'C'), ('B', 'B', 'B'), ('B', 'B', 'C'), ('B', 'C', 'C'), ('C', 'C', 'C')] +assert list(itertools.combinations_with_replacement("ABCD", 0)) == [()] +assert list(itertools.combinations_with_replacement("ABCD", 1)) == [("A",), ("B",), ("C",), ("D",)] +assert list(itertools.combinations_with_replacement("ABC", 2)) == [ + ("A", "A"), + ("A", "B"), + ("A", "C"), + ("B", "B"), + ("B", "C"), + ("C", "C"), +] +assert list(itertools.combinations_with_replacement("ABC", 3)) == [ + ("A", "A", "A"), + ("A", "A", "B"), + ("A", "A", "C"), + ("A", "B", "B"), + ("A", "B", "C"), + ("A", "C", "C"), + ("B", "B", "B"), + ("B", "B", "C"), + ("B", "C", "C"), + ("C", "C", "C"), +] # compress -assert tuple(itertools.compress('ABCDEF', (1,0,1,0,1,1))) == ('A', 'C', 'E', 'F') -assert tuple(itertools.compress('ABCDEF', (1,0,1,1,0))) == ('A', 'C', 'D') +assert tuple(itertools.compress("ABCDEF", (1, 0, 1, 0, 1, 1))) == ("A", "C", "E", "F") +assert tuple(itertools.compress("ABCDEF", (1, 0, 1, 1, 0))) == ("A", "C", "D") # count it = itertools.count(4, 3) @@ -44,53 +74,120 @@ for _ in range(200): assert list(next(it) for _ in range(5)) == [604, 607, 610, 613, 616] # cycle -it = itertools.cycle(iter('abcde')) -assert list(next(it) for _ in range(12)) == ['a', 'b', 'c', 'd', 'e', 'a', 'b', 'c', 'd', 'e', 'a', 'b'] -it = itertools.cycle([2, 4, 'x']) -assert list(next(it) for _ in range(7)) == [2, 4, 'x', 2, 4, 'x', 2] +it = itertools.cycle(iter("abcde")) +assert list(next(it) for _ in range(12)) == [ + "a", + "b", + "c", + "d", + "e", + "a", + "b", + "c", + "d", + "e", + "a", + "b", +] +it = itertools.cycle([2, 4, "x"]) +assert list(next(it) for _ in range(7)) == [2, 4, "x", 2, 4, "x", 2] # dropwhile -assert list(itertools.dropwhile(lambda x: x<5, [1,4,6,4,1])) == [6, 4, 1] -assert list(itertools.dropwhile(lambda x: ord(x)<118, '')) == [] -assert list(itertools.dropwhile(lambda x: ord(x)<118, 'dropwhile')) == ['w', 'h', 'i', 'l', 'e'] +assert list(itertools.dropwhile(lambda x: x < 5, [1, 4, 6, 4, 1])) == [6, 4, 1] +assert list(itertools.dropwhile(lambda x: ord(x) < 118, "")) == [] +assert list(itertools.dropwhile(lambda x: ord(x) < 118, "dropwhile")) == ["w", "h", "i", "l", "e"] # filterfalse -assert list(itertools.filterfalse(lambda x: ord(x)<110, 'dropwhile')) == ['r', 'o', 'p', 'w'] +assert list(itertools.filterfalse(lambda x: ord(x) < 110, "dropwhile")) == ["r", "o", "p", "w"] # groupby -assert list(((k,''.join(g)) for k,g in itertools.groupby('aaaabbbccdaa'))) == [('a', 'aaaa'), ('b', 'bbb'), ('c', 'cc'), ('d', 'd'), ('a', 'aa')] +assert list(((k, "".join(g)) for k, g in itertools.groupby("aaaabbbccdaa"))) == [ + ("a", "aaaa"), + ("b", "bbb"), + ("c", "cc"), + ("d", "d"), + ("a", "aa"), +] # islice -assert ''.join(itertools.islice('', 2, 5)) == '' -assert ''.join(itertools.islice('abcdefgh', 2, 5)) == 'cde' -assert ''.join(itertools.islice('abcdefghij', 2, None, 3)) == 'cfi' -assert ''.join(itertools.islice('abcdefghij', 2, None)) == 'cdefghij' -assert ''.join(itertools.islice('abcdefghij', 6)) == 'abcdef' -assert ''.join(itertools.islice('abcdefghij', 6, 6)) == '' +assert "".join(itertools.islice("", 2, 5)) == "" +assert "".join(itertools.islice("abcdefgh", 2, 5)) == "cde" +assert "".join(itertools.islice("abcdefghij", 2, None, 3)) == "cfi" +assert "".join(itertools.islice("abcdefghij", 2, None)) == "cdefghij" +assert "".join(itertools.islice("abcdefghij", 6)) == "abcdef" +assert "".join(itertools.islice("abcdefghij", 6, 6)) == "" assert list(itertools.islice(range(10), 2, 6, 2)) == [2, 4] assert list(itertools.islice(itertools.cycle([1, 2, 3]), 10)) == [1, 2, 3, 1, 2, 3, 1, 2, 3, 1] # pairwise assert list(itertools.pairwise(range(5))) == [(0, 1), (1, 2), (2, 3), (3, 4)] -assert list((''.join(t) for t in itertools.pairwise('abcdefg'))) == ['ab', 'bc', 'cd', 'de', 'ef', 'fg'] -assert list((''.join(t) for t in itertools.pairwise('ab'))) == ['ab'] -assert list((''.join(t) for t in itertools.pairwise('a'))) == [] -assert list((''.join(t) for t in itertools.pairwise(''))) == [] +assert list(("".join(t) for t in itertools.pairwise("abcdefg"))) == [ + "ab", + "bc", + "cd", + "de", + "ef", + "fg", +] +assert list(("".join(t) for t in itertools.pairwise("ab"))) == ["ab"] +assert list(("".join(t) for t in itertools.pairwise("a"))) == [] +assert list(("".join(t) for t in itertools.pairwise(""))) == [] # permutations -assert list(itertools.permutations('', 1)) == [] -assert list((''.join(t) for t in itertools.permutations('a', 2))) == [] -assert list((''.join(t) for t in itertools.permutations('abcd', 0))) == [''] -assert list((''.join(t) for t in itertools.permutations('ab', 2))) == ['ab', 'ba'] -assert list((''.join(t) for t in itertools.permutations('abcd', 2))) == ['ab', 'ac', 'ad', 'ba', 'bc', 'bd', 'ca', 'cb', 'cd', 'da', 'db', 'dc'] -assert list(itertools.permutations(range(3))) == [(0, 1, 2), (0, 2, 1), (1, 0, 2), (1, 2, 0), (2, 0, 1), (2, 1, 0)] +assert list(itertools.permutations("", 1)) == [] +assert list(("".join(t) for t in itertools.permutations("a", 2))) == [] +assert list(("".join(t) for t in itertools.permutations("abcd", 0))) == [""] +assert list(("".join(t) for t in itertools.permutations("ab", 2))) == ["ab", "ba"] +assert list(("".join(t) for t in itertools.permutations("abcd", 2))) == [ + "ab", + "ac", + "ad", + "ba", + "bc", + "bd", + "ca", + "cb", + "cd", + "da", + "db", + "dc", +] +assert list(itertools.permutations(range(3))) == [ + (0, 1, 2), + (0, 2, 1), + (1, 0, 2), + (1, 2, 0), + (2, 0, 1), + (2, 1, 0), +] # product assert list(itertools.product()) == [()] assert list(itertools.product(range(2), repeat=0)) == [()] -assert list((''.join(t) for t in itertools.product('ABC', 'xy'))) == ['Ax', 'Ay', 'Bx', 'By', 'Cx', 'Cy'] -assert list((''.join(t) for t in itertools.product('A', 'xy', repeat=2))) == ['AxAx', 'AxAy', 'AyAx', 'AyAy'] -assert list((''.join(map(str,t)) for t in itertools.product(range(2), repeat=3))) == ['000','001','010','011','100','101','110','111'] +assert list(("".join(t) for t in itertools.product("ABC", "xy"))) == [ + "Ax", + "Ay", + "Bx", + "By", + "Cx", + "Cy", +] +assert list(("".join(t) for t in itertools.product("A", "xy", repeat=2))) == [ + "AxAx", + "AxAy", + "AyAx", + "AyAy", +] +assert list(("".join(map(str, t)) for t in itertools.product(range(2), repeat=3))) == [ + "000", + "001", + "010", + "011", + "100", + "101", + "110", + "111", +] # repeat assert list(itertools.repeat(10, 0)) == [] @@ -99,29 +196,42 @@ assert list(itertools.repeat(10, 3)) == [10, 10, 10] # starmap assert list(itertools.starmap(pow, [])) == [] -assert list(itertools.starmap(pow, [(2,5), (3,2), (10,3)])) == [32, 9, 1000] +assert list(itertools.starmap(pow, [(2, 5), (3, 2), (10, 3)])) == [32, 9, 1000] assert list(itertools.starmap(lambda x, y: x * y, [[1, 2], [2, 3], [3, 4]])) == [2, 6, 12] # takewhile -assert list(itertools.takewhile(lambda x: x<5, [1,4,6,4,1])) == [1, 4] -assert list(itertools.takewhile(lambda x: ord(x)<118, 'dropwhile')) == ['d', 'r', 'o', 'p'] +assert list(itertools.takewhile(lambda x: x < 5, [1, 4, 6, 4, 1])) == [1, 4] +assert list(itertools.takewhile(lambda x: ord(x) < 118, "dropwhile")) == ["d", "r", "o", "p"] # tee def genx(n): - i=1; + i = 1 while True: - yield i; i+=1 - if i >n: + yield i + i += 1 + if i > n: return -it1, it2, it3 = itertools.tee(genx(1000), 3); _ = next(it1) # case of iterable that is unique; iterate once -assert [min(it1), max(it2), sum(it3)] == [2, 1000, 500500] -it1, it2, it3 = itertools.tee(range(2,10), 3); _ = next(it1) # iterable that is not unique; iterate once -assert [min(it1), max(it2), sum(it3)] == [3, 9, 44] # the min is increased, other iterators remained full + + +it1, it2, it3 = itertools.tee(genx(1000), 3) +_ = next(it1) # case of iterable that is unique; iterate once +assert [min(it1), max(it2), sum(it3)] == [2, 1000, 500500] +it1, it2, it3 = itertools.tee(range(2, 10), 3) +_ = next(it1) # iterable that is not unique; iterate once +assert [min(it1), max(it2), sum(it3)] == [ + 3, + 9, + 44, +] # the min is increased, other iterators remained full # zip_longest -assert list(itertools.zip_longest('', '')) == [] -assert list(itertools.zip_longest('', '', fillvalue='-')) == [] -assert list(itertools.zip_longest('', 'xy')) == [(None, 'x'), (None, 'y')] -assert list(itertools.zip_longest('', 'xy', fillvalue='-')) == [('-', 'x'), ('-', 'y')] -assert list(itertools.zip_longest('ABCD', 'xy', fillvalue='-')) == [('A','x'),('B','y'),('C','-'),('D','-')] - +assert list(itertools.zip_longest("", "")) == [] +assert list(itertools.zip_longest("", "", fillvalue="-")) == [] +assert list(itertools.zip_longest("", "xy")) == [(None, "x"), (None, "y")] +assert list(itertools.zip_longest("", "xy", fillvalue="-")) == [("-", "x"), ("-", "y")] +assert list(itertools.zip_longest("ABCD", "xy", fillvalue="-")) == [ + ("A", "x"), + ("B", "y"), + ("C", "-"), + ("D", "-"), +]