1. Lynn Rees
  2. knife

Commits

Lynn Rees  committed 520a41c

- reduce

  • Participants
  • Parent commits ed1af88
  • Branches pu

Comments (0)

Files changed (8)

File knife/_active.py

-# -*- coding: utf-8 -*-
-'''active knives'''
-
-from threading import local
-from collections import deque
-from contextlib import contextmanager
-
-from stuf.deep import clsname
-from stuf.utils import loads, optimize, moptimize
-
-
-class _ActiveMixin(local):
-
-    '''active knife mixin'''
-
-    def __init__(self, *things, **kw):
-        '''
-        :argument things: incoming things
-
-        :keyword integer snapshots: snapshots to keep (default: ``5``)
-        '''
-        incoming = deque()
-        incoming.extend(things)
-        super(_ActiveMixin, self).__init__(incoming, deque(), **kw)
-        # working things
-        self._work = deque()
-        # holding things
-        self._hold = deque()
-
-    @property
-    @contextmanager
-    def _chain(self, d=moptimize):
-        # take snapshot
-        snapshot = d(self._in)
-        # rebalance incoming with outcoming
-        if self._history:
-            self._in.clear()
-            self._in.extend(self._out)
-        # make snapshot original snapshot?
-        else:
-            self._original = snapshot
-        # place snapshot at beginning of snapshot stack
-        self._history.appendleft(snapshot)
-        # move incoming things to working things
-        self._work.extend(self._in)
-        yield
-        out = self._out
-        # clear outgoing things
-        out.clear()
-        # extend outgoing things with holding things
-        out.extend(self._hold)
-        # clear working things
-        self._work.clear()
-        # clear holding things
-        self._hold.clear()
-
-    @property
-    def _iterable(self):
-        # derived from Raymond Hettinger Python Cookbook recipe # 577155
-        call = self._work.popleft
-        try:
-            while 1:
-                yield call()
-        except IndexError:
-            pass
-
-    def _append(self, thing):
-        # append thing after other holding things
-        self._hold.append(thing)
-        return self
-
-    def _xtend(self, things):
-        # place things after holding things
-        self._hold.extend(things)
-        return self
-
-    def _prependit(self, things, d=moptimize):
-        # take snapshot
-        snapshot = d(self._in)
-        # make snapshot original snapshot?
-        if self._original is None:
-            self._original = snapshot
-        # place snapshot at beginning of snapshot stack
-        self._history.appendleft(snapshot)
-        # place thing before other holding things
-        self._in.extendleft(reversed(things))
-        return self
-
-    def _appendit(self, things, d=moptimize):
-        # take snapshot
-        snapshot = d(self._in)
-        # make snapshot original snapshot?
-        if self._original is None:
-            self._original = snapshot
-        # place snapshot at beginning of snapshot stack
-        self._history.appendleft(snapshot)
-        # place things after other incoming things
-        self._in.extend(things)
-        return self
-
-    def _pipeit(self, knife):
-        knife.clear()
-        knife._history.clear()
-        knife._history.extend(self._history)
-        knife._original = self._original
-        knife._baseline = self._baseline
-        knife._out.extend(self._out)
-        knife._worker = self._worker
-        knife._args = self._args
-        knife._kw = self._kw
-        knife._wrapper = self._wrapper
-        knife._pipe = self
-        return knife
-
-    def _unpipeit(self):
-        piped = self._pipe
-        piped.clear()
-        piped._history.clear()
-        piped._history.extend(self._history)
-        piped._original = self._original
-        piped._baseline = self._baseline
-        piped._out.extend(self._out)
-        piped._worker = self._worker
-        piped._args = self._args
-        piped._kw = self._kw
-        piped._wrapper = self._wrapper
-        self.clear()
-        return piped
-
-    def _repr(self, clsname_=clsname, list_=list):
-        # object representation
-        return self._REPR.format(
-            self.__module__,
-            clsname_(self),
-            list_(self._in),
-            list_(self._work),
-            list_(self._hold),
-            list_(self._out),
-        )
-
-    def _len(self, len=len):
-        # length of incoming things
-        return len(self._in)
-
-
-class _OutMixin(_ActiveMixin):
-
-    '''active output mixin'''
-
-    def _undo(self, snapshot=0, loads_=loads):
-        # clear everything
-        self.clear()
-        # if specified, use a specific snapshot
-        if snapshot:
-            self._history.rotate(-(snapshot - 1))
-        try:
-            self._in.extend(loads_(self._history.popleft()))
-        except IndexError:
-            raise IndexError('nothing to undo')
-        return self
-
-    def _snapshot(self, d=optimize):
-        # take baseline snapshot of incoming things
-        self._baseline = d(self._in)
-        return self
-
-    def _rollback(self, loads_=loads):
-        # clear everything
-        self.clear()
-        # clear snapshots
-        self._history.clear()
-        # revert to baseline snapshot of incoming things
-        self._in.extend(loads_(self._baseline))
-        return self
-
-    def _revert(self, loads_=loads):
-        # clear everything
-        self.clear()
-        # clear snapshots
-        self._history.clear()
-        # clear baseline
-        self._baseline = None
-        # restore original snapshot of incoming things
-        self._in.extend(loads_(self._original))
-        return self
-
-    def _clear(self, list_=list):
-        # clear worker
-        self._worker = None
-        # clear worker positional arguments
-        self._args = ()
-        # clear worker keyword arguments
-        self._kw = {}
-        # default iterable wrapper
-        self._wrapper = list_
-        # clear pipe
-        self._pipe = None
-        # clear incoming things
-        self._in.clear()
-        # clear working things
-        self._work.clear()
-        # clear holding things
-        self._hold.clear()
-        # clear outgoing things
-        self._out.clear()
-        return self
-
-    def _iterate(self, iter_=iter):
-        return iter_(self._out)
-
-    def _peek(self, len_=len, list_=list):
-        wrap, out = self._wrapper, self._in
-        value = list_(wrap(i) for i in out) if self._each else wrap(out)
-        self._each = False
-        self._wrapper = list_
-        return value[0] if len_(value) == 1 else value
-
-    def _get(self, len_=len, list_=list):
-        wrap, out = self._wrapper, self._out
-        value = list_(wrap(i) for i in out) if self._each else wrap(out)
-        self._each = False
-        self._wrapper = list_
-        return value[0] if len_(value) == 1 else value

File knife/_base.py

-# -*- coding: utf-8 -*-
-'''base base knife mixins'''
-
-from operator import truth
-from threading import local
-from collections import deque
-
-from stuf.six import identity
-from stuf.utils import memoize
-from stuf.patterns import searcher
-
-SLOTS = (
-    '_in _work _hold _out _original _baseline _each _kw _history _worker '
-    '_wrapper _args _pipe'
-).split()
-
-
-class _KnifeMixin(local):
-
-    '''Base knife mixin.'''
-
-    def __init__(self, ins, outs, **kw):
-        super(_KnifeMixin, self).__init__()
-        # incoming things
-        self._in = ins
-        # outgoing things
-        self._out = outs
-        # pipe out default
-        self._pipe = None
-        # default output default
-        self._each = False
-        # original and baseline snapshots
-        self._original = self._baseline = None
-        # maximum number of history snapshots to keep (default: 5)
-        self._history = deque(maxlen=kw.pop('snapshots', 5))
-        # worker default
-        self._worker = None
-        # position arguments default
-        self._args = ()
-        # keyword arguments default
-        self._kw = {}
-        # default wrapper default
-        self._wrapper = list
-
-    @property
-    def _identity(self):
-        # use  generic identity function for worker if no worker assigned
-        return self._worker if self._worker is not None else identity
-
-    @property
-    def _test(self, truth_=truth):
-        # use truth operator function for worker if no worker assigned
-        return self._worker if self._worker is not None else truth_
-
-    @staticmethod
-    @memoize
-    def _pattern(pat, type, flags, s=searcher):
-        # compile glob pattern into regex
-        return s((type, pat), flags)
-
-    _REPR = '{0}.{1} ([IN: ({2}) => WORK: ({3}) => HOLD: ({4}) => OUT: ({5})])'

File knife/_lazy.py

-# -*- coding: utf-8 -*-
-'''lazily evaluated knives'''
-
-from threading import local
-from itertools import tee, chain
-from contextlib import contextmanager
-
-from stuf.deep import clsname
-from stuf.iterable import count
-
-
-class _LazyMixin(local):
-
-    '''Lazy knife mixin.'''
-
-    def __init__(self, *things, **kw):
-        '''
-        :argument things: incoming things
-        :keyword integer snapshots: snapshots to keep (default: ``5``)
-        '''
-        incoming = (
-            (things[0],).__iter__() if len(things) == 1 else things.__iter__()
-        )
-        super(_LazyMixin, self).__init__(incoming, ().__iter__(), **kw)
-        # working things
-        self._work = ().__iter__()
-        # holding things
-        self._hold = ().__iter__()
-
-    @property
-    @contextmanager
-    def _chain(self, tee_=tee):
-        # take snapshot
-        self._in, snapshot = tee_(self._in)
-        # rebalance incoming with outcoming
-        if self._history:
-            self._in, self._out = tee_(self._out)
-        # make snapshot original snapshot?
-        else:
-            self._original = snapshot
-        # place snapshot at beginning of snapshot stack
-        self._history.appendleft(snapshot)
-        # move incoming things to working things
-        work, self._in = tee_(self._in)
-        self._work = work
-        yield
-        # extend outgoing things with holding things
-        self._out = self._hold
-        # clear working things
-        del self._work
-        self._work = ().__iter__()
-        # clear holding things
-        del self._hold
-        self._hold = ().__iter__()
-
-    @property
-    def _iterable(self):
-        # iterable derived from link in chain
-        return self._work
-
-    def _xtend(self, things, chain_=chain):
-        # place things after holding things
-        self._hold = chain_(things, self._hold)
-        return self
-
-    def _append(self, things, chain_=chain):
-        # append thing after other holding things
-        self._hold = chain_(self._hold, (things,).__iter__())
-        return self
-
-    def _prependit(self, things, tee_=tee, chain_=chain):
-        # take snapshot
-        self._in, snapshot = tee_(self._in)
-        # make snapshot original snapshot?
-        if self._original is None:
-            self._original = snapshot
-        # place snapshot at beginning of snapshot stack
-        self._history.appendleft(snapshot)
-        # place things before other incoming things
-        self._in = chain_(things, self._in)
-        return self
-
-    def _appendit(self, things, tee_=tee, chain_=chain):
-        # take snapshot
-        self._in, snapshot = tee_(self._in)
-        # make snapshot original snapshot?
-        if self._original is None:
-            self._original = snapshot
-        # place snapshot at beginning of snapshot stack
-        self._history.appendleft(snapshot)
-        # place things before other incoming things
-        self._in = chain_(self._in, things)
-        return self
-
-    def _pipeit(self, knife):
-        knife.clear()
-        knife._history.clear()
-        knife._history.extend(self._history)
-        knife._original = self._original
-        knife._baseline = self._baseline
-        knife._out = self._out
-        knife._worker = self._worker
-        knife._args = self._args
-        knife._kw = self._kw
-        knife._wrapper = self._wrapper
-        knife._pipe = self
-        return knife
-
-    def _unpipeit(self):
-        piped = self._pipe
-        piped.clear()
-        piped._history.clear()
-        piped._history.extend(self._history)
-        piped._original = self._original
-        piped._baseline = self._baseline
-        piped._out = self._out
-        piped._worker = self._worker
-        piped._args = self._args
-        piped._kw = self._kw
-        piped._wrapper = self._wrapper
-        return piped
-
-    def _repr(self, tee_=tee, l=list, clsname_=clsname):
-        # object representation
-        self._in, in2 = tee_(self._in)
-        self._out, out2 = tee_(self._out)
-        self._work, work2 = tee_(self._work)
-        self._hold, hold2 = tee_(self._hold)
-        return self._REPR.format(
-            self.__module__,
-            clsname_(self),
-            l(in2),
-            l(work2),
-            l(hold2),
-            l(out2),
-        )
-
-    def _len(self, tee_=tee, count_=count):
-        # length of incoming things
-        self._in, incoming = tee_(self._in)
-        return count_(incoming)
-
-
-class _OutMixin(_LazyMixin):
-
-    '''Lazy output mixin.'''
-
-    def _undo(self, snapshot=0):
-        # clear everything
-        self.clear()
-        # if specified, use a specific snapshot
-        if snapshot:
-            self._history.rotate(-(snapshot - 1))
-        try:
-            self._in = self._history.popleft()
-        except IndexError:
-            raise IndexError('nothing to undo')
-        # clear outgoing things
-        del self._out
-        self._out = ().__iter__()
-        return self
-
-    def _snapshot(self, tee_=tee):
-        # take baseline snapshot of incoming things
-        self._in, self._baseline = tee_(self._in)
-        return self
-
-    def _rollback(self, tee_=tee):
-        # clear everything
-        self.clear()
-        # clear snapshots
-        self._history.clear()
-        # revert to baseline snapshot of incoming things
-        self._in, self._baseline = tee_(self._baseline)
-        return self
-
-    def _revert(self, tee_=tee):
-        # clear everything
-        self.clear()
-        # clear snapshots
-        self._history.clear()
-        # clear baseline
-        self._baseline = None
-        # restore original snapshot of incoming things
-        self._in, self._original = tee_(self._original)
-        return self
-
-    def _clear(self, list_=list):
-        # clear worker
-        self._worker = None
-        # clear worker positional arguments
-        self._args = ()
-        # clear worker keyword arguments
-        self._kw = {}
-        # revert to default iterable wrapper
-        self._wrapper = list_
-        # clear pipe
-        self._pipe = None
-        # clear incoming things
-        del self._in
-        self._in = ().__iter__()
-        # clear working things
-        del self._work
-        self._work = ().__iter__()
-        # clear holding things
-        del self._hold
-        self._hold = ().__iter__()
-        # clear outgoing things
-        del self._out
-        self._out = ().__iter__()
-        return self
-
-    def _iterate(self, tee_=tee):
-        self._out, outs = tee_(self._out)
-        return outs
-
-    def _peek(self, tee_=tee, list_=list, count_=count):
-        tell, self._in, out = tee_(self._in, 3)
-        wrap = self._wrapper
-        value = list_(wrap(i) for i in out) if self._each else wrap(out)
-        # reset each flag
-        self._each = False
-        # reset wrapper
-        self._wrapper = list_
-        return value[0] if count_(tell) == 1 else value
-
-    def _get(self, tee_=tee, list_=list, count_=count):
-        tell, self._out, out = tee_(self._out, 3)
-        wrap = self._wrapper
-        value = list_(wrap(i) for i in out) if self._each else wrap(out)
-        # reset each flag
-        self._each = False
-        # reset wrapper
-        self._wrapper = list_
-        return value[0] if count_(tell) == 1 else value

File knife/_mixins.py

-# -*- coding: utf-8 -*-
-'''specific knife mixins'''
-
-from math import fsum
-from copy import deepcopy
-from threading import local
-from functools import reduce
-from inspect import getmro, isclass
-from random import randrange, shuffle
-from collections import deque, namedtuple
-from operator import attrgetter, itemgetter, methodcaller, truediv
-from itertools import (
-    chain, combinations, groupby, islice, repeat, permutations, starmap, tee)
-
-from stuf.utils import memoize
-from stuf.deep import selfname, members
-from stuf.six.moves import filterfalse, zip_longest  # @UnresolvedImport
-from stuf.iterable import deferfunc, deferiter, count
-from stuf.collects import OrderedDict, Counter, ChainMap
-from stuf.six import filter, items, keys, map, isstring, values, next
-
-Count = namedtuple('Count', 'least most overall')
-GroupBy = namedtuple('Group', 'keys groups')
-MinMax = namedtuple('MinMax', 'min max')
-TrueFalse = namedtuple('TrueFalse', 'true false')
-slicer = lambda x, y: next(islice(x, y, None))
-
-
-class _CmpMixin(local):
-
-    @memoize
-    def _all(self):
-        # invoke worker on each item to yield truth
-        return self._append(all(map(self._test, self._iterable)))
-
-    @memoize
-    def _any(self):
-        # invoke worker on each item to yield truth
-        return self._append(any(map(self._test, self._iterable)))
-
-    @memoize
-    def _difference(self, symmetric):
-        if symmetric:
-            test = lambda x, y: set(x).symmetric_difference(y)
-        else:
-            test = lambda x, y: set(x).difference(y)
-        return self._xtend(reduce(test, self._iterable))
-
-    def _intersection(self):
-        return self._xtend(
-            reduce(lambda x, y: set(x).intersection(y), self._iterable)
-        )
-
-    def _union(self):
-        return self._xtend(
-            reduce(lambda x, y: set(x).union(y), self._iterable)
-        )
-
-    @memoize
-    def _unique(self):
-        def unique(key, iterable):
-            seen = set()
-            seenadd = seen.add
-            try:
-                while 1:
-                    element = key(next(iterable))
-                    if element not in seen:
-                        yield element
-                        seenadd(element)
-            except StopIteration:
-                pass
-        return self._xtend(unique(self._identity, self._iterable))
-
-
-class _MathMixin(local):
-
-    def _average(self):
-        i1, i2 = tee(self._iterable)
-        return self._append(truediv(sum(i1, 0.0), count(i2)))
-
-    def _count(self):
-        cnt = Counter(self._iterable).most_common
-        commonality = cnt()
-        return self._append(Count(
-            # least common
-            commonality[:-2:-1][0][0],
-            # most common (mode)
-            cnt(1)[0][0],
-            # overall commonality
-            commonality,
-        ))
-
-    @memoize
-    def _max(self):
-        return self._append(max(self._iterable, key=self._identity))
-
-    def _median(self):
-        i1, i2 = tee(sorted(self._iterable))
-        result = truediv(count(i1) - 1, 2)
-        pint = int(result)
-        if result % 2 == 0:
-            return self._append(slicer(i2, pint))
-        i3, i4 = tee(i2)
-        return self._append(
-            truediv(slicer(i3, pint) + slicer(i4, pint + 1), 2)
-        )
-
-    def _minmax(self):
-        i1, i2 = tee(self._iterable)
-        return self._append(MinMax(min(i1), max(i2)))
-
-    def _range(self):
-        i1, i2 = tee(sorted(self._iterable))
-        return self._append(deque(i1, maxlen=1).pop() - next(i2))
-
-    @memoize
-    def _min(self):
-        return self._append(min(self._iterable, key=self._identity))
-
-    @memoize
-    def _sum(self, start, floats):
-        return self._append(
-            fsum(self._iterable) if floats else sum(self._iterable, start)
-        )
-
-
-class _OrderMixin(local):
-
-    @memoize
-    def _group(self):
-        def grouper(call, iterable):
-            try:
-                it = groupby(sorted(iterable, key=call), call)
-                while 1:
-                    k, v = next(it)
-                    yield GroupBy(k, tuple(v))
-            except StopIteration:
-                pass
-        return self._xtend(grouper(self._identity, self._iterable))
-
-    def _reverse(self):
-        return self._xtend(reversed(tuple(self._iterable)))
-
-    def _shuffle(self):
-        iterable = list(self._iterable)
-        shuffle(iterable)
-        return self._xtend(iterable)
-
-    @memoize
-    def _sort(self):
-        return self._xtend(sorted(self._iterable, key=self._identity))
-
-
-class _RepeatMixin(local):
-
-    def _combinate(self, n):
-        return self._xtend(combinations(self._iterable, n))
-
-    def _copy(self):
-        return self._xtend(map(deepcopy, self._iterable))
-
-    def _permute(self, n):
-        return self._xtend(permutations(self._iterable, n))
-
-    @memoize
-    def _repeat(self, n, use):
-        call = self._identity
-        if use:
-            return self._xtend(starmap(call, repeat(tuple(self._iterable), n)))
-        return self._xtend(repeat(tuple(self._iterable), n))
-
-
-class _MapMixin(local):
-
-    @memoize
-    def _argmap(self, curr):
-        call = self._identity
-        if curr:
-            def argmap(*args):
-                return call(*(args + self._args))
-            return self._xtend(starmap(argmap, self._iterable))
-        return self._xtend(starmap(call, self._iterable))
-
-    @memoize
-    def _invoke(self, name):
-        def invoke(thing, caller=methodcaller(name, *self._args, **self._kw)):
-            read = caller(thing)
-            return thing if read is None else read
-        return self._xtend(map(invoke, self._iterable))
-
-    @memoize
-    def _kwargmap(self, curr):
-        call = self._identity
-        if curr:
-            def kwargmap(*params):
-                args, kwargs = params
-                kwargs.update(self._kw)
-                return call(*(args + self._args), **kwargs)
-        else:
-            kwargmap = lambda x, y: call(*x, **y)
-        return self._xtend(starmap(kwargmap, self._iterable))
-
-    @memoize
-    def _map(self):
-        return self._xtend(map(self._identity, self._iterable))
-
-    @memoize
-    def _mapping(self, key, value):
-        if key:
-            return self._xtend(map(
-                self._identity, chain.from_iterable(map(keys, self._iterable))
-            ))
-        elif value:
-            return self._xtend(map(self._identity, chain.from_iterable(
-                map(values, self._iterable)
-            )))
-        call = (lambda x, y: (x, y)) if self._worker is None else self._worker
-        return self._xtend(starmap(
-            call, chain.from_iterable(map(items, self._iterable)))
-        )
-
-
-class _FilterMixin(local):
-
-    @memoize
-    def _attrs(self, names):
-        def attrs(iterable, ):
-            try:
-                get = attrgetter(*names)
-                nx = next
-                while 1:
-                    try:
-                        yield get(nx(iterable))
-                    except AttributeError:
-                        pass
-            except StopIteration:
-                pass
-        return self._xtend(attrs(self._iterable))
-
-    @memoize
-    def _duality(self):
-        truth, false = tee(self._iterable)
-        call = self._test
-        return self._append(TrueFalse(
-            tuple(filter(call, truth)), tuple(filterfalse(call, false))
-        ))
-
-    @memoize
-    def _filter(self, false):
-        return self._xtend(
-            (filterfalse if false else filter)(self._worker, self._iterable)
-        )
-
-    @memoize
-    def _items(self, keys):
-        def itemz(iterable):
-            try:
-                get = itemgetter(*keys)
-                nx = next
-                while 1:
-                    try:
-                        yield get(nx(iterable))
-                    except (IndexError, KeyError, TypeError):
-                        pass
-            except StopIteration:
-                pass
-        return self._xtend(itemz(self._iterable))
-
-    @memoize
-    def _traverse(self, invert):
-        if self._worker is None:
-            test = lambda x: not x[0].startswith('__')
-        else:
-            test = self._identity
-        ifilter = filterfalse if invert else filter
-        def members(iterable):  # @IgnorePep8
-            mro = getmro(iterable)
-            names = iter(dir(iterable))
-            beenthere = set()
-            adder = beenthere.add
-            try:
-                OD = OrderedDict
-                vz = vars
-                cn = chain
-                ga = getattr
-                ic = isclass
-                nx = next
-                while 1:
-                    name = nx(names)
-                    # yes, it's really supposed to be a tuple
-                    for base in cn([iterable], mro):
-                        var = vz(base)
-                        if name in var:
-                            obj = var[name]
-                            break
-                    else:
-                        obj = ga(iterable, name)
-                    if (name, obj) in beenthere:
-                        continue
-                    else:
-                        adder((name, obj))
-                    if ic(obj):
-                        yield name, OD((k, v) for k, v in ifilter(
-                            test, members(obj),
-                        ))
-                    else:
-                        yield name, obj
-            except StopIteration:
-                pass
-        def traverse(iterable):  # @IgnorePep8
-            try:
-                iterable = iter(iterable)
-                OD = OrderedDict
-                sn = selfname
-                nx = next
-                while 1:
-                    iterator = nx(iterable)
-                    chaining = ChainMap()
-                    chaining['classname'] = sn(iterator)
-                    cappend = chaining.maps.append
-                    for k, v in ifilter(test, members(iterator)):
-                        if isinstance(v, OD):
-                            v['classname'] = k
-                            cappend(v)
-                        else:
-                            chaining[k] = v
-                    yield chaining
-            except StopIteration:
-                pass
-        return self._xtend(traverse(self._iterable))
-
-    def _mro(self):
-        return self._xtend(chain.from_iterable(map(getmro, self._iterable)))
-
-    def _members(self, invert):
-        if self._worker is None:
-            test = lambda x: not x[0].startswith('__')
-        else:
-            test = self._identity
-        ifilter = filterfalse if invert else filter
-        return self._xtend(ifilter(
-            test, chain.from_iterable(map(members, self._iterable)),
-        ))
-
-
-class _ReduceMixin(local):
-
-    def _flatten(self):
-        def flatten(iterable):
-            nx = next
-            st = isstring
-            next_ = iterable.__iter__()
-            try:
-                while 1:
-                    item = nx(next_)
-                    try:
-                        # don't recur over strings
-                        if st(item):
-                            yield item
-                        else:
-                            # do recur over other things
-                            for j in flatten(item):
-                                yield j
-                    except (AttributeError, TypeError):
-                        # does not recur
-                        yield item
-            except StopIteration:
-                pass
-        return self._xtend(flatten(self._iterable))
-
-    def _merge(self):
-        return self._xtend(chain.from_iterable(self._iterable))
-
-    @memoize
-    def _reduce(self, initial, reverse):
-        call = self._identity
-        if reverse:
-            if initial is None:
-                return self._append(
-                    reduce(lambda x, y: call(y, x), self._iterable)
-                )
-            return self._append(
-                reduce(lambda x, y: call(y, x), self._iterable, initial)
-            )
-        if initial is None:
-            return self._append(reduce(call, self._iterable))
-        return self._append(reduce(call, self._iterable, initial))
-
-    def _zip(self, zip_=zip_longest):
-        return self._xtend(zip_(*self._iterable))
-
-
-class _SliceMixin(local):
-
-    @memoize
-    def _at(self, n, default):
-        return self._append(next(islice(self._iterable, n, None), default))
-
-    @memoize
-    def _choice(self):
-        i1, i2 = tee(self._iterable)
-        return self._append(islice(i1, randrange(0, count(i2)), None))
-
-    @memoize
-    def _dice(self, n, fill):
-        return self._xtend(
-            zip_longest(fillvalue=fill, *[self._iterable.__iter__()] * n)
-        )
-
-    @memoize
-    def _first(self, n=0):
-        return self._xtend(
-            islice(self._iterable, n) if n else deferiter(self._iterable),
-        )
-
-    def _initial(self):
-        i1, i2 = tee(self._iterable)
-        return self._xtend(islice(i1, count(i2) - 1))
-
-    @memoize
-    def _last(self, n):
-        if n:
-            i1, i2 = tee(self._iterable)
-            return self._xtend(islice(i1, count(i2) - n, None))
-        return self._xtend(deferfunc(deque(self._iterable, maxlen=1).pop))
-
-    def _rest(self, iz=islice):
-        return self._xtend(iz(self._iterable, 1, None))
-
-    @memoize
-    def _sample(self, n):
-        i1, i2 = tee(self._iterable)
-        length = count(i1)
-        return self._xtend(
-            map(lambda x: slice(x, randrange(0, length)), tee(i2, n))
-        )
-
-    def _slice(self, start, stop, step):
-        if stop and step:
-            return self._xtend(islice(self._iterable, start, stop, step))
-        elif stop:
-            return self._xtend(islice(self._iterable, start, stop))
-        return self._xtend(islice(self._iterable, start))

File knife/active.py

View file
 # -*- coding: utf-8 -*-
 '''Actively evaluated knives.'''
 
-from knife.base import OutMixin
+from threading import local
+from collections import deque
+from contextlib import contextmanager
+
+from stuf.deep import clsname
+from stuf.utils import loads, optimize, moptimize
+
+from knife.base import KnifeMixin, SLOTS
 from knife.mixins import (
     RepeatMixin, MapMixin, SliceMixin, ReduceMixin, FilterMixin, MathMixin,
     CmpMixin, OrderMixin)
 
-from knife._active import _OutMixin
-from knife._base import SLOTS, _KnifeMixin
-from knife._mixins import (
-    _RepeatMixin, _MapMixin, _SliceMixin, _ReduceMixin, _FilterMixin,
-    _MathMixin, _CmpMixin, _OrderMixin)
+
+class _ActiveMixin(local):
+
+    '''active knife mixin'''
+
+    def __init__(self, *things, **kw):
+        '''
+        :argument things: incoming things
+
+        :keyword integer snapshots: snapshots to keep (default: ``5``)
+        '''
+        incoming = deque()
+        incoming.extend(things)
+        super(_ActiveMixin, self).__init__(incoming, deque(), **kw)
+        # working things
+        self._work = deque()
+        # holding things
+        self._hold = deque()
+
+    @property
+    @contextmanager
+    def _chain(self, d=moptimize):
+        # take snapshot
+        snapshot = d(self._in)
+        # rebalance incoming with outcoming
+        if self._history:
+            self._in.clear()
+            self._in.extend(self._out)
+        # make snapshot original snapshot?
+        else:
+            self._original = snapshot
+        # place snapshot at beginning of snapshot stack
+        self._history.appendleft(snapshot)
+        # move incoming things to working things
+        self._work.extend(self._in)
+        yield
+        out = self._out
+        # clear outgoing things
+        out.clear()
+        # extend outgoing things with holding things
+        out.extend(self._hold)
+        # clear working things
+        self._work.clear()
+        # clear holding things
+        self._hold.clear()
+
+    @property
+    def _iterable(self):
+        # derived from Raymond Hettinger Python Cookbook recipe # 577155
+        call = self._work.popleft
+        try:
+            while 1:
+                yield call()
+        except IndexError:
+            pass
+
+    def _append(self, thing):
+        # append thing after other holding things
+        self._hold.append(thing)
+        return self
+
+    def _xtend(self, things):
+        # place things after holding things
+        self._hold.extend(things)
+        return self
+
+    def _prependit(self, things, d=moptimize):
+        # take snapshot
+        snapshot = d(self._in)
+        # make snapshot original snapshot?
+        if self._original is None:
+            self._original = snapshot
+        # place snapshot at beginning of snapshot stack
+        self._history.appendleft(snapshot)
+        # place thing before other holding things
+        self._in.extendleft(reversed(things))
+        return self
+
+    def _appendit(self, things, d=moptimize):
+        # take snapshot
+        snapshot = d(self._in)
+        # make snapshot original snapshot?
+        if self._original is None:
+            self._original = snapshot
+        # place snapshot at beginning of snapshot stack
+        self._history.appendleft(snapshot)
+        # place things after other incoming things
+        self._in.extend(things)
+        return self
+
+    def _pipeit(self, knife):
+        knife.clear()
+        knife._history.clear()
+        knife._history.extend(self._history)
+        knife._original = self._original
+        knife._baseline = self._baseline
+        knife._out.extend(self._out)
+        knife._worker = self._worker
+        knife._args = self._args
+        knife._kw = self._kw
+        knife._wrapper = self._wrapper
+        knife._pipe = self
+        return knife
+
+    def _unpipeit(self):
+        piped = self._pipe
+        piped.clear()
+        piped._history.clear()
+        piped._history.extend(self._history)
+        piped._original = self._original
+        piped._baseline = self._baseline
+        piped._out.extend(self._out)
+        piped._worker = self._worker
+        piped._args = self._args
+        piped._kw = self._kw
+        piped._wrapper = self._wrapper
+        self.clear()
+        return piped
+
+    def _repr(self, clsname_=clsname, list_=list):
+        # object representation
+        return self._REPR.format(
+            self.__module__,
+            clsname_(self),
+            list_(self._in),
+            list_(self._work),
+            list_(self._hold),
+            list_(self._out),
+        )
+
+    def _len(self, len=len):
+        # length of incoming things
+        return len(self._in)
+
+    def _undo(self, snapshot=0, loads_=loads):
+        # clear everything
+        self.clear()
+        # if specified, use a specific snapshot
+        if snapshot:
+            self._history.rotate(-(snapshot - 1))
+        try:
+            self._in.extend(loads_(self._history.popleft()))
+        except IndexError:
+            raise IndexError('nothing to undo')
+        return self
+
+    def _snapshot(self, d=optimize):
+        # take baseline snapshot of incoming things
+        self._baseline = d(self._in)
+        return self
+
+    def _rollback(self, loads_=loads):
+        # clear everything
+        self.clear()
+        # clear snapshots
+        self._history.clear()
+        # revert to baseline snapshot of incoming things
+        self._in.extend(loads_(self._baseline))
+        return self
+
+    def _revert(self, loads_=loads):
+        # clear everything
+        self.clear()
+        # clear snapshots
+        self._history.clear()
+        # clear baseline
+        self._baseline = None
+        # restore original snapshot of incoming things
+        self._in.extend(loads_(self._original))
+        return self
+
+    def _clear(self, list_=list):
+        # clear worker
+        self._worker = None
+        # clear worker positional arguments
+        self._args = ()
+        # clear worker keyword arguments
+        self._kw = {}
+        # default iterable wrapper
+        self._wrapper = list_
+        # clear pipe
+        self._pipe = None
+        # clear incoming things
+        self._in.clear()
+        # clear working things
+        self._work.clear()
+        # clear holding things
+        self._hold.clear()
+        # clear outgoing things
+        self._out.clear()
+        return self
+
+    def _iterate(self, iter_=iter):
+        return iter_(self._out)
+
+    def _peek(self, len_=len, list_=list):
+        wrap, out = self._wrapper, self._in
+        value = list_(wrap(i) for i in out) if self._each else wrap(out)
+        self._each = False
+        self._wrapper = list_
+        return value[0] if len_(value) == 1 else value
+
+    def _get(self, len_=len, list_=list):
+        wrap, out = self._wrapper, self._out
+        value = list_(wrap(i) for i in out) if self._each else wrap(out)
+        self._each = False
+        self._wrapper = list_
+        return value[0] if len_(value) == 1 else value
 
 
 class activeknife(
-    _OutMixin, _KnifeMixin, _CmpMixin, _FilterMixin, _MapMixin, _MathMixin,
-    _OrderMixin, _ReduceMixin, _SliceMixin, _RepeatMixin,
-    OutMixin, FilterMixin, MapMixin, ReduceMixin, OrderMixin, RepeatMixin,
-    MathMixin, SliceMixin, CmpMixin,
+    _ActiveMixin, KnifeMixin, FilterMixin, MapMixin, ReduceMixin,
+    OrderMixin, RepeatMixin, MathMixin, SliceMixin, CmpMixin,
 ):
 
     '''
     __slots__ = SLOTS
 
 
-class cmpknife(_OutMixin, _KnifeMixin, OutMixin, CmpMixin, _CmpMixin):
+class cmpknife(_ActiveMixin, KnifeMixin, CmpMixin):
 
     '''
     Actively evaluated comparing knife. Provides comparison operations for
     __slots__ = SLOTS
 
 
-class filterknife(_OutMixin, _KnifeMixin, OutMixin, FilterMixin, _FilterMixin):
+class filterknife(_ActiveMixin, KnifeMixin, FilterMixin):
 
     '''
     Actively evaluated filtering knife. Provides filtering operations for
     __slots__ = SLOTS
 
 
-class mapknife(_OutMixin, _KnifeMixin, OutMixin, MapMixin, _MapMixin):
+class mapknife(_ActiveMixin, KnifeMixin, MapMixin):
 
     '''
     Actively evaluated mapping knife. Provides `mapping <http://docs.python.org
     __slots__ = SLOTS
 
 
-class mathknife(_OutMixin, _KnifeMixin, OutMixin, MathMixin, _MathMixin):
+class mathknife(_ActiveMixin, KnifeMixin, MathMixin):
 
     '''
     Actively evaluated mathing knife. Provides numeric and statistical
     __slots__ = SLOTS
 
 
-class orderknife(_OutMixin, _KnifeMixin, OutMixin, OrderMixin, _OrderMixin):
+class orderknife(_ActiveMixin, KnifeMixin, OrderMixin):
 
     '''
     Actively evaluated ordering knife. Provides sorting and grouping operations
     __slots__ = SLOTS
 
 
-class reduceknife(_OutMixin, _KnifeMixin, OutMixin, ReduceMixin, _ReduceMixin):
+class reduceknife(_ActiveMixin, KnifeMixin, ReduceMixin):
 
     '''
     Actively evaluated reducing knife. Provides `reducing <http://docs.python.
     __slots__ = SLOTS
 
 
-class repeatknife(_OutMixin, _KnifeMixin, OutMixin, RepeatMixin, _RepeatMixin):
+class repeatknife(_ActiveMixin, KnifeMixin, RepeatMixin):
 
     '''
     Actively evaluated repeating knife. Provides repetition operations for
     __slots__ = SLOTS
 
 
-class sliceknife(_OutMixin, _KnifeMixin, OutMixin, SliceMixin, _SliceMixin):
+class sliceknife(_ActiveMixin, KnifeMixin, SliceMixin):
 
     '''
     Actively evaluated slicing knife. Provides `slicing <http://docs.python.

File knife/base.py

View file
 # -*- coding: utf-8 -*-
-'''base knife mixins'''
+'''Base knife mixins.'''
 
 from threading import local
 from functools import partial
+from operator import truth
+from collections import deque
 
-from stuf.six import tounicode, tobytes
+from stuf.utils import memoize
+from stuf.patterns import searcher
+from stuf.six import identity, tounicode, tobytes
+
+SLOTS = (
+    '_in _work _hold _out _original _baseline _each _kw _history _worker _args'
+    '_wrapper _pipe'
+).split()
 
 
 class KnifeMixin(local):
 
     '''Base knife mixin.'''
 
+    _REPR = '{0}.{1} ([IN: ({2}) => WORK: ({3}) => HOLD: ({4}) => OUT: ({5})])'
+
+    def __init__(self, ins, outs, **kw):
+        super(KnifeMixin, self).__init__()
+        # incoming things
+        self._in = ins
+        # outgoing things
+        self._out = outs
+        # pipe out default
+        self._pipe = None
+        # default output default
+        self._each = False
+        # original and baseline snapshots
+        self._original = self._baseline = None
+        # maximum number of history snapshots to keep (default: 5)
+        self._history = deque(maxlen=kw.pop('snapshots', 5))
+        # worker default
+        self._worker = None
+        # position arguments default
+        self._args = ()
+        # keyword arguments default
+        self._kw = {}
+        # default wrapper default
+        self._wrapper = list
+
+    @property
+    def _identity(self):
+        # use  generic identity function for worker if no worker assigned
+        return self._worker if self._worker is not None else identity
+
+    @property
+    def _test(self, truth_=truth):
+        # use truth operator function for worker if no worker assigned
+        return self._worker if self._worker is not None else truth_
+
+    @staticmethod
+    @memoize
+    def _pattern(pat, type, flags, s=searcher):
+        # compile glob pattern into regex
+        return s((type, pat), flags)
+
     def apply(self, worker, *args, **kw):
         '''
         Assign :func:`callable` used to work on incoming things plus any
         '''String representation.'''
         return self._repr()
 
-
-class OutMixin(KnifeMixin):
-
-    '''Output mixin.'''
-
     def __iter__(self):
         '''Iterate over outgoing things.'''
         return self._iterate()

File knife/lazy.py

View file
 # -*- coding: utf-8 -*-
 '''Lazier evaluated knives.'''
 
-from knife.base import OutMixin
+from threading import local
+from itertools import tee, chain
+from contextlib import contextmanager
+
+from stuf.deep import clsname
+from stuf.iterable import count
+
+from knife.base import SLOTS, KnifeMixin
 from knife.mixins import (
     RepeatMixin, MapMixin, SliceMixin, ReduceMixin, FilterMixin, MathMixin,
     CmpMixin, OrderMixin)
 
-from knife._lazy import _OutMixin
-from knife._base import SLOTS, _KnifeMixin
-from knife._mixins import (
-    _RepeatMixin, _MapMixin, _SliceMixin, _ReduceMixin, _FilterMixin,
-    _MathMixin, _CmpMixin, _OrderMixin)
+
+class _LazyMixin(local):
+
+    '''Lazy knife mixin.'''
+
+    def __init__(self, *things, **kw):
+        '''
+        :argument things: incoming things
+        :keyword integer snapshots: snapshots to keep (default: ``5``)
+        '''
+        incoming = (
+            (things[0],).__iter__() if len(things) == 1 else things.__iter__()
+        )
+        super(_LazyMixin, self).__init__(incoming, ().__iter__(), **kw)
+        # working things
+        self._work = ().__iter__()
+        # holding things
+        self._hold = ().__iter__()
+
+    @property
+    @contextmanager
+    def _chain(self, tee_=tee):
+        # take snapshot
+        self._in, snapshot = tee_(self._in)
+        # rebalance incoming with outcoming
+        if self._history:
+            self._in, self._out = tee_(self._out)
+        # make snapshot original snapshot?
+        else:
+            self._original = snapshot
+        # place snapshot at beginning of snapshot stack
+        self._history.appendleft(snapshot)
+        # move incoming things to working things
+        work, self._in = tee_(self._in)
+        self._work = work
+        yield
+        # extend outgoing things with holding things
+        self._out = self._hold
+        # clear working things
+        del self._work
+        self._work = ().__iter__()
+        # clear holding things
+        del self._hold
+        self._hold = ().__iter__()
+
+    @property
+    def _iterable(self):
+        # iterable derived from link in chain
+        return self._work
+
+    def _xtend(self, things, chain_=chain):
+        # place things after holding things
+        self._hold = chain_(things, self._hold)
+        return self
+
+    def _append(self, things, chain_=chain):
+        # append thing after other holding things
+        self._hold = chain_(self._hold, (things,).__iter__())
+        return self
+
+    def _prependit(self, things, tee_=tee, chain_=chain):
+        # take snapshot
+        self._in, snapshot = tee_(self._in)
+        # make snapshot original snapshot?
+        if self._original is None:
+            self._original = snapshot
+        # place snapshot at beginning of snapshot stack
+        self._history.appendleft(snapshot)
+        # place things before other incoming things
+        self._in = chain_(things, self._in)
+        return self
+
+    def _appendit(self, things, tee_=tee, chain_=chain):
+        # take snapshot
+        self._in, snapshot = tee_(self._in)
+        # make snapshot original snapshot?
+        if self._original is None:
+            self._original = snapshot
+        # place snapshot at beginning of snapshot stack
+        self._history.appendleft(snapshot)
+        # place things before other incoming things
+        self._in = chain_(self._in, things)
+        return self
+
+    def _pipeit(self, knife):
+        knife.clear()
+        knife._history.clear()
+        knife._history.extend(self._history)
+        knife._original = self._original
+        knife._baseline = self._baseline
+        knife._out = self._out
+        knife._worker = self._worker
+        knife._args = self._args
+        knife._kw = self._kw
+        knife._wrapper = self._wrapper
+        knife._pipe = self
+        return knife
+
+    def _unpipeit(self):
+        piped = self._pipe
+        piped.clear()
+        piped._history.clear()
+        piped._history.extend(self._history)
+        piped._original = self._original
+        piped._baseline = self._baseline
+        piped._out = self._out
+        piped._worker = self._worker
+        piped._args = self._args
+        piped._kw = self._kw
+        piped._wrapper = self._wrapper
+        return piped
+
+    def _repr(self, tee_=tee, l=list, clsname_=clsname):
+        # object representation
+        self._in, in2 = tee_(self._in)
+        self._out, out2 = tee_(self._out)
+        self._work, work2 = tee_(self._work)
+        self._hold, hold2 = tee_(self._hold)
+        return self._REPR.format(
+            self.__module__,
+            clsname_(self),
+            l(in2),
+            l(work2),
+            l(hold2),
+            l(out2),
+        )
+
+    def _len(self, tee_=tee, count_=count):
+        # length of incoming things
+        self._in, incoming = tee_(self._in)
+        return count_(incoming)
+
+    def _undo(self, snapshot=0):
+        # clear everything
+        self.clear()
+        # if specified, use a specific snapshot
+        if snapshot:
+            self._history.rotate(-(snapshot - 1))
+        try:
+            self._in = self._history.popleft()
+        except IndexError:
+            raise IndexError('nothing to undo')
+        # clear outgoing things
+        del self._out
+        self._out = ().__iter__()
+        return self
+
+    def _snapshot(self, tee_=tee):
+        # take baseline snapshot of incoming things
+        self._in, self._baseline = tee_(self._in)
+        return self
+
+    def _rollback(self, tee_=tee):
+        # clear everything
+        self.clear()
+        # clear snapshots
+        self._history.clear()
+        # revert to baseline snapshot of incoming things
+        self._in, self._baseline = tee_(self._baseline)
+        return self
+
+    def _revert(self, tee_=tee):
+        # clear everything
+        self.clear()
+        # clear snapshots
+        self._history.clear()
+        # clear baseline
+        self._baseline = None
+        # restore original snapshot of incoming things
+        self._in, self._original = tee_(self._original)
+        return self
+
+    def _clear(self, list_=list):
+        # clear worker
+        self._worker = None
+        # clear worker positional arguments
+        self._args = ()
+        # clear worker keyword arguments
+        self._kw = {}
+        # revert to default iterable wrapper
+        self._wrapper = list_
+        # clear pipe
+        self._pipe = None
+        # clear incoming things
+        del self._in
+        self._in = ().__iter__()
+        # clear working things
+        del self._work
+        self._work = ().__iter__()
+        # clear holding things
+        del self._hold
+        self._hold = ().__iter__()
+        # clear outgoing things
+        del self._out
+        self._out = ().__iter__()
+        return self
+
+    def _iterate(self, tee_=tee):
+        self._out, outs = tee_(self._out)
+        return outs
+
+    def _peek(self, tee_=tee, list_=list, count_=count):
+        tell, self._in, out = tee_(self._in, 3)
+        wrap = self._wrapper
+        value = list_(wrap(i) for i in out) if self._each else wrap(out)
+        # reset each flag
+        self._each = False
+        # reset wrapper
+        self._wrapper = list_
+        return value[0] if count_(tell) == 1 else value
+
+    def _get(self, tee_=tee, list_=list, count_=count):
+        tell, self._out, out = tee_(self._out, 3)
+        wrap = self._wrapper
+        value = list_(wrap(i) for i in out) if self._each else wrap(out)
+        # reset each flag
+        self._each = False
+        # reset wrapper
+        self._wrapper = list_
+        return value[0] if count_(tell) == 1 else value
 
 
 class lazyknife(
-    _OutMixin, _KnifeMixin, _CmpMixin, _FilterMixin, _MapMixin, _MathMixin,
-    _OrderMixin, _ReduceMixin, _SliceMixin, _RepeatMixin,
-    OutMixin, FilterMixin, MapMixin, ReduceMixin, OrderMixin, RepeatMixin,
-    MathMixin, SliceMixin, CmpMixin,
+    _LazyMixin, KnifeMixin, FilterMixin, MapMixin, ReduceMixin, OrderMixin,
+    RepeatMixin, MathMixin, SliceMixin, CmpMixin,
 ):
 
     '''
     __slots__ = SLOTS
 
 
-class cmpknife(_OutMixin, _KnifeMixin, OutMixin, CmpMixin, _CmpMixin):
+class cmpknife(_LazyMixin, KnifeMixin, CmpMixin):
 
     '''
     Lazier evaluated comparing knife. Provides comparison operations for
     __slots__ = SLOTS
 
 
-class filterknife(_OutMixin, _KnifeMixin, OutMixin, FilterMixin, _FilterMixin):
+class filterknife(_LazyMixin, KnifeMixin, FilterMixin):
 
     '''
     Lazier evaluated filtering knife. Provides filtering operations for
     __slots__ = SLOTS
 
 
-class mapknife(_OutMixin, _KnifeMixin, OutMixin, MapMixin, _MapMixin):
+class mapknife(_LazyMixin, KnifeMixin, MapMixin):
 
     '''
     Lazier evaluated mapping knife. Provides `mapping <http://docs.python.org
     __slots__ = SLOTS
 
 
-class mathknife(_OutMixin, _KnifeMixin, OutMixin, MathMixin, _MathMixin):
+class mathknife(_LazyMixin, KnifeMixin, MathMixin):
 
     '''
     Lazier evaluated mathing knife. Provides numeric and statistical
     __slots__ = SLOTS
 
 
-class orderknife(_OutMixin, _KnifeMixin, OutMixin, OrderMixin, _OrderMixin):
+class orderknife(_LazyMixin, KnifeMixin, OrderMixin):
 
     '''
     Lazier evaluated ordering knife. Provides sorting and grouping operations
     __slots__ = SLOTS
 
 
-class reduceknife(_OutMixin, _KnifeMixin, OutMixin, ReduceMixin, _ReduceMixin):
+class reduceknife(_LazyMixin, KnifeMixin, ReduceMixin):
 
     '''
     Lazier evaluated reducing knife. Provides `reducing <http://docs.python.
     __slots__ = SLOTS
 
 
-class repeatknife(_OutMixin, _KnifeMixin, OutMixin, RepeatMixin, _RepeatMixin):
+class repeatknife(_LazyMixin, KnifeMixin, RepeatMixin):
 
     '''
     Lazier evaluated repeating knife. Provides repetition operations for
     __slots__ = SLOTS
 
 
-class sliceknife(_OutMixin, _KnifeMixin, OutMixin, SliceMixin, _SliceMixin):
+class sliceknife(_LazyMixin, KnifeMixin, SliceMixin):
 
     '''
     Lazier evaluated slicing knife. Provides `slicing <http://docs.python.

File knife/mixins.py

View file
 # -*- coding: utf-8 -*-
 '''knife mixins.'''
 
+from math import fsum
+from copy import deepcopy
 from threading import local
+from functools import reduce
+from inspect import getmro, isclass
+from random import randrange, shuffle
+from collections import deque, namedtuple
+from operator import attrgetter, itemgetter, methodcaller, truediv
+from itertools import (
+    chain, combinations, groupby, islice, repeat, permutations, starmap, tee)
+
+from stuf.deep import selfname, members
+from stuf.six.moves import filterfalse, zip_longest  # @UnresolvedImport
+from stuf.iterable import deferfunc, deferiter, count
+from stuf.collects import OrderedDict, Counter, ChainMap
+from stuf.six import (
+    filter, items, keys as keyz, map, isstring, values as valuez, next)
+
+Count = namedtuple('Count', 'least most overall')
+GroupBy = namedtuple('Group', 'keys groups')
+MinMax = namedtuple('MinMax', 'min max')
+TrueFalse = namedtuple('TrueFalse', 'true false')
+slicer = lambda x, y: next(islice(x, y, None))
 
 
 class CmpMixin(local):
             function in Underscore.php
         '''
         with self._chain:
-            return self._all()
+            return self._append(all(map(self._test, self._iterable)))
 
     def any(self):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._any()
+            return self._append(any(map(self._test, self._iterable)))
 
     def difference(self, symmetric=False):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._difference(symmetric)
+            if symmetric:
+                test = lambda x, y: set(x).symmetric_difference(y)
+            else:
+                test = lambda x, y: set(x).difference(y)
+            return self._xtend(reduce(test, self._iterable))
 
     def intersection(self):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._intersection()
+            return self._xtend(
+                reduce(lambda x, y: set(x).intersection(y), self._iterable)
+            )
 
     def union(self):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._union()
+            return self._xtend(
+                reduce(lambda x, y: set(x).union(y), self._iterable)
+            )
 
     def unique(self):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._unique()
+            def unique(key, iterable):
+                seen = set()
+                seenadd = seen.add
+                try:
+                    while 1:
+                        element = key(next(iterable))
+                        if element not in seen:
+                            yield element
+                            seenadd(element)
+                except StopIteration:
+                    pass
+            return self._xtend(unique(self._identity, self._iterable))
 
 
 class MathMixin(local):
         31.666666666666668
         '''
         with self._chain:
-            return self._average()
+            i1, i2 = tee(self._iterable)
+            return self._append(truediv(sum(i1, 0.0), count(i2)))
 
     def count(self):
         '''
         [(11, 3), (3, 2), (5, 2), (7, 1)]
         '''
         with self._chain:
-            return self._count()
+            cnt = Counter(self._iterable).most_common
+            commonality = cnt()
+            return self._append(Count(
+                # least common
+                commonality[:-2:-1][0][0],
+                # most common (mode)
+                cnt(1)[0][0],
+                # overall commonality
+                commonality,
+            ))
 
     def max(self):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._max()
+            return self._append(max(self._iterable, key=self._identity))
 
     def median(self):
         '''
         4.5
         '''
         with self._chain:
-            return self._median()
+            i1, i2 = tee(sorted(self._iterable))
+            result = truediv(count(i1) - 1, 2)
+            pint = int(result)
+            if result % 2 == 0:
+                return self._append(slicer(i2, pint))
+            i3, i4 = tee(i2)
+            return self._append(
+                truediv(slicer(i3, pint) + slicer(i4, pint + 1), 2)
+            )
 
     def min(self):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._min()
+            return self._append(min(self._iterable, key=self._identity))
 
     def minmax(self):
         '''
         4
         '''
         with self._chain:
-            return self._minmax()
+            i1, i2 = tee(self._iterable)
+            return self._append(MinMax(min(i1), max(i2)))
 
     def range(self):
         '''
         8
         '''
         with self._chain:
-            return self._range()
+            i1, i2 = tee(sorted(self._iterable))
+            return self._append(deque(i1, maxlen=1).pop() - next(i2))
 
     def sum(self, start=0, precision=False):
         '''
             function in Python standard library
         '''
         with self._chain:
-            return self._sum(start, precision)
+            return self._append(
+                fsum(self._iterable) if precision
+                else sum(self._iterable, start)
+            )
 
 
 class OrderMixin(local):
             function in Underscore.php
         '''
         with self._chain:
-            return self._group()
+            def grouper(call, iterable):
+                try:
+                    it = groupby(sorted(iterable, key=call), call)
+                    while 1:
+                        k, v = next(it)
+                        yield GroupBy(k, tuple(v))
+                except StopIteration:
+                    pass
+            return self._xtend(grouper(self._identity, self._iterable))
 
     def reverse(self):
         '''
             function in Underscore.lua
         '''
         with self._chain:
-            return self._reverse()
+            return self._xtend(reversed(tuple(self._iterable)))
 
     def shuffle(self):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._shuffle()
+            iterable = list(self._iterable)
+            shuffle(iterable)
+            return self._xtend(iterable)
 
     def sort(self):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._sort()
+            return self._xtend(sorted(self._iterable, key=self._identity))
 
 
 class RepeatMixin(local):
             function in Python standard library
         '''
         with self._chain:
-            return self._combinate(n)
+            return self._xtend(combinations(self._iterable, n))
 
     def copy(self):
         '''
             function in Python standard library
         '''
         with self._chain:
-            return self._copy()
+            return self._xtend(map(deepcopy, self._iterable))
 
     def permutate(self, n):
         '''
             function in Python standard library
         '''
         with self._chain:
-            return self._permute(n)
+            return self._xtend(permutations(self._iterable, n))
 
     def repeat(self, n=None, call=False):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._repeat(n, call)
+            call = self._identity
+            if call:
+                return self._xtend(
+                    starmap(call, repeat(tuple(self._iterable), n))
+                )
+            return self._xtend(repeat(tuple(self._iterable), n))
 
 
 class MapMixin(local):
             function in Python standard library
         '''
         with self._chain:
-            return self._argmap(merge)
+            call = self._identity
+            if merge:
+                def argmap(*args):
+                    return call(*(args + self._args))
+                return self._xtend(starmap(argmap, self._iterable))
+            return self._xtend(starmap(call, self._iterable))
 
     def invoke(self, name):
         '''
           `invoke <http://brianhaveri.github.com/Underscore.php/#invoke>`_
             function in Underscore.php
         '''
+        caller = methodcaller(name, *self._args, **self._kw)
         with self._chain:
-            return self._invoke(name)
+            def invoke(thing, caller=caller):
+                read = caller(thing)
+                return thing if read is None else read
+            return self._xtend(map(invoke, self._iterable))
 
     def kwargmap(self, merge=False):
         '''
         [270, 330, 390]
         '''
         with self._chain:
-            return self._kwargmap(merge)
+            call = self._identity
+            if merge:
+                def kwargmap(*params):
+                    args, kwargs = params
+                    kwargs.update(self._kw)
+                    return call(*(args + self._args), **kwargs)
+            else:
+                kwargmap = lambda x, y: call(*x, **y)
+            return self._xtend(starmap(kwargmap, self._iterable))
 
     def map(self):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._map()
+            return self._xtend(map(self._identity, self._iterable))
 
     def mapping(self, keys=False, values=False):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._mapping(keys, values)
+            if keys:
+                return self._xtend(map(
+                    self._identity,
+                    chain.from_iterable(map(keyz, self._iterable))
+                ))
+            elif values:
+                return self._xtend(map(self._identity, chain.from_iterable(
+                    map(valuez, self._iterable)
+                )))
+            call = (
+                (lambda x, y: (x, y)) if self._worker is None else self._worker
+            )
+            return self._xtend(starmap(
+                call, chain.from_iterable(map(items, self._iterable)))
+            )
 
 
 class FilterMixin(local):
             function in Underscore.js
         '''
         with self._chain:
-            return self._attrs(names)
+            def attrs(iterable, ):
+                try:
+                    get = attrgetter(*names)
+                    nx = next
+                    while 1:
+                        try:
+                            yield get(nx(iterable))
+                        except AttributeError:
+                            pass
+                except StopIteration:
+                    pass
+            return self._xtend(attrs(self._iterable))
 
     def duality(self):
         '''
         (1, 3, 5)
         '''
         with self._chain:
-            return self._duality()
+            truth, false = tee(self._iterable)
+            call = self._test
+            return self._append(TrueFalse(
+                tuple(filter(call, truth)), tuple(filterfalse(call, false))
+            ))
 
     def filter(self, invert=False):
         '''
             function in Underscore.php
         '''
         with self._chain:
-            return self._filter(invert)
+            return self._xtend(
+            (filterfalse if invert else filter)(self._worker, self._iterable)
+            )
 
     def items(self, *keys):
         '''
             function in Underscore.js
         '''
         with self._chain:
-            return self._items(keys)
+            def itemz(iterable):
+                try:
+                    get = itemgetter(*keys)
+                    nx = next
+                    while 1:
+                        try:
+                            yield get(nx(iterable))
+                        except (IndexError, KeyError, TypeError):
+                            pass
+                except StopIteration:
+                    pass
+            return self._xtend(itemz(self._iterable))
 
     def traverse(self, invert=False):
         '''
         OrderedDict([('age', 969), ('classname', 'stooge4')]))]
         '''
         with self._chain:
-            return self._traverse(invert)
+            if self._worker is None:
+                test = lambda x: not x[0].startswith('__')
+            else:
+                test = self._identity
+            ifilter = filterfalse if invert else filter
+            def members(iterable):  # @IgnorePep8
+                mro = getmro(iterable)
+                names = iter(dir(iterable))
+                beenthere = set()
+                adder = beenthere.add
+                try:
+                    OD = OrderedDict
+                    vz = vars
+                    cn = chain
+                    ga = getattr
+                    ic = isclass
+                    nx = next
+                    while 1:
+                        name = nx(names)
+                        # yes, it's really supposed to be a tuple
+                        for base in cn([iterable], mro):
+                            var = vz(base)
+                            if name in var:
+                                obj = var[name]
+                                break
+                        else:
+                            obj = ga(iterable, name)
+                        if (name, obj) in beenthere:
+                            continue
+                        else:
+                            adder((name, obj))
+                        if ic(obj):
+                            yield name, OD((k, v) for k, v in ifilter(
+                                test, members(obj),
+                            ))
+                        else:
+                            yield name, obj
+                except StopIteration:
+                    pass
+            def traverse(iterable):  # @IgnorePep8
+                try:
+                    iterable = iter(iterable)
+                    OD = OrderedDict
+                    sn = selfname
+                    nx = next
+                    while 1:
+                        iterator = nx(iterable)
+                        chaining = ChainMap()
+                        chaining['classname'] = sn(iterator)
+                        cappend = chaining.maps.append
+                        for k, v in ifilter(test, members(iterator)):
+                            if isinstance(v, OD):
+                                v['classname'] = k
+                                cappend(v)
+                            else:
+                                chaining[k] = v
+                        yield chaining
+                except StopIteration:
+                    pass
+            return self._xtend(traverse(self._iterable))
 
     def members(self, inverse=False):
         '''
         :rtype: :obj:`knife` :term:`object`
         '''
         with self._chain:
-            return self._members(inverse)
+            if self._worker is None:
+                test = lambda x: not x[0].startswith('__')
+            else:
+                test = self._identity
+            ifilter = filterfalse if inverse else filter
+            return self._xtend(ifilter(
+                test, chain.from_iterable(map(members, self._iterable)),
+            ))
 
     def mro(self):
         with self._chain:
-            return self._mro()
+            return self._xtend(
+                chain.from_iterable(map(getmro, self._iterable))
+            )
 
 
 class ReduceMixin(local):
             function in Underscore.php
         '''
         with self._chain:
-            return self._flatten()
+            def flatten(iterable):
+                nx = next
+                st = isstring
+                next_ = iterable.__iter__()
+                try:
+                    while 1:
+                        item = nx(next_)
+                        try:
+                            # don't recur over strings
+                            if st(item):
+                                yield item
+                            else:
+                                # do recur over other things
+                                for j in flatten(item):
+                                    yield j
+                        except (AttributeError, TypeError):
+                            # does not recur
+                            yield item
+                except StopIteration:
+                    pass
+            return self._xtend(flatten(self._iterable))