Commits

Lynn Rees committed 381af4a

- extend functionality

Comments (0)

Files changed (14)

thingq/active/mixins.py

         '''swap for auto-synchronizing context'''
         self.swap(context=self.autoctx, **kw)
         getr_ = lambda x: getattr(self, x)
-        outq = getr_(self._OUTQ)
-        utilq = getr_(self._UTILQ)
-        workq = getr_(self._WORKQ)
-        inq = getr_(self._INQ)
+        inq, workq = getr_(self._INQ), getr_(self._WORKQ)
+        utilq,  outq = getr_(self._UTILQ), getr_(self._OUTQ)
         # clear work things
         workq.clear()
         # extend work things with incoming things
         @param attr: things to iterate over
         '''
         dq = getattr(self, attr)
-        length = len(dq)
-        call = dq.popleft
+        length, call = len(dq), dq.popleft
         for i in repeat(None, length):  # @UnusedVariable
             yield call()
 
 
     def snapshot(self):
         '''snapshot of current outgoing things'''
-        wrap = copy(getattr(self, self._OUTQ))
+        wrap = copy(self.outgoing)
         return wrap.pop() if len(wrap) == 1 else self._wrapper(wrap)
 
     def out(self):
         # return to default context
         self.unswap()
         wrap, outgoing = self._wrapper, self.outgoing
-        wrap = self.outgoing.pop() if len(outgoing) == 1 else wrap(outgoing)
+        wrap = outgoing.pop() if len(outgoing) == 1 else wrap(outgoing)
         # clear outgoing things
         self.clearout()
         return wrap

thingq/filtering.py

 # -*- coding: utf-8 -*-
 '''thingq filtering mixins'''
 
+import re
 from inspect import getmro
 from threading import local
 from functools import reduce
 from collections import deque
+from json import dumps, loads
 from itertools import tee, islice
+from htmlentitydefs import name2codepoint
+from xml.sax.saxutils import escape, unescape
 from operator import attrgetter, itemgetter, truth
 
-from thingq.support import ifilter, ichain, imap, filterfalse
+from thingq.support import (
+    ifilter, ichain, imap, filterfalse, tounicode, tobytes)
 
 
 class CollectMixin(local):
             return self._xtend(
                 filterfalse(lambda y: y in things, self._iterable)
             )
+    
+    def ascii(self, errors='strict'):
+        '''
+        encode each incoming thing as ascii string (regardless of type)
+
+        @param errors: error handling (default: 'strict')
+        '''
+        with self._context():
+            return self._xtend(imap(
+                lambda x: tobytes(x, 'ascii', errors), self._iterable,
+            ))
+    
+    def bytes(self, encoding='utf-8', errors='strict'):
+        '''
+        encode each incoming thing as byte string (regardless of type)
+
+        @param encoding: encoding for things (default: 'utf-8')
+        @param errors: error handling (default: 'strict')
+        '''
+        with self._context():
+            return self._xtend(imap(
+                lambda x: tobytes(x, encoding, errors), self._iterable,
+            ))
+
+    def unicode(self, encoding='utf-8', errors='strict'):
+        '''
+        decode each incoming thing as unicode string (regardless of type)
+
+        @param encoding: encoding for things (default: 'utf-8')
+        @param errors: error handling (default: 'strict')
+        '''
+        with self._context():
+            return self._xtend(imap(
+                lambda x: tounicode(x, encoding, errors), self._iterable,
+            ))
+    
+    def match(self, pattern, flags=0):
+        '''
+        look for pattern among incoming strings
+        
+        @param pattern: search pattern 
+        '''
+        search = re.compile(pattern, flags)
+        def find(x):
+            return True if search.search(x) else False
+        with self._context():
+            return self._xtend(imap(find, self._iterable))
+        
+    def extract(self, pattern, flags=0):
+        '''
+        look for pattern among incoming strings
+        
+        @param pattern: search pattern 
+        '''
+        search = re.compile(pattern, flags)
+        def find(x):
+            results = search.search(x)
+            if not results:
+                return None, None
+            # extract any named results
+            named = results.groupdict()
+            # extract any positional arguments
+            positions = tuple(i for i in search.groups() if i not in named)
+            return positions, named
+        with self._context():
+            return self._xtend(ifilter(
+                lambda x, y: x is not None and y is not None,  
+                imap(find, self._iterable),
+            ))
+
+    def sub(self, pattern, repl, flags=0, count=0):
+        '''
+        replace strings matching pattern with replacement
+        
+        @param pattern: search pattern 
+        @param repl: replacement string
+        '''
+        search = re.compile(pattern, flags)
+        def find(x):
+            return search.sub(repl, x, count)
+        with self._context():
+            return self._xtend(imap(find, self._iterable))
+
+    def htmlescape(self):
+        '''escape HTML (&, <, >, ", and ')'''
+        with self._context():
+            return self._xtend(imap(
+                lambda x: escape(x, {'"':"&quot;", "'":'&#39;'}),
+                self._iterable,
+            ))
+
+    def htmlunescape(self):
+        '''
+        unescape HTML
+        
+        from -> John J. Lee 
+        http://groups.google.com/group/comp.lang.python/msg/ce3fc3330cbbac0a
+        '''
+        def unescape_charref(ref): 
+            name = ref[2:-1] 
+            base = 10 
+            if name.startswith("x"): 
+                name = name[1:] 
+                base = 16 
+            return unichr(int(name, base)) 
+        def replace_entities(match): 
+            ent = match.group() 
+            if ent[1] == "#": 
+                return unescape_charref(ent) 
+            repl = name2codepoint.get(ent[1:-1]) 
+            if repl is not None: 
+                repl = unichr(repl) 
+            else: 
+                repl = ent 
+            return repl 
+        def unescape(data): 
+            return re.sub(r"&#?[A-Za-z0-9]+?;", replace_entities, data) 
+        with self._context():
+            return self._xtend(imap(unescape, self._iterable)) 
+
+    def jsunescape(self):
+        '''javascript/json unescape each incoming things'''
+        with self._context():
+            return self._xtend(imap(loads, self._iterable))
+
+    def jsescape(self):
+        '''javascript/json unescape each incoming things'''
+        with self._context():
+            return self._xtend(imap(dumps, self._iterable))
+
+    def xmlunescape(self):
+        '''xml unexcape each incoming things'''
+        with self._context():
+            return self._xtend(imap(unescape, self._iterable))
+
+    def xmlescape(self):
+        '''xml excape each incoming things'''
+        with self._context():
+            return self._xtend(imap(escape, self._iterable))

thingq/mapping.py

 from copy import deepcopy
 from threading import local
 from operator import methodcaller
-from itertools import starmap, repeat
+from itertools import starmap, repeat, product, combinations, permutations
 
 from thingq.support import imap, ichain, items, xrange
 
 
     '''repetition mixin'''
 
+    def combinations(self, n):
+        '''
+        repeat every combination for `n` of incoming things
+
+        @param n: number of repetitions
+        '''
+        with self._context():
+            return self._xtend(combinations(self._iterable, n))
+
     def copy(self):
         '''copy each incoming thing'''
         with self._context():
             return self._xtend(imap(deepcopy, self._iterable))
 
+    def product(self, n=1):
+        '''
+        nested for each loops repeated `n` times
+
+        @param n: number of repetitions (default: 1)
+        '''
+        with self._context():
+            return self._xtend(product(*self._iterable, repeat=n))
+
+    def permutations(self, n):
+        '''
+        repeat every permutation for every `n` of incoming things
+
+        @param n: length of thing to permutate
+        '''
+        with self._context():
+            return self._xtend(permutations(self._iterable, n))
+
     def range(self, start, stop=0, step=1):
         '''
         put sequence of numbers in incoming things
             de = delay_each
             call_ = lambda x, y: de(x, y, wait, call)
         else:
-            
+
             call_ = lambda x, y: call(*x, **y)
         with self._context():
             return self._xtend(starmap(call_, self._iterable))

thingq/ordering.py

 '''thingq ordering mixins'''
 
 from threading import local
-from itertools import product, groupby
+from itertools import groupby
 from random import choice, shuffle, sample
 
 from thingq.support import zip_longest, imap
         with self._context():
             return self._xtend(
                 zip_longest(fillvalue=fill, *[iter(self._iterable)] * n)
-            )
-
-    def product(self, n=1):
-        '''
-        nested for each loops repeated `n` times
-
-        @param n: number of repetitions (default: 1)
-        '''
-        with self._context():
-            return self._xtend(product(*self._iterable, repeat=n))
+            )  
 
     def reverse(self):
         '''reverse order of incoming things'''

thingq/reducing.py

 from operator import contains, truediv
 from itertools import cycle, tee, islice
 
-from stuf.six import strings
-from thingq.support import Counter, imap, zip, ichain
+from stuf.six import strings, u
+from thingq.support import Counter, imap, zip, ichain, tounicode
 
 
 class MathMixin(local):
     def flatten(self):
         '''flatten deeply nested incoming things'''
         def smash(iterable):
-            smash_ = smash
+            smash_, strings_, isinst_ = smash, strings, isinstance
             for item in iterable:
                 try:
-                    if isinstance(item, strings):
+                    # don't recur over strings
+                    if isinst_(item, strings_):
                         yield item
                     else:
+                        # do recur over other things
                         for j in smash_(item):
                             yield j
                 except TypeError:
+                    # does not recur
                     yield item
         with self._context():
             return self._xtend(smash(self._iterable))
 
+    def join(self, sep=u(''), encoding='utf-8', errors='strict'):
+        '''
+        join incoming things into one unicode string (regardless of type)
+
+        @param sep: join separator (default: '')
+        @param encoding: encoding for things (default: 'utf-8')
+        @param errors: error handling (default: 'strict')
+        '''
+        with self._context():
+            return self._append(tounicode(sep.join(imap(
+                tounicode, self._iterable,
+            )), encoding, errors))
+
     def pairwise(self):
-        '''every two incoming things as a tuple'''
+        '''every two incoming things as a `tuple`'''
         with self._context():
             i1, i2 = tee(self._iterable)
             next(i2, None)
 
     def reduce(self, initial=None):
         '''
-        reduce incoming things to one thing using call (from left side of
-        incoming things)
+        reduce incoming things to one thing using current callable (from left
+        side of incoming things)
 
         @param initial: initial thing (default: None)
         '''
     def reduce_right(self, initial=None):
         '''
         reduce incoming things to one thing from right side of incoming things
-        using call
+        using current callable
 
         @param initial: initial thing (default: None)
         '''

thingq/support.py

         return six.printf(*args, **kw)
 
 
+isbinary = port.isbinary
 isstring = port.isstring
 isunicode = port.isunicode
+texts = six.texts
+
+
+def tounicode(thing, encoding='utf-8', errors='strict'):
+    return (
+        thing.decode(encoding, errors) if isbinary(thing) else
+        texts(texts(thing).encode(encoding, errors), encoding, errors)
+    )
+
+
+def tobytes(thing, encoding='utf-8', errors='strict'):
+    return (
+        texts(thing).encode(encoding, errors) if not isbinary(thing) else thing
+    )
+
 
 import sys
 if not sys.version_info[0] == 2 and sys.version_info[1] < 7:

thingq/tests/auto/filtering.py

         self.assertEqual(
             self.qclass(1, 2, 1, 0, 3, 1, 4).without(0, 1).end(), [2, 3, 4],
         )
+        
+    def test_ascii(self):
+        from stuf.six import u, b
+        self.assertEqual(
+            self.qclass(
+                [1], True, r't', b('i'), u('g'), None, (1,)
+            ).ascii().end(),
+            [b('[1]'), b('True'), b('t'), b('i'), b('g'), b('None'), b('(1,)')]
+        )
+
+    def test_bytes(self):
+        from stuf.six import u, b
+        self.assertEqual(
+            self.qclass(
+                [1], True, r't',  b('i'), u('g'), None, (1,)
+            ).bytes().end(),
+            [b('[1]'), b('True'), b('t'), b('i'), b('g'), b('None'), b('(1,)')]
+        )
+        
+    def test_unicode(self):
+        from stuf.six import u, b
+        self.assertEqual(
+            self.qclass(
+                [1], True, r't', b('i'), u('g'), None, (1,)
+            ).unicode().end(),
+            [u('[1]'), u('True'), u('t'), u('i'), u('g'), u('None'), u('(1,)')]
+        )
 
 __all__ = sorted(name for name, obj in port.items(locals()) if not any([
     name.startswith('_'), ismodule(obj), name in ['ismodule', 'port']

thingq/tests/auto/mapping.py

         self.assertFalse(newlist[1] is testlist[1])
         self.assertListEqual(newlist[1], testlist[1])
 
+    def test_permutations(self):
+        self.assertEqual(
+            self.qclass(40, 50, 60).permutations(2).end(),
+            [(40, 50), (40, 60), (50, 40), (50, 60), (60, 40), (60, 50)],
+        )
+
+    def test_combination(self):
+        self.assertEqual(
+            self.qclass(40, 50, 60).combinations(2).end(),
+            [(40, 50), (40, 60), (50, 60)],
+        )
+
+    def test_product(self):
+        foo = self.qclass('ABCD', 'xy').product().out()
+        self.assertEqual(
+            foo,
+            [('A', 'x'), ('A', 'y'), ('B', 'x'), ('B', 'y'), ('C', 'x'),
+            ('C', 'y'), ('D', 'x'), ('D', 'y')],
+            foo,
+        )
+
 
 class AMapQMixin(ARepeatQMixin):
 

thingq/tests/auto/ordering.py

             [2, 3, 4, 4, 6, 63, 65],
         )
 
-    def test_product(self):
-        foo = self.qclass('ABCD', 'xy').product().out()
-        self.assertEqual(
-            foo,
-            [('A', 'x'), ('A', 'y'), ('B', 'x'), ('B', 'y'), ('C', 'x'),
-            ('C', 'y'), ('D', 'x'), ('D', 'y')],
-            foo,
-        )
-
 
 __all__ = sorted(name for name, obj in port.items(locals()) if not any([
     name.startswith('_'), ismodule(obj), name in ['ismodule', 'port']

thingq/tests/auto/reducing.py

         self.assertEqual(
             self.qclass([[1, [2], [3, [[4]]]]]).flatten().end(), [1, 2, 3, 4],
         )
+        
+    def test_join(self):
+        from stuf.six import u, b
+        self.assertEqual(
+            self.qclass([1], True, b('thing'), None, (1,)).join(u(', ')).end(),
+            u('[1], True, thing, None, (1,)')
+        )
 
     def test_pairwise(self):
         self.assertEqual(

thingq/tests/man/filtering.py

             self.assertEqual,
             [2, 3, 4],
         )
+        
+    def test_ascii(self):
+        from stuf.six import u, b
+        self._true_true_false(
+            self.qclass([1], True, r't', b('i'), u('g'), None, (1,)).ascii(),
+            self.assertEqual,
+            [b('[1]'), b('True'), b('t'), b('i'), b('g'), b('None'), b('(1,)')]
+        )
+
+    def test_bytes(self):
+        from stuf.six import u, b
+        self._true_true_false(
+            self.qclass([1], True, r't', b('i'), u('g'), None, (1,)).bytes(),
+            self.assertEqual,
+            [b('[1]'), b('True'), b('t'), b('i'),  b('g'), b('None'), b('(1,)')]
+        )
+        
+    def test_unicode(self):
+        from stuf.six import u, b
+        self._true_true_false(
+            self.qclass([1], True, r't', b('i'), u('g'), None, (1,)).unicode(),
+            self.assertEqual,
+            [u('[1]'), u('True'), u('t'), u('i'), u('g'), u('None'), u('(1,)')]
+        )
 
 
 __all__ = sorted(name for name, obj in port.items(locals()) if not any([

thingq/tests/man/mapping.py

             [(40, 50, 60), (40, 50, 60), (40, 50, 60)],
         )
 
+    def test_permutations(self):
+        self._false_true_false(
+            self.qclass(40, 50, 60).permutations(2),
+            self.assertEqual,
+            [(40, 50), (40, 60), (50, 40), (50, 60), (60, 40), (60, 50)],
+        )
+
+    def test_combination(self):
+        self._true_true_false(
+            self.qclass(40, 50, 60).combinations(2),
+            self.assertEqual,
+            [(40, 50), (40, 60), (50, 60)],
+        )
+
     def test_times(self):
         def test(*args):
             return list(args)
         self.assertListEqual(newlist[1], testlist[1])
         self.assertTrue(manq.balanced)
 
+    def test_product(self):
+        self._false_true_false(
+            self.qclass('ABCD', 'xy').product(),
+            self.assertListEqual,
+            [('A', 'x'), ('A', 'y'), ('B', 'x'), ('B', 'y'), ('C', 'x'),
+            ('C', 'y'), ('D', 'x'), ('D', 'y')]
+        )
+
 
 class MMapQMixin(MRepeatQMixin):
 

thingq/tests/man/ordering.py

             [2, 3, 4, 4, 6, 63, 65],
         )
 
-    def test_product(self):
-        self._false_true_false(
-            self.qclass('ABCD', 'xy').product(),
-            self.assertListEqual,
-            [('A', 'x'), ('A', 'y'), ('B', 'x'), ('B', 'y'), ('C', 'x'),
-            ('C', 'y'), ('D', 'x'), ('D', 'y')]
-        )
-
 
 __all__ = sorted(name for name, obj in port.items(locals()) if not any([
     name.startswith('_'), ismodule(obj), name in ['ismodule', 'port']

thingq/tests/man/reducing.py

             self.assertEqual,
             [1, 2, 3, 4],
         )
+        
+    def test_join(self):
+        from stuf.six import u, b
+        self._false_true_false(
+            self.qclass([1], True, b('thing'), None, (1,)).join(u(', ')),
+            self.assertEqual,
+            u('[1], True, thing, None, (1,)')
+        )
 
     def test_pairwise(self):
         self._false_true_false(