Commits

Lynn Rees  committed 4dd0857

- cleanup

  • Participants
  • Parent commits f2adc87
  • Branches pu

Comments (0)

Files changed (20)

 except ImportError:
     from distutils.core import setup
 
-install_requires = ['httplib2', 'stuf']
+install_requires = ['urllib3', 'stuf', 'omnijson']
 if sys.version_info <= (2, 7):
     install_requires.append('importlib')
 

File unfinished/avro_talk.py

-'''AVRO <-> Python (warning: experimental)'''
-
-import zlib
-from wire.util import random_string
-#try:
-#    from cStringIO import StringIO
-#except ImportError:
-from StringIO import StringIO
-
-try:
-    from avro import io, schema, datafile as df
-except ImportError:
-    raise ImportError('requires "avro" library')
-
-CODEC_KEY = df.CODEC_KEY
-META_SCHEMA = df.META_SCHEMA
-MAGIC = df.MAGIC
-
-
-class Talker(object):
-
-    type = 'avro/binary'
-
-    def dumps(self, data, **kw):
-        codec = kw.get('avro_codec', 'null')
-        rawschema = kw.get('avro_schema')
-        wschema = schema.parse(rawschema)
-        data_writer = io.DatumWriter(wschema)
-        writer = StringIO()
-        encoder = io.BinaryEncoder(writer)
-        meta = {}
-        if not all(i for i in ('deflate', 'null')):
-            raise df.DataFileException('Unknown codec: %r' % codec)
-        meta['avro.codec'] = codec
-        meta['avro.schema'] = str(wschema)
-        data_writer.writers_schema = wschema
-        sync = random_string()
-        data_writer.write_data(
-            META_SCHEMA,
-            dict(magic=MAGIC, meta=meta, sync=sync),
-            encoder,
-        )
-
-        # write number of items in block
-        encoder.write_long(1)
-        KEY = CODEC_KEY
-        # write block contents
-        if meta[KEY] == 'null':
-            out = writer.getvalue()
-        elif meta[KEY] == 'deflate':
-            # The first two characters and last character are zlib
-            # wrappers around deflate data.
-            out = zlib.compress(writer.getvalue())[2:-1]
-        encoder.write_long(len(out))
-        writer.write(out)
-#        writer.write(sync)
-        return writer.getvalue()
-
-    def loads(self, data, **kw):
-        reader = StringIO(data)
-        data_reader = io.DatumReader()
-        decoder = io.BinaryDecoder(reader)
-        # seek to the beginning of the file to get magic block
-        reader.seek(0, 0)
-        # read header into a dict
-        header = data_reader.read_data(META_SCHEMA, META_SCHEMA, decoder)
-        # check magic number
-        if header.get('magic') != df.MAGIC:
-            raise schema.AvroException(
-                "Not an Avro data file: %s doesn't match %s" % (
-                    header.get('magic'), df.MAGIC,
-                )
-            )
-        # set metadata
-        meta = header['meta']
-        # ensure codec is valid
-        codec = meta.get('avro.codec')
-        if codec is None:
-            self.codec = 'null'
-        if codec not in df.VALID_CODECS:
-            raise df.DataFileException('Unknown codec:'+codec)
-        # get ready to read
-        data_reader.writers_schema = schema.parse(meta.get(df.SCHEMA_KEY))
-        if codec == 'null':
-            # Skip a long; we don't need to use the length.
-            decoder.skip_long()
-        else:
-            # Compressed data is stored as (length, data), which
-            # corresponds to how the 'bytes' type is encoded.
-            data = decoder.read_bytes()
-            # -15 is the log of the window size; negative indicates
-            # 'raw' (no zlib headers) decompression.    See zlib.h.
-            decoder = io.BinaryDecoder(StringIO(zlib.decompress(data, -15)))
-        return data_reader.read(decoder)

File unfinished/curl_client.py

             cmd = cmd
             for k, v in headers:
                 cmd.extend(['--header', '"{0}: {1}"'.format(k, v)])
-            for url in urls: cmd.extend([])
+            for url in urls:
+                cmd.extend([])
             cmd.extend(
                 ['--url', url, '--request', method, '--data-binary',
                 '"{0}"'.format(body)]
         return Response
 
     def _request(self, **kw):
-        if self._randomua: self.headers.user_agent = self.randomua
+        if self._randomua:
+            self.headers.user_agent = self.randomua
         url = kw.get('url', self._url)
         data = kw.get('data', self.body)
         headers = kw.get('headers', dict(self.headers))

File unfinished/test_avro.py

-'''avro tests'''
-
-import unittest
-
-
-class TestAvro(unittest.TestCase):
-
-    @property
-    def _makeone(self):
-        from wire.talk.avro_talk import Talker
-        return Talker
-
-    def setUp(self):
-        self.schema = u'''{
-        "name": "TestDict",
-        "namespace": "talker.avro",
-        "type": "record",
-        "fields": [
-            {"name": "name", "type": "string"},
-            {"name": "types", "type":{"type": "array", "items":"int"}}
-        ]
-        }'''
-        self.talker = self._makeone()
-        self.obj = {u'name': u'obj', u'types': [1, 3, 3]}
-
-    def test_talk(self):
-        dumped = self.talker.dumps(self.obj, avro_schema=self.schema)
-        self.assertIsInstance(dumped, basestring)
-        newobj = self.talker._loads(dumped)
-        self.assertEqual(self.obj, newobj, newobj)
-
-
-class TestAvroWithLoader(unittest.TestCase):
-
-    @property
-    def _makeone(self):
-        from wire.talk.app import TALKERS
-        return TALKERS('avro')
-
-    def setUp(self):
-        self.schema = '''{
-        "name": "TestDict",
-        "namespace": "talker.avro",
-        "type": "record",
-        "fields": [
-            {"name": "name", "type": "string"},
-            {"name": "types", "type":{"type": "array", "items":"int"}}
-        ]
-        }'''
-        self.talker = self._makeone()
-        self.obj = {u'name': u'obj', u'types':[1, 3, 3]}
-
-    def test_talk(self):
-        dumped = self.talker.dumps(self.obj, avro_schema=self.schema)
-        self.assertIsInstance(dumped, basestring)
-        newobj = self.talker._loads(dumped)
-        self.assertEqual(self.obj, newobj, newobj)
-
-
-if __name__ == '__main__':  unittest.main()

File wire/http/apps.py

-
-
-# HTTP clients
-HTTPLIST = dict(
-    urllib2='ul2_client.Request',
-    httplib2='hl2_client.Request',
-    pycurl='pycurl_client.Request',
-)
-#HTTP = Loader(HTTPLIST, package='wire.http.client')
-
-# REST clients
-RESTLIST = dict(
-    urllib2='ul2_client.REST',
-    httplib2='hl2_client.REST',
-    pycurl='pycurl_client.REST',
-)
-#REST = Loader(RESTLIST, package='wire.http.client')

File wire/http/chains.py

-# -*- coding: utf-8 -*-
-'''wire http call chain'''
-
-from callchain.root.chain import callchain
-from wire.util import missing
-
-
-class http(callchain):
-
-    '''HTTP client call chain'''
-
-    def __init__(self, pattern=None, required=None, defaults=None, **kw):
-        '''
-        init
-
-        @param pattern: pattern configuration or appspace label (default: None)
-        @param required: required settings (default: None)
-        @param defaults: default settings (default: None)
-        '''
-        super(http, self).__init__(pattern, required, defaults, **kw)
-        # set data to something
-        self._data = ''
-        # wire protocol
-        self._format = ''
-        # files something
-        self._files = kw.pop('files', {})
-        # custom headers
-        self._headers = kw.pop('headers', {})
-        # custom cookie
-        self._cookie = kw.pop('cookie', {})
-        # default timeout
-        self._timeout = kw.pop('timeout', 100)
-        # use random user agent
-        self._randomua = kw.pop('randomua', False)
-        # preserve any original data
-        self._original = kw.pop('data', '')
-        # url, if any
-        self._url = kw.pop('url', '')
-        # files demand multipart as default otherwise go with urlencode
-        self._originalfmt = kw.pop(
-            'format', 'multipart' if self._files else 'url'
-        )
-        # authentication
-        self._auth = kw.pop('auth', {})
-        if self._auth:
-            # verify
-            if missing(self._auth, ('username', 'password')):
-                raise TypeError(
-                    u'authentication missing both "username" and "password"',
-                )
-        # proxy
-        self._proxy = kw.pop('proxy', {})
-        if self._proxy:
-            if missing(self._proxy, ('server', 'host')):
-                raise TypeError(u'proxy missing both "server" and "host"')
-        # misc. options, if any
-        self._options = kw
-        self._changed = False
-
-    def __call__(self, *urls):
-        '''
-        load urls
-
-        @param urls: Uniform Resource Locations (URLs)
-        '''
-        self._urls = urls
-        return self

File wire/http/drivers/__init__.py

Empty file added.

File wire/http/drivers/apps.py

+
+
+# HTTP clients
+HTTPLIST = dict(
+    urllib2='ul2_client.Request',
+    httplib2='hl2_client.Request',
+    pycurl='pycurl_client.Request',
+)
+#HTTP = Loader(HTTPLIST, package='wire.http.client')
+
+# REST clients
+RESTLIST = dict(
+    urllib2='ul2_client.REST',
+    httplib2='hl2_client.REST',
+    pycurl='pycurl_client.REST',
+)
+#REST = Loader(RESTLIST, package='wire.http.client')

File wire/http/drivers/hl2.py

+'''httplib2 wire HTTP client wrapper'''
+
+#FIXME: don't import eventlet in test mode
+try:
+    import eventlet
+    httplib2 = eventlet.import_patched('httplib2')
+except ImportError:
+    import httplib2
+
+from wire.util import lazy, lazier
+from wire.http.response import CoreResponse
+from wire.http.request import CoreRequest, CoreREST
+
+
+class Response(CoreResponse):
+
+    '''httplib2 response object'''
+
+    def __init__(self, resp, **kw):
+        response, content = resp
+        self._hset('_original', content)
+        h = self._hset('_headers', dict(response))
+        self._hset('_code', h.pop('status', '500'))
+        self._hset('url', h.pop('content-location', None))
+        super(Response, self).__init__(**kw)
+
+
+class Request(CoreRequest):
+
+    def _factory(self):
+        auth = self._auth
+        proxy = self._proxy
+        if proxy:
+            try:
+                import socks
+                h = httplib2.Http(
+                    '.cache',
+                    proxy_info=httplib2.ProxyInfo(
+                        socks.PROXY_TYPE_HTTP, proxy['server'], proxy['host'],
+                    ),
+                )
+            except ImportError:
+                raise ImportWarning(
+                    u'httplib2 proxy support requires "socks" library'
+                )
+        else:
+            h = httplib2.Http('.cache')
+        if auth: h.add_credentials(*(auth['username'], auth['password']))
+        return h
+
+    def _request(self, **kw):
+        try:
+            factory = self._requestor
+            kw, newkw, respkw = self._kwfy(kw)
+            # temporary auth
+            auth = newkw.pop('auth', self._auth)
+            if auth: factory.add_credentials(
+                *(auth['username'], auth['password'])
+            )
+            # temporary timeout
+            factory.timeout = newkw.pop('timeout', self._timeout)
+            result = factory.request(
+                newkw.pop('url'),
+                method=kw.get('method', 'GET'),
+                body=newkw.pop('data', ''),
+                headers=dict(newkw.pop('headers')),
+            )
+            return self._response(result, **respkw)
+        except httplib2.RelativeURIError, e:
+            raise ValueError(e)
+        finally:
+            # clear temporary auth
+            if auth: factory.clear_credentials()
+            # clear temporary timeout
+            factory.timeout = None
+
+    @lazy
+    def _requestor(self):
+        return self._factory()
+
+    @lazier
+    def _response(self):
+        return Response
+
+
+class RESTClient(Request, CoreREST):
+
+    '''httplib2 REST client'''

File wire/http/drivers/pc.py

+'''p wire HTTP client wrapper'''
+
+from StringIO import StringIO
+from httplib import HTTPMessage
+# what pycurl needs...
+try:
+    import signal
+    from signal import SIGPIPE, SIG_IGN
+    signal.signal(SIGPIPE, SIG_IGN)
+except ImportError:
+    pass
+
+try:
+    import pycurl as p
+except ImportError:
+    raise ImportError('pycurl_client requires the "p" library')
+
+from wire.util import lazy, lazier
+from wire.http.response import CoreResponse
+from wire.http.request import CoreRequest, CoreREST
+
+
+class Request(CoreRequest):
+
+    @lazy
+    def _factory(self):
+        factory = p.Curl()
+        proxy = self._proxy
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY
+        if proxy:
+            factory.setopt(p.PROXY, ':'.join([proxy['server'], proxy['host']]))
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTFOLLOWLOCATION
+        factory.setopt(p.FOLLOWLOCATION, 1)
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTMAXREDIRS
+        factory.setopt(p.MAXREDIRS, 5)
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUT
+        factory.setopt(p.CONNECTTIMEOUT, 15)
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTNOSIGNAL
+        factory.setopt(p.NOSIGNAL, 1)
+        # deprecated?
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTRANSFERTEXT
+        factory.setopt(p.TRANSFERTEXT, 1)
+        auth = self._auth
+        if auth:
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTHTTPAUTH
+            factory.setopt(p.HTTPAUTH, p.HTTPAUTH_BASIC)
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUSERPWD
+            factory.setopt(
+                p.USERPWD, ':'.join(auth['username'], auth['password'])
+            )
+        return factory
+
+#    @lazy
+#    def multiple(self):
+#        def func(urls, **kw):
+#            multi = p.CurlMulti()
+#            makehandler = self._makehandler
+#            num_urls = len(urls)
+#            handlers = list(makehandler() for i in xrange(urls))
+#            return func
+
+    @lazier
+    def _response(self):
+        return Response
+
+#    def _makehandler(self, **kw):
+#        factory = p.Curl()
+#        getinfo = factory.getinfo
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTFOLLOWLOCATION
+#        factory.setopt(p.FOLLOWLOCATION, 1)
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTMAXREDIRS
+#        factory.setopt(p.MAXREDIRS, 5)
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUT
+#        factory.setopt(p.CONNECTTIMEOUT, 15)
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTNOSIGNAL
+#        factory.setopt(p.NOSIGNAL, 1)
+#        # deprecated?
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTRANSFERTEXT
+#        factory.setopt(p.TRANSFERTEXT, 1)
+#        # handle proxy
+#        proxy = kw.get('proxy', self._proxy)
+#        if proxy:
+#            factory.setopt(
+#                p.PROXY, ':'.join([proxy['server'], proxy['host']]),
+#            )
+#            # handle proxy auth
+#            factory.setopt(p.PROXYAUTH, p.HTTPAUTH_BASIC)
+#        # handle auth
+#        auth = kw.get('auth', self._auth)
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUSERPWD
+#        if auth:
+#            factory.setopt(p.HTTPAUTH, p.HTTPAUTH_BASIC)
+#            factory.setopt(
+#                p.USERPWD, ':'.join(auth['username'], auth['password'])
+#            )
+#        # handle timeout
+#        timeout = kw.get('timeout', self._timeout)
+#        if timeout: factory.setopt(p.TIMEOUT, timeout)
+#        data = kw.get('data', '')
+#        # handle method
+#        method = kw.get('method', 'get').lower()
+#        headers = list(
+#            ': '.join(i) for i in kw.get('headers', list(self.h))
+#        )
+#        if method == 'post':
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPOST
+#            factory.setopt(p.POST, 1)
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPOSTFIELDS
+#            factory.setopt(p.POSTFIELDS, data)
+#        elif method == 'put':
+#            # deprecated in favor of UPLOAD?
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPUT
+#            factory.setopt(p.PUT, 1)
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUPLOAD
+#            factory.setopt(p.UPLOAD, 1)
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTINFILESIZE
+#            factory.setopt(p.INFILESIZE, len(data))
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTREADDATA
+#            factory.setopt(p.READDATA, data)
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTNOBODY
+#        elif method == 'head':
+#            factory.setopt(p.NOBODY, 1)
+## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCUSTOMREQUEST
+#        elif method == 'delete':
+#            factory.setopt(p.CUSTOMREQUEST, 'DELETE')
+#        elif method == 'options':
+#            factory.setopt(p.CUSTOMREQUEST, 'OPTIONS')
+#        return factory
+
+    def _request(self, **kw):
+        try:
+            factory = self._factory
+            getinfo = factory.getinfo
+            # handle proxy
+            proxy = kw.pop('proxy', self._proxy)
+            if proxy:
+                factory.setopt(
+                    p.PROXY, ':'.join([proxy['server'], proxy['host']]),
+                )
+                # handle proxy auth
+                factory.setopt(p.PROXYAUTH, p.HTTPAUTH_BASIC)
+            # handle auth
+            auth = kw.pop('auth', self._auth)
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUSERPWD
+            if auth:
+                factory.setopt(
+                    p.USERPWD, ':'.join([auth['username'], auth['password']])
+                )
+            # handle timeout
+            timeout = kw.pop('timeout', self._timeout)
+            if timeout: factory.setopt(p.TIMEOUT, timeout)
+            data = kw.pop('data', '')
+            # handle method
+            method = kw.pop('method', 'get').lower()
+            headers = list(
+                ': '.join(i) for i in kw.pop('headers', list(self.headers))
+            )
+            if method == 'post':
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPOST
+                factory.setopt(p.POST, 1)
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPOSTFIELDS
+                factory.setopt(p.POSTFIELDS, data)
+            elif method == 'put':
+                # deprecated in favor of UPLOAD?
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPUT
+                factory.setopt(p.PUT, 1)
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUPLOAD
+                factory.setopt(p.UPLOAD, 1)
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTINFILESIZE
+                factory.setopt(p.INFILESIZE, len(data))
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTREADDATA
+                factory.setopt(p.READDATA, data)
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTNOBODY
+            elif method == 'head':
+                factory.setopt(p.NOBODY, 1)
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCUSTOMREQUEST
+            elif method == 'delete':
+                factory.setopt(p.CUSTOMREQUEST, 'DELETE')
+            elif method == 'options':
+                factory.setopt(p.CUSTOMREQUEST, 'OPTIONS')
+            # handle headers
+            header = StringIO()
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUSERAGENT
+            factory.setopt(p.USERAGENT, self.headers.user_agent)
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTHTTPHEADER
+            factory.setopt(p.HTTPHEADER, list(headers))
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTHEADERFUNCTION
+            factory.setopt(p.HEADERFUNCTION, header.write)
+            # handle data
+            body = StringIO()
+# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTWRITEFUNCTION
+            factory.setopt(p.WRITEFUNCTION, body.write)
+            # handle url
+            factory.setopt(p.URL, kw.pop('url'))
+            factory.perform()
+# INFO: http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html
+            result = dict(
+                headers=header,
+                data=data,
+                status=getinfo(p.HTTP_CODE),
+                content_location=getinfo(p.EFFECTIVE_URL),
+            )
+        except p.error, e:
+# INFO: http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html
+            result = dict(
+                headers=header,
+                data=data,
+                status=getinfo(p.HTTP_CODE),
+                url=getinfo(p.EFFECTIVE_URL),
+                redirect_url=getinfo(p.REDIRECT_URL),
+                errors='{0} {1}'.format(e[0], e[1]),
+            )
+        finally:
+            factory.close()
+        return result
+
+
+class RESTClient(Request, CoreREST):
+
+    '''pycurl REST client'''
+
+
+class Response(CoreResponse):
+
+    '''pycurl response object'''
+
+    def __init__(self, resp, **kw):
+        # data
+        data = resp.pop('data', StringIO())
+        data.seek(0)
+        # preserve original data
+        self._original = data.read()
+        # headers...
+        headers = resp.pop('headers', StringIO())
+        headers.seek(0)
+        # clear first status line
+        headers.readline()
+        # yum...headers
+        self._headers = h = dict(i for i in HTTPMessage(headers).items())
+        # it's an int
+        self._code = str(h.pop('status', 500))
+        # final url
+        self.url = h.pop('url', None)
+        super(Response, self).__init__(**kw)
+
+
+#m = pycurl.CurlMulti()
+#m.handles = []
+#for i in range(num_conn):
+#    c = pycurl.Curl()
+#    c.fp = None
+#    c.setopt(pycurl.FOLLOWLOCATION, 1)
+#    c.setopt(pycurl.MAXREDIRS, 5)
+#    c.setopt(pycurl.CONNECTTIMEOUT, 30)
+#    c.setopt(pycurl.TIMEOUT, 300)
+#    c.setopt(pycurl.NOSIGNAL, 1)
+#    m.handles.append(c)
+#
+#
+## Main loop
+#freelist = m.handles[:]
+#num_processed = 0
+#while num_processed < num_urls:
+#    # If there is an url to process and a free curl object, add to multi stack
+#    while queue and freelist:
+#        url, filename = queue.pop(0)
+#        c = freelist.pop()
+#        c.fp = open(filename, "wb")
+#        c.setopt(pycurl.URL, url)
+#        c.setopt(pycurl.WRITEDATA, c.fp)
+#        m.add_handle(c)
+#        # store some info
+#        c.filename = filename
+#        c.url = url
+#    # Run the internal curl state machine for the multi stack
+#    while 1:
+#        ret, num_handles = m.perform()
+#        if ret != pycurl.E_CALL_MULTI_PERFORM:
+#            break
+#    # Check for curl objects which have terminated, and add them to the freelist
+#    while 1:
+#        num_q, ok_list, err_list = m.info_read()
+#        for c in ok_list:
+#            c.fp.close()
+#            c.fp = None
+#            m.remove_handle(c)
+#            print "Success:", c.filename, c.url, c.getinfo(pycurl.EFFECTIVE_URL)
+#            freelist.append(c)
+#        for c, errno, errmsg in err_list:
+#            c.fp.close()
+#            c.fp = None
+#            m.remove_handle(c)
+#            print "Failed: ", c.filename, c.url, errno, errmsg
+#            freelist.append(c)
+#        num_processed = num_processed + len(ok_list) + len(err_list)
+#        if num_q == 0:
+#            break
+#    # Currently no more I/O is pending, could do something in the meantime
+#    # (display a progress bar, etc.).
+#    # We just call select() to sleep until some more data is available.
+#    m.select(1.0)
+#
+#
+## Cleanup
+#for c in m.handles:
+#    if c.fp is not None:
+#        c.fp.close()
+#        c.fp = None
+#    c.close()
+#m.close()

File wire/http/drivers/ul2.py

+'''urllib2 client wrapper'''
+
+import urllib2
+from collections import deque
+from stuf.utils import lazy_class
+from wire.http.client.response import CoreResponse
+
+#FIXME: don't import eventlet in test mode
+try:
+    import eventlet
+    ulib = eventlet.import_patched('urllib2')
+except ImportError:
+    ulib = urllib2
+
+from stuf import stuf
+
+
+class Response(CoreResponse):
+
+    def __init__(self, response, **kw):
+        self._original = response.read()
+        self._code = response.code
+        self._headers = response.headers
+        self.url = response.url
+        super(Response, self).__init__(**kw)
+
+
+class Request(CoreRequest):
+
+    def _factory(self, **kw):
+        if kw.pop('test', False):
+            handlers = deque([
+                ulib.HTTPHandler,
+                ulib.HTTPDefaultErrorHandler,
+                ulib.HTTPRedirectHandler,
+                ulib.FileHandler,
+                ulib.HTTPErrorProcessor,
+                ulib.HTTPSHandler,
+            ])
+            proxy = self._proxy
+            if proxy:
+                proxy_set = '{0} {1}'.format(proxy['server'], proxy['host'])
+                handlers.appendleft(
+                    ulib.ProxyHandler(dict(http=proxy_set, https=proxy_set))
+                )
+            auth = kw.pop('auth', self._auth)
+            if auth:
+                password_mgr = ulib.HTTPPasswordMgrWithDefaultRealm()
+                password_mgr.add_password(
+                    None, kw.get('url'), auth['username'], auth['password'],
+                )
+                handlers.append(ulib.HTTPBasicAuthHandler(password_mgr))
+            ulib.install_opener(ulib.build_opener(*tuple(h for h in handlers)))
+
+    @lazy_class
+    def _response(self):
+        return Response
+
+    def _request(self, **kw):
+        kw, newkw, respkw = self._kwfy(kw)
+        factory = self._factory(**newkw)
+        url = newkw.pop('url')
+        data = newkw.pop('data')
+        headers = newkw.pop('headers', dict(self.headers))
+        request = urllib2.Request(url=url, data=data, headers=headers)
+        request.get_method = lambda: kw.get('method', 'GET')
+        try:
+            return self._response(factory.open(request), **respkw)
+        except ulib.HTTPError, e:
+            return stuf(
+                read=e.fp.read,
+                code=e.code,
+                headers=e.hdrs,
+                url=e.filename,
+            )
+
+
+class RESTClient(Request, CoreREST):
+
+    '''urllib2 REST client'''

File wire/http/drivers/ul3.py

+'''urllib2 client wrapper'''
+
+
+#from collections import deque
+from stuf.utils import lazy_class
+from wire.http.response import CoreResponse
+
+from stuf import stuf
+import urllib3
+
+
+class Response(CoreResponse):
+
+    def __init__(self, response, **kw):
+        self._original = response.read()
+        self._code = response.code
+        self._headers = response.headers
+        self.url = response.url
+        super(Response, self).__init__(**kw)
+
+
+class Request(Request):
+
+#    def _factory(self, **kw):
+#        if kw.pop('test', False):
+#            handlers = deque([
+#                urllib3.HTTPHandler,
+#                urllib3.HTTPDefaultErrorHandler,
+#                urllib3.HTTPRedirectHandler,
+#                urllib3.FileHandler,
+#                urllib3.HTTPErrorProcessor,
+#                urllib3.HTTPSHandler,
+#            ])
+#            proxy = self._proxy
+#            if proxy:
+#                proxy_set = '{0} {1}'.format(proxy['server'], proxy['host'])
+#                handlers.appendleft(
+#                    urllib3.ProxyHandler(dict(http=proxy_set, https=proxy_set))
+#                )
+#            auth = kw.pop('auth', self._auth)
+#            if auth:
+#                password_mgr = urllib3.HTTPPasswordMgrWithDefaultRealm()
+#                password_mgr.add_password(
+#                    None, kw.get('url'), auth['username'], auth['password'],
+#                )
+#                handlers.append(urllib3.HTTPBasicAuthHandler(password_mgr))
+#            urllib3.install_opener(urllib3.build_opener(*tuple(h for h in handlers)))
+
+    @lazy_class
+    def _response(self):
+        return Response
+
+    def _request(self, **kw):
+        kw, newkw, respkw = self._kwfy(kw)
+        factory = self._factory(**newkw)
+        url = newkw.pop('url')
+        data = newkw.pop('data')
+        headers = newkw.pop('headers', dict(self.headers))
+        request = urllib3(url=url, data=data, headers=headers)
+        request.get_method = lambda: kw.get('method', 'GET')
+        try:
+            return self._response(factory.open(request), **respkw)
+        except urllib3.HTTPError, e:
+            return stuf(
+                read=e.fp.read,
+                code=e.code,
+                headers=e.hdrs,
+                url=e.filename,
+            )
+

File wire/http/ext/__init__.py

Empty file removed.

File wire/http/ext/hl2_client.py

-'''httplib2 wire HTTP client wrapper'''
-
-#FIXME: don't import eventlet in test mode
-try:
-    import eventlet
-    httplib2 = eventlet.import_patched('httplib2')
-except ImportError:
-    import httplib2
-
-from wire.util import lazy, lazier
-from wire.http.response import CoreResponse
-from wire.http.request import CoreRequest, CoreREST
-
-
-class Response(CoreResponse):
-
-    '''httplib2 response object'''
-
-    def __init__(self, resp, **kw):
-        response, content = resp
-        self._hset('_original', content)
-        h = self._hset('_headers', dict(response))
-        self._hset('_code', h.pop('status', '500'))
-        self._hset('url', h.pop('content-location', None))
-        super(Response, self).__init__(**kw)
-
-
-class Request(CoreRequest):
-
-    def _factory(self):
-        auth = self._auth
-        proxy = self._proxy
-        if proxy:
-            try:
-                import socks
-                h = httplib2.Http(
-                    '.cache',
-                    proxy_info=httplib2.ProxyInfo(
-                        socks.PROXY_TYPE_HTTP, proxy['server'], proxy['host'],
-                    ),
-                )
-            except ImportError:
-                raise ImportWarning(
-                    u'httplib2 proxy support requires "socks" library'
-                )
-        else:
-            h = httplib2.Http('.cache')
-        if auth: h.add_credentials(*(auth['username'], auth['password']))
-        return h
-
-    def _request(self, **kw):
-        try:
-            factory = self._requestor
-            kw, newkw, respkw = self._kwfy(kw)
-            # temporary auth
-            auth = newkw.pop('auth', self._auth)
-            if auth: factory.add_credentials(
-                *(auth['username'], auth['password'])
-            )
-            # temporary timeout
-            factory.timeout = newkw.pop('timeout', self._timeout)
-            result = factory.request(
-                newkw.pop('url'),
-                method=kw.get('method', 'GET'),
-                body=newkw.pop('data', ''),
-                headers=dict(newkw.pop('headers')),
-            )
-            return self._response(result, **respkw)
-        except httplib2.RelativeURIError, e:
-            raise ValueError(e)
-        finally:
-            # clear temporary auth
-            if auth: factory.clear_credentials()
-            # clear temporary timeout
-            factory.timeout = None
-
-    @lazy
-    def _requestor(self):
-        return self._factory()
-
-    @lazier
-    def _response(self):
-        return Response
-
-
-class RESTClient(Request, CoreREST):
-
-    '''httplib2 REST client'''

File wire/http/ext/pycurl_client.py

-'''p wire HTTP client wrapper'''
-
-from StringIO import StringIO
-from httplib import HTTPMessage
-# what pycurl needs...
-try:
-    import signal
-    from signal import SIGPIPE, SIG_IGN
-    signal.signal(SIGPIPE, SIG_IGN)
-except ImportError:
-    pass
-
-try:
-    import pycurl as p
-except ImportError:
-    raise ImportError('pycurl_client requires the "p" library')
-
-from wire.util import lazy, lazier
-from wire.http.response import CoreResponse
-from wire.http.request import CoreRequest, CoreREST
-
-
-class Request(CoreRequest):
-
-    @lazy
-    def _factory(self):
-        factory = p.Curl()
-        proxy = self._proxy
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY
-        if proxy:
-            factory.setopt(p.PROXY, ':'.join([proxy['server'], proxy['host']]))
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTFOLLOWLOCATION
-        factory.setopt(p.FOLLOWLOCATION, 1)
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTMAXREDIRS
-        factory.setopt(p.MAXREDIRS, 5)
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUT
-        factory.setopt(p.CONNECTTIMEOUT, 15)
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTNOSIGNAL
-        factory.setopt(p.NOSIGNAL, 1)
-        # deprecated?
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTRANSFERTEXT
-        factory.setopt(p.TRANSFERTEXT, 1)
-        auth = self._auth
-        if auth:
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTHTTPAUTH
-            factory.setopt(p.HTTPAUTH, p.HTTPAUTH_BASIC)
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUSERPWD
-            factory.setopt(
-                p.USERPWD, ':'.join(auth['username'], auth['password'])
-            )
-        return factory
-
-#    @lazy
-#    def multiple(self):
-#        def func(urls, **kw):
-#            multi = p.CurlMulti()
-#            makehandler = self._makehandler
-#            num_urls = len(urls)
-#            handlers = list(makehandler() for i in xrange(urls))
-#            return func
-
-    @lazier
-    def _response(self):
-        return Response
-
-#    def _makehandler(self, **kw):
-#        factory = p.Curl()
-#        getinfo = factory.getinfo
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTFOLLOWLOCATION
-#        factory.setopt(p.FOLLOWLOCATION, 1)
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTMAXREDIRS
-#        factory.setopt(p.MAXREDIRS, 5)
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUT
-#        factory.setopt(p.CONNECTTIMEOUT, 15)
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTNOSIGNAL
-#        factory.setopt(p.NOSIGNAL, 1)
-#        # deprecated?
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTRANSFERTEXT
-#        factory.setopt(p.TRANSFERTEXT, 1)
-#        # handle proxy
-#        proxy = kw.get('proxy', self._proxy)
-#        if proxy:
-#            factory.setopt(
-#                p.PROXY, ':'.join([proxy['server'], proxy['host']]),
-#            )
-#            # handle proxy auth
-#            factory.setopt(p.PROXYAUTH, p.HTTPAUTH_BASIC)
-#        # handle auth
-#        auth = kw.get('auth', self._auth)
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUSERPWD
-#        if auth:
-#            factory.setopt(p.HTTPAUTH, p.HTTPAUTH_BASIC)
-#            factory.setopt(
-#                p.USERPWD, ':'.join(auth['username'], auth['password'])
-#            )
-#        # handle timeout
-#        timeout = kw.get('timeout', self._timeout)
-#        if timeout: factory.setopt(p.TIMEOUT, timeout)
-#        data = kw.get('data', '')
-#        # handle method
-#        method = kw.get('method', 'get').lower()
-#        headers = list(
-#            ': '.join(i) for i in kw.get('headers', list(self.h))
-#        )
-#        if method == 'post':
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPOST
-#            factory.setopt(p.POST, 1)
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPOSTFIELDS
-#            factory.setopt(p.POSTFIELDS, data)
-#        elif method == 'put':
-#            # deprecated in favor of UPLOAD?
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPUT
-#            factory.setopt(p.PUT, 1)
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUPLOAD
-#            factory.setopt(p.UPLOAD, 1)
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTINFILESIZE
-#            factory.setopt(p.INFILESIZE, len(data))
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTREADDATA
-#            factory.setopt(p.READDATA, data)
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTNOBODY
-#        elif method == 'head':
-#            factory.setopt(p.NOBODY, 1)
-## http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCUSTOMREQUEST
-#        elif method == 'delete':
-#            factory.setopt(p.CUSTOMREQUEST, 'DELETE')
-#        elif method == 'options':
-#            factory.setopt(p.CUSTOMREQUEST, 'OPTIONS')
-#        return factory
-
-    def _request(self, **kw):
-        try:
-            factory = self._factory
-            getinfo = factory.getinfo
-            # handle proxy
-            proxy = kw.pop('proxy', self._proxy)
-            if proxy:
-                factory.setopt(
-                    p.PROXY, ':'.join([proxy['server'], proxy['host']]),
-                )
-                # handle proxy auth
-                factory.setopt(p.PROXYAUTH, p.HTTPAUTH_BASIC)
-            # handle auth
-            auth = kw.pop('auth', self._auth)
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUSERPWD
-            if auth:
-                factory.setopt(
-                    p.USERPWD, ':'.join([auth['username'], auth['password']])
-                )
-            # handle timeout
-            timeout = kw.pop('timeout', self._timeout)
-            if timeout: factory.setopt(p.TIMEOUT, timeout)
-            data = kw.pop('data', '')
-            # handle method
-            method = kw.pop('method', 'get').lower()
-            headers = list(
-                ': '.join(i) for i in kw.pop('headers', list(self.headers))
-            )
-            if method == 'post':
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPOST
-                factory.setopt(p.POST, 1)
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPOSTFIELDS
-                factory.setopt(p.POSTFIELDS, data)
-            elif method == 'put':
-                # deprecated in favor of UPLOAD?
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPUT
-                factory.setopt(p.PUT, 1)
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUPLOAD
-                factory.setopt(p.UPLOAD, 1)
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTINFILESIZE
-                factory.setopt(p.INFILESIZE, len(data))
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTREADDATA
-                factory.setopt(p.READDATA, data)
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTNOBODY
-            elif method == 'head':
-                factory.setopt(p.NOBODY, 1)
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCUSTOMREQUEST
-            elif method == 'delete':
-                factory.setopt(p.CUSTOMREQUEST, 'DELETE')
-            elif method == 'options':
-                factory.setopt(p.CUSTOMREQUEST, 'OPTIONS')
-            # handle headers
-            header = StringIO()
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUSERAGENT
-            factory.setopt(p.USERAGENT, self.headers.user_agent)
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTHTTPHEADER
-            factory.setopt(p.HTTPHEADER, list(headers))
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTHEADERFUNCTION
-            factory.setopt(p.HEADERFUNCTION, header.write)
-            # handle data
-            body = StringIO()
-# http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTWRITEFUNCTION
-            factory.setopt(p.WRITEFUNCTION, body.write)
-            # handle url
-            factory.setopt(p.URL, kw.pop('url'))
-            factory.perform()
-# INFO: http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html
-            result = dict(
-                headers=header,
-                data=data,
-                status=getinfo(p.HTTP_CODE),
-                content_location=getinfo(p.EFFECTIVE_URL),
-            )
-        except p.error, e:
-# INFO: http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html
-            result = dict(
-                headers=header,
-                data=data,
-                status=getinfo(p.HTTP_CODE),
-                url=getinfo(p.EFFECTIVE_URL),
-                redirect_url=getinfo(p.REDIRECT_URL),
-                errors='{0} {1}'.format(e[0], e[1]),
-            )
-        finally:
-            factory.close()
-        return result
-
-
-class RESTClient(Request, CoreREST):
-
-    '''pycurl REST client'''
-
-
-class Response(CoreResponse):
-
-    '''pycurl response object'''
-
-    def __init__(self, resp, **kw):
-        # data
-        data = resp.pop('data', StringIO())
-        data.seek(0)
-        # preserve original data
-        self._original = data.read()
-        # headers...
-        headers = resp.pop('headers', StringIO())
-        headers.seek(0)
-        # clear first status line
-        headers.readline()
-        # yum...headers
-        self._headers = h = dict(i for i in HTTPMessage(headers).items())
-        # it's an int
-        self._code = str(h.pop('status', 500))
-        # final url
-        self.url = h.pop('url', None)
-        super(Response, self).__init__(**kw)
-
-
-#m = pycurl.CurlMulti()
-#m.handles = []
-#for i in range(num_conn):
-#    c = pycurl.Curl()
-#    c.fp = None
-#    c.setopt(pycurl.FOLLOWLOCATION, 1)
-#    c.setopt(pycurl.MAXREDIRS, 5)
-#    c.setopt(pycurl.CONNECTTIMEOUT, 30)
-#    c.setopt(pycurl.TIMEOUT, 300)
-#    c.setopt(pycurl.NOSIGNAL, 1)
-#    m.handles.append(c)
-#
-#
-## Main loop
-#freelist = m.handles[:]
-#num_processed = 0
-#while num_processed < num_urls:
-#    # If there is an url to process and a free curl object, add to multi stack
-#    while queue and freelist:
-#        url, filename = queue.pop(0)
-#        c = freelist.pop()
-#        c.fp = open(filename, "wb")
-#        c.setopt(pycurl.URL, url)
-#        c.setopt(pycurl.WRITEDATA, c.fp)
-#        m.add_handle(c)
-#        # store some info
-#        c.filename = filename
-#        c.url = url
-#    # Run the internal curl state machine for the multi stack
-#    while 1:
-#        ret, num_handles = m.perform()
-#        if ret != pycurl.E_CALL_MULTI_PERFORM:
-#            break
-#    # Check for curl objects which have terminated, and add them to the freelist
-#    while 1:
-#        num_q, ok_list, err_list = m.info_read()
-#        for c in ok_list:
-#            c.fp.close()
-#            c.fp = None
-#            m.remove_handle(c)
-#            print "Success:", c.filename, c.url, c.getinfo(pycurl.EFFECTIVE_URL)
-#            freelist.append(c)
-#        for c, errno, errmsg in err_list:
-#            c.fp.close()
-#            c.fp = None
-#            m.remove_handle(c)
-#            print "Failed: ", c.filename, c.url, errno, errmsg
-#            freelist.append(c)
-#        num_processed = num_processed + len(ok_list) + len(err_list)
-#        if num_q == 0:
-#            break
-#    # Currently no more I/O is pending, could do something in the meantime
-#    # (display a progress bar, etc.).
-#    # We just call select() to sleep until some more data is available.
-#    m.select(1.0)
-#
-#
-## Cleanup
-#for c in m.handles:
-#    if c.fp is not None:
-#        c.fp.close()
-#        c.fp = None
-#    c.close()
-#m.close()

File wire/http/ext/ul2_client.py

-'''urllib2 client wrapper'''
-
-import urllib2
-from collections import deque
-from stuf.utils import lazy_class
-from wire.http.client.response import CoreResponse
-
-#FIXME: don't import eventlet in test mode
-try:
-    import eventlet
-    ulib = eventlet.import_patched('urllib2')
-except ImportError:
-    ulib = urllib2
-
-from stuf import stuf
-
-
-class Response(CoreResponse):
-
-    def __init__(self, response, **kw):
-        self._original = response.read()
-        self._code = response.code
-        self._headers = response.headers
-        self.url = response.url
-        super(Response, self).__init__(**kw)
-
-
-class Request(CoreRequest):
-
-    def _factory(self, **kw):
-        if kw.pop('test', False):
-            handlers = deque([
-                ulib.HTTPHandler,
-                ulib.HTTPDefaultErrorHandler,
-                ulib.HTTPRedirectHandler,
-                ulib.FileHandler,
-                ulib.HTTPErrorProcessor,
-                ulib.HTTPSHandler,
-            ])
-            proxy = self._proxy
-            if proxy:
-                proxy_set = '{0} {1}'.format(proxy['server'], proxy['host'])
-                handlers.appendleft(
-                    ulib.ProxyHandler(dict(http=proxy_set, https=proxy_set))
-                )
-            auth = kw.pop('auth', self._auth)
-            if auth:
-                password_mgr = ulib.HTTPPasswordMgrWithDefaultRealm()
-                password_mgr.add_password(
-                    None, kw.get('url'), auth['username'], auth['password'],
-                )
-                handlers.append(ulib.HTTPBasicAuthHandler(password_mgr))
-            ulib.install_opener(ulib.build_opener(*tuple(h for h in handlers)))
-
-    @lazy_class
-    def _response(self):
-        return Response
-
-    def _request(self, **kw):
-        kw, newkw, respkw = self._kwfy(kw)
-        factory = self._factory(**newkw)
-        url = newkw.pop('url')
-        data = newkw.pop('data')
-        headers = newkw.pop('headers', dict(self.headers))
-        request = urllib2.Request(url=url, data=data, headers=headers)
-        request.get_method = lambda: kw.get('method', 'GET')
-        try:
-            return self._response(factory.open(request), **respkw)
-        except ulib.HTTPError, e:
-            return stuf(
-                read=e.fp.read,
-                code=e.code,
-                headers=e.hdrs,
-                url=e.filename,
-            )
-
-
-class RESTClient(Request, CoreREST):
-
-    '''urllib2 REST client'''

File wire/http/header.py

         while 0 <= i < n:
             # Start looking for a cookie
             match = patt.search(cookie, i)
-            if not match: break  # No more cookies
+            if not match:
+                break  # No more cookies
             k, v = match.group('key'), match.group('val').rstrip(', ')
             i = match.end(0)
             # Parse the key, value in case it's metainfo
             if k[0] == '$':
-                if m: m._morsel[k[1:]] = v
+                if m:
+                    m._morsel[k[1:]] = v
             elif k.lower() in reserved:
-                if m: m._morsel[k] = unquote(v)
+                if m:
+                    m._morsel[k] = unquote(v)
             else:
                 rval, cval = self.value_decode(v)
                 self._set(k, rval, cval)
         if isinstance(data, str):
             self._parse(data)
         else:
-            for k, v in data.iteritems(): self.__setitem__(k, v)
+            for k, v in data.iteritems():
+                self.__setitem__(k, v)
 
     def output(self):
         return partial(super(CoreCookie, self).output, header='cookie:')
         return frozenset([
             'accept', 'accept-charset', 'accept-encoding', 'accept-language',
             'allow', 'cache-control', 'content-encoding', 'content-language',
-            'if-match', 'if-none-match', 'pragma', 'vary','via',
+            'if-match', 'if-none-match', 'pragma', 'vary', 'via',
         ])
 
     @lazy_class
             r'(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)'
             r'\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])'
             r'|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|'
-            r'mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))'
-            r'(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&amp;%\$#\=~_\-]+))*$',
+            r'mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2})'
+            r')(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&amp;%\$#\=~_\-]+))*$',
         # RFC 1123 time format validation
-        rfc1123=r'(Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} (Jan|Feb|Mar|Apr|May|Jun'
-            r'|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} GMT',
+        rfc1123=r'(Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} (Jan|Feb|Mar|Apr|May|'
+            r'Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} GMT',
         # Content type HTTP header validation
         content_type=r'bytes\s\d+-\d+/(\d+|\*)',
         # HTTP header name check (from wsgiref)
     def _isurl(cls, data):
         '''Validates data is a valid URL'''
         return cls._regeval('url', data)
-    
+
     def final(self):
         return '\r\n'.join(': '.join([k, v]) for k, v in self)

File wire/http/request.py

 from itertools import chain
 from datetime import datetime
 from functools import partial
+from collections import deque
 from email.header import Header
 
-from stuf.utils import lazy_class
-from appspace.keys import appifies
-from callchain.root.linked import chainlink
+from appspace.keys import appifies, imap
+from callchain.root.chain import callchain
+from stuf.utils import lazy_class, OrderedDict, exhaust, iterexcept
+
+from wire.util import missing
 
 from wire.http.util import httpdate
-from wire.util import OrderedDict, subdictremove, hiddenhand
 from wire.http.header import CoreHeaders, HTTPHeaderException
 from wire.http.keys.request import KRequest, KRequestData, KRequestHeaders
+from callchain.root.linked import chainlink
 
 
 @appifies(KRequestHeaders)
     def __init__(self, headers=()):
         super(Headers, self).__init__()
         # "best practice" sends HTTP headers in order: general, request, entity
-        hiddenhand(self, '_general', OrderedDict())
-        hiddenhand(self, '_entity', OrderedDict())
-        hiddenhand(self, '_request', OrderedDict())
-        hiddenhand(self, '_custom', OrderedDict())
+        self._general = OrderedDict()
+        self._entity = OrderedDict()
+        self._request = OrderedDict()
+        self._custom = OrderedDict()
         # populate headers
-        self.update(headers)
+        self.headers(headers)
+        if self.G.randomua:
+            self.user_agent(self._randomua())
 
-    def __setitem__(self, key, value):
+    def _getheaders(self, key):
+        '''
+        Gets the list and tracker for a header's category.
+
+        @param: key Header key
+        '''
+        if key in self._GENERAL:
+            return self._general
+        elif key in self._REQUEST:
+            return self._request
+        elif key in self._ENTITY:
+            return self._entity
+        return self._custom
+
+    def _get(self, key):
+        key = key.lower().replace('_', '-')
+        headers = self._getheaders(key)
+        return headers.get(key)
+
+    def _set(self, key, value):
         key = key.lower().replace('_', '-')
         # validate header value
         self._validate(key, value)
         else:
             headers[key] = value
 
-    __setattr__ = __setitem__
-
-    def __delitem__(self, key):
-        key = key.lower()
-        headers = self._getheaders(key)
+    def _delete(self, key):
+        key = key.lower().replace('_', '-')
+        headers = self._get(key)
         del headers[key]
 
-    def __getattr__(self, key):
-        try:
-            return object.__getattribute__(self, key)
-        except AttributeError:
-            tkey = self._headattr.get(key)
-            # Cast to str to cast email.Header to str type
-            if tkey is not None:
-                value = self.get(tkey)
-                # Don't make 'None' a string
-                return str(value) if value is not None else value
-            raise AttributeError(key)
-
-    def __delattr__(self, key):
-        try:
-            self.__delitem__(self._headattr.get(key))
-        except KeyError:
-            try:
-                self.__delitem__(key)
-            except KeyError:
-                raise AttributeError(key)
-
-    def __iter__(self):
-        # Cast any header objects as strings (calling Header.decode)
-        return ((h[0].lower(), str(h[1])) for h in chain(
-            list((k, v) for k, v in self._general.iteritems()),
-            list((k, v) for k, v in self._request.iteritems()),
-            list((k, v) for k, v in self._entity.iteritems()),
-            list((k, v) for k, v in self._custom.iteritems()),
-            list(self.cookies),
-        ) if h is not None)
-
-    def _headerfy(self, headers, **kw):
-        if headers:
-            new_headers = Headers(i for i in headers.iteritems())
-        else:
-            new_headers = Headers()
-        if kw.get('randomua', self._randomua):
-            new_headers.user_agent = new_headers._randomua()
-        return new_headers
-
     @lazy_class
     def _badheaders(self):
-        '''Bad header detecting regex from wsgiref'''
+        '''bad header detecting regex from wsgiref'''
         return re.compile(r'[\000-\037]')
 
     def _format(self, key, value):
             value = httpdate(value)
         return value
 
-    def _getheaders(self, key):
-        '''
-        Gets the list and tracker for a header's category.
-
-        @param: key Header key
-        '''
-        if key in self._GENERAL:
-            return self._general
-        elif key in self._REQUEST:
-            return self._request
-        elif key in self._ENTITY:
-            return self._entity
-        return self._custom
-
     @lazy_class
     def _langs(self):
-        '''Language set.'''
+        '''language set.'''
         from locale import locale_alias
         return frozenset(i.replace('_', '-') for i in locale_alias)
 
         if err_msg is not None:
             raise HTTPHeaderException(err_msg)
 
+    def content_type(self, mime, charset=None):
+        '''
+        Utility method to set the "Content-Type" header.
+
+        @param mime Media type
+        @param charset Optional charset (default: None)
+        '''
+        if charset is None:
+            self['Content-Type'] = mime
+        else:
+            self['Content-Type'] = '{0}; charset={1}'.format(mime, charset)
+        return self
+
+    def end(self):
+        # Cast any header objects as strings (calling Header.decode)
+        return ((h[0].lower(), str(h[1])) for h in chain(
+            list((k, v) for k, v in self._general.iteritems()),
+            list((k, v) for k, v in self._request.iteritems()),
+            list((k, v) for k, v in self._entity.iteritems()),
+            list((k, v) for k, v in self._custom.iteritems()),
+            list(self.cookies),
+        ) if h is not None)
+
     def md5(self, data):
         '''
         Take a MD5 hash of data and set the "Content-MD5" header.
         @param length Instance length
         '''
         if length == '*':
-            self['Content-Range'] = 'bytes %d-%d/*' % (first, last)
+            self._set('Content-Range', 'bytes %d-%d/*' % (first, last))
         else:
-            self['Content-Range'] = 'bytes %d-%d/%d' % (first, last, length)
+            self._set(
+                'Content-Range', 'bytes %d-%d/%d' % (first, last, length)
+            )
         return self
 
     def rfc2822(self, key, value, charset=None):
         @param charset HTTP charset (default: None)
         '''
         # Use an existing email.Header instance
-        existing = self.get(key)
+        existing = self._get(key)
         if existing is not None:
             try:
                 existing.append(value, charset)
                 pass
         else:
             # Override existing value if not Header instance or add new value
-            self[key] = Header(value, charset)
+            self._set(key, Header(value, charset))
         return self
 
-    def content_type(self, mime, charset=None):
-        '''
-        Utility method to set the "Content-Type" header.
-
-        @param mime Media type
-        @param charset Optional charset (default: None)
-        '''
-        if charset is None:
-            self['Content-Type'] = mime
-        else:
-            self['Content-Type'] = '{0}; charset={1}'.format(mime, charset)
+    def header(self, key, value):
+        self._set(key, value)
         return self
 
-    def update(self, headers=(), **kw):
+    def headers(self, headers):
         '''update header values'''
-        setitem = self.__setitem__
-        if kw:
-            for k, v in kw.iteritems():
-                setitem(k, v)
-        if headers:
-            for h in headers:
-                setitem(h[0], h[1])
+        setitem = self.header
+        exhaust(imap(setitem, iterexcept(headers, IndexError)))
         return self
 
-#    def headers(self):
-#        '''
-#        HTTP header manager
-#        '''
-#        return self._headerfy(self._headers)
-
 
 @appifies(KRequestData)
 class Data(chainlink):
         super(Data, self).__init__(root)
         self._data = ''
 
+    def files(self, files):
+        # files demand multipart as default otherwise go with urlencode
+        self._format = 'multipart'
+        self._files = files
+        return self
+
     @property
     def _thedata(self):
         return self._data or self._original
 
-    def data(self):
+    def data(self, data):
+        # preserve any original data
+        self._original = data
         if not self._data:
             data = self._thedata
             if data:
     def _datafy(self, data, headers, fmt, **kw):
         if fmt in self._dumpers.apps:
             files = kw.pop('files', self._files)
-            if files: data = dict(form=data, files=files)
+            if files:
+                data = dict(form=data, files=files)
             type_, data = self.dumps(data, fmt, **self._kwfy(kw)[-1])
             headers.content_type = type_
             headers.accept = type_
         return data
-    
 
-@appifies(KRequest)
+
 class Request(object):
 
-    '''core HTTP request'''
-
-
-
-    def __getattr__(self, k):
-        try:
-            return object.__getattribute__(self, k)
-        except AttributeError:
-            if k == self.format and not self._changed:
-                v = self.data
-            else:
-                # clear data
-                self._data = ''
-                v = self._datafy(
-                    self._thedata, self.headers, k, **self._options
-                )
-                self._data = v
-                # override existing format
-                self._format = k
-                self._changed = True
-                # cache
-            object.__setattr__(self, k, v)
-            return v
-
-    def __setattr__(self, k, v):
-        if k in self._dumpers.apps:
-            v = self._datafy(v, self.headers, k, **self._options)
-            self._data = v
-            self._format = k
-            self._changed = True
-        object.__setattr__(self, k, v)
+    def __init__(self, url):
+        self._data = ''
+        params = None
+        data = None
+        headers = None
+        cookies = None
+        files = None
+        auth = None
+        timeout = None
+        allow_redirects = False
+        proxies = None
+        hooks = None
+        return_response = True
+        config = None
 
     def _request(self, **kw):
         '''make HTTP request'''
         self._raw = self._request(**kw)
         return self
 
-    # properties
 
-    def _kwfy(self, kw):
-        # remove consumed data
-        newkw = subdictremove(
-            kw, ('auth', 'timeout', 'url', 'data', 'headers')
-        )
-        # preserve HTTP method
-        respkw = dict((k, v) for k, v in kw.iteritems() if k not in ('method',))
-        # use default misc if necessary
-        if not respkw:
-            respkw = self._options
-        return kw, newkw, respkw
+@appifies(KRequest)
+class http(callchain):
+
+    '''HTTP client call chain'''
+
+    def __init__(self, pattern=None, required=None, defaults=None, **kw):
+        '''
+        init
+
+        @param pattern: pattern configuration or appspace label (default: None)
+        @param required: required settings (default: None)
+        @param defaults: default settings (default: None)
+        '''
+        super(http, self).__init__(pattern, required, defaults, **kw)
+        # preserve any original data
+        self._original = ''
+        # url, if any
+        self._urls = deque()
+        # files demand multipart as default otherwise go with urlencode
+        self._format = kw.get('format', self.defaults.format)
+        # set data to something
+        self._data = ''
+        # files something
+        self._files = {}
+        # custom headers
+        self._headers = {}
+        # custom cookies
+        self._cookies = {}
+        # default timeout
+        self._timeout = kw.get('timeout', self.defaults.timeout)
+        # use random user agent
+        self._randomua = kw.pop('randomua', self.defaults.random_user_agent)
+        # authentication
+        self._auth = self.defaults.authentication
+        # proxy
+        self._proxy = self.defaults.proxy
+
+    def __call__(self, *urls, **kw):
+        '''
+        load urls
+
+        @param urls: Uniform Resource Locations (URLs)
+        '''
+        self.clear()
+        # urls, if any
+        self._urls = deque(urls)
+        # default timeout
+        self._timeout = kw.pop('timeout', self._timeout)
+        # use random user agent
+        self._randomua = kw.pop('randomua', self._randomua)
+        # authentication
+        self._auth = kw.pop('auth', {})
+        if self._auth:
+            # verify
+            if missing(self._auth, ('username', 'password')):
+                raise TypeError(
+                    u'authentication missing both "username" and "password"',
+                )
+                # proxy
+        self._proxy = kw.pop('proxy', {})
+        if self._proxy:
+            if missing(self._proxy, ('server', 'host')):
+                raise TypeError(u'proxy missing both "server" and "host"')
+        return self
 
     def get(self):
         '''
         http://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9.3
         '''
         return partial(self._request, method='GET')
-    
+
     def post(self):
         '''
         *POST* HTTP method
         http://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9.7
         '''
         return partial(self._request, method='DELETE')
-    
+
     def put(self):
         '''
         *PUT* HTTP method

File wire/http/response.py

 '''wire core HTTP response'''
 
 from StringIO import StringIO
-from collections import defaultdict
 
 from stuf.utils import lazy_class
 from callchain.root.linked import chainlink
 
-from wire.util import Immutable, lru_cache, hiddenhand
+from wire.util import Immutable, hiddenhand
 
+from wire.http.header import CoreHeaders
 from wire.http.util import statuscode, datehttp
-from wire.http.header import CoreHeaders, CoreCookie, CoreMorsel
 
 
-class Morsel(Immutable):
-
-    '''HTTP response specific morsel'''
-
-    def __init__(self):
-        super(Morsel, self).__init__()
-        self._hset('_morsel', CoreMorsel())
-
-    def __getattr__(self, key):
-        try:
-            return object.__getattribute__(self, key)
-        except AttributeError:
-            try:
-                return self._morsel.__getitem__(key)
-            except KeyError:
-                try:
-                    return self._morsel.__getitem__(key.replace('_', '-'))
-                except KeyError:
-                    raise AttributeError(key)
-
-    def __getitem__(self, key):
-        return self._morsel.__getitem__(key)
-
-    def _set(self):
-        return self._morsel.set
-
-    def output(self):
-        return self._morsel.output
-
-    def value(self):
-        return self._morsel.value
-
-
-class Cookie(object):
-
-    '''HTTP response specific cookie'''
-
-    __slots__ = ('_cookie')
-
-    def __init__(self, cookie):
-        self._cookie = self._class()(cookie)
-
-    def __getattr__(self, key):
-        try:
-            return object.__getattribute__(self, key)
-        except AttributeError:
-            try:
-                return self.__getitem__(key)
-            except KeyError:
-                raise AttributeError(key)
-
-    def __getitem__(self, key):
-        return self._cookie.__getitem__(key)
-
-    def __iter__(self):
-        return self._cookie.__iter__()
-
-    def __str__(self):
-        return self._cookie.output()
-
-    @classmethod
-    def _class(cls):
-        class _ResponseCookie(CoreCookie):
-            def _set(self, k, real_value, coded_value):
-                '''invisible setters'''
-                m = self.get(k, Morsel())
-                m._set(k, real_value, coded_value)
-                dict.__setitem__(self, k, m)
-        return _ResponseCookie
-
-
-class Headers(Immutable, CoreHeaders):
+class Headers(CoreHeaders):
 
     '''HTTP response headers'''
 
         # combine cookies
         self._hset('_cookies', ', '.join(cookies) if cookies else '')
 
-    def __getattr__(self, key):
-        try:
-            return object.__getattribute__(self, key)
-        except AttributeError:
-            value = self.get(key)
-            if value is not None: return value
-            raise AttributeError(key)
-
-    def __iter__(self):
-        return iter((k, self.get(k)) for k in self._headers)
-
     @classmethod
     def _set(cls, key, value):
         '''
             elif key in cls.COMMA_SEP:
                 # Split comma-separatable values
                 newval = value.replace(', ', ',').replace(' ,', ',').split(',')
-                if len(newval) > 1: value = newval
+                if len(newval) > 1:
+                    value = newval
             # Reformat HTTP dates as datetime objects
             elif key in cls.DATES:
                 value = datehttp(value)
         '''Python name -> header name mapping'''
         return dict((self._checkname(h), h) for h in self.RES_HEADERS)
 
-    def cookies(self):
-        '''HTTP cookies'''
-        if self._cookies: return Cookie(self._cookies)
-        return ''
-
-    def get(self, k, default=None):
+    def header(self, k, default=None):
         '''
         Return an item associated with a key or a default value
 
         self._headers[k] = v = self._set(k, v)
         return self._hset(tk, v)
 
-    def items(self):
-        '''all the header fields and values'''
-        return lru_cache()(self._items)
 
-    def keys(self):
-        '''list of all header keys'''
-        return lru_cache()(self._keys)
+class Data(chainlink):
 
-    def values(self):
-        '''a list of all header values'''
-        return lru_cache()(self._values)
-    
-    
-class Data(chainlink):
-    
     def data(self):
         '''data, serialized if supported, original if not'''
-        return self._hset(
-            self.format,
-            self._loads(self._original, self.format, **self._options),
-        )
-        
-    @lazy_class
-    def _mime(self):
-        '''built-in serialization mimetypes'''
-        MIME = defaultdict(lambda: 'application/octet-stream')
-        MIME.update({
-            'application/bson': 'bson',  # not official
-            'application/x-bson': 'bson', # not official
-            'text/csv': 'csv', # usual
-            'text/comma-separated-values': 'csv',
-            'application/vnd.ms-excel': 'excel', # accurate
-            'application/msexcel': 'excel',
-            'application/x-msexcel': 'excel',
-            'application/x-ms-excel': 'excel',
-            'application/x-excel': 'excel',
-            'application/xls': 'excel',
-            'application/x-xls': 'excel',
-            'text/html': 'html',
-            'application/xhtml': 'html',
-            'application/xhtml+xml': 'html',
-            'application/json': 'json',
-            'application/x-python-marshal': 'marshal', # made up
-            'multipart/form-data': 'multipart',
-            'application/python-pickle': 'pickle', # for django piston
-            'application/x-python-pickle': 'pickle', # probably more correct
-            'application/x-protobuf': 'protobuf', # not official
-            'application/vnd.google.protobuf': 'protobuf', # not official
-            'application/x-thrift': 'thrift',
-            'application/x-www-form-urlencoded': 'url',
-            'text/xml': 'xml',
-            'application/xml': 'xml',
-            'application/x-yaml': 'yaml',
-        })
-        return MIME
-        
-    def format(self):
-        return self._options.pop('format', self._mime[self.content_type])
-    
-    def raw_data(self):
         return self._original
-    
+
     def touched(self):
         '''Verify MD5 digest of response data against "Content-MD5" digest'''
         digest = self.headers.content_md5
             from base64 import b64decode
             digest = b64decode(digest)
             body_digest = md5.new(self._original).digest()
-            if digest == body_digest: 
+            if digest == body_digest:
                 return False
         return True
-    
+
     def read(self):
         '''read original response content'''
         return StringIO(self._original).read()
         super(CoreResponse, self).__init__()
         hiddenhand(self, '_options', kw)
 
-    def __getattr__(self, k):
-        try:
-            return object.__getattribute__(self, k)
-        except AttributeError:
-            if k in self._loaders.apps:
-                return self._hset(
-                    k, self._loads(self._original, k, **self._options)
-                )
-
-    def __len__(self):