Commits

Anonymous committed 9917f35

started new namespace.

Comments (0)

Files changed (10)

+##############################################################################
+#
+# Copyright (c) 2006 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Bootstrap a buildout-based project
+
+Simply run this script in a directory containing a buildout.cfg.
+The script accepts buildout command-line options, so you can
+use the -c option to specify an alternate configuration file.
+
+$Id: bootstrap.py 105417 2009-11-01 15:15:20Z tarek $
+"""
+
+import os, shutil, sys, tempfile, urllib2
+from optparse import OptionParser
+
+tmpeggs = tempfile.mkdtemp()
+
+is_jython = sys.platform.startswith('java')
+
+# parsing arguments
+parser = OptionParser()
+parser.add_option("-v", "--version", dest="version",
+                          help="use a specific zc.buildout version")
+parser.add_option("-d", "--distribute",
+                   action="store_true", dest="distribute", default=False,
+                   help="Use Disribute rather than Setuptools.")
+
+parser.add_option("-c", None, action="store", dest="config_file",
+                   help=("Specify the path to the buildout configuration "
+                         "file to be used."))
+
+options, args = parser.parse_args()
+
+# if -c was provided, we push it back into args for buildout' main function
+if options.config_file is not None:
+    args += ['-c', options.config_file]
+
+if options.version is not None:
+    VERSION = '==%s' % options.version
+else:
+    VERSION = ''
+
+USE_DISTRIBUTE = options.distribute
+args = args + ['bootstrap']
+
+try:
+    import pkg_resources
+    import setuptools
+    if not hasattr(pkg_resources, '_distribute'):
+        raise ImportError
+except ImportError:
+    ez = {}
+    if USE_DISTRIBUTE:
+        exec urllib2.urlopen('http://python-distribute.org/distribute_setup.py'
+                         ).read() in ez
+        ez['use_setuptools'](to_dir=tmpeggs, download_delay=0, no_fake=True)
+    else:
+        exec urllib2.urlopen('http://peak.telecommunity.com/dist/ez_setup.py'
+                             ).read() in ez
+        ez['use_setuptools'](to_dir=tmpeggs, download_delay=0)
+
+    reload(sys.modules['pkg_resources'])
+    import pkg_resources
+
+if sys.platform == 'win32':
+    def quote(c):
+        if ' ' in c:
+            return '"%s"' % c # work around spawn lamosity on windows
+        else:
+            return c
+else:
+    def quote (c):
+        return c
+
+cmd = 'from setuptools.command.easy_install import main; main()'
+ws  = pkg_resources.working_set
+
+if USE_DISTRIBUTE:
+    requirement = 'distribute'
+else:
+    requirement = 'setuptools'
+
+if is_jython:
+    import subprocess
+
+    assert subprocess.Popen([sys.executable] + ['-c', quote(cmd), '-mqNxd',
+           quote(tmpeggs), 'zc.buildout' + VERSION],
+           env=dict(os.environ,
+               PYTHONPATH=
+               ws.find(pkg_resources.Requirement.parse(requirement)).location
+               ),
+           ).wait() == 0
+
+else:
+    assert os.spawnle(
+        os.P_WAIT, sys.executable, quote (sys.executable),
+        '-c', quote (cmd), '-mqNxd', quote (tmpeggs), 'zc.buildout' + VERSION,
+        dict(os.environ,
+            PYTHONPATH=
+            ws.find(pkg_resources.Requirement.parse(requirement)).location
+            ),
+        ) == 0
+
+ws.add_entry(tmpeggs)
+ws.require('zc.buildout' + VERSION)
+import zc.buildout.buildout
+zc.buildout.buildout.main(args)
+shutil.rmtree(tmpeggs)
+[buildout]
+parts =
+    paster
+    httpclient
+    test
+    pep8
+develop = 
+    httpclient
+versions = versions
+
+[versions]
+lxml=2.2.4
+
+[paster]
+recipe = zc.recipe.egg
+eggs = 
+    PasteScript
+
+[httpclient]
+recipe = zc.recipe.egg
+eggs =
+    aodag.httpclient
+interpreter = python
+
+[test]
+recipe = pbp.recipe.noserunner
+defaults = httpclient --with-doctest --traverse-namespace
+eggs = ${httpclient:eggs}
+
+[pep8]
+recipe = zc.recipe.egg
+eggs = pep8

httpclient/MANIFEST.in

+include README.rst

httpclient/README.rst

+aodag.httpclient
+===============================
+
+Motivation
+-------------------------------
+
+Example
+--------------------------------
+
+

httpclient/aodag/__init__.py

+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
+

httpclient/aodag/httpclient/__init__.py

+#
+import urlparse
+import urllib
+import httplib2
+from lxml import html
+import logging
+import logging.handlers
+import itertools
+from StringIO import StringIO
+from Cookie import SimpleCookie
+
+
+def parse_html(contents):
+    """
+    parse html string
+    """
+
+    f = StringIO(contents)
+    tree = html.parse(f)
+    return tree
+
+
+class HTTPClient(object):
+    def __init__(self):
+        self.currentUrl = None
+        self.http = httplib2.Http(cache=False)
+        self.contents = None
+        self.response = None
+        self.cookies = dict()
+        self.logger = logging.getLogger("httpclient_logger")
+        self.logger.setLevel(logging.ERROR)
+        self.encoding = 'utf-8'
+
+    def debug(self):
+        """
+        logging debug mode
+        """
+        self.logger.setLevel(logging.DEBUG)
+
+    def verbose(self):
+        """
+        logging verbose mode
+        """
+        self.logger.setLevel(logging.INFO)
+
+    def quiet(self):
+        self.logger.setLevel(logging.CRITICAL)
+
+    def go(self, url):
+        self._request(url)
+
+    def save(self, file):
+        if self.contents is None:
+            return
+        if hasattr(file, 'write'):
+            file.write(self.contents)
+        else:
+            f = open(file, "wb")
+            f.write(self.contents)
+            f.close()
+
+    @property
+    def tree(self):
+        return parse_html(self.contents)
+
+    def links(self):
+        if self.contents is None:
+            return None
+        return [e for e in self.tree.xpath('//a/@href')]
+
+    def _request(self, url, headers={}, method='GET', body=None):
+        url = urlparse.urljoin(self.currentUrl, url)
+        cookie = self.cookie
+        if cookie:
+            headers['Cookie'] = ";".join(['%s="%s"' % (k, v) for k, v in cookie.iteritems()])
+
+        self.response, self.contents = self.http.request(url,
+                                                         method=method,
+                                                         body=body,
+                                                         headers=headers)
+        self.currentUrl = url
+
+        o = urlparse.urlparse(url)
+        self.domain = o[1]
+        cookie = self.response.get('set-cookie')
+        if cookie:
+            self.add_cookie(self.domain, cookie)
+
+        if self.response.status == 302:
+            return self._request(self.response['location'])
+
+        self.forms = [Form(f, self) for f in self.tree.xpath('//form')]
+
+        return self.response, self.contents
+
+    def add_cookie(self, domain, new_cookie):
+        cookie = SimpleCookie()
+        cookie.load(new_cookie)
+        cookies = dict([(c, cookie[c].value) for c in cookie])
+        domain_cookies = self.cookies.get(domain, {})
+        domain_cookies.update(cookies)
+        self.cookies[domain] = domain_cookies
+
+    @property
+    def status(self):
+        if self.response is None:
+            return None
+        return self.response.status
+
+    @property
+    def cookie(self):
+        if self.currentUrl is None:
+            return None
+        if self.domain is None:
+            return None
+        return self.cookies.get(self.domain)
+
+    def get(self, action, query):
+        headers = {}
+        self._request(action, method='GET', body=body, headers=headers)
+
+    def post(self, action, body,
+            content_type='application/x-www-form-urlencoded'):
+
+        headers = {'Content-type': content_type,
+                   'Content-length': '%d' % len(body),
+                   }
+
+        self._request(action, method='POST', body=body, headers=headers)
+
+
+class Form(object):
+    """
+    >>> import lxml.etree
+    >>> e = lxml.etree.XML('<input type="text" name="input1" value="test" />')
+    """
+
+    def __init__(self, form_element, client):
+        self.client = client
+        self.element = form_element
+        self.action = form_element.attrib.get('action')
+        self.method = form_element.attrib.get('method', 'get')
+        self.fields = dict(self.get_input_fields(self.element)
+                + self.get_select_fields(self.element)
+                + self.get_radio_field(self.element))
+
+    def set(self, dct):
+        """
+        >>> import lxml.etree
+        >>> e = lxml.etree.XML('<form>'
+        ...        '<input type="text" name="input1" value="test" />'
+        ...        '<select name="select1">'
+        ...        '<option value="1" selected="selected"></option></select>'
+        ...        '<input type="radio" name="radio1" value="a" />'
+        ...        '<input type="radio" name="radio1" value="b" />'
+        ...        '<input type="radio" name="radio1" value="c" checked="checked" />'
+        ...        '<input type="radio" name="radio2" value="A" />'
+        ...        '<input type="radio" name="radio2" value="B" checked="checked" />'
+        ...        '<input type="radio" name="radio2" value="C" />'
+        ...        '<input type="submit" value="post" name="post"/></form>')
+        >>> form = Form(e, HTTPClient())
+        >>> form.set({'input1':'changed', 'radio1':'a', 'select1':'2'})
+        >>> form.form_values()
+        [('input1', 'changed'), ('radio1', 'a'), ('select1', '2'), ('radio2', 'B'), ('post', 'post')]
+        """
+        for k, v in dct.iteritems():
+            self.fields[k].value = v
+
+    def get_input_fields(self, form_element):
+
+        return [(e.attrib.get('name'), Field(e))
+                for e in form_element.xpath('.//input')
+                if e.attrib.get('name')
+                and e.attrib.get('type') != 'radio']
+
+    def get_select_fields(self, form_element):
+
+        return [(e.attrib.get('name'), SelectField(e))
+                for e in form_element.xpath('.//select')
+                if e.attrib.get('name')]
+
+    def get_radio_field(self, form_element):
+        radio_fields = form_element.xpath(".//input[@type='radio']")
+        key_func = lambda e: e.attrib.get('name')
+        g = itertools.groupby(sorted(radio_fields, key=key_func), key_func)
+        return [(name, RadioButtonField(list(inputs))) for name, inputs in g]
+
+
+    def form_values(self, submit_name=None):
+        """
+        >>> import lxml.etree
+        >>> e = lxml.etree.XML('<form>'
+        ...        '<input type="text" name="input1" value="test" />'
+        ...        '<select name="select1">'
+        ...        '<option value="1" selected="selected"></option></select>'
+        ...        '<input type="radio" name="radio1" value="a" />'
+        ...        '<input type="radio" name="radio1" value="b" />'
+        ...        '<input type="radio" name="radio1" value="c" checked="checked" />'
+        ...        '<input type="radio" name="radio2" value="A" />'
+        ...        '<input type="radio" name="radio2" value="B" checked="checked" />'
+        ...        '<input type="radio" name="radio2" value="C" />'
+        ...        '<input type="submit" value="post" name="post"/></form>')
+        >>> form = Form(e, HTTPClient())
+        >>> form.form_values()
+        [('input1', 'test'), ('radio1', 'c'), ('select1', '1'), ('radio2', 'B'), ('post', 'post')]
+        """
+        vars = [(f.name, f.value.encode(self.client.encoding))
+                for n, f in self.fields.iteritems()
+                if f.name is not None and f.value is not None
+                and f.type != 'submit']
+
+        submitbuttons = [f for n, f in self.fields.iteritems()
+                if f.type == 'submit']
+
+        if submitbuttons and submit_name is None:
+            submit = submitbuttons[0]
+            submit_value = submit.value.encode(self.client.encoding)
+            vars = vars + [(submit.name, submit_value)]
+        else:
+            submit = [f for f in submitbuttons if f.name == submit_name]
+            if submit:
+                submit = submit[0]
+                submit_value = submit.value.encode(self.client.encoding)
+                vars = vars + [(submit.name, submit_value)]
+
+        return vars
+
+    def submit(self, name=None):
+        vars = self.form_values(name)
+        self.request(vars)
+
+    def request(self, vars):
+        body = urllib.urlencode(vars)
+        if self.method == 'get':
+            self.client.get(self.action, body)
+        elif self.method == 'post':
+            self.client.post(self.action, body)
+        else:
+            raise Exception('Unknown method %s' % self.method)
+
+
+class Field(object):
+    """
+    >>> import lxml.etree
+    >>> e = lxml.etree.XML('<input type="text" name="input1" value="test" />')
+    >>> f = Field(e)
+    >>> f.name
+    'input1'
+    >>> f.value
+    'test'
+    >>> f.type
+    'text'
+    """
+
+    def __init__(self, field_element):
+        self.element = field_element
+        self.value = self.element.attrib.get('value')
+        self.name = self.element.attrib.get('name')
+        self.type = self.element.attrib.get('type')
+
+    def __repr__(self):
+        return str((self.name, self.value))
+
+
+class SelectField(object):
+    """
+    >>> import lxml.etree
+    >>> e = lxml.etree.XML('<select name="input1">'
+    ...        '<option value="1">A</option>'
+    ...        '<option value="2" selected="selected">B</option>'
+    ...        '</select>')
+    >>> f = SelectField(e)
+    >>> f.name
+    'input1'
+    >>> f.value
+    '2'
+    >>> f.value = "1"
+    >>> f.value
+    '1'
+    """
+
+    def __init__(self, field_element):
+        self.element = field_element
+        self.value = None
+        self.type = 'select'
+        selected = self.element.xpath('.//option[@selected]')
+        if selected:
+            self.value = selected[0].attrib.get('value')
+
+        self.name = self.element.attrib.get('name')
+
+    def __repr__(self):
+        return str((self.name, self.value))
+
+
+class RadioButtonField(object):
+    """
+    >>> import lxml.etree
+    >>> def create_element(v):
+    ...     html_tmpl = '<input type="radiobutton" value="%s" name="test" %s/>'
+    ...     return lxml.etree.XML(html_tmpl % (v, 'checked="checked"' if v == 'd' else ''))
+    >>> elements = [create_element(v) 
+    ...        for v in 'abcde']
+    >>> f = RadioButtonField(elements)
+    >>> f.name
+    'test'
+    >>> f.value
+    'd'
+    >>> f.value = "a"
+    >>> f.value
+    'a'
+    """
+
+    def __init__(self, field_elements):
+        assert len(field_elements) > 0
+        self.elements = field_elements
+        self.value = None
+        self.type = 'radiobutton'
+        checked = [e for e in self.elements if e.get('checked')]
+        if checked:
+            self.value = checked[0].attrib.get('value')
+
+        self.name = self.elements[0].attrib.get('name')
+
+    def __repr__(self):
+        return str((self.name, self.value))
+
+def main():
+    """
+    main script runner
+    """

httpclient/setup.cfg

+[egg_info]
+tag_build = dev
+tag_svn_revision = true

httpclient/setup.py

+from setuptools import setup, find_packages
+import sys, os
+
+version = '0.0'
+
+requires = [
+        'lxml',
+        'httplib2',
+        ]
+major, minor, micro, releaselevel, serial = sys.version_info
+if major == 2 and minor < 6:
+    requires += ['multiprocessing']
+
+long_description = open(os.path.join(os.path.dirname(__file__), 'README.rst')).read()
+
+setup(name='aodag.httpclient',
+      version=version,
+      description="",
+      long_description=long_description,
+      classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+      keywords='',
+      author='',
+      author_email='',
+      url='',
+      license='',
+      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
+      namespace_packages=["aodag"],
+      include_package_data=True,
+      package_data={
+          '':['*.rst'],
+      },
+      zip_safe=False,
+      install_requires=requires,
+      entry_points="""
+      # -*- Entry points: -*-
+      """,
+      )

httpclient/tests/__init__.py

+#def test_failure():
+#    assert False
+
+import httpclient
+import lxml.html
+from StringIO import StringIO
+
+
+def test_simpleform():
+    html = """\
+    <form>
+        <input type="text" name="input1" value="x">
+        <input type="text" name="input2" >
+    </form>
+    """
+    f = StringIO(html)
+    tree = lxml.html.parse(f)
+
+    form = httpclient.Form(tree.xpath('.')[0], None)
+    assert len(form.fields) == 2
+
+def test_selectionform():
+    html = """\
+    <form>
+        <select name="select1">
+            <option value="1">A</option>
+            <option value="2" selected="selected">B</option>
+        </select>
+    </form>
+    """
+
+    f = StringIO(html)
+    tree = lxml.html.parse(f)
+
+    form = httpclient.Form(tree.xpath('.')[0], None)
+    assert len(form.fields) == 1
+    print form.fields['select1'].value
+    assert form.fields['select1'].value == '2'
+

httpclient/tests/test_functional.py

+from wsgiref.simple_server import make_server
+from multiprocessing import Process
+from httpclient import HTTPClient
+from StringIO import StringIO
+import tempfile
+import os
+import shutil
+from Cookie import SimpleCookie
+
+HTTP_PORT = 8000
+
+class ContentsApplication(object):
+    def __init__(self, contents):
+        self.contents = contents
+
+    def __call__(self, environ, start_response):
+        start_response("200 OK",
+                [('Content-type', 'text/plain')])
+        return [self.contents]
+
+
+def runserver(port, app):
+    httpd = make_server('', port, app)
+    httpd.serve_forever()
+
+def with_app(port, app):
+    def wrap(inner_test):
+        def test():
+            test_site = Process(target=runserver, args=(port, app))
+            test_site.start()
+            url = 'http://localhost:%d/' % port
+            try:
+                inner_test(url)
+
+            finally:
+                test_site.terminate()
+        return test
+    return wrap
+
+class CookieApplication(object):
+    def __init__(self, cookies):
+        self.cookies = cookies
+
+    def __call__(self, environ, start_response):
+        start_response("200 OK",
+                [('Content-type', 'text/plain')]
+                + [('Set-Cookie', "%s=%s" % (k, v)) for k, v in self.cookies.iteritems()])
+
+        cookie = environ.get('HTTP_COOKIE')
+        if cookie:
+            c = SimpleCookie()
+            c.load(cookie)
+            return ["\n".join(["%s = %s" % (c_name, c[c_name].value) for c_name in self.cookies])]
+        return ["<html></html>"]
+
+SIMPLE_CONTENTS = """\
+This is Test
+"""
+
+@with_app(HTTP_PORT, ContentsApplication(SIMPLE_CONTENTS))
+def test_save(url):
+    c = HTTPClient()
+    c.go(url)
+    f = StringIO()
+    c.save(f)
+    assert f.getvalue() == SIMPLE_CONTENTS
+    d = tempfile.mkdtemp()
+    t = os.path.join(d, 'test.txt')
+    try:
+        c.save(t)
+        assert open(t).read() == SIMPLE_CONTENTS
+    finally:
+        shutil.rmtree(d)
+
+@with_app(HTTP_PORT, CookieApplication({'test_cookie':'"this is test"', 'test2':'"second cookie"'}))
+def test_cookie(url):
+    """
+    """
+    c = HTTPClient()
+    c.go(url)
+    assert c.contents == "<html></html>"
+    c.go(url)
+    print c.contents
+    assert c.contents == """\
+test_cookie = this is test
+test2 = second cookie"""
+