Commits

Mike Bayer  committed a629df3

- merged -r481:499 of py3k branch.
- Python 3 support is added ! See README.py3k
for installation and testing notes.
[ticket:119]

  • Participants
  • Parent commits 4d91d76

Comments (0)

Files changed (27)

 0.3
 - Python 2.3 support is dropped. [ticket:123]
+
+- Python 3 support is added ! See README.py3k
+  for installation and testing notes.
+  [ticket:119]
   
 - Unit tests now run with nose.  [ticket:127]
 
+=================
+PYTHON 3 SUPPORT
+=================
+
+Python 3 support in Mako is provided by the Python 2to3 script.
+
+Installing Distribute
+---------------------
+
+Distribute should be installed with the Python3 installation.  The
+distribute bootloader is included.
+
+Running as a user with permission to modify the Python distribution,
+install Distribute:
+
+    python3 distribute_setup.py
+    
+
+Installing Mako in Python 3
+---------------------------------
+
+Once Distribute is installed, Mako can be installed directly.  
+The 2to3 process will kick in which takes several minutes:
+
+    python3 setup.py install
+
+Converting Tests, Examples, Source to Python 3
+----------------------------------------------
+
+To convert all files in the source distribution, run 
+the 2to3 script:
+
+    2to3 --no-diffs -w lib test examples
+
+The above will rewrite all files in-place in Python 3 format.
+
+Running Tests
+-------------
+
+To run the unit tests, ensure Distribute is installed as above,
+and also that at least the ./lib/ and ./test/ directories have been converted
+to Python 3 using the source tool above.   A Python 3 version of Nose
+can be acquired from Bitbucket using Mercurial:
+
+    hg clone http://bitbucket.org/jpellerin/nose3/
+    cd nose3
+    python3 setup.py install
+
+The tests can then be run using the "nosetests3" script installed by the above.
+

File distribute_setup.py

+#!python
+"""Bootstrap distribute installation
+
+If you want to use setuptools in your package's setup.py, just include this
+file in the same directory with it, and add this to the top of your setup.py::
+
+    from distribute_setup import use_setuptools
+    use_setuptools()
+
+If you want to require a specific version of setuptools, set a download
+mirror, or use an alternate download directory, you can do so by supplying
+the appropriate options to ``use_setuptools()``.
+
+This file can also be run as a script to install or upgrade setuptools.
+"""
+import os
+import sys
+import time
+import fnmatch
+import tempfile
+import tarfile
+from distutils import log
+
+try:
+    from site import USER_SITE
+except ImportError:
+    USER_SITE = None
+
+try:
+    import subprocess
+
+    def _python_cmd(*args):
+        args = (sys.executable,) + args
+        return subprocess.call(args) == 0
+
+except ImportError:
+    # will be used for python 2.3
+    def _python_cmd(*args):
+        args = (sys.executable,) + args
+        # quoting arguments if windows
+        if sys.platform == 'win32':
+            def quote(arg):
+                if ' ' in arg:
+                    return '"%s"' % arg
+                return arg
+            args = [quote(arg) for arg in args]
+        return os.spawnl(os.P_WAIT, sys.executable, *args) == 0
+
+DEFAULT_VERSION = "0.6.10"
+DEFAULT_URL = "http://pypi.python.org/packages/source/d/distribute/"
+SETUPTOOLS_FAKED_VERSION = "0.6c11"
+
+SETUPTOOLS_PKG_INFO = """\
+Metadata-Version: 1.0
+Name: setuptools
+Version: %s
+Summary: xxxx
+Home-page: xxx
+Author: xxx
+Author-email: xxx
+License: xxx
+Description: xxx
+""" % SETUPTOOLS_FAKED_VERSION
+
+
+def _install(tarball):
+    # extracting the tarball
+    tmpdir = tempfile.mkdtemp()
+    log.warn('Extracting in %s', tmpdir)
+    old_wd = os.getcwd()
+    try:
+        os.chdir(tmpdir)
+        tar = tarfile.open(tarball)
+        _extractall(tar)
+        tar.close()
+
+        # going in the directory
+        subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
+        os.chdir(subdir)
+        log.warn('Now working in %s', subdir)
+
+        # installing
+        log.warn('Installing Distribute')
+        if not _python_cmd('setup.py', 'install'):
+            log.warn('Something went wrong during the installation.')
+            log.warn('See the error message above.')
+    finally:
+        os.chdir(old_wd)
+
+
+def _build_egg(egg, tarball, to_dir):
+    # extracting the tarball
+    tmpdir = tempfile.mkdtemp()
+    log.warn('Extracting in %s', tmpdir)
+    old_wd = os.getcwd()
+    try:
+        os.chdir(tmpdir)
+        tar = tarfile.open(tarball)
+        _extractall(tar)
+        tar.close()
+
+        # going in the directory
+        subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
+        os.chdir(subdir)
+        log.warn('Now working in %s', subdir)
+
+        # building an egg
+        log.warn('Building a Distribute egg in %s', to_dir)
+        _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
+
+    finally:
+        os.chdir(old_wd)
+    # returning the result
+    log.warn(egg)
+    if not os.path.exists(egg):
+        raise IOError('Could not build the egg.')
+
+
+def _do_download(version, download_base, to_dir, download_delay):
+    egg = os.path.join(to_dir, 'distribute-%s-py%d.%d.egg'
+                       % (version, sys.version_info[0], sys.version_info[1]))
+    if not os.path.exists(egg):
+        tarball = download_setuptools(version, download_base,
+                                      to_dir, download_delay)
+        _build_egg(egg, tarball, to_dir)
+    sys.path.insert(0, egg)
+    import setuptools
+    setuptools.bootstrap_install_from = egg
+
+
+def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
+                   to_dir=os.curdir, download_delay=15, no_fake=True):
+    # making sure we use the absolute path
+    to_dir = os.path.abspath(to_dir)
+    was_imported = 'pkg_resources' in sys.modules or \
+        'setuptools' in sys.modules
+    try:
+        try:
+            import pkg_resources
+            if not hasattr(pkg_resources, '_distribute'):
+                if not no_fake:
+                    _fake_setuptools()
+                raise ImportError
+        except ImportError:
+            return _do_download(version, download_base, to_dir, download_delay)
+        try:
+            pkg_resources.require("distribute>="+version)
+            return
+        except pkg_resources.VersionConflict:
+            e = sys.exc_info()[1]
+            if was_imported:
+                sys.stderr.write(
+                "The required version of distribute (>=%s) is not available,\n"
+                "and can't be installed while this script is running. Please\n"
+                "install a more recent version first, using\n"
+                "'easy_install -U distribute'."
+                "\n\n(Currently using %r)\n" % (version, e.args[0]))
+                sys.exit(2)
+            else:
+                del pkg_resources, sys.modules['pkg_resources']    # reload ok
+                return _do_download(version, download_base, to_dir,
+                                    download_delay)
+        except pkg_resources.DistributionNotFound:
+            return _do_download(version, download_base, to_dir,
+                                download_delay)
+    finally:
+        if not no_fake:
+            _create_fake_setuptools_pkg_info(to_dir)
+
+def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
+                        to_dir=os.curdir, delay=15):
+    """Download distribute from a specified location and return its filename
+
+    `version` should be a valid distribute version number that is available
+    as an egg for download under the `download_base` URL (which should end
+    with a '/'). `to_dir` is the directory where the egg will be downloaded.
+    `delay` is the number of seconds to pause before an actual download
+    attempt.
+    """
+    # making sure we use the absolute path
+    to_dir = os.path.abspath(to_dir)
+    try:
+        from urllib.request import urlopen
+    except ImportError:
+        from urllib2 import urlopen
+    tgz_name = "distribute-%s.tar.gz" % version
+    url = download_base + tgz_name
+    saveto = os.path.join(to_dir, tgz_name)
+    src = dst = None
+    if not os.path.exists(saveto):  # Avoid repeated downloads
+        try:
+            log.warn("Downloading %s", url)
+            src = urlopen(url)
+            # Read/write all in one block, so we don't create a corrupt file
+            # if the download is interrupted.
+            data = src.read()
+            dst = open(saveto, "wb")
+            dst.write(data)
+        finally:
+            if src:
+                src.close()
+            if dst:
+                dst.close()
+    return os.path.realpath(saveto)
+
+
+def _patch_file(path, content):
+    """Will backup the file then patch it"""
+    existing_content = open(path).read()
+    if existing_content == content:
+        # already patched
+        log.warn('Already patched.')
+        return False
+    log.warn('Patching...')
+    _rename_path(path)
+    f = open(path, 'w')
+    try:
+        f.write(content)
+    finally:
+        f.close()
+    return True
+
+
+def _same_content(path, content):
+    return open(path).read() == content
+
+def _no_sandbox(function):
+    def __no_sandbox(*args, **kw):
+        try:
+            from setuptools.sandbox import DirectorySandbox
+            def violation(*args):
+                pass
+            DirectorySandbox._old = DirectorySandbox._violation
+            DirectorySandbox._violation = violation
+            patched = True
+        except ImportError:
+            patched = False
+
+        try:
+            return function(*args, **kw)
+        finally:
+            if patched:
+                DirectorySandbox._violation = DirectorySandbox._old
+                del DirectorySandbox._old
+
+    return __no_sandbox
+
+@_no_sandbox
+def _rename_path(path):
+    new_name = path + '.OLD.%s' % time.time()
+    log.warn('Renaming %s into %s', path, new_name)
+    os.rename(path, new_name)
+    return new_name
+
+def _remove_flat_installation(placeholder):
+    if not os.path.isdir(placeholder):
+        log.warn('Unkown installation at %s', placeholder)
+        return False
+    found = False
+    for file in os.listdir(placeholder):
+        if fnmatch.fnmatch(file, 'setuptools*.egg-info'):
+            found = True
+            break
+    if not found:
+        log.warn('Could not locate setuptools*.egg-info')
+        return
+
+    log.warn('Removing elements out of the way...')
+    pkg_info = os.path.join(placeholder, file)
+    if os.path.isdir(pkg_info):
+        patched = _patch_egg_dir(pkg_info)
+    else:
+        patched = _patch_file(pkg_info, SETUPTOOLS_PKG_INFO)
+
+    if not patched:
+        log.warn('%s already patched.', pkg_info)
+        return False
+    # now let's move the files out of the way
+    for element in ('setuptools', 'pkg_resources.py', 'site.py'):
+        element = os.path.join(placeholder, element)
+        if os.path.exists(element):
+            _rename_path(element)
+        else:
+            log.warn('Could not find the %s element of the '
+                     'Setuptools distribution', element)
+    return True
+
+
+def _after_install(dist):
+    log.warn('After install bootstrap.')
+    placeholder = dist.get_command_obj('install').install_purelib
+    _create_fake_setuptools_pkg_info(placeholder)
+
+@_no_sandbox
+def _create_fake_setuptools_pkg_info(placeholder):
+    if not placeholder or not os.path.exists(placeholder):
+        log.warn('Could not find the install location')
+        return
+    pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1])
+    setuptools_file = 'setuptools-%s-py%s.egg-info' % \
+            (SETUPTOOLS_FAKED_VERSION, pyver)
+    pkg_info = os.path.join(placeholder, setuptools_file)
+    if os.path.exists(pkg_info):
+        log.warn('%s already exists', pkg_info)
+        return
+
+    log.warn('Creating %s', pkg_info)
+    f = open(pkg_info, 'w')
+    try:
+        f.write(SETUPTOOLS_PKG_INFO)
+    finally:
+        f.close()
+
+    pth_file = os.path.join(placeholder, 'setuptools.pth')
+    log.warn('Creating %s', pth_file)
+    f = open(pth_file, 'w')
+    try:
+        f.write(os.path.join(os.curdir, setuptools_file))
+    finally:
+        f.close()
+
+def _patch_egg_dir(path):
+    # let's check if it's already patched
+    pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO')
+    if os.path.exists(pkg_info):
+        if _same_content(pkg_info, SETUPTOOLS_PKG_INFO):
+            log.warn('%s already patched.', pkg_info)
+            return False
+    _rename_path(path)
+    os.mkdir(path)
+    os.mkdir(os.path.join(path, 'EGG-INFO'))
+    pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO')
+    f = open(pkg_info, 'w')
+    try:
+        f.write(SETUPTOOLS_PKG_INFO)
+    finally:
+        f.close()
+    return True
+
+
+def _before_install():
+    log.warn('Before install bootstrap.')
+    _fake_setuptools()
+
+
+def _under_prefix(location):
+    if 'install' not in sys.argv:
+        return True
+    args = sys.argv[sys.argv.index('install')+1:]
+    for index, arg in enumerate(args):
+        for option in ('--root', '--prefix'):
+            if arg.startswith('%s=' % option):
+                top_dir = arg.split('root=')[-1]
+                return location.startswith(top_dir)
+            elif arg == option:
+                if len(args) > index:
+                    top_dir = args[index+1]
+                    return location.startswith(top_dir)
+            elif option == '--user' and USER_SITE is not None:
+                return location.startswith(USER_SITE)
+    return True
+
+
+def _fake_setuptools():
+    log.warn('Scanning installed packages')
+    try:
+        import pkg_resources
+    except ImportError:
+        # we're cool
+        log.warn('Setuptools or Distribute does not seem to be installed.')
+        return
+    ws = pkg_resources.working_set
+    try:
+        setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools',
+                                  replacement=False))
+    except TypeError:
+        # old distribute API
+        setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools'))
+
+    if setuptools_dist is None:
+        log.warn('No setuptools distribution found')
+        return
+    # detecting if it was already faked
+    setuptools_location = setuptools_dist.location
+    log.warn('Setuptools installation detected at %s', setuptools_location)
+
+    # if --root or --preix was provided, and if
+    # setuptools is not located in them, we don't patch it
+    if not _under_prefix(setuptools_location):
+        log.warn('Not patching, --root or --prefix is installing Distribute'
+                 ' in another location')
+        return
+
+    # let's see if its an egg
+    if not setuptools_location.endswith('.egg'):
+        log.warn('Non-egg installation')
+        res = _remove_flat_installation(setuptools_location)
+        if not res:
+            return
+    else:
+        log.warn('Egg installation')
+        pkg_info = os.path.join(setuptools_location, 'EGG-INFO', 'PKG-INFO')
+        if (os.path.exists(pkg_info) and
+            _same_content(pkg_info, SETUPTOOLS_PKG_INFO)):
+            log.warn('Already patched.')
+            return
+        log.warn('Patching...')
+        # let's create a fake egg replacing setuptools one
+        res = _patch_egg_dir(setuptools_location)
+        if not res:
+            return
+    log.warn('Patched done.')
+    _relaunch()
+
+
+def _relaunch():
+    log.warn('Relaunching...')
+    # we have to relaunch the process
+    args = [sys.executable] + sys.argv
+    sys.exit(subprocess.call(args))
+
+
+def _extractall(self, path=".", members=None):
+    """Extract all members from the archive to the current working
+       directory and set owner, modification time and permissions on
+       directories afterwards. `path' specifies a different directory
+       to extract to. `members' is optional and must be a subset of the
+       list returned by getmembers().
+    """
+    import copy
+    import operator
+    from tarfile import ExtractError
+    directories = []
+
+    if members is None:
+        members = self
+
+    for tarinfo in members:
+        if tarinfo.isdir():
+            # Extract directories with a safe mode.
+            directories.append(tarinfo)
+            tarinfo = copy.copy(tarinfo)
+            tarinfo.mode = 448 # decimal for oct 0700
+        self.extract(tarinfo, path)
+
+    # Reverse sort directories.
+    if sys.version_info < (2, 4):
+        def sorter(dir1, dir2):
+            return cmp(dir1.name, dir2.name)
+        directories.sort(sorter)
+        directories.reverse()
+    else:
+        directories.sort(key=operator.attrgetter('name'), reverse=True)
+
+    # Set correct owner, mtime and filemode on directories.
+    for tarinfo in directories:
+        dirpath = os.path.join(path, tarinfo.name)
+        try:
+            self.chown(tarinfo, dirpath)
+            self.utime(tarinfo, dirpath)
+            self.chmod(tarinfo, dirpath)
+        except ExtractError:
+            e = sys.exc_info()[1]
+            if self.errorlevel > 1:
+                raise
+            else:
+                self._dbg(1, "tarfile: %s" % e)
+
+
+def main(argv, version=DEFAULT_VERSION):
+    """Install or upgrade setuptools and EasyInstall"""
+    tarball = download_setuptools()
+    _install(tarball)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
-"""utilities for analyzing expressions and blocks of Python code, as well as generating Python from AST nodes"""
+"""utilities for analyzing expressions and blocks of Python 
+code, as well as generating Python from AST nodes"""
 
 from mako import exceptions, pyparser, util
 import re
         
         # note that an identifier can be in both the undeclared and declared lists.
 
-        # using AST to parse instead of using code.co_varnames, code.co_names has several advantages:
-        # - we can locate an identifier as "undeclared" even if its declared later in the same block of code
-        # - AST is less likely to break with version changes (for example, the behavior of co_names changed a little bit
+        # using AST to parse instead of using code.co_varnames, 
+        # code.co_names has several advantages:
+        # - we can locate an identifier as "undeclared" even if 
+        # its declared later in the same block of code
+        # - AST is less likely to break with version changes 
+        # (for example, the behavior of co_names changed a little bit
         # in python version 2.5)
         if isinstance(code, basestring):
             expr = pyparser.parse(code.lstrip(), "exec", **exception_kwargs)
     def __init__(self, code, **exception_kwargs):
         m = re.match(r'^(\w+)(?:\s+(.*?))?:\s*(#|$)', code.strip(), re.S)
         if not m:
-            raise exceptions.CompileException("Fragment '%s' is not a partial control statement" % code, **exception_kwargs)
+            raise exceptions.CompileException(
+                            "Fragment '%s' is not a partial control statement" % 
+                            code, **exception_kwargs)
         if m.group(3):
             code = code[:m.start(3)]
         (keyword, expr) = m.group(1,2)
         elif keyword == 'except':
             code = "try:pass\n" + code + "pass"
         else:
-            raise exceptions.CompileException("Unsupported control keyword: '%s'" % keyword, **exception_kwargs)
+            raise exceptions.CompileException(
+                                "Unsupported control keyword: '%s'" % 
+                                keyword, **exception_kwargs)
         super(PythonFragment, self).__init__(code, **exception_kwargs)
         
         
         f = pyparser.ParseFunc(self, **exception_kwargs)
         f.visit(expr)
         if not hasattr(self, 'funcname'):
-            raise exceptions.CompileException("Code '%s' is not a function declaration" % code, **exception_kwargs)
+            raise exceptions.CompileException(
+                                "Code '%s' is not a function declaration" % code,
+                                **exception_kwargs)
         if not allow_kwargs and self.kwargs:
-            raise exceptions.CompileException("'**%s' keyword argument not allowed here" % self.argnames[-1], **exception_kwargs)
+            raise exceptions.CompileException(
+                                "'**%s' keyword argument not allowed here" % 
+                                self.argnames[-1], **exception_kwargs)
             
     def get_argument_expressions(self, include_defaults=True):
         """return the argument declarations of this FunctionDecl as a printable list."""
+        
         namedecls = []
         defaults = [d for d in self.defaults]
         kwargs = self.kwargs
             else:
                 default = len(defaults) and defaults.pop() or None
             if include_defaults and default:
-                namedecls.insert(0, "%s=%s" % (arg, pyparser.ExpressionGenerator(default).value()))
+                namedecls.insert(0, "%s=%s" % 
+                            (arg, 
+                            pyparser.ExpressionGenerator(default).value()
+                            )
+                        )
             else:
                 namedecls.insert(0, arg)
         return namedecls
 
 class FunctionArgs(FunctionDecl):
     """the argument portion of a function declaration"""
+    
     def __init__(self, code, **kwargs):
         super(FunctionArgs, self).__init__("def ANON(%s):pass" % code, **kwargs)

File mako/codegen.py

     """Generate module source code given a parsetree node, 
       uri, and optional source filename"""
 
+    # if on Py2K, push the "source_encoding" string to be
+    # a bytestring itself, as we will be embedding it into 
+    # the generated source and we don't want to coerce the 
+    # result into a unicode object, in "disable_unicode" mode
+    if not util.py3k and isinstance(source_encoding, unicode):
+        source_encoding = source_encoding.encode(source_encoding)
+        
+        
     buf = util.FastEncodingBuffer()
 
     printer = PythonPrinter(buf)
             if not self.in_def and len(self.identifiers.locally_assigned) > 0:
                 # if we are the "template" def, fudge locally declared/modified variables into the "__M_locals" dictionary,
                 # which is used for def calls within the same template, to simulate "enclosing scope"
-                self.printer.writeline('__M_locals.update(__M_dict_builtin([(__M_key, __M_locals_builtin()[__M_key]) for __M_key in [%s] if __M_key in __M_locals_builtin()]))' % ','.join([repr(x) for x in node.declared_identifiers()]))
-                
+                self.printer.writeline('__M_locals_builtin_stored = __M_locals_builtin()')
+                self.printer.writeline('__M_locals.update(__M_dict_builtin([(__M_key, __M_locals_builtin_stored[__M_key]) for __M_key in [%s] if __M_key in __M_locals_builtin_stored]))' % ','.join([repr(x) for x in node.declared_identifiers()]))
+
     def visitIncludeTag(self, node):
         self.write_source_comment(node)
         args = node.attributes.get('args')

File mako/exceptions.py

     if filename is None:
         return " at line: %d char: %d" % (lineno, pos)
     else:
-        return " in file '%s' at line: %d char: %d" % (filename, lineno, pos)     
+        return " in file '%s' at line: %d char: %d" % (filename, lineno, pos)
+        
+        
 class CompileException(MakoException):
     def __init__(self, message, source, lineno, pos, filename):
         MakoException.__init__(self, message + _format_filepos(lineno, pos, filename))
         self.pos = pos
         self.filename = filename
         self.source = source
+
+class UnsupportedError(MakoException):
+    """raised when a retired feature is used."""
         
 class TemplateLookupException(MakoException):
     pass
                     template_filename = info.template_filename or filename
                 except KeyError:
                     # A normal .py file (not a Template)
-                    try:
-                        fp = open(filename)
-                        encoding = util.parse_encoding(fp)
-                        fp.close()
-                    except IOError:
-                        encoding = None
-                    if encoding:
-                        line = line.decode(encoding)
-                    else:
-                        line = line.decode('ascii', 'replace')
-                    new_trcback.append((filename, lineno, function, line, None, None, None, None))
+                    if not util.py3k:
+                        try:
+                            fp = open(filename, 'rb')
+                            encoding = util.parse_encoding(fp)
+                            fp.close()
+                        except IOError:
+                            encoding = None
+                        if encoding:
+                            line = line.decode(encoding)
+                        else:
+                            line = line.decode('ascii', 'replace')
+                    new_trcback.append((filename, lineno, function, line, 
+                                            None, None, None, None))
                     continue
 
                 template_ln = module_ln = 1
                 template_line = template_lines[template_ln - 1]
             else:
                 template_line = None
-            new_trcback.append((filename, lineno, function, line, template_filename, template_ln, template_line, template_source))
+            new_trcback.append((filename, lineno, function, 
+                                line, template_filename, template_ln, 
+                                template_line, template_source))
         if not self.source:
             for l in range(len(new_trcback)-1, 0, -1):
                 if new_trcback[l][5]:
             else:
                 try:
                     # A normal .py file (not a Template)
-                    fp = open(new_trcback[-1][0])
+                    fp = open(new_trcback[-1][0], 'rb')
                     encoding = util.parse_encoding(fp)
                     fp.seek(0)
                     self.source = fp.read()

File mako/filters.py

 
 import re, cgi, urllib, htmlentitydefs, codecs
 from StringIO import StringIO
+from mako import util
 
 xml_escapes = {
     '&' : '&amp;',
     'str':'str',
     'n':'n'
 }
-    
+
+if util.py3k:
+    DEFAULT_ESCAPES.update({
+        'unicode':'str'
+    })
 

File mako/lexer.py

 """provides the Lexer class for parsing template strings into parse trees."""
 
 import re, codecs
-from mako import parsetree, exceptions
+from mako import parsetree, exceptions, util
 from mako.pygen import adjust_whitespace
 
 _regexp_cache = {}
         self.control_line = []
         self.disable_unicode = disable_unicode
         self.encoding = input_encoding
+        
+        if util.py3k and disable_unicode:
+            raise exceptions.UnsupportedError(
+                                    "Mako for Python 3 does not "
+                                    "support disabling Unicode")
+        
         if preprocessor is None:
             self.preprocessor = []
         elif not hasattr(preprocessor, '__iter__'):
                 'filename':self.filename}
     
     def match(self, regexp, flags=None):
-        """match the given regular expression string and flags to the current text position.
+        """compile the given regexp, cache the reg, and call match_reg()."""
         
-        if a match occurs, update the current text and line position."""
-        mp = self.match_position
         try:
             reg = _regexp_cache[(regexp, flags)]
         except KeyError:
             else:
                 reg = re.compile(regexp)
             _regexp_cache[(regexp, flags)] = reg
+        
+        return self.match_reg(reg)
+        
+    def match_reg(self, reg):
+        """match the given regular expression object to the current text position.
+        
+        if a match occurs, update the current text and line position.
+        
+        """
+
+        mp = self.match_position
 
         match = reg.match(self.text, self.match_position)
         if match:
                                 (node.keyword, self.control_line[-1].keyword),
                                 **self.exception_kwargs)
 
-    def parse(self):
-        for preproc in self.preprocessor:
-            self.text = preproc(self.text)
-            
-        if not isinstance(self.text, unicode) and self.text.startswith(codecs.BOM_UTF8):
-            self.text = self.text[len(codecs.BOM_UTF8):]
+    _coding_re = re.compile(r'#.*coding[:=]\s*([-\w.]+).*\r?\n')
+
+    def decode_raw_stream(self, text, decode_raw, known_encoding, filename):
+        """given string/unicode or bytes/string, determine encoding
+           from magic encoding comment, return body as unicode
+           or raw if decode_raw=False
+
+        """
+        if isinstance(text, unicode):
+            m = self._coding_re.match(text)
+            encoding = m and m.group(1) or known_encoding or 'ascii'
+            return encoding, text
+
+        if text.startswith(codecs.BOM_UTF8):
+            text = text[len(codecs.BOM_UTF8):]
             parsed_encoding = 'utf-8'
-            me = self.match_encoding()
-            if me is not None and me != 'utf-8':
+            m = self._coding_re.match(text.decode('utf-8', 'ignore'))
+            if m is not None and m.group(1) != 'utf-8':
                 raise exceptions.CompileException(
                                 "Found utf-8 BOM in file, with conflicting "
-                                "magic encoding comment of '%s'" % me, 
-                                self.text.decode('utf-8', 'ignore'), 
-                                0, 0, self.filename)
+                                "magic encoding comment of '%s'" % m.group(1), 
+                                text.decode('utf-8', 'ignore'), 
+                                0, 0, filename)
         else:
-            parsed_encoding = self.match_encoding()
-            
-        if parsed_encoding:
-            self.encoding = parsed_encoding
-            
-        if not self.disable_unicode and not isinstance(self.text, unicode):
-            if self.encoding:
-                try:
-                    self.text = self.text.decode(self.encoding)
-                except UnicodeDecodeError, e:
-                    raise exceptions.CompileException(
-                                    "Unicode decode operation of encoding '%s' failed" %
-                                    self.encoding, 
-                                    self.text.decode('utf-8', 'ignore'), 
-                                    0, 0, self.filename)
+            m = self._coding_re.match(text.decode('utf-8', 'ignore'))
+            if m:
+                parsed_encoding = m.group(1)
             else:
-                try:
-                    self.text = self.text.decode()
-                except UnicodeDecodeError, e:
-                    raise exceptions.CompileException(
-                                    "Could not read template using encoding of 'ascii'.  "
-                                    "Did you forget a magic encoding comment?",
-                                    self.text.decode('utf-8', 'ignore'), 0, 0, self.filename)
+                parsed_encoding = known_encoding or 'ascii'
+
+        if decode_raw:
+            try:
+                text = text.decode(parsed_encoding)
+            except UnicodeDecodeError, e:
+                raise exceptions.CompileException(
+                                "Unicode decode operation of encoding '%s' failed" %
+                                parsed_encoding, 
+                                text.decode('utf-8', 'ignore'), 
+                                0, 0, filename)
+
+        return parsed_encoding, text
+
+    def parse(self):
+        self.encoding, self.text = self.decode_raw_stream(self.text, 
+                                        not self.disable_unicode, 
+                                        self.encoding,
+                                        self.filename,)
 
+        for preproc in self.preprocessor:
+            self.text = preproc(self.text)
+        
+        # push the match marker past the 
+        # encoding comment.
+        self.match_reg(self._coding_re)
+        
         self.textlength = len(self.text)
             
         while (True):
                                             self.control_line[-1].pos, self.filename)
         return self.template
 
-    def match_encoding(self):
-        match = self.match(r'#.*coding[:=]\s*([-\w.]+).*\r?\n')
-        if match:
-            return match.group(1)
-        else:
-            return None
-            
     def match_tag_start(self):
         match = self.match(r'''
             \<%     # opening tag

File mako/parsetree.py

         
         attributes - raw dictionary of attribute key/value pairs
         
-        expressions - a set of identifiers that are legal attributes, which can also contain embedded expressions
+        expressions - a set of identifiers that are legal attributes, 
+            which can also contain embedded expressions
         
-        nonexpressions - a set of identifiers that are legal attributes, which cannot contain embedded expressions
+        nonexpressions - a set of identifiers that are legal attributes, 
+            which cannot contain embedded expressions
         
-        **kwargs - other arguments passed to the Node superclass (lineno, pos)
+        \**kwargs - other arguments passed to the Node superclass (lineno, pos)
         
         """
         super(Tag, self).__init__(**kwargs)
                     m = re.match(r'^\${(.+?)}$', x)
                     if m:
                         code = ast.PythonCode(m.group(1), **self.exception_kwargs)
-                        undeclared_identifiers = undeclared_identifiers.union(code.undeclared_identifiers)
+                        undeclared_identifiers = undeclared_identifiers.union(
+                                                        code.undeclared_identifiers
+                                                    )
                         expr.append("(%s)" % m.group(1))
                     else:
                         if x:
             elif key in nonexpressions:
                 if re.search(r'${.+?}', self.attributes[key]):
                     raise exceptions.CompileException(
-                            "Attibute '%s' in tag '%s' does not allow embedded expressions"  % (key, self.keyword), 
+                            "Attibute '%s' in tag '%s' does not allow embedded "
+                            "expressions"  % (key, self.keyword), 
                             **self.exception_kwargs)
                 self.parsed_attributes[key] = repr(self.attributes[key])
             else:
-                raise exceptions.CompileException("Invalid attribute for tag '%s': '%s'" %(self.keyword, key), **self.exception_kwargs)
+                raise exceptions.CompileException(
+                                    "Invalid attribute for tag '%s': '%s'" %
+                                    (self.keyword, key), 
+                                    **self.exception_kwargs)
         self.expression_undeclared_identifiers = undeclared_identifiers
 
     def declared_identifiers(self):
                                         self.keyword, 
                                         util.sorted_dict_repr(self.attributes),
                                         (self.lineno, self.pos), 
-                                        [repr(x) for x in self.nodes]
+                                        self.nodes
                                     )
         
 class IncludeTag(Tag):
     __keyword__ = 'include'
 
     def __init__(self, keyword, attributes, **kwargs):
-        super(IncludeTag, self).__init__(keyword, attributes, ('file', 'import', 'args'), (), ('file',), **kwargs)
-        self.page_args = ast.PythonCode("__DUMMY(%s)" % attributes.get('args', ''), **self.exception_kwargs)
+        super(IncludeTag, self).__init__(
+                                    keyword, 
+                                    attributes, 
+                                    ('file', 'import', 'args'), 
+                                    (), ('file',), **kwargs)
+        self.page_args = ast.PythonCode(
+                                "__DUMMY(%s)" % attributes.get('args', ''),
+                                 **self.exception_kwargs)
 
     def declared_identifiers(self):
         return []
     __keyword__ = 'namespace'
 
     def __init__(self, keyword, attributes, **kwargs):
-        super(NamespaceTag, self).__init__(keyword, attributes, (), ('name','inheritable','file','import','module'), (), **kwargs)
+        super(NamespaceTag, self).__init__(
+                                        keyword, attributes, 
+                                        (), 
+                                        ('name','inheritable',
+                                        'file','import','module'), 
+                                        (), **kwargs)
+                                        
         self.name = attributes.get('name', '__anon_%s' % hex(abs(id(self))))
         if not 'name' in attributes and not 'import' in attributes:
-            raise exceptions.CompileException("'name' and/or 'import' attributes are required for <%namespace>", **self.exception_kwargs)
+            raise exceptions.CompileException(
+                                "'name' and/or 'import' attributes are required "
+                                "for <%namespace>", 
+                                **self.exception_kwargs)
 
     def declared_identifiers(self):
         return []
     __keyword__ = 'text'
 
     def __init__(self, keyword, attributes, **kwargs):
-        super(TextTag, self).__init__(keyword, attributes, (), ('filter'), (), **kwargs)
-        self.filter_args = ast.ArgumentList(attributes.get('filter', ''), **self.exception_kwargs)
+        super(TextTag, self).__init__(
+                                    keyword, 
+                                    attributes, (), 
+                                    ('filter'), (), **kwargs)
+        self.filter_args = ast.ArgumentList(
+                                    attributes.get('filter', ''), 
+                                    **self.exception_kwargs)
         
 class DefTag(Tag):
     __keyword__ = 'def'
         super(DefTag, self).__init__(
                 keyword, 
                 attributes, 
-                ('buffered', 'cached', 'cache_key', 'cache_timeout', 'cache_type', 'cache_dir', 'cache_url'), 
+                ('buffered', 'cached', 'cache_key', 'cache_timeout', 
+                    'cache_type', 'cache_dir', 'cache_url'), 
                 ('name','filter', 'decorator'), 
                 ('name',), 
                 **kwargs)
         name = attributes['name']
         if re.match(r'^[\w_]+$',name):
-            raise exceptions.CompileException("Missing parenthesis in %def", **self.exception_kwargs)
+            raise exceptions.CompileException(
+                                "Missing parenthesis in %def", 
+                                **self.exception_kwargs)
         self.function_decl = ast.FunctionDecl("def " + name + ":pass", **self.exception_kwargs)
         self.name = self.function_decl.funcname
         self.decorator = attributes.get('decorator', '')
-        self.filter_args = ast.ArgumentList(attributes.get('filter', ''), **self.exception_kwargs)
+        self.filter_args = ast.ArgumentList(
+                                attributes.get('filter', ''), 
+                                **self.exception_kwargs)
 
     def declared_identifiers(self):
         return self.function_decl.argnames
         res = []
         for c in self.function_decl.defaults:
             res += list(ast.PythonCode(c, **self.exception_kwargs).undeclared_identifiers)
-        return res + list(self.filter_args.undeclared_identifiers.difference(set(filters.DEFAULT_ESCAPES.keys())))
+        return res + list(self.filter_args.\
+                            undeclared_identifiers.\
+                            difference(filters.DEFAULT_ESCAPES.keys())
+                        )
 
 class CallTag(Tag):
     __keyword__ = 'call'
 
     def __init__(self, keyword, attributes, **kwargs):
-        super(CallTag, self).__init__(keyword, attributes, ('args'), ('expr',), ('expr',), **kwargs)
+        super(CallTag, self).__init__(keyword, attributes, 
+                                    ('args'), ('expr',), ('expr',), **kwargs)
         self.expression = attributes['expr']
         self.code = ast.PythonCode(self.expression, **self.exception_kwargs)
         self.body_decl = ast.FunctionArgs(attributes.get('args', ''), **self.exception_kwargs)
                     (), 
                     (), 
                     **kwargs)
-        self.expression = "%s.%s(%s)" % (namespace, defname, ",".join(["%s=%s" % (k, v) for k, v in self.parsed_attributes.iteritems() if k != 'args']))
+                    
+        self.expression = "%s.%s(%s)" % (
+                                namespace, 
+                                defname, 
+                                ",".join(["%s=%s" % (k, v) for k, v in
+                                            self.parsed_attributes.iteritems() 
+                                            if k != 'args'])
+                            )
         self.code = ast.PythonCode(self.expression, **self.exception_kwargs)
-        self.body_decl = ast.FunctionArgs(attributes.get('args', ''), **self.exception_kwargs)
+        self.body_decl = ast.FunctionArgs(
+                                    attributes.get('args', ''), 
+                                    **self.exception_kwargs)
 
     def declared_identifiers(self):
         return self.code.declared_identifiers.union(self.body_decl.argnames)
     __keyword__ = 'inherit'
 
     def __init__(self, keyword, attributes, **kwargs):
-        super(InheritTag, self).__init__(keyword, attributes, ('file',), (), ('file',), **kwargs)
+        super(InheritTag, self).__init__(
+                                keyword, attributes, 
+                                ('file',), (), ('file',), **kwargs)
 
 class PageTag(Tag):
     __keyword__ = 'page'
         super(PageTag, self).__init__(
                 keyword, 
                 attributes, 
-                ('cached', 'cache_key', 'cache_timeout', 'cache_type', 'cache_dir', 'cache_url', 'args', 'expression_filter'), 
+                ('cached', 'cache_key', 'cache_timeout', 
+                'cache_type', 'cache_dir', 'cache_url', 
+                'args', 'expression_filter'), 
                 (), 
                 (), 
                 **kwargs)
         self.body_decl = ast.FunctionArgs(attributes.get('args', ''), **self.exception_kwargs)
-        self.filter_args = ast.ArgumentList(attributes.get('expression_filter', ''), **self.exception_kwargs)
+        self.filter_args = ast.ArgumentList(
+                                attributes.get('expression_filter', ''),
+                                **self.exception_kwargs)
 
     def declared_identifiers(self):
         return self.body_decl.argnames

File mako/pygen.py

 
 import re, string
 from StringIO import StringIO
+from mako import exceptions
 
 class PythonPrinter(object):
     def __init__(self, stream):
                 # probably put extra closures - the resulting
                 # module wont compile.  
                 if len(self.indent_detail) == 0:  
-                    raise "Too many whitespace closures"
+                    raise exceptions.SyntaxException("Too many whitespace closures")
                 self.indent_detail.pop()
         
         if line is None:
             if self._in_multi_line(entry):
                 self.stream.write(entry + "\n")
             else:
-                entry = string.expandtabs(entry)
+                entry = entry.expandtabs()
                 if stripspace is None and re.search(r"^[ \t]*[^# \t]", entry):
                     stripspace = re.match(r"^([ \t]*)", entry).group(1)
                 self.stream.write(self._indent_line(entry, stripspace) + "\n")
         if in_multi_line(line):
             lines.append(line)
         else:
-            line = string.expandtabs(line)
+            line = line.expandtabs()
             if stripspace is None and re.search(r"^[ \t]*[^# \t]", line):
                 stripspace = re.match(r"^([ \t]*)", line).group(1)
             lines.append(_indent_line(line, stripspace))

File mako/pyparser.py

 
 from StringIO import StringIO
 from mako import exceptions, util
+import operator
+
+if util.py3k:
+    # words that cannot be assigned to (notably 
+    # smaller than the total keys in __builtins__)
+    reserved = set(['True', 'False', 'None', 'print'])
+
+    # the "id" attribute on a function node
+    arg_id = operator.attrgetter('arg')
+else:
+    # words that cannot be assigned to (notably 
+    # smaller than the total keys in __builtins__)
+    reserved = set(['True', 'False', 'None'])
+    
+    # the "id" attribute on a function node
+    arg_id = operator.attrgetter('id')
 
-# words that cannot be assigned to (notably smaller than the total keys in __builtins__)
-reserved = set(['True', 'False', 'None'])
 
 try:
     import _ast
             for n in node.targets:
                 self.visit(n)
             self.in_assign_targets = in_a
+        
+        if util.py3k:
+            # ExceptHandler is in Python 2, but this
+            # block only works in Python 3 (and is required there)
+            def visit_ExceptHandler(self, node):
+                if node.name is not None:
+                    self._add_declared(node.name)
+                if node.type is not None:
+                    self.listener.undeclared_identifiers.add(node.type.id)
+                for statement in node.body:
+                    self.visit(statement)
+                
         def visit_FunctionDef(self, node):
             self._add_declared(node.name)
             # push function state onto stack.  dont log any
             saved = {}
             inf = self.in_function
             self.in_function = True
+            
             for arg in node.args.args:
-                if arg.id in self.local_ident_stack:
-                    saved[arg.id] = True
+                if arg_id(arg) in self.local_ident_stack:
+                    saved[arg_id(arg)] = True
                 else:
-                    self.local_ident_stack[arg.id] = True
+                    self.local_ident_stack[arg_id(arg)] = True
             for n in node.body:
                 self.visit(n)
             self.in_function = inf
             for arg in node.args.args:
-                if arg.id not in saved:
-                    del self.local_ident_stack[arg.id]
+                if arg_id(arg) not in saved:
+                    del self.local_ident_stack[arg_id(arg)]
+                    
         def visit_For(self, node):
             # flip around visit
             self.visit(node.iter)
         def visit_Name(self, node):
             if isinstance(node.ctx, _ast.Store):
                 self._add_declared(node.id)
-            if node.id not in reserved and node.id not in self.listener.declared_identifiers and node.id not in self.local_ident_stack:
+            if node.id not in reserved and \
+                        node.id not in self.listener.declared_identifiers and \
+                        node.id not in self.local_ident_stack:
                 self.listener.undeclared_identifiers.add(node.id)
         def visit_Import(self, node):
             for name in node.names:
         def __init__(self, listener, **exception_kwargs):
             self.listener = listener
             self.exception_kwargs = exception_kwargs
+            
         def visit_FunctionDef(self, node):
             self.listener.funcname = node.name
-            argnames = [arg.id for arg in node.args.args]
+            argnames = [arg_id(arg) for arg in node.args.args]
             if node.args.vararg:
                 argnames.append(node.args.vararg)
             if node.args.kwarg:

File mako/runtime.py

         self._data.update(data)
         self._kwargs = data.copy()
         self._with_template = None
+        self._outputting_as_unicode = None
         self.namespaces = {}
         
         # "capture" function which proxies to the generic "capture" function
         # "caller" stack used by def calls with content
         self.caller_stack = self._data['caller'] = CallerStack()
         
-    lookup = property(lambda self:self._with_template.lookup)
-    kwargs = property(lambda self:self._kwargs.copy())
+    @property
+    def lookup(self):
+        return self._with_template.lookup
+        
+    @property
+    def kwargs(self):
+        return self._kwargs.copy()
     
     def push_caller(self, caller):
         self.caller_stack.append(caller)
         c._orig = self._orig
         c._kwargs = self._kwargs
         c._with_template = self._with_template
+        c._outputting_as_unicode = self._outputting_as_unicode
         c.namespaces = self.namespaces
         c.caller_stack = self.caller_stack
         return c
     else:
         buf = util.StringIO()
     context = Context(buf, **data)
+    context._outputting_as_unicode = as_unicode
     context._with_template = template
     _render_context(template, callable_, context, *args, **_kwargs_for_callable(callable_, data))
     return context._pop_buffer().getvalue()
         try:
             callable_(context, *args, **kwargs)
         except Exception, e:
-            error = e
+            _render_error(template, context, e)
         except:                
             e = sys.exc_info()[0]
-            error = e
-        if error:
-            if template.error_handler:
-                result = template.error_handler(context, error)
-                if not result:
-                    raise error
-            else:
-                error_template = exceptions.html_error_template()
-                context._buffer_stack[:] = [util.FastEncodingBuffer(error_template.output_encoding, error_template.encoding_errors)]
-                context._with_template = error_template
-                error_template.render_context(context, error=error)
+            _render_error(template, context, e)
     else:
         callable_(context, *args, **kwargs)
+
+
+def _render_error(template, context, error):
+    if template.error_handler:
+        result = template.error_handler(context, error)
+        if not result:
+            raise error
+    else:
+        error_template = exceptions.html_error_template()
+        if context._outputting_as_unicode:
+            context._buffer_stack[:] = [util.FastEncodingBuffer(unicode=True)]
+        else:
+            context._buffer_stack[:] = [util.FastEncodingBuffer(error_template.output_encoding, error_template.encoding_errors)]
+        context._with_template = error_template
+        error_template.render_context(context, error=error)

File mako/template.py

         self.output_encoding = output_encoding
         self.encoding_errors = encoding_errors
         self.disable_unicode = disable_unicode
+
+        if util.py3k and disable_unicode:
+            raise exceptions.UnsupportedError(
+                                    "Mako for Python 3 does not "
+                                    "support disabling Unicode")
+        
         if default_filters is None:
-            if self.disable_unicode:
+            if util.py3k or self.disable_unicode:
                 self.default_filters = ['str']
             else:
                 self.default_filters = ['unicode']
                             os.stat(path)[stat.ST_MTIME] < filemtime:
                     _compile_module_file(
                                 self, 
-                                file(filename).read(), 
+                                open(filename, 'rb').read(), 
                                 filename, 
                                 path)
-                module = imp.load_source(self.module_id, path, file(path))
+                module = imp.load_source(self.module_id, path, open(path, 'rb'))
                 del sys.modules[self.module_id]
                 if module._magic_number != codegen.MAGIC_NUMBER:
                     _compile_module_file(
                                 self, 
-                                file(filename).read(), 
+                                open(filename, 'rb').read(), 
                                 filename, 
                                 path)
-                    module = imp.load_source(self.module_id, path, file(path))
+                    module = imp.load_source(self.module_id, path, open(path, 'rb'))
                     del sys.modules[self.module_id]
                 ModuleInfo(module, path, self, filename, None, None)
             else:
                 # in memory
                 (code, module) = _compile_text(
                                     self, 
-                                    file(filename).read(), 
+                                    open(filename, 'rb').read(), 
                                     filename)
                 self._source = None
                 self._code = code
         if self.module_source is not None:
             return self.module_source
         else:
-            return file(self.module_filename).read()
+            return open(self.module_filename).read()
     
     @property
     def source(self):
                 return self.template_source
         else:
             if self.module._source_encoding:
-                return file(self.template_filename).read().\
+                return open(self.template_filename, 'rb').read().\
                                 decode(self.module._source_encoding)
             else:
-                return file(self.template_filename).read()
+                return open(self.template_filename).read()
         
 def _compile_text(template, text, filename):
     identifier = template.module_id
                             generate_magic_comment=template.disable_unicode)
 
     cid = identifier
-    if isinstance(cid, unicode):
+    if not util.py3k and isinstance(cid, unicode):
         cid = cid.encode()
     module = types.ModuleType(cid)
     code = compile(source, cid, 'exec')

File mako/util.py

 
 import sys
 
-try:
-    from cStringIO import StringIO
-except:
-    from StringIO import StringIO
 
 py3k = getattr(sys, 'py3kwarning', False) or sys.version_info >= (3, 0)
 jython = sys.platform.startswith('java')
 win32 = sys.platform.startswith('win')
 
-import codecs, re, weakref, os, time
+if py3k:
+    from io import StringIO
+else:
+    try:
+        from cStringIO import StringIO
+    except:
+        from StringIO import StringIO
+
+import codecs, re, weakref, os, time, operator
 
 try:
     import threading
     else:
         return x
 
+
+    
+
+
 class SetLikeDict(dict):
     """a dictionary that has some setlike methods on it"""
     def union(self, other):
         self.unicode = unicode
         self.errors = errors
         self.write = self.data.append
+    
+    def truncate(self):
+        self.data =[]
         
     def getvalue(self):
         if self.encoding:
     
     def _manage_size(self):
         while len(self) > self.capacity + self.capacity * self.threshold:
-            bytime = dict.values(self)
-            bytime.sort(lambda a, b: cmp(b.timestamp, a.timestamp))
+            bytime = sorted(dict.values(self), 
+                            key=operator.attrgetter('timestamp'), reverse=True)
             for item in bytime[self.capacity:]:
                 try:
                     del self[item.key]
     re.VERBOSE)
 
 def parse_encoding(fp):
-    """Deduce the encoding of a source file from magic comment.
+    """Deduce the encoding of a Python source file (binary mode) from magic comment.
 
     It does this in the same way as the `Python interpreter`__
 
     .. __: http://docs.python.org/ref/encodings.html
 
-    The ``fp`` argument should be a seekable file object.
+    The ``fp`` argument should be a seekable file object in binary mode.
     """
     pos = fp.tell()
     fp.seek(0)
         if has_bom:
             line1 = line1[len(codecs.BOM_UTF8):]
 
-        m = _PYTHON_MAGIC_COMMENT_re.match(line1)
+        m = _PYTHON_MAGIC_COMMENT_re.match(line1.decode('ascii', 'ignore'))
         if not m:
             try:
                 import parser
-                parser.suite(line1)
+                parser.suite(line1.decode('ascii', 'ignore'))
             except (ImportError, SyntaxError):
                 # Either it's a real syntax error, in which case the source
                 # is not valid python source, or line2 is a continuation of
                 pass
             else:
                 line2 = fp.readline()
-                m = _PYTHON_MAGIC_COMMENT_re.match(line2)
+                m = _PYTHON_MAGIC_COMMENT_re.match(line2.decode('ascii', 'ignore'))
 
         if has_bom:
             if m:
 from setuptools import setup, find_packages
 import os
 import re
+import sys
+
+extra = {}
+if sys.version_info >= (3, 0):
+    extra.update(
+        use_2to3=True,
+    )
 
 v = file(os.path.join(os.path.dirname(__file__), 'mako', '__init__.py'))
 VERSION = re.compile(r".*__version__ = '(.*?)'", re.S).match(v.read()).group(1)
 
 """,
       classifiers=[
-      "Development Status :: 5 - Production/Stable",
+      'Development Status :: 5 - Production/Stable',
       'Environment :: Web Environment',
       'Intended Audience :: Developers',
       'Programming Language :: Python',
+      'Programming Language :: Python :: 3',
       'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
       ],
       keywords='wsgi myghty mako',

File test/__init__.py

         return Template(uri=filename, filename=filepath, module_directory=module_base, **kw)
     
     def _file_path(self, filename):
+        name, ext = os.path.splitext(filename)
+        
+        if py3k:
+            py3k_path = os.path.join(template_base, name + "_py3k" + ext)
+            if os.path.exists(py3k_path):
+                return py3k_path
+        
         return os.path.join(template_base, filename)
         
     def _do_file_test(self, filename, expected, filters=None, 
                         unicode_=True, template_args=None, **kw):
         t1 = self._file_template(filename, **kw)
-        self._do_test(t1, expected, filters=filters, unicode_=unicode_, template_args=template_args)
+        self._do_test(t1, expected, filters=filters, 
+                        unicode_=unicode_, template_args=template_args)
     
     def _do_memory_test(self, source, expected, filters=None, 
                         unicode_=True, template_args=None, **kw):
         t1 = Template(text=source, **kw)
-        self._do_test(t1, expected, filters=filters, unicode_=unicode_, template_args=template_args)
+        self._do_test(t1, expected, filters=filters, 
+                        unicode_=unicode_, template_args=template_args)
     
     def _do_test(self, template, expected, filters=None, template_args=None, unicode_=True):
         if template_args is None:

File test/templates/chs_unicode_py3k.html

+## -*- encoding:utf8 -*-
+<%
+ msg = '新中国的主席'
+%>
+
+<%def name="welcome(who, place='北京')">
+Welcome ${who} to ${place}.
+</%def>
+
+${name} 是 ${msg}<br/>
+${welcome('你')}

File test/templates/read_unicode_py3k.html

+<% 
+try:
+    file_content = open(path)
+except:
+    raise "Should never execute here"
+doc_content = ''.join(file_content.readlines())
+file_content.close()
+%>
+
+${bytes(doc_content, encoding='utf-8')}

File test/templates/unicode_arguments_py3k.html

+# coding: utf-8
+
+<%def name="my_def(x)">
+    x is: ${x}
+</%def>
+
+${my_def('drôle de petit voix m’a réveillé')}
+<%self:my_def x='drôle de petit voix m’a réveillé'/>
+<%self:my_def x="${'drôle de petit voix m’a réveillé'}"/>
+<%call expr="my_def('drôle de petit voix m’a réveillé')"/>

File test/templates/unicode_code_py3k.html

+## -*- coding: utf-8 -*-
+<%
+    x = "drôle de petit voix m’a réveillé."
+%>
+% if x=="drôle de petit voix m’a réveillé.":
+    hi, ${x}
+% endif

File test/templates/unicode_expr_py3k.html

+## -*- coding: utf-8 -*-
+${"Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petit voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »"}

File test/test_ast.py

 import unittest
 
 from mako import ast, exceptions, pyparser, util
+from test import eq_
 
 exception_kwargs = {'source':'', 'lineno':0, 'pos':0, 'filename':''}
 
 class AstParseTest(unittest.TestCase):
-    def setUp(self):
-        pass
-    def tearDown(self):
-        pass
+
     def test_locate_identifiers(self):
         """test the location of identifiers in a python code string"""
         code = """
 for lar in (1,2,3):
     gh = 5
     x = 12
-print "hello world, ", a, b
-print "Another expr", c
+("hello world, ", a, b)
+("Another expr", c)
 """
         parsed = ast.PythonCode(code, **exception_kwargs)
         assert parsed.declared_identifiers == set(['a','b','c', 'g', 'h', 'i', 'u', 'k', 'j', 'gh', 'lar', 'x'])
         code = """
 x = x + 5
 for y in range(1, y):
-    print "hi"
+    ("hi",)
 [z for z in range(1, z)]
 (q for q in range (1, q))
 """
         assert parsed.undeclared_identifiers == set(['x', 'y', 'z', 'q', 'range'])
     
     def test_locate_identifiers_4(self):
-        code = """
+        if util.py3k:
+            code = """
+x = 5
+(y, )
+def mydef(mydefarg):
+    print("mda is", mydefarg)
+"""    
+        else:
+            code = """
 x = 5
-print y
+(y, )
 def mydef(mydefarg):
     print "mda is", mydefarg
 """    
         assert parsed.declared_identifiers == set(['mydef', 'x'])
     
     def test_locate_identifiers_5(self):
-        code = """
+        if util.py3k:
+            code = """
+try:
+    print(x)
+except:
+    print(y)
+"""
+        else:
+            
+            code = """
 try:
     print x
 except:
 """
         parsed = ast.PythonCode(code, **exception_kwargs)
         assert parsed.undeclared_identifiers == set(['bar'])
-    
-        code = """
+        
+        if util.py3k:
+            code = """
+def lala(x, y):
+    return x, y, z
+print(x)
+"""
+        else:
+            code = """
 def lala(x, y):
     return x, y, z
 print x
         parsed = ast.PythonCode(code, **exception_kwargs)
         assert parsed.undeclared_identifiers == set(['z', 'x'])
         assert parsed.declared_identifiers == set(['lala'])
-    
-        code = """
+        
+        if util.py3k:
+            code = """
+def lala(x, y):
+    def hoho():
+        def bar():
+            z = 7
+print(z)
+"""
+        else:
+            code = """
 def lala(x, y):
     def hoho():
         def bar():
 from foo import *
 import x as bar
 """
-        try:
-            parsed = ast.PythonCode(code, **exception_kwargs)
-            assert False
-        except exceptions.CompileException, e:
-            assert str(e).startswith("'import *' is not supported")
+        self.assertRaises(exceptions.CompileException, ast.PythonCode, code, **exception_kwargs)
             
     def test_python_fragment(self):
         parsed = ast.PythonFragment("for x in foo:", **exception_kwargs)
         
         parsed = ast.PythonFragment("try:", **exception_kwargs)
 
-        parsed = ast.PythonFragment("except MyException, e:", **exception_kwargs)
-        assert parsed.declared_identifiers == set(['e'])
-        assert parsed.undeclared_identifiers == set(['MyException'])
+        if util.py3k:
+            parsed = ast.PythonFragment("except MyException as e:", **exception_kwargs)
+        else:
+            parsed = ast.PythonFragment("except MyException, e:", **exception_kwargs)
+        eq_(parsed.declared_identifiers, set(['e']))
+        eq_(parsed.undeclared_identifiers, set(['MyException']))
     
     def test_argument_list(self):
         parsed = ast.ArgumentList("3, 5, 'hi', x+5, context.get('lala')", **exception_kwargs)
         code = "str((x+7*y) / foo.bar(5,6)) + lala('ho')"
         astnode = pyparser.parse(code)
         newcode = pyparser.ExpressionGenerator(astnode).value()
-        #print "newcode:" + newcode
-        #print "result:" + eval(code, local_dict)
         assert (eval(code, local_dict) == eval(newcode, local_dict))
         
         a = ["one", "two", "three"]
         code = "a[2] + hoho['somevalue'] + repr(g[3:5]) + repr(g[3:]) + repr(g[:5])"
         astnode = pyparser.parse(code)
         newcode = pyparser.ExpressionGenerator(astnode).value()
-        #print newcode
-        #print "result:", eval(code, local_dict)
         assert(eval(code, local_dict) == eval(newcode, local_dict))
         
         local_dict={'f':lambda :9, 'x':7}
             local_dict={}
             astnode = pyparser.parse(code)
             newcode = pyparser.ExpressionGenerator(astnode).value()
-            #print code, newcode
             assert(eval(code, local_dict)) == eval(newcode, local_dict), "%s != %s" % (code, newcode)
 
     

File test/test_def.py

 from mako.template import Template
 from mako import lookup
-import unittest
+from test import TemplateTest
 from util import flatten_result, result_lines
 
-class DefTest(unittest.TestCase):
+class DefTest(TemplateTest):
     def test_def_noargs(self):
         template = Template("""
         
 
     def test_toplevel(self):
         """test calling a def from the top level"""
+
         template = Template("""
         
             this is the body
                 this is b, ${x} ${y}
             </%def>
                 
-        """, output_encoding='utf-8')
-        assert flatten_result(template.get_def("a").render()) == "this is a"
-        assert flatten_result(template.get_def("b").render(x=10, y=15)) == "this is b, 10 15"
-        assert flatten_result(template.get_def("body").render()) == "this is the body"
+        """)
+        
+        self._do_test(template.get_def("a"), "this is a", filters=flatten_result)
+        self._do_test(template.get_def("b"), "this is b, 10 15", 
+                                                            template_args={'x':10, 'y':15}, 
+                                                            filters=flatten_result)
+        self._do_test(template.get_def("body"), "this is the body", filters=flatten_result)
+        
         
-class ScopeTest(unittest.TestCase):
+class ScopeTest(TemplateTest):
     """test scoping rules.  The key is, enclosing scope always takes precedence over contextual scope."""
+    
     def test_scope_one(self):
-        t = Template("""
+        self._do_memory_test("""
         <%def name="a()">
             this is a, and y is ${y}
         </%def>
 
         ${a()}
 
-""")
-        assert flatten_result(t.render(y=None)) == "this is a, and y is None this is a, and y is 7"
+""",
+            "this is a, and y is None this is a, and y is 7",
+            filters=flatten_result,
+            template_args={'y':None}
+        )
 
     def test_scope_two(self):
         t = Template("""
             "this is a, x is 15"
         ]
 
-class NestedDefTest(unittest.TestCase):
+class NestedDefTest(TemplateTest):
     def test_nested_def(self):
         t = Template("""
 
 """)
         assert flatten_result(t.render(x=5)) == "b. c. x is 10. a: x is 5 x is 5"
             
-class ExceptionTest(unittest.TestCase):
+class ExceptionTest(TemplateTest):
     def test_raise(self):
         template = Template("""
             <%

File test/test_exceptions.py

 import sys
 import unittest
 
-from mako import exceptions
+from mako import exceptions, util
 from mako.template import Template
 from mako.lookup import TemplateLookup
 from util import result_lines
 """
         try:
             template = Template(code)
-            template.render()
+            template.render_unicode()
         except exceptions.CompileException, ce:
-            html_error = exceptions.html_error_template().render()
+            html_error = exceptions.html_error_template().render_unicode()
             assert ("CompileException: Fragment 'i = 0' is not a partial "
                     "control statement") in html_error
             assert '<style>' in html_error
             assert html_error_stripped.startswith('<html>')
             assert html_error_stripped.endswith('</html>')
 
-            not_full = exceptions.html_error_template().render(full=False)
+            not_full = exceptions.html_error_template().render_unicode(full=False)
             assert '<html>' not in not_full
             assert '</html>' not in not_full
             assert '<style>' in not_full
             assert '</style>' in not_full
 
-            no_css = exceptions.html_error_template().render(css=False)
+            no_css = exceptions.html_error_template().render_unicode(css=False)
             assert '<style>' not in no_css
             assert '</style>' not in no_css
         else:
 
     def test_utf8_html_error_template(self):
         """test the html_error_template with a Template containing utf8 chars"""
-        code = """# -*- coding: utf-8 -*-
+        
+        if util.py3k:
+            code = """# -*- coding: utf-8 -*-
+% if 2 == 2: /an error
+${'привет'}
+% endif
+"""
+        else:
+            code = """# -*- coding: utf-8 -*-
 % if 2 == 2: /an error
 ${u'привет'}
 % endif
 """
         try:
             template = Template(code)
-            template.render()
+            template.render_unicode()
         except exceptions.CompileException, ce:
             html_error = exceptions.html_error_template().render()
             assert ("CompileException: Fragment 'if 2 == 2: /an "
                     "error' is not a partial control "
-                    "statement at line: 2 char: 1") in html_error
-            assert u"3 ${u'привет'}".encode(sys.getdefaultencoding(),
+                    "statement at line: 2 char: 1") in html_error.decode('utf-8')
+                    
+            if util.py3k:
+                assert u"3 ${'привет'}".encode(sys.getdefaultencoding(),
+                                            'htmlentityreplace') in html_error
+            else:
+                assert u"3 ${u'привет'}".encode(sys.getdefaultencoding(),
                                             'htmlentityreplace') in html_error
         else:
             assert False, ("This function should trigger a CompileException, "
             raise RuntimeError('test')
         except:
             html_error = exceptions.html_error_template().render()
-            assert 'RuntimeError: test' in html_error
-            assert "foo = u'&#x65E5;&#x672C;'" in html_error
+            if util.py3k:
+                assert 'RuntimeError: test' in html_error.decode('utf-8')
+                assert u"foo = '日本'" in html_error.decode('utf-8')
+            else:
+                assert 'RuntimeError: test' in html_error
+                assert "foo = u'&#x65E5;&#x672C;'" in html_error
 
 
     def test_py_unicode_error_html_error_template(self):
             raise RuntimeError(u'日本')
         except:
             html_error = exceptions.html_error_template().render()
-            assert 'RuntimeError: &#x65E5;&#x672C;' in html_error
-            assert "RuntimeError(u'&#x65E5;&#x672C;')" in html_error
+            assert u"RuntimeError: 日本".encode('ascii', 'ignore') in html_error
 
     def test_format_exceptions(self):
         l = TemplateLookup(format_exceptions=True)
         ${self.body()}
         """)
 
-        assert '<div class="sourceline">${foobar}</div>' in result_lines(l.get_template("foo.html").render())
+        assert '<div class="sourceline">${foobar}</div>' in result_lines(l.get_template("foo.html").render_unicode())
     
     def test_utf8_format_exceptions(self):
         """test that htmlentityreplace formatting is applied to exceptions reported with format_exceptions=True"""
         
         l = TemplateLookup(format_exceptions=True)
+        if util.py3k:
+            l.put_string("foo.html", """# -*- coding: utf-8 -*-\n${'привет' + foobar}""")
+        else:
+            l.put_string("foo.html", """# -*- coding: utf-8 -*-\n${u'привет' + foobar}""")
 
-        l.put_string("foo.html", """# -*- coding: utf-8 -*-
-${u'привет' + foobar}
-""")
-
-        assert '''<div class="highlight">2 ${u\'&#x43F;&#x440;&#x438;&#x432;&#x435;&#x442;\' + foobar}</div>''' in result_lines(l.get_template("foo.html").render())
+        if util.py3k:
+            assert u'<div class="sourceline">${\'привет\' + foobar}</div>'\
+                in result_lines(l.get_template("foo.html").render().decode('utf-8'))
+        else:
+            assert '<div class="highlight">2 ${u\'&#x43F;&#x440;&#x438;&#x432;&#x435;&#x442;\' + foobar}</div>' \
+                in result_lines(l.get_template("foo.html").render().decode('utf-8'))
         

File test/test_inheritance.py

 from mako.template import Template
-from mako import lookup
+from mako import lookup, util
 import unittest
 from util import flatten_result, result_lines
 
             this is the base.
 
             <%
-            sorted = pageargs.items()
-            sorted.sort()
+            sorted_ = pageargs.items()
+            sorted_ = sorted(sorted_)
             %>
-            pageargs: (type: ${type(pageargs)}) ${sorted}
+            pageargs: (type: ${type(pageargs)}) ${sorted_}
             <%def name="foo()">
                 ${next.body(**context.kwargs)}
             </%def>
             <%page args="x, y, z=7"/>
             print ${x}, ${y}, ${z}
         """)
-        assert result_lines(collection.get_template('index').render(x=5,y=10)) == [
-            "this is the base.",
-            "pageargs: (type: <type 'dict'>) [('x', 5), ('y', 10)]",
-            "print 5, 10, 7"
-        ]
+        
+        if util.py3k:
+            assert result_lines(collection.get_template('index').render_unicode(x=5,y=10)) == [
+                "this is the base.",
+                "pageargs: (type: <class 'dict'>) [('x', 5), ('y', 10)]",
+                "print 5, 10, 7"
+            ]
+        else:
+            assert result_lines(collection.get_template('index').render_unicode(x=5,y=10)) == [
+                "this is the base.",
+                "pageargs: (type: <type 'dict'>) [('x', 5), ('y', 10)]",
+                "print 5, 10, 7"
+            ]
+        
     def test_pageargs_2(self):
         collection = lookup.TemplateLookup()
         collection.put_string("base", """

File test/test_lexer.py

 import unittest
 
 from mako.lexer import Lexer
-from mako import exceptions
+from mako import exceptions, util
 from util import flatten_result, result_lines
 from mako.template import Template
 import re
-from test import TemplateTest, template_base, skip_if
-
-
+from test import TemplateTest, template_base, skip_if, eq_
+
+# create fake parsetree classes which are constructed
+# exactly as the repr() of a real parsetree object.
+# this allows us to use a Python construct as the source
+# of a comparable repr(), which is also hit by the 2to3 tool.
+
+def repr_arg(x):
+    if isinstance(x, dict):
+        return util.sorted_dict_repr(x)
+    else:
+        return repr(x)
+