Henning Schröder avatar Henning Schröder committed 4603218

added cythonized tokenize module

Comments (0)

Files changed (3)

ctokenize/setup.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from distutils.core import setup, Extension
+from Cython.Distutils import build_ext
+
+if __name__ == "__main__":
+    setup(name = "tokenize",
+      version = "2.7",
+      description = "Python module for tokenizing source",
+      cmdclass = {'build_ext': build_ext},
+      ext_modules = [ 
+          Extension("tokenize", ["tokenize.pyx"])
+      ]
+    )

ctokenize/tokenize.c

+/* Generated by Cython 0.14.1 on Tue Dec  6 20:23:19 2011 */
+
+#define PY_SSIZE_T_CLEAN
+#include "Python.h"
+#ifndef Py_PYTHON_H
+    #error Python headers needed to compile C extensions, please install development version of Python.
+#else
+
+#include <stddef.h> /* For offsetof */
+#ifndef offsetof
+#define offsetof(type, member) ( (size_t) & ((type*)0) -> member )
+#endif
+
+#if !defined(WIN32) && !defined(MS_WINDOWS)
+  #ifndef __stdcall
+    #define __stdcall
+  #endif
+  #ifndef __cdecl
+    #define __cdecl
+  #endif
+  #ifndef __fastcall
+    #define __fastcall
+  #endif
+#endif
+
+#ifndef DL_IMPORT
+  #define DL_IMPORT(t) t
+#endif
+#ifndef DL_EXPORT
+  #define DL_EXPORT(t) t
+#endif
+
+#ifndef PY_LONG_LONG
+  #define PY_LONG_LONG LONG_LONG
+#endif
+
+#if PY_VERSION_HEX < 0x02040000
+  #define METH_COEXIST 0
+  #define PyDict_CheckExact(op) (Py_TYPE(op) == &PyDict_Type)
+  #define PyDict_Contains(d,o)   PySequence_Contains(d,o)
+#endif
+
+#if PY_VERSION_HEX < 0x02050000
+  typedef int Py_ssize_t;
+  #define PY_SSIZE_T_MAX INT_MAX
+  #define PY_SSIZE_T_MIN INT_MIN
+  #define PY_FORMAT_SIZE_T ""
+  #define PyInt_FromSsize_t(z) PyInt_FromLong(z)
+  #define PyInt_AsSsize_t(o)   PyInt_AsLong(o)
+  #define PyNumber_Index(o)    PyNumber_Int(o)
+  #define PyIndex_Check(o)     PyNumber_Check(o)
+  #define PyErr_WarnEx(category, message, stacklevel) PyErr_Warn(category, message)
+#endif
+
+#if PY_VERSION_HEX < 0x02060000
+  #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt)
+  #define Py_TYPE(ob)   (((PyObject*)(ob))->ob_type)
+  #define Py_SIZE(ob)   (((PyVarObject*)(ob))->ob_size)
+  #define PyVarObject_HEAD_INIT(type, size) \
+          PyObject_HEAD_INIT(type) size,
+  #define PyType_Modified(t)
+
+  typedef struct {
+     void *buf;
+     PyObject *obj;
+     Py_ssize_t len;
+     Py_ssize_t itemsize;
+     int readonly;
+     int ndim;
+     char *format;
+     Py_ssize_t *shape;
+     Py_ssize_t *strides;
+     Py_ssize_t *suboffsets;
+     void *internal;
+  } Py_buffer;
+
+  #define PyBUF_SIMPLE 0
+  #define PyBUF_WRITABLE 0x0001
+  #define PyBUF_FORMAT 0x0004
+  #define PyBUF_ND 0x0008
+  #define PyBUF_STRIDES (0x0010 | PyBUF_ND)
+  #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES)
+  #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES)
+  #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES)
+  #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES)
+
+#endif
+
+#if PY_MAJOR_VERSION < 3
+  #define __Pyx_BUILTIN_MODULE_NAME "__builtin__"
+#else
+  #define __Pyx_BUILTIN_MODULE_NAME "builtins"
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+  #define Py_TPFLAGS_CHECKTYPES 0
+  #define Py_TPFLAGS_HAVE_INDEX 0
+#endif
+
+#if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3)
+  #define Py_TPFLAGS_HAVE_NEWBUFFER 0
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+  #define PyBaseString_Type            PyUnicode_Type
+  #define PyStringObject               PyUnicodeObject
+  #define PyString_Type                PyUnicode_Type
+  #define PyString_Check               PyUnicode_Check
+  #define PyString_CheckExact          PyUnicode_CheckExact
+#endif
+
+#if PY_VERSION_HEX < 0x02060000
+  #define PyBytesObject                PyStringObject
+  #define PyBytes_Type                 PyString_Type
+  #define PyBytes_Check                PyString_Check
+  #define PyBytes_CheckExact           PyString_CheckExact
+  #define PyBytes_FromString           PyString_FromString
+  #define PyBytes_FromStringAndSize    PyString_FromStringAndSize
+  #define PyBytes_FromFormat           PyString_FromFormat
+  #define PyBytes_DecodeEscape         PyString_DecodeEscape
+  #define PyBytes_AsString             PyString_AsString
+  #define PyBytes_AsStringAndSize      PyString_AsStringAndSize
+  #define PyBytes_Size                 PyString_Size
+  #define PyBytes_AS_STRING            PyString_AS_STRING
+  #define PyBytes_GET_SIZE             PyString_GET_SIZE
+  #define PyBytes_Repr                 PyString_Repr
+  #define PyBytes_Concat               PyString_Concat
+  #define PyBytes_ConcatAndDel         PyString_ConcatAndDel
+#endif
+
+#if PY_VERSION_HEX < 0x02060000
+  #define PySet_Check(obj)             PyObject_TypeCheck(obj, &PySet_Type)
+  #define PyFrozenSet_Check(obj)       PyObject_TypeCheck(obj, &PyFrozenSet_Type)
+#endif
+#ifndef PySet_CheckExact
+  #define PySet_CheckExact(obj)        (Py_TYPE(obj) == &PySet_Type)
+#endif
+
+#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type)
+
+#if PY_MAJOR_VERSION >= 3
+  #define PyIntObject                  PyLongObject
+  #define PyInt_Type                   PyLong_Type
+  #define PyInt_Check(op)              PyLong_Check(op)
+  #define PyInt_CheckExact(op)         PyLong_CheckExact(op)
+  #define PyInt_FromString             PyLong_FromString
+  #define PyInt_FromUnicode            PyLong_FromUnicode
+  #define PyInt_FromLong               PyLong_FromLong
+  #define PyInt_FromSize_t             PyLong_FromSize_t
+  #define PyInt_FromSsize_t            PyLong_FromSsize_t
+  #define PyInt_AsLong                 PyLong_AsLong
+  #define PyInt_AS_LONG                PyLong_AS_LONG
+  #define PyInt_AsSsize_t              PyLong_AsSsize_t
+  #define PyInt_AsUnsignedLongMask     PyLong_AsUnsignedLongMask
+  #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+  #define PyBoolObject                 PyLongObject
+#endif
+
+
+#if PY_MAJOR_VERSION >= 3
+  #define __Pyx_PyNumber_Divide(x,y)         PyNumber_TrueDivide(x,y)
+  #define __Pyx_PyNumber_InPlaceDivide(x,y)  PyNumber_InPlaceTrueDivide(x,y)
+#else
+  #define __Pyx_PyNumber_Divide(x,y)         PyNumber_Divide(x,y)
+  #define __Pyx_PyNumber_InPlaceDivide(x,y)  PyNumber_InPlaceDivide(x,y)
+#endif
+
+#if (PY_MAJOR_VERSION < 3) || (PY_VERSION_HEX >= 0x03010300)
+  #define __Pyx_PySequence_GetSlice(obj, a, b) PySequence_GetSlice(obj, a, b)
+  #define __Pyx_PySequence_SetSlice(obj, a, b, value) PySequence_SetSlice(obj, a, b, value)
+  #define __Pyx_PySequence_DelSlice(obj, a, b) PySequence_DelSlice(obj, a, b)
+#else
+  #define __Pyx_PySequence_GetSlice(obj, a, b) (unlikely(!(obj)) ? \
+        (PyErr_SetString(PyExc_SystemError, "null argument to internal routine"), (PyObject*)0) : \
+        (likely((obj)->ob_type->tp_as_mapping) ? (PySequence_GetSlice(obj, a, b)) : \
+            (PyErr_Format(PyExc_TypeError, "'%.200s' object is unsliceable", (obj)->ob_type->tp_name), (PyObject*)0)))
+  #define __Pyx_PySequence_SetSlice(obj, a, b, value) (unlikely(!(obj)) ? \
+        (PyErr_SetString(PyExc_SystemError, "null argument to internal routine"), -1) : \
+        (likely((obj)->ob_type->tp_as_mapping) ? (PySequence_SetSlice(obj, a, b, value)) : \
+            (PyErr_Format(PyExc_TypeError, "'%.200s' object doesn't support slice assignment", (obj)->ob_type->tp_name), -1)))
+  #define __Pyx_PySequence_DelSlice(obj, a, b) (unlikely(!(obj)) ? \
+        (PyErr_SetString(PyExc_SystemError, "null argument to internal routine"), -1) : \
+        (likely((obj)->ob_type->tp_as_mapping) ? (PySequence_DelSlice(obj, a, b)) : \
+            (PyErr_Format(PyExc_TypeError, "'%.200s' object doesn't support slice deletion", (obj)->ob_type->tp_name), -1)))
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+  #define PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func))
+#endif
+
+#if PY_VERSION_HEX < 0x02050000
+  #define __Pyx_GetAttrString(o,n)   PyObject_GetAttrString((o),((char *)(n)))
+  #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),((char *)(n)),(a))
+  #define __Pyx_DelAttrString(o,n)   PyObject_DelAttrString((o),((char *)(n)))
+#else
+  #define __Pyx_GetAttrString(o,n)   PyObject_GetAttrString((o),(n))
+  #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),(n),(a))
+  #define __Pyx_DelAttrString(o,n)   PyObject_DelAttrString((o),(n))
+#endif
+
+#if PY_VERSION_HEX < 0x02050000
+  #define __Pyx_NAMESTR(n) ((char *)(n))
+  #define __Pyx_DOCSTR(n)  ((char *)(n))
+#else
+  #define __Pyx_NAMESTR(n) (n)
+  #define __Pyx_DOCSTR(n)  (n)
+#endif
+
+#ifdef __cplusplus
+#define __PYX_EXTERN_C extern "C"
+#else
+#define __PYX_EXTERN_C extern
+#endif
+
+#if defined(WIN32) || defined(MS_WINDOWS)
+#define _USE_MATH_DEFINES
+#endif
+#include <math.h>
+#define __PYX_HAVE_API__tokenize
+
+#ifdef PYREX_WITHOUT_ASSERTIONS
+#define CYTHON_WITHOUT_ASSERTIONS
+#endif
+
+
+/* inline attribute */
+#ifndef CYTHON_INLINE
+  #if defined(__GNUC__)
+    #define CYTHON_INLINE __inline__
+  #elif defined(_MSC_VER)
+    #define CYTHON_INLINE __inline
+  #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+    #define CYTHON_INLINE inline
+  #else
+    #define CYTHON_INLINE
+  #endif
+#endif
+
+/* unused attribute */
+#ifndef CYTHON_UNUSED
+# if defined(__GNUC__)
+#   if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4))
+#     define CYTHON_UNUSED __attribute__ ((__unused__))
+#   else
+#     define CYTHON_UNUSED
+#   endif
+# elif defined(__ICC) || defined(__INTEL_COMPILER)
+#   define CYTHON_UNUSED __attribute__ ((__unused__))
+# else
+#   define CYTHON_UNUSED
+# endif
+#endif
+
+typedef struct {PyObject **p; char *s; const long n; const char* encoding; const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; /*proto*/
+
+
+/* Type Conversion Predeclarations */
+
+#define __Pyx_PyBytes_FromUString(s) PyBytes_FromString((char*)s)
+#define __Pyx_PyBytes_AsUString(s)   ((unsigned char*) PyBytes_AsString(s))
+
+#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False))
+static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*);
+static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x);
+
+static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*);
+static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t);
+static CYTHON_INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*);
+
+#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x))
+
+
+#ifdef __GNUC__
+/* Test for GCC > 2.95 */
+#if __GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))
+#define likely(x)   __builtin_expect(!!(x), 1)
+#define unlikely(x) __builtin_expect(!!(x), 0)
+#else /* __GNUC__ > 2 ... */
+#define likely(x)   (x)
+#define unlikely(x) (x)
+#endif /* __GNUC__ > 2 ... */
+#else /* __GNUC__ */
+#define likely(x)   (x)
+#define unlikely(x) (x)
+#endif /* __GNUC__ */
+    
+static PyObject *__pyx_m;
+static PyObject *__pyx_b;
+static PyObject *__pyx_empty_tuple;
+static PyObject *__pyx_empty_bytes;
+static int __pyx_lineno;
+static int __pyx_clineno = 0;
+static const char * __pyx_cfilenm= __FILE__;
+static const char *__pyx_filename;
+
+
+static const char *__pyx_f[] = {
+  "tokenize.pyx",
+};
+
+/* Type declarations */
+
+#ifndef CYTHON_REFNANNY
+  #define CYTHON_REFNANNY 0
+#endif
+
+#if CYTHON_REFNANNY
+  typedef struct {
+    void (*INCREF)(void*, PyObject*, int);
+    void (*DECREF)(void*, PyObject*, int);
+    void (*GOTREF)(void*, PyObject*, int);
+    void (*GIVEREF)(void*, PyObject*, int);
+    void* (*SetupContext)(const char*, int, const char*);
+    void (*FinishContext)(void**);
+  } __Pyx_RefNannyAPIStruct;
+  static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL;
+  static __Pyx_RefNannyAPIStruct * __Pyx_RefNannyImportAPI(const char *modname) {
+    PyObject *m = NULL, *p = NULL;
+    void *r = NULL;
+    m = PyImport_ImportModule((char *)modname);
+    if (!m) goto end;
+    p = PyObject_GetAttrString(m, (char *)"RefNannyAPI");
+    if (!p) goto end;
+    r = PyLong_AsVoidPtr(p);
+  end:
+    Py_XDECREF(p);
+    Py_XDECREF(m);
+    return (__Pyx_RefNannyAPIStruct *)r;
+  }
+  #define __Pyx_RefNannySetupContext(name)           void *__pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__)
+  #define __Pyx_RefNannyFinishContext()           __Pyx_RefNanny->FinishContext(&__pyx_refnanny)
+  #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
+  #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
+  #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
+  #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
+  #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r);} } while(0)
+#else
+  #define __Pyx_RefNannySetupContext(name)
+  #define __Pyx_RefNannyFinishContext()
+  #define __Pyx_INCREF(r) Py_INCREF(r)
+  #define __Pyx_DECREF(r) Py_DECREF(r)
+  #define __Pyx_GOTREF(r)
+  #define __Pyx_GIVEREF(r)
+  #define __Pyx_XDECREF(r) Py_XDECREF(r)
+#endif /* CYTHON_REFNANNY */
+#define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);} } while(0)
+#define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r);} } while(0)
+
+static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/
+
+static CYTHON_INLINE int __Pyx_CheckKeywordStrings(PyObject *kwdict,
+    const char* function_name, int kw_allowed); /*proto*/
+
+static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact,
+    Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/
+
+static void __Pyx_RaiseDoubleKeywordsError(
+    const char* func_name, PyObject* kw_name); /*proto*/
+
+static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],     PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,     const char* function_name); /*proto*/
+
+static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index);
+
+static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected);
+
+static PyObject *__Pyx_UnpackItem(PyObject *, Py_ssize_t index); /*proto*/
+static int __Pyx_EndUnpack(PyObject *, Py_ssize_t expected); /*proto*/
+
+static CYTHON_INLINE PyObject* __Pyx_PyObject_Append(PyObject* L, PyObject* x) {
+    if (likely(PyList_CheckExact(L))) {
+        if (PyList_Append(L, x) < 0) return NULL;
+        Py_INCREF(Py_None);
+        return Py_None; /* this is just to have an accurate signature */
+    }
+    else {
+        PyObject *r, *m;
+        m = __Pyx_GetAttrString(L, "append");
+        if (!m) return NULL;
+        r = PyObject_CallFunctionObjArgs(m, x, NULL);
+        Py_DECREF(m);
+        return r;
+    }
+}
+
+static CYTHON_INLINE PyObject* __Pyx_PyObject_Pop(PyObject* L) {
+    PyObject *r, *m;
+#if PY_VERSION_HEX >= 0x02040000
+    if (likely(PyList_CheckExact(L))
+            /* Check that both the size is positive and no reallocation shrinking needs to be done. */
+            && likely(PyList_GET_SIZE(L) > (((PyListObject*)L)->allocated >> 1))) {
+        Py_SIZE(L) -= 1;
+        return PyList_GET_ITEM(L, PyList_GET_SIZE(L));
+    }
+#endif
+    m = __Pyx_GetAttrString(L, "pop");
+    if (!m) return NULL;
+    r = PyObject_CallObject(m, NULL);
+    Py_DECREF(m);
+    return r;
+}
+
+
+static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) {
+    PyObject *r;
+    if (!j) return NULL;
+    r = PyObject_GetItem(o, j);
+    Py_DECREF(j);
+    return r;
+}
+
+
+#define __Pyx_GetItemInt_List(o, i, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \
+                                                    __Pyx_GetItemInt_List_Fast(o, i) : \
+                                                    __Pyx_GetItemInt_Generic(o, to_py_func(i)))
+
+static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i) {
+    if (likely(o != Py_None)) {
+        if (likely((0 <= i) & (i < PyList_GET_SIZE(o)))) {
+            PyObject *r = PyList_GET_ITEM(o, i);
+            Py_INCREF(r);
+            return r;
+        }
+        else if ((-PyList_GET_SIZE(o) <= i) & (i < 0)) {
+            PyObject *r = PyList_GET_ITEM(o, PyList_GET_SIZE(o) + i);
+            Py_INCREF(r);
+            return r;
+        }
+    }
+    return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i));
+}
+
+#define __Pyx_GetItemInt_Tuple(o, i, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \
+                                                    __Pyx_GetItemInt_Tuple_Fast(o, i) : \
+                                                    __Pyx_GetItemInt_Generic(o, to_py_func(i)))
+
+static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i) {
+    if (likely(o != Py_None)) {
+        if (likely((0 <= i) & (i < PyTuple_GET_SIZE(o)))) {
+            PyObject *r = PyTuple_GET_ITEM(o, i);
+            Py_INCREF(r);
+            return r;
+        }
+        else if ((-PyTuple_GET_SIZE(o) <= i) & (i < 0)) {
+            PyObject *r = PyTuple_GET_ITEM(o, PyTuple_GET_SIZE(o) + i);
+            Py_INCREF(r);
+            return r;
+        }
+    }
+    return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i));
+}
+
+
+#define __Pyx_GetItemInt(o, i, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \
+                                                    __Pyx_GetItemInt_Fast(o, i) : \
+                                                    __Pyx_GetItemInt_Generic(o, to_py_func(i)))
+
+static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i) {
+    PyObject *r;
+    if (PyList_CheckExact(o) && ((0 <= i) & (i < PyList_GET_SIZE(o)))) {
+        r = PyList_GET_ITEM(o, i);
+        Py_INCREF(r);
+    }
+    else if (PyTuple_CheckExact(o) && ((0 <= i) & (i < PyTuple_GET_SIZE(o)))) {
+        r = PyTuple_GET_ITEM(o, i);
+        Py_INCREF(r);
+    }
+    else if (Py_TYPE(o)->tp_as_sequence && Py_TYPE(o)->tp_as_sequence->sq_item && (likely(i >= 0))) {
+        r = PySequence_GetItem(o, i);
+    }
+    else {
+        r = __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i));
+    }
+    return r;
+}
+
+static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); /*proto*/
+
+static CYTHON_INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/
+static CYTHON_INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/
+
+static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb); /*proto*/
+
+static CYTHON_INLINE long __Pyx_NegateNonNeg(long b) { return unlikely(b < 0) ? b : !b; }
+static CYTHON_INLINE PyObject* __Pyx_PyBoolOrNull_FromLong(long b) {
+    return unlikely(b < 0) ? NULL : __Pyx_PyBool_FromLong(b);
+}
+
+static CYTHON_INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb); /*proto*/
+static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb); /*proto*/
+
+static int __Pyx_Print(PyObject*, PyObject *, int); /*proto*/
+#if PY_MAJOR_VERSION >= 3
+static PyObject* __pyx_print = 0;
+static PyObject* __pyx_print_kwargs = 0;
+#endif
+
+static int __Pyx_PrintOne(PyObject* stream, PyObject *o); /*proto*/
+
+static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list); /*proto*/
+
+static PyObject *__Pyx_FindPy2Metaclass(PyObject *bases); /*proto*/
+
+static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *name,
+                                   PyObject *modname); /*proto*/
+
+#define __pyx_binding_PyCFunctionType_USED 1
+
+typedef struct {
+    PyCFunctionObject func;
+} __pyx_binding_PyCFunctionType_object;
+
+static PyTypeObject __pyx_binding_PyCFunctionType_type;
+static PyTypeObject *__pyx_binding_PyCFunctionType = NULL;
+
+static PyObject *__pyx_binding_PyCFunctionType_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module); /* proto */
+#define __pyx_binding_PyCFunctionType_New(ml, self) __pyx_binding_PyCFunctionType_NewEx(ml, self, NULL)
+
+static int __pyx_binding_PyCFunctionType_init(void); /* proto */
+
+static CYTHON_INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *);
+
+static CYTHON_INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject *);
+
+static CYTHON_INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject *);
+
+static CYTHON_INLINE char __Pyx_PyInt_AsChar(PyObject *);
+
+static CYTHON_INLINE short __Pyx_PyInt_AsShort(PyObject *);
+
+static CYTHON_INLINE int __Pyx_PyInt_AsInt(PyObject *);
+
+static CYTHON_INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject *);
+
+static CYTHON_INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject *);
+
+static CYTHON_INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject *);
+
+static CYTHON_INLINE int __Pyx_PyInt_AsLongDouble(PyObject *);
+
+static CYTHON_INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject *);
+
+static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject *);
+
+static CYTHON_INLINE long __Pyx_PyInt_AsLong(PyObject *);
+
+static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject *);
+
+static CYTHON_INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *);
+
+static CYTHON_INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *);
+
+static CYTHON_INLINE int __Pyx_StrEq(const char *, const char *); /*proto*/
+
+static void __Pyx_AddTraceback(const char *funcname); /*proto*/
+
+static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/
+/* Module declarations from tokenize */
+
+#define __Pyx_MODULE_NAME "tokenize"
+static int __pyx_module_is_main_tokenize = 0;
+
+/* Implementation of tokenize */
+static PyObject *__pyx_builtin_map;
+static PyObject *__pyx_builtin_Exception;
+static PyObject *__pyx_builtin_open;
+static PyObject *__pyx_builtin_StopIteration;
+static PyObject *__pyx_builtin_IndentationError;
+static char __pyx_k_1[] = "(";
+static char __pyx_k_2[] = "|";
+static char __pyx_k_3[] = ")";
+static char __pyx_k_4[] = "*";
+static char __pyx_k_5[] = "?";
+static char __pyx_k_6[] = "%d,%d-%d,%d:\t%s\t%s";
+static char __pyx_k_8[] = " ";
+static char __pyx_k_9[] = "";
+static char __pyx_k_10[] = "EOF in multi-line string";
+static char __pyx_k_12[] = "\\\n";
+static char __pyx_k_13[] = "\\\r\n";
+static char __pyx_k_14[] = "\t";
+static char __pyx_k_15[] = "\014";
+static char __pyx_k_16[] = "#\r\n";
+static char __pyx_k_17[] = "#";
+static char __pyx_k_18[] = "\r\n";
+static char __pyx_k_20[] = "unindent does not match any outer indentation level";
+static char __pyx_k_21[] = "<tokenize>";
+static char __pyx_k_22[] = "EOF in multi-line statement";
+static char __pyx_k_24[] = ".";
+static char __pyx_k_25[] = "\n";
+static char __pyx_k_28[] = "\\";
+static char __pyx_k_29[] = "([{";
+static char __pyx_k_30[] = ")]}";
+static char __pyx_k_31[] = "Tokenization help for Python programs.\n\ngenerate_tokens(readline) is a generator that breaks a stream of\ntext into Python tokens.  It accepts a readline-like method which is called\nrepeatedly to get the next line of input (or \"\" for EOF).  It generates\n5-tuples with these members:\n\n    the token type (see token.py)\n    the token (a string)\n    the starting (row, column) indices of the token (a 2-tuple of ints)\n    the ending (row, column) indices of the token (a 2-tuple of ints)\n    the original line (string)\n\nIt is designed to match the working of the Python tokenizer exactly, except\nthat it produces COMMENT tokens for comments and gives type OP for all\noperators\n\nOlder entry points\n    tokenize_loop(readline, tokeneater)\n    tokenize(readline, tokeneater=printtoken)\nare the same, except instead of generating tokens, tokeneater is a callback\nfunction to which the 5 fields described above are passed as 5 arguments,\neach time a new token is found.";
+static char __pyx_k_32[] = "Ka-Ping Yee <ping@lfw.org>";
+static char __pyx_k_33[] = "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger";
+static char __pyx_k_34[] = "ATTENTION: using optimized tokenize";
+static char __pyx_k_36[] = "[ \\f\\t]*";
+static char __pyx_k_37[] = "#[^\\r\\n]*";
+static char __pyx_k_38[] = "\\\\\\r?\\n";
+static char __pyx_k_39[] = "[a-zA-Z_]\\w*";
+static char __pyx_k_40[] = "0[xX][\\da-fA-F]+[lL]?";
+static char __pyx_k_41[] = "(0[oO][0-7]+)|(0[0-7]*)[lL]?";
+static char __pyx_k_42[] = "0[bB][01]+[lL]?";
+static char __pyx_k_43[] = "[1-9]\\d*[lL]?";
+static char __pyx_k_44[] = "[eE][-+]?\\d+";
+static char __pyx_k_45[] = "\\d+\\.\\d*";
+static char __pyx_k_46[] = "\\.\\d+";
+static char __pyx_k_48[] = "\\d+";
+static char __pyx_k_49[] = "\\d+[jJ]";
+static char __pyx_k_50[] = "[jJ]";
+static char __pyx_k_51[] = "[^'\\\\]*(?:\\\\.[^'\\\\]*)*'";
+static char __pyx_k_52[] = "[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*\"";
+static char __pyx_k_53[] = "[^'\\\\]*(?:(?:\\\\.|'(?!''))[^'\\\\]*)*'''";
+static char __pyx_k_54[] = "[^\"\\\\]*(?:(?:\\\\.|\"(?!\"\"))[^\"\\\\]*)*\"\"\"";
+static char __pyx_k_55[] = "[uU]?[rR]?'''";
+static char __pyx_k_56[] = "[uU]?[rR]?\"\"\"";
+static char __pyx_k_58[] = "[uU]?[rR]?'[^\\n'\\\\]*(?:\\\\.[^\\n'\\\\]*)*'";
+static char __pyx_k_59[] = "[uU]?[rR]?\"[^\\n\"\\\\]*(?:\\\\.[^\\n\"\\\\]*)*\"";
+static char __pyx_k_61[] = "\\*\\*=?";
+static char __pyx_k_62[] = ">>=?";
+static char __pyx_k_63[] = "<<=?";
+static char __pyx_k_64[] = "<>";
+static char __pyx_k_65[] = "!=";
+static char __pyx_k_66[] = "//=?";
+static char __pyx_k_67[] = "[+\\-*/%&|^=<>]=?";
+static char __pyx_k_68[] = "~";
+static char __pyx_k_70[] = "[][(){}]";
+static char __pyx_k_71[] = "\\r?\\n";
+static char __pyx_k_72[] = "[:;.,`@]";
+static char __pyx_k_74[] = "[uU]?[rR]?'[^\\n'\\\\]*(?:\\\\.[^\\n'\\\\]*)*";
+static char __pyx_k_75[] = "'";
+static char __pyx_k_77[] = "[uU]?[rR]?\"[^\\n\"\\\\]*(?:\\\\.[^\\n\"\\\\]*)*";
+static char __pyx_k_78[] = "\"";
+static char __pyx_k_80[] = "'''";
+static char __pyx_k_81[] = "\"\"\"";
+static char __pyx_k_82[] = "r'''";
+static char __pyx_k_83[] = "r\"\"\"";
+static char __pyx_k_84[] = "u'''";
+static char __pyx_k_85[] = "u\"\"\"";
+static char __pyx_k_86[] = "ur'''";
+static char __pyx_k_87[] = "ur\"\"\"";
+static char __pyx_k_88[] = "R'''";
+static char __pyx_k_89[] = "R\"\"\"";
+static char __pyx_k_90[] = "U'''";
+static char __pyx_k_91[] = "U\"\"\"";
+static char __pyx_k_92[] = "uR'''";
+static char __pyx_k_93[] = "uR\"\"\"";
+static char __pyx_k_94[] = "Ur'''";
+static char __pyx_k_95[] = "Ur\"\"\"";
+static char __pyx_k_96[] = "UR'''";
+static char __pyx_k_97[] = "UR\"\"\"";
+static char __pyx_k_98[] = "b'''";
+static char __pyx_k_99[] = "b\"\"\"";
+static char __pyx_k__B[] = "B";
+static char __pyx_k__R[] = "R";
+static char __pyx_k__U[] = "U";
+static char __pyx_k___[] = "_";
+static char __pyx_k__b[] = "b";
+static char __pyx_k__r[] = "r";
+static char __pyx_k__t[] = "t";
+static char __pyx_k__u[] = "u";
+static char __pyx_k__x[] = "x";
+static char __pyx_k_100[] = "br'''";
+static char __pyx_k_101[] = "br\"\"\"";
+static char __pyx_k_102[] = "B'''";
+static char __pyx_k_103[] = "B\"\"\"";
+static char __pyx_k_104[] = "bR'''";
+static char __pyx_k_105[] = "bR\"\"\"";
+static char __pyx_k_106[] = "Br'''";
+static char __pyx_k_107[] = "Br\"\"\"";
+static char __pyx_k_108[] = "BR'''";
+static char __pyx_k_109[] = "BR\"\"\"";
+static char __pyx_k_111[] = "r'";
+static char __pyx_k_112[] = "r\"";
+static char __pyx_k_113[] = "R'";
+static char __pyx_k_114[] = "R\"";
+static char __pyx_k_115[] = "u'";
+static char __pyx_k_116[] = "u\"";
+static char __pyx_k_117[] = "U'";
+static char __pyx_k_118[] = "U\"";
+static char __pyx_k_119[] = "ur'";
+static char __pyx_k_120[] = "ur\"";
+static char __pyx_k_121[] = "Ur'";
+static char __pyx_k_122[] = "Ur\"";
+static char __pyx_k_123[] = "uR'";
+static char __pyx_k_124[] = "uR\"";
+static char __pyx_k_125[] = "UR'";
+static char __pyx_k_126[] = "UR\"";
+static char __pyx_k_127[] = "b'";
+static char __pyx_k_128[] = "b\"";
+static char __pyx_k_129[] = "B'";
+static char __pyx_k_130[] = "B\"";
+static char __pyx_k_131[] = "br'";
+static char __pyx_k_132[] = "br\"";
+static char __pyx_k_133[] = "Br'";
+static char __pyx_k_134[] = "Br\"";
+static char __pyx_k_135[] = "bR'";
+static char __pyx_k_136[] = "bR\"";
+static char __pyx_k_137[] = "BR'";
+static char __pyx_k_138[] = "BR\"";
+static char __pyx_k__NL[] = "NL";
+static char __pyx_k__OP[] = "OP";
+static char __pyx_k__re[] = "re";
+static char __pyx_k__any[] = "any";
+static char __pyx_k__end[] = "end";
+static char __pyx_k__map[] = "map";
+static char __pyx_k__sys[] = "sys";
+static char __pyx_k__NAME[] = "NAME";
+static char __pyx_k__Name[] = "Name";
+static char __pyx_k__argv[] = "argv";
+static char __pyx_k__join[] = "join";
+static char __pyx_k__line[] = "line";
+static char __pyx_k__open[] = "open";
+static char __pyx_k__self[] = "self";
+static char __pyx_k__span[] = "span";
+static char __pyx_k__type[] = "type";
+static char __pyx_k__Funny[] = "Funny";
+static char __pyx_k__Token[] = "Token";
+static char __pyx_k__group[] = "group";
+static char __pyx_k__match[] = "match";
+static char __pyx_k__maybe[] = "maybe";
+static char __pyx_k__start[] = "start";
+static char __pyx_k__stdin[] = "stdin";
+static char __pyx_k__token[] = "token";
+static char __pyx_k__DEDENT[] = "DEDENT";
+static char __pyx_k__Double[] = "Double";
+static char __pyx_k__INDENT[] = "INDENT";
+static char __pyx_k__Ignore[] = "Ignore";
+static char __pyx_k__NUMBER[] = "NUMBER";
+static char __pyx_k__Number[] = "Number";
+static char __pyx_k__STRING[] = "STRING";
+static char __pyx_k__Single[] = "Single";
+static char __pyx_k__String[] = "String";
+static char __pyx_k__Triple[] = "Triple";
+static char __pyx_k__append[] = "append";
+static char __pyx_k__compat[] = "compat";
+static char __pyx_k__rstrip[] = "rstrip";
+static char __pyx_k__string[] = "string";
+static char __pyx_k__tokens[] = "tokens";
+static char __pyx_k__Bracket[] = "Bracket";
+static char __pyx_k__COMMENT[] = "COMMENT";
+static char __pyx_k__Comment[] = "Comment";
+static char __pyx_k__ContStr[] = "ContStr";
+static char __pyx_k__Double3[] = "Double3";
+static char __pyx_k__NEWLINE[] = "NEWLINE";
+static char __pyx_k__Single3[] = "Single3";
+static char __pyx_k__Special[] = "Special";
+static char __pyx_k____all__[] = "__all__";
+static char __pyx_k__compile[] = "compile";
+static char __pyx_k__tabsize[] = "tabsize";
+static char __pyx_k__Expfloat[] = "Expfloat";
+static char __pyx_k__Exponent[] = "Exponent";
+static char __pyx_k__N_TOKENS[] = "N_TOKENS";
+static char __pyx_k__Operator[] = "Operator";
+static char __pyx_k____init__[] = "__init__";
+static char __pyx_k____main__[] = "__main__";
+static char __pyx_k____name__[] = "__name__";
+static char __pyx_k____test__[] = "__test__";
+static char __pyx_k__endprogs[] = "endprogs";
+static char __pyx_k__endswith[] = "endswith";
+static char __pyx_k__iterable[] = "iterable";
+static char __pyx_k__prev_col[] = "prev_col";
+static char __pyx_k__prev_row[] = "prev_row";
+static char __pyx_k__readline[] = "readline";
+static char __pyx_k__tok_name[] = "tok_name";
+static char __pyx_k__tokenize[] = "tokenize";
+static char __pyx_k__Binnumber[] = "Binnumber";
+static char __pyx_k__Decnumber[] = "Decnumber";
+static char __pyx_k__ENDMARKER[] = "ENDMARKER";
+static char __pyx_k__Exception[] = "Exception";
+static char __pyx_k__Hexnumber[] = "Hexnumber";
+static char __pyx_k__Intnumber[] = "Intnumber";
+static char __pyx_k__Octnumber[] = "Octnumber";
+static char __pyx_k__erow_ecol[] = "erow_ecol";
+static char __pyx_k__srow_scol[] = "srow_scol";
+static char __pyx_k__tokenprog[] = "tokenprog";
+static char __pyx_k__0123456789[] = "0123456789";
+static char __pyx_k__ERRORTOKEN[] = "ERRORTOKEN";
+static char __pyx_k__Imagnumber[] = "Imagnumber";
+static char __pyx_k__PlainToken[] = "PlainToken";
+static char __pyx_k__Pointfloat[] = "Pointfloat";
+static char __pyx_k__TokenError[] = "TokenError";
+static char __pyx_k__Whitespace[] = "Whitespace";
+static char __pyx_k____author__[] = "__author__";
+static char __pyx_k__printtoken[] = "printtoken";
+static char __pyx_k__pseudoprog[] = "pseudoprog";
+static char __pyx_k__startswith[] = "startswith";
+static char __pyx_k__tokeneater[] = "tokeneater";
+static char __pyx_k__untokenize[] = "untokenize";
+static char __pyx_k__Floatnumber[] = "Floatnumber";
+static char __pyx_k__PseudoToken[] = "PseudoToken";
+static char __pyx_k__Untokenizer[] = "Untokenizer";
+static char __pyx_k____credits__[] = "__credits__";
+static char __pyx_k__double3prog[] = "double3prog";
+static char __pyx_k__single3prog[] = "single3prog";
+static char __pyx_k__PseudoExtras[] = "PseudoExtras";
+static char __pyx_k__StopIteration[] = "StopIteration";
+static char __pyx_k__ascii_letters[] = "ascii_letters";
+static char __pyx_k__single_quoted[] = "single_quoted";
+static char __pyx_k__tokenize_loop[] = "tokenize_loop";
+static char __pyx_k__triple_quoted[] = "triple_quoted";
+static char __pyx_k__StopTokenizing[] = "StopTokenizing";
+static char __pyx_k__add_whitespace[] = "add_whitespace";
+static char __pyx_k__generate_tokens[] = "generate_tokens";
+static char __pyx_k__IndentationError[] = "IndentationError";
+static PyObject *__pyx_kp_s_1;
+static PyObject *__pyx_kp_s_10;
+static PyObject *__pyx_kp_s_100;
+static PyObject *__pyx_kp_s_101;
+static PyObject *__pyx_kp_s_102;
+static PyObject *__pyx_kp_s_103;
+static PyObject *__pyx_kp_s_104;
+static PyObject *__pyx_kp_s_105;
+static PyObject *__pyx_kp_s_106;
+static PyObject *__pyx_kp_s_107;
+static PyObject *__pyx_kp_s_108;
+static PyObject *__pyx_kp_s_109;
+static PyObject *__pyx_kp_s_111;
+static PyObject *__pyx_kp_s_112;
+static PyObject *__pyx_kp_s_113;
+static PyObject *__pyx_kp_s_114;
+static PyObject *__pyx_kp_s_115;
+static PyObject *__pyx_kp_s_116;
+static PyObject *__pyx_kp_s_117;
+static PyObject *__pyx_kp_s_118;
+static PyObject *__pyx_kp_s_119;
+static PyObject *__pyx_kp_s_12;
+static PyObject *__pyx_kp_s_120;
+static PyObject *__pyx_kp_s_121;
+static PyObject *__pyx_kp_s_122;
+static PyObject *__pyx_kp_s_123;
+static PyObject *__pyx_kp_s_124;
+static PyObject *__pyx_kp_s_125;
+static PyObject *__pyx_kp_s_126;
+static PyObject *__pyx_kp_s_127;
+static PyObject *__pyx_kp_s_128;
+static PyObject *__pyx_kp_s_129;
+static PyObject *__pyx_kp_s_13;
+static PyObject *__pyx_kp_s_130;
+static PyObject *__pyx_kp_s_131;
+static PyObject *__pyx_kp_s_132;
+static PyObject *__pyx_kp_s_133;
+static PyObject *__pyx_kp_s_134;
+static PyObject *__pyx_kp_s_135;
+static PyObject *__pyx_kp_s_136;
+static PyObject *__pyx_kp_s_137;
+static PyObject *__pyx_kp_s_138;
+static PyObject *__pyx_kp_s_14;
+static PyObject *__pyx_kp_s_15;
+static PyObject *__pyx_kp_s_16;
+static PyObject *__pyx_kp_s_17;
+static PyObject *__pyx_kp_s_18;
+static PyObject *__pyx_kp_s_2;
+static PyObject *__pyx_kp_s_20;
+static PyObject *__pyx_kp_s_21;
+static PyObject *__pyx_kp_s_22;
+static PyObject *__pyx_kp_s_24;
+static PyObject *__pyx_kp_s_25;
+static PyObject *__pyx_kp_s_28;
+static PyObject *__pyx_kp_s_29;
+static PyObject *__pyx_kp_s_3;
+static PyObject *__pyx_kp_s_30;
+static PyObject *__pyx_kp_s_32;
+static PyObject *__pyx_kp_s_33;
+static PyObject *__pyx_kp_s_34;
+static PyObject *__pyx_kp_s_36;
+static PyObject *__pyx_kp_s_37;
+static PyObject *__pyx_kp_s_38;
+static PyObject *__pyx_kp_s_39;
+static PyObject *__pyx_kp_s_4;
+static PyObject *__pyx_kp_s_40;
+static PyObject *__pyx_kp_s_41;
+static PyObject *__pyx_kp_s_42;
+static PyObject *__pyx_kp_s_43;
+static PyObject *__pyx_kp_s_44;
+static PyObject *__pyx_kp_s_45;
+static PyObject *__pyx_kp_s_46;
+static PyObject *__pyx_kp_s_48;
+static PyObject *__pyx_kp_s_49;
+static PyObject *__pyx_kp_s_5;
+static PyObject *__pyx_kp_s_50;
+static PyObject *__pyx_kp_s_51;
+static PyObject *__pyx_kp_s_52;
+static PyObject *__pyx_kp_s_53;
+static PyObject *__pyx_kp_s_54;
+static PyObject *__pyx_kp_s_55;
+static PyObject *__pyx_kp_s_56;
+static PyObject *__pyx_kp_s_58;
+static PyObject *__pyx_kp_s_59;
+static PyObject *__pyx_kp_s_6;
+static PyObject *__pyx_kp_s_61;
+static PyObject *__pyx_kp_s_62;
+static PyObject *__pyx_kp_s_63;
+static PyObject *__pyx_kp_s_64;
+static PyObject *__pyx_kp_s_65;
+static PyObject *__pyx_kp_s_66;
+static PyObject *__pyx_kp_s_67;
+static PyObject *__pyx_kp_s_68;
+static PyObject *__pyx_kp_s_70;
+static PyObject *__pyx_kp_s_71;
+static PyObject *__pyx_kp_s_72;
+static PyObject *__pyx_kp_s_74;
+static PyObject *__pyx_kp_s_75;
+static PyObject *__pyx_kp_s_77;
+static PyObject *__pyx_kp_s_78;
+static PyObject *__pyx_kp_s_8;
+static PyObject *__pyx_kp_s_80;
+static PyObject *__pyx_kp_s_81;
+static PyObject *__pyx_kp_s_82;
+static PyObject *__pyx_kp_s_83;
+static PyObject *__pyx_kp_s_84;
+static PyObject *__pyx_kp_s_85;
+static PyObject *__pyx_kp_s_86;
+static PyObject *__pyx_kp_s_87;
+static PyObject *__pyx_kp_s_88;
+static PyObject *__pyx_kp_s_89;
+static PyObject *__pyx_kp_s_9;
+static PyObject *__pyx_kp_s_90;
+static PyObject *__pyx_kp_s_91;
+static PyObject *__pyx_kp_s_92;
+static PyObject *__pyx_kp_s_93;
+static PyObject *__pyx_kp_s_94;
+static PyObject *__pyx_kp_s_95;
+static PyObject *__pyx_kp_s_96;
+static PyObject *__pyx_kp_s_97;
+static PyObject *__pyx_kp_s_98;
+static PyObject *__pyx_kp_s_99;
+static PyObject *__pyx_kp_s__0123456789;
+static PyObject *__pyx_n_s__B;
+static PyObject *__pyx_n_s__Binnumber;
+static PyObject *__pyx_n_s__Bracket;
+static PyObject *__pyx_n_s__COMMENT;
+static PyObject *__pyx_n_s__Comment;
+static PyObject *__pyx_n_s__ContStr;
+static PyObject *__pyx_n_s__DEDENT;
+static PyObject *__pyx_n_s__Decnumber;
+static PyObject *__pyx_n_s__Double;
+static PyObject *__pyx_n_s__Double3;
+static PyObject *__pyx_n_s__ENDMARKER;
+static PyObject *__pyx_n_s__ERRORTOKEN;
+static PyObject *__pyx_n_s__Exception;
+static PyObject *__pyx_n_s__Expfloat;
+static PyObject *__pyx_n_s__Exponent;
+static PyObject *__pyx_n_s__Floatnumber;
+static PyObject *__pyx_n_s__Funny;
+static PyObject *__pyx_n_s__Hexnumber;
+static PyObject *__pyx_n_s__INDENT;
+static PyObject *__pyx_n_s__Ignore;
+static PyObject *__pyx_n_s__Imagnumber;
+static PyObject *__pyx_n_s__IndentationError;
+static PyObject *__pyx_n_s__Intnumber;
+static PyObject *__pyx_n_s__NAME;
+static PyObject *__pyx_n_s__NEWLINE;
+static PyObject *__pyx_n_s__NL;
+static PyObject *__pyx_n_s__NUMBER;
+static PyObject *__pyx_n_s__N_TOKENS;
+static PyObject *__pyx_n_s__Name;
+static PyObject *__pyx_n_s__Number;
+static PyObject *__pyx_n_s__OP;
+static PyObject *__pyx_n_s__Octnumber;
+static PyObject *__pyx_n_s__Operator;
+static PyObject *__pyx_n_s__PlainToken;
+static PyObject *__pyx_n_s__Pointfloat;
+static PyObject *__pyx_n_s__PseudoExtras;
+static PyObject *__pyx_n_s__PseudoToken;
+static PyObject *__pyx_n_s__R;
+static PyObject *__pyx_n_s__STRING;
+static PyObject *__pyx_n_s__Single;
+static PyObject *__pyx_n_s__Single3;
+static PyObject *__pyx_n_s__Special;
+static PyObject *__pyx_n_s__StopIteration;
+static PyObject *__pyx_n_s__StopTokenizing;
+static PyObject *__pyx_n_s__String;
+static PyObject *__pyx_n_s__Token;
+static PyObject *__pyx_n_s__TokenError;
+static PyObject *__pyx_n_s__Triple;
+static PyObject *__pyx_n_s__U;
+static PyObject *__pyx_n_s__Untokenizer;
+static PyObject *__pyx_n_s__Whitespace;
+static PyObject *__pyx_n_s___;
+static PyObject *__pyx_n_s____all__;
+static PyObject *__pyx_n_s____author__;
+static PyObject *__pyx_n_s____credits__;
+static PyObject *__pyx_n_s____init__;
+static PyObject *__pyx_n_s____main__;
+static PyObject *__pyx_n_s____name__;
+static PyObject *__pyx_n_s____test__;
+static PyObject *__pyx_n_s__add_whitespace;
+static PyObject *__pyx_n_s__any;
+static PyObject *__pyx_n_s__append;
+static PyObject *__pyx_n_s__argv;
+static PyObject *__pyx_n_s__ascii_letters;
+static PyObject *__pyx_n_s__b;
+static PyObject *__pyx_n_s__compat;
+static PyObject *__pyx_n_s__compile;
+static PyObject *__pyx_n_s__double3prog;
+static PyObject *__pyx_n_s__end;
+static PyObject *__pyx_n_s__endprogs;
+static PyObject *__pyx_n_s__endswith;
+static PyObject *__pyx_n_s__erow_ecol;
+static PyObject *__pyx_n_s__generate_tokens;
+static PyObject *__pyx_n_s__group;
+static PyObject *__pyx_n_s__iterable;
+static PyObject *__pyx_n_s__join;
+static PyObject *__pyx_n_s__line;
+static PyObject *__pyx_n_s__map;
+static PyObject *__pyx_n_s__match;
+static PyObject *__pyx_n_s__maybe;
+static PyObject *__pyx_n_s__open;
+static PyObject *__pyx_n_s__prev_col;
+static PyObject *__pyx_n_s__prev_row;
+static PyObject *__pyx_n_s__printtoken;
+static PyObject *__pyx_n_s__pseudoprog;
+static PyObject *__pyx_n_s__r;
+static PyObject *__pyx_n_s__re;
+static PyObject *__pyx_n_s__readline;
+static PyObject *__pyx_n_s__rstrip;
+static PyObject *__pyx_n_s__self;
+static PyObject *__pyx_n_s__single3prog;
+static PyObject *__pyx_n_s__single_quoted;
+static PyObject *__pyx_n_s__span;
+static PyObject *__pyx_n_s__srow_scol;
+static PyObject *__pyx_n_s__start;
+static PyObject *__pyx_n_s__startswith;
+static PyObject *__pyx_n_s__stdin;
+static PyObject *__pyx_n_s__string;
+static PyObject *__pyx_n_s__sys;
+static PyObject *__pyx_n_s__t;
+static PyObject *__pyx_n_s__tabsize;
+static PyObject *__pyx_n_s__tok_name;
+static PyObject *__pyx_n_s__token;
+static PyObject *__pyx_n_s__tokeneater;
+static PyObject *__pyx_n_s__tokenize;
+static PyObject *__pyx_n_s__tokenize_loop;
+static PyObject *__pyx_n_s__tokenprog;
+static PyObject *__pyx_n_s__tokens;
+static PyObject *__pyx_n_s__triple_quoted;
+static PyObject *__pyx_n_s__type;
+static PyObject *__pyx_n_s__u;
+static PyObject *__pyx_n_s__untokenize;
+static PyObject *__pyx_n_s__x;
+static PyObject *__pyx_int_0;
+static PyObject *__pyx_int_1;
+static PyObject *__pyx_int_2;
+static PyObject *__pyx_int_8;
+static PyObject *__pyx_k_7;
+static PyObject *__pyx_k_tuple_11;
+static PyObject *__pyx_k_tuple_19;
+static PyObject *__pyx_k_tuple_23;
+static PyObject *__pyx_k_tuple_26;
+static PyObject *__pyx_k_tuple_27;
+static PyObject *__pyx_k_tuple_35;
+static PyObject *__pyx_k_tuple_47;
+static PyObject *__pyx_k_tuple_57;
+static PyObject *__pyx_k_tuple_60;
+static PyObject *__pyx_k_tuple_69;
+static PyObject *__pyx_k_tuple_73;
+static PyObject *__pyx_k_tuple_76;
+static PyObject *__pyx_k_tuple_79;
+static PyObject *__pyx_k_tuple_110;
+static PyObject *__pyx_k_tuple_139;
+
+/* "tokenize.pyx":46
+ * N_TOKENS += 2
+ * 
+ * def group(*choices): return '(' + '|'.join(choices) + ')'             # <<<<<<<<<<<<<<
+ * def any(*choices): return group(*choices) + '*'
+ * def maybe(*choices): return group(*choices) + '?'
+ */
+
+static PyObject *__pyx_pf_8tokenize_group(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static PyMethodDef __pyx_mdef_8tokenize_group = {__Pyx_NAMESTR("group"), (PyCFunction)__pyx_pf_8tokenize_group, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)};
+static PyObject *__pyx_pf_8tokenize_group(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  PyObject *__pyx_v_choices = 0;
+  PyObject *__pyx_r = NULL;
+  PyObject *__pyx_t_1 = NULL;
+  PyObject *__pyx_t_2 = NULL;
+  PyObject *__pyx_t_3 = NULL;
+  __Pyx_RefNannySetupContext("group");
+  __pyx_self = __pyx_self;
+  if (unlikely(__pyx_kwds) && unlikely(PyDict_Size(__pyx_kwds) > 0) && unlikely(!__Pyx_CheckKeywordStrings(__pyx_kwds, "group", 0))) return NULL;
+  __Pyx_INCREF(__pyx_args);
+  __pyx_v_choices = __pyx_args;
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_2), __pyx_n_s__join); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(((PyObject *)__pyx_t_2));
+  __Pyx_INCREF(((PyObject *)__pyx_v_choices));
+  PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)__pyx_v_choices));
+  __Pyx_GIVEREF(((PyObject *)__pyx_v_choices));
+  __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_3);
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+  __pyx_t_2 = PyNumber_Add(((PyObject *)__pyx_kp_s_1), __pyx_t_3); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+  __pyx_t_3 = PyNumber_Add(__pyx_t_2, ((PyObject *)__pyx_kp_s_3)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_3);
+  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+  __pyx_r = __pyx_t_3;
+  __pyx_t_3 = 0;
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_XDECREF(__pyx_t_3);
+  __Pyx_AddTraceback("tokenize.group");
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_DECREF(__pyx_v_choices);
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "tokenize.pyx":47
+ * 
+ * def group(*choices): return '(' + '|'.join(choices) + ')'
+ * def any(*choices): return group(*choices) + '*'             # <<<<<<<<<<<<<<
+ * def maybe(*choices): return group(*choices) + '?'
+ * 
+ */
+
+static PyObject *__pyx_pf_8tokenize_1any(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static PyMethodDef __pyx_mdef_8tokenize_1any = {__Pyx_NAMESTR("any"), (PyCFunction)__pyx_pf_8tokenize_1any, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)};
+static PyObject *__pyx_pf_8tokenize_1any(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  PyObject *__pyx_v_choices = 0;
+  PyObject *__pyx_r = NULL;
+  PyObject *__pyx_t_1 = NULL;
+  PyObject *__pyx_t_2 = NULL;
+  PyObject *__pyx_t_3 = NULL;
+  __Pyx_RefNannySetupContext("any");
+  __pyx_self = __pyx_self;
+  if (unlikely(__pyx_kwds) && unlikely(PyDict_Size(__pyx_kwds) > 0) && unlikely(!__Pyx_CheckKeywordStrings(__pyx_kwds, "any", 0))) return NULL;
+  __Pyx_INCREF(__pyx_args);
+  __pyx_v_choices = __pyx_args;
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__group); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_t_2 = PySequence_Tuple(((PyObject *)__pyx_v_choices)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(((PyObject *)__pyx_t_2));
+  __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_3);
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+  __pyx_t_2 = PyNumber_Add(__pyx_t_3, ((PyObject *)__pyx_kp_s_4)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+  __pyx_r = __pyx_t_2;
+  __pyx_t_2 = 0;
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_XDECREF(__pyx_t_3);
+  __Pyx_AddTraceback("tokenize.any");
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_DECREF(__pyx_v_choices);
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "tokenize.pyx":48
+ * def group(*choices): return '(' + '|'.join(choices) + ')'
+ * def any(*choices): return group(*choices) + '*'
+ * def maybe(*choices): return group(*choices) + '?'             # <<<<<<<<<<<<<<
+ * 
+ * Whitespace = r'[ \f\t]*'
+ */
+
+static PyObject *__pyx_pf_8tokenize_2maybe(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static PyMethodDef __pyx_mdef_8tokenize_2maybe = {__Pyx_NAMESTR("maybe"), (PyCFunction)__pyx_pf_8tokenize_2maybe, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)};
+static PyObject *__pyx_pf_8tokenize_2maybe(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  PyObject *__pyx_v_choices = 0;
+  PyObject *__pyx_r = NULL;
+  PyObject *__pyx_t_1 = NULL;
+  PyObject *__pyx_t_2 = NULL;
+  PyObject *__pyx_t_3 = NULL;
+  __Pyx_RefNannySetupContext("maybe");
+  __pyx_self = __pyx_self;
+  if (unlikely(__pyx_kwds) && unlikely(PyDict_Size(__pyx_kwds) > 0) && unlikely(!__Pyx_CheckKeywordStrings(__pyx_kwds, "maybe", 0))) return NULL;
+  __Pyx_INCREF(__pyx_args);
+  __pyx_v_choices = __pyx_args;
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__group); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 48; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_t_2 = PySequence_Tuple(((PyObject *)__pyx_v_choices)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 48; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(((PyObject *)__pyx_t_2));
+  __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 48; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_3);
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+  __pyx_t_2 = PyNumber_Add(__pyx_t_3, ((PyObject *)__pyx_kp_s_5)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 48; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+  __pyx_r = __pyx_t_2;
+  __pyx_t_2 = 0;
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_XDECREF(__pyx_t_3);
+  __Pyx_AddTraceback("tokenize.maybe");
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_DECREF(__pyx_v_choices);
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "tokenize.pyx":151
+ * class StopTokenizing(Exception): pass
+ * 
+ * def printtoken(type, token, srow_scol, erow_ecol, line): # for testing             # <<<<<<<<<<<<<<
+ *     srow, scol = srow_scol
+ *     erow, ecol = erow_ecol
+ */
+
+static PyObject *__pyx_pf_8tokenize_3printtoken(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static PyMethodDef __pyx_mdef_8tokenize_3printtoken = {__Pyx_NAMESTR("printtoken"), (PyCFunction)__pyx_pf_8tokenize_3printtoken, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)};
+static PyObject *__pyx_pf_8tokenize_3printtoken(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  PyObject *__pyx_v_type = 0;
+  PyObject *__pyx_v_token = 0;
+  PyObject *__pyx_v_srow_scol = 0;
+  PyObject *__pyx_v_erow_ecol = 0;
+  PyObject *__pyx_v_line = 0;
+  PyObject *__pyx_v_srow;
+  PyObject *__pyx_v_scol;
+  PyObject *__pyx_v_erow;
+  PyObject *__pyx_v_ecol;
+  PyObject *__pyx_r = NULL;
+  PyObject *__pyx_t_1 = NULL;
+  PyObject *__pyx_t_2 = NULL;
+  PyObject *__pyx_t_3 = NULL;
+  static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__type,&__pyx_n_s__token,&__pyx_n_s__srow_scol,&__pyx_n_s__erow_ecol,&__pyx_n_s__line,0};
+  __Pyx_RefNannySetupContext("printtoken");
+  __pyx_self = __pyx_self;
+  if (unlikely(__pyx_kwds)) {
+    Py_ssize_t kw_args = PyDict_Size(__pyx_kwds);
+    PyObject* values[5] = {0,0,0,0,0};
+    switch (PyTuple_GET_SIZE(__pyx_args)) {
+      case  5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4);
+      case  4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3);
+      case  3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
+      case  2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
+      case  1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+      case  0: break;
+      default: goto __pyx_L5_argtuple_error;
+    }
+    switch (PyTuple_GET_SIZE(__pyx_args)) {
+      case  0:
+      values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__type);
+      if (likely(values[0])) kw_args--;
+      else goto __pyx_L5_argtuple_error;
+      case  1:
+      values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__token);
+      if (likely(values[1])) kw_args--;
+      else {
+        __Pyx_RaiseArgtupleInvalid("printtoken", 1, 5, 5, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 151; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+      }
+      case  2:
+      values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__srow_scol);
+      if (likely(values[2])) kw_args--;
+      else {
+        __Pyx_RaiseArgtupleInvalid("printtoken", 1, 5, 5, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 151; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+      }
+      case  3:
+      values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__erow_ecol);
+      if (likely(values[3])) kw_args--;
+      else {
+        __Pyx_RaiseArgtupleInvalid("printtoken", 1, 5, 5, 3); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 151; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+      }
+      case  4:
+      values[4] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__line);
+      if (likely(values[4])) kw_args--;
+      else {
+        __Pyx_RaiseArgtupleInvalid("printtoken", 1, 5, 5, 4); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 151; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+      }
+    }
+    if (unlikely(kw_args > 0)) {
+      if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "printtoken") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 151; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+    }
+    __pyx_v_type = values[0];
+    __pyx_v_token = values[1];
+    __pyx_v_srow_scol = values[2];
+    __pyx_v_erow_ecol = values[3];
+    __pyx_v_line = values[4];
+  } else if (PyTuple_GET_SIZE(__pyx_args) != 5) {
+    goto __pyx_L5_argtuple_error;
+  } else {
+    __pyx_v_type = PyTuple_GET_ITEM(__pyx_args, 0);
+    __pyx_v_token = PyTuple_GET_ITEM(__pyx_args, 1);
+    __pyx_v_srow_scol = PyTuple_GET_ITEM(__pyx_args, 2);
+    __pyx_v_erow_ecol = PyTuple_GET_ITEM(__pyx_args, 3);
+    __pyx_v_line = PyTuple_GET_ITEM(__pyx_args, 4);
+  }
+  goto __pyx_L4_argument_unpacking_done;
+  __pyx_L5_argtuple_error:;
+  __Pyx_RaiseArgtupleInvalid("printtoken", 1, 5, 5, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 151; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __pyx_L3_error:;
+  __Pyx_AddTraceback("tokenize.printtoken");
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+  __pyx_L4_argument_unpacking_done:;
+  __pyx_v_srow = Py_None; __Pyx_INCREF(Py_None);
+  __pyx_v_scol = Py_None; __Pyx_INCREF(Py_None);
+  __pyx_v_erow = Py_None; __Pyx_INCREF(Py_None);
+  __pyx_v_ecol = Py_None; __Pyx_INCREF(Py_None);
+
+  /* "tokenize.pyx":152
+ * 
+ * def printtoken(type, token, srow_scol, erow_ecol, line): # for testing
+ *     srow, scol = srow_scol             # <<<<<<<<<<<<<<
+ *     erow, ecol = erow_ecol
+ *     print "%d,%d-%d,%d:\t%s\t%s" % \
+ */
+  if (PyTuple_CheckExact(__pyx_v_srow_scol) && likely(PyTuple_GET_SIZE(__pyx_v_srow_scol) == 2)) {
+    PyObject* tuple = __pyx_v_srow_scol;
+    __pyx_t_1 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_t_1);
+    __pyx_t_2 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_t_2);
+    __Pyx_DECREF(__pyx_v_srow);
+    __pyx_v_srow = __pyx_t_1;
+    __pyx_t_1 = 0;
+    __Pyx_DECREF(__pyx_v_scol);
+    __pyx_v_scol = __pyx_t_2;
+    __pyx_t_2 = 0;
+  } else {
+    __pyx_t_3 = PyObject_GetIter(__pyx_v_srow_scol); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 152; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_3);
+    __pyx_t_1 = __Pyx_UnpackItem(__pyx_t_3, 0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 152; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_1);
+    __pyx_t_2 = __Pyx_UnpackItem(__pyx_t_3, 1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 152; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_2);
+    if (__Pyx_EndUnpack(__pyx_t_3, 2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 152; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+    __Pyx_DECREF(__pyx_v_srow);
+    __pyx_v_srow = __pyx_t_1;
+    __pyx_t_1 = 0;
+    __Pyx_DECREF(__pyx_v_scol);
+    __pyx_v_scol = __pyx_t_2;
+    __pyx_t_2 = 0;
+  }
+
+  /* "tokenize.pyx":153
+ * def printtoken(type, token, srow_scol, erow_ecol, line): # for testing
+ *     srow, scol = srow_scol
+ *     erow, ecol = erow_ecol             # <<<<<<<<<<<<<<
+ *     print "%d,%d-%d,%d:\t%s\t%s" % \
+ *         (srow, scol, erow, ecol, tok_name[type], repr(token))
+ */
+  if (PyTuple_CheckExact(__pyx_v_erow_ecol) && likely(PyTuple_GET_SIZE(__pyx_v_erow_ecol) == 2)) {
+    PyObject* tuple = __pyx_v_erow_ecol;
+    __pyx_t_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_t_2);
+    __pyx_t_1 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_t_1);
+    __Pyx_DECREF(__pyx_v_erow);
+    __pyx_v_erow = __pyx_t_2;
+    __pyx_t_2 = 0;
+    __Pyx_DECREF(__pyx_v_ecol);
+    __pyx_v_ecol = __pyx_t_1;
+    __pyx_t_1 = 0;
+  } else {
+    __pyx_t_3 = PyObject_GetIter(__pyx_v_erow_ecol); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_3);
+    __pyx_t_2 = __Pyx_UnpackItem(__pyx_t_3, 0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_2);
+    __pyx_t_1 = __Pyx_UnpackItem(__pyx_t_3, 1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_1);
+    if (__Pyx_EndUnpack(__pyx_t_3, 2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+    __Pyx_DECREF(__pyx_v_erow);
+    __pyx_v_erow = __pyx_t_2;
+    __pyx_t_2 = 0;
+    __Pyx_DECREF(__pyx_v_ecol);
+    __pyx_v_ecol = __pyx_t_1;
+    __pyx_t_1 = 0;
+  }
+
+  /* "tokenize.pyx":155
+ *     erow, ecol = erow_ecol
+ *     print "%d,%d-%d,%d:\t%s\t%s" % \
+ *         (srow, scol, erow, ecol, tok_name[type], repr(token))             # <<<<<<<<<<<<<<
+ * 
+ * def tokenize(readline, tokeneater=printtoken):
+ */
+  __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__tok_name); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_t_2 = PyObject_GetItem(__pyx_t_1, __pyx_v_type); if (!__pyx_t_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  __pyx_t_1 = PyObject_Repr(__pyx_v_token); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_t_3 = PyTuple_New(6); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(((PyObject *)__pyx_t_3));
+  __Pyx_INCREF(__pyx_v_srow);
+  PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_srow);
+  __Pyx_GIVEREF(__pyx_v_srow);
+  __Pyx_INCREF(__pyx_v_scol);
+  PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_v_scol);
+  __Pyx_GIVEREF(__pyx_v_scol);
+  __Pyx_INCREF(__pyx_v_erow);
+  PyTuple_SET_ITEM(__pyx_t_3, 2, __pyx_v_erow);
+  __Pyx_GIVEREF(__pyx_v_erow);
+  __Pyx_INCREF(__pyx_v_ecol);
+  PyTuple_SET_ITEM(__pyx_t_3, 3, __pyx_v_ecol);
+  __Pyx_GIVEREF(__pyx_v_ecol);
+  PyTuple_SET_ITEM(__pyx_t_3, 4, __pyx_t_2);
+  __Pyx_GIVEREF(__pyx_t_2);
+  PyTuple_SET_ITEM(__pyx_t_3, 5, __pyx_t_1);
+  __Pyx_GIVEREF(__pyx_t_1);
+  __pyx_t_2 = 0;
+  __pyx_t_1 = 0;
+  __pyx_t_1 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_6), ((PyObject *)__pyx_t_3)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 154; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(((PyObject *)__pyx_t_1));
+  __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0;
+  if (__Pyx_PrintOne(0, ((PyObject *)__pyx_t_1)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 154; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_XDECREF(__pyx_t_3);
+  __Pyx_AddTraceback("tokenize.printtoken");
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_DECREF(__pyx_v_srow);
+  __Pyx_DECREF(__pyx_v_scol);
+  __Pyx_DECREF(__pyx_v_erow);
+  __Pyx_DECREF(__pyx_v_ecol);
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "tokenize.pyx":157
+ *         (srow, scol, erow, ecol, tok_name[type], repr(token))
+ * 
+ * def tokenize(readline, tokeneater=printtoken):             # <<<<<<<<<<<<<<
+ *     """
+ *     The tokenize() function accepts two parameters: one representing the
+ */
+
+static PyObject *__pyx_pf_8tokenize_4tokenize(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static char __pyx_doc_8tokenize_4tokenize[] = "\n    The tokenize() function accepts two parameters: one representing the\n    input stream, and one providing an output mechanism for tokenize().\n\n    The first parameter, readline, must be a callable object which provides\n    the same interface as the readline() method of built-in file objects.\n    Each call to the function should return one line of input as a string.\n\n    The second parameter, tokeneater, must also be a callable object. It is\n    called once for each token, with five arguments, corresponding to the\n    tuples generated by generate_tokens().\n    ";
+static PyMethodDef __pyx_mdef_8tokenize_4tokenize = {__Pyx_NAMESTR("tokenize"), (PyCFunction)__pyx_pf_8tokenize_4tokenize, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(__pyx_doc_8tokenize_4tokenize)};
+static PyObject *__pyx_pf_8tokenize_4tokenize(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  PyObject *__pyx_v_readline = 0;
+  PyObject *__pyx_v_tokeneater = 0;
+  PyObject *__pyx_r = NULL;
+  PyObject *__pyx_t_1 = NULL;
+  PyObject *__pyx_t_2 = NULL;
+  PyObject *__pyx_t_3 = NULL;
+  int __pyx_t_4;
+  static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__readline,&__pyx_n_s__tokeneater,0};
+  __Pyx_RefNannySetupContext("tokenize");
+  __pyx_self = __pyx_self;
+  if (unlikely(__pyx_kwds)) {
+    Py_ssize_t kw_args = PyDict_Size(__pyx_kwds);
+    PyObject* values[2] = {0,0};
+    values[1] = __pyx_k_7;
+    switch (PyTuple_GET_SIZE(__pyx_args)) {
+      case  2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
+      case  1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+      case  0: break;
+      default: goto __pyx_L5_argtuple_error;
+    }
+    switch (PyTuple_GET_SIZE(__pyx_args)) {
+      case  0:
+      values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__readline);
+      if (likely(values[0])) kw_args--;
+      else goto __pyx_L5_argtuple_error;
+      case  1:
+      if (kw_args > 0) {
+        PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__tokeneater);
+        if (value) { values[1] = value; kw_args--; }
+      }
+    }
+    if (unlikely(kw_args > 0)) {
+      if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "tokenize") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 157; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+    }
+    __pyx_v_readline = values[0];
+    __pyx_v_tokeneater = values[1];
+  } else {
+    __pyx_v_tokeneater = __pyx_k_7;
+    switch (PyTuple_GET_SIZE(__pyx_args)) {
+      case  2: __pyx_v_tokeneater = PyTuple_GET_ITEM(__pyx_args, 1);
+      case  1: __pyx_v_readline = PyTuple_GET_ITEM(__pyx_args, 0);
+      break;
+      default: goto __pyx_L5_argtuple_error;
+    }
+  }
+  goto __pyx_L4_argument_unpacking_done;
+  __pyx_L5_argtuple_error:;
+  __Pyx_RaiseArgtupleInvalid("tokenize", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 157; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __pyx_L3_error:;
+  __Pyx_AddTraceback("tokenize.tokenize");
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+  __pyx_L4_argument_unpacking_done:;
+
+  /* "tokenize.pyx":170
+ *     tuples generated by generate_tokens().
+ *     """
+ *     try:             # <<<<<<<<<<<<<<
+ *         tokenize_loop(readline, tokeneater)
+ *     except StopTokenizing:
+ */
+  {
+    PyObject *__pyx_save_exc_type, *__pyx_save_exc_value, *__pyx_save_exc_tb;
+    __Pyx_ExceptionSave(&__pyx_save_exc_type, &__pyx_save_exc_value, &__pyx_save_exc_tb);
+    __Pyx_XGOTREF(__pyx_save_exc_type);
+    __Pyx_XGOTREF(__pyx_save_exc_value);
+    __Pyx_XGOTREF(__pyx_save_exc_tb);
+    /*try:*/ {
+
+      /* "tokenize.pyx":171
+ *     """
+ *     try:
+ *         tokenize_loop(readline, tokeneater)             # <<<<<<<<<<<<<<
+ *     except StopTokenizing:
+ *         pass
+ */
+      __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__tokenize_loop); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L6_error;}
+      __Pyx_GOTREF(__pyx_t_1);
+      __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L6_error;}
+      __Pyx_GOTREF(((PyObject *)__pyx_t_2));
+      __Pyx_INCREF(__pyx_v_readline);
+      PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_readline);
+      __Pyx_GIVEREF(__pyx_v_readline);
+      __Pyx_INCREF(__pyx_v_tokeneater);
+      PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_tokeneater);
+      __Pyx_GIVEREF(__pyx_v_tokeneater);
+      __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L6_error;}
+      __Pyx_GOTREF(__pyx_t_3);
+      __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+      __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+      __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+    }
+    __Pyx_XDECREF(__pyx_save_exc_type); __pyx_save_exc_type = 0;
+    __Pyx_XDECREF(__pyx_save_exc_value); __pyx_save_exc_value = 0;
+    __Pyx_XDECREF(__pyx_save_exc_tb); __pyx_save_exc_tb = 0;
+    goto __pyx_L13_try_end;
+    __pyx_L6_error:;
+    __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
+    __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
+    __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
+
+    /* "tokenize.pyx":172
+ *     try:
+ *         tokenize_loop(readline, tokeneater)
+ *     except StopTokenizing:             # <<<<<<<<<<<<<<
+ *         pass
+ * 
+ */
+    __pyx_t_3 = __Pyx_GetName(__pyx_m, __pyx_n_s__StopTokenizing); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L8_except_error;}
+    __Pyx_GOTREF(__pyx_t_3);
+    __pyx_t_4 = PyErr_ExceptionMatches(__pyx_t_3);
+    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+    if (__pyx_t_4) {
+      PyErr_Restore(0,0,0);
+      goto __pyx_L7_exception_handled;
+    }
+    __pyx_L8_except_error:;
+    __Pyx_XGIVEREF(__pyx_save_exc_type);
+    __Pyx_XGIVEREF(__pyx_save_exc_value);
+    __Pyx_XGIVEREF(__pyx_save_exc_tb);
+    __Pyx_ExceptionReset(__pyx_save_exc_type, __pyx_save_exc_value, __pyx_save_exc_tb);
+    goto __pyx_L1_error;
+    __pyx_L7_exception_handled:;
+    __Pyx_XGIVEREF(__pyx_save_exc_type);
+    __Pyx_XGIVEREF(__pyx_save_exc_value);
+    __Pyx_XGIVEREF(__pyx_save_exc_tb);
+    __Pyx_ExceptionReset(__pyx_save_exc_type, __pyx_save_exc_value, __pyx_save_exc_tb);
+    __pyx_L13_try_end:;
+  }
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_XDECREF(__pyx_t_3);
+  __Pyx_AddTraceback("tokenize.tokenize");
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "tokenize.pyx":176
+ * 
+ * # backwards compatible interface
+ * def tokenize_loop(readline, tokeneater):             # <<<<<<<<<<<<<<
+ *     for token_info in generate_tokens(readline):
+ *         tokeneater(*token_info)
+ */
+
+static PyObject *__pyx_pf_8tokenize_5tokenize_loop(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static PyMethodDef __pyx_mdef_8tokenize_5tokenize_loop = {__Pyx_NAMESTR("tokenize_loop"), (PyCFunction)__pyx_pf_8tokenize_5tokenize_loop, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)};
+static PyObject *__pyx_pf_8tokenize_5tokenize_loop(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  PyObject *__pyx_v_readline = 0;
+  PyObject *__pyx_v_tokeneater = 0;
+  PyObject *__pyx_v_token_info;
+  PyObject *__pyx_r = NULL;
+  Py_ssize_t __pyx_t_1;
+  PyObject *__pyx_t_2 = NULL;
+  PyObject *__pyx_t_3 = NULL;
+  PyObject *__pyx_t_4 = NULL;
+  static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__readline,&__pyx_n_s__tokeneater,0};
+  __Pyx_RefNannySetupContext("tokenize_loop");
+  __pyx_self = __pyx_self;
+  if (unlikely(__pyx_kwds)) {
+    Py_ssize_t kw_args = PyDict_Size(__pyx_kwds);
+    PyObject* values[2] = {0,0};
+    switch (PyTuple_GET_SIZE(__pyx_args)) {
+      case  2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
+      case  1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+      case  0: break;
+      default: goto __pyx_L5_argtuple_error;
+    }
+    switch (PyTuple_GET_SIZE(__pyx_args)) {
+      case  0:
+      values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__readline);
+      if (likely(values[0])) kw_args--;
+      else goto __pyx_L5_argtuple_error;
+      case  1:
+      values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__tokeneater);
+      if (likely(values[1])) kw_args--;
+      else {
+        __Pyx_RaiseArgtupleInvalid("tokenize_loop", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 176; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+      }
+    }
+    if (unlikely(kw_args > 0)) {
+      if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "tokenize_loop") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 176; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+    }
+    __pyx_v_readline = values[0];
+    __pyx_v_tokeneater = values[1];
+  } else if (PyTuple_GET_SIZE(__pyx_args) != 2) {
+    goto __pyx_L5_argtuple_error;
+  } else {
+    __pyx_v_readline = PyTuple_GET_ITEM(__pyx_args, 0);
+    __pyx_v_tokeneater = PyTuple_GET_ITEM(__pyx_args, 1);
+  }
+  goto __pyx_L4_argument_unpacking_done;
+  __pyx_L5_argtuple_error:;
+  __Pyx_RaiseArgtupleInvalid("tokenize_loop", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 176; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __pyx_L3_error:;
+  __Pyx_AddTraceback("tokenize.tokenize_loop");
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+  __pyx_L4_argument_unpacking_done:;
+  __pyx_v_token_info = Py_None; __Pyx_INCREF(Py_None);
+
+  /* "tokenize.pyx":177
+ * # backwards compatible interface
+ * def tokenize_loop(readline, tokeneater):
+ *     for token_info in generate_tokens(readline):             # <<<<<<<<<<<<<<
+ *         tokeneater(*token_info)
+ * 
+ */
+  __pyx_t_2 = __Pyx_GetName(__pyx_m, __pyx_n_s__generate_tokens); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 177; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 177; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(((PyObject *)__pyx_t_3));
+  __Pyx_INCREF(__pyx_v_readline);
+  PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_readline);
+  __Pyx_GIVEREF(__pyx_v_readline);
+  __pyx_t_4 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 177; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_4);
+  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+  __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0;
+  if (PyList_CheckExact(__pyx_t_4) || PyTuple_CheckExact(__pyx_t_4)) {
+    __pyx_t_1 = 0; __pyx_t_3 = __pyx_t_4; __Pyx_INCREF(__pyx_t_3);
+  } else {
+    __pyx_t_1 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_t_4); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 177; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_3);
+  }
+  __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+  for (;;) {
+    if (likely(PyList_CheckExact(__pyx_t_3))) {
+      if (__pyx_t_1 >= PyList_GET_SIZE(__pyx_t_3)) break;
+      __pyx_t_4 = PyList_GET_ITEM(__pyx_t_3, __pyx_t_1); __Pyx_INCREF(__pyx_t_4); __pyx_t_1++;
+    } else if (likely(PyTuple_CheckExact(__pyx_t_3))) {
+      if (__pyx_t_1 >= PyTuple_GET_SIZE(__pyx_t_3)) break;
+      __pyx_t_4 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_1); __Pyx_INCREF(__pyx_t_4); __pyx_t_1++;
+    } else {
+      __pyx_t_4 = PyIter_Next(__pyx_t_3);
+      if (!__pyx_t_4) {
+        if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 177; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        break;
+      }
+      __Pyx_GOTREF(__pyx_t_4);
+    }
+    __Pyx_DECREF(__pyx_v_token_info);
+    __pyx_v_token_info = __pyx_t_4;
+    __pyx_t_4 = 0;
+
+    /* "tokenize.pyx":178
+ * def tokenize_loop(readline, tokeneater):
+ *     for token_info in generate_tokens(readline):
+ *         tokeneater(*token_info)             # <<<<<<<<<<<<<<
+ * 
+ * class Untokenizer:
+ */
+    __pyx_t_4 = PySequence_Tuple(__pyx_v_token_info); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 178; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(((PyObject *)__pyx_t_4));
+    __pyx_t_2 = PyObject_Call(__pyx_v_tokeneater, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 178; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_2);
+    __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0;
+    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+  }
+  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_XDECREF(__pyx_t_3);
+  __Pyx_XDECREF(__pyx_t_4);
+  __Pyx_AddTraceback("tokenize.tokenize_loop");
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_DECREF(__pyx_v_token_info);
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "tokenize.pyx":182
+ * class Untokenizer:
+ * 
+ *     def __init__(self):             # <<<<<<<<<<<<<<
+ *         self.tokens = []
+ *         self.prev_row = 1
+ */
+
+static PyObject *__pyx_pf_8tokenize_11Untokenizer___init__(PyObject *__pyx_self, PyObject *__pyx_v_self); /*proto*/
+static PyMethodDef __pyx_mdef_8tokenize_11Untokenizer___init__ = {__Pyx_NAMESTR("__init__"), (PyCFunction)__pyx_pf_8tokenize_11Untokenizer___init__, METH_O, __Pyx_DOCSTR(0)};
+static PyObject *__pyx_pf_8tokenize_11Untokenizer___init__(PyObject *__pyx_self, PyObject *__pyx_v_self) {
+  PyObject *__pyx_r = NULL;
+  PyObject *__pyx_t_1 = NULL;
+  __Pyx_RefNannySetupContext("__init__");
+  __pyx_self = __pyx_self;
+
+  /* "tokenize.pyx":183
+ * 
+ *     def __init__(self):
+ *         self.tokens = []             # <<<<<<<<<<<<<<
+ *         self.prev_row = 1
+ *         self.prev_col = 0
+ */
+  __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 183; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(((PyObject *)__pyx_t_1));
+  if (PyObject_SetAttr(__pyx_v_self, __pyx_n_s__tokens, ((PyObject *)__pyx_t_1)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 183; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
+
+  /* "tokenize.pyx":184
+ *     def __init__(self):
+ *         self.tokens = []
+ *         self.prev_row = 1             # <<<<<<<<<<<<<<
+ *         self.prev_col = 0
+ * 
+ */
+  if (PyObject_SetAttr(__pyx_v_self, __pyx_n_s__prev_row, __pyx_int_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 184; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+
+  /* "tokenize.pyx":185
+ *         self.tokens = []
+ *         self.prev_row = 1
+ *         self.prev_col = 0             # <<<<<<<<<<<<<<
+ * 
+ *     def add_whitespace(self, start):
+ */
+  if (PyObject_SetAttr(__pyx_v_self, __pyx_n_s__prev_col, __pyx_int_0) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_AddTraceback("tokenize.Untokenizer.__init__");
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "tokenize.pyx":187
+ *         self.prev_col = 0
+ * 
+ *     def add_whitespace(self, start):             # <<<<<<<<<<<<<<
+ *         row, col = start
+ *         assert row <= self.prev_row
+ */
+
+static PyObject *__pyx_pf_8tokenize_11Untokenizer_1add_whitespace(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static PyMethodDef __pyx_mdef_8tokenize_11Untokenizer_1add_whitespace = {__Pyx_NAMESTR("add_whitespace"), (PyCFunction)__pyx_pf_8tokenize_11Untokenizer_1add_whitespace, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)};
+static PyObject *__pyx_pf_8tokenize_11Untokenizer_1add_whitespace(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  PyObject *__pyx_v_self = 0;
+  PyObject *__pyx_v_start = 0;
+  PyObject *__pyx_v_row;
+  PyObject *__pyx_v_col;
+  PyObject *__pyx_v_col_offset;
+  PyObject *__pyx_r = NULL;
+  PyObject *__pyx_t_1 = NULL;
+  PyObject *__pyx_t_2 = NULL;
+  PyObject *__pyx_t_3 = NULL;
+  int __pyx_t_4;
+  static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__self,&__pyx_n_s__start,0};
+  __Pyx_RefNannySetupContext("add_whitespace");
+  __pyx_self = __pyx_self;
+  if (unlikely(__pyx_kwds)) {
+    Py_ssize_t kw_args = PyDict_Size(__pyx_kwds);
+    PyObject* values[2] = {0,0};
+    switch (PyTuple_GET_SIZE(__pyx_args)) {
+      case  2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
+      case  1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+      case  0: break;
+      default: goto __pyx_L5_argtuple_error;
+    }
+    switch (PyTuple_GET_SIZE(__pyx_args)) {
+      case  0:
+      values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__self);
+      if (likely(values[0])) kw_args--;
+      else goto __pyx_L5_argtuple_error;
+      case  1:
+      values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__start);
+      if (likely(values[1])) kw_args--;
+      else {
+        __Pyx_RaiseArgtupleInvalid("add_whitespace", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+      }
+    }
+    if (unlikely(kw_args > 0)) {
+      if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "add_whitespace") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+    }
+    __pyx_v_self = values[0];
+    __pyx_v_start = values[1];
+  } else if (PyTuple_GET_SIZE(__pyx_args) != 2) {
+    goto __pyx_L5_argtuple_error;
+  } else {
+    __pyx_v_self = PyTuple_GET_ITEM(__pyx_args, 0);
+    __pyx_v_start = PyTuple_GET_ITEM(__pyx_args, 1);
+  }
+  goto __pyx_L4_argument_unpacking_done;
+  __pyx_L5_argtuple_error:;
+  __Pyx_RaiseArgtupleInvalid("add_whitespace", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __pyx_L3_error:;
+  __Pyx_AddTraceback("tokenize.Untokenizer.add_whitespace");
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+  __pyx_L4_argument_unpacking_done:;
+  __pyx_v_row = Py_None; __Pyx_INCREF(Py_None);
+  __pyx_v_col = Py_None; __Pyx_INCREF(Py_None);
+  __pyx_v_col_offset = Py_None; __Pyx_INCREF(Py_None);
+
+  /* "tokenize.pyx":188
+ * 
+ *     def add_whitespace(self, start):
+ *         row, col = start             # <<<<<<<<<<<<<<
+ *         assert row <= self.prev_row
+ *         col_offset = col - self.prev_col
+ */
+  if (PyTuple_CheckExact(__pyx_v_start) && likely(PyTuple_GET_SIZE(__pyx_v_start) == 2)) {
+    PyObject* tuple = __pyx_v_start;
+    __pyx_t_1 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_t_1);
+    __pyx_t_2 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_t_2);
+    __Pyx_DECREF(__pyx_v_row);
+    __pyx_v_row = __pyx_t_1;
+    __pyx_t_1 = 0;
+    __Pyx_DECREF(__pyx_v_col);
+    __pyx_v_col = __pyx_t_2;
+    __pyx_t_2 = 0;
+  } else {
+    __pyx_t_3 = PyObject_GetIter(__pyx_v_start); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_3);
+    __pyx_t_1 = __Pyx_UnpackItem(__pyx_t_3, 0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_1);
+    __pyx_t_2 = __Pyx_UnpackItem(__pyx_t_3, 1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_2);
+    if (__Pyx_EndUnpack(__pyx_t_3, 2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+    __Pyx_DECREF(__pyx_v_row);
+    __pyx_v_row = __pyx_t_1;
+    __pyx_t_1 = 0;
+    __Pyx_DECREF(__pyx_v_col);
+    __pyx_v_col = __pyx_t_2;
+    __pyx_t_2 = 0;
+  }
+
+  /* "tokenize.pyx":189
+ *     def add_whitespace(self, start):
+ *         row, col = start
+ *         assert row <= self.prev_row             # <<<<<<<<<<<<<<
+ *         col_offset = col - self.prev_col
+ *         if col_offset:
+ */
+  #ifndef CYTHON_WITHOUT_ASSERTIONS
+  __pyx_t_2 = PyObject_GetAttr(__pyx_v_self, __pyx_n_s__prev_row); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 189; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __pyx_t_1 = PyObject_RichCompare(__pyx_v_row, __pyx_t_2, Py_LE); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 189; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+  __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 189; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  if (unlikely(!__pyx_t_4)) {
+    PyErr_SetNone(PyExc_AssertionError);
+    {__pyx_filename = __pyx_f[0]; __pyx_lineno = 189; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  }
+  #endif
+
+  /* "tokenize.pyx":190
+ *         row, col = start
+ *         assert row <= self.prev_row
+ *         col_offset = col - self.prev_col             # <<<<<<<<<<<<<<
+ *         if col_offset:
+ *             self.tokens.append(" " * col_offset)
+ */
+  __pyx_t_1 = PyObject_GetAttr(__pyx_v_self, __pyx_n_s__prev_col); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_t_2 = PyNumber_Subtract(__pyx_v_col, __pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  __Pyx_DECREF(__pyx_v_col_offset);
+  __pyx_v_col_offset = __pyx_t_2;
+  __pyx_t_2 = 0;
+
+  /* "tokenize.pyx":191
+ *         assert row <= self.prev_row
+ *         col_offset = col - self.prev_col
+ *         if col_offset:             # <<<<<<<<<<<<<<
+ *             self.tokens.append(" " * col_offset)
+ * 
+ */
+  __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_col_offset); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (__pyx_t_4) {
+
+    /* "tokenize.pyx":192
+ *         col_offset = col - self.prev_col
+ *         if col_offset:
+ *             self.tokens.append(" " * col_offset)             # <<<<<<<<<<<<<<
+ * 
+ *     def untokenize(self, iterable):
+ */
+    __pyx_t_2 = PyObject_GetAttr(__pyx_v_self, __pyx_n_s__tokens); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_2);
+    __pyx_t_1 = PyNumber_Multiply(((PyObject *)__pyx_kp_s_8), __pyx_v_col_offset); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(((PyObject *)__pyx_t_1));
+    __pyx_t_3 = __Pyx_PyObject_Append(__pyx_t_2, ((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_3);
+    __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+    __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
+    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+    goto __pyx_L6;
+  }
+  __pyx_L6:;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_XDECREF(__pyx_t_3);
+  __Pyx_AddTraceback("tokenize.Untokenizer.add_whitespace");
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_DECREF(__pyx_v_row);
+  __Pyx_DECREF(__pyx_v_col);
+  __Pyx_DECREF(__pyx_v_col_offset);
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "tokenize.pyx":194
+ *             self.tokens.append(" " * col_offset)
+ * 
+ *     def untokenize(self, iterable):             # <<<<<<<<<<<<<<
+ *         for t in iterable:
+ *             if len(t) == 2:
+ */
+
+static PyObject *__pyx_pf_8tokenize_11Untokenizer_2untokenize(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static PyMethodDef __pyx_mdef_8tokenize_11Untokenizer_2untokenize = {__Pyx_NAMESTR("untokenize"), (PyCFunction)__pyx_pf_8tokenize_11Untokenizer_2untokenize, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)};
+static PyObject *__pyx_pf_8tokenize_11Untokenizer_2untokenize(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  PyObject *__pyx_v_self = 0;
+  PyObject *__pyx_v_iterable = 0;
+  PyObject *__pyx_v_t;
+  PyObject *__pyx_v_tok_type;
+  PyObject *__pyx_v_token;
+  PyObject *__pyx_v_start;
+  PyObject *__pyx_v_end;
+  PyObject *__pyx_v_line;
+  PyObject *__pyx_r = NULL;
+  Py_ssize_t __pyx_t_1;
+  PyObject *__pyx_t_2 = NULL;
+  PyObject *__pyx_t_3 = NULL;
+  Py_ssize_t __pyx_t_4;
+  int __pyx_t_5;
+  PyObject *__pyx_t_6 = NULL;
+  PyObject *__pyx_t_7 = NULL;
+  PyObject *__pyx_t_8 = NULL;
+  PyObject *__pyx_t_9 = NULL;
+  PyObject *__pyx_t_10 = NULL;
+  int __pyx_t_11;
+  int __pyx_t_12;
+  static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__self,&__pyx_n_s__iterable,0};
+  __Pyx_RefNannySetupContext("untokenize");
+  __pyx_self = __pyx_self;
+  if (unlikely(__pyx_kwds)) {
+    Py_ssize_t kw_args = PyDict_Size(__pyx_kwds);
+    PyObject* values[2] = {0,0};
+    switch (PyTuple_GET_SIZE(__pyx_args)) {
+      case  2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
+      case  1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+      case  0: break;
+      default: goto __pyx_L5_argtuple_error;
+    }
+    switch (PyTuple_GET_SIZE(__pyx_args)) {
+      case  0:
+      values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__self);
+      if (likely(values[0])) kw_args--;
+      else goto __pyx_L5_argtuple_error;
+      case  1:
+      values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__iterable);
+      if (likely(values[1])) kw_args--;
+      else {
+        __Pyx_RaiseArgtupleInvalid("untokenize", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 194; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+      }
+    }
+    if (unlikely(kw_args > 0)) {
+      if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "untokenize") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 194; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+    }
+    __pyx_v_self = values[0];
+    __pyx_v_iterable = values[1];
+  } else if (PyTuple_GET_SIZE(__pyx_args) != 2) {
+    goto __pyx_L5_argtuple_error;
+  } else {
+    __pyx_v_self = PyTuple_GET_ITEM(__pyx_args, 0);
+    __pyx_v_iterable = PyTuple_GET_ITEM(__pyx_args, 1);
+  }
+  goto __pyx_L4_argument_unpacking_done;
+  __pyx_L5_argtuple_error:;
+  __Pyx_RaiseArgtupleInvalid("untokenize", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 194; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __pyx_L3_error:;
+  __Pyx_AddTraceback("tokenize.Untokenizer.untokenize");
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+  __pyx_L4_argument_unpacking_done:;
+  __pyx_v_t = Py_None; __Pyx_INCREF(Py_None);
+  __pyx_v_tok_type = Py_None; __Pyx_INCREF(Py_None);
+  __pyx_v_token = Py_None; __Pyx_INCREF(Py_None);
+  __pyx_v_start = Py_None; __Pyx_INCREF(Py_None);
+  __pyx_v_end = Py_None; __Pyx_INCREF(Py_None);
+  __pyx_v_line = Py_None; __Pyx_INCREF(Py_None);
+
+  /* "tokenize.pyx":195
+ * 
+ *     def untokenize(self, iterable):
+ *         for t in iterable:             # <<<<<<<<<<<<<<
+ *             if len(t) == 2:
+ *                 self.compat(t, iterable)
+ */
+  if (PyList_CheckExact(__pyx_v_iterable) || PyTuple_CheckExact(__pyx_v_iterable)) {
+    __pyx_t_1 = 0; __pyx_t_2 = __pyx_v_iterable; __Pyx_INCREF(__pyx_t_2);
+  } else {
+    __pyx_t_1 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_iterable); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_2);
+  }
+  for (;;) {
+    if (likely(PyList_CheckExact(__pyx_t_2))) {
+      if (__pyx_t_1 >= PyList_GET_SIZE(__pyx_t_2)) break;
+      __pyx_t_3 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_1); __Pyx_INCREF(__pyx_t_3); __pyx_t_1++;
+    } else if (likely(PyTuple_CheckExact(__pyx_t_2))) {
+      if (__pyx_t_1 >= PyTuple_GET_SIZE(__pyx_t_2)) break;
+      __pyx_t_3 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_1); __Pyx_INCREF(__pyx_t_3); __pyx_t_1++;
+    } else {
+      __pyx_t_3 = PyIter_Next(__pyx_t_2);
+      if (!__pyx_t_3) {
+        if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        break;
+      }
+      __Pyx_GOTREF(__pyx_t_3);
+    }
+    __Pyx_DECREF(__pyx_v_t);
+    __pyx_v_t = __pyx_t_3;
+    __pyx_t_3 = 0;
+
+    /* "tokenize.pyx":196
+ *     def untokenize(self, iterable):
+ *         for t in iterable:
+ *             if len(t) == 2:             # <<<<<<<<<<<<<<
+ *                 self.compat(t, iterable)
+ *                 break
+ */
+    __pyx_t_4 = PyObject_Length(__pyx_v_t); if (unlikely(__pyx_t_4 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 196; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_5 = (__pyx_t_4 == 2);
+    if (__pyx_t_5) {
+
+      /* "tokenize.pyx":197
+ *         for t in iterable:
+ *             if len(t) == 2:
+ *                 self.compat(t, iterable)             # <<<<<<<<<<<<<<
+ *                 break
+ *             tok_type, token, start, end, line = t
+ */
+      __pyx_t_3 = PyObject_GetAttr(__pyx_v_self, __pyx_n_s__compat); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_GOTREF(__pyx_t_3);
+      __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_GOTREF(((PyObject *)__pyx_t_6));
+      __Pyx_INCREF(__pyx_v_t);
+      PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_v_t);
+      __Pyx_GIVEREF(__pyx_v_t);
+      __Pyx_INCREF(__pyx_v_iterable);
+      PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_v_iterable);
+      __Pyx_GIVEREF(__pyx_v_iterable);
+      __pyx_t_7 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_GOTREF(__pyx_t_7);
+      __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+      __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0;
+      __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
+
+      /* "tokenize.pyx":198
+ *             if len(t) == 2:
+ *                 self.compat(t, iterable)
+ *                 break             # <<<<<<<<<<<<<<
+ *             tok_type, token, start, end, line = t
+ *             self.add_whitespace(start)
+ */
+      goto __pyx_L7_break;
+      goto __pyx_L8;
+    }
+    __pyx_L8:;
+
+    /* "tokenize.pyx":199
+ *                 self.compat(t, iterable)
+ *                 break
+ *             tok_type, token, start, end, line = t             # <<<<<<<<<<<<<<
+ *             self.add_whitespace(start)
+ *             self.tokens.append(token)
+ */
+    if (PyTuple_CheckExact(__pyx_v_t) && likely(PyTuple_GET_SIZE(__pyx_v_t) == 5)) {
+      PyObject* tuple = __pyx_v_t;
+      __pyx_t_7 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_t_7);
+      __pyx_t_6 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_t_6);
+      __pyx_t_3 = PyTuple_GET_ITEM(tuple, 2); __Pyx_INCREF(__pyx_t_3);
+      __pyx_t_8 = PyTuple_GET_ITEM(tuple, 3); __Pyx_INCREF(__pyx_t_8);
+      __pyx_t_9 = PyTuple_GET_ITEM(tuple, 4); __Pyx_INCREF(__pyx_t_9);
+      __Pyx_DECREF(__pyx_v_tok_type);
+      __pyx_v_tok_type = __pyx_t_7;
+      __pyx_t_7 = 0;
+      __Pyx_DECREF(__pyx_v_token);
+      __pyx_v_token = __pyx_t_6;
+      __pyx_t_6 = 0;
+      __Pyx_DECREF(__pyx_v_start);
+      __pyx_v_start = __pyx_t_3;
+      __pyx_t_3 = 0;
+      __Pyx_DECREF(__pyx_v_end);
+      __pyx_v_end = __pyx_t_8;
+      __pyx_t_8 = 0;
+      __Pyx_DECREF(__pyx_v_line);
+      __pyx_v_line = __pyx_t_9;
+      __pyx_t_9 = 0;
+    } else {
+      __pyx_t_10 = PyObject_GetIter(__pyx_v_t); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_GOTREF(__pyx_t_10);
+      __pyx_t_7 = __Pyx_UnpackItem(__pyx_t_10, 0); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_GOTREF(__pyx_t_7);
+      __pyx_t_6 = __Pyx_UnpackItem(__pyx_t_10, 1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_GOTREF(__pyx_t_6);
+      __pyx_t_3 = __Pyx_UnpackItem(__pyx_t_10, 2); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_GOTREF(__pyx_t_3);
+      __pyx_t_8 = __Pyx_UnpackItem(__pyx_t_10, 3); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_GOTREF(__pyx_t_8);
+      __pyx_t_9 = __Pyx_UnpackItem(__pyx_t_10, 4); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_GOTREF(__pyx_t_9);
+      if (__Pyx_EndUnpack(__pyx_t_10, 5) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
+      __Pyx_DECREF(__pyx_v_tok_type);
+      __pyx_v_tok_type = __pyx_t_7;
+      __pyx_t_7 = 0;
+      __Pyx_DECREF(__pyx_v_token);
+      __pyx_v_token = __pyx_t_6;
+      __pyx_t_6 = 0;
+      __Pyx_DECREF(__pyx_v_start);
+      __pyx_v_start = __pyx_t_3;
+      __pyx_t_3 = 0;
+      __Pyx_DECREF(__pyx_v_end);
+      __pyx_v_end = __pyx_t_8;
+      __pyx_t_8 = 0;
+      __Pyx_DECREF(__pyx_v_line);
+      __pyx_v_line = __pyx_t_9;
+      __pyx_t_9 = 0;
+    }
+
+    /* "tokenize.pyx":200
+ *                 break
+ *             tok_type, token, start, end, line = t
+ *             self.add_whitespace(start)             # <<<<<<<<<<<<<<
+ *             self.tokens.append(token)
+ *             self.prev_row, self.prev_col = end
+ */
+    __pyx_t_9 = PyObject_GetAttr(__pyx_v_self, __pyx_n_s__add_whitespace); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_9);
+    __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(((PyObject *)__pyx_t_8));
+    __Pyx_INCREF(__pyx_v_start);
+    PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v_start);
+    __Pyx_GIVEREF(__pyx_v_start);
+    __pyx_t_3 = PyObject_Call(__pyx_t_9, ((PyObject *)__pyx_t_8), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_3);
+    __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+    __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0;
+    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+
+    /* "tokenize.pyx":201
+ *             tok_type, token, start, end, line = t
+ *             self.add_whitespace(start)