Commits

Mark Shannon committed fa0ef7b

Working version of lowering interpreter. Approx x5 slowdown

Comments (0)

Files changed (18)

    call frame it setup.
 */
 #define CO_NOFREE       0x0040
+/* This function will not appear in backtraces (HotPy internal functions) */
+#define CO_INVISIBLE    0x0080
 
 /* These are no longer used. */
 #if 0
 
 #define CO_FUTURE_BARRY_AS_BDFL  0x40000
 
+
 /* This value is found in the co_cell2arg array when the associated cell
    variable does not correspond to an argument. The maximum number of
    arguments is 255 (indexed up to 254), so 255 work as a special flag.*/
 #define INPLACE_FLOOR_DIVIDE 28
 #define INPLACE_TRUE_DIVIDE 29
 
+/* HotPy opcodes */
+#define GET_CLASS_ATTR 33
+#define HAS_CLASS_ATTR 34
+#define BIND 35
+#define HAS_DICT 36
+#define GET_TYPE 37
+#define POP_CALLER_BLOCK 38
+#define RETURN_VALUE_IF_IN_OBJECT_DICT 39
+#define SET_IN_OBJECT_DICT 40
+#define GEN_ENTER 41
+#define GEN_SEND 42
+#define GEN_STARTED 43
+#define GEN_EXIT 44
+#define OVERRIDES 45
+#define RETURN_VOID 46
+#define C_BIND 47
+#define DESCRIPTOR_GET 48
+#define DESCRIPTOR_SET 49
+
 #define STORE_MAP       54
 #define INPLACE_ADD     55
 #define INPLACE_SUBTRACT        56
 
 #define LOAD_GLOBAL     116     /* Index in name list */
 
+/* HotPy opcodes */
+#define LOAD_SPECIAL 117
+#define HAS_SPECIAL 118
+
 #define CONTINUE_LOOP   119     /* Start of loop (absolute) */
 #define SETUP_LOOP      120     /* Target address (relative) */
 #define SETUP_EXCEPT    121     /* "" */
 #define STORE_FAST      125     /* Local variable number */
 #define DELETE_FAST     126     /* Local variable number */
 
+/* HotPy opcodes */
+#define LOAD_REGISTER   127     /* Index of register */
+#define STORE_REGISTER  128     /* Index of register */
+
 #define RAISE_VARARGS   130     /* Number of raise arguments (1, 2 or 3) */
 /* CALL_FUNCTION_XXX opcodes defined below depend on this definition */
 #define CALL_FUNCTION   131     /* #args + (#kwargs<<8) */
 
 #define MAKE_CLOSURE    134     /* same as MAKE_FUNCTION */
 #define LOAD_CLOSURE    135     /* Load free variable from closure */
-#define LOAD_DEREF      136     /* Load and dereference from closure cell */ 
-#define STORE_DEREF     137     /* Store into cell */ 
-#define DELETE_DEREF    138     /* Delete closure cell */ 
+#define LOAD_DEREF      136     /* Load and dereference from closure cell */
+#define STORE_DEREF     137     /* Store into cell */
+#define DELETE_DEREF    138     /* Delete closure cell */
 
 /* The next 3 opcodes must be contiguous and satisfy
    (CALL_FUNCTION_VAR - CALL_FUNCTION) & 3 == 1  */
 #define EXCEPT_HANDLER 257
 
 
-enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE, PyCmp_GT=Py_GT, PyCmp_GE=Py_GE,
-             PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};
+enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE,
+             PyCmp_GT=Py_GT, PyCmp_GE=Py_GE, PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS,
+             PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_SUB_TYPE, PyCmp_BAD};
 
 #define HAS_ARG(op) ((op) >= HAVE_ARGUMENT)
 

Include/pystate.h

 #ifdef Py_LIMITED_API
 typedef struct _ts PyThreadState;
 #else
+#define HOTPY_REGISTER_COUNT 32
+
 typedef struct _ts {
     /* See Python/ceval.c for comments explaining most fields */
 
     PyObject *async_exc; /* Asynchronous exception to raise */
     long thread_id; /* Thread id where this tstate was created */
 
+    PyObject *registers[HOTPY_REGISTER_COUNT];
     /* XXX signal handlers should also be here */
 
 } PyThreadState;
            "HAVE_ARGUMENT", "EXTENDED_ARG"]
 
 cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is',
-        'is not', 'exception match', 'BAD')
+        'is not', 'exception match', 'subtype', 'BAD')
 
 hasconst = []
 hasname = []
 def_op('INPLACE_FLOOR_DIVIDE', 28)
 def_op('INPLACE_TRUE_DIVIDE', 29)
 
+#HotPy low-level bytecodes
+def_op('GET_CLASS_ATTR', 33)
+def_op('HAS_CLASS_ATTR', 34)
+def_op('BIND', 35)
+def_op('HAS_DICT', 36)
+def_op('GET_TYPE', 37)
+def_op('POP_CALLER_BLOCK', 38)
+def_op('RETURN_VALUE_IF_IN_OBJECT_DICT', 39)
+def_op('SET_IN_OBJECT_DICT', 40)
+def_op('GEN_ENTER', 41)
+def_op('GEN_SEND', 42)
+def_op('GEN_STARTED', 43)
+def_op('GEN_EXIT', 44)
+def_op('OVERRIDES', 45)
+def_op('RETURN_VOID', 46)
+def_op('C_BIND', 47)
+def_op('DESCRIPTOR_GET', 48)
+def_op('DESCRIPTOR_SET', 49)
+
 def_op('STORE_MAP', 54)
 def_op('INPLACE_ADD', 55)
 def_op('INPLACE_SUBTRACT', 56)
 
 name_op('LOAD_GLOBAL', 116)     # Index in name list
 
+#HotPy low-level bytecodes
+name_op('LOAD_SPECIAL', 117)    # Index in name list
+name_op('HAS_SPECIAL', 118)     # Index in name list
+
 jabs_op('CONTINUE_LOOP', 119)   # Target address
 jrel_op('SETUP_LOOP', 120)      # Distance to target address
 jrel_op('SETUP_EXCEPT', 121)    # ""
 def_op('DELETE_FAST', 126)      # Local variable number
 haslocal.append(126)
 
+#HotPy low-level bytecodes
+name_op('LOAD_REGISTER', 127)    # Index of register
+name_op('STORE_REGISTER', 128)     # Index of register
+
 def_op('RAISE_VARARGS', 130)    # Number of raise arguments (1, 2, or 3)
 def_op('CALL_FUNCTION', 131)    # #args + (#kwargs << 8)
 def_op('MAKE_FUNCTION', 132)    # Number of args with default values

Lib/test/test_class.py

 
         class E:
             __eq__ = property(booh)
-        E() == E() # In debug mode, caused a C-level assert() to fail
+        try:
+            E() == E() # In debug mode, caused a C-level assert() to fail
+        except AttributeError:
+            pass
 
         class I:
             __init__ = property(booh)

Lib/test/test_sys.py

         def f():
             f()
         try:
-            for i in (50, 1000):
+            for i in (60, 1000):
                 # Issue #5392: stack overflow after hitting recursion limit twice
                 sys.setrecursionlimit(i)
                 self.assertRaises(RuntimeError, f)
         ncells = len(x.f_code.co_cellvars)
         nfrees = len(x.f_code.co_freevars)
         extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
-                  ncells + nfrees - 1
+                  ncells + nfrees - 1 + 1 #One extra for HotPy overhead.
         check(x, size(vh + '13P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
         # function
         def func(): pass

Objects/exceptions.c

     PyDoc_STR(EXCDOC), (traverseproc)BaseException_traverse, \
     (inquiry)BaseException_clear, 0, 0, 0, 0, 0, 0, 0, &_ ## EXCBASE, \
     0, 0, 0, offsetof(PyBaseExceptionObject, dict), \
-    (initproc)BaseException_init, 0, BaseException_new,\
+    (initproc)BaseException_init, 0, 0,\
 }; \
 PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME
 

Objects/frameobject.c

 #define OFF(x) offsetof(PyFrameObject, x)
 
 static PyMemberDef frame_memberlist[] = {
-    {"f_back",          T_OBJECT,       OFF(f_back),      READONLY},
     {"f_code",          T_OBJECT,       OFF(f_code),      READONLY},
     {"f_builtins",      T_OBJECT,       OFF(f_builtins),  READONLY},
     {"f_globals",       T_OBJECT,       OFF(f_globals),   READONLY},
 };
 
 static PyObject *
+frame_getf_back(PyFrameObject *f, void *closure)
+{
+    PyFrameObject *back = f->f_back;
+    while (back != NULL && (back->f_code->co_flags & CO_INVISIBLE))
+        back = back->f_back;
+    if (back == NULL)
+        Py_RETURN_NONE;
+    Py_INCREF(back);
+    return (PyObject *)back;
+}
+
+
+
+static PyObject *
 frame_getlocals(PyFrameObject *f, void *closure)
 {
     PyFrame_FastToLocals(f);
 
 
 static PyGetSetDef frame_getsetlist[] = {
+    {"f_back",          (getter)frame_getf_back, NULL, NULL},
     {"f_locals",        (getter)frame_getlocals, NULL, NULL},
     {"f_lineno",        (getter)frame_getlineno,
                     (setter)frame_setlineno, NULL},
 
     ncells = PyTuple_GET_SIZE(f->f_code->co_cellvars);
     nfrees = PyTuple_GET_SIZE(f->f_code->co_freevars);
-    extras = f->f_code->co_stacksize + f->f_code->co_nlocals +
+    extras = f->f_code->co_stacksize + 1 + f->f_code->co_nlocals +
              ncells + nfrees;
     /* subtract one as it is already included in PyFrameObject */
     res = sizeof(PyFrameObject) + (extras-1) * sizeof(PyObject *);
         Py_ssize_t extras, ncells, nfrees;
         ncells = PyTuple_GET_SIZE(code->co_cellvars);
         nfrees = PyTuple_GET_SIZE(code->co_freevars);
-        extras = code->co_stacksize + code->co_nlocals + ncells +
+        /* +1 here is to make a little headroom fro HotPy transforms */
+        extras = code->co_stacksize + 1 + code->co_nlocals + ncells +
             nfrees;
         if (free_list == NULL) {
             f = PyObject_GC_NewVar(PyFrameObject, &PyFrame_Type,

Objects/genobject.c

     assert(f->f_back == tstate->frame);
     Py_CLEAR(f->f_back);
 
-    /* If the generator just returned (as opposed to yielding), signal
-     * that the generator is exhausted. */
-    if (result && f->f_stacktop == NULL) {
-        if (result == Py_None) {
-            /* Delay exception instantiation if we can */
-            PyErr_SetNone(PyExc_StopIteration);
-        } else {
-            PyObject *e = PyStopIteration_Create(result);
-            if (e != NULL) {
-                PyErr_SetObject(PyExc_StopIteration, e);
-                Py_DECREF(e);
-            }
-        }
-        Py_CLEAR(result);
-    }
-
-    if (!result || f->f_stacktop == NULL) {
+    if (!result) {
         /* generator can't be rerun, so release the frame */
         /* first clean reference cycle through stored exception traceback */
         PyObject *t, *v, *tb;
 gen_close_iter(PyObject *yf)
 {
     PyObject *retval = NULL;
-    
+
     if (PyGen_CheckExact(yf)) {
         retval = gen_close((PyGenObject *)yf, NULL);
         if (retval == NULL) {
     }
     Py_XDECREF(retval);
     return 0;
-}       
+}
 
 static PyObject *
 gen_close(PyGenObject *gen, PyObject *args)
 PyGen_FetchStopIterationValue(PyObject **pvalue) {
     PyObject *et, *ev, *tb;
     PyObject *value = NULL;
-    
+
     if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
         PyErr_Fetch(&et, &ev, &tb);
         Py_XDECREF(et);
 
     /* Any block type besides a loop requires cleanup. */
     for (i = 0; i < f->f_iblock; i++)
-        if (f->f_blockstack[i].b_type != SETUP_LOOP)
+        if (f->f_blockstack[i].b_type != SETUP_LOOP &&
+            f->f_blockstack[i].b_type != FOR_ITER)
             return 1;
 
     /* No blocks except loops, it's safe to skip finalization. */
 PyObject_Print(PyObject *op, FILE *fp, int flags)
 {
     int ret = 0;
+    PyObject *type, *value, *traceback;
     if (PyErr_CheckSignals())
         return -1;
+    PyErr_Fetch(&type, &value, &traceback);
 #ifdef USE_STACKCHECK
     if (PyOS_CheckStack()) {
         PyErr_SetString(PyExc_MemoryError, "stack overflow");
             ret = -1;
         }
     }
+    PyErr_Restore(type, value, traceback);
     return ret;
 }
 
     if (PyType_Ready(&PyEllipsis_Type) < 0)
         Py_FatalError("Can't initialize ellipsis type");
 
+    if (PyType_Ready(&PyMethodDescr_Type) < 0)
+        Py_FatalError("Can't initialize method descriptor type");
+
     if (PyType_Ready(&PyMemberDescr_Type) < 0)
         Py_FatalError("Can't initialize member descriptor type");
 
+    if (PyType_Ready(&PyClassMethodDescr_Type) < 0)
+        Py_FatalError("Can't initialize class-method descriptor type");
+
     if (PyType_Ready(&PyFilter_Type) < 0)
         Py_FatalError("Can't initialize filter type");
 

Objects/typeobject.c

     if (type == NULL) {
         /* Call super(), without args -- fill in from __class__
            and first local variable on the stack. */
-        PyFrameObject *f = PyThreadState_GET()->frame;
+        PyFrameObject *f = _PyThreadState_GetFrame(PyThreadState_GET());
         PyCodeObject *co = f->f_code;
         Py_ssize_t i, n;
         if (co == NULL) {

Python/_warnings.c

     PyObject *globals;
 
     /* Setup globals and lineno. */
-    PyFrameObject *f = PyThreadState_GET()->frame;
-    while (--stack_level > 0 && f != NULL)
+    PyFrameObject *f = _PyThreadState_GetFrame(PyThreadState_GET());
+    while (f != NULL &&
+          ((f->f_code->co_flags & CO_INVISIBLE) ||  --stack_level > 0))
+        f = f->f_back;
+
+    while (f != NULL && (f->f_code->co_flags & CO_INVISIBLE))
         f = f->f_back;
 
     if (f == NULL) {

Python/binaryoperators.c

         binary_func_count++;
         _Py_binary_functions[hash & MASK] = f;
     }
-    assert(hash & MASK] != 0);
+    assert((hash & MASK) != 0);
     assert(_Py_binary_functions[hash & MASK] == f);
     return hash & MASK;
 }
                                       PyObject *);
 static PyObject * update_star_args(int, int, PyObject *, PyObject ***);
 static PyObject * load_args(PyObject ***, int);
+static PyFrameObject *fast_frame(PyObject *, PyObject **, int);
+static PyFrameObject *make_frame(PyObject *, PyObject *, PyObject *);
+static PyObject *build_parameters(PyObject *, PyObject ***, int , int, int, PyObject**);
+static PyObject *make_traced_c_call(PyThreadState*, PyObject*, PyObject*, PyObject*);
+
 #define CALL_FLAG_VAR 1
 #define CALL_FLAG_KW 2
 
    fast_next_opcode*/
 static int _Py_TracingPossible = 0;
 
-
+static PyObject _VoidStruct = {
+    _PyObject_EXTRA_INIT
+    1, &PyBaseObject_Type
+};
+
+#define VOID_VALUE &_VoidStruct
 
 PyObject *
 PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)
     return PyEval_EvalFrameEx(f, 0);
 }
 
+static int py_surrogates_loaded = 0;
+static int py_surrogates_loading = 0;
+
+static PyObject *
+load_surrogate(char *name) {
+    static PyObject *module_dict = NULL;
+    PyObject *surrogate;
+    if (module_dict == NULL) {
+        PyObject *module = PyImport_ImportModule("hotpy.surrogates");
+        if (module == NULL) {
+            fprintf(stderr, "Cannot import hotpy.surrogates module\n");
+            return NULL;
+        }
+        module_dict = PyObject_GetAttrString(module, "__dict__");
+        Py_DECREF(module);
+        if (module_dict == NULL) {
+            fprintf(stderr, "Cannot import hotpy.surrogates module\n");
+            return NULL;
+        }
+    }
+    surrogate = PyDict_GetItemString(module_dict, name);
+    Py_XINCREF(surrogate);
+    return surrogate;
+}
+
+static PyObject *PyEval_EvalFrame_C(PyFrameObject *f, int throwflag);
+static PyObject *PyEval_EvalFrame_Py(PyFrameObject *f);
+int _PyEval_Load_Surrogates(void);
+
 PyObject *
 PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 {
+    if (py_surrogates_loaded && throwflag == 0) {
+        Py_INCREF(f); /* PyEval_EvalFrame_Py consumes a ref to f */
+        return PyEval_EvalFrame_Py(f);
+    }
+    else {
+        return PyEval_EvalFrame_C(f, throwflag);
+    }
+}
+
+static PyObject *
+PyEval_EvalFrame_C(PyFrameObject *f, int throwflag)
+{
 #ifdef DXPAIRS
     int lastopcode = 0;
 #endif
     register unsigned char *next_instr;
     register int opcode;        /* Current opcode */
     register int oparg;         /* Current opcode argument, if any */
-    register enum why_code why; /* Reason for block stack unwind */
+    register enum why_code why = WHY_NOT; /* Reason for block stack unwind */
     register int err;           /* Error status -- nonzero if error */
     register PyObject *x;       /* Result object -- NULL if error */
     register PyObject *v;       /* Temporary objects popped off stack */
     register PyObject **fastlocals, **freevars;
     PyObject *retval = NULL;            /* Return value */
     PyThreadState *tstate = PyThreadState_GET();
-    PyCodeObject *co;
+    PyCodeObject *co = f->f_code;
 
     /* when tracing we set things up so that
 
 #ifdef LLTRACE
 #define PUSH(v)         { (void)(BASIC_PUSH(v), \
                           lltrace && prtrace(TOP(), "push")); \
-                          assert(STACK_LEVEL() <= co->co_stacksize); }
+                          assert(STACK_LEVEL() <= co->co_stacksize+1); }
 #define POP()           ((void)(lltrace && prtrace(TOP(), "pop")), \
-                         BASIC_POP())
-#define STACKADJ(n)     { (void)(BASIC_STACKADJ(n), \
-                          lltrace && prtrace(TOP(), "stackadj")); \
-                          assert(STACK_LEVEL() <= co->co_stacksize); }
+                         assert(STACK_LEVEL() >= 1), BASIC_POP())
+#define STACKADJ         BASIC_STACKADJ
+
 #define EXT_POP(STACK_POINTER) ((void)(lltrace && \
                                 prtrace((STACK_POINTER)[-1], "ext_pop")), \
                                 *--(STACK_POINTER))
 
 
 #define UNWIND_BLOCK(b) \
+do { \
+    assert(STACK_LEVEL() >= (b)->b_level); \
     while (STACK_LEVEL() > (b)->b_level) { \
         PyObject *v = POP(); \
         Py_XDECREF(v); \
-    }
+    } \
+} while (0)
 
 #define UNWIND_EXCEPT_HANDLER(b) \
     { \
         }
     }
 
-    co = f->f_code;
     names = co->co_names;
     consts = co->co_consts;
     fastlocals = f->f_localsplus;
     lltrace = PyDict_GetItemString(f->f_globals, "__lltrace__") != NULL;
 #endif
 
-    why = WHY_NOT;
     err = 0;
     x = Py_None;        /* Not a reference, just anything non-NULL */
     w = NULL;
         READ_TIMESTAMP(loop0);
 #endif
         assert(stack_pointer >= f->f_valuestack); /* else underflow */
-        assert(STACK_LEVEL() <= co->co_stacksize);  /* else overflow */
+        assert(STACK_LEVEL() <= co->co_stacksize+1);  /* else overflow */
 
         /* Do periodic things.  Doing this every time through
            the loop would add too much overhead, so we do it
            x to NULL, err to nonzero, or why to anything but WHY_NOT,
            and that no operation that succeeds does this! */
 
-        TARGET(NOP)
-            FAST_DISPATCH();
-
-        TARGET(LOAD_FAST)
-            x = GETLOCAL(oparg);
-            if (x != NULL) {
-                Py_INCREF(x);
-                PUSH(x);
-                FAST_DISPATCH();
-            }
-            format_exc_check_arg(PyExc_UnboundLocalError,
-                UNBOUNDLOCAL_ERROR_MSG,
-                PyTuple_GetItem(co->co_varnames, oparg));
-            break;
-
-        TARGET(LOAD_CONST)
-            x = GETITEM(consts, oparg);
-            Py_INCREF(x);
-            PUSH(x);
-            FAST_DISPATCH();
-
-        PREDICTED_WITH_ARG(STORE_FAST);
-        TARGET(STORE_FAST)
-            v = POP();
-            SETLOCAL(oparg, v);
-            FAST_DISPATCH();
-
-        TARGET(POP_TOP)
-            v = POP();
-            Py_DECREF(v);
-            FAST_DISPATCH();
-
-        TARGET(ROT_TWO)
-            v = TOP();
-            w = SECOND();
-            SET_TOP(w);
-            SET_SECOND(v);
-            FAST_DISPATCH();
-
-        TARGET(ROT_THREE)
-            v = TOP();
-            w = SECOND();
-            x = THIRD();
-            SET_TOP(w);
-            SET_SECOND(x);
-            SET_THIRD(v);
-            FAST_DISPATCH();
-
-        TARGET(DUP_TOP)
-            v = TOP();
-            Py_INCREF(v);
-            PUSH(v);
-            FAST_DISPATCH();
-
-        TARGET(DUP_TOP_TWO)
-            x = TOP();
-            Py_INCREF(x);
-            w = SECOND();
-            Py_INCREF(w);
-            STACKADJ(2);
-            SET_TOP(x);
-            SET_SECOND(w);
-            FAST_DISPATCH();
+#include "atomic_ops.inc"
 
         TARGET(UNARY_POSITIVE)
             v = TOP();
             break;
         }
 
-        TARGET(LIST_APPEND)
-            w = POP();
-            v = PEEK(oparg);
-            err = PyList_Append(v, w);
-            Py_DECREF(w);
-            if (err == 0) {
-                PREDICT(JUMP_ABSOLUTE);
-                DISPATCH();
-            }
-            break;
-
-        TARGET(SET_ADD)
-            w = POP();
-            v = stack_pointer[-oparg];
-            err = PySet_Add(v, w);
-            Py_DECREF(w);
-            if (err == 0) {
-                PREDICT(JUMP_ABSOLUTE);
-                DISPATCH();
-            }
-            break;
 
         TARGET(INPLACE_POWER)
         {
             if (err == 0) DISPATCH();
             break;
 
-        TARGET(PRINT_EXPR)
-            v = POP();
-            w = PySys_GetObject("displayhook");
-            if (w == NULL) {
-                PyErr_SetString(PyExc_RuntimeError,
-                                "lost sys.displayhook");
-                err = -1;
-                x = NULL;
-            }
-            if (err == 0) {
-                x = PyTuple_Pack(1, v);
-                if (x == NULL)
-                    err = -1;
-            }
-            if (err == 0) {
-                w = PyEval_CallObject(w, x);
-                Py_XDECREF(w);
-                if (w == NULL)
-                    err = -1;
-            }
-            Py_DECREF(v);
-            Py_XDECREF(x);
-            break;
+#include "high_level_ops.inc"
 
 #ifdef CASE_TOO_BIG
         default: switch (opcode) {
 #endif
-        TARGET(RAISE_VARARGS)
-            v = w = NULL;
-            switch (oparg) {
-            case 2:
-                v = POP(); /* cause */
-            case 1:
-                w = POP(); /* exc */
-            case 0: /* Fallthrough */
-                why = do_raise(w, v);
-                break;
-            default:
-                PyErr_SetString(PyExc_SystemError,
-                           "bad RAISE_VARARGS oparg");
-                why = WHY_EXCEPTION;
-                break;
-            }
-            break;
-
-        TARGET(STORE_LOCALS)
-            x = POP();
-            v = f->f_locals;
-            Py_XDECREF(v);
-            f->f_locals = x;
-            DISPATCH();
-
-        TARGET(RETURN_VALUE)
-            retval = POP();
-            why = WHY_RETURN;
-            goto fast_block_end;
-
-        TARGET(YIELD_FROM)
-            u = POP();
-            x = PyObject_GetIter(u);
-            Py_DECREF(u);
-            if (x == NULL)
-                break;
-            /* x is now the iterator, make the first next() call */
-            retval = (*Py_TYPE(x)->tp_iternext)(x);
-            if (!retval) {
-                PyObject *et, *ev, *tb;
-                /* iter may be exhausted */
-                Py_CLEAR(x);
-                if (PyErr_Occurred() &&
-                    !PyErr_ExceptionMatches(PyExc_StopIteration)) {
-                    /* some other exception */
-                    break;
-                }
-                /* try to get return value from exception */
-                PyErr_Fetch(&et, &ev, &tb);
-                Py_XDECREF(et);
-                Py_XDECREF(tb);
-                /* u is return value */
-                u = NULL;
-                if (ev) {
-                    u = PyObject_GetAttrString(ev, "value");
-                    Py_DECREF(ev);
-                    if (u == NULL) {
-                        if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
-                            /* some other exception */
-                            break;
-                        }
-                        PyErr_Clear();
-                    }
-                }
-                if (u == NULL) {
-                    u = Py_None;
-                    Py_INCREF(u);
-                }
-                PUSH(u);
-                continue;
-            }
-            /* x is iterator, retval is value to be yielded */
-            f->f_yieldfrom = x;
-            f->f_stacktop = stack_pointer;
-            why = WHY_YIELD;
-            goto fast_yield;
-
-        TARGET(YIELD_VALUE)
-            retval = POP();
-            f->f_stacktop = stack_pointer;
-            why = WHY_YIELD;
-            goto fast_yield;
-
-        TARGET(POP_EXCEPT)
-            {
-                PyTryBlock *b = PyFrame_BlockPop(f);
-                if (b->b_type != EXCEPT_HANDLER) {
-                    PyErr_SetString(PyExc_SystemError,
-                        "popped block is not an except handler");
-                    why = WHY_EXCEPTION;
-                    break;
-                }
-                UNWIND_EXCEPT_HANDLER(b);
-            }
-            DISPATCH();
-
-        TARGET(POP_BLOCK)
-            {
-                PyTryBlock *b = PyFrame_BlockPop(f);
-                UNWIND_BLOCK(b);
-            }
-            DISPATCH();
-
-        PREDICTED(END_FINALLY);
-        TARGET(END_FINALLY)
-            v = POP();
-            if (PyLong_Check(v)) {
-                why = (enum why_code) PyLong_AS_LONG(v);
-                assert(why != WHY_YIELD);
-                if (why == WHY_RETURN ||
-                    why == WHY_CONTINUE)
-                    retval = POP();
-                if (why == WHY_SILENCED) {
-                    /* An exception was silenced by 'with', we must
-                    manually unwind the EXCEPT_HANDLER block which was
-                    created when the exception was caught, otherwise
-                    the stack will be in an inconsistent state. */
-                    PyTryBlock *b = PyFrame_BlockPop(f);
-                    assert(b->b_type == EXCEPT_HANDLER);
-                    UNWIND_EXCEPT_HANDLER(b);
-                    why = WHY_NOT;
-                }
-            }
-            else if (PyExceptionClass_Check(v)) {
-                w = POP();
-                u = POP();
-                PyErr_Restore(v, w, u);
-                why = WHY_RERAISE;
-                break;
-            }
-            else if (v != Py_None) {
-                PyErr_SetString(PyExc_SystemError,
-                    "'finally' pops bad exception");
-                why = WHY_EXCEPTION;
-            }
-            Py_DECREF(v);
-            break;
-
-        TARGET(LOAD_BUILD_CLASS)
-            x = PyDict_GetItemString(f->f_builtins,
-                                     "__build_class__");
-            if (x == NULL) {
-                PyErr_SetString(PyExc_ImportError,
-                                "__build_class__ not found");
-                break;
-            }
-            Py_INCREF(x);
-            PUSH(x);
-            break;
-
-        TARGET(STORE_NAME)
-            w = GETITEM(names, oparg);
-            v = POP();
-            if ((x = f->f_locals) != NULL) {
-                if (PyDict_CheckExact(x))
-                    err = PyDict_SetItem(x, w, v);
-                else
-                    err = PyObject_SetItem(x, w, v);
-                Py_DECREF(v);
-                if (err == 0) DISPATCH();
-                break;
-            }
-            PyErr_Format(PyExc_SystemError,
-                         "no locals found when storing %R", w);
-            break;
-
-        TARGET(DELETE_NAME)
-            w = GETITEM(names, oparg);
-            if ((x = f->f_locals) != NULL) {
-                if ((err = PyObject_DelItem(x, w)) != 0)
-                    format_exc_check_arg(PyExc_NameError,
-                                         NAME_ERROR_MSG,
-                                         w);
-                break;
-            }
-            PyErr_Format(PyExc_SystemError,
-                         "no locals when deleting %R", w);
-            break;
-
-        PREDICTED_WITH_ARG(UNPACK_SEQUENCE);
-        TARGET(UNPACK_SEQUENCE)
-            v = POP();
-            if (PyTuple_CheckExact(v) &&
-                PyTuple_GET_SIZE(v) == oparg) {
-                PyObject **items = \
-                    ((PyTupleObject *)v)->ob_item;
-                while (oparg--) {
-                    w = items[oparg];
-                    Py_INCREF(w);
-                    PUSH(w);
-                }
-                Py_DECREF(v);
-                DISPATCH();
-            } else if (PyList_CheckExact(v) &&
-                       PyList_GET_SIZE(v) == oparg) {
-                PyObject **items = \
-                    ((PyListObject *)v)->ob_item;
-                while (oparg--) {
-                    w = items[oparg];
-                    Py_INCREF(w);
-                    PUSH(w);
-                }
-            } else if (unpack_iterable(v, oparg, -1,
-                                       stack_pointer + oparg)) {
-                STACKADJ(oparg);
-            } else {
-                /* unpack_iterable() raised an exception */
-                why = WHY_EXCEPTION;
-            }
-            Py_DECREF(v);
-            break;
-
-        TARGET(UNPACK_EX)
-        {
-            int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
-            v = POP();
-
-            if (unpack_iterable(v, oparg & 0xFF, oparg >> 8,
-                                stack_pointer + totalargs)) {
-                stack_pointer += totalargs;
-            } else {
-                why = WHY_EXCEPTION;
-            }
-            Py_DECREF(v);
-            break;
-        }
 
         TARGET(STORE_ATTR)
             w = GETITEM(names, oparg);
             Py_DECREF(v);
             break;
 
-        TARGET(STORE_GLOBAL)
-            w = GETITEM(names, oparg);
-            v = POP();
-            err = PyDict_SetItem(f->f_globals, w, v);
-            Py_DECREF(v);
-            if (err == 0) DISPATCH();
-            break;
-
-        TARGET(DELETE_GLOBAL)
-            w = GETITEM(names, oparg);
-            if ((err = PyDict_DelItem(f->f_globals, w)) != 0)
-                format_exc_check_arg(
-                    PyExc_NameError, GLOBAL_NAME_ERROR_MSG, w);
-            break;
-
-        TARGET(LOAD_NAME)
-            w = GETITEM(names, oparg);
-            if ((v = f->f_locals) == NULL) {
-                PyErr_Format(PyExc_SystemError,
-                             "no locals when loading %R", w);
-                why = WHY_EXCEPTION;
-                break;
-            }
-            if (PyDict_CheckExact(v)) {
-                x = PyDict_GetItem(v, w);
-                Py_XINCREF(x);
-            }
-            else {
-                x = PyObject_GetItem(v, w);
-                if (x == NULL && PyErr_Occurred()) {
-                    if (!PyErr_ExceptionMatches(
-                                    PyExc_KeyError))
-                        break;
-                    PyErr_Clear();
-                }
-            }
-            if (x == NULL) {
-                x = PyDict_GetItem(f->f_globals, w);
-                if (x == NULL) {
-                    x = PyDict_GetItem(f->f_builtins, w);
-                    if (x == NULL) {
-                        format_exc_check_arg(
-                                    PyExc_NameError,
-                                    NAME_ERROR_MSG, w);
-                        break;
-                    }
-                }
-                Py_INCREF(x);
-            }
-            PUSH(x);
-            DISPATCH();
-
-        TARGET(LOAD_GLOBAL)
-            w = GETITEM(names, oparg);
-            if (PyUnicode_CheckExact(w)) {
-                /* Inline the PyDict_GetItem() calls.
-                   WARNING: this is an extreme speed hack.
-                   Do not try this at home. */
-                Py_hash_t hash = ((PyASCIIObject *)w)->hash;
-                if (hash != -1) {
-                    PyDictObject *d;
-                    PyDictEntry *e;
-                    d = (PyDictObject *)(f->f_globals);
-                    e = d->ma_lookup(d, w, hash);
-                    if (e == NULL) {
-                        x = NULL;
-                        break;
-                    }
-                    x = e->me_value;
-                    if (x != NULL) {
-                        Py_INCREF(x);
-                        PUSH(x);
-                        DISPATCH();
-                    }
-                    d = (PyDictObject *)(f->f_builtins);
-                    e = d->ma_lookup(d, w, hash);
-                    if (e == NULL) {
-                        x = NULL;
-                        break;
-                    }
-                    x = e->me_value;
-                    if (x != NULL) {
-                        Py_INCREF(x);
-                        PUSH(x);
-                        DISPATCH();
-                    }
-                    goto load_global_error;
-                }
-            }
-            /* This is the un-inlined version of the code above */
-            x = PyDict_GetItem(f->f_globals, w);
-            if (x == NULL) {
-                x = PyDict_GetItem(f->f_builtins, w);
-                if (x == NULL) {
-                  load_global_error:
-                    format_exc_check_arg(
-                                PyExc_NameError,
-                                GLOBAL_NAME_ERROR_MSG, w);
-                    break;
-                }
-            }
-            Py_INCREF(x);
-            PUSH(x);
-            DISPATCH();
-
-        TARGET(DELETE_FAST)
-            x = GETLOCAL(oparg);
-            if (x != NULL) {
-                SETLOCAL(oparg, NULL);
-                DISPATCH();
-            }
-            format_exc_check_arg(
-                PyExc_UnboundLocalError,
-                UNBOUNDLOCAL_ERROR_MSG,
-                PyTuple_GetItem(co->co_varnames, oparg)
-                );
-            break;
-
-        TARGET(DELETE_DEREF)
-            x = freevars[oparg];
-            if (PyCell_GET(x) != NULL) {
-                PyCell_Set(x, NULL);
-                DISPATCH();
-            }
-            err = -1;
-            format_exc_unbound(co, oparg);
-            break;
-
-        TARGET(LOAD_CLOSURE)
-            x = freevars[oparg];
-            Py_INCREF(x);
-            PUSH(x);
-            if (x != NULL) DISPATCH();
-            break;
-
-        TARGET(LOAD_DEREF)
-            x = freevars[oparg];
-            w = PyCell_Get(x);
-            if (w != NULL) {
-                PUSH(w);
-                DISPATCH();
-            }
-            err = -1;
-            format_exc_unbound(co, oparg);
-            break;
-
-        TARGET(STORE_DEREF)
-            w = POP();
-            x = freevars[oparg];
-            PyCell_Set(x, w);
-            Py_DECREF(w);
-            DISPATCH();
-
-        TARGET(BUILD_TUPLE)
-            x = PyTuple_New(oparg);
-            if (x != NULL) {
-                for (; --oparg >= 0;) {
-                    w = POP();
-                    PyTuple_SET_ITEM(x, oparg, w);
-                }
-                PUSH(x);
-                DISPATCH();
-            }
-            break;
-
-        TARGET(BUILD_LIST)
-            x =  PyList_New(oparg);
-            if (x != NULL) {
-                for (; --oparg >= 0;) {
-                    w = POP();
-                    PyList_SET_ITEM(x, oparg, w);
-                }
-                PUSH(x);
-                DISPATCH();
-            }
-            break;
-
-        TARGET(BUILD_SET)
-            x = PySet_New(NULL);
-            if (x != NULL) {
-                for (; --oparg >= 0;) {
-                    w = POP();
-                    if (err == 0)
-                        err = PySet_Add(x, w);
-                    Py_DECREF(w);
-                }
-                if (err != 0) {
-                    Py_DECREF(x);
-                    break;
-                }
-                PUSH(x);
-                DISPATCH();
-            }
-            break;
-
-        TARGET(BUILD_MAP)
-            x = _PyDict_NewPresized((Py_ssize_t)oparg);
-            PUSH(x);
-            if (x != NULL) DISPATCH();
-            break;
-
-        TARGET(STORE_MAP)
-            w = TOP();     /* key */
-            u = SECOND();  /* value */
-            v = THIRD();   /* dict */
-            STACKADJ(-2);
-            assert (PyDict_CheckExact(v));
-            err = PyDict_SetItem(v, w, u);  /* v[w] = u */
-            Py_DECREF(u);
-            Py_DECREF(w);
-            if (err == 0) DISPATCH();
-            break;
-
-        TARGET(MAP_ADD)
-            w = TOP();     /* key */
-            u = SECOND();  /* value */
-            STACKADJ(-2);
-            v = stack_pointer[-oparg];  /* dict */
-            assert (PyDict_CheckExact(v));
-            err = PyDict_SetItem(v, w, u);  /* v[w] = u */
-            Py_DECREF(u);
-            Py_DECREF(w);
-            if (err == 0) {
-                PREDICT(JUMP_ABSOLUTE);
-                DISPATCH();
-            }
-            break;
-
         TARGET(LOAD_ATTR)
             w = GETITEM(names, oparg);
             v = TOP();
             DISPATCH();
         }
 
-        TARGET(IMPORT_NAME)
-            w = GETITEM(names, oparg);
-            x = PyDict_GetItemString(f->f_builtins, "__import__");
-            if (x == NULL) {
-                PyErr_SetString(PyExc_ImportError,
-                                "__import__ not found");
-                break;
-            }
-            Py_INCREF(x);
-            v = POP();
-            u = TOP();
-            if (PyLong_AsLong(u) != -1 || PyErr_Occurred())
-                w = PyTuple_Pack(5,
-                            w,
-                            f->f_globals,
-                            f->f_locals == NULL ?
-                                  Py_None : f->f_locals,
-                            v,
-                            u);
-            else
-                w = PyTuple_Pack(4,
-                            w,
-                            f->f_globals,
-                            f->f_locals == NULL ?
-                                  Py_None : f->f_locals,
-                            v);
-            Py_DECREF(v);
-            Py_DECREF(u);
-            if (w == NULL) {
-                u = POP();
-                Py_DECREF(x);
-                x = NULL;
-                break;
-            }
-            READ_TIMESTAMP(intr0);
-            v = x;
-            x = PyEval_CallObject(v, w);
-            Py_DECREF(v);
-            READ_TIMESTAMP(intr1);
-            Py_DECREF(w);
-            SET_TOP(x);
-            if (x != NULL) DISPATCH();
-            break;
-
-        TARGET(IMPORT_STAR)
-            v = POP();
-            PyFrame_FastToLocals(f);
-            if ((x = f->f_locals) == NULL) {
-                PyErr_SetString(PyExc_SystemError,
-                    "no locals found during 'import *'");
-                break;
-            }
-            READ_TIMESTAMP(intr0);
-            err = import_all_from(x, v);
-            READ_TIMESTAMP(intr1);
-            PyFrame_LocalsToFast(f, 0);
-            Py_DECREF(v);
-            if (err == 0) DISPATCH();
-            break;
-
-        TARGET(IMPORT_FROM)
-            w = GETITEM(names, oparg);
-            v = TOP();
-            READ_TIMESTAMP(intr0);
-            x = import_from(v, w);
-            READ_TIMESTAMP(intr1);
-            PUSH(x);
-            if (x != NULL) DISPATCH();
-            break;
-
-        TARGET(JUMP_FORWARD)
-            JUMPBY(oparg);
-            FAST_DISPATCH();
-
         PREDICTED_WITH_ARG(POP_JUMP_IF_FALSE);
         TARGET(POP_JUMP_IF_FALSE)
             w = POP();
                 break;
             DISPATCH();
 
-        PREDICTED_WITH_ARG(JUMP_ABSOLUTE);
-        TARGET(JUMP_ABSOLUTE)
-            JUMPTO(oparg);
-#if FAST_LOOPS
-            /* Enabling this path speeds-up all while and for-loops by bypassing
-               the per-loop checks for signals.  By default, this should be turned-off
-               because it prevents detection of a control-break in tight loops like
-               "while 1: pass".  Compile with this option turned-on when you need
-               the speed-up and do not need break checking inside tight loops (ones
-               that contain only instructions ending with FAST_DISPATCH).
-            */
-            FAST_DISPATCH();
-#else
-            DISPATCH();
-#endif
-
         TARGET(GET_ITER)
             /* before: [obj]; after [getiter(obj)] */
             v = TOP();
             JUMPBY(oparg);
             DISPATCH();
 
-        TARGET(BREAK_LOOP)
-            why = WHY_BREAK;
-            goto fast_block_end;
-
-        TARGET(CONTINUE_LOOP)
-            retval = PyLong_FromLong(oparg);
-            if (!retval) {
-                x = NULL;
-                break;
-            }
-            why = WHY_CONTINUE;
-            goto fast_block_end;
-
-        TARGET_WITH_IMPL(SETUP_LOOP, _setup_finally)
-        TARGET_WITH_IMPL(SETUP_EXCEPT, _setup_finally)
-        TARGET(SETUP_FINALLY)
-        _setup_finally:
-            /* NOTE: If you add any new block-setup opcodes that
-               are not try/except/finally handlers, you may need
-               to update the PyGen_NeedsFinalizing() function.
-               */
-
-            PyFrame_BlockSetup(f, opcode, INSTR_OFFSET() + oparg,
-                               STACK_LEVEL());
-            DISPATCH();
-
-        TARGET(SETUP_WITH)
-        {
-            _Py_IDENTIFIER(__exit__);
-            _Py_IDENTIFIER(__enter__);
-            w = TOP();
-            x = special_lookup(w, &PyId___exit__);
-            if (!x)
-                break;
-            SET_TOP(x);
-            u = special_lookup(w, &PyId___enter__);
-            Py_DECREF(w);
-            if (!u) {
-                x = NULL;
-                break;
-            }
-            x = PyObject_CallFunctionObjArgs(u, NULL);
-            Py_DECREF(u);
-            if (!x)
-                break;
-            /* Setup the finally block before pushing the result
-               of __enter__ on the stack. */
-            PyFrame_BlockSetup(f, SETUP_FINALLY, INSTR_OFFSET() + oparg,
-                               STACK_LEVEL());
-
-            PUSH(x);
-            DISPATCH();
-        }
-
-        TARGET(WITH_CLEANUP)
-        {
-            /* At the top of the stack are 1-3 values indicating
-               how/why we entered the finally clause:
-               - TOP = None
-               - (TOP, SECOND) = (WHY_{RETURN,CONTINUE}), retval
-               - TOP = WHY_*; no retval below it
-               - (TOP, SECOND, THIRD) = exc_info()
-                 (FOURTH, FITH, SIXTH) = previous exception for EXCEPT_HANDLER
-               Below them is EXIT, the context.__exit__ bound method.
-               In the last case, we must call
-                 EXIT(TOP, SECOND, THIRD)
-               otherwise we must call
-                 EXIT(None, None, None)
-
-               In the first two cases, we remove EXIT from the
-               stack, leaving the rest in the same order.  In the
-               third case, we shift the bottom 3 values of the
-               stack down, and replace the empty spot with NULL.
-
-               In addition, if the stack represents an exception,
-               *and* the function call returns a 'true' value, we
-               push WHY_SILENCED onto the stack.  END_FINALLY will
-               then not re-raise the exception.  (But non-local
-               gotos should still be resumed.)
-            */
-
-            PyObject *exit_func;
-            u = TOP();
-            if (u == Py_None) {
-                (void)POP();
-                exit_func = TOP();
-                SET_TOP(u);
-                v = w = Py_None;
-            }
-            else if (PyLong_Check(u)) {
-                (void)POP();
-                switch(PyLong_AsLong(u)) {
-                case WHY_RETURN:
-                case WHY_CONTINUE:
-                    /* Retval in TOP. */
-                    exit_func = SECOND();
-                    SET_SECOND(TOP());
-                    SET_TOP(u);
-                    break;
-                default:
-                    exit_func = TOP();
-                    SET_TOP(u);
-                    break;
-                }
-                u = v = w = Py_None;
-            }
-            else {
-                PyObject *tp, *exc, *tb;
-                PyTryBlock *block;
-                v = SECOND();
-                w = THIRD();
-                tp = FOURTH();
-                exc = PEEK(5);
-                tb = PEEK(6);
-                exit_func = PEEK(7);
-                SET_VALUE(7, tb);
-                SET_VALUE(6, exc);
-                SET_VALUE(5, tp);
-                /* UNWIND_EXCEPT_HANDLER will pop this off. */
-                SET_FOURTH(NULL);
-                /* We just shifted the stack down, so we have
-                   to tell the except handler block that the
-                   values are lower than it expects. */
-                block = &f->f_blockstack[f->f_iblock - 1];
-                assert(block->b_type == EXCEPT_HANDLER);
-                block->b_level--;
-            }
-            /* XXX Not the fastest way to call it... */
-            x = PyObject_CallFunctionObjArgs(exit_func, u, v, w,
-                                             NULL);
-            Py_DECREF(exit_func);
-            if (x == NULL)
-                break; /* Go to error exit */
-
-            if (u != Py_None)
-                err = PyObject_IsTrue(x);
-            else
-                err = 0;
-            Py_DECREF(x);
-
-            if (err < 0)
-                break; /* Go to error exit */
-            else if (err > 0) {
-                err = 0;
-                /* There was an exception and a True return */
-                PUSH(PyLong_FromLong((long) WHY_SILENCED));
-            }
-            PREDICT(END_FINALLY);
-            break;
-        }
-
         TARGET(CALL_FUNCTION)
         {
             PyObject **sp;
             break;
         }
 
-        TARGET_WITH_IMPL(MAKE_CLOSURE, _make_function)
-        TARGET(MAKE_FUNCTION)
-        _make_function:
-        {
-            int posdefaults = oparg & 0xff;
-            int kwdefaults = (oparg>>8) & 0xff;
-            int num_annotations = (oparg >> 16) & 0x7fff;
-
-            w = POP(); /* qualname */
-            v = POP(); /* code object */
-            x = PyFunction_NewWithQualName(v, f->f_globals, w);
-            Py_DECREF(v);
-            Py_DECREF(w);
-
-            if (x != NULL && opcode == MAKE_CLOSURE) {
-                v = POP();
-                if (PyFunction_SetClosure(x, v) != 0) {
-                    /* Can't happen unless bytecode is corrupt. */
-                    why = WHY_EXCEPTION;
+        TARGET(LOAD_SPECIAL)
+            {
+                PyObject *name = GETITEM(names, oparg);
+                v = TOP();
+                x = _PyType_Lookup(Py_TYPE(v), name);
+                if (x != NULL) {
+                    descrgetfunc f;
+                    if ((f = Py_TYPE(x)->tp_descr_get) == NULL)
+                        Py_INCREF(x);
+                    else
+                        x = f(x, v, (PyObject *)(Py_TYPE(v)));
                 }
                 Py_DECREF(v);
-            }
-
-            if (x != NULL && num_annotations > 0) {
-                Py_ssize_t name_ix;
-                u = POP(); /* names of args with annotations */
-                v = PyDict_New();
-                if (v == NULL) {
-                    Py_DECREF(x);
-                    x = NULL;
+                SET_TOP(x);
+                if (x == NULL) {
+                    if (!PyErr_Occurred())
+                        PyErr_SetObject(PyExc_AttributeError, name);
                     break;
                 }
-                name_ix = PyTuple_Size(u);
-                assert(num_annotations == name_ix+1);
-                while (name_ix > 0) {
-                    --name_ix;
-                    t = PyTuple_GET_ITEM(u, name_ix);
-                    w = POP();
-                    /* XXX(nnorwitz): check for errors */
-                    PyDict_SetItem(v, t, w);
-                    Py_DECREF(w);
-                }
-
-                if (PyFunction_SetAnnotations(x, v) != 0) {
-                    /* Can't happen unless
-                       PyFunction_SetAnnotations changes. */
-                    why = WHY_EXCEPTION;
-                }
-                Py_DECREF(v);
-                Py_DECREF(u);
+                DISPATCH();
             }
 
-            /* XXX Maybe this should be a separate opcode? */
-            if (x != NULL && posdefaults > 0) {
-                v = PyTuple_New(posdefaults);
-                if (v == NULL) {
-                    Py_DECREF(x);
-                    x = NULL;
-                    break;
-                }
-                while (--posdefaults >= 0) {
-                    w = POP();
-                    PyTuple_SET_ITEM(v, posdefaults, w);
-                }
-                if (PyFunction_SetDefaults(x, v) != 0) {
-                    /* Can't happen unless
-                       PyFunction_SetDefaults changes. */
-                    why = WHY_EXCEPTION;
-                }
-                Py_DECREF(v);
-            }
-            if (x != NULL && kwdefaults > 0) {
-                v = PyDict_New();
-                if (v == NULL) {
-                    Py_DECREF(x);
-                    x = NULL;
-                    break;
-                }
-                while (--kwdefaults >= 0) {
-                    w = POP(); /* default value */
-                    u = POP(); /* kw only arg name */
-                    /* XXX(nnorwitz): check for errors */
-                    PyDict_SetItem(v, u, w);
-                    Py_DECREF(w);
-                    Py_DECREF(u);
-                }
-                if (PyFunction_SetKwDefaults(x, v) != 0) {
-                    /* Can't happen unless
-                       PyFunction_SetKwDefaults changes. */
-                    why = WHY_EXCEPTION;
-                }
-                Py_DECREF(v);
-            }
-            PUSH(x);
-            break;
-        }
-
-        TARGET(BUILD_SLICE)
-            if (oparg == 3)
-                w = POP();
-            else
-                w = NULL;
-            v = POP();
-            u = TOP();
-            x = PySlice_New(u, v, w);
-            Py_DECREF(u);
-            Py_DECREF(v);
-            Py_XDECREF(w);
-            SET_TOP(x);
-            if (x != NULL) DISPATCH();
-            break;
-
-        TARGET(EXTENDED_ARG)
-            opcode = NEXTOP();
-            oparg = oparg<<16 | NEXTARG();
-            goto dispatch_opcode;
-
 #if USE_COMPUTED_GOTOS
         _unknown_opcode:
 #endif
         /* Log traceback info if this is a real exception */
 
         if (why == WHY_EXCEPTION) {
-            PyTraceBack_Here(f);
-
-            if (tstate->c_tracefunc != NULL)
-                call_exc_trace(tstate->c_tracefunc,
-                               tstate->c_traceobj, f);
+            if ((f->f_code->co_flags & CO_INVISIBLE) == 0) {
+                PyTraceBack_Here(f);
+
+                if (tstate->c_tracefunc != NULL)
+                    call_exc_trace(tstate->c_tracefunc,
+                                   tstate->c_traceobj, f);
+            }
         }
 
         /* For the rest, treat WHY_RERAISE as WHY_EXCEPTION */
             PyTryBlock *b = &f->f_blockstack[f->f_iblock - 1];
 
             assert(why != WHY_YIELD);
+            if (b->b_type == FOR_ITER) {
+                f->f_iblock--;
+                UNWIND_BLOCK(b);
+                if (why == WHY_EXCEPTION &&
+                    tstate->curexc_type == PyExc_StopIteration) {
+                    int handler = b->b_handler;
+                    why = WHY_NOT;
+                    PyErr_Clear();
+                    JUMPTO(handler);
+                    break;
+                }
+                continue;
+            }
             if (b->b_type == SETUP_LOOP && why == WHY_CONTINUE) {
                 why = WHY_NOT;
                 JUMPTO(PyLong_AS_LONG(retval));
                    Python main loop. */
                 PyErr_NormalizeException(
                     &exc, &val, &tb);
-                PyException_SetTraceback(val, tb);
+                if (tb != NULL)
+                    PyException_SetTraceback(val, tb);
                 Py_INCREF(exc);
                 tstate->exc_type = exc;
                 Py_INCREF(val);
     Py_LeaveRecursiveCall();
     tstate->frame = f->f_back;
 
+    if (co->co_flags & CO_GENERATOR && why == WHY_RETURN) {
+        if (retval == Py_None) {
+            /* Delay exception instantiation if we can */
+            PyErr_SetNone(PyExc_StopIteration);
+        } else {
+            PyObject *e = PyStopIteration_Create(retval);
+            if (e != NULL) {
+                PyErr_SetObject(PyExc_StopIteration, e);
+                Py_DECREF(e);
+            }
+        }
+        Py_CLEAR(retval);
+    }
     return retval;
 }
 
+static PyObject *call_surrogates; /* A dictionary */
+static PyObject *type_call_surrogates; /* A dictionary */
+static PyObject *surrogate_pow;
+static PyObject *surrogate_mul;
+static PyObject *surrogate_truediv;
+static PyObject *surrogate_floordiv;
+static PyObject *surrogate_mod;
+static PyObject *surrogate_add;
+static PyObject *surrogate_and;
+static PyObject *surrogate_or;
+static PyObject *surrogate_xor;
+static PyObject *surrogate_sub;
+static PyObject *surrogate_lshift;
+static PyObject *surrogate_rshift;
+static PyObject *surrogate_inplace_pow;
+static PyObject *surrogate_inplace_mul;
+static PyObject *surrogate_inplace_truediv;
+static PyObject *surrogate_inplace_floordiv;
+static PyObject *surrogate_inplace_mod;
+static PyObject *surrogate_inplace_add;
+static PyObject *surrogate_inplace_and;
+static PyObject *surrogate_inplace_or;
+static PyObject *surrogate_inplace_xor;
+static PyObject *surrogate_inplace_sub;
+static PyObject *surrogate_inplace_lshift;
+static PyObject *surrogate_inplace_rshift;
+static PyObject *surrogate_positive;
+static PyObject *surrogate_negative;
+static PyObject *surrogate_invert;
+static PyObject *surrogate_truth;
+static PyObject *surrogate_delete_attr;
+static PyObject *surrogate_load_special;
+static PyObject *surrogate_get_iter;
+static PyObject *surrogate_delete_subscr;
+static PyObject *surrogate_load_attr;
+static PyObject *surrogate_store_subscr;
+static PyObject *surrogate_store_attr;
+static PyObject *surrogate_binary_subscr;
+static PyObject *surrogate_for_iter;
+static PyObject *surrogate_type_call;
+static PyObject *surrogate_compares[PyCmp_EXC_MATCH];
+
+#define LOAD_SURROGATE(name) \
+    surrogate_##name = load_surrogate("surrogate_" #name); \
+    if (surrogate_##name == NULL) { \
+        failure = #name; \
+        goto fail; \
+    }
+
+#define LOAD_COMPARE(op, name) \
+    x = load_surrogate("surrogate_" #name); \
+    if (x == NULL) { \
+        failure = #name; \
+        goto fail; \
+    } \
+    surrogate_compares[op] = x
+
+
+int _PyEval_Load_Surrogates(void) {
+    char *failure = "";
+    PyObject *x;
+    if (py_surrogates_loading || py_surrogates_loaded)
+        return 0;
+    py_surrogates_loading = 1;
+    call_surrogates = PyDict_New();
+    if (call_surrogates == NULL) {
+        goto fail;
+    }
+    type_call_surrogates = PyDict_New();
+    if (type_call_surrogates == NULL) {
+        goto fail;
+    }
+    LOAD_SURROGATE(pow);
+    LOAD_SURROGATE(mul);
+    LOAD_SURROGATE(truediv);
+    LOAD_SURROGATE(floordiv);
+    LOAD_SURROGATE(mod);
+    LOAD_SURROGATE(add);
+    LOAD_SURROGATE(and);
+    LOAD_SURROGATE(or);
+    LOAD_SURROGATE(xor);
+    LOAD_SURROGATE(sub);
+    LOAD_SURROGATE(lshift);
+    LOAD_SURROGATE(rshift);
+    LOAD_SURROGATE(inplace_pow);
+    LOAD_SURROGATE(inplace_mul);
+    LOAD_SURROGATE(inplace_truediv);
+    LOAD_SURROGATE(inplace_floordiv);
+    LOAD_SURROGATE(inplace_mod);
+    LOAD_SURROGATE(inplace_add);
+    LOAD_SURROGATE(inplace_and);
+    LOAD_SURROGATE(inplace_or);
+    LOAD_SURROGATE(inplace_xor);
+    LOAD_SURROGATE(inplace_sub);
+    LOAD_SURROGATE(inplace_lshift);
+    LOAD_SURROGATE(inplace_rshift);
+    LOAD_SURROGATE(positive);
+    LOAD_SURROGATE(negative);
+    LOAD_SURROGATE(invert);
+    LOAD_SURROGATE(truth);
+    LOAD_SURROGATE(delete_attr);
+    LOAD_SURROGATE(load_special);
+    LOAD_SURROGATE(get_iter);
+    LOAD_SURROGATE(delete_subscr);
+    LOAD_SURROGATE(load_attr);
+    LOAD_SURROGATE(store_subscr);
+    LOAD_SURROGATE(store_attr);
+    LOAD_SURROGATE(binary_subscr);
+    LOAD_SURROGATE(for_iter);
+    LOAD_SURROGATE(type_call);
+    if (PyDict_SetItem(call_surrogates, (PyObject*)&PyType_Type,
+                       surrogate_type_call) < 0) {
+        goto fail;
+    }
+    LOAD_COMPARE(PyCmp_EQ, eq);
+    LOAD_COMPARE(PyCmp_NE, ne);
+    LOAD_COMPARE(PyCmp_LT, lt);
+    LOAD_COMPARE(PyCmp_GT, gt);
+    LOAD_COMPARE(PyCmp_LE, le);
+    LOAD_COMPARE(PyCmp_GE, ge);
+    py_surrogates_loaded = 1;
+    py_surrogates_loading = 0;
+    return 0;
+fail:
+    Py_XDECREF(call_surrogates);
+    return -1;
+}
+
 static void
 format_missing(const char *kind, PyCodeObject *co, PyObject *names)
 {
     Py_DECREF(kwonly_sig);
 }
 
-/* This is gonna seem *real weird*, but if you put some other code between
-   PyEval_EvalFrame() and PyEval_EvalCodeEx() you will need to adjust
-   the test in the if statements in Misc/gdbinit (pystack and pystackv). */
-
-PyObject *
-PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals,
+static PyFrameObject *
+make_frame_ex(PyObject *_co, PyObject *globals, PyObject *locals,
            PyObject **args, int argcount, PyObject **kws, int kwcount,
            PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure)
 {
     PyCodeObject* co = (PyCodeObject*)_co;
     register PyFrameObject *f;
-    register PyObject *retval = NULL;
     register PyObject **fastlocals, **freevars;
     PyThreadState *tstate = PyThreadState_GET();
     PyObject *x, *u;
 
     if (globals == NULL) {
         PyErr_SetString(PyExc_SystemError,
-                        "PyEval_EvalCodeEx: NULL globals");
+                        "make_frame: NULL globals");
         return NULL;
     }
 
         freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o;
     }
 
+    return f;
+
+fail: /* Jump here from prelude on failure */
+
+    /* decref'ing the frame can cause __del__ methods to get invoked,
+       which can call back into Python.  While we're done with the
+       current Python frame (f), the associated C stack is still in use,
+       so recursion_depth must be boosted for the duration.
+    */
+    ++tstate->recursion_depth;
+    Py_DECREF(f);
+    --tstate->recursion_depth;
+    return NULL;
+}
+
+/* Consumes references to all n args */
+static PyFrameObject *
+fast_frame(PyObject *func, PyObject **args, int n) {
+     int i;
+    PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func);
+    PyObject *globals = PyFunction_GET_GLOBALS(func);
+    PyObject *argdefs = PyFunction_GET_DEFAULTS(func);
+    PyObject *kwdefs = PyFunction_GET_KW_DEFAULTS(func);
+    PyObject **d = NULL;
+    int nd = 0;
+    PyFrameObject *f;
+
+    if (argdefs == NULL && co->co_argcount == n &&
+        co->co_kwonlyargcount == 0 &&
+        co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) {
+        PyThreadState *tstate = PyThreadState_GET();
+        PyObject **fastlocals;
+        int i;
+
+        assert(globals != NULL);
+        /* XXX Perhaps we should create a specialized
+           PyFrame_New() that doesn't take locals, but does
+           take builtins without sanity checking them.
+        */
+        assert(tstate != NULL);
+        f = PyFrame_New(tstate, co, globals, NULL);
+        if (f == NULL) {
+            for (i = 0; i < n; i++) {
+                Py_DECREF(args[i]);
+            }
+            return NULL;
+        }
+        fastlocals = f->f_localsplus;
+
+        for (i = 0; i < n; i++) {
+            fastlocals[i] = args[i];
+        }
+    }
+    else {
+        if (argdefs != NULL) {
+            d = &PyTuple_GET_ITEM(argdefs, 0);
+            nd = Py_SIZE(argdefs);
+        }
+        f = make_frame_ex((PyObject*)co, globals, NULL, args, n, args+n, 0,
+                            d, nd, kwdefs, PyFunction_GET_CLOSURE(func));
+        for (i = 0; i < n; i++) {
+            Py_DECREF(args[i]);
+        }
+    }
+    return f;
+}
+
+static PyFrameObject *
+make_frame(PyObject *func, PyObject *arg, PyObject *kw)
+{
+    PyFrameObject *result;
+    PyObject *argdefs;
+    PyObject *kwtuple = NULL;
+    PyObject **d, **k;
+    Py_ssize_t nk, nd;
+
+    argdefs = PyFunction_GET_DEFAULTS(func);
+    if (argdefs != NULL && PyTuple_Check(argdefs)) {
+        d = &PyTuple_GET_ITEM((PyTupleObject *)argdefs, 0);
+        nd = PyTuple_GET_SIZE(argdefs);
+    }
+    else {
+        d = NULL;
+        nd = 0;
+    }
+
+    if (kw != NULL && PyDict_Check(kw)) {
+        Py_ssize_t pos, i;
+        nk = PyDict_Size(kw);
+        kwtuple = PyTuple_New(2*nk);
+        if (kwtuple == NULL)
+            return NULL;
+        k = &PyTuple_GET_ITEM(kwtuple, 0);
+        pos = i = 0;
+        while (PyDict_Next(kw, &pos, &k[i], &k[i+1])) {
+            Py_INCREF(k[i]);
+            Py_INCREF(k[i+1]);
+            i += 2;
+        }
+        nk = i/2;
+    }
+    else {
+        k = NULL;
+        nk = 0;
+    }
+
+    result = make_frame_ex(
+        PyFunction_GET_CODE(func),
+        PyFunction_GET_GLOBALS(func), (PyObject *)NULL,
+        &PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg),
+        k, nk, d, nd,
+        PyFunction_GET_KW_DEFAULTS(func),
+        PyFunction_GET_CLOSURE(func));
+
+    Py_XDECREF(kwtuple);
+
+    return result;
+}
+
+static PyObject *
+build_parameters(PyObject *func, PyObject ***pp_stack,
+               int flags, int na, int nk, PyObject **kws)
+{
+    int nstar = 0;
+    PyObject *callargs = NULL;
+    PyObject *stararg = NULL;
+    PyObject *kwdict = NULL;
+
+    if (flags & CALL_FLAG_KW) {
+        kwdict = EXT_POP(*pp_stack);
+        if (!PyDict_Check(kwdict)) {
+            PyObject *d;
+            d = PyDict_New();
+            if (d == NULL)
+                goto fail;
+            if (PyDict_Update(d, kwdict) != 0) {
+                Py_DECREF(d);
+                /* PyDict_Update raises attribute
+                 * error (percolated from an attempt
+                 * to get 'keys' attribute) instead of
+                 * a type error if its second argument
+                 * is not a mapping.
+                 */
+                if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
+                    PyErr_Format(PyExc_TypeError,
+                                 "%.200s%.200s argument after ** "
+                                 "must be a mapping, not %.200s",
+                                 PyEval_GetFuncName(func),
+                                 PyEval_GetFuncDesc(func),
+                                 kwdict->ob_type->tp_name);
+                }
+                goto fail;
+            }
+            Py_DECREF(kwdict);
+            kwdict = d;
+        }
+    }
+    if (flags & CALL_FLAG_VAR) {
+        stararg = EXT_POP(*pp_stack);
+        if (!PyTuple_Check(stararg)) {
+            PyObject *t = NULL;
+            t = PySequence_Tuple(stararg);
+            if (t == NULL) {
+                if (PyErr_ExceptionMatches(PyExc_TypeError)) {
+                    PyErr_Format(PyExc_TypeError,
+                                 "%.200s%.200s argument after * "
+                                 "must be a sequence, not %.200s",
+                                 PyEval_GetFuncName(func),
+                                 PyEval_GetFuncDesc(func),
+                                 stararg->ob_type->tp_name);
+                }
+                goto fail;
+            }
+            Py_DECREF(stararg);
+            stararg = t;
+        }
+        nstar = PyTuple_GET_SIZE(stararg);
+    }
+    if (nk > 0) {
+        kwdict = update_keyword_args(kwdict, nk, pp_stack, func);
+        if (kwdict == NULL)
+            goto fail;
+    }
+    callargs = update_star_args(na, nstar, stararg, pp_stack);
+    if (callargs == NULL)
+        goto fail;
+    Py_XDECREF(stararg);
+    *kws = kwdict;
+    return callargs;
+fail:
+    Py_XDECREF(callargs);
+    Py_XDECREF(kwdict);
+    Py_XDECREF(stararg);
+    return NULL;
+}
+
+
+#define CALL_SURROGATE(func, n) call_surrogate(func, #func, stack_pointer, n)
+
+static PyObject *
+call_surrogate(PyObject *func, char *name, PyObject **args, int n) {
+    PyFrameObject *f = fast_frame(func, args, n);
+    PyObject *result;
+    // printf("@@@@ %s\n", name);
+    if (f == NULL)
+        return NULL;
+    result = PyEval_EvalFrame_Py(f);
+    return result;
+}
+
+#undef PREDICT
+#undef PREDICTED
+#undef PREDICTED_WITH_ARG
+#define PREDICT(x) ((void)0)
+#define PREDICTED(x) ((void)0)
+#define PREDICTED_WITH_ARG(x) ((void)0)
+
+/* In order to make calls to PyEval_EvalFrame_Py tail calls,
+   PyEval_EvalFrame_Py consumes a reference to f */
+static PyObject *PyEval_EvalFrame_Py(PyFrameObject *f)
+{
+#ifdef DXPAIRS
+    int lastopcode = 0;
+#endif
+    register PyObject **stack_pointer;  /* Next free slot in value stack */
+    register unsigned char *next_instr;
+    register int opcode;        /* Current opcode */
+    register int oparg;         /* Current opcode argument, if any */
+    register enum why_code why = WHY_NOT; /* Reason for block stack unwind */
+    register int err;           /* Error status -- nonzero if error */
+    register PyObject *x;       /* Result object -- NULL if error */
+    register PyObject *v;       /* Temporary objects popped off stack */
+    register PyObject *w;
+    register PyObject *u;
+    register PyObject *t;
+    register PyObject **fastlocals, **freevars;
+    PyObject *retval = NULL;            /* Return value */
+    PyThreadState *tstate = PyThreadState_GET();
+    PyCodeObject *co = f->f_code;
+
+    /* when tracing we set things up so that
+
+           not (instr_lb <= current_bytecode_offset < instr_ub)
+
+       is true when the line being executed has changed.  The
+       initial values are such as to make this false the first
+       time it is tested. */
+    int instr_ub = -1, instr_lb = 0, instr_prev = -1;
+
+    unsigned char *first_instr;
+    PyObject *names;
+    PyObject *consts;
+
+#if USE_COMPUTED_GOTOS
+/* Import the static jump table */
+#include "opcode_targets.h"
+#endif
+
+    /* push frame */
+    if (Py_EnterRecursiveCall("")) {
+        Py_DECREF(f);
+        return NULL;
+    }
+
+    tstate->frame = f;
+
+    if (tstate->use_tracing) {
+        if (tstate->c_tracefunc != NULL) {
+            /* tstate->c_tracefunc, if defined, is a
+               function that will be called on *every* entry
+               to a code block.  Its return value, if not
+               None, is a function that will be called at
+               the start of each executed line of code.
+               (Actually, the function must return itself
+               in order to continue tracing.)  The trace
+               functions are called with three arguments:
+               a pointer to the current frame, a string
+               indicating why the function is called, and
+               an argument which depends on the situation.
+               The global trace function is also called
+               whenever an exception is detected. */
+            if (call_trace_protected(tstate->c_tracefunc,
+                                     tstate->c_traceobj,
+                                     f, PyTrace_CALL, Py_None)) {
+                /* Trace function raised an error */
+                goto exit_eval_frame;
+            }
+        }
+        if (tstate->c_profilefunc != NULL) {
+            /* Similar for c_profilefunc, except it needn't
+               return itself and isn't called for "line" events */
+            if (call_trace_protected(tstate->c_profilefunc,
+                                     tstate->c_profileobj,
+                                     f, PyTrace_CALL, Py_None)) {
+                /* Profile function raised an error */
+                goto exit_eval_frame;
+            }
+        }
+    }
+
+    names = co->co_names;
+    consts = co->co_consts;
+    fastlocals = f->f_localsplus;
+    freevars = f->f_localsplus + co->co_nlocals;
+    first_instr = (unsigned char*) PyBytes_AS_STRING(co->co_code);
+    /* An explanation is in order for the next line.
+
+       f->f_lasti now refers to the index of the last instruction
+       executed.  You might think this was obvious from the name, but
+       this wasn't always true before 2.3!  PyFrame_New now sets
+       f->f_lasti to -1 (i.e. the index *before* the first instruction)
+       and YIELD_VALUE doesn't fiddle with f_lasti any more.  So this
+       does work.  Promise.
+
+       When the PREDICT() macros are enabled, some opcode pairs follow in
+       direct succession without updating f->f_lasti.  A successful
+       prediction effectively links the two codes together as if they
+       were a single new opcode; accordingly,f->f_lasti will point to
+       the first code in the pair (for instance, GET_ITER followed by
+       FOR_ITER is effectively a single opcode and f->f_lasti will point
+       at to the beginning of the combined pair.)
+    */
+    next_instr = first_instr + f->f_lasti + 1;
+    stack_pointer = f->f_stacktop;
+    assert(stack_pointer != NULL);
+    f->f_stacktop = NULL;       /* remains NULL unless yield suspends frame */
+
+    if (co->co_flags & CO_GENERATOR) {
+        if (f->f_exc_type != NULL && f->f_exc_type != Py_None) {
+            /* We were in an except handler when we left,
+               restore the exception state which was put aside
+               (see YIELD_VALUE). */
+            swap_exc_state(tstate, f);
+        }
+        else
+            save_exc_state(tstate, f);
+    }
+
+#ifdef LLTRACE
+    lltrace = PyDict_GetItemString(f->f_globals, "__lltrace__") != NULL;
+#endif
+
+
+    err = 0;
+    x = Py_None;        /* Not a reference, just anything non-NULL */
+    w = NULL;
+
+    for (;;) {
+#ifdef WITH_TSC
+        if (inst1 == 0) {
+            /* Almost surely, the opcode executed a break
+               or a continue, preventing inst1 from being set
+               on the way out of the loop.
+            */
+            READ_TIMESTAMP(inst1);
+            loop1 = inst1;
+        }
+        dump_tsc(opcode, ticked, inst0, inst1, loop0, loop1,
+                 intr0, intr1);
+        ticked = 0;
+        inst1 = 0;
+        intr0 = 0;
+        intr1 = 0;
+        READ_TIMESTAMP(loop0);
+#endif
+        assert(stack_pointer >= f->f_valuestack); /* else underflow */
+        assert(STACK_LEVEL() <= co->co_stacksize+1);  /* else overflow */
+
+        /* Do periodic things.  Doing this every time through
+           the loop would add too much overhead, so we do it
+           only every Nth instruction.  We also do it if
+           ``pendingcalls_to_do'' is set, i.e. when an asynchronous