Anonymous avatar Anonymous committed 258a13b

Various updates

Comments (0)

Files changed (5)

inline_feedback.patch

-diff -r 873fbe103aea Python/eval.cc
---- a/Python/eval.cc	Mon Feb 15 12:02:13 2010 +0100
-+++ b/Python/eval.cc	Mon Feb 15 12:17:23 2010 +0100
-@@ -325,6 +325,7 @@
+diff -r 10fa728f253a Python/eval.cc
+--- a/Python/eval.cc	Mon Mar 29 06:38:31 2010 +0200
++++ b/Python/eval.cc	Mon Mar 29 21:07:34 2010 +0200
+@@ -349,6 +349,7 @@
  /* Record data for use in generating optimized machine code. */
  static void record_type(PyCodeObject *, int, int, int, PyObject *);
  static void record_func(PyCodeObject *, int, int, int, PyObject *);
  static void record_object(PyCodeObject *, int, int, int, PyObject *);
  static void inc_feedback_counter(PyCodeObject *, int, int, int);
  #endif  /* WITH_LLVM */
-@@ -944,6 +945,8 @@
+@@ -968,6 +969,8 @@
  	if(rec_feedback){record_object(co, opcode, f->f_lasti, arg_index, obj);}
  #define RECORD_FUNC(obj) \
  	if(rec_feedback){record_func(co, opcode, f->f_lasti, 0, obj);}
  #define RECORD_TRUE() \
  	if(rec_feedback){inc_feedback_counter(co, opcode, f->f_lasti, PY_FDO_JUMP_TRUE);}
  #define RECORD_FALSE() \
-@@ -956,6 +959,7 @@
+@@ -980,6 +983,7 @@
  #define RECORD_TYPE(arg_index, obj)
  #define RECORD_OBJECT(arg_index, obj)
  #define RECORD_FUNC(obj)
  #define RECORD_TRUE()
  #define RECORD_FALSE()
  #define RECORD_NONBOOLEAN()
-@@ -2829,7 +2833,12 @@
+@@ -2830,17 +2834,25 @@
  				/* Duplicate this bit of logic from
  				 * _PyEval_CallFunction(). */
  				PyObject **func = stack_pointer - num_args - 1;
 -				RECORD_FUNC(*func);
-+				if (PyCFunction_Check(*func)) {
-+					RECORD_FUNC(*func);
-+				}
-+				else {
-+					RECORD_INLINE(*func);
-+				}
+-				/* For C functions, record the types passed, 
+-				 * in order to do potential inlining. */
+-				if (PyCFunction_Check(*func) &&
+-					(PyCFunction_GET_FLAGS(*func) &
+-					    METH_ARG_RANGE)) {
++                                if (PyCFunction_Check(*func)) {
++                                    RECORD_FUNC(*func);
++                                    /* For C functions, record the types
++                                     * passed, in order to do potential 
++                                     * inlining. */
++                                    if (PyCFunction_GET_FLAGS(*func) &
++					    METH_ARG_RANGE) {
+ 					for(int i = 0; i < num_args; i++) {
+ 						RECORD_TYPE(i + 1,
+ 						       stack_pointer[-i-1]);
+ 					}
+-				}
++                                    }
++                                }
++                                else {
++                                    if (PyFunction_Check(*func) ||
++                                        PyMethod_Check(*func)) {
++                                        RECORD_INLINE(*func);
++                                    }
++                                }
  			}
  #endif
  			x = _PyEval_CallFunction(stack_pointer,
-@@ -4825,6 +4834,23 @@
+@@ -4867,6 +4879,23 @@
  	feedback.AddFuncSeen(func);
  }
  
  // Records obj into the feedback array. Only use this on long-lived objects,
  // since the feedback system will keep any object live forever.
  void record_object(PyCodeObject *co, int expected_opcode,
-diff -r 873fbe103aea Python/llvm_fbuilder.cc
---- a/Python/llvm_fbuilder.cc	Mon Feb 15 12:02:13 2010 +0100
-+++ b/Python/llvm_fbuilder.cc	Mon Feb 15 12:17:23 2010 +0100
-@@ -2160,16 +2160,15 @@
+diff -r 10fa728f253a Python/llvm_fbuilder.cc
+--- a/Python/llvm_fbuilder.cc	Mon Mar 29 06:38:31 2010 +0200
++++ b/Python/llvm_fbuilder.cc	Mon Mar 29 21:07:34 2010 +0200
+@@ -2205,6 +2205,12 @@
          return;
      }
  
--    // Only optimize monomorphic callsites.
--    llvm::SmallVector<PyMethodDef*, 3> fdo_data;
--    feedback->GetSeenFuncsInto(fdo_data);
--
-     if (feedback->FuncsOverflowed()) {
-         CF_INC_STATS(no_opt_polymorphic);
-         this->CALL_FUNCTION_safe(oparg);
-         return;
-     }
- 
-+    // Only optimize monomorphic callsites.
-+    llvm::SmallVector<PyMethodDef*, 3> fdo_data;
-+    feedback->GetSeenFuncsInto(fdo_data);
-     if (fdo_data.size() != 1) {
- #ifdef Py_WITH_INSTRUMENTATION
-         if (fdo_data.size() == 0)
-diff -r 873fbe103aea Util/RuntimeFeedback.cc
---- a/Util/RuntimeFeedback.cc	Mon Feb 15 12:02:13 2010 +0100
-+++ b/Util/RuntimeFeedback.cc	Mon Feb 15 12:17:23 2010 +0100
-@@ -14,12 +14,92 @@
++    if (feedback->FuncsOverflowed()) {
++        CF_INC_STATS(no_opt_polymorphic);
++        this->CALL_FUNCTION_safe(oparg);
++        return;
++    }
++
+     // Only optimize monomorphic callsites.
+     llvm::SmallVector<PyMethodDef*, 3> fdo_data;
+     feedback->GetSeenFuncsInto(fdo_data);
+diff -r 10fa728f253a Util/RuntimeFeedback.cc
+--- a/Util/RuntimeFeedback.cc	Mon Mar 29 06:38:31 2010 +0200
++++ b/Util/RuntimeFeedback.cc	Mon Mar 29 21:07:34 2010 +0200
+@@ -14,12 +14,97 @@
  using llvm::SmallVector;
  
  
 +        this->flags |= PY_INLINE_FUNCTION;
 +    }
 +
++    assert(PyFunction_Check(func));
++
 +    if (((PyFunctionObject *)func)->func_defaults)
-+        this->defaults = PyTuple_Size(((PyFunctionObject *)func)->func_defaults);
++        this->defaults =
++            PyTuple_Size(((PyFunctionObject *)func)->func_defaults);
 +    else
 +        this->defaults = 0;
 +
 +        }
 +    }
 +    else if (PyMethod_Check(a)) {
-+        if (PyMethod_GET_SELF(a))
++        if (PyMethod_GET_SELF(a)) {
 +            if ((record->flags & PY_INLINE_METHOD) == 0)
 +                return false;
-+        else
++        }
++        else {
 +            if ((record->flags & PY_INLINE_UNBOUND) == 0)
 +                return false;
++        }
 +
 +        func = (PyFunctionObject *)PyMethod_GET_FUNCTION(a);
 +        if ((PyCodeObject *)PyFunction_GET_CODE(func) != record->code) {
  PyLimitedFeedback::PyLimitedFeedback()
  {
  }
-@@ -32,6 +112,15 @@
+@@ -32,6 +117,15 @@
              Py_XINCREF(value);
              this->data_[i] = src.data_[i];
          }
          else {
              this->data_[i] = src.data_[i];
          }
-@@ -109,11 +198,20 @@
+@@ -109,11 +203,20 @@
  void
  PyLimitedFeedback::Clear()
  {
          this->data_[i].setPointer(NULL);
          this->data_[i].setInt(0);
      }
-@@ -164,6 +262,12 @@
+@@ -164,6 +267,12 @@
  void
  PyLimitedFeedback::AddFuncSeen(PyObject *obj)
  {
      assert(this->InFuncMode());
      this->SetFlagBit(FUNC_MODE_BIT, true);
  
-@@ -173,10 +277,8 @@
+@@ -173,8 +282,10 @@
          this->SetFlagBit(SAW_A_NULL_OBJECT_BIT, true);
          return;
      }
--    if (!PyCFunction_Check(obj)) {
--        this->SetMegamorphic();
-+    if (!PyCFunction_Check(obj)) 
+-    if (!PyCFunction_Check(obj))
++    if (!PyCFunction_Check(obj)) {
++        this->SetMegamorphic();
          return;
--    }
++    }
  
      for (int i = 0; i < PyLimitedFeedback::NUM_POINTERS; ++i) {
          PyMethodDef *value = (PyMethodDef *)this->data_[i].getPointer();
-@@ -197,9 +299,12 @@
+@@ -195,9 +306,12 @@
  PyLimitedFeedback::GetSeenFuncsInto(
      SmallVector<PyMethodDef*, 3> &result) const
  {
      if (this->GetFlagBit(SAW_A_NULL_OBJECT_BIT)) {
          // Saw a NULL value, so add NULL to the result.
          result.push_back(NULL);
-@@ -212,6 +317,70 @@
+@@ -210,9 +324,80 @@
      }
  }
  
 +        return;
 +    }
 +
-+    if (!(PyFunction_Check(obj) || PyMethod_Check(obj))) {
++    if (PyMethod_Check(obj)) {
++        if (!PyFunction_Check(PyMethod_GET_FUNCTION(obj))) {
++            this->SetMegamorphic();
++            return;
++        }
++    }
++    else if (!PyFunction_Check(obj)) {
 +        this->SetMegamorphic();
 +        return;
 +    }
  
  PyFullFeedback::PyFullFeedback()
      : counters_(/* Zero out the array. */),
-diff -r 873fbe103aea Util/RuntimeFeedback.h
---- a/Util/RuntimeFeedback.h	Mon Feb 15 12:02:13 2010 +0100
-+++ b/Util/RuntimeFeedback.h	Mon Feb 15 12:17:23 2010 +0100
++      megamorphic_(false),
+       usage_(UnknownMode)
+ {
+ }
+@@ -230,6 +415,7 @@
+         }
+         this->data_.insert(obj);
+     }
++    this->megamorphic_ = src.megamorphic_;
+ }
+ 
+ PyFullFeedback::~PyFullFeedback()
+@@ -305,8 +491,10 @@
+     this->usage_ = FuncMode;
+ 
+     // We only record C functions for now.
+-    if (!PyCFunction_Check(obj))
++    if (!PyCFunction_Check(obj)) {
++        this->SetMegamorphic();
+         return;
++    }
+     if (obj == NULL)
+         this->data_.insert(NULL);
+     else if (!this->data_.count(obj)) {
+diff -r 10fa728f253a Util/RuntimeFeedback.h
+--- a/Util/RuntimeFeedback.h	Mon Mar 29 06:38:31 2010 +0200
++++ b/Util/RuntimeFeedback.h	Mon Mar 29 21:07:34 2010 +0200
 @@ -39,6 +39,19 @@
  // PY_FDO_JUMP_FALSE - PY_FDO_JUMP_NON_BOOLEAN).
  enum { PY_FDO_JUMP_TRUE = 0, PY_FDO_JUMP_FALSE, PY_FDO_JUMP_NON_BOOLEAN };
      // There are three counters available.  Their storage space
      // overlaps with the object record, so you can't use both.  They
      // saturate rather than wrapping on overflow.
-@@ -100,7 +121,8 @@
+@@ -73,6 +94,10 @@
+     // Assignment copies the list of collected objects, fixing up refcounts.
+     PyLimitedFeedback &operator=(PyLimitedFeedback rhs);
+ 
++    void SetMegamorphic() {
++        SetFlagBit(SAW_MORE_THAN_THREE_OBJS_BIT, true);
++    }
++
+ private:
+     // 'index' must be between 0 and 5 inclusive.
+     void SetFlagBit(unsigned index, bool value);
+@@ -96,7 +121,8 @@
          OBJECT_MODE_BIT = 3,
      //   4: True if this instance is being used in function-gathering mode.
          FUNC_MODE_BIT = 4,
      };
      //
      // The pointers in this array start out NULL and are filled from
-@@ -109,17 +131,26 @@
+@@ -105,17 +131,26 @@
      llvm::PointerIntPair<void*, /*bits used from bottom of pointer=*/2>
          data_[NUM_POINTERS];
  
      }
  };
  
+@@ -129,13 +164,13 @@
+     void AddObjectSeen(PyObject *obj);
+     // Clears result and fills it with the set of seen objects.
+     void GetSeenObjectsInto(llvm::SmallVector<PyObject*, 3> &result) const;
+-    bool ObjectsOverflowed() const { return false; }
++    bool ObjectsOverflowed() const { return this->megamorphic_; }
+ 
+     // Record that a given function was called.
+     void AddFuncSeen(PyObject *obj);
+     // Clears result and fills it with the set of seen function objects.
+     void GetSeenFuncsInto(llvm::SmallVector<PyMethodDef*, 3> &result) const;
+-    bool FuncsOverflowed() const { return false; }
++    bool FuncsOverflowed() const { return this->megamorphic_; }
+ 
+     void IncCounter(unsigned counter_id);
+     uintptr_t GetCounter(unsigned counter_id) const;
+@@ -146,6 +181,10 @@
+     // Assignment copies the list of collected objects, fixing up refcounts.
+     PyFullFeedback &operator=(PyFullFeedback rhs);
+ 
++    void SetMegamorphic() {
++        this->megamorphic_ = true;
++    }
++
+ private:
+     // Assume three pointers in the set to start with. We store either
+     // PyObject *s (when in object mode) or PyMethodDef *s (when in function
+@@ -156,6 +195,7 @@
+ 
+     ObjSet data_;
+     uintptr_t counters_[3];
++    bool megamorphic_;
+ 
+     enum UsageMode {
+         UnknownMode,

jit_recompilation.patch

 -#endif /* !Py_LLVMFUNCTIONOBJECT_H */
 diff -r a994d3c424c0 Include/code.h
 --- a/Include/code.h	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Include/code.h	Sun Mar 14 19:59:40 2010 +0100
++++ b/Include/code.h	Sat Mar 20 21:42:20 2010 +0100
 @@ -3,25 +3,13 @@
  #ifndef Py_CODE_H
  #define Py_CODE_H
  #ifdef __cplusplus
 diff -r a994d3c424c0 Include/dictobject.h
 --- a/Include/dictobject.h	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Include/dictobject.h	Sun Mar 14 19:59:40 2010 +0100
++++ b/Include/dictobject.h	Sat Mar 20 21:42:20 2010 +0100
 @@ -155,16 +155,18 @@
  /* Register the given code object as depending on the state of this dict.
     The same code object may be registered multiple times without error. Returns
  #ifdef __cplusplus
 diff -r a994d3c424c0 Include/frameobject.h
 --- a/Include/frameobject.h	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Include/frameobject.h	Sun Mar 14 19:59:40 2010 +0100
++++ b/Include/frameobject.h	Sat Mar 20 21:42:20 2010 +0100
 @@ -55,6 +55,7 @@
         Without this, Python code can cause active generators to flip between
         LLVM and the interpreter at will, which is a bad idea. */
 -#endif // PYTHON_LLVM_COMPILE_H
 diff -r a994d3c424c0 Include/nativefunctionobject.h
 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
-+++ b/Include/nativefunctionobject.h	Sun Mar 14 19:59:40 2010 +0100
++++ b/Include/nativefunctionobject.h	Sat Mar 20 21:42:20 2010 +0100
 @@ -0,0 +1,40 @@
 +#ifndef Py_NATIVEFUNCTIONOBJECT_H
 +#define Py_NATIVEFUNCTIONOBJECT_H
 +#endif /* !Py_NATIVEFUNCTIONOBJECT_H */
 diff -r a994d3c424c0 Include/nativefunctionobject_fwd.h
 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
-+++ b/Include/nativefunctionobject_fwd.h	Sun Mar 14 19:59:40 2010 +0100
++++ b/Include/nativefunctionobject_fwd.h	Sat Mar 20 21:42:20 2010 +0100
 @@ -0,0 +1,104 @@
 +
 +#ifndef Py_NATIVEFUNCTIONOBJECT_FWD_H
 +
 diff -r a994d3c424c0 Include/object.h
 --- a/Include/object.h	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Include/object.h	Sun Mar 14 19:59:40 2010 +0100
++++ b/Include/object.h	Sat Mar 20 21:42:20 2010 +0100
 @@ -456,7 +456,7 @@
  PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *);
  PyAPI_FUNC(unsigned int) PyType_ClearCache(void);
  PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int);
 diff -r a994d3c424c0 Lib/test/test_llvm.py
 --- a/Lib/test/test_llvm.py	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Lib/test/test_llvm.py	Sun Mar 14 19:59:40 2010 +0100
++++ b/Lib/test/test_llvm.py	Sat Mar 20 21:42:20 2010 +0100
 @@ -161,7 +161,7 @@
      @at_each_optimization_level
      def test_llvm_compile(self, level):
          # to cause invalidation due to no-op assignments to the globals dict
 diff -r a994d3c424c0 Lib/test/test_sys.py
 --- a/Lib/test/test_sys.py	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Lib/test/test_sys.py	Sun Mar 14 19:59:40 2010 +0100
++++ b/Lib/test/test_sys.py	Sat Mar 20 21:42:20 2010 +0100
 @@ -609,7 +609,7 @@
          extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
                   ncells + nfrees - 1
          # function
 diff -r a994d3c424c0 Makefile.pre.in
 --- a/Makefile.pre.in	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Makefile.pre.in	Sun Mar 14 19:59:40 2010 +0100
++++ b/Makefile.pre.in	Sat Mar 20 21:42:20 2010 +0100
 @@ -387,7 +387,7 @@
  
  ifneq (@BUILD_LLVM@, disabled)
  		Util/EventTimer.h \
 diff -r a994d3c424c0 Modules/_llvm.c
 --- a/Modules/_llvm.c	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Modules/_llvm.c	Sun Mar 14 19:59:40 2010 +0100
++++ b/Modules/_llvm.c	Sat Mar 20 21:42:20 2010 +0100
 @@ -7,8 +7,7 @@
  */
  
 -};
 diff -r a994d3c424c0 Objects/codeobject.c
 --- a/Objects/codeobject.c	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Objects/codeobject.c	Sun Mar 14 19:59:40 2010 +0100
++++ b/Objects/codeobject.c	Sat Mar 20 21:42:20 2010 +0100
 @@ -1,6 +1,5 @@
  #include "Python.h"
  #include "code.h"
  	PyObject_DEL(co);
 diff -r a994d3c424c0 Objects/dictobject.c
 --- a/Objects/dictobject.c	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Objects/dictobject.c	Sun Mar 14 19:59:40 2010 +0100
++++ b/Objects/dictobject.c	Sat Mar 20 21:42:20 2010 +0100
 @@ -10,6 +10,7 @@
  #include "Python.h"
  
  // We split the real work of notify_watchers() out into a separate function so
 diff -r a994d3c424c0 Objects/frameobject.c
 --- a/Objects/frameobject.c	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Objects/frameobject.c	Sun Mar 14 19:59:40 2010 +0100
++++ b/Objects/frameobject.c	Sat Mar 20 21:42:20 2010 +0100
 @@ -434,6 +434,10 @@
  	Py_CLEAR(f->f_exc_value);
  	Py_CLEAR(f->f_exc_traceback);
  #endif
 diff -r a994d3c424c0 Objects/nativefunctionobject.cc
 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
-+++ b/Objects/nativefunctionobject.cc	Sun Mar 14 19:59:40 2010 +0100
-@@ -0,0 +1,487 @@
++++ b/Objects/nativefunctionobject.cc	Sat Mar 20 21:42:20 2010 +0100
+@@ -0,0 +1,492 @@
 +
 +#include "Python.h"
 +#include "nativefunctionobject_fwd.h"
 +
 +    if (func->code) {
 +        _PyCode_InvalidateMachineCode(func->code, reason);
++        assert(func->code->co_native == func);
++        Py_CLEAR(func->code->co_native);
++
++        func->code = NULL;
 +    }
 +}
 +
 +    }
 +
 +    if (func->watching == NULL ||
++        func->watching[WATCHING_GLOBALS] == NULL ||
 +        (func->watching[WATCHING_GLOBALS] == globals &&
 +         func->watching[WATCHING_BUILTINS] == builtins)) {
 +        return 1;
 +
 diff -r a994d3c424c0 Objects/typeobject.c
 --- a/Objects/typeobject.c	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Objects/typeobject.c	Sun Mar 14 19:59:40 2010 +0100
++++ b/Objects/typeobject.c	Sat Mar 20 21:42:20 2010 +0100
 @@ -110,9 +110,10 @@
  
  #ifdef WITH_LLVM
  int
 diff -r a994d3c424c0 Python/eval.cc
 --- a/Python/eval.cc	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Python/eval.cc	Sun Mar 14 19:59:40 2010 +0100
++++ b/Python/eval.cc	Sat Mar 20 21:42:20 2010 +0100
 @@ -17,7 +17,6 @@
  #include "code.h"
  #include "frameobject.h"
  	f->f_use_llvm = co->co_use_llvm;
 diff -r a994d3c424c0 Python/global_llvm_data.cc
 --- a/Python/global_llvm_data.cc	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Python/global_llvm_data.cc	Sun Mar 14 19:59:40 2010 +0100
++++ b/Python/global_llvm_data.cc	Sat Mar 20 21:42:20 2010 +0100
 @@ -9,7 +9,7 @@
  #include "Util/PyAliasAnalysis.h"
  #include "Util/SingleFunctionInliner.h"
      llvm::cl::ParseEnvironmentOptions("python", "PYTHONLLVMFLAGS", "", true);
 diff -r a994d3c424c0 Python/global_llvm_data_fwd.h
 --- a/Python/global_llvm_data_fwd.h	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Python/global_llvm_data_fwd.h	Sun Mar 14 19:59:40 2010 +0100
++++ b/Python/global_llvm_data_fwd.h	Sat Mar 20 21:42:20 2010 +0100
 @@ -21,9 +21,6 @@
  #define Py_DEFAULT_JIT_OPT_LEVEL 2
  #define Py_MAX_LLVM_OPT_LEVEL 3
      struct PyGlobalLlvmData *);
 diff -r a994d3c424c0 Python/llvm_compile.cc
 --- a/Python/llvm_compile.cc	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Python/llvm_compile.cc	Sun Mar 14 19:59:40 2010 +0100
++++ b/Python/llvm_compile.cc	Sat Mar 20 21:42:20 2010 +0100
 @@ -1,7 +1,5 @@
  #include "Python.h"
 -#include "llvm_compile.h"
 +
 diff -r a994d3c424c0 Python/llvm_fbuilder.cc
 --- a/Python/llvm_fbuilder.cc	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Python/llvm_fbuilder.cc	Sun Mar 14 19:59:40 2010 +0100
++++ b/Python/llvm_fbuilder.cc	Sat Mar 20 21:42:20 2010 +0100
 @@ -4,6 +4,7 @@
  #include "code.h"
  #include "opcode.h"
  
 diff -r a994d3c424c0 Python/llvm_fbuilder.h
 --- a/Python/llvm_fbuilder.h	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Python/llvm_fbuilder.h	Sun Mar 14 19:59:40 2010 +0100
++++ b/Python/llvm_fbuilder.h	Sat Mar 20 21:42:20 2010 +0100
 @@ -22,6 +22,7 @@
  
  struct PyCodeObject;
      llvm::Function *const function_;
 diff -r a994d3c424c0 Python/llvm_inline_functions.c
 --- a/Python/llvm_inline_functions.c	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Python/llvm_inline_functions.c	Sun Mar 14 19:59:40 2010 +0100
++++ b/Python/llvm_inline_functions.c	Sat Mar 20 21:42:20 2010 +0100
 @@ -10,6 +10,7 @@
  
  #include "Python.h"
  
 diff -r a994d3c424c0 Unittests/DictTest.cc
 --- a/Unittests/DictTest.cc	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Unittests/DictTest.cc	Sun Mar 14 19:59:40 2010 +0100
++++ b/Unittests/DictTest.cc	Sat Mar 20 21:42:20 2010 +0100
 @@ -1,6 +1,10 @@
  #include "Python.h"
 +#include "nativefunctionobject.h"
      DictWatcherTest()
 diff -r a994d3c424c0 Util/PyTypeBuilder.h
 --- a/Util/PyTypeBuilder.h	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Util/PyTypeBuilder.h	Sun Mar 14 19:59:40 2010 +0100
++++ b/Util/PyTypeBuilder.h	Sat Mar 20 21:42:20 2010 +0100
 @@ -10,6 +10,7 @@
  #include "code.h"
  #include "frameobject.h"
      DEFINE_FIELD(PyFrameObject, f_iblock)
 diff -r a994d3c424c0 Util/RuntimeFeedback.cc
 --- a/Util/RuntimeFeedback.cc	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Util/RuntimeFeedback.cc	Sun Mar 14 19:59:40 2010 +0100
++++ b/Util/RuntimeFeedback.cc	Sat Mar 20 21:42:20 2010 +0100
 @@ -173,8 +173,10 @@
          this->SetFlagBit(SAW_A_NULL_OBJECT_BIT, true);
          return;
      else if (!this->data_.count(obj)) {
 diff -r a994d3c424c0 Util/RuntimeFeedback.h
 --- a/Util/RuntimeFeedback.h	Wed Mar 10 22:29:28 2010 +0100
-+++ b/Util/RuntimeFeedback.h	Sun Mar 14 19:59:40 2010 +0100
++++ b/Util/RuntimeFeedback.h	Sat Mar 20 21:42:20 2010 +0100
 @@ -73,6 +73,10 @@
      // Assignment copies the list of collected objects, fixing up refcounts.
      PyLimitedFeedback &operator=(PyLimitedFeedback rhs);

python_python_inlineing.patch

-diff -r 2817eef6f379 Include/frameobject.h
---- a/Include/frameobject.h	Mon Feb 15 12:17:24 2010 +0100
-+++ b/Include/frameobject.h	Mon Feb 15 12:23:04 2010 +0100
-@@ -100,6 +100,9 @@
+diff -r 9d3de5a4d8ae Include/frameobject.h
+--- a/Include/frameobject.h	Mon Mar 29 21:07:34 2010 +0200
++++ b/Include/frameobject.h	Sun Jun 13 09:15:53 2010 +0200
+@@ -96,6 +96,9 @@
      _PYGUARD_CFUNC,
      _PYGUARD_BRANCH,
      _PYGUARD_STORE_SUBSCR,
 +    _PYGUARD_INLINE_DEFAULTS,
  };
  
- enum _PyRecordType {
-diff -r 2817eef6f379 Lib/test/test_inlineing.py
+ /* Standard object interface */
+diff -r 9d3de5a4d8ae Lib/test/test_inlineing.py
 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
-+++ b/Lib/test/test_inlineing.py	Mon Feb 15 12:23:04 2010 +0100
++++ b/Lib/test/test_inlineing.py	Sun Jun 13 09:15:53 2010 +0200
 @@ -0,0 +1,616 @@
 +# Tests for our minimal LLVM wrappers
 +import __future__
 +""", optimization_level=None)
 +
 +            spin_until_hot(foo, [[], lambda: None], [[], lambda: 1])
-+            self.assertEqual(foo.__code__.__use_llvm__, True)
++            self.assertEqual(foo.__code__.co_use_jit, True)
 +
 +            def change_builtins():
-+                self.assertEqual(foo.__code__.__use_llvm__, True)
++                self.assertEqual(foo.__code__.co_use_jit, True)
 +                __builtin__.len = lambda x: 7
 +
 +            def run_test():
-+                self.assertEqual(foo.__code__.__use_llvm__, True)
++                self.assertEqual(foo.__code__.co_use_jit, True)
 +                self.assertEqual(foo.__code__.co_fatalbailcount, 0)
 +                #~ self.assertEqual(foo([], change_builtins), 7)
 +                sys.setbailerror(True)
 +                finally:
 +                    sys.setbailerror(False)
 +                self.assertEqual(foo.__code__.co_fatalbailcount, 1)
-+                self.assertEqual(foo.__code__.__use_llvm__, False)
++                self.assertEqual(foo.__code__.co_use_jit, False)
 +            run_test()
 +
 +
 +
 +if __name__ == "__main__":
 +    test_main()
-diff -r 2817eef6f379 Lib/test/test_llvm.py
---- a/Lib/test/test_llvm.py	Mon Feb 15 12:17:24 2010 +0100
-+++ b/Lib/test/test_llvm.py	Mon Feb 15 12:23:04 2010 +0100
-@@ -2603,7 +2603,8 @@
+diff -r 9d3de5a4d8ae Lib/test/test_llvm.py
+--- a/Lib/test/test_llvm.py	Mon Mar 29 21:07:34 2010 +0200
++++ b/Lib/test/test_llvm.py	Sun Jun 13 09:15:53 2010 +0200
+@@ -2605,7 +2605,7 @@
      return len(x)
  """, optimization_level=None)
  
 -            spin_until_hot(foo, [[], lambda: None])
-+            # do not trigger inlineing
 +            spin_until_hot(foo, [[], lambda: None], [[], lambda: 1])
-             self.assertEqual(foo.__code__.__use_llvm__, True)
+             self.assertEqual(foo.__code__.co_use_jit, True)
  
              def change_builtins():
-diff -r 2817eef6f379 Python/eval.cc
---- a/Python/eval.cc	Mon Feb 15 12:17:24 2010 +0100
-+++ b/Python/eval.cc	Mon Feb 15 12:23:04 2010 +0100
-@@ -268,6 +268,9 @@
+diff -r 9d3de5a4d8ae Python/eval.cc
+--- a/Python/eval.cc	Mon Mar 29 21:07:34 2010 +0200
++++ b/Python/eval.cc	Sun Jun 13 09:15:53 2010 +0200
+@@ -292,6 +292,9 @@
  			GUARD_CASE(_PYGUARD_CFUNC)
  			GUARD_CASE(_PYGUARD_BRANCH)
  			GUARD_CASE(_PYGUARD_STORE_SUBSCR)
  			default:
  				wrapper << ((int)frame->f_guard_type);
  		}
-diff -r 2817eef6f379 Python/llvm_compile.cc
---- a/Python/llvm_compile.cc	Mon Feb 15 12:17:24 2010 +0100
-+++ b/Python/llvm_compile.cc	Mon Feb 15 12:23:04 2010 +0100
-@@ -424,3 +424,87 @@
-     return fbuilder.function();
+diff -r 9d3de5a4d8ae Python/llvm_compile.cc
+--- a/Python/llvm_compile.cc	Mon Mar 29 21:07:34 2010 +0200
++++ b/Python/llvm_compile.cc	Sun Jun 13 09:15:53 2010 +0200
+@@ -174,30 +174,98 @@
+     return 0;
  }
  
+-extern "C" _LlvmFunction *
+-_PyCode_ToLlvmIr(PyCodeObject *code)
 +bool is_inlineable_callsite(PyCodeObject *code, size_t lasti)
-+{
+ {
+-    if (!PyCode_Check(code)) {
+-        PyErr_Format(PyExc_TypeError, "Expected code object, not '%.500s'",
+-                     code->ob_type->tp_name);
+-        return NULL;
+-    }
+-    if (!PyString_Check(code->co_code)) {
+-        PyErr_SetString(PyExc_SystemError,
+-                        "non-string codestring in code object");
+-        return NULL;
 +    const PyFeedbackMap *map = code->co_runtime_feedback;
 +    if (map == NULL)
 +        return false;
 +        // Iline recursive calls at most once.
 +        if (inlineing->inlined_code_obj.count(code))
 +            return false;
-+    }
-+
+     }
+ 
+-    PyGlobalLlvmData *global_data = PyGlobalLlvmData::Get();
+-    global_data->MaybeCollectUnusedGlobals();
 +    PyBytecodeIterator iter(code->co_code);
 +    for (; !iter.Done() && !iter.Error(); iter.Advance()) {
 +        switch (iter.Opcode()) {
-+
+ 
+-    py::LlvmFunctionBuilder fbuilder(global_data, code);
 +        // Opcodes, which make the function un-inlineable
 +#define OPCODE(opname) \
 +        case opname: \
 +    return true;
 +}
 +
++void
++_PyCode_ToInlineLlvmIr(py::LlvmFunctionBuilder &fbuilder, PyCodeObject *code)
++{
+     std::vector<InstrInfo> instr_info(PyString_GET_SIZE(code->co_code));
+     if (-1 == set_line_numbers(code, instr_info)) {
+-        return NULL;
++        return;
+     }
+     if (-1 == find_basic_blocks(code->co_code, fbuilder, instr_info)) {
+-        return NULL;
++        return;
+     }
+ 
+     PyBytecodeIterator iter(code->co_code);
+@@ -376,11 +444,11 @@
+             PyErr_Format(PyExc_SystemError,
+                          "Invalid opcode %d in LLVM IR generation",
+                          iter.Opcode());
+-            return NULL;
++            return;
+         }
+     }
+     if (iter.Error()) {
+-        return NULL;
++        return;
+     }
+     // Make sure the last block has a terminator, even though it should
+     // be unreachable.
+@@ -401,6 +469,28 @@
+         }
+     }
+ 
++}
 +
-diff -r 2817eef6f379 Python/llvm_fbuilder.cc
---- a/Python/llvm_fbuilder.cc	Mon Feb 15 12:17:24 2010 +0100
-+++ b/Python/llvm_fbuilder.cc	Mon Feb 15 12:23:04 2010 +0100
-@@ -11,6 +11,7 @@
++extern "C" _LlvmFunction *
++_PyCode_ToLlvmIr(PyCodeObject *code)
++{
++    if (!PyCode_Check(code)) {
++        PyErr_Format(PyExc_TypeError, "Expected code object, not '%.500s'",
++                     code->ob_type->tp_name);
++        return NULL;
++    }
++    if (!PyString_Check(code->co_code)) {
++        PyErr_SetString(PyExc_SystemError,
++                        "non-string codestring in code object");
++        return NULL;
++    }
++
++    PyGlobalLlvmData *global_data = PyGlobalLlvmData::Get();
++    global_data->MaybeCollectUnusedGlobals();
++
++    py::LlvmFunctionBuilder fbuilder(global_data, code);
++    _PyCode_ToInlineLlvmIr(fbuilder, code);
++
+     if (llvm::verifyFunction(*fbuilder.function(), llvm::PrintMessageAction)) {
+         PyErr_SetString(PyExc_SystemError, "invalid LLVM IR produced");
+         return NULL;
+diff -r 9d3de5a4d8ae Python/llvm_fbuilder.cc
+--- a/Python/llvm_fbuilder.cc	Mon Mar 29 21:07:34 2010 +0200
++++ b/Python/llvm_fbuilder.cc	Sun Jun 13 09:15:53 2010 +0200
+@@ -10,6 +10,7 @@
  #include "Util/ConstantMirror.h"
  #include "Util/EventTimer.h"
  #include "Util/PyTypeBuilder.h"
  
  #include "llvm/ADT/DenseMap.h"
  #include "llvm/ADT/STLExtras.h"
-@@ -38,6 +39,9 @@
+@@ -37,6 +38,9 @@
  
  struct PyExcInfo;
  
  namespace Intrinsic = llvm::Intrinsic;
  using llvm::BasicBlock;
  using llvm::CallInst;
-@@ -63,11 +67,19 @@
+@@ -68,12 +72,21 @@
      ~CallFunctionStats() {
          errs() << "\nCALL_FUNCTION optimization:\n";
          errs() << "Total opcodes: " << this->total << "\n";
 +        errs() << "--------------------\n";
-         errs() << "Optimized opcodes: " << this->optimized << "\n";
+         errs() << "Direct C calls: " << this->direct_calls << "\n";
+         errs() << "Inlined: " << this->inlined << "\n";
          errs() << "No opt: callsite kwargs: " << this->no_opt_kwargs << "\n";
          errs() << "No opt: function params: " << this->no_opt_params << "\n";
          errs() << "No opt: no data: " << this->no_opt_no_data << "\n";
 +        errs() << "Not inlined: heuristics: " << this->inline_heuristic << "\n";
 +        errs() << "Not inlined: no data: " << this->inline_no_data << "\n";
 +        errs() << "Not inlined: polymorphic: " << this->inline_poly << "\n";
-+        errs() << "Inlined calls:" << this->inlined << "\n";
++        errs() << "Inlined calls:" << this->py_inlined << "\n";
++
      }
  
      // How many CALL_FUNCTION opcodes were compiled.
-@@ -84,6 +96,16 @@
+@@ -93,6 +106,16 @@
      unsigned no_opt_no_data;
      // We only optimize monomorphic callsites so far.
      unsigned no_opt_polymorphic;
 +    // Number of functions not inlined because of the heuristic
 +    unsigned inline_heuristic;
 +    // Number of functions inlined
-+    unsigned inlined;
++    unsigned py_inlined;
 +    unsigned inline_no_data;
 +    unsigned inline_poly;
  };
  
  static llvm::ManagedStatic<CallFunctionStats> call_function_stats;
-@@ -340,7 +362,8 @@
+@@ -408,7 +431,8 @@
                              code_object->co_firstlineno,
                              llvm::DIType(),
                              false,  // Not local to unit.
  {
      Function::arg_iterator args = this->function_->arg_begin();
      this->frame_ = args++;
-@@ -442,6 +465,7 @@
+@@ -508,6 +532,7 @@
      Value *frame_code = this->builder_.CreateLoad(
          FrameTy::f_code(this->builder_, this->frame_),
          "frame->f_code");
 +    this->f_code_ = frame_code;
-     this->use_llvm_addr_ = CodeTy::co_use_llvm(this->builder_, frame_code);
+     this->use_jit_addr_ = CodeTy::co_use_jit(this->builder_, frame_code);
  #ifndef NDEBUG
      // Assert that the code object we pull out of the frame is the
-@@ -984,7 +1008,6 @@
+@@ -1047,7 +1072,6 @@
      BasicBlock *pop_loop = this->CreateBasicBlock("pop_loop");
      BasicBlock *pop_block = this->CreateBasicBlock("pop_stack");
      BasicBlock *pop_done = this->CreateBasicBlock("pop_done");
      this->FallThroughTo(pop_loop);
      Value *stack_pointer = this->builder_.CreateLoad(this->stack_pointer_addr_);
      Value *finished_popping = this->builder_.CreateICmpULE(
-@@ -992,6 +1015,7 @@
+@@ -1055,6 +1079,7 @@
      this->builder_.CreateCondBr(finished_popping, pop_done, pop_block);
  
      this->builder_.SetInsertPoint(pop_block);
      this->XDecRef(this->Pop());
      this->builder_.CreateBr(pop_loop);
  
-@@ -1408,7 +1432,9 @@
-     // A code object might not have CO_FDO_GLOBALS set if
+@@ -1463,7 +1488,8 @@
      // a) it was compiled by setting co_optimization, or
      // b) we couldn't watch the globals/builtins dicts.
--    if (this->code_object_->co_flags & CO_FDO_GLOBALS)
-+
-+    // Inlineing does not like this optimization
-+    if (this->code_object_->co_flags & CO_FDO_GLOBALS && this->inlineing_ == NULL)
+     PyObject **watching = this->code_object_->co_watching;
+-    if (watching && watching[WATCHING_GLOBALS] && watching[WATCHING_BUILTINS])
++    if (watching && watching[WATCHING_GLOBALS] && watching[WATCHING_BUILTINS] &&
++        this->inlineing_ == NULL)
          this->LOAD_GLOBAL_fast(name_index);
      else
          this->LOAD_GLOBAL_safe(name_index);
-@@ -1851,9 +1877,9 @@
+@@ -1896,9 +1922,9 @@
  
      this->builder_.SetInsertPoint(unbound_local);
      Function *do_raise =
      this->PropagateException();
  
      this->builder_.SetInsertPoint(success);
-@@ -2147,6 +2173,638 @@
+@@ -2192,6 +2218,611 @@
  }
  
  void
 +        return;
 +    }
 +
-+    CF_INC_STATS(inlined);
++    CF_INC_STATS(py_inlined);
 +
 +    BasicBlock *inline_abort =
 +        this->CreateBasicBlock("CALL_FUNCTION_inline_abort");
 +    this->inlineing_->f_lineno_addr = this->f_lineno_addr_;
 +    this->inlineing_->f_lasti_addr = this->f_lasti_addr_;
 +
-+    this->inlineing_->f_record_lasti_addr = this->f_record_lasti_addr_;
-+    this->inlineing_->f_record_type_addr = this->f_record_type_addr_;
-+
-+    this->f_record_lasti_addr_ = this->CreateAllocaInEntryBlock(
-+        PyTypeBuilder<int>::get(this->context_),
-+        NULL, "inline_record_lasti");
-+    this->builder_.CreateStore(
-+        ConstantInt::get(PyTypeBuilder<int>::get(this->context_), 0),
-+        this->f_record_lasti_addr_);
-+
-+    this->f_record_type_addr_ = this->CreateAllocaInEntryBlock(
-+        PyTypeBuilder<int>::get(this->context_),
-+        NULL, "inline_record_type");
-+    this->builder_.CreateStore(
-+        ConstantInt::get(PyTypeBuilder<int>::get(this->context_), 0),
-+        this->f_record_type_addr_);
-+
-+
 +    this->f_lineno_addr_ = this->CreateAllocaInEntryBlock(
 +        PyTypeBuilder<int>::get(this->context_),
 +        NULL, "inline_lineno");
 +}
 +
 +void
-+LlvmFunctionBuilder::FillInlineBailBlock(llvm::Value *func, PyCodeObject *co, llvm::BasicBlock *end_block)
++LlvmFunctionBuilder::FillInlineBailBlock(llvm::Value *func, PyCodeObject *co,
++                                         llvm::BasicBlock *end_block)
 +{
 +    BasicBlock *bb = this->CreateBasicBlock("InlineBailBlock");
 +    BasicBlock *pass_block = this->CreateBasicBlock("InlineBail_Pass_Block");
 +
 +    llvm::Value *callee_frame = FillInlineFrameBlock(func, co);
 +
-+    llvm::Value *bail_reason = this->builder_.CreateLoad(this->inlineing_->bail_reason);
++    llvm::Value *bail_reason =
++        this->builder_.CreateLoad(this->inlineing_->bail_reason);
 +    this->builder_.CreateStore(
 +            bail_reason,
 +            FrameTy::f_bailed_from_llvm(this->builder_, callee_frame));
 +
-+    llvm::Value *guard_type = this->builder_.CreateLoad(this->inlineing_->guard_type);
++    llvm::Value *guard_type =
++        this->builder_.CreateLoad(this->inlineing_->guard_type);
 +    this->builder_.CreateStore(
 +            guard_type,
 +            FrameTy::f_guard_type(this->builder_, callee_frame));
 +
-+    llvm::Value *record_lasti = this->builder_.CreateLoad(this->f_record_lasti_addr_);
-+    this->builder_.CreateStore(
-+            record_lasti,
-+            FrameTy::f_record_lasti(this->builder_, callee_frame));
-+
-+    llvm::Value *record_type = this->builder_.CreateLoad(this->f_record_type_addr_);
-+    this->builder_.CreateStore(
-+            record_type,
-+            FrameTy::f_record_type(this->builder_, callee_frame));
-+
-+    llvm::Value *native = this->builder_.CreateLoad(
-+        FrameTy::f_native(this->builder_, this->frame_));
-+    this->CreateCall(
-+        this->GetGlobalFunction<void(PyObject *)>("_PyLlvm_WrapXIncref"),
-+        this->builder_.CreateBitCast(native,
-+            PyTypeBuilder<PyObject*>::get(this->context_)));
-+    this->builder_.CreateStore(native,
-+        FrameTy::f_native(this->builder_, callee_frame));
-+
 +    {
 +        BasicBlock *pop_loop = this->CreateBasicBlock("pop_loop");
 +        BasicBlock *pop_block = this->CreateBasicBlock("pop_stack");
 +        BasicBlock *pop_done = this->CreateBasicBlock("pop_done");
 +
-+        Value *target_stack_pointer = this->builder_.CreateLoad(this->stack_pointer_addr_);
-+        Value *counter = this->builder_.CreateAlloca(PyTypeBuilder<PyObject **>::get(this->context_), NULL, "counter");
-+        Value *target_counter = this->builder_.CreateAlloca(PyTypeBuilder<PyObject **>::get(this->context_), NULL, "target_counter");
++        Value *target_stack_pointer =
++            this->builder_.CreateLoad(this->stack_pointer_addr_);
++
++        Value *counter = this->builder_.CreateAlloca(
++            PyTypeBuilder<PyObject **>::get(this->context_), NULL, "counter");
++
++        Value *target_counter = this->builder_.CreateAlloca(
++            PyTypeBuilder<PyObject **>::get(this->context_),
++            NULL, "target_counter");
 +
 +        this->builder_.CreateStore(this->stack_bottom_, counter);
 +        this->builder_.CreateStore(
 +        callee_frame);
 +
 +
-+    this->builder_.CreateStore(this->inlineing_->stack_pointer, this->stack_pointer_addr_);
++    this->builder_.CreateStore(this->inlineing_->stack_pointer,
++                               this->stack_pointer_addr_);
 +
-+    if (!this->inlineing_->method && this->inlineing_->unbound) {
++    if (!this->inlineing_->method || this->inlineing_->unbound) {
 +        this->DecRef(this->Pop());
 +    }
 +
  LlvmFunctionBuilder::CALL_FUNCTION_fast(int oparg,
                                          const PyRuntimeFeedback *feedback)
  {
-@@ -2349,10 +3007,18 @@
+@@ -2461,10 +3092,18 @@
  LlvmFunctionBuilder::CALL_FUNCTION(int oparg)
  {
      const PyRuntimeFeedback *feedback = this->GetFeedback();
  }
  
  
-@@ -2721,9 +3387,16 @@
+@@ -2545,7 +3184,7 @@
+ {
+     IMPORT_NAME_INC_STATS(total);
+ 
+-    if (this->IMPORT_NAME_fast())
++    if (this->inlineing_ == NULL && this->IMPORT_NAME_fast())
+         return;
+ 
+     Value *mod_name = this->Pop();
+@@ -2883,9 +3522,16 @@
          // -1 so that next_instr gets set right in EvalFrame.
          this->GetSigned<int>(bail_idx - 1),
          this->f_lasti_addr_);
      this->builder_.CreateBr(this->GetBailBlock());
  }
  
-@@ -2731,9 +3404,16 @@
+@@ -2893,9 +3539,16 @@
  LlvmFunctionBuilder::CreateGuardBailPoint(unsigned bail_idx, char reason)
  {
  #ifdef Py_WITH_INSTRUMENTATION
  #endif
      this->CreateBailPoint(bail_idx, _PYFRAME_GUARD_FAIL);
  }
-@@ -2758,9 +3438,9 @@
+@@ -2920,9 +3573,9 @@
  
      this->builder_.SetInsertPoint(failure);
      Function *do_raise = this->GetGlobalFunction<
          ConstantInt::getSigned(PyTypeBuilder<int>::get(this->context_), index));
      this->PropagateException();
  
-@@ -2816,6 +3496,7 @@
+@@ -2978,6 +3631,7 @@
      this->builder_.SetInsertPoint(next_null);
      Value *err_occurred = this->CreateCall(
          this->GetGlobalFunction<PyObject*()>("PyErr_Occurred"));
      BasicBlock *iter_ended = this->CreateBasicBlock("iter_ended");
      BasicBlock *exception = this->CreateBasicBlock("exception");
      this->builder_.CreateCondBr(this->IsNull(err_occurred),
-@@ -3046,8 +3727,13 @@
+@@ -3208,8 +3862,13 @@
      // terminators causes problems.
      BasicBlock *dead_code = this->CreateBasicBlock("dead_code");
  
  
      this->builder_.SetInsertPoint(dead_code);
  }
-diff -r 2817eef6f379 Python/llvm_fbuilder.h
---- a/Python/llvm_fbuilder.h	Mon Feb 15 12:17:24 2010 +0100
-+++ b/Python/llvm_fbuilder.h	Mon Feb 15 12:23:04 2010 +0100
-@@ -24,6 +24,34 @@
+diff -r 9d3de5a4d8ae Python/llvm_fbuilder.h
+--- a/Python/llvm_fbuilder.h	Mon Mar 29 21:07:34 2010 +0200
++++ b/Python/llvm_fbuilder.h	Sun Jun 13 09:15:53 2010 +0200
+@@ -25,6 +25,32 @@
  
  namespace py {
  
 +    llvm::BasicBlock *propagate_exception_block;
 +    llvm::Value *f_lineno_addr;
 +    llvm::Value *f_lasti_addr;
-+    llvm::Value *f_record_lasti_addr;
-+    llvm::Value *f_record_type_addr;
 +    InlineingInformation *inlineing;
 +
 +    llvm::SmallPtrSet<PyCodeObject*, 5> inlined_code_obj;
  /// Helps the compiler build LLVM functions corresponding to Python
  /// functions.  This class maintains the IRBuilder and several Value*s
  /// set up in the entry block.
-@@ -548,6 +576,7 @@
+@@ -554,6 +580,7 @@
      // running through the eval loop to omit as much flexibility as possible.
      void CALL_FUNCTION_safe(int num_args);
      void CALL_FUNCTION_fast(int num_args, const PyRuntimeFeedback *);
 +    void CALL_FUNCTION_inline(int num_args, const PyRuntimeFeedback *);
  
-     // LOAD/STORE_ATTR_safe always works, while LOAD/STORE_ATTR_fast is
-     // optimized to skip the descriptor/method lookup on the type if the object
-@@ -686,11 +715,22 @@
+     // Specialized version of CALL_FUNCTION for len() on certain types.
+     void CALL_FUNCTION_fast_len(llvm::Value *actual_func,
+@@ -703,11 +730,23 @@
      /// exception.
      llvm::BasicBlock *GetExceptionBlock() const;
  
-+
 +    /// Inlineing related helper functions
-+    void EnterInlineingMode(llvm::Value *func, PyCodeObject *code, int num_args, int defaults);
++    void EnterInlineingMode(llvm::Value *func, PyCodeObject *code,
++                            int num_args, int defaults);
 +    void LeaveInlineingMode(PyCodeObject *code);
 +    llvm::Value *InlineGetCodeObject(llvm::Value *func);
 +    llvm::Value *InlineGetGlobalsObject(llvm::Value *func);
 +
 +    void FillInlineExceptionBlock(llvm::Value *func, PyCodeObject *co);
-+    void FillInlineBailBlock(llvm::Value *func, PyCodeObject *co, llvm::BasicBlock *end_block);
++    void FillInlineBailBlock(llvm::Value *func, PyCodeObject *co,
++                             llvm::BasicBlock *end_block);
 +    llvm::Value *FillInlineFrameBlock(llvm::Value *func, PyCodeObject *co);
 +
      PyGlobalLlvmData *const llvm_data_;
      // string.
 -    PyCodeObject *const code_object_;
 +    PyCodeObject *code_object_;
-     PyNativeFunctionObject *native_object_;
      llvm::LLVMContext &context_;
      llvm::Module *const module_;
-@@ -712,6 +752,9 @@
+     llvm::Function *const function_;
+@@ -728,6 +767,9 @@
      // entry block. They're constant after construction.
      llvm::Value *frame_;
  
 +    // Pointer to the passed in code_object
 +    llvm::Value *f_code_;
 +
-     // Address of code_object_->co_use_llvm, used for guards.
-     llvm::Value *use_llvm_addr_;
+     // Address of code_object_->co_use_jit, used for guards.
+     llvm::Value *use_jit_addr_;
  
-@@ -767,6 +810,8 @@
+@@ -779,6 +821,8 @@
      llvm::Value *retval_addr_;
  
      llvm::SmallPtrSet<PyTypeObject*, 5> types_used_;
  };
  
  }  // namespace py
-diff -r 2817eef6f379 Python/llvm_inline_functions.c
---- a/Python/llvm_inline_functions.c	Mon Feb 15 12:17:24 2010 +0100
-+++ b/Python/llvm_inline_functions.c	Mon Feb 15 12:23:04 2010 +0100
-@@ -29,6 +29,12 @@
+diff -r 9d3de5a4d8ae Python/llvm_inline_functions.c
+--- a/Python/llvm_inline_functions.c	Mon Mar 29 21:07:34 2010 +0200
++++ b/Python/llvm_inline_functions.c	Sun Jun 13 09:15:53 2010 +0200
+@@ -28,6 +28,12 @@
  }
  
  void __attribute__((always_inline))
  _PyLlvm_WrapDecref(PyObject *obj)
  {
      Py_DECREF(obj);
-@@ -52,6 +58,75 @@
+@@ -51,6 +57,75 @@
      return PyCFunction_Check(obj);
  }
  

recompilation_feedback.patch

-diff -r a9de87c4a9bc Include/code.h
---- a/Include/code.h	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Include/code.h	Thu Mar 11 15:40:51 2010 +0100
-@@ -10,18 +10,6 @@
- #endif
- 
- 
--#ifdef WITH_LLVM
--/* Why we're watching a given dict. We malloc a list of PyObject*s to hold the
--   dicts being watched; these serve as indicies into that list. */
--typedef enum {
--    WATCHING_BUILTINS = 0,
--    WATCHING_GLOBALS,
--    WATCHING_SYS_MODULES,
--    NUM_WATCHING_REASONS,
--} ReasonWatched;
--#endif
--
--
- /* Bytecode object.  Keep this in sync with Util/PyTypeBuilder.h. */
- typedef struct PyCodeObject {
-     PyObject_HEAD
-@@ -52,6 +40,7 @@
-        only for ease of testing; the flag that matters is f_use_llvm on the
-        frame object, which is influenced by co_use_llvm. */
-     char co_use_llvm;
-+    char co_globals_changes;
-     /* Stores which optimizations have been applied to this code
-        object.  Each level corresponds to an argument to
-        PyGlobalLlvmData::Optimize().  Starts at -1 for unoptimized
-@@ -66,8 +55,7 @@
-     /* Measure of how hot this code object is. This is used to decide
-        which code objects are worth sending through LLVM. */
-     long co_hotness;
--    /* Keep track of which dicts this code object is watching. */
--    //~XXX PyObject **co_watching;
-+    long co_hotness_threshold;
- 
-     PyNativeFunctionObject *co_native;
- #endif
-@@ -76,13 +64,23 @@
- /* If co_fatalbailcount >= PY_MAX_FATAL_BAIL_COUNT, force this code to use the
-    eval loop forever after. See the comment on the co_fatalbailcount field
-    for more details. */
--#define PY_MAX_FATALBAILCOUNT 1
-+#define PY_MAX_FATALBAILCOUNT 10
-+
-+/* How often we want to recompile, before disabling FDO_GLOBAL */
-+#define PY_MAX_GLOBALS_CHANGES 2
-+
-+/* After a native function fails PY_MAX_GUARD_FAILS times, recompile it. */
-+#define PY_MAX_GUARD_FAILS 100
- 
- /* The threshold for co_hotness before the code object is considered "hot". */
- #define PY_HOTNESS_THRESHOLD 100000
- 
-+/* After a native code object becomes invalid, we want to use the eval loop
-+   to gather additional feedback. */
-+#define PY_HOTNESS_ADDITIONAL 10000
-+
- /* Masks for co_flags above.  If you update these, consider updating the
-- * fast_function fast path in eval.cc.  */
-+   fast_function fast path in eval.cc.  */
- #define CO_OPTIMIZED    (1 << 0)
- #define CO_NEWLOCALS    (1 << 1)
- #define CO_VARARGS      (1 << 2)
-@@ -158,8 +156,8 @@
- PyAPI_FUNC(int) _PyCode_CheckLineNumber(PyCodeObject* co,
-                                         int lasti, PyAddrPair *bounds);
- 
--PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts,
--                                      PyObject *names, PyObject *lineno_obj);
-+//~ PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts,
-+                                      //~ PyObject *names, PyObject *lineno_obj);
- 
- #ifdef WITH_LLVM
- /* Compile a given function to LLVM IR, and apply a set of optimization passes.
-@@ -172,7 +170,7 @@
- 
-    This should eventually be able to *re*compile bytecode to LLVM IR. See
-    http://code.google.com/p/unladen-swallow/issues/detail?id=41. */
--PyAPI_FUNC(int) _PyCode_ToOptimizedLlvmIr(PyCodeObject *code, int opt_level);
-+//~ PyAPI_FUNC(int) _PyCode_ToOptimizedLlvmIr(PyCodeObject *code, int opt_level);
- 
- /* Register a code object to receive updates if its globals or builtins change.
-    If the globals or builtins change, co_use_llvm will be set to 0; this causes
-@@ -183,24 +181,32 @@
-    one) and needs to be conveyed to the user. There are recoverable failure
-    modes (globals == NULL, builtins == NULL, etc) that merely disable the
-    optimization and return 0. */
--PyAPI_FUNC(int) _PyCode_WatchDict(PyCodeObject *code, ReasonWatched reason,
--                                  PyObject *dict);
-+//~ PyAPI_FUNC(int) _PyCode_WatchDict(PyCodeObject *code, ReasonWatched reason,
-+                                  //~ PyObject *dict);
- 
- /* Stop watching a dict for changes. Returns 0 on success, -1 on failure. */
--PyAPI_FUNC(int) _PyCode_IgnoreDict(PyCodeObject *code, ReasonWatched reason);
-+//~ PyAPI_FUNC(int) _PyCode_IgnoreDict(PyCodeObject *code, ReasonWatched reason);
- 
- /* Internal helper function to get the number of dicts being watched. */
--PyAPI_FUNC(Py_ssize_t) _PyCode_WatchingSize(PyCodeObject *code);
-+//~ PyAPI_FUNC(Py_ssize_t) _PyCode_WatchingSize(PyCodeObject *code);
- 
- /* Ignore all dicts this code object is currently watching. */
--PyAPI_FUNC(void) _PyCode_IgnoreWatchedDicts(PyCodeObject *code);
-+//~ PyAPI_FUNC(void) _PyCode_IgnoreWatchedDicts(PyCodeObject *code);
-+
-+typedef enum co_invalid_reason {
-+    CO_INVALID_GUARDS,
-+    CO_INVALID_GLOBALS,
-+    CO_INVALID_TYPES,
-+} co_invalid_reason;
- 
- /* Perform any steps needed to mark a function's machine code as invalid.
-    Individual fatal guard failures may need to do extra work on their own to
-    clean up any special references/data they may have created, but calling this
-    function will ensure that `code`'s machine code equivalent will not be
-    called again. */
--PyAPI_FUNC(void) _PyCode_InvalidateMachineCode(PyCodeObject *code);
-+PyAPI_FUNC(void) _PyCode_InvalidateMachineCode(
-+                                PyCodeObject *code,
-+                                co_invalid_reason reason);
- #endif
- 
- #ifdef __cplusplus
-diff -r a9de87c4a9bc Include/nativefunctionobject.h
---- a/Include/nativefunctionobject.h	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Include/nativefunctionobject.h	Thu Mar 11 15:40:51 2010 +0100
-@@ -16,6 +16,9 @@
- 
- struct PyNativeFunctionObject {
-     PyObject_HEAD
-+    /* This does not hold a reference to a code object. It's only valid 
-+       (not null) when the nativefuncobj is held by a code object.
-+       When a nativefuncobj is invalidated the field gets cleared */
-     PyCodeObject *code;
-     llvm::Function *function;
-     PyEvalFrameFunction native;
-@@ -24,16 +27,10 @@
- 
-     int bailcount;
- 
--    /* Because the globals dict is set on the frame, we record *which* globals
--       dict we're assuming. */
--    PyObject *assumed_globals;
--    /* Because the builtins dict is set on the frame, we record *which* builtins
--       dict we're assuming. */
--    PyObject *assumed_builtins;
-+    /* Keep track of which dicts this code object is watching. */
-+    PyObject **watching;
- };
- 
--#define PyNativeFunction_Check(op) (Py_TYPE(op) == &PyNativeFunction_Type)
--
- #endif  /* WITH_LLVM */
- 
- #ifdef __cplusplus
-diff -r a9de87c4a9bc Include/nativefunctionobject_fwd.h
---- /dev/null	Thu Jan 01 00:00:00 1970 +0000
-+++ b/Include/nativefunctionobject_fwd.h	Thu Mar 11 15:40:51 2010 +0100
-@@ -0,0 +1,68 @@
-+
-+#ifndef Py_NATIVEFUNCTIONOBJECT_FWD_H
-+#define Py_NATIVEFUNCTIONOBJECT_FWD_H
-+
-+#ifdef __cplusplus
-+extern "C" {
-+#endif
-+
-+
-+#ifdef WITH_LLVM
-+/* Why we're watching a given dict. We malloc a list of PyObject*s to hold the
-+   dicts being watched; these serve as indicies into that list. */
-+typedef enum {
-+    WATCHING_BUILTINS = 0,
-+    WATCHING_GLOBALS,
-+    WATCHING_SYS_MODULES,
-+    NUM_WATCHING_REASONS,
-+} ReasonWatched;
-+
-+struct PyCodeObject;
-+struct _dictobject;
-+
-+typedef struct PyNativeFunctionObject PyNativeFunctionObject;
-+
-+PyAPI_DATA(void) _PyNativeFunction_ClearCode(PyNativeFunctionObject *func);
-+
-+PyAPI_DATA(PyObject *) _PyNativeFunction_Compile(
-+                            struct PyCodeObject *co,
-+                            int target_optimization,
-+                            PyObject *globals,
-+                            PyObject *builtins);
-+
-+PyAPI_DATA(PyObject *) _PyNativeFunction_GetOptimization(
-+                            PyNativeFunctionObject *obj);
-+
-+PyAPI_DATA(PyObject *) _PyNativeFunction_GetIR(
-+                            struct PyCodeObject *co,
-+                            int target_optimization);
-+
-+PyAPI_DATA(void) _PyNativeFunction_Invalidate(
-+                            PyNativeFunctionObject *func);
-+
-+PyAPI_DATA(void) _PyNativeFunction_IgnoreWatchedDicts(
-+                            PyNativeFunctionObject *func);
-+
-+PyAPI_DATA(int) _PyNativeFunction_IgnoreDict(
-+                            PyNativeFunctionObject *func, 
-+                            ReasonWatched reason);
-+
-+PyAPI_DATA(int) _PyNativeFunction_CheckGlobals(
-+                            PyNativeFunctionObject *func,
-+                            PyObject *globals,
-+                            PyObject *builtins);
-+
-+PyAPI_DATA(PyTypeObject) PyNativeFunction_Type;
-+
-+typedef PyObject *(*PyEvalFrameFunction)(struct _frame *);
-+
-+#define PyNativeFunction_Check(op) (Py_TYPE(op) == &PyNativeFunction_Type)
-+
-+#endif  /* WITH_LLVM */
-+
-+#ifdef __cplusplus
-+}
-+#endif
-+
-+#endif
-+
-diff -r a9de87c4a9bc Lib/test/test_llvm.py
---- a/Lib/test/test_llvm.py	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Lib/test/test_llvm.py	Thu Mar 11 15:40:51 2010 +0100
-@@ -2753,11 +2753,7 @@
-         sys.setbailerror(False)
-         self.assertEqual(foo(lambda x: 7), 7)
- 
--    def test_guard_failure_blocks_native_code(self):
--        # Until we can recompile things, failing a guard should force use of the
--        # eval loop forever after. Even once we can recompile things, we should
--        # limit how often we're willing to recompile highly-dynamic functions.
--        # test_mutants has a good example of this.
-+    def test_guard_failure_triggers_recompilation(self):
- 
-         # Compile like this so we get a new code object every time.
-         foo = compile_for_llvm("foo", "def foo(): return len([])",
-@@ -2768,13 +2764,20 @@
-         self.assertEqual(foo(), 0)
- 
-         with test_support.swap_attr(__builtin__, "len", lambda x: 7):
-+            # Changing globals should invalidate the machine code
-             self.assertEqual(foo.__code__.__use_llvm__, False)
-             self.assertEqual(foo.__code__.co_fatalbailcount, 1)
--            # Since we can't recompile things yet, __use_llvm__ should be left
--            # at False and execution should use the eval loop.
-+
-+            sys.setbailerror(False)
-+
-+            # Do not recompile immediately after invalidation
-+            for i in xrange(100):
-+                foo()
-+            self.assertEqual(foo.__code__.__use_llvm__, False)
-+
-             spin_until_hot(foo, [])
--            self.assertEqual(foo.__code__.__use_llvm__, False)
--            self.assertEqual(foo(), 7)
-+            self.assertEqual(foo.__code__.__use_llvm__, True)
-+            self.assertEqual(foo.__code__.co_fatalbailcount, 1)
- 
-     def test_fast_calls_method(self):
-         # This used to crash at one point while developing CALL_FUNCTION's
-@@ -3496,6 +3499,35 @@
-         # Make sure we compiled both branches to LLVM IR.
-         self.assertContains("@hex", str(foo.__code__.co_llvm))
- 
-+    def test_cold_branch_recompilation(self):
-+        # If we have runtime feedback, we'd like to be able to omit untaken
-+        # branches to a) reduce code size, b) give the optimizers less input.
-+        # We only do this if the branch is consistent (always 100%
-+        # taken/not-taken).
-+        foo = compile_for_llvm("foo", """
-+def foo(x):
-+    return x or len([1, 2])
-+""", optimization_level=None)
-+
-+        spin_until_hot(foo, [True])
-+
-+        self.assertTrue(foo.__code__.__use_llvm__)
-+        self.assertEqual(foo(True), True)
-+        self.assertRaises(RuntimeError, foo, False)
-+        self.assertEqual(foo.__code__.co_fatalbailcount, 0)
-+
-+        # Spin with the cold path to trigger recompilation
-+        sys.setbailerror(False)
-+        self.assertTrue(foo.__code__.__use_llvm__)
-+        spin_until_hot(foo, [False])
-+
-+        # Make sure both paths are now compiled
-+        sys.setbailerror(True)
-+        foo(False) # Should not bail
-+
-+        self.assertTrue(foo.__code__.__use_llvm__)
-+        self.assertTrue("@len" in str(foo.__code__.co_llvm))
-+
-     def test_import_does_not_bail(self):
-         # Regression test: this simple import (which hits sys.modules!) used
-         # to cause invalidation due to no-op assignments to the globals dict
-diff -r a9de87c4a9bc Makefile.pre.in
---- a/Makefile.pre.in	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Makefile.pre.in	Thu Mar 11 15:40:51 2010 +0100
-@@ -387,7 +387,7 @@
- 
- ifneq (@BUILD_LLVM@, disabled)
- 	OBJECT_OBJS +=	\
--        Objects/nativefunctionobject.o
-+		Objects/nativefunctionobject.o
- endif
- 
- 
-@@ -748,6 +748,8 @@
- 		Include/unicodeobject.h \
- 		Include/warnings.h \
- 		Include/weakrefobject.h \
-+		Include/nativefunctionobject.h \
-+		Include/nativefunctionobject_fwd.h \
- 		Python/global_llvm_data.h \
- 		Python/global_llvm_data_fwd.h \
- 		Python/llvm_fbuilder.h \
-diff -r a9de87c4a9bc Objects/codeobject.c
---- a/Objects/codeobject.c	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Objects/codeobject.c	Thu Mar 11 15:40:51 2010 +0100
-@@ -111,8 +111,10 @@
- 		// save memory for code that is never executed.
- 		co->co_runtime_feedback = NULL;
- 		co->co_use_llvm = 0;
-+		co->co_globals_changes = 0;
- 		co->co_optimization = -1;
- 		co->co_hotness = 0;
-+		co->co_hotness_threshold = PY_HOTNESS_THRESHOLD;
- 		co->co_fatalbailcount = 0;
- 		//~ XXX co->co_watching = NULL;
- 		co->co_native = NULL;
-@@ -160,7 +162,6 @@
- static int
- code_set_optimization(PyCodeObject *code, PyObject *new_opt_level_obj)
- {
--	PyNativeFunctionObject *old;
- 	long new_opt_level = PyInt_AsLong(new_opt_level_obj);
- 	if (new_opt_level == -1 && PyErr_Occurred())
- 		return -1;
-@@ -169,8 +170,8 @@
- 		return 0;
- 
- 	code->co_optimization = new_opt_level;
--	_PyNativeFunction_ClearCode(co->native);
--	Py_CLEAR(co->native);
-+	_PyNativeFunction_ClearCode(code->co_native);
-+	Py_CLEAR(code->co_native);
- 	code->co_native = (PyNativeFunctionObject *)
- 		_PyNativeFunction_Compile(code, new_opt_level, NULL, NULL);
- 
-@@ -190,7 +191,7 @@
- 		Py_RETURN_NONE;
- 
- 	ir = _PyNativeFunction_GetIR(code, code->co_optimization);
--	if (ir) {
-+	if (ir != NULL) {
- 		return ir;
- 	}
- 	else {
-@@ -205,92 +206,20 @@
- 	{NULL} /* Sentinel */
- };
- 
--
--static PyObject **
--new_watch_list(void)
--{
--	PyObject **list = PyMem_New(PyObject *, NUM_WATCHING_REASONS);
--	if (list == NULL) {
--		PyErr_NoMemory();
--		return NULL;
--	}
--	memset(list, 0, sizeof(PyObject *) * NUM_WATCHING_REASONS);
--	return list;
--}
--
--
--//~ int
--//~ _PyCode_WatchDict(PyCodeObject *code, ReasonWatched reason, PyObject *dict)
--//~ {
--	//~ if (code->co_watching == NULL) {
--		//~ code->co_watching = new_watch_list();
--		//~ if (code->co_watching == NULL)
--			//~ return -1;
--	//~ }
--//~ 
--	//~ if (code->co_watching[reason] != NULL) {
--		//~ _PyDict_DropWatcher(code->co_watching[reason], code);
--	//~ }
--	//~ /* Note that we do not hold a reference to these dicts. If one of these
--	   //~ dicts is deleted, it will notify all dependent code objects.
--	   //~ Likewise, if this code object is deleted, it will remove itself from
--	   //~ the dictionaries' watcher arrays. */
--	//~ code->co_watching[reason] = dict;
--	//~ _PyDict_AddWatcher(dict, code);
--//~ 
--	//~ return 0;
--//~ }
--//~ 
--//~ int
--//~ _PyCode_IgnoreDict(PyCodeObject *code, ReasonWatched reason)
--//~ {
--	//~ if (code->co_watching == NULL || code->co_watching[reason] == NULL)
--		//~ return 0;
--	//~ _PyDict_DropWatcher(code->co_watching[reason], code);
--	//~ code->co_watching[reason] = NULL;
--	//~ return 0;
--//~ }
--//~ 
--//~ Py_ssize_t
--//~ _PyCode_WatchingSize(PyCodeObject *code)
--//~ {
--	//~ Py_ssize_t i, n = 0;
--	//~ if (code->co_watching == NULL)
--		//~ return 0;
--//~ 
--	//~ for (i = 0; i < NUM_WATCHING_REASONS; ++i) {
--		//~ n += (code->co_watching[i] != NULL);
--	//~ }
--	//~ return n;
--//~ }
--//~ 
--//~ void
--//~ _PyCode_IgnoreWatchedDicts(PyCodeObject *code)
--//~ {
--	//~ Py_ssize_t i;
--	//~ if (code->co_watching == NULL)
--		//~ return;
--//~ 
--	//~ for (i = 0; i < NUM_WATCHING_REASONS; ++i) {
--		//~ if (code->co_watching[i] != NULL) {
--			//~ _PyDict_DropWatcher(code->co_watching[i], code);
--		//~ }
--		//~ code->co_watching[i] = NULL;
--	//~ }
--//~ }
--
- void
--_PyCode_InvalidateMachineCode(PyCodeObject *code)
-+_PyCode_InvalidateMachineCode(PyCodeObject *code, co_invalid_reason reason)
- {
- 	/* This will cause the LLVM-generated code to bail back to the
- 	   interpreter. The LLVM code won't be re-entered until it is
- 	   recompiled. */
- 	code->co_use_llvm = 0;
- 	code->co_fatalbailcount++;
-+	if (reason == CO_INVALID_GLOBALS) {
-+		code->co_globals_changes += 1;
-+	}
-+	code->co_hotness_threshold = code->co_hotness + PY_HOTNESS_ADDITIONAL;
- 	/* This is a no-op if not configured with --with-instrumentation. */
- 	_PyEval_RecordFatalBail(code);
--	/* The machine code is invalid, no need to keep watching these dicts. */
--	_PyCode_IgnoreWatchedDicts(code);
- }
- 
- //~ int
-@@ -487,11 +416,11 @@
- 	_PyNativeFunction_ClearCode(co->co_native);
- 	Py_XDECREF(co->co_native);
- 
--	if (co->co_watching) {
-+	//~ if (co->co_watching) {
- 		//~ XXX _PyCode_IgnoreWatchedDicts(co);
--		PyMem_Free(co->co_watching);
--		co->co_watching = NULL;
--	}
-+		//~ PyMem_Free(co->co_watching);
-+		//~ co->co_watching = NULL;
-+	//~ }
- 	PyFeedbackMap_Del(co->co_runtime_feedback);
- #endif
- 	PyObject_DEL(co);
-diff -r a9de87c4a9bc Objects/dictobject.c
---- a/Objects/dictobject.c	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Objects/dictobject.c	Thu Mar 11 15:40:51 2010 +0100
-@@ -10,6 +10,7 @@
- #include "Python.h"
- 
- #include "Util/PySmallPtrSet.h"
-+#include "nativefunctionobject_fwd.h"
- 
- 
- /* Set a key error with the specified argument, wrapping it in a
-@@ -2077,8 +2078,8 @@
- static void
- notify_watcher_callback(PyObject *obj, void *unused)
- {
--	assert(PyCode_Check(obj));
--	//~ XXX _PyCode_InvalidateMachineCode((PyCodeObject *)obj);
-+	assert(PyNativeFunction_Check(obj));
-+	_PyNativeFunction_Invalidate((PyNativeFunctionObject *)obj);
- }
- 
- // We split the real work of notify_watchers() out into a separate function so
-diff -r a9de87c4a9bc Objects/nativefunctionobject.cc
---- a/Objects/nativefunctionobject.cc	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Objects/nativefunctionobject.cc	Thu Mar 11 15:40:51 2010 +0100
-@@ -1,5 +1,6 @@
- 
- #include "Python.h"
-+#include "nativefunctionobject_fwd.h"
- #include "nativefunctionobject.h"
- 
- #include "Python/global_llvm_data.h"
-@@ -28,7 +29,8 @@
- using llvm::Type;
- using llvm::Value;
- 
--llvm::Function *_PyCode_ToLlvmIrFunction(PyCodeObject *code, PyNativeFunctionObject *native = NULL);
-+llvm::Function *_PyCode_ToLlvmIrFunction(PyCodeObject *code,
-+                        PyNativeFunctionObject *native = NULL);
- 
- 
- #ifdef Py_WITH_INSTRUMENTATION
-@@ -113,40 +115,81 @@
-         globals_vect.begin(), globals_vect.end(), "", entry);
- }
- 
-+static PyObject **
-+new_watch_list(void)
-+{
-+    PyObject **list = PyMem_New(PyObject *, NUM_WATCHING_REASONS);
-+    if (list == NULL) {
-+        PyErr_NoMemory();
-+        return NULL;
-+    }
-+    memset(list, 0, sizeof(PyObject *) * NUM_WATCHING_REASONS);
-+    return list;
-+}
-+
-+
- int
--_PyNativeFunction_WatchGlobals(PyNativeFunctionObject *func,
--        PyObject *globals, PyObject *builtins)
-+_PyNativeFunction_WatchDict(PyNativeFunctionObject *func,
-+                            ReasonWatched reason,
-+                            PyObject *dict)
- {
--    if (func->assumed_globals) {
--        _PyDict_DropWatcher(func->assumed_globals, func);
--        _PyDict_DropWatcher(func->assumed_builtins, func);
--        func->assumed_globals = NULL;
--        func->assumed_builtins = NULL;
-+    if (func->watching == NULL) {
-+        func->watching = new_watch_list();
-+        if (func->watching == NULL)
-+            return -1;
-     }
--
--    /* If any of these checks fail, we need this optimization off. */
--    if (globals == NULL || builtins == NULL ||
--        !PyDict_CheckExact(globals) || !PyDict_CheckExact(builtins)) {
--        return 0;
-+    if (func->watching[reason] != NULL) {
-+       _PyDict_DropWatcher(func->watching[reason], func);
-     }
--    if (_PyDict_AddWatcher(globals, func)) {
--        return 0;
--    }
--    if (_PyDict_AddWatcher(builtins, func)) {
--        _PyDict_DropWatcher(globals, func);
--        return 0;
--    }
--
-     /* Note that we do not hold a reference to these dicts. If one of these
-        dicts is deleted, it will notify all dependent code objects.
-        Likewise, if this code object is deleted, it will remove itself from
-        the dictionaries' watcher arrays. */
--    func->assumed_globals = globals;
--    func->assumed_builtins = builtins;
--    return 1;
-+    func->watching[reason] = dict;
-+    _PyDict_AddWatcher(dict, func);
-+
-+    return 0;
- }
- 
--PyObject *
-+void
-+_PyNativeFunction_IgnoreWatchedDicts(PyNativeFunctionObject *func)
-+{
-+    Py_ssize_t i;
-+    if (func->watching == NULL)
-+        return;
-+
-+    for (i = 0; i < NUM_WATCHING_REASONS; ++i) {
-+        if (func->watching[i] != NULL) {
-+            _PyDict_DropWatcher(func->watching[i], func);
-+        }
-+        func->watching[i] = NULL;
-+    }
-+}
-+
-+int
-+_PyNativeFunction_IgnoreDict(PyNativeFunctionObject *func, ReasonWatched reason)
-+{
-+    if (func->watching == NULL || func->watching[reason] == NULL)
-+        return 0;
-+    _PyDict_DropWatcher(func->watching[reason], func);
-+    func->watching[reason] = NULL;
-+    return 0;
-+}
-+
-+Py_ssize_t
-+_PyNative_WatchingSize(PyNativeFunctionObject *func)
-+{
-+    Py_ssize_t i, n = 0;
-+    if (func->watching == NULL)
-+        return 0;
-+
-+    for (i = 0; i < NUM_WATCHING_REASONS; ++i) {
-+        n += (func->watching[i] != NULL);
-+    }
-+    return n;
-+}
-+
-+PyNativeFunctionObject *
- _PyNativeFunction_New()
- {
-     PyNativeFunctionObject *result =
-@@ -156,18 +199,17 @@
-     result->native = NULL;
-     result->optimization = 0;
-     result->bailcount = 0;
--    result->assumed_globals = NULL;
--    result->assumed_builtins = NULL;
-+    result->watching = NULL;
- 
--    return (PyObject *)result;
-+    return result;
- }
- 
- PyObject *
--_PyNativeFunction_Compile(PyCodeObject *co, int target_optimization, PyObject *globals, PyObject *builtins)
-+_PyNativeFunction_Compile(PyCodeObject *co, int target_optimization,
-+                          PyObject *globals, PyObject *builtins)
- {
- 
--    PyNativeFunctionObject *result =
--        (PyNativeFunctionObject *)_PyNativeFunction_New();
-+    PyNativeFunctionObject *result = _PyNativeFunction_New();
- 
-     if (result == NULL) {
-         return NULL;
-@@ -177,12 +219,9 @@
-     // haven't already done that.
-     PY_LOG_TSC_EVENT(EVAL_COMPILE_START);
- 
--    if (_PyNativeFunction_WatchGlobals(result, globals,
--                             builtins)) {
--        co->co_flags |= CO_FDO_GLOBALS;
--    }
--    else {
--        co->co_flags &= ~CO_FDO_GLOBALS;
-+    if (co->co_globals_changes < PY_MAX_GLOBALS_CHANGES) {
-+        _PyNativeFunction_WatchDict(result, WATCHING_GLOBALS, globals);
-+        _PyNativeFunction_WatchDict(result, WATCHING_BUILTINS, builtins);
-     }
- 
-     /* Large functions take a very long time to translate to LLVM
-@@ -202,11 +241,11 @@
-         return NULL;
-     }
- 
--    struct PyGlobalLlvmData *global_llvm_data = PyThreadState_GET()->interp->global_llvm_data;
-+    struct PyGlobalLlvmData *global_llvm_data = PyGlobalLlvmData_GET();
-     
-     PY_LOG_TSC_EVENT(LLVM_COMPILE_START);
-     llvm::Function *llvm_function = _PyCode_ToLlvmIrFunction(co, result);
--    // Forwards to global_data->Optimize(llvm_function->lf_function, level);
-+
-     if (global_llvm_data->Optimize(*llvm_function,
-                         target_optimization) < 0) {
-         PyErr_Format(PyExc_SystemError,
-@@ -274,16 +313,10 @@
-     if (co->co_flags & CO_USES_EXEC)
-         return NULL;
- 
--    bool has_global_opt = co->co_flags & CO_FDO_GLOBALS;
--    co->co_flags &= ~CO_FDO_GLOBALS;
--    
--    struct PyGlobalLlvmData *global_llvm_data = PyThreadState_GET()->interp->global_llvm_data;
-+    struct PyGlobalLlvmData *global_llvm_data = PyGlobalLlvmData_GET();
- 
-     llvm::Function *llvm_function = _PyCode_ToLlvmIrFunction(co);
- 
--    if (has_global_opt)
--        co->co_flags |= CO_FDO_GLOBALS;
--    // Forwards to global_data->Optimize(llvm_function->lf_function, level);
-     if (global_llvm_data->Optimize(*llvm_function,
-                         target_optimization) < 0) {
-         PyErr_Format(PyExc_SystemError,
-@@ -311,29 +344,13 @@
- }
- 
- void
--_PyNativeFunction_Invalidate(PyDictObject *dict, PyNativeFunctionObject *func)
-+_PyNativeFunction_Invalidate(PyNativeFunctionObject *func)
- {
--    PyObject *pyself = (PyObject *)dict;
--    /* Clean up the watcher list for the other dict
--       this code object is watching. This prevents the
--       other dict from potentially corrupting memory during
--       notify_watchers(). */
--    if (func->assumed_globals == pyself) {
--        _PyDict_DropWatcher(func->assumed_builtins,
--                            func);
-+    _PyNativeFunction_IgnoreWatchedDicts(func);
-+
-+    if (func->code) {
-+        _PyCode_InvalidateMachineCode(func->code, CO_INVALID_GUARDS);
-     }
--    else if (func->assumed_builtins == pyself) {
--        _PyDict_DropWatcher(func->assumed_globals,
--                            func);
--    }
--    else {
--        assert(0 && "NativeFunction isn't watching this dict!");
--    }
--    func->assumed_globals = NULL;
--    func->assumed_builtins = NULL;
--
--    if (func->code)
--        _PyCode_InvalidateMachineCode(func->code);
- }
- 
- void
-@@ -346,14 +363,16 @@
- int
- _PyNativeFunction_CheckGlobals(PyNativeFunctionObject *func, PyObject *globals, PyObject *builtins)
- {
--    if (func && 
--        func->assumed_globals && 
--        func->assumed_builtins) {
-+    if (func == NULL) {
-+        return 0;
-+    }
- 
--        return (func->assumed_globals == globals &&
--                func->assumed_builtins == builtins);
-+    if (func->watching == NULL ||
-+        (func->watching[WATCHING_GLOBALS] == globals &&
-+         func->watching[WATCHING_BUILTINS] == builtins)) {
-+        return 1;
-     }
--    return 1;
-+    return 0;
- }
- 
- PyDoc_STRVAR(nativefunction_doc,
-@@ -367,10 +386,8 @@
-     if (func->function) {
-         func->function->setLinkage(llvm::GlobalValue::InternalLinkage);
-     }
--    if (func->assumed_globals)
--        _PyDict_DropWatcher(func->assumed_globals, func);
--    if (func->assumed_builtins)
--        _PyDict_DropWatcher(func->assumed_builtins, func);
-+
-+    _PyNativeFunction_IgnoreWatchedDicts(func);
- }
- 
- static PyObject *
-diff -r a9de87c4a9bc Objects/typeobject.c
---- a/Objects/typeobject.c	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Objects/typeobject.c	Thu Mar 11 15:40:51 2010 +0100
-@@ -112,7 +112,7 @@
- 			if (code != Py_None) {
- 				assert(PyCode_Check(code));
- 				_PyCode_InvalidateMachineCode(
--                                    (PyCodeObject*)code);
-+					(PyCodeObject*)code, CO_INVALID_TYPES);
- 			}
- #endif  /* WITH_LLVM */
- 			Py_DECREF(code);
-diff -r a9de87c4a9bc Python/eval.cc
---- a/Python/eval.cc	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Python/eval.cc	Thu Mar 11 15:40:51 2010 +0100
-@@ -1170,8 +1170,21 @@
- 		case _PYFRAME_CALL_PROFILE:
- 		case _PYFRAME_LINE_TRACE:
- 		case _PYFRAME_FATAL_GUARD_FAIL:
-+			/* These are handled by the opcode dispatch loop. */
-+			break;
- 		case _PYFRAME_GUARD_FAIL:
--			/* These are handled by the opcode dispatch loop. */
-+			if (f->f_native) {
-+				f->f_native->bailcount += 1;
-+				if (f->f_native->bailcount >=
-+					PY_MAX_GUARD_FAILS &&
-+					f->f_native->code)
-+				{
-+					_PyCode_InvalidateMachineCode(
-+						f->f_native->code,
-+						CO_INVALID_GUARDS);
-+					f->f_native->bailcount = 0;
-+				}
-+			}
- 			break;
- 
- 		default:
-@@ -4300,7 +4313,7 @@
- 	}
- 
- 	bool is_hot = false;
--	if (co->co_hotness > PY_HOTNESS_THRESHOLD) {
-+	if (co->co_hotness > co->co_hotness_threshold) {
- 		is_hot = true;
- #ifdef Py_WITH_INSTRUMENTATION
- 		hot_code->AddHotCode(co);
-@@ -4381,8 +4394,10 @@
- 		if (co->co_native == NULL) {
- 			co->co_use_llvm = 0;
- 			target_optimization = -1;
--			if (PyErr_Occured()) {
--				PyErr_Clear();
-+			if (PyErr_Occurred()) {
-+				co->co_optimization = -1;
-+				f->f_use_llvm = 0;
-+				return -1;
- 			}
- 		}
- 		co->co_optimization = target_optimization;
-diff -r a9de87c4a9bc Python/llvm_compile.cc
---- a/Python/llvm_compile.cc	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Python/llvm_compile.cc	Thu Mar 11 15:40:51 2010 +0100
-@@ -173,7 +173,7 @@
- }
- 
- void
--_PyCode_ToInlineLlvmIr(py::LlvmFunctionBuilder &fbuilder, PyCodeObject *code)
-+_PyCode_ToLlvmIr(py::LlvmFunctionBuilder &fbuilder, PyCodeObject *code)
- {
- 
-     PyGlobalLlvmData *global_data = PyGlobalLlvmData::Get();
-@@ -418,7 +418,7 @@
-     global_data->MaybeCollectUnusedGlobals();
- 
-     py::LlvmFunctionBuilder fbuilder(global_data, code, native);
--    _PyCode_ToInlineLlvmIr(fbuilder, code);
-+    _PyCode_ToLlvmIr(fbuilder, code);
- 
-     if (llvm::verifyFunction(*fbuilder.function(), llvm::PrintMessageAction)) {
-         PyErr_SetString(PyExc_SystemError, "invalid LLVM IR produced");
-diff -r a9de87c4a9bc Python/llvm_fbuilder.cc
---- a/Python/llvm_fbuilder.cc	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Python/llvm_fbuilder.cc	Thu Mar 11 15:40:51 2010 +0100
-@@ -1412,7 +1412,8 @@
-     PyCodeObject *code = this->code_object_;
- 
-     PyNativeFunctionObject *native = this->native_object_;
--    assert(native != NULL && "FDO_GLOBAL set but no native object encountered.");
-+    assert(native != NULL &&
-+           "FDO_GLOBAL set but no native object encountered.");
-     assert(native->watching != NULL);
-     assert(native->watching[WATCHING_GLOBALS]);
-     assert(native->watching[WATCHING_BUILTINS]);
-@@ -1477,10 +1478,14 @@
- void
- LlvmFunctionBuilder::LOAD_GLOBAL(int name_index)
- {
--    // A code object might not have co_watching set if
-+    // A native object might not be present when generating IR for output
-+    if (this->native_object_ == NULL)
-+        this->LOAD_GLOBAL_safe(name_index);
-+
-+    // A native object might not have watching set if
-     // a) it was compiled by setting co_optimization, or
-     // b) we couldn't watch the globals/builtins dicts.
--    PyObject **watching = this->code_object_->co_watching;
-+    PyObject **watching = this->native_object_->watching;
-     if (watching && watching[WATCHING_GLOBALS] && watching[WATCHING_BUILTINS])
-         this->LOAD_GLOBAL_fast(name_index);
-     else
-diff -r a9de87c4a9bc Python/llvm_fbuilder.h
---- a/Python/llvm_fbuilder.h	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Python/llvm_fbuilder.h	Thu Mar 11 15:40:51 2010 +0100
-@@ -34,7 +34,9 @@
-     void operator=(const LlvmFunctionBuilder &);  // Not implemented.
- 
- public:
--    LlvmFunctionBuilder(PyGlobalLlvmData *global_data, PyCodeObject *code, PyNativeFunctionObject *native = NULL);
-+    LlvmFunctionBuilder(PyGlobalLlvmData *global_data,
-+                        PyCodeObject *code,
-+                        PyNativeFunctionObject *native);
- 
-     llvm::Function *function() { return function_; }
-     typedef llvm::IRBuilder<true, llvm::TargetFolder> BuilderT;
-diff -r a9de87c4a9bc Unittests/DictTest.cc
---- a/Unittests/DictTest.cc	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Unittests/DictTest.cc	Thu Mar 11 15:40:51 2010 +0100
-@@ -2,6 +2,9 @@
- #include "nativefunctionobject.h"
- #include "gtest/gtest.h"
- 
-+
-+PyNativeFunctionObject *_PyNativeFunction_New();
-+
- class DictWatcherTest : public testing::Test {
- protected:
-     DictWatcherTest()
-diff -r a9de87c4a9bc Util/RuntimeFeedback.cc
---- a/Util/RuntimeFeedback.cc	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Util/RuntimeFeedback.cc	Thu Mar 11 15:40:51 2010 +0100
-@@ -173,8 +173,10 @@
-         this->SetFlagBit(SAW_A_NULL_OBJECT_BIT, true);
-         return;
-     }
--    if (!PyCFunction_Check(obj))
-+    if (!PyCFunction_Check(obj)) {
-+        this->SetMegamorphic();
-         return;
-+    }
- 
-     for (int i = 0; i < PyLimitedFeedback::NUM_POINTERS; ++i) {
-         PyMethodDef *value = (PyMethodDef *)this->data_[i].getPointer();
-@@ -213,6 +215,7 @@
- 
- PyFullFeedback::PyFullFeedback()
-     : counters_(/* Zero out the array. */),
-+      megamorphic(false),
-       usage_(UnknownMode)
- {
- }
-@@ -230,6 +233,7 @@
-         }
-         this->data_.insert(obj);
-     }
-+    this->megamorphic = src.megamorphic;
- }
- 
- PyFullFeedback::~PyFullFeedback()
-@@ -305,8 +309,10 @@
-     this->usage_ = FuncMode;
- 
-     // We only record C functions for now.
--    if (!PyCFunction_Check(obj))
-+    if (!PyCFunction_Check(obj)) {
-+        this->SetMegamorphic();
-         return;
-+    }
-     if (obj == NULL)
-         this->data_.insert(NULL);
-     else if (!this->data_.count(obj)) {
-diff -r a9de87c4a9bc Util/RuntimeFeedback.h
---- a/Util/RuntimeFeedback.h	Thu Mar 11 15:30:04 2010 +0100
-+++ b/Util/RuntimeFeedback.h	Thu Mar 11 15:40:51 2010 +0100
-@@ -73,6 +73,10 @@
-     // Assignment copies the list of collected objects, fixing up refcounts.
-     PyLimitedFeedback &operator=(PyLimitedFeedback rhs);
- 
-+    void SetMegamorphic() {
-+        SetFlagBit(SAW_MORE_THAN_THREE_OBJS_BIT, true);
-+    }
-+
- private:
-     // 'index' must be between 0 and 5 inclusive.
-     void SetFlagBit(unsigned index, bool value);
-@@ -129,13 +133,13 @@
-     void AddObjectSeen(PyObject *obj);
-     // Clears result and fills it with the set of seen objects.
-     void GetSeenObjectsInto(llvm::SmallVector<PyObject*, 3> &result) const;
--    bool ObjectsOverflowed() const { return false; }
-+    bool ObjectsOverflowed() const { return this->megamorphic; }
- 
-     // Record that a given function was called.
-     void AddFuncSeen(PyObject *obj);
-     // Clears result and fills it with the set of seen function objects.
-     void GetSeenFuncsInto(llvm::SmallVector<PyMethodDef*, 3> &result) const;
--    bool FuncsOverflowed() const { return false; }
-+    bool FuncsOverflowed() const { return this->megamorphic; }
- 
-     void IncCounter(unsigned counter_id);
-     uintptr_t GetCounter(unsigned counter_id) const;
-@@ -146,6 +150,10 @@
-     // Assignment copies the list of collected objects, fixing up refcounts.
-     PyFullFeedback &operator=(PyFullFeedback rhs);
-