Commits

ebo  committed 2abea14

Fixed a bug that produced incorrect traces. Changed getValue definition.

  • Participants
  • Parent commits 8a07a20

Comments (0)

Files changed (1)

 # HG changeset patch
 # Parent 7b530b3ed07bbc09b6ea3ef291ce796f06f54f0b
 
+diff --git a/Include/opcode.h b/Include/opcode.h
+--- a/Include/opcode.h
++++ b/Include/opcode.h
+@@ -146,7 +146,7 @@
+    as we want it to be available to both frameobject.c and ceval.c, while
+    remaining private.*/
+ #define EXCEPT_HANDLER 257
+-
++#define JIT_CALL 258
+ 
+ enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE, PyCmp_GT=Py_GT, PyCmp_GE=Py_GE,
+              PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};
 diff --git a/JIT/PyTypeBuilder.h b/JIT/PyTypeBuilder.h
 --- a/JIT/PyTypeBuilder.h
 +++ b/JIT/PyTypeBuilder.h
 +      JITFunctionState::BuilderT &builder = fbuilder.builder();
 +      JITFunctionState *state = fbuilder.getState();
 +
-+      Value *naked0 = arg0_->getStaticType()->getNakedValue(fbuilder);
-+      Value *naked1 = arg1_->getStaticType()->getNakedValue(fbuilder);
++      Value *naked0 = arg0_->getNakedValue(fbuilder);
++      Value *naked1 = arg1_->getNakedValue(fbuilder);
 +      Value *naked_result = builder.CreateFAdd(naked0, naked1);
 +
 +      Value *func = state->GetGlobalFunction<PyObject*(double)>("PyFloat_FromDouble");
 +      JITFunctionState *state = fbuilder.getState();
 +      Value *func = state->GetGlobalFunction<PyObject*(PyObject*, PyObject*)>("PyNumber_Add");
 +      llvm::Value *value = state->CreateCall(func,
-+                                             arg0_->getValue(),
-+                                             arg1_->getValue());
++                                             arg0_->getValue(fbuilder),
++                                             arg1_->getValue(fbuilder));
 +      result->init(value);
 +    }
 +
 +      JITFunctionState *state = fbuilder.getState();
 +      Value *func = state->GetGlobalFunction<PyObject*(PyObject*, PyObject*)>("PyNumber_InPlaceAdd");
 +      llvm::Value *value = state->CreateCall(func,
-+                                             arg0_->getValue(),
-+                                             arg1_->getValue());
++                                             arg0_->getValue(fbuilder),
++                                             arg1_->getValue(fbuilder));
 +      result->init(value);
 +    }
 +
 +      JITFunctionState::BuilderT &builder = fbuilder.builder();
 +      JITFunctionState *state = fbuilder.getState();
 +
-+      Value *naked0 = arg0_->getStaticType()->getNakedValue(fbuilder);
-+      Value *naked1 = arg1_->getStaticType()->getNakedValue(fbuilder);
++      Value *naked0 = arg0_->getNakedValue(fbuilder);
++      Value *naked1 = arg1_->getNakedValue(fbuilder);
 +      Value *naked_result = builder.CreateFSub(naked0, naked1);
 +
 +      Value *func = state->GetGlobalFunction<PyObject*(double)>("PyFloat_FromDouble");
 +      JITFunctionState *state = fbuilder.getState();
 +      Value *func = state->GetGlobalFunction<PyObject*(PyObject*, PyObject*)>("PyNumber_Subtract");
 +      llvm::Value *value = state->CreateCall(func,
-+                                             arg0_->getValue(),
-+                                             arg1_->getValue());
++                                             arg0_->getValue(fbuilder),
++                                             arg1_->getValue(fbuilder));
 +      result->init(value);
 +    }
 +
 +      JITFunctionState *state = fbuilder.getState();
 +      Value *func = state->GetGlobalFunction<PyObject*(PyObject*, PyObject*)>("PyNumber_InPlaceSubtract");
 +      llvm::Value *value = state->CreateCall(func,
-+                                             arg0_->getValue(),
-+                                             arg1_->getValue());
++                                             arg0_->getValue(fbuilder),
++                                             arg1_->getValue(fbuilder));
 +      result->init(value);
 +    }
 +
 +      JITFunctionState::BuilderT &builder = fbuilder.builder();
 +      JITFunctionState *state = fbuilder.getState();
 +
-+      Value *naked0 = arg0_->getStaticType()->getNakedValue(fbuilder);
-+      Value *naked1 = arg1_->getStaticType()->getNakedValue(fbuilder);
++      Value *naked0 = arg0_->getNakedValue(fbuilder);
++      Value *naked1 = arg1_->getNakedValue(fbuilder);
 +      Value *naked_result = builder.CreateFMul(naked0, naked1);
 +
 +      Value *func = state->GetGlobalFunction<PyObject*(double)>("PyFloat_FromDouble");
 +      JITFunctionState *state = fbuilder.getState();
 +      Value *func = state->GetGlobalFunction<PyObject*(PyObject*, PyObject*)>("PyNumber_Multiply");
 +      llvm::Value *value = state->CreateCall(func,
-+                                             arg0_->getValue(),
-+                                             arg1_->getValue());
++                                             arg0_->getValue(fbuilder),
++                                             arg1_->getValue(fbuilder));
 +      result->init(value);
 +    }
 +
 +      JITFunctionState *state = fbuilder.getState();
 +      Value *func = state->GetGlobalFunction<PyObject*(PyObject*, PyObject*)>("PyNumber_InPlaceMultiply");
 +      llvm::Value *value = state->CreateCall(func,
-+                                             arg0_->getValue(),
-+                                             arg1_->getValue());
++                                             arg0_->getValue(fbuilder),
++                                             arg1_->getValue(fbuilder));
 +      result->init(value);
 +    }
 +
 +      JITFunctionState::BuilderT &builder = fbuilder.builder();
 +      JITFunctionState *state = fbuilder.getState();
 +
-+      Value *naked0 = arg0_->getStaticType()->getNakedValue(fbuilder);
-+      Value *naked1 = arg1_->getStaticType()->getNakedValue(fbuilder);
++      Value *naked0 = arg0_->getNakedValue(fbuilder);
++      Value *naked1 = arg1_->getNakedValue(fbuilder);
 +      Value *naked_result = builder.CreateFDiv(naked0, naked1);
 +
 +      Value *func = state->GetGlobalFunction<PyObject*(double)>("PyFloat_FromDouble");
 +      JITFunctionState *state = fbuilder.getState();
 +      Value *func = state->GetGlobalFunction<PyObject*(PyObject*, PyObject*)>("PyNumber_FloorDivide");
 +      llvm::Value *value = state->CreateCall(func,
-+                                             arg0_->getValue(),
-+                                             arg1_->getValue());
++                                             arg0_->getValue(fbuilder),
++                                             arg1_->getValue(fbuilder));
 +      result->init(value);
 +    }
 +
 +      JITFunctionState *state = fbuilder.getState();
 +      Value *func = state->GetGlobalFunction<PyObject*(PyObject*, PyObject*)>("PyNumber_InPlaceFloorDivide");
 +      llvm::Value *value = state->CreateCall(func,
-+                                             arg0_->getValue(),
-+                                             arg1_->getValue());
++                                             arg0_->getValue(fbuilder),
++                                             arg1_->getValue(fbuilder));
 +      result->init(value);
 +    }
 +
    JITTraceAnalysis::JITTraceAnalysis()
      : stack_counter_(0)
    {
-@@ -206,37 +242,50 @@
+@@ -206,37 +242,52 @@
      }
    }
  
 -      JITValue *v = stack_.back();
 -      stack_.pop_back();
 -      return v;
++    JITStack stack_before((stack));
++
 +    int opcode = entry.opcode;
 +    JITValue *w = stack.Pop();
 +    JITValue *v = stack.Pop();
 +    if (impl->inlined()) {
 +      if (!type_v->hasGuard()) {
 +        type_v->setGuard(true);
-+        this->createGuardType(v, BH_NO_BLACKHOLE, entry.pc, stack);
++        this->createGuardType(v, BH_NO_BLACKHOLE, entry.pc, stack_before);
 +      }
 +
 +      if (!type_w->hasGuard()) {
 +        type_w->setGuard(true);
-+        this->createGuardType(w, BH_NO_BLACKHOLE, entry.pc, stack);
++        this->createGuardType(w, BH_NO_BLACKHOLE, entry.pc, stack_before);
 +      }
      }
  
      for (size_t i = 0; i < trace.size(); ++i) {
        TraceEntry &entry = trace[i];
        int opcode = entry.opcode;
-@@ -249,94 +298,79 @@
+@@ -249,101 +300,86 @@
  
  
        case POP_JUMP_IF_FALSE:
    }
  
    void
-@@ -351,6 +385,7 @@
+   JITForIter::emit(JITFunctionBuilder &fbuilder)
+   {
+     JITFunctionState::BuilderT &builder = fbuilder.builder();
+-    llvm::Value *v = arg_->getValue();
++    llvm::Value *v = arg_->getValue(fbuilder);
+     llvm::Value *v_type =
+       builder.CreateLoad(ObjectTy::ob_type(builder, v), "v_type");
+     llvm::Value *tp_iternext =
+@@ -351,13 +387,14 @@
                           "tp_iternext");
  
      value_ = fbuilder.getState()->CreateCall(tp_iternext, v);
    }
  
    void
-@@ -370,6 +405,7 @@
+   JITStoreFast::emit(JITFunctionBuilder &fbuilder)
+   {
+     JITFunctionState::BuilderT &builder = fbuilder.builder();
+-    llvm::Value *v = arg_->getValue();
++    llvm::Value *v = arg_->getValue(fbuilder);
+     llvm::Value *localptr = fbuilder.LocalPtr(oparg_);
+     llvm::Value *old = builder.CreateLoad(localptr);
+     builder.CreateStore(v, localptr);
+@@ -370,6 +407,7 @@
      JITFunctionState::BuilderT &builder = fbuilder.builder();
      llvm::Value *localptr = fbuilder.LocalPtr(oparg_);
      value_ = builder.CreateLoad(localptr);
    }
  
    void
-@@ -378,69 +414,32 @@
+@@ -378,71 +416,34 @@
      JITFunctionState::BuilderT &builder = fbuilder.builder();
      llvm::Value *localptr = fbuilder.ConstPtr(oparg_);
      value_ = builder.CreateLoad(localptr);
 -    value_ = state->CreateCall(func,
 -                               arg0_->getValue(),
 -                               arg1_->getValue());
+-    state->DecRef(arg0_->getValue());
+-    state->DecRef(arg1_->getValue());
 +    if (inplace_) {
 +      if (impl_->inlined())
 +        impl_->emit_inplace_inline(fbuilder, this->type_);
 +      else
 +        impl_->emit(fbuilder, this->type_);
 +    }
-     state->DecRef(arg0_->getValue());
-     state->DecRef(arg1_->getValue());
++    state->DecRef(arg0_->getValue(fbuilder));
++    state->DecRef(arg1_->getValue(fbuilder));
    }
-@@ -457,6 +456,38 @@
  
-     state->DecRef(arg0_->getValue());
-     state->DecRef(arg1_->getValue());
+   void
+@@ -452,11 +453,43 @@
+ 
+     Value *func = state->GetGlobalFunction<PyObject*(PyObject*, PyObject*, int)>("PyObject_RichCompare");
+     value_ = state->CreateCall(func,
+-      arg0_->getValue(), arg1_->getValue(),
++      arg0_->getValue(fbuilder), arg1_->getValue(fbuilder),
+       ConstantInt::getSigned(Type::getInt32Ty(state->context()), oparg_));
+ 
+-    state->DecRef(arg0_->getValue());
+-    state->DecRef(arg1_->getValue());
++    state->DecRef(arg0_->getValue(fbuilder));
++    state->DecRef(arg1_->getValue(fbuilder));
 +    this->getStaticType()->init(value_);
 +  }
 +
 +  JITGuardType::emit(JITFunctionBuilder &fbuilder)
 +  {
 +    JITType *type = arg_->getStaticType();
-+    llvm::Value *value = type->getValue();
++    llvm::Value *value = arg_->getValue(fbuilder);
 +
 +    JITFunctionState::BuilderT &builder = fbuilder.builder();
 +    JITFunctionState *state = fbuilder.getState();
 +
 +    builder.SetInsertPoint(bail);
 +    for (size_t i = 0; i < stack_.size(); ++i) {
-+      fbuilder.Push(stack_[i]->getValue());
++      fbuilder.Push(stack_[i]->getValue(fbuilder));
 +    }
 +    fbuilder.SetLastI(lasti_);
 +    fbuilder.SetBailReason(reason_);
    }
  
    void
-@@ -566,4 +597,19 @@
-     state->IncRef(arg_->getValue());
+@@ -468,13 +501,13 @@
+     llvm::BasicBlock *trace_cont = state->CreateBasicBlock("trace_cont");
+     llvm::BasicBlock *bail = state->CreateBasicBlock("bail");
+ 
+-    builder.CreateCondBr(state->IsNull(arg_->getValue()),
++    builder.CreateCondBr(state->IsNull(arg_->getValue(fbuilder)),
+                          bail, trace_cont);
+ 
+     builder.SetInsertPoint(bail);
+ 
+     for (size_t i = 0; i < stack_.size(); ++i) {
+-      fbuilder.Push(stack_[i]->getValue());
++      fbuilder.Push(stack_[i]->getValue(fbuilder));
+     }
+     fbuilder.SetLastI(lasti_);
+     fbuilder.SetBailReason(reason_);
+@@ -497,7 +530,7 @@
+     llvm::BasicBlock *bail_exc = state->CreateBasicBlock("bail_exc");
+     llvm::BasicBlock *bail_cont = state->CreateBasicBlock("bail_cont");
+ 
+-    Value *v = arg_->getValue();
++    Value *v = arg_->getValue(fbuilder);
+     Value *py_true = state->EmbedPointer<PyObject*>(&_Py_TrueStruct);
+     Value *py_false = state->EmbedPointer<PyObject*>(&_Py_FalseStruct);
+ 
+@@ -541,7 +574,7 @@
+ 
+     state->DecRef(v);
+     for (size_t i = 0; i < stack_.size(); ++i) {
+-      fbuilder.Push(stack_[i]->getValue());
++      fbuilder.Push(stack_[i]->getValue(fbuilder));
+     }
+     fbuilder.SetLastI(lasti_);
+     fbuilder.CreateBailBr();
+@@ -554,7 +587,7 @@
+   JITTraceEnd::emit(JITFunctionBuilder &fbuilder)
+   {
+     for (size_t i = 0; i < stack_.size(); ++i) {
+-      fbuilder.Push(stack_[i]->getValue());
++      fbuilder.Push(stack_[i]->getValue(fbuilder));
+     }
+     fbuilder.CreateEntryBr();
+   }
+@@ -563,7 +596,22 @@
+   JITIncRef::emit(JITFunctionBuilder &fbuilder)
+   {
+     JITFunctionState *state = fbuilder.getState();
+-    state->IncRef(arg_->getValue());
++    state->IncRef(arg_->getValue(fbuilder));
    }
  
 +  const int JITOpcodeInfo::ID = 0;
 diff --git a/JIT/jit_tracer.h b/JIT/jit_tracer.h
 --- a/JIT/jit_tracer.h
 +++ b/JIT/jit_tracer.h
-@@ -9,24 +9,76 @@
+@@ -9,43 +9,136 @@
  #include "JIT/global_jit_data.h"
  #include "JIT/jit_blackhole.h"
  #include "JIT/jit_fbuilder.h"
    public:
      virtual void dump() = 0;
      virtual void emit(JITFunctionBuilder &builder) = 0;
+-    virtual llvm::Value *getValue() const = 0;
 +
 +    virtual uintptr_t type() = 0;
 +  };
 +  class JITValue : public JITOpcode {
 +  public:
 +    // virtual llvm::Value *getOrMaterializeValue(JITFunctionBuilder &builder) = 0;
-     virtual llvm::Value *getValue() const = 0;
++    virtual llvm::Value *getValue(JITFunctionBuilder &fbuilder) const = 0;
++    virtual llvm::Value *getNakedValue(JITFunctionBuilder &fbuilder) const = 0;
++
 +    virtual JITType *getStaticType() const = 0;
 +    virtual void registerStaticType(const PyTypeObject *) = 0;
 +  };
      virtual void dump()
      {
        std::cout << "Stack Value\n";
-@@ -34,18 +86,52 @@
+     }
      virtual void emit(JITFunctionBuilder &builder);
-     virtual llvm::Value *getValue() const
+-    virtual llvm::Value *getValue() const
++    virtual llvm::Value *getValue(JITFunctionBuilder &fbuilder) const
      {
 -      return value_;
 +      return getStaticType()->getValue();
      }
  
++    virtual llvm::Value *getNakedValue(JITFunctionBuilder &fbuilder) const
++    {
++      return getStaticType()->getNakedValue(fbuilder);
++    }
++
 +    virtual JITType *getStaticType() const
 +    {
 +      return type_;
      virtual void dump()
      {
        std::cout << "FOR_ITER\n";
-@@ -55,17 +141,47 @@
+@@ -53,19 +146,54 @@
  
-     virtual llvm::Value *getValue() const
+     virtual void emit(JITFunctionBuilder &builder);
+ 
+-    virtual llvm::Value *getValue() const
++    virtual llvm::Value *getValue(JITFunctionBuilder &fbuilder) const
      {
 -      return value_;
 +      return getStaticType()->getValue();
      }
 +
++    virtual llvm::Value *getNakedValue(JITFunctionBuilder &fbuilder) const
++    {
++      return getStaticType()->getNakedValue(fbuilder);
++    }
++
 +    virtual JITType *getStaticType() const
 +    {
 +      return type_;
      {}
  
      virtual void dump()
-@@ -75,22 +191,28 @@
+@@ -75,22 +203,28 @@
  
      virtual void emit(JITFunctionBuilder &builder);
  
      virtual void dump()
      {
        std::cout << "LOAD_FAST\n";
-@@ -100,19 +222,54 @@
+@@ -98,21 +232,61 @@
  
-     virtual llvm::Value *getValue() const
+     virtual void emit(JITFunctionBuilder &builder);
+ 
+-    virtual llvm::Value *getValue() const
++    virtual llvm::Value *getValue(JITFunctionBuilder &fbuilder) const
      {
 -      return value_;
 +      return getStaticType()->getValue();
      }
 +
++    virtual llvm::Value *getNakedValue(JITFunctionBuilder &fbuilder) const
++    {
++      return getStaticType()->getNakedValue(fbuilder);
++    }
++
 +    virtual JITType *getStaticType() const
 +    {
 +      return type_;
      virtual void dump()
      {
        std::cout << "LOAD_CONST\n";
-@@ -124,133 +281,112 @@
+@@ -120,137 +294,128 @@
+ 
+     virtual void emit(JITFunctionBuilder &builder);
+ 
+-    virtual llvm::Value *getValue() const
++    virtual llvm::Value *getValue(JITFunctionBuilder &fbuilder) const
      {
-       return value_;
+-      return value_;
++      return getStaticType()->getValue();
      }
 +
++    virtual llvm::Value *getNakedValue(JITFunctionBuilder &fbuilder) const
++    {
++      return getStaticType()->getNakedValue(fbuilder);
++    }
++
 +    virtual JITType *getStaticType() const
 +    {
 +      return type_;
 +          JITType *old_type = type_;
 +          type_ = new_type;
 +          delete old_type;
++
++          type_->setGuard(true);
 +        }
 +      }
 +      else {
  
      virtual void emit(JITFunctionBuilder &builder);
  
-     virtual llvm::Value *getValue() const
+-    virtual llvm::Value *getValue() const
++    virtual llvm::Value *getValue(JITFunctionBuilder &fbuilder) const
      {
 -      return value_;
 +      return getStaticType()->getValue();
      }
 +
++    virtual llvm::Value *getNakedValue(JITFunctionBuilder &fbuilder) const
++    {
++      return getStaticType()->getNakedValue(fbuilder);
++    }
++
 +    virtual JITType *getStaticType() const
 +    {
 +      return type_;
      virtual void dump()
      {
        std::cout << "COMPARE_OP " << oparg_ << "\n";
-@@ -262,16 +398,33 @@
+@@ -258,20 +423,55 @@
+ 
+     virtual void emit(JITFunctionBuilder &builder);
+ 
+-    virtual llvm::Value *getValue() const
++    virtual JITType *getStaticType() const
      {
-       return value_;
+-      return value_;
++      return type_;
      }
 +
-+    virtual JITType *getStaticType() const
++    virtual llvm::Value *getValue(JITFunctionBuilder &fbuilder) const
 +    {
-+      return type_;
++      return getStaticType()->getValue();
 +    }
 +
-+    virtual void registerStaticType(const PyTypeObject *)
++    virtual llvm::Value *getNakedValue(JITFunctionBuilder &fbuilder) const
 +    {
++      return getStaticType()->getNakedValue(fbuilder);
++    }
++
++    virtual void registerStaticType(const PyTypeObject *t)
++    {
++      if (type_ == NULL || type_->canSpecialize(t)) {
++        JITType *new_type = JITType::createBoxedType(t);
++        if (new_type != NULL) {
++          JITType *old_type = type_;
++          type_ = new_type;
++          delete old_type;
++        }
++      }
++      else {
++        if (!type_->matchesType(t)) {
++          std::cout << "Trace Type mismatch\n";
++        }
++      }
 +    }
 +
 +    virtual uintptr_t type()
      {}
  
      virtual void dump()
-@@ -281,21 +434,50 @@
+@@ -281,21 +481,50 @@
  
      virtual void emit(JITFunctionBuilder &builder);
  
      {}
  
      virtual void dump()
-@@ -305,24 +487,25 @@
+@@ -305,24 +534,25 @@
  
      virtual void emit(JITFunctionBuilder &builder);
  
      {}
  
      virtual void dump()
-@@ -332,22 +515,23 @@
+@@ -332,22 +562,23 @@
  
      virtual void emit(JITFunctionBuilder &builder);
  
      {}
  
      virtual void dump()
-@@ -357,24 +541,39 @@
+@@ -357,24 +588,39 @@
  
      virtual void emit(JITFunctionBuilder &builder);
  
      void dump()
      {
        for (size_t i = 0; i < loaded_stack_.size(); ++i) {
-@@ -387,6 +586,15 @@
+@@ -387,6 +633,15 @@
  
      void emit(JITFunctionState &state);
  
      JITForIter *createForIter(JITValue *v)
      {
        JITForIter *r = new JITForIter(v);
-@@ -394,37 +602,9 @@
+@@ -394,37 +649,9 @@
        return r;
      }
  
        trace_.push_back(r);
        return r;
      }
-@@ -436,16 +616,23 @@
+@@ -436,16 +663,23 @@
        return r;
      }
  
        trace_.push_back(r);
        return r;
      }
-@@ -478,19 +665,18 @@
+@@ -478,19 +712,18 @@
        return r;
      }
  
      why = WHY_NOT;
      err = 0;
      x = Py_None;        /* Not a reference, just anything non-NULL */
-@@ -1315,11 +1322,31 @@
+@@ -1315,11 +1322,33 @@
  
          /* Extract opcode and argument */
  
 +        if (co->co_jit && f->f_lasti == co->co_jit->opcode) {
++          trace->trace(co, f->f_lasti, JIT_CALL, 0);
++
 +          f->f_stacktop = stack_pointer;
 +          int reason = co->co_jit->trace(f);
 +          int val = jit::PyJIT_BlackHoleEval(f, reason);
      dispatch_opcode:
  #ifdef DYNAMIC_EXECUTION_PROFILE
  #ifdef DXPAIRS
-@@ -1364,6 +1391,7 @@
+@@ -1364,6 +1393,7 @@
                  PUSH(x);
                  FAST_DISPATCH();
              }
              format_exc_check_arg(PyExc_UnboundLocalError,
                  UNBOUNDLOCAL_ERROR_MSG,
                  PyTuple_GetItem(co->co_varnames, oparg));
-@@ -1378,6 +1406,7 @@
+@@ -1378,6 +1408,7 @@
          PREDICTED_WITH_ARG(STORE_FAST);
          TARGET(STORE_FAST)
              v = POP();
              SETLOCAL(oparg, v);
              FAST_DISPATCH();
  
-@@ -1468,41 +1497,53 @@
+@@ -1468,41 +1499,53 @@
              Py_DECREF(w);
              SET_TOP(x);
              if (x != NULL) DISPATCH();
              if (PyUnicode_CheckExact(v))
                  x = PyUnicode_Format(v, w);
              else
-@@ -1511,11 +1552,14 @@
+@@ -1511,11 +1554,14 @@
              Py_DECREF(w);
              SET_TOP(x);
              if (x != NULL) DISPATCH();
              if (PyUnicode_CheckExact(v) &&
                       PyUnicode_CheckExact(w)) {
                  x = unicode_concatenate(v, w, f, next_instr);
-@@ -1530,26 +1574,33 @@
+@@ -1530,26 +1576,33 @@
              Py_DECREF(w);
              SET_TOP(x);
              if (x != NULL) DISPATCH();
              break;
  
          TARGET(BINARY_LSHIFT)
-@@ -1627,6 +1678,8 @@
+@@ -1627,6 +1680,8 @@
          TARGET(INPLACE_POWER)
              w = POP();
              v = TOP();
              x = PyNumber_InPlacePower(v, w, Py_None);
              Py_DECREF(v);
              Py_DECREF(w);
-@@ -1637,6 +1690,8 @@
+@@ -1637,6 +1692,8 @@
          TARGET(INPLACE_MULTIPLY)
              w = POP();
              v = TOP();
              x = PyNumber_InPlaceMultiply(v, w);
              Py_DECREF(v);
              Py_DECREF(w);
-@@ -1647,6 +1702,8 @@
+@@ -1647,6 +1704,8 @@
          TARGET(INPLACE_TRUE_DIVIDE)
              w = POP();
              v = TOP();
              x = PyNumber_InPlaceTrueDivide(v, w);
              Py_DECREF(v);
              Py_DECREF(w);
-@@ -1657,6 +1714,8 @@
+@@ -1657,6 +1716,8 @@
          TARGET(INPLACE_FLOOR_DIVIDE)
              w = POP();
              v = TOP();
              x = PyNumber_InPlaceFloorDivide(v, w);
              Py_DECREF(v);
              Py_DECREF(w);
-@@ -1667,6 +1726,8 @@
+@@ -1667,6 +1728,8 @@
          TARGET(INPLACE_MODULO)
              w = POP();
              v = TOP();
              x = PyNumber_InPlaceRemainder(v, w);
              Py_DECREF(v);
              Py_DECREF(w);
-@@ -1677,6 +1738,8 @@
+@@ -1677,6 +1740,8 @@
          TARGET(INPLACE_ADD)
              w = POP();
              v = TOP();
              if (PyUnicode_CheckExact(v) &&
                       PyUnicode_CheckExact(w)) {
                  x = unicode_concatenate(v, w, f, next_instr);
-@@ -1696,6 +1759,8 @@
+@@ -1696,6 +1761,8 @@
          TARGET(INPLACE_SUBTRACT)
              w = POP();
              v = TOP();
              x = PyNumber_InPlaceSubtract(v, w);
              Py_DECREF(v);
              Py_DECREF(w);
-@@ -2382,16 +2447,22 @@
+@@ -2382,16 +2449,22 @@
              if (w == Py_False) {
                  Py_DECREF(w);
                  JUMPTO(oparg);
              DISPATCH();
  
          PREDICTED_WITH_ARG(POP_JUMP_IF_TRUE);
-@@ -2468,6 +2539,8 @@
+@@ -2468,6 +2541,8 @@
          PREDICTED_WITH_ARG(JUMP_ABSOLUTE);
          TARGET(JUMP_ABSOLUTE)
              JUMPTO(oparg);
  #if FAST_LOOPS
              /* Enabling this path speeds-up all while and for-loops by bypassing
                 the per-loop checks for signals.  By default, this should be turned-off
-@@ -2507,11 +2580,14 @@
+@@ -2507,11 +2582,14 @@
              }
              if (PyErr_Occurred()) {
                  if (!PyErr_ExceptionMatches(