Commits

Maciej Fijalkowski committed ecb0ec3 Merge

(cfbolz, fijal) Merge virtual-arguments.

This branch improve handling of **kwargs and **args, making them virtual in
some cases. An example of code that gets good speedups:

f(**{'a': 'b'})

Comments (0)

Files changed (73)

pypy/annotation/binaryop.py

File contents unchanged.

pypy/interpreter/argument.py

             make_sure_not_resized(self.keywords_w)
 
         make_sure_not_resized(self.arguments_w)
-        if w_stararg is not None:
-            self._combine_starargs_wrapped(w_stararg)
-        # if we have a call where **args are used at the callsite
-        # we shouldn't let the JIT see the argument matching
-        self._dont_jit = (w_starstararg is not None and
-                          self._combine_starstarargs_wrapped(w_starstararg))
+        self._combine_wrapped(w_stararg, w_starstararg)
+        # a flag that specifies whether the JIT can unroll loops that operate
+        # on the keywords
+        self._jit_few_keywords = self.keywords is None or jit.isconstant(len(self.keywords))
 
     def __repr__(self):
         """ NOT_RPYTHON """
 
     ###  Manipulation  ###
 
-    @jit.look_inside_iff(lambda self: not self._dont_jit)
+    @jit.look_inside_iff(lambda self: self._jit_few_keywords)
     def unpack(self): # slowish
         "Return a ([w1,w2...], {'kw':w3...}) pair."
         kwds_w = {}
         keywords, values_w = space.view_as_kwargs(w_starstararg)
         if keywords is not None: # this path also taken for empty dicts
             if self.keywords is None:
-                self.keywords = keywords[:] # copy to make non-resizable
-                self.keywords_w = values_w[:]
+                self.keywords = keywords
+                self.keywords_w = values_w
             else:
-                self._check_not_duplicate_kwargs(keywords, values_w)
+                _check_not_duplicate_kwargs(
+                    self.space, self.keywords, keywords, values_w)
                 self.keywords = self.keywords + keywords
                 self.keywords_w = self.keywords_w + values_w
-            return not jit.isconstant(len(self.keywords))
+            return
         if space.isinstance_w(w_starstararg, space.w_dict):
             keys_w = space.unpackiterable(w_starstararg)
         else:
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-        return True
-
-    def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
-        space = self.space
         keywords_w = [None] * len(keys_w)
         keywords = [None] * len(keys_w)
-        i = 0
-        for w_key in keys_w:
-            try:
-                key = space.str_w(w_key)
-            except OperationError, e:
-                if e.match(space, space.w_TypeError):
-                    raise OperationError(
-                        space.w_TypeError,
-                        space.wrap("keywords must be strings"))
-                if e.match(space, space.w_UnicodeEncodeError):
-                    # Allow this to pass through
-                    key = None
-                else:
-                    raise
-            else:
-                if self.keywords and key in self.keywords:
-                    raise operationerrfmt(self.space.w_TypeError,
-                                          "got multiple values "
-                                          "for keyword argument "
-                                          "'%s'", key)
-            keywords[i] = key
-            keywords_w[i] = space.getitem(w_starstararg, w_key)
-            i += 1
+        _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords, keywords_w, self.keywords)
+        self.keyword_names_w = keys_w
         if self.keywords is None:
             self.keywords = keywords
             self.keywords_w = keywords_w
         else:
             self.keywords = self.keywords + keywords
             self.keywords_w = self.keywords_w + keywords_w
-        self.keyword_names_w = keys_w
 
-    @jit.look_inside_iff(lambda self, keywords, keywords_w:
-            jit.isconstant(len(keywords) and
-            jit.isconstant(self.keywords)))
-    def _check_not_duplicate_kwargs(self, keywords, keywords_w):
-        # looks quadratic, but the JIT should remove all of it nicely.
-        # Also, all the lists should be small
-        for key in keywords:
-            for otherkey in self.keywords:
-                if otherkey == key:
-                    raise operationerrfmt(self.space.w_TypeError,
-                                          "got multiple values "
-                                          "for keyword argument "
-                                          "'%s'", key)
 
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
 
     ###  Parsing for function calls  ###
 
-    # XXX: this should be @jit.look_inside_iff, but we need key word arguments,
-    # and it doesn't support them for now.
+    @jit.unroll_safe
     def _match_signature(self, w_firstarg, scope_w, signature, defaults_w=None,
                          blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
         or raise an ArgErr in case of failure.
-        Return the number of arguments filled in.
         """
-        if jit.we_are_jitted() and self._dont_jit:
-            return self._match_signature_jit_opaque(w_firstarg, scope_w,
-                                                    signature, defaults_w,
-                                                    blindargs)
-        return self._really_match_signature(w_firstarg, scope_w, signature,
-                                            defaults_w, blindargs)
-
-    @jit.dont_look_inside
-    def _match_signature_jit_opaque(self, w_firstarg, scope_w, signature,
-                                    defaults_w, blindargs):
-        return self._really_match_signature(w_firstarg, scope_w, signature,
-                                            defaults_w, blindargs)
-
-    @jit.unroll_safe
-    def _really_match_signature(self, w_firstarg, scope_w, signature,
-                                defaults_w=None, blindargs=0):
-        #
+        #   w_firstarg = a first argument to be inserted (e.g. self) or None
         #   args_w = list of the normal actual parameters, wrapped
-        #   kwds_w = real dictionary {'keyword': wrapped parameter}
-        #   argnames = list of formal parameter names
         #   scope_w = resulting list of wrapped values
         #
 
         # so all values coming from there can be assumed constant. It assumes
         # that the length of the defaults_w does not vary too much.
         co_argcount = signature.num_argnames() # expected formal arguments, without */**
-        has_vararg = signature.has_vararg()
-        has_kwarg = signature.has_kwarg()
-        extravarargs = None
-        input_argcount =  0
 
+        # put the special w_firstarg into the scope, if it exists
         if w_firstarg is not None:
             upfront = 1
             if co_argcount > 0:
                 scope_w[0] = w_firstarg
-                input_argcount = 1
-            else:
-                extravarargs = [w_firstarg]
         else:
             upfront = 0
 
         args_w = self.arguments_w
         num_args = len(args_w)
+        avail = num_args + upfront
 
         keywords = self.keywords
-        keywords_w = self.keywords_w
         num_kwds = 0
         if keywords is not None:
             num_kwds = len(keywords)
 
-        avail = num_args + upfront
 
+        # put as many positional input arguments into place as available
+        input_argcount = upfront
         if input_argcount < co_argcount:
-            # put as many positional input arguments into place as available
-            if avail > co_argcount:
-                take = co_argcount - input_argcount
-            else:
-                take = num_args
+            take = min(num_args, co_argcount - upfront)
 
             # letting the JIT unroll this loop is safe, because take is always
             # smaller than co_argcount
             input_argcount += take
 
         # collect extra positional arguments into the *vararg
-        if has_vararg:
+        if signature.has_vararg():
             args_left = co_argcount - upfront
             if args_left < 0:  # check required by rpython
-                assert extravarargs is not None
-                starargs_w = extravarargs
+                starargs_w = [w_firstarg]
                 if num_args:
                     starargs_w = starargs_w + args_w
             elif num_args > args_left:
                 starargs_w = []
             scope_w[co_argcount] = self.space.newtuple(starargs_w)
         elif avail > co_argcount:
-            raise ArgErrCount(avail, num_kwds,
-                              co_argcount, has_vararg, has_kwarg,
-                              defaults_w, 0)
+            raise ArgErrCount(avail, num_kwds, signature, defaults_w, 0)
 
-        # the code assumes that keywords can potentially be large, but that
-        # argnames is typically not too large
-        num_remainingkwds = num_kwds
-        used_keywords = None
-        if keywords:
-            # letting JIT unroll the loop is *only* safe if the callsite didn't
-            # use **args because num_kwds can be arbitrarily large otherwise.
-            used_keywords = [False] * num_kwds
-            for i in range(num_kwds):
-                name = keywords[i]
-                # If name was not encoded as a string, it could be None. In that
-                # case, it's definitely not going to be in the signature.
-                if name is None:
-                    continue
-                j = signature.find_argname(name)
-                if j < 0:
-                    continue
-                elif j < input_argcount:
-                    # check that no keyword argument conflicts with these. note
-                    # that for this purpose we ignore the first blindargs,
-                    # which were put into place by prepend().  This way,
-                    # keywords do not conflict with the hidden extra argument
-                    # bound by methods.
-                    if blindargs <= j:
-                        raise ArgErrMultipleValues(name)
+        # if a **kwargs argument is needed, create the dict
+        w_kwds = None
+        if signature.has_kwarg():
+            w_kwds = self.space.newdict(kwargs=True)
+            scope_w[co_argcount + signature.has_vararg()] = w_kwds
+
+        # handle keyword arguments
+        num_remainingkwds = 0
+        keywords_w = self.keywords_w
+        kwds_mapping = None
+        if num_kwds:
+            # kwds_mapping maps target indexes in the scope (minus input_argcount)
+            # to positions in the keywords_w list
+            cnt = (co_argcount - input_argcount)
+            if cnt < 0:
+                cnt = 0
+            kwds_mapping = [0] * cnt
+            # initialize manually, for the JIT :-(
+            for i in range(len(kwds_mapping)):
+                kwds_mapping[i] = -1
+            # match the keywords given at the call site to the argument names
+            # the called function takes
+            # this function must not take a scope_w, to make the scope not
+            # escape
+            num_remainingkwds = _match_keywords(
+                    signature, blindargs, input_argcount, keywords,
+                    kwds_mapping, self._jit_few_keywords)
+            if num_remainingkwds:
+                if w_kwds is not None:
+                    # collect extra keyword arguments into the **kwarg
+                    _collect_keyword_args(
+                            self.space, keywords, keywords_w, w_kwds,
+                            kwds_mapping, self.keyword_names_w, self._jit_few_keywords)
                 else:
-                    assert scope_w[j] is None
-                    scope_w[j] = keywords_w[i]
-                    used_keywords[i] = True # mark as used
-                    num_remainingkwds -= 1
+                    if co_argcount == 0:
+                        raise ArgErrCount(avail, num_kwds, signature, defaults_w, 0)
+                    raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
+                                            kwds_mapping, self.keyword_names_w)
+
+        # check for missing arguments and fill them from the kwds,
+        # or with defaults, if available
         missing = 0
         if input_argcount < co_argcount:
             def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
+            j = 0
+            kwds_index = -1
             for i in range(input_argcount, co_argcount):
-                if scope_w[i] is not None:
-                    continue
+                if kwds_mapping is not None:
+                    kwds_index = kwds_mapping[j]
+                    j += 1
+                    if kwds_index >= 0:
+                        scope_w[i] = keywords_w[kwds_index]
+                        continue
                 defnum = i - def_first
                 if defnum >= 0:
                     scope_w[i] = defaults_w[defnum]
                 else:
-                    # error: not enough arguments.  Don't signal it immediately
-                    # because it might be related to a problem with */** or
-                    # keyword arguments, which will be checked for below.
                     missing += 1
-
-        # collect extra keyword arguments into the **kwarg
-        if has_kwarg:
-            w_kwds = self.space.newdict(kwargs=True)
-            if num_remainingkwds:
-                #
-                limit = len(keywords)
-                if self.keyword_names_w is not None:
-                    limit -= len(self.keyword_names_w)
-                for i in range(len(keywords)):
-                    if not used_keywords[i]:
-                        if i < limit:
-                            w_key = self.space.wrap(keywords[i])
-                        else:
-                            w_key = self.keyword_names_w[i - limit]
-                        self.space.setitem(w_kwds, w_key, keywords_w[i])
-                #
-            scope_w[co_argcount + has_vararg] = w_kwds
-        elif num_remainingkwds:
-            if co_argcount == 0:
-                raise ArgErrCount(avail, num_kwds,
-                              co_argcount, has_vararg, has_kwarg,
-                              defaults_w, missing)
-            raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
-                                    used_keywords, self.keyword_names_w)
-
-        if missing:
-            raise ArgErrCount(avail, num_kwds,
-                              co_argcount, has_vararg, has_kwarg,
-                              defaults_w, missing)
-
-        return co_argcount + has_vararg + has_kwarg
+            if missing:
+                raise ArgErrCount(avail, num_kwds, signature, defaults_w, missing)
 
 
 
         scope_w must be big enough for signature.
         """
         try:
-            return self._match_signature(w_firstarg,
-                                         scope_w, signature, defaults_w, 0)
+            self._match_signature(w_firstarg,
+                                  scope_w, signature, defaults_w, 0)
         except ArgErr, e:
             raise operationerrfmt(self.space.w_TypeError,
                                   "%s() %s", fnname, e.getmsg())
+        return signature.scope_length()
 
     def _parse(self, w_firstarg, signature, defaults_w, blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
                 space.setitem(w_kwds, w_key, self.keywords_w[i])
         return w_args, w_kwds
 
+# JIT helper functions
+# these functions contain functionality that the JIT is not always supposed to
+# look at. They should not get a self arguments, which makes the amount of
+# arguments annoying :-(
+
+@jit.look_inside_iff(lambda space, existingkeywords, keywords, keywords_w:
+        jit.isconstant(len(keywords) and
+        jit.isconstant(existingkeywords)))
+def _check_not_duplicate_kwargs(space, existingkeywords, keywords, keywords_w):
+    # looks quadratic, but the JIT should remove all of it nicely.
+    # Also, all the lists should be small
+    for key in keywords:
+        for otherkey in existingkeywords:
+            if otherkey == key:
+                raise operationerrfmt(space.w_TypeError,
+                                      "got multiple values "
+                                      "for keyword argument "
+                                      "'%s'", key)
+
+def _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords,
+        keywords_w, existingkeywords):
+    i = 0
+    for w_key in keys_w:
+        try:
+            key = space.str_w(w_key)
+        except OperationError, e:
+            if e.match(space, space.w_TypeError):
+                raise OperationError(
+                    space.w_TypeError,
+                    space.wrap("keywords must be strings"))
+            if e.match(space, space.w_UnicodeEncodeError):
+                # Allow this to pass through
+                key = None
+            else:
+                raise
+        else:
+            if existingkeywords and key in existingkeywords:
+                raise operationerrfmt(space.w_TypeError,
+                                      "got multiple values "
+                                      "for keyword argument "
+                                      "'%s'", key)
+        keywords[i] = key
+        keywords_w[i] = space.getitem(w_starstararg, w_key)
+        i += 1
+
+@jit.look_inside_iff(
+    lambda signature, blindargs, input_argcount,
+           keywords, kwds_mapping, jiton: jiton)
+def _match_keywords(signature, blindargs, input_argcount,
+                    keywords, kwds_mapping, _):
+    # letting JIT unroll the loop is *only* safe if the callsite didn't
+    # use **args because num_kwds can be arbitrarily large otherwise.
+    num_kwds = num_remainingkwds = len(keywords)
+    for i in range(num_kwds):
+        name = keywords[i]
+        # If name was not encoded as a string, it could be None. In that
+        # case, it's definitely not going to be in the signature.
+        if name is None:
+            continue
+        j = signature.find_argname(name)
+        # if j == -1 nothing happens, because j < input_argcount and
+        # blindargs > j
+        if j < input_argcount:
+            # check that no keyword argument conflicts with these. note
+            # that for this purpose we ignore the first blindargs,
+            # which were put into place by prepend().  This way,
+            # keywords do not conflict with the hidden extra argument
+            # bound by methods.
+            if blindargs <= j:
+                raise ArgErrMultipleValues(name)
+        else:
+            kwds_mapping[j - input_argcount] = i # map to the right index
+            num_remainingkwds -= 1
+    return num_remainingkwds
+
+@jit.look_inside_iff(
+    lambda space, keywords, keywords_w, w_kwds, kwds_mapping,
+        keyword_names_w, jiton: jiton)
+def _collect_keyword_args(space, keywords, keywords_w, w_kwds, kwds_mapping,
+                          keyword_names_w, _):
+    limit = len(keywords)
+    if keyword_names_w is not None:
+        limit -= len(keyword_names_w)
+    for i in range(len(keywords)):
+        # again a dangerous-looking loop that either the JIT unrolls
+        # or that is not too bad, because len(kwds_mapping) is small
+        for j in kwds_mapping:
+            if i == j:
+                break
+        else:
+            if i < limit:
+                w_key = space.wrap(keywords[i])
+            else:
+                w_key = keyword_names_w[i - limit]
+            space.setitem(w_kwds, w_key, keywords_w[i])
+
 class ArgumentsForTranslation(Arguments):
     def __init__(self, space, args_w, keywords=None, keywords_w=None,
                  w_stararg=None, w_starstararg=None):
 
 class ArgErrCount(ArgErr):
 
-    def __init__(self, got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
+    def __init__(self, got_nargs, nkwds, signature,
                  defaults_w, missing_args):
-        self.expected_nargs = expected_nargs
-        self.has_vararg = has_vararg
-        self.has_kwarg = has_kwarg
+        self.signature = signature
 
         self.num_defaults = 0 if defaults_w is None else len(defaults_w)
         self.missing_args = missing_args
         self.num_kwds = nkwds
 
     def getmsg(self):
-        n = self.expected_nargs
+        n = self.signature.num_argnames()
         if n == 0:
             msg = "takes no arguments (%d given)" % (
                 self.num_args + self.num_kwds)
         else:
             defcount = self.num_defaults
-            has_kwarg = self.has_kwarg
+            has_kwarg = self.signature.has_kwarg()
             num_args = self.num_args
             num_kwds = self.num_kwds
-            if defcount == 0 and not self.has_vararg:
+            if defcount == 0 and not self.signature.has_vararg():
                 msg1 = "exactly"
                 if not has_kwarg:
                     num_args += num_kwds
 
 class ArgErrUnknownKwds(ArgErr):
 
-    def __init__(self, space, num_remainingkwds, keywords, used_keywords,
+    def __init__(self, space, num_remainingkwds, keywords, kwds_mapping,
                  keyword_names_w):
         name = ''
         self.num_kwds = num_remainingkwds
         if num_remainingkwds == 1:
             for i in range(len(keywords)):
-                if not used_keywords[i]:
+                if i not in kwds_mapping:
                     name = keywords[i]
                     if name is None:
                         # We'll assume it's unicode. Encode it.

pypy/interpreter/astcompiler/assemble.py

File contents unchanged.

pypy/interpreter/baseobjspace.py

File contents unchanged.

pypy/interpreter/pyframe.py

File contents unchanged.

pypy/interpreter/test/test_argument.py

     def __nonzero__(self):
         raise NotImplementedError
 
+class kwargsdict(dict):
+    pass
+
 class DummySpace(object):
     def newtuple(self, items):
         return tuple(items)
         return list(it)
 
     def view_as_kwargs(self, x):
+        if len(x) == 0:
+            return [], []
         return None, None
 
     def newdict(self, kwargs=False):
+        if kwargs:
+            return kwargsdict()
         return {}
 
     def newlist(self, l=[]):
             args._match_signature(None, l, Signature(["a", "b", "c"], None, "**"))
             assert l == [1, 2, 3, {'d': 4}]
 
+    def test_match_kwds_creates_kwdict(self):
+        space = DummySpace()
+        kwds = [("c", 3), ('d', 4)]
+        for i in range(4):
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            w_kwds = dummy_wrapped_dict(kwds[i:])
+            if i == 3:
+                w_kwds = None
+            args = Arguments(space, [1, 2], keywords, keywords_w, w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"], None, "**"))
+            assert l == [1, 2, 3, {'d': 4}]
+            assert isinstance(l[-1], kwargsdict)
+
     def test_duplicate_kwds(self):
         space = DummySpace()
         excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
         # defaults_w, missing_args
-        err = ArgErrCount(1, 0, 0, False, False, None, 0)
+        sig = Signature([], None, None)
+        err = ArgErrCount(1, 0, sig, None, 0)
         s = err.getmsg()
         assert s == "takes no arguments (1 given)"
-        err = ArgErrCount(0, 0, 1, False, False, [], 1)
+
+        sig = Signature(['a'], None, None)
+        err = ArgErrCount(0, 0, sig, [], 1)
         s = err.getmsg()
         assert s == "takes exactly 1 argument (0 given)"
-        err = ArgErrCount(3, 0, 2, False, False, [], 0)
+
+        sig = Signature(['a', 'b'], None, None)
+        err = ArgErrCount(3, 0, sig, [], 0)
         s = err.getmsg()
         assert s == "takes exactly 2 arguments (3 given)"
-        err = ArgErrCount(3, 0, 2, False, False, ['a'], 0)
+        err = ArgErrCount(3, 0, sig, ['a'], 0)
         s = err.getmsg()
         assert s == "takes at most 2 arguments (3 given)"
-        err = ArgErrCount(1, 0, 2, True, False, [], 1)
+
+        sig = Signature(['a', 'b'], '*', None)
+        err = ArgErrCount(1, 0, sig, [], 1)
         s = err.getmsg()
         assert s == "takes at least 2 arguments (1 given)"
-        err = ArgErrCount(0, 1, 2, True, False, ['a'], 1)
+        err = ArgErrCount(0, 1, sig, ['a'], 1)
         s = err.getmsg()
         assert s == "takes at least 1 non-keyword argument (0 given)"
-        err = ArgErrCount(2, 1, 1, False, True, [], 0)
+
+        sig = Signature(['a'], None, '**')
+        err = ArgErrCount(2, 1, sig, [], 0)
         s = err.getmsg()
         assert s == "takes exactly 1 non-keyword argument (2 given)"
-        err = ArgErrCount(0, 1, 1, False, True, [], 1)
+        err = ArgErrCount(0, 1, sig, [], 1)
         s = err.getmsg()
         assert s == "takes exactly 1 non-keyword argument (0 given)"
-        err = ArgErrCount(0, 1, 1, True, True, [], 1)
+
+        sig = Signature(['a'], '*', '**')
+        err = ArgErrCount(0, 1, sig, [], 1)
         s = err.getmsg()
         assert s == "takes at least 1 non-keyword argument (0 given)"
-        err = ArgErrCount(2, 1, 1, False, True, ['a'], 0)
+
+        sig = Signature(['a'], None, '**')
+        err = ArgErrCount(2, 1, sig, ['a'], 0)
         s = err.getmsg()
         assert s == "takes at most 1 non-keyword argument (2 given)"
 
 
     def test_unknown_keywords(self):
         space = DummySpace()
-        err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [True, False], None)
+        err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [0], None)
         s = err.getmsg()
         assert s == "got an unexpected keyword argument 'b'"
+        err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [1], None)
+        s = err.getmsg()
+        assert s == "got an unexpected keyword argument 'a'"
         err = ArgErrUnknownKwds(space, 2, ['a', 'b', 'c'],
-                                [True, False, False], None)
+                                [0], None)
         s = err.getmsg()
         assert s == "got 2 unexpected keyword arguments"
 
                 defaultencoding = 'utf-8'
         space = DummySpaceUnicode()
         err = ArgErrUnknownKwds(space, 1, ['a', None, 'b', 'c'],
-                                [True, False, True, True],
+                                [0, 3, 2],
                                 [unichr(0x1234), u'b', u'c'])
         s = err.getmsg()
         assert s == "got an unexpected keyword argument '\xe1\x88\xb4'"

pypy/jit/backend/llgraph/llimpl.py

 
 def do_new_array(arraynum, count):
     TYPE = symbolic.Size2Type[arraynum]
+    assert count >= 0 # explode if it's not
     x = lltype.malloc(TYPE, count, zero=True)
     return cast_to_ptr(x)
 

pypy/jit/backend/x86/assembler.py

     genop_cast_ptr_to_int = genop_same_as
     genop_cast_int_to_ptr = genop_same_as
 
+    def genop_int_force_ge_zero(self, op, arglocs, resloc):
+        self.mc.TEST(arglocs[0], arglocs[0])
+        self.mov(imm0, resloc)
+        self.mc.CMOVNS(arglocs[0], resloc)
+
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
             self.mc.CDQ()

pypy/jit/backend/x86/regalloc.py

     consider_cast_ptr_to_int = consider_same_as
     consider_cast_int_to_ptr = consider_same_as
 
+    def consider_int_force_ge_zero(self, op):
+        argloc = self.loc(op.getarg(0))
+        resloc = self.force_allocate_reg(op.result, [op.getarg(0)])
+        self.possibly_free_var(op.getarg(0))
+        self.Perform(op, [argloc], resloc)
+
     def consider_strlen(self, op):
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)

pypy/jit/backend/x86/regloc.py

     # Avoid XCHG because it always implies atomic semantics, which is
     # slower and does not pair well for dispatch.
     #XCHG = _binaryop('XCHG')
+    CMOVNS = _binaryop('CMOVNS')
 
     PUSH = _unaryop('PUSH')
     POP = _unaryop('POP')

pypy/jit/backend/x86/rx86.py

     NOT_r = insn(rex_w, '\xF7', register(1), '\xD0')
     NOT_b = insn(rex_w, '\xF7', orbyte(2<<3), stack_bp(1))
 
+    CMOVNS_rr = insn(rex_w, '\x0F\x49', register(2, 8), register(1), '\xC0')
+
     # ------------------------------ Misc stuff ------------------------------
 
     NOP = insn('\x90')

pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py

                 # CALL_j is actually relative, so tricky to test
                 (instrname == 'CALL' and argmodes == 'j') or
                 # SET_ir must be tested manually
-                (instrname == 'SET' and argmodes == 'ir')
+                (instrname == 'SET' and argmodes == 'ir') or
+                # asm gets CMOVNS args the wrong way
+                (instrname.startswith('CMOV'))
         )
 
 

pypy/jit/codewriter/jtransform.py

 
     def do_fixed_newlist(self, op, args, arraydescr):
         v_length = self._get_initial_newlist_length(op, args)
-        return SpaceOperation('new_array', [arraydescr, v_length], op.result)
+        v = Variable('new_length')
+        v.concretetype = lltype.Signed
+        return [SpaceOperation('int_force_ge_zero', [v_length], v),
+            SpaceOperation('new_array', [arraydescr, v], op.result)]
 
     def do_fixed_list_len(self, op, args, arraydescr):
         if args[0] in self.vable_array_vars:     # virtualizable array

pypy/jit/codewriter/test/test_codewriter.py

     assert 'setarrayitem_raw_i' in s
     assert 'getarrayitem_raw_i' in s
     assert 'residual_call_ir_v $<* fn _ll_1_raw_free__arrayPtr>' in s
+
+def test_newlist_negativ():
+    def f(n):
+        l = [0] * n
+        return len(l)
+
+    rtyper = support.annotate(f, [-1])
+    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
+    cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd])
+    cw.find_all_graphs(FakePolicy())
+    cw.make_jitcodes(verbose=True)
+    s = jitdriver_sd.mainjitcode.dump()
+    assert 'int_force_ge_zero' in s
+    assert 'new_array' in s

pypy/jit/metainterp/blackhole.py

     @arguments("i", "i", "i", returns="i")
     def bhimpl_int_between(a, b, c):
         return a <= b < c
+    @arguments("i", returns="i")
+    def bhimpl_int_force_ge_zero(i):
+        if i < 0:
+            return 0
+        return i
 
     @arguments("i", "i", returns="i")
     def bhimpl_uint_lt(a, b):

pypy/jit/metainterp/optimizeopt/virtualize.py

File contents unchanged.

pypy/jit/metainterp/optimizeopt/vstring.py

File contents unchanged.

pypy/jit/metainterp/pyjitpl.py

                     'float_neg', 'float_abs',
                     'cast_ptr_to_int', 'cast_int_to_ptr',
                     'convert_float_bytes_to_longlong',
-                    'convert_longlong_bytes_to_float',
+                    'convert_longlong_bytes_to_float', 'int_force_ge_zero',
                     ]:
         exec py.code.Source('''
             @arguments("box")

pypy/jit/metainterp/resoperation.py

     'INT_IS_TRUE/1b',
     'INT_NEG/1',
     'INT_INVERT/1',
+    'INT_FORCE_GE_ZERO/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1',

pypy/jit/metainterp/resume.py

File contents unchanged.

pypy/jit/metainterp/test/test_dict.py

                            'guard_no_exception': 8, 'new': 2,
                            'guard_false': 2, 'int_is_true': 2})
 
+    def test_unrolling_of_dict_iter(self):
+        driver = JitDriver(greens = [], reds = ['n'])
+        
+        def f(n):
+            while n > 0:
+                driver.jit_merge_point(n=n)
+                d = {1: 1}
+                for elem in d:
+                    n -= elem
+            return n
+
+        res = self.meta_interp(f, [10], listops=True)
+        assert res == 0
+        self.check_simple_loop({'int_sub': 1, 'int_gt': 1, 'guard_true': 1,
+                                'jump': 1})
+
 
 class TestOOtype(DictTests, OOJitMixin):
     pass

pypy/jit/metainterp/test/test_list.py

         self.meta_interp(f, [10], listops=True)
         self.check_resops(new_array=0, call=0)
 
+    def test_list_mul(self):
+        def f(i):
+            l = [0] * i
+            return len(l)
+
+        r = self.interp_operations(f, [3])
+        assert r == 3
+        r = self.interp_operations(f, [-1])
+        assert r == 0
+
 class TestOOtype(ListTests, OOJitMixin):
     pass
 

pypy/jit/metainterp/test/test_tracingopts.py

File contents unchanged.

pypy/jit/tl/pypyjit_demo.py

 import pypyjit
 pypyjit.set_param(threshold=200)
 
+kwargs = {"z": 1}
 
-def g(*args):
-    return len(args)
+def f(*args, **kwargs):
+    result = g(1, *args, **kwargs)
+    return result + 2
 
-def f(n):
-    s = 0
-    for i in range(n):
-        l = [i, n, 2]
-        s += g(*l)
-    return s
+def g(x, y, z=2):
+    return x - y + z
+
+def main():
+    res = 0
+    i = 0
+    while i < 10000:
+        res = f(res, z=i)
+        g(1, res, **kwargs)
+        i += 1
+    return res
+
 
 try:
-    print f(301)
+    print main()
 
 except Exception, e:
     print "Exception: ", type(e)

pypy/jit/tl/spli/autopath.py

File contents unchanged.

pypy/jit/tl/spli/examples.py

File contents unchanged.

pypy/jit/tl/spli/execution.py

File contents unchanged.

pypy/jit/tl/spli/interpreter.py

File contents unchanged.

pypy/jit/tl/spli/objects.py

File contents unchanged.

pypy/jit/tl/spli/pycode.py

File contents unchanged.

pypy/jit/tl/spli/serializer.py

File contents unchanged.

pypy/jit/tl/spli/targetspli.py

File contents unchanged.

pypy/jit/tl/spli/test/test_interpreter.py

File contents unchanged.

pypy/jit/tl/spli/test/test_jit.py

File contents unchanged.

pypy/jit/tl/spli/test/test_serializer.py

File contents unchanged.

pypy/jit/tl/spli/test/test_translated.py

File contents unchanged.

pypy/jit/tl/tlc.py

File contents unchanged.

pypy/module/__builtin__/functional.py

File contents unchanged.

pypy/module/__pypy__/__init__.py

         'do_what_I_mean'            : 'interp_magic.do_what_I_mean',
         'list_strategy'             : 'interp_magic.list_strategy',
         'validate_fd'               : 'interp_magic.validate_fd',
+        'newdict'                   : 'interp_dict.newdict',
+        'dictstrategy'              : 'interp_dict.dictstrategy',
     }
     if sys.platform == 'win32':
         interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'

pypy/module/__pypy__/bytebuffer.py

File contents unchanged.

pypy/module/__pypy__/interp_dict.py

+
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.error import operationerrfmt, OperationError
+from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+
+@unwrap_spec(type=str)
+def newdict(space, type):
+    if type == 'module':
+        return space.newdict(module=True)
+    elif type == 'instance':
+        return space.newdict(instance=True)
+    elif type == 'kwargs':
+        return space.newdict(kwargs=True)
+    elif type == 'strdict':
+        return space.newdict(strdict=True)
+    else:
+        raise operationerrfmt(space.w_TypeError, "unknown type of dict %s",
+                              type)
+
+def dictstrategy(space, w_obj):
+    if not isinstance(w_obj, W_DictMultiObject):
+        raise OperationError(space.w_TypeError,
+                             space.wrap("expecting dict object"))
+    return space.wrap('%r' % (w_obj.strategy,))

pypy/module/_io/interp_bufferedio.py

File contents unchanged.

pypy/module/_io/interp_bytesio.py

File contents unchanged.

pypy/module/_io/interp_stringio.py

File contents unchanged.

pypy/module/_random/interp_random.py

File contents unchanged.

pypy/module/_sre/interp_sre.py

File contents unchanged.

pypy/module/cpyext/listobject.py

File contents unchanged.

pypy/module/cpyext/tupleobject.py

File contents unchanged.

pypy/module/itertools/interp_itertools.py

File contents unchanged.

pypy/module/micronumpy/interp_iter.py

File contents unchanged.

pypy/module/micronumpy/interp_numarray.py

File contents unchanged.

pypy/module/micronumpy/strides.py

File contents unchanged.

pypy/module/pypyjit/test_pypy_c/test_call.py

 import py
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+from pypy.module.pypyjit.test_pypy_c.model import OpMatcher
 
 class TestCall(BaseTestPyPyC):
 
             setfield_gc(p26, ConstPtr(ptr22), descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_strategy .*>)
             setarrayitem_gc(p24, 0, p26, descr=<ArrayP .>)
             setfield_gc(p22, p24, descr=<FieldP .*Arguments.inst_arguments_w .*>)
+            setfield_gc(p22, 1, descr=<FieldU .*Arguments.inst__jit_few_keywords .*>)
             p32 = call_may_force(11376960, p18, p22, descr=<Callr . rr EF=6>)
             ...
         """)
             return res""", [1000])
         assert log.result == 500
         loop, = log.loops_by_id('call')
-        print loop.ops_by_id('call')
         assert loop.match("""
             i65 = int_lt(i58, i29)
             guard_true(i65, descr=...)
             jump(..., descr=...)
         """)
 
+    def test_kwargs_virtual3(self):
+        log = self.run("""
+        def f(a, b, c):
+            pass
+
+        def main(stop):
+            i = 0
+            while i < stop:
+                d = {'a': 2, 'b': 3, 'c': 4}
+                f(**d) # ID: call
+                i += 1
+            return 13
+        """, [1000])
+        assert log.result == 13
+        loop, = log.loops_by_id('call')
+        allops = loop.allops()
+        calls = [op for op in allops if op.name.startswith('call')]
+        assert len(calls) == 0
+        assert len([op for op in allops if op.name.startswith('new')]) == 0
+
+    def test_kwargs_non_virtual(self):
+        log = self.run("""
+        def f(a, b, c):
+            pass
+
+        def main(stop):
+            d = {'a': 2, 'b': 3, 'c': 4}
+            i = 0
+            while i < stop:
+                f(**d) # ID: call
+                i += 1
+            return 13
+        """, [1000])
+        assert log.result == 13
+        loop, = log.loops_by_id('call')
+        allops = loop.allops()
+        calls = [op for op in allops if op.name.startswith('call')]
+        assert OpMatcher(calls).match('''
+        p93 = call(ConstClass(view_as_kwargs), p35, p12, descr=<.*>)
+        i103 = call(ConstClass(_match_keywords), ConstPtr(ptr52), 0, 0, p94, p98, 0, descr=<.*>)
+        ''')
+        assert len([op for op in allops if op.name.startswith('new')]) == 1
+        # 1 alloc
+
+    def test_complex_case(self):
+        log = self.run("""
+        def f(x, y, a, b, c=3, d=4):
+            pass
+
+        def main(stop):
+            i = 0
+            while i < stop:
+                a = [1, 2]
+                d = {'a': 2, 'b': 3, 'd':4}
+                f(*a, **d) # ID: call
+                i += 1
+            return 13        
+        """, [1000])
+        loop, = log.loops_by_id('call')
+        assert loop.match_by_id('call', '''
+        guard_not_invalidated(descr=<.*>)
+        i1 = force_token()
+        ''')
+
+    def test_complex_case_global(self):
+        log = self.run("""
+        def f(x, y, a, b, c=3, d=4):
+            pass
+
+        a = [1, 2]
+        d = {'a': 2, 'b': 3, 'd':4}
+
+        def main(stop):
+            i = 0
+            while i < stop:
+                f(*a, **d) # ID: call
+                i += 1
+            return 13        
+        """, [1000])
+
+    def test_complex_case_loopconst(self):
+        log = self.run("""
+        def f(x, y, a, b, c=3, d=4):
+            pass
+
+        def main(stop):
+            i = 0
+            a = [1, 2]
+            d = {'a': 2, 'b': 3, 'd':4}
+            while i < stop:
+                f(*a, **d) # ID: call
+                i += 1
+            return 13        
+        """, [1000])

pypy/module/unicodedata/interp_ucd.py

File contents unchanged.

pypy/objspace/std/bytearrayobject.py

File contents unchanged.

pypy/objspace/std/dictmultiobject.py

 from pypy.rlib.debug import mark_dict_non_null
 
 from pypy.rlib import rerased
+from pypy.rlib import jit
 
 def _is_str(space, w_key):
     return space.is_w(space.type(w_key), space.w_str)
             space.is_w(w_lookup_type, space.w_float)
             )
 
+
+DICT_CUTOFF = 5
+
+@specialize.call_location()
+def w_dict_unrolling_heuristic(w_dct):
+    """ In which cases iterating over dict items can be unrolled.
+    Note that w_dct is an instance of W_DictMultiObject, not necesarilly
+    an actual dict
+    """
+    return jit.isvirtual(w_dct) or (jit.isconstant(w_dct) and
+                                    w_dct.length() <= DICT_CUTOFF)
+
 class W_DictMultiObject(W_Object):
     from pypy.objspace.std.dicttype import dict_typedef as typedef
 
 
         elif kwargs:
             assert w_type is None
-            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
-            strategy = space.fromcache(KwargsDictStrategy)
+            from pypy.objspace.std.kwargsdict import EmptyKwargsDictStrategy
+            strategy = space.fromcache(EmptyKwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
         if w_type is None:
         for w_k, w_v in list_pairs_w:
             w_self.setitem(w_k, w_v)
 
+    def view_as_kwargs(self):
+        return self.strategy.view_as_kwargs(self)
+
 def _add_indirections():
     dict_methods = "setitem setitem_str getitem \
                     getitem_str delitem length \
                     clear w_keys values \
                     items iter setdefault \
-                    popitem listview_str listview_int \
-                    view_as_kwargs".split()
+                    popitem listview_str listview_int".split()
 
     def make_method(method):
         def f(self, *args):
     def w_keys(self, w_dict):
         return self.space.newlist_str(self.listview_str(w_dict))
 
+    @jit.look_inside_iff(lambda self, w_dict:
+                         w_dict_unrolling_heuristic(w_dict))
+    def view_as_kwargs(self, w_dict):
+        d = self.unerase(w_dict.dstorage)
+        l = len(d)
+        keys, values = [None] * l, [None] * l
+        i = 0
+        for key, val in d.iteritems():
+            keys[i] = key
+            values[i] = val
+            i += 1
+        return keys, values
 
 class _WrappedIteratorMixin(object):
     _mixin_ = True

pypy/objspace/std/kwargsdict.py

 
 from pypy.rlib import rerased, jit
 from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               EmptyDictStrategy,
                                                IteratorImplementation,
                                                ObjectDictStrategy,
                                                StringDictStrategy)
 
 
+class EmptyKwargsDictStrategy(EmptyDictStrategy):
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(KwargsDictStrategy)
+        storage = strategy.get_empty_storage()
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+
 class KwargsDictStrategy(DictStrategy):
     erase, unerase = rerased.new_erasing_pair("kwargsdict")
     erase = staticmethod(erase)
         w_dict.dstorage = storage
 
     def view_as_kwargs(self, w_dict):
-        return self.unerase(w_dict.dstorage)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        return keys[:], values_w[:] # copy to make non-resizable
 
 
 class KwargsDictIterator(IteratorImplementation):

pypy/objspace/std/listobject.py

File contents unchanged.

pypy/objspace/std/specialisedtupleobject.py

File contents unchanged.

pypy/objspace/std/test/test_dictmultiobject.py

         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance)
 
+    def view_as_kwargs(self, w_d):
+        return w_d.view_as_kwargs() # assume it's a multidict
+
     def finditem_str(self, w_dict, s):
         return w_dict.getitem_str(s) # assume it's a multidict
 
         assert self.impl.getitem(s) == 1000
         assert s.unwrapped
 
+    def test_view_as_kwargs(self):
+        self.fill_impl()
+        assert self.fakespace.view_as_kwargs(self.impl) == (["fish", "fish2"], [1000, 2000])
+
 ## class TestMeasuringDictImplementation(BaseTestRDictImplementation):
 ##     ImplementionClass = MeasuringDictImplementation
 ##     DevolvedClass = MeasuringDictImplementation

pypy/objspace/std/test/test_kwargsdict.py

     d = W_DictMultiObject(space, strategy, storage)
     w_l = d.w_keys() # does not crash
 
+def test_view_as_kwargs():
+    from pypy.objspace.std.dictmultiobject import EmptyDictStrategy
+    strategy = KwargsDictStrategy(space)
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert (space.view_as_kwargs(d) == keys, values)
+
+    strategy = EmptyDictStrategy(space)
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    assert (space.view_as_kwargs(d) == [], [])
+
+def test_from_empty_to_kwargs():
+    strategy = EmptyKwargsDictStrategy(space)
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    d.setitem_str("a", 3)
+    assert isinstance(d.strategy, KwargsDictStrategy)
+
 
 from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
 def get_impl(self):
             return args
         d = f(a=1)
         assert "KwargsDictStrategy" in self.get_strategy(d)
+        d = f()
+        assert "EmptyKwargsDictStrategy" in self.get_strategy(d)
 

pypy/objspace/std/tupleobject.py

File contents unchanged.

pypy/objspace/std/unicodeobject.py

File contents unchanged.
             thing._annspecialcase_ = "specialize:call_location"
 
         args = _get_args(func)
+        predicateargs = _get_args(predicate)
+        assert len(args) == len(predicateargs), "%s and predicate %s need the same numbers of arguments" % (func, predicate)
         d = {
             "dont_look_inside": dont_look_inside,
             "predicate": predicate,

pypy/rlib/rbigint.py

File contents unchanged.

pypy/rlib/test/test_runicode.py

File contents unchanged.

pypy/rpython/lltypesystem/rdict.py

 
 def _make_ll_dictnext(kind):
     # make three versions of the following function: keys, values, items
+    @jit.look_inside_iff(lambda RETURNTYPE, iter: jit.isvirtual(iter)
+                         and (iter.dict is None or
+                              jit.isvirtual(iter.dict)))
+    @jit.oopspec("dictiter.next%s(iter)" % kind)
     def ll_dictnext(RETURNTYPE, iter):
         # note that RETURNTYPE is None for keys and values
         dict = iter.dict
             # clear the reference to the dict and prevent restarts
             iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
         raise StopIteration
-    ll_dictnext.oopspec = 'dictiter.next%s(iter)' % kind
     return ll_dictnext
 
 ll_dictnext_group = {'keys'  : _make_ll_dictnext('keys'),

pypy/rpython/memory/test/test_gc.py

File contents unchanged.

pypy/rpython/rlist.py

File contents unchanged.

pypy/rpython/test/test_llinterp.py

File contents unchanged.

pypy/rpython/test/test_rlist.py

File contents unchanged.

pypy/rpython/test/test_rstr.py

File contents unchanged.

pypy/translator/backendopt/test/test_writeanalyze.py

File contents unchanged.