Commits

Armin Rigo committed 7273161

(arigo, fijal)

Merge branch/stringbuilder2:

Simplifies a lot the implementation of stringbuilder,
getting rid of various realloc_* and resizable_buffer_*
operations. Replaces it with just one operation:
'shrink_array'.

  • Participants
  • Parent commits f0986e9

Comments (0)

Files changed (26)

File pypy/rlib/rgc.py

         hop.exception_cannot_occur()
         return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype)
 
-def resizable_buffer_of_shape(T, init_size):
-    """ Pre-allocates structure of type T (varsized) with possibility
-    to reallocate it further by resize_buffer.
-    """
-    from pypy.rpython.lltypesystem import lltype
-    return lltype.malloc(T, init_size)
-
-class ResizableBufferOfShapeEntry(ExtRegistryEntry):
-    _about_ = resizable_buffer_of_shape
-
-    def compute_result_annotation(self, s_T, s_init_size):
-        from pypy.annotation import model as annmodel
-        from pypy.rpython.lltypesystem import rffi, lltype
-        assert s_T.is_constant()
-        assert isinstance(s_init_size, annmodel.SomeInteger)
-        T = s_T.const
-        # limit ourselves to structs and to a fact that var-sized element
-        # does not contain pointers.
-        assert isinstance(T, lltype.Struct)
-        assert isinstance(getattr(T, T._arrayfld).OF, lltype.Primitive)
-        return annmodel.SomePtr(lltype.Ptr(T))
-
-    def specialize_call(self, hop):
-        from pypy.rpython.lltypesystem import lltype
-        flags = {'flavor': 'gc'}
-        vlist = [hop.inputarg(lltype.Void, 0),
-                 hop.inputconst(lltype.Void, flags),
-                 hop.inputarg(lltype.Signed, 1)]
-        hop.exception_is_here()
-        return hop.genop('malloc_resizable_buffer', vlist,
-                         resulttype=hop.r_result.lowleveltype)
-
-def resize_buffer(ptr, old_size, new_size):
-    """ Resize raw buffer returned by resizable_buffer_of_shape to new size
-    """
-    from pypy.rpython.lltypesystem import lltype
-    T = lltype.typeOf(ptr).TO
-    arrayfld = T._arrayfld
-    arr = getattr(ptr, arrayfld)
-    # we don't have any realloc on top of cpython
-    new_ptr = lltype.malloc(T, new_size)
-    new_ar = getattr(new_ptr, arrayfld)
-    for i in range(old_size):
-        new_ar[i] = arr[i]
-    return new_ptr
-
-class ResizeBufferEntry(ExtRegistryEntry):
-    _about_ = resize_buffer
-
-    def compute_result_annotation(self, s_ptr, s_old_size, s_new_size):
-        from pypy.annotation import model as annmodel
-        from pypy.rpython.lltypesystem import rffi
-        assert isinstance(s_ptr, annmodel.SomePtr)
-        assert isinstance(s_new_size, annmodel.SomeInteger)
-        assert isinstance(s_old_size, annmodel.SomeInteger)
-        return s_ptr
-
-    def specialize_call(self, hop):
-        from pypy.rpython.lltypesystem import lltype
-        vlist = [hop.inputarg(hop.args_r[0], 0),
-                 hop.inputarg(lltype.Signed, 1),
-                 hop.inputarg(lltype.Signed, 2)]
-        hop.exception_is_here()
-        return hop.genop('resize_buffer', vlist,
-                         resulttype=hop.r_result.lowleveltype)
-
-def finish_building_buffer(ptr, final_size):
-    """ Finish building resizable buffer returned by resizable_buffer_of_shape
-    """
-    return ptr
-
-class FinishBuildingBufferEntry(ExtRegistryEntry):
-    _about_ = finish_building_buffer
-
-    def compute_result_annotation(self, s_arr, s_final_size):
-        from pypy.annotation.model import SomePtr, s_ImpossibleValue,\
-             SomeInteger
-        assert isinstance(s_arr, SomePtr)
-        assert isinstance(s_final_size, SomeInteger)
-        return s_arr
-
-    def specialize_call(self, hop):
-        from pypy.rpython.lltypesystem import lltype
-        vlist = [hop.inputarg(hop.args_r[0], 0),
-                 hop.inputarg(hop.args_r[1], 1)]
-        hop.exception_cannot_occur()
-        return hop.genop('finish_building_buffer', vlist,
-                         resulttype=hop.r_result.lowleveltype)
-
 def ll_arraycopy(source, dest, source_start, dest_start, length):
     from pypy.rpython.lltypesystem.lloperation import llop
     from pypy.rpython.lltypesystem import lltype, llmemory
 ll_arraycopy._annspecialcase_ = 'specialize:ll'
 ll_arraycopy._jit_look_inside_ = False
 
+def ll_shrink_array(p, smallerlength):
+    from pypy.rpython.lltypesystem.lloperation import llop
+    from pypy.rpython.lltypesystem import lltype, llmemory
+    from pypy.rlib.objectmodel import keepalive_until_here
+
+    if llop.shrink_array(lltype.Bool, p, smallerlength):
+        return p    # done by the GC
+    # XXX we assume for now that the type of p is GcStruct containing a
+    # variable array, with no further pointers anywhere, and exactly one
+    # field in the fixed part -- like STR and UNICODE.
+
+    TP = lltype.typeOf(p).TO
+    newp = lltype.malloc(TP, smallerlength)
+
+    assert len(TP._names) == 2
+    field = getattr(p, TP._names[0])
+    setattr(newp, TP._names[0], field)
+
+    ARRAY = getattr(TP, TP._arrayfld)
+    offset = (llmemory.offsetof(TP, TP._arrayfld) +
+              llmemory.itemoffsetof(ARRAY, 0))
+    source_addr = llmemory.cast_ptr_to_adr(p)    + offset
+    dest_addr   = llmemory.cast_ptr_to_adr(newp) + offset
+    llmemory.raw_memcopy(source_addr, dest_addr, 
+                         llmemory.sizeof(ARRAY.OF) * smallerlength)
+
+    keepalive_until_here(p)
+    keepalive_until_here(newp)
+    return newp
+ll_shrink_array._annspecialcase_ = 'specialize:ll'
+ll_shrink_array._jit_look_inside_ = False
+
+
 def no_collect(func):
     func._dont_inline_ = True
     func._gc_no_collect_ = True

File pypy/rlib/test/test_rgc.py

     
     assert res == True
     
-def test_resizable_buffer():
-    from pypy.rpython.lltypesystem.rstr import STR
-    from pypy.rpython.annlowlevel import hlstr
-    
-    def f():
-        ptr = rgc.resizable_buffer_of_shape(STR, 1)
-        ptr.chars[0] = 'a'
-        ptr = rgc.resize_buffer(ptr, 1, 2)
-        ptr.chars[1] = 'b'
-        return hlstr(rgc.finish_building_buffer(ptr, 2))
-
-    assert f() == 'ab'
-
 def test_ll_arraycopy_1():
     TYPE = lltype.GcArray(lltype.Signed)
     a1 = lltype.malloc(TYPE, 10)
             assert a2[i] == a1[i+2]
         else:
             assert a2[i] == org2[i]
+
+def test_ll_shrink_array_1():
+    py.test.skip("implement ll_shrink_array for GcStructs or GcArrays that "
+                 "don't have the shape of STR or UNICODE")
+
+def test_ll_shrink_array_2():
+    S = lltype.GcStruct('S', ('x', lltype.Signed),
+                             ('vars', lltype.Array(lltype.Signed)))
+    s1 = lltype.malloc(S, 5)
+    s1.x = 1234
+    for i in range(5):
+        s1.vars[i] = 50 + i
+    s2 = rgc.ll_shrink_array(s1, 3)
+    assert lltype.typeOf(s2) == lltype.Ptr(S)
+    assert s2.x == 1234
+    assert len(s2.vars) == 3
+    for i in range(3):
+        assert s2.vars[i] == 50 + i

File pypy/rpython/llinterp.py

         except MemoryError:
             self.make_llexception()
             
-    def op_malloc_nonmovable(self, obj, flags):
+    def op_malloc_nonmovable(self, TYPE, flags):
         flavor = flags['flavor']
         assert flavor == 'gc'
         zero = flags.get('zero', False)
-        return self.heap.malloc_nonmovable(obj, zero=zero)
+        return self.heap.malloc_nonmovable(TYPE, zero=zero)
         
-    def op_malloc_nonmovable_varsize(self, obj, flags, size):
+    def op_malloc_nonmovable_varsize(self, TYPE, flags, size):
         flavor = flags['flavor']
         assert flavor == 'gc'
         zero = flags.get('zero', False)
-        return self.heap.malloc_nonmovable(obj, size, zero=zero)
-
-    def op_malloc_resizable_buffer(self, obj, flags, size):
-        return self.heap.malloc_resizable_buffer(obj, size)
-
-    def op_resize_buffer(self, obj, old_size, new_size):
-        return self.heap.resize_buffer(obj, old_size, new_size)
-
-    def op_finish_building_buffer(self, obj, size):
-        return self.heap.finish_building_buffer(obj, size)
+        return self.heap.malloc_nonmovable(TYPE, size, zero=zero)
 
     def op_free(self, obj, flavor):
         assert isinstance(flavor, str)
             self.llinterpreter.remember_free(obj)
         self.heap.free(obj, flavor=flavor)
 
+    def op_shrink_array(self, obj, smallersize):
+        return self.heap.shrink_array(obj, smallersize)
+
     def op_zero_gc_pointers_inside(self, obj):
         raise NotImplementedError("zero_gc_pointers_inside")
 
         assert lltype.typeOf(size) == lltype.Signed
         return llmemory.raw_malloc(size)
 
-    def op_raw_realloc_grow(self, addr, old_size, size):
-        assert lltype.typeOf(size) == lltype.Signed
-        return llmemory.raw_realloc_grow(addr, old_size, size)
-
-    def op_raw_realloc_shrink(self, addr, old_size, size):
-        assert lltype.typeOf(size) == lltype.Signed
-        return llmemory.raw_realloc_shrink(addr, old_size, size)
-
     op_boehm_malloc = op_boehm_malloc_atomic = op_raw_malloc
 
     def op_boehm_register_finalizer(self, p, finalizer):

File pypy/rpython/lltypesystem/llarena.py

         Arena.object_arena_location[container] = self, offset
         Arena.old_object_arena_location[container] = self, offset
 
+    def shrink_obj(self, offset, newsize):
+        oldbytes = self.objectsizes[offset]
+        newbytes = llmemory.raw_malloc_usage(newsize)
+        assert newbytes <= oldbytes
+        # fix self.objectsizes
+        for i in range(newbytes):
+            adr = offset + i
+            if adr in self.objectsizes:
+                assert self.objectsizes[adr] == oldbytes - i
+                self.objectsizes[adr] = newbytes - i
+        # fix self.usagemap
+        for i in range(offset + newbytes, offset + oldbytes):
+            assert self.usagemap[i] == 'x'
+            self.usagemap[i] = '#'
+
 class fakearenaaddress(llmemory.fakeaddress):
 
     def __init__(self, arena, offset):
                          % (addr.offset,))
     addr.arena.allocate_object(addr.offset, size)
 
+def arena_shrink_obj(addr, newsize):
+    """ Mark object as shorter than it was
+    """
+    addr = _getfakearenaaddress(addr)
+    addr.arena.shrink_obj(addr.offset, newsize)
+
 def round_up_for_allocation(size, minsize=0):
     """Round up the size in order to preserve alignment of objects
     following an object.  For arenas containing heterogenous objects.
                   llfakeimpl=arena_reserve,
                   sandboxsafe=True)
 
+def llimpl_arena_shrink_obj(addr, newsize):
+    pass
+register_external(arena_shrink_obj, [llmemory.Address, int], None,
+                  'll_arena.arena_shrink_obj',
+                  llimpl=llimpl_arena_shrink_obj,
+                  llfakeimpl=arena_shrink_obj,
+                  sandboxsafe=True)
+
 llimpl_round_up_for_allocation = rffi.llexternal('ROUND_UP_FOR_ALLOCATION',
                                                 [lltype.Signed, lltype.Signed],
                                                  lltype.Signed,

File pypy/rpython/lltypesystem/llheap.py

 
 malloc_nonmovable = malloc
 
-def malloc_resizable_buffer(TP, size):
-    return malloc(TP, size)
+def shrink_array(p, smallersize):
+    return False
 
-def resize_buffer(buf, old_size, new_size):
-    ll_str = malloc(typeOf(buf).TO, new_size)
-    for i in range(old_size):
-        ll_str.chars[i] = buf.chars[i]
-    return ll_str
-
-def finish_building_buffer(buf, final_size):
-    ll_str = malloc(typeOf(buf).TO, final_size)
-    for i in range(final_size):
-        ll_str.chars[i] = buf.chars[i]
-    return ll_str
 
 def thread_prepare():
     pass

File pypy/rpython/lltypesystem/llmemory.py

         raise NotImplementedError(size)
     return size._raw_malloc([], zero=False)
 
-def raw_realloc_grow(addr, old_size, size):
-    new_area = size._raw_malloc([], zero=False)
-    raw_memcopy(addr, new_area, old_size)
-    raw_free(addr)
-    return new_area
-
-def raw_realloc_shrink(addr, old_size, size):
-    new_area = size._raw_malloc([], zero=False)
-    raw_memcopy(addr, new_area, size)
-    raw_free(addr)
-    return new_area
-
 def raw_free(adr):
     # try to free the whole object if 'adr' is the address of the header
     from pypy.rpython.memory.gcheader import GCHeaderBuilder

File pypy/rpython/lltypesystem/lloperation.py

     'malloc_varsize':       LLOp(canraise=(MemoryError,), canunwindgc=True),
     'malloc_nonmovable':    LLOp(canraise=(MemoryError,), canunwindgc=True),
     'malloc_nonmovable_varsize':LLOp(canraise=(MemoryError,),canunwindgc=True),
-    'malloc_resizable_buffer': LLOp(canraise=(MemoryError,),canunwindgc=True),
-    'resize_buffer':        LLOp(canraise=(MemoryError,), canunwindgc=True),
-    'finish_building_buffer' : LLOp(canraise=(MemoryError,), canunwindgc=True),
+    'shrink_array':         LLOp(canrun=True),
     'zero_gc_pointers_inside': LLOp(),
     'free':                 LLOp(),
     'getfield':             LLOp(sideeffects=False, canrun=True),
     'boehm_register_finalizer': LLOp(),
     'boehm_disappearing_link': LLOp(),
     'raw_malloc':           LLOp(),
-    'raw_realloc_grow':     LLOp(),
-    'raw_realloc_shrink':   LLOp(),
     'raw_malloc_usage':     LLOp(sideeffects=False),
     'raw_free':             LLOp(),
     'raw_memclear':         LLOp(),

File pypy/rpython/lltypesystem/lltype.py

     def getlength(self):
         return len(self.items)
 
+    def shrinklength(self, newlength):
+        del self.items[newlength:]
+
     def getbounds(self):
         stop = len(self.items)
         return 0, stop
     def setitem(self, index, value):
         assert index == 0
         if value != self.array.getlength():
-            raise Exception("can't change the length of an array")
+            if value > self.array.getlength():
+                raise Exception("can't grow an array in-place")
+            self.array.shrinklength(value)
 
     def _makeptr(array, solid=False):
         try:

File pypy/rpython/lltypesystem/opimpl.py

 def op_gc_assume_young_pointers(addr):
     pass
 
+def op_shrink_array(array, smallersize):
+    return False
+
 # ____________________________________________________________
 
 def get_op_impl(opname):

File pypy/rpython/lltypesystem/rbuilder.py

 
 from pypy.rpython.rbuilder import AbstractStringBuilderRepr
-from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.lltypesystem import lltype, rstr
 from pypy.rpython.lltypesystem.rstr import STR, UNICODE, char_repr,\
      string_repr, unichar_repr, unicode_repr
 from pypy.rpython.annlowlevel import llstr
 from pypy.rpython.lltypesystem.lltype import staticAdtMethod
 from pypy.tool.sourcetools import func_with_new_name
 
+# Think about heuristics below, maybe we can come up with something
+# better or at least compare it with list heuristics
+
 GROW_FAST_UNTIL = 100*1024*1024      # 100 MB
 
-def new_grow_func(name):
+def new_grow_func(name, mallocfn, copycontentsfn):
     def stringbuilder_grow(ll_builder, needed):
         allocated = ll_builder.allocated
-        if allocated < GROW_FAST_UNTIL:
-            new_allocated = allocated << 1
-        else:
-            extra_size = allocated >> 2
-            try:
-                new_allocated = ovfcheck(allocated + extra_size)
-            except OverflowError:
-                raise MemoryError
+        #if allocated < GROW_FAST_UNTIL:
+        #    new_allocated = allocated << 1
+        #else:
+        extra_size = allocated >> 2
         try:
+            new_allocated = ovfcheck(allocated + extra_size)
             new_allocated = ovfcheck(new_allocated + needed)
         except OverflowError:
             raise MemoryError
-        ll_builder.buf = rgc.resize_buffer(ll_builder.buf, ll_builder.used,
-                                           new_allocated)
+        newbuf = mallocfn(new_allocated)
+        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+        ll_builder.buf = newbuf
         ll_builder.allocated = new_allocated
     return func_with_new_name(stringbuilder_grow, name)
 
-stringbuilder_grow = new_grow_func('stringbuilder_grow')
-unicodebuilder_grow = new_grow_func('unicodebuilder_grow')
+stringbuilder_grow = new_grow_func('stringbuilder_grow', rstr.mallocstr,
+                                   rstr.copy_string_contents)
+unicodebuilder_grow = new_grow_func('unicodebuilder_grow', rstr.mallocunicode,
+                                    rstr.copy_unicode_contents)
 
 STRINGBUILDER = lltype.GcStruct('stringbuilder',
-                              ('allocated', lltype.Signed),
-                              ('used', lltype.Signed),
-                              ('buf', lltype.Ptr(STR)),
-                                adtmeths={'grow':staticAdtMethod(stringbuilder_grow)})
+                               ('allocated', lltype.Signed),
+                               ('used', lltype.Signed),
+                               ('buf', lltype.Ptr(STR)),
+                               adtmeths={'grow':staticAdtMethod(stringbuilder_grow)})
 
 UNICODEBUILDER = lltype.GcStruct('unicodebuilder',
                                  ('allocated', lltype.Signed),
                                  ('used', lltype.Signed),
                                  ('buf', lltype.Ptr(UNICODE)),
-                                 adtmeths={'grow':staticAdtMethod(unicodebuilder_grow)})
+                              adtmeths={'grow':staticAdtMethod(unicodebuilder_grow)})
 
 MAX = 16*1024*1024
 
         ll_builder = lltype.malloc(cls.lowleveltype.TO)
         ll_builder.allocated = init_size
         ll_builder.used = 0
-        ll_builder.buf = rgc.resizable_buffer_of_shape(cls.basetp, init_size)
+        ll_builder.buf = cls.mallocfn(init_size)
         return ll_builder
 
     @staticmethod
         used = ll_builder.used
         lgt = len(ll_str.chars)
         needed = lgt + used
-        if needed >= ll_builder.allocated:
+        if needed > ll_builder.allocated:
             ll_builder.grow(ll_builder, lgt)
         ll_str.copy_contents(ll_str, ll_builder.buf, 0, used, lgt)
         ll_builder.used = needed
     def ll_append_slice(ll_builder, ll_str, start, end):
         needed = end - start
         used = ll_builder.used
-        if needed + used >= ll_builder.allocated:
+        if needed + used > ll_builder.allocated:
             ll_builder.grow(ll_builder, needed)
         assert needed >= 0
         ll_str.copy_contents(ll_str, ll_builder.buf, start, used, needed)
     @staticmethod
     def ll_append_multiple_char(ll_builder, char, times):
         used = ll_builder.used
-        if times + used >= ll_builder.allocated:
+        if times + used > ll_builder.allocated:
             ll_builder.grow(ll_builder, times)
         for i in range(times):
             ll_builder.buf.chars[used] = char
     @staticmethod
     def ll_build(ll_builder):
         final_size = ll_builder.used
-        return rgc.finish_building_buffer(ll_builder.buf, final_size)
+        assert final_size >= 0
+        if final_size == ll_builder.allocated:
+            return ll_builder.buf
+        return rgc.ll_shrink_array(ll_builder.buf, final_size)
 
 class StringBuilderRepr(BaseStringBuilderRepr):
     lowleveltype = lltype.Ptr(STRINGBUILDER)
     basetp = STR
+    mallocfn = staticmethod(rstr.mallocstr)
     string_repr = string_repr
     char_repr = char_repr
 
 class UnicodeBuilderRepr(BaseStringBuilderRepr):
     lowleveltype = lltype.Ptr(UNICODEBUILDER)
     basetp = UNICODE
+    mallocfn = staticmethod(rstr.mallocunicode)
     string_repr = unicode_repr
     char_repr = unichar_repr
 

File pypy/rpython/lltypesystem/test/test_llarena.py

 from pypy.rpython.lltypesystem.llarena import arena_reserve, arena_free
 from pypy.rpython.lltypesystem.llarena import round_up_for_allocation
 from pypy.rpython.lltypesystem.llarena import ArenaError, arena_new_view
+from pypy.rpython.lltypesystem.llarena import arena_shrink_obj
 
 def test_arena():
     S = lltype.Struct('S', ('x',lltype.Signed))
     fn = compile(test_look_inside_object, [])
     res = fn()
     assert res == 42
+
+def test_shrink_obj():
+    from pypy.rpython.memory.gcheader import GCHeaderBuilder
+    HDR = lltype.Struct('HDR', ('h', lltype.Signed))
+    gcheaderbuilder = GCHeaderBuilder(HDR)
+    size_gc_header = gcheaderbuilder.size_gc_header
+    S = lltype.GcStruct('S', ('x', lltype.Signed),
+                             ('a', lltype.Array(lltype.Signed)))
+    myarenasize = 200
+    a = arena_malloc(myarenasize, False)
+    arena_reserve(a, size_gc_header + llmemory.sizeof(S, 10))
+    arena_shrink_obj(a, size_gc_header + llmemory.sizeof(S, 5))
+    arena_reset(a, size_gc_header + llmemory.sizeof(S, 5), False)

File pypy/rpython/lltypesystem/test/test_llmemory.py

     # the following line crashes if the array is dead
     ptr1 = cast_adr_to_ptr(adr, lltype.Ptr(lltype.FixedSizeArray(Address, 1)))
     ptr1[0] = NULL
-
-def test_realloc():
-    A = lltype.Array(lltype.Float)
-    adr = raw_malloc(sizeof(A, 10))
-    ptr = cast_adr_to_ptr(adr, lltype.Ptr(A))
-    for i in range(10):
-        ptr[i] = float(i)
-    adr2 = raw_realloc_shrink(adr, sizeof(A, 10), sizeof(A, 5))
-    ptr2 = cast_adr_to_ptr(adr2, lltype.Ptr(A))
-    assert len(ptr2) == 5
-    assert ptr2[3] == 3.0
-    assert ptr2[1] == 1.0
-
-def test_realloc_struct():
-    S = lltype.Struct('x', ('one', lltype.Signed),
-                      ('a', lltype.Array(lltype.Float)))
-    adr = raw_malloc(sizeof(S, 5))
-    ptr = cast_adr_to_ptr(adr, lltype.Ptr(S))
-    for i in range(5):
-        ptr.a[i] = float(i)
-    ptr.one = 3
-    adr2 = raw_realloc_grow(adr, sizeof(S, 5), sizeof(S, 10))
-    ptr2 = cast_adr_to_ptr(adr2, lltype.Ptr(S))
-    assert len(ptr2.a) == 10
-    assert ptr2.a[3] == 3.0
-    assert ptr2.a[0] == 0.0
-    assert ptr2.one == 3
-    

File pypy/rpython/memory/gc/base.py

     needs_write_barrier = False
     malloc_zero_filled = False
     prebuilt_gc_objects_are_static_roots = True
-    can_realloc = False
     object_minimal_size = 0
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE):

File pypy/rpython/memory/gc/hybrid.py

 # Object lists:
 #   * gen2_rawmalloced_objects
 #   * gen3_rawmalloced_objects
-#   * gen2_resizable_objects
 #   * old_objects_pointing_to_young: gen2or3 objs that point to gen1 objs
 #   * last_generation_root_objects: gen3 objs that point to gen1or2 objs
 #
 # Some invariants:
 #   * gen3 are either GCFLAG_NO_HEAP_PTRS or in 'last_generation_root_objects'
 #   * between collections, GCFLAG_UNVISITED set exactly for gen2_rawmalloced
-#   * objects in gen2_resizable_objects are part of the generation 2 but never
-#     explicitly listed in gen2_rawmalloced_objects.
 #
 # A malloc_varsize() of large objects returns objects that are external
 # but initially of generation 2.  Old objects from the semispaces are
 # moved to external objects directly as generation 3.
 
-# gen2_resizable_objects is for objects that are resizable
-
 # The "age" of an object is the number of times it survived a full
 # collections, without counting the step that moved it out of the nursery.
 # When a semispace-based object would grow older than MAX_SEMISPACE_AGE,
     """
     first_unused_gcflag = _gcflag_next_bit
     prebuilt_gc_objects_are_static_roots = True
-    can_realloc = False
 
     # the following values override the default arguments of __init__ when
     # translating to a real backend.
 
         self.gen2_rawmalloced_objects = self.AddressStack()
         self.gen3_rawmalloced_objects = self.AddressStack()
-        self.gen2_resizable_objects = self.AddressStack()
         GenerationGC.setup(self)
 
     def set_max_heap_size(self, size):
                                                 llmemory.GCREF)
         return self.malloc_varsize_slowpath(typeid, length)
 
-    def malloc_varsize_slowpath(self, typeid, length, force_nonmovable=False,
-                                resizable=False):
+    def malloc_varsize_slowpath(self, typeid, length, force_nonmovable=False):
         # For objects that are too large, or when the nursery is exhausted.
         # In order to keep malloc_varsize_clear() as compact as possible,
         # we recompute what we need in this slow path instead of passing
         else:
             nonlarge_max = self.nonlarge_max
         if force_nonmovable or raw_malloc_usage(totalsize) > nonlarge_max:
-            result = self.malloc_varsize_marknsweep(totalsize, resizable)
+            result = self.malloc_varsize_marknsweep(totalsize)
             flags = self.GCFLAGS_FOR_NEW_EXTERNAL_OBJECTS | GCFLAG_UNVISITED
         else:
             result = self.malloc_varsize_collecting_nursery(totalsize)
     def malloc_varsize_nonmovable(self, typeid, length):
         return self.malloc_varsize_slowpath(typeid, length, True)
 
-    def malloc_varsize_resizable(self, typeid, length):
-        return self.malloc_varsize_slowpath(typeid, length, True, True)
-
     def malloc_nonmovable(self, typeid, length, zero):
         # helper for testing, same as GCBase.malloc
         if self.is_varsize(typeid):
             raise NotImplementedError("Not supported")
         return llmemory.cast_ptr_to_adr(gcref)
 
-    def realloc(self, ptr, newlength, fixedsize, itemsize, lengthofs, grow):
-        size_gc_header = self.size_gc_header()
-        addr = llmemory.cast_ptr_to_adr(ptr)
-        ll_assert(self.header(addr).tid & GCFLAG_EXTERNAL,
-                  "realloc() on a non-external object")
-        nonvarsize = size_gc_header + fixedsize
-        try:
-            varsize = ovfcheck(itemsize * newlength)
-            tot_size = ovfcheck(nonvarsize + varsize)
-        except OverflowError:
-            raise MemoryError()
-        oldlength = (addr + lengthofs).signed[0]
-        old_tot_size = size_gc_header + fixedsize + oldlength * itemsize
-        source_addr = addr - size_gc_header
-        self.gen2_resizable_objects.remove(addr)
-        if grow:
-            result = llop.raw_realloc_grow(llmemory.Address, source_addr,
-                                           old_tot_size, tot_size)
-        else:
-            result = llop.raw_realloc_shrink(llmemory.Address, source_addr,
-                                             old_tot_size, tot_size)
-        if not result:
-            self.gen2_resizable_objects.append(addr)
-            raise MemoryError()
-        if grow:
-            self.gen2_resizable_objects.append(result + size_gc_header)
-        else:
-            self.gen2_rawmalloced_objects.append(result + size_gc_header)
-        self._check_rawsize_alloced(raw_malloc_usage(tot_size) -
-                                    raw_malloc_usage(old_tot_size),
-                                    can_collect = not grow)
-        (result + size_gc_header + lengthofs).signed[0] = newlength
-        return llmemory.cast_adr_to_ptr(result + size_gc_header, llmemory.GCREF)
-
     def can_move(self, addr):
         tid = self.header(addr).tid
         return not (tid & GCFLAG_EXTERNAL)
             self.semispace_collect()
             debug_stop("gc-rawsize-collect")
 
-    def malloc_varsize_marknsweep(self, totalsize, resizable=False):
+    def malloc_varsize_marknsweep(self, totalsize):
         # In order to free the large objects from time to time, we
         # arbitrarily force a full collect() if none occurs when we have
         # allocated 'self.space_size' bytes of large objects.
         # need to follow suit.
         llmemory.raw_memclear(result, totalsize)
         size_gc_header = self.gcheaderbuilder.size_gc_header
-        if resizable:
-            self.gen2_resizable_objects.append(result + size_gc_header)
-        else:
-            self.gen2_rawmalloced_objects.append(result + size_gc_header)
+        self.gen2_rawmalloced_objects.append(result + size_gc_header)
         return result
 
     def allocate_external_object(self, totalsize):
         if self.is_collecting_gen3():
             self.sweep_rawmalloced_objects(generation=3)
         self.sweep_rawmalloced_objects(generation=2)
-        self.sweep_rawmalloced_objects(generation=-2)
         # As we just collected, it's fine to raw_malloc'ate up to space_size
         # bytes again before we should force another collect.
         self.large_objects_collect_trigger = self.space_size
             gen3roots.delete()
             self.last_generation_root_objects = newgen3roots
         else:
-            # mostly a hack: the generation number -2 is the part of the
-            # generation 2 that lives in gen2_resizable_objects
-            ll_assert(generation == -2, "bogus 'generation'")
-            objects = self.gen2_resizable_objects
+            ll_assert(False, "bogus 'generation'")
+            return
 
         surviving_objects = self.AddressStack()
         # Help the flow space
                         tid |= GCFLAG_UNVISITED
                         surviving_objects.append(obj)
                     self.header(obj).tid = tid
-                elif generation == -2:
-                    # the object stays in generation -2
-                    tid |= GCFLAG_UNVISITED
-                    surviving_objects.append(obj)
-                    self.header(obj).tid = tid
         objects.delete()
         if generation == 2:
             self.gen2_rawmalloced_objects = surviving_objects
         elif generation == 3:
             self.gen3_rawmalloced_objects = surviving_objects
-        elif generation == -2:
-            self.gen2_resizable_objects = surviving_objects
         debug_print("| [hyb] gen", generation,
                     "nonmoving now alive: ",
                     alive_size, "bytes in",

File pypy/rpython/memory/gc/semispace.py

         self.free = result + llarena.round_up_for_allocation(totalsize)
         return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF)
 
+    def shrink_array(self, addr, smallerlength):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        if self._is_in_the_space(addr - size_gc_header):
+            typeid = self.get_type_id(addr)
+            totalsmallersize = (
+                size_gc_header + self.fixed_size(typeid) +
+                self.varsize_item_sizes(typeid) * smallerlength)
+            llarena.arena_shrink_obj(addr - size_gc_header, totalsmallersize)
+            #
+            offset_to_length = self.varsize_offset_to_length(typeid)
+            (addr + offset_to_length).signed[0] = smallerlength
+            return True
+        else:
+            return False
+
     def obtain_free_space(self, needed):
         # a bit of tweaking to maximize the performance and minimize the
         # amount of code in an inlined version of malloc_fixedsize_clear()
     def _is_external(self, obj):
         return (self.header(obj).tid & GCFLAG_EXTERNAL) != 0
 
+    def _is_in_the_space(self, obj):
+        return self.tospace <= obj < self.free
+
     def debug_check_object(self, obj):
         """Check the invariants about 'obj' that should be true
         between collections."""

File pypy/rpython/memory/gc/test/test_direct.py

 class TestSemiSpaceGC(DirectGCTest):
     from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass
 
+    def test_shrink_array(self):
+        S1 = lltype.GcStruct('S1', ('h', lltype.Char),
+                                   ('v', lltype.Array(lltype.Char)))
+        p1 = self.malloc(S1, 2)
+        p1.h = '?'
+        for i in range(2):
+            p1.v[i] = chr(50 + i)
+        addr = llmemory.cast_ptr_to_adr(p1)
+        ok = self.gc.shrink_array(addr, 1)
+        assert ok
+        assert p1.h == '?'
+        assert len(p1.v) == 1
+        for i in range(1):
+            assert p1.v[i] == chr(50 + i)
+
+
 class TestGenerationGC(TestSemiSpaceGC):
     from pypy.rpython.memory.gc.generation import GenerationGC as GCClass
 
 
         gc.collect(9)
         assert calls == [('semispace_collect', True)]
-        calls = []                        
+        calls = []
 
 
 class TestMarkCompactGC(DirectGCTest):

File pypy/rpython/memory/gctransform/boehm.py

 
         mh = mallocHelpers()
         mh.allocate = lambda size: llop.boehm_malloc(llmemory.Address, size)
-        c_realloc = rffi.llexternal('GC_REALLOC', [rffi.VOIDP, rffi.INT],
-                                    rffi.VOIDP, sandboxsafe=True)
-        def _realloc(ptr, size):
-            return llmemory.cast_ptr_to_adr(c_realloc(rffi.cast(rffi.VOIDP, ptr), size))
-        mh.realloc = _realloc
         ll_malloc_fixedsize = mh._ll_malloc_fixedsize
 
         # XXX, do we need/want an atomic version of this function?
         ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length
         ll_malloc_varsize = mh.ll_malloc_varsize
 
-        ll_realloc = mh.ll_realloc
-
         HDRPTR = lltype.Ptr(self.HDR)
 
         def ll_identityhash(addr):
                 inline=False)
             self.weakref_deref_ptr = self.inittime_helper(
                 ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.Address)
-            self.realloc_ptr = self.inittime_helper(
-                ll_realloc, [llmemory.Address] + [lltype.Signed] * 4,
-                llmemory.Address)
             self.identityhash_ptr = self.inittime_helper(
                 ll_identityhash, [llmemory.Address], lltype.Signed,
                 inline=False)
     def pop_alive_nopyobj(self, var, llops):
         pass
 
-    def _can_realloc(self):
-        return True
-
-    def perform_realloc(self, hop, v_ptr, v_newlgt, c_const_size, c_item_size,
-                        c_lengthofs, c_grow):
-        args = [self.realloc_ptr, v_ptr, v_newlgt, c_const_size,
-                c_item_size, c_lengthofs]
-        return hop.genop('direct_call', args, resulttype=llmemory.Address)
-
     def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size):
         # XXX same behavior for zero=True: in theory that's wrong
         if TYPE._is_atomic():

File pypy/rpython/memory/gctransform/framework.py

                                   [s_gc, annmodel.SomeAddress()],
                                   annmodel.SomeBool())
 
+        if hasattr(GCClass, 'shrink_array'):
+            self.shrink_array_ptr = getfn(
+                GCClass.shrink_array.im_func,
+                [s_gc, annmodel.SomeAddress(),
+                 annmodel.SomeInteger(nonneg=True)], annmodel.s_Bool)
+        else:
+            self.shrink_array_ptr = None
+
         if hasattr(GCClass, 'assume_young_pointers'):
             # xxx should really be a noop for gcs without generations
             self.assume_young_pointers_ptr = getfn(
         else:
             self.malloc_varsize_nonmovable_ptr = None
 
-        if getattr(GCClass, 'malloc_varsize_resizable', False):
-            malloc_resizable = func_with_new_name(
-                GCClass.malloc_varsize_resizable.im_func,
-                "malloc_varsize_resizable")
-            self.malloc_varsize_resizable_ptr = getfn(
-                malloc_resizable,
-                [s_gc, s_typeid16,
-                 annmodel.SomeInteger(nonneg=True)], s_gcref)
-        else:
-            self.malloc_varsize_resizable_ptr = None
-
-        if getattr(GCClass, 'realloc', False):
-            self.realloc_ptr = getfn(
-                GCClass.realloc.im_func,
-                [s_gc, s_gcref] +
-                [annmodel.SomeInteger(nonneg=True)] * 4 +
-                [annmodel.SomeBool()],
-                s_gcref)
-
         self.identityhash_ptr = getfn(GCClass.identityhash.im_func,
                                       [s_gc, s_gcref],
                                       annmodel.SomeInteger(),
                                               info_varsize.ofstolength)
             c_varitemsize = rmodel.inputconst(lltype.Signed,
                                               info_varsize.varitemsize)
-            if flags.get('resizable') and self.malloc_varsize_resizable_ptr:
-                assert c_can_collect.value
-                malloc_ptr = self.malloc_varsize_resizable_ptr
-                args = [self.c_const_gc, c_type_id, v_length]                
-            elif flags.get('nonmovable') and self.malloc_varsize_nonmovable_ptr:
+            if flags.get('nonmovable') and self.malloc_varsize_nonmovable_ptr:
                 # we don't have tests for such cases, let's fail
                 # explicitely
                 assert c_can_collect.value
         hop.genop("direct_call", [self.can_move_ptr, self.c_const_gc, v_addr],
                   resultvar=op.result)
 
+    def gct_shrink_array(self, hop):
+        if self.shrink_array_ptr is None:
+            return GCTransformer.gct_shrink_array(self, hop)
+        op = hop.spaceop
+        v_addr = hop.genop('cast_ptr_to_adr',
+                           [op.args[0]], resulttype=llmemory.Address)
+        v_length = op.args[1]
+        hop.genop("direct_call", [self.shrink_array_ptr, self.c_const_gc,
+                                  v_addr, v_length],
+                  resultvar=op.result)
+
     def gct_gc_assume_young_pointers(self, hop):
         op = hop.spaceop
         v_addr = op.args[0]
                              resulttype=llmemory.Address)
         hop.genop('adr_add', [v_gc_adr, c_ofs], resultvar=op.result)
 
-    def _can_realloc(self):
-        return self.gcdata.gc.can_realloc
-
-    def perform_realloc(self, hop, v_ptr, v_newsize, c_const_size,
-                        c_itemsize, c_lengthofs, c_grow):
-        vlist = [self.realloc_ptr, self.c_const_gc, v_ptr, v_newsize,
-                 c_const_size, c_itemsize, c_lengthofs, c_grow]
-        livevars = self.push_roots(hop)
-        v_result = hop.genop('direct_call', vlist,
-                             resulttype=llmemory.GCREF)
-        self.pop_roots(hop, livevars)
-        return v_result
-
     def gct_gc_x_swap_pool(self, hop):
         op = hop.spaceop
         [v_malloced] = op.args

File pypy/rpython/memory/gctransform/transform.py

         return result
     mh.ll_malloc_varsize_no_length_zero = _ll_malloc_varsize_no_length_zero
 
-    def ll_realloc(ptr, length, constsize, itemsize, lengthoffset):
-        size = constsize + length * itemsize
-        result = mh.realloc(ptr, size)
-        if not result:
-            raise MemoryError()
-        (result + lengthoffset).signed[0] = length
-        return result
-    mh.ll_realloc = ll_realloc
-
     return mh
 
 class GCTransformer(BaseGCTransformer):
     def gct_malloc_nonmovable_varsize(self, *args, **kwds):
         return self.gct_malloc_varsize(*args, **kwds)
 
-    def gct_malloc_resizable_buffer(self, hop):
-        flags = hop.spaceop.args[1].value
-        flags['varsize'] = True
-        flags['nonmovable'] = True
-        flags['resizable'] = True
-        flavor = flags['flavor']
-        assert flavor != 'cpy', "cannot malloc CPython objects directly"
-        meth = getattr(self, 'gct_fv_%s_malloc_varsize' % flavor, None)
-        assert meth, "%s has no support for malloc_varsize with flavor %r" % (self, flavor) 
-        return self.varsize_malloc_helper(hop, flags, meth, [])
-
-    def gct_resize_buffer(self, hop):
-        op = hop.spaceop
-        if self._can_realloc():
-            self._gct_resize_buffer_realloc(hop, op.args[2], True)
-        else:
-            self._gct_resize_buffer_no_realloc(hop, op.args[1])
-
-    def _can_realloc(self):
-        return False
-
-    def _gct_resize_buffer_realloc(self, hop, v_newsize, grow=True):
-        def intconst(c): return rmodel.inputconst(lltype.Signed, c)
-        op = hop.spaceop
-        flags = {'flavor':'gc', 'varsize': True}
-        TYPE = op.args[0].concretetype.TO
-        ARRAY = TYPE._flds[TYPE._arrayfld]
-        offset_to_length = llmemory.FieldOffset(TYPE, TYPE._arrayfld) + \
-                           llmemory.ArrayLengthOffset(ARRAY)
-        c_const_size = intconst(llmemory.sizeof(TYPE, 0))
-        c_item_size = intconst(llmemory.sizeof(ARRAY.OF))
-
-        c_lengthofs = intconst(offset_to_length)
-        v_ptr = op.args[0]
-        v_ptr = gen_cast(hop.llops, llmemory.GCREF, v_ptr)
-        c_grow = rmodel.inputconst(lltype.Bool, grow)
-        v_raw = self.perform_realloc(hop, v_ptr, v_newsize, c_const_size,
-                                     c_item_size, c_lengthofs, c_grow)
-        hop.cast_result(v_raw)
-
-    def _gct_resize_buffer_no_realloc(self, hop, v_lgt):
-        op = hop.spaceop
-        meth = self.gct_fv_gc_malloc_varsize
-        flags = {'flavor':'gc', 'varsize': True, 'keep_current_args': True}
-        self.varsize_malloc_helper(hop, flags, meth, [])
-        # fish resvar
-        v_newbuf = hop.llops[-1].result
-        v_src = op.args[0]
-        TYPE = v_src.concretetype.TO
-        c_fldname = rmodel.inputconst(lltype.Void, TYPE._arrayfld)
-        v_adrsrc = hop.genop('cast_ptr_to_adr', [v_src],
-                             resulttype=llmemory.Address)
-        v_adrnewbuf = hop.genop('cast_ptr_to_adr', [v_newbuf],
-                                resulttype=llmemory.Address)
-        ofs = (llmemory.offsetof(TYPE, TYPE._arrayfld) +
-               llmemory.itemoffsetof(getattr(TYPE, TYPE._arrayfld), 0))
-        v_ofs = rmodel.inputconst(lltype.Signed, ofs)
-        v_adrsrc = hop.genop('adr_add', [v_adrsrc, v_ofs],
-                             resulttype=llmemory.Address)
-        v_adrnewbuf = hop.genop('adr_add', [v_adrnewbuf, v_ofs],
-                                resulttype=llmemory.Address)
-        size = llmemory.sizeof(getattr(TYPE, TYPE._arrayfld).OF)
-        c_size = rmodel.inputconst(lltype.Signed, size)
-        v_lgtsym = hop.genop('int_mul', [c_size, v_lgt],
-                             resulttype=lltype.Signed) 
-        vlist = [v_adrsrc, v_adrnewbuf, v_lgtsym]
-        hop.genop('raw_memcopy', vlist)
-
-    def gct_finish_building_buffer(self, hop):
-        op = hop.spaceop
-        if self._can_realloc():
-            return self._gct_resize_buffer_realloc(hop, op.args[1], False)
-        else:
-            return self._gct_resize_buffer_no_realloc(hop, op.args[1])
-
     def varsize_malloc_helper(self, hop, flags, meth, extraargs):
         def intconst(c): return rmodel.inputconst(lltype.Signed, c)
         op = hop.spaceop
 
     def gct_gc_can_move(self, hop):
         return hop.cast_result(rmodel.inputconst(lltype.Bool, False))
+
+    def gct_shrink_array(self, hop):
+        return hop.cast_result(rmodel.inputconst(lltype.Bool, False))

File pypy/rpython/memory/gcwrapper.py

 from pypy.rpython.memory import gctypelayout
 from pypy.objspace.flow.model import Constant
 
-
 class GCManagedHeap(object):
 
     def __init__(self, llinterp, flowgraphs, gc_class, GC_PARAMS={}):
             gctypelayout.zero_gc_pointers(result)
         return result
 
-    def malloc_resizable_buffer(self, TYPE, n):
-        typeid = self.get_type_id(TYPE)
-        addr = self.gc.malloc(typeid, n)
-        result = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE))
-        if not self.gc.malloc_zero_filled:
-            gctypelayout.zero_gc_pointers(result)
-        return result
-
-    def resize_buffer(self, obj, old_size, new_size):
-        T = lltype.typeOf(obj).TO
-        buf = self.malloc_resizable_buffer(T, new_size)
-        # copy contents
-        arrayfld = T._arrayfld
-        new_arr = getattr(buf, arrayfld)
-        old_arr = getattr(obj, arrayfld)
-        for i in range(old_size):
-            new_arr[i] = old_arr[i]
-        return buf
-
-    def finish_building_buffer(self, obj, size):
-        return obj
+    def shrink_array(self, p, smallersize):
+        if hasattr(self.gc, 'shrink_array'):
+            addr = llmemory.cast_ptr_to_adr(p)
+            return self.gc.shrink_array(addr, smallersize)
+        return False
 
     def free(self, TYPE, flavor='gc'):
         assert flavor != 'gc'

File pypy/rpython/memory/test/test_gc.py

 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import compute_unique_id, keepalive_until_here
 from pypy.rlib import rgc
-
+from pypy.rlib.rstring import StringBuilder
 
 def stdout_ignore_ll_functions(msg):
     strmsg = str(msg)
     GC_PARAMS = {}
     GC_CAN_MOVE = False
     GC_CANNOT_MALLOC_NONMOVABLE = False
+    GC_CAN_SHRINK_ARRAY = False
 
     def setup_class(cls):
         cls._saved_logstate = py.log._getstate()
 
         assert self.interpret(func, []) == int(self.GC_CANNOT_MALLOC_NONMOVABLE)
 
-    def test_resizable_buffer(self):
+    def test_shrink_array(self):
         from pypy.rpython.lltypesystem.rstr import STR
-        from pypy.rpython.annlowlevel import hlstr
+        GC_CAN_SHRINK_ARRAY = self.GC_CAN_SHRINK_ARRAY
 
-        def f():
-            ptr = rgc.resizable_buffer_of_shape(STR, 1)
-            ptr.chars[0] = 'a'
-            ptr = rgc.resize_buffer(ptr, 1, 2)
-            ptr.chars[1] = 'b'
-            return len(hlstr(rgc.finish_building_buffer(ptr, 2)))
+        def f(n, m):
+            ptr = lltype.malloc(STR, n)
+            ptr.hash = 0x62
+            ptr.chars[0] = 'A'
+            ptr.chars[1] = 'B'
+            ptr.chars[2] = 'C'
+            ptr2 = rgc.ll_shrink_array(ptr, 2)
+            assert (ptr == ptr2) == GC_CAN_SHRINK_ARRAY
+            rgc.collect()
+            return ( ord(ptr2.chars[0])       +
+                    (ord(ptr2.chars[1]) << 8) +
+                    (len(ptr2.chars)   << 16) +
+                    (ptr2.hash         << 24))
 
-        assert self.interpret(f, []) == 2
+        assert self.interpret(f, [3, 0]) == 0x62024241
+        # don't test with larger numbers of top of the Hybrid GC, because
+        # the default settings make it a too-large varsized object that
+        # gets allocated outside the semispace
+        if not isinstance(self, TestHybridGC):
+            assert self.interpret(f, [12, 0]) == 0x62024241
 
     def test_tagged_simple(self):
         from pypy.rlib.objectmodel import UnboxedValue
 
         self.interpret(fn, [])
 
+    def test_stringbuilder(self):
+        def fn():
+            s = StringBuilder(4)
+            s.append("abcd")
+            s.append("defg")
+            s.append("rty")
+            s.append_multiple_char('y', 1000)
+            rgc.collect()
+            s.append_multiple_char('y', 1000)
+            res = s.build()[1000]
+            rgc.collect()
+            return ord(res)
+        res = self.interpret(fn, [])
+        assert res == ord('y')
 
 from pypy.rlib.objectmodel import UnboxedValue
 
     from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass
     GC_CAN_MOVE = True
     GC_CANNOT_MALLOC_NONMOVABLE = True
+    GC_CAN_SHRINK_ARRAY = True
 
 class TestGrowingSemiSpaceGC(TestSemiSpaceGC):
     GC_PARAMS = {'space_size': 64}

File pypy/rpython/memory/test/test_transformed_gc.py

             return run
         
 class GenericGCTests(GCTest):
+    GC_CAN_SHRINK_ARRAY = False
 
     def heap_usage(self, statistics):
         try:
         run = self.runner("malloc_nonmovable_fixsize")
         assert run([]) == int(self.GC_CANNOT_MALLOC_NONMOVABLE)
 
-    def define_resizable_buffer(cls):
+    def define_shrink_array(cls):
         from pypy.rpython.lltypesystem.rstr import STR
-        from pypy.rpython.annlowlevel import hlstr
 
         def f():
-            ptr = rgc.resizable_buffer_of_shape(STR, 2)
-            ptr.chars[0] = 'a'
-            ptr = rgc.resize_buffer(ptr, 1, 200)
-            ptr.chars[1] = 'b'
-            return hlstr(rgc.finish_building_buffer(ptr, 2)) == "ab"
-
+            ptr = lltype.malloc(STR, 3)
+            ptr.hash = 0x62
+            ptr.chars[0] = '0'
+            ptr.chars[1] = 'B'
+            ptr.chars[2] = 'C'
+            ptr2 = rgc.ll_shrink_array(ptr, 2)
+            return ((ptr == ptr2)             +
+                     ord(ptr2.chars[0])       +
+                    (ord(ptr2.chars[1]) << 8) +
+                    (len(ptr2.chars)   << 16) +
+                    (ptr2.hash         << 24))
         return f
 
-    def test_resizable_buffer(self):
-        run = self.runner("resizable_buffer")
-        assert run([]) == 1
+    def test_shrink_array(self):
+        run = self.runner("shrink_array")
+        if self.GC_CAN_SHRINK_ARRAY:
+            expected = 0x62024231
+        else:
+            expected = 0x62024230
+        assert run([]) == expected
 
     def define_string_builder_over_allocation(cls):
         import gc
 
 class TestSemiSpaceGC(GenericMovingGCTests):
     gcname = "semispace"
+    GC_CAN_SHRINK_ARRAY = True
 
     class gcpolicy(gc.FrameworkGcPolicy):
         class transformerclass(framework.FrameworkGCTransformer):
 
 class TestGenerationGC(GenericMovingGCTests):
     gcname = "generation"
+    GC_CAN_SHRINK_ARRAY = True
 
     class gcpolicy(gc.FrameworkGcPolicy):
         class transformerclass(framework.FrameworkGCTransformer):

File pypy/rpython/test/test_rbuilder.py

 
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
+from pypy.rpython.lltypesystem.rbuilder import *
+from pypy.rpython.annlowlevel import llstr, hlstr
 from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
 
+
+class TestStringBuilderDirect(object):
+    def test_simple(self):
+        sb = StringBuilderRepr.ll_new(3)
+        StringBuilderRepr.ll_append_char(sb, 'x')
+        StringBuilderRepr.ll_append(sb, llstr("abc"))
+        StringBuilderRepr.ll_append_slice(sb, llstr("foobar"), 2, 5)
+        StringBuilderRepr.ll_append_multiple_char(sb, 'y', 3)
+        s = StringBuilderRepr.ll_build(sb)
+        assert hlstr(s) == "xabcobayyy"
+
+    def test_nooveralloc(self):
+        sb = StringBuilderRepr.ll_new(3)
+        StringBuilderRepr.ll_append(sb, llstr("abc"))
+        assert StringBuilderRepr.ll_build(sb) == sb.buf
+
 class BaseTestStringBuilder(BaseRtypingTest):
     def test_simple(self):
         def func():
         assert res == 'aabcabcdefbuuuu'
         assert isinstance(res, unicode)
 
-
 class TestLLtype(BaseTestStringBuilder, LLRtypeMixin):
     pass
 

File pypy/translator/c/src/mem.h

 
 #define OP_RAW_MALLOC_USAGE(size, r) r = size
 
-#define OP_RAW_REALLOC_SHRINK(p, old_size, size, r) r = PyObject_Realloc((void*)p, size)
-
-#define OP_RAW_REALLOC_GROW(p, old_size, size, r) r = PyObject_Realloc((void*)p, size)
-
 #ifdef MS_WINDOWS
 #define alloca  _alloca
 #endif

File pypy/translator/c/test/test_boehm.py

         run = self.getcompiled(func)
         assert run() == 0
 
-    def test_resizable_buffer(self):
+    def test_shrink_array(self):
         from pypy.rpython.lltypesystem.rstr import STR
-        from pypy.rpython.annlowlevel import hlstr
         from pypy.rlib import rgc
 
         def f():
-            ptr = rgc.resizable_buffer_of_shape(STR, 2)
-            ptr.chars[0] = 'a'
-            ptr = rgc.resize_buffer(ptr, 1, 200)
-            ptr.chars[1] = 'b'
-            return hlstr(rgc.finish_building_buffer(ptr, 2)) == "ab"
+            ptr = lltype.malloc(STR, 3)
+            ptr.hash = 0x62
+            ptr.chars[0] = '0'
+            ptr.chars[1] = 'B'
+            ptr.chars[2] = 'C'
+            ptr2 = rgc.ll_shrink_array(ptr, 2)
+            return ((ptr == ptr2)             +
+                     ord(ptr2.chars[0])       +
+                    (ord(ptr2.chars[1]) << 8) +
+                    (len(ptr2.chars)   << 16) +
+                    (ptr2.hash         << 24))
 
         run = self.getcompiled(f)
-        assert run() == True
+        assert run() == 0x62024230
 
     def test_assume_young_pointers_nop(self):
         S = lltype.GcStruct('S', ('x', lltype.Signed))

File pypy/translator/c/test/test_newgc.py

     taggedpointers = False
     GC_CAN_MOVE = False
     GC_CANNOT_MALLOC_NONMOVABLE = False
+    GC_CAN_SHRINK_ARRAY = False
 
     _isolated_func = None
 
 
     def define_resizable_buffer(cls):
         from pypy.rpython.lltypesystem.rstr import STR
-        from pypy.rpython.annlowlevel import hlstr
 
         def f():
-            ptr = rgc.resizable_buffer_of_shape(STR, 2)
-            ptr.chars[0] = 'a'
-            ptr = rgc.resize_buffer(ptr, 1, 200)
-            ptr.chars[1] = 'b'
-            return hlstr(rgc.finish_building_buffer(ptr, 2)) == "ab"
-
+            ptr = lltype.malloc(STR, 3)
+            ptr.hash = 0x62
+            ptr.chars[0] = '0'
+            ptr.chars[1] = 'B'
+            ptr.chars[2] = 'C'
+            ptr2 = rgc.ll_shrink_array(ptr, 2)
+            return ((ptr == ptr2)             +
+                     ord(ptr2.chars[0])       +
+                    (ord(ptr2.chars[1]) << 8) +
+                    (len(ptr2.chars)   << 16) +
+                    (ptr2.hash         << 24))
         return f
 
     def test_resizable_buffer(self):
-        assert self.run('resizable_buffer')
+        res = self.run('resizable_buffer')
+        if self.GC_CAN_SHRINK_ARRAY:
+            expected = 0x62024231
+        else:
+            expected = 0x62024230
+        assert res == expected
 
     def define_hash_preservation(cls):
         from pypy.rlib.objectmodel import compute_hash
     should_be_moving = True
     GC_CAN_MOVE = True
     GC_CANNOT_MALLOC_NONMOVABLE = True
+    GC_CAN_SHRINK_ARRAY = True
 
     # for snippets
     large_tests_ok = True
 class TestMarkCompactGC(TestSemiSpaceGC):
     gcpolicy = "markcompact"
     should_be_moving = True
+    GC_CAN_SHRINK_ARRAY = False
 
     def setup_class(cls):
         py.test.skip("Disabled for now")