Commits

Anonymous committed ed6fe5a

Replaced direct usages of array.tostring/fromstring with compat function. See issue #257. (cherrypicked changeset d65ec5e266aa from default branch)

Comments (0)

Files changed (9)

src/whoosh/codec/base.py

 from struct import Struct, pack
 from bisect import bisect_right
 
-from whoosh.compat import loads, dumps, b, bytes_type, string_type, xrange
+from whoosh.compat import (loads, dumps, b, bytes_type, string_type, xrange,
+                           array_frombytes, array_tobytes)
 from whoosh.filedb.compound import CompoundStorage
 from whoosh.matching import Matcher, ReadTooFar
 from whoosh.reading import TermInfo
             arry = array(typecode, iter(arry))
         if not IS_LITTLE:
             arry.byteswap()
-        string = arry.tostring()
+        string = array_tobytes(arry)
     if compression:
         string = compress(string, compression)
     return (typecode, string)
         return loads(string)
     else:
         arry = array(typecode)
-        arry.fromstring(string)
+        array_frombytes(arry, string)
         if not IS_LITTLE:
             arry.byteswap()
         return arry
     else:
         if not IS_LITTLE:
             weights.byteswap()
-        string = weights.tostring()
+        string = array_tobytes(weights)
     if string and compression:
         string = compress(string, compression)
     return string
     if compression:
         string = decompress(string)
     arry = array("f")
-    arry.fromstring(string)
+    array_frombytes(arry, string)
     if not IS_LITTLE:
         arry.byteswap()
     return arry

src/whoosh/codec/legacy.py

 from array import array
 from struct import Struct
 
-from whoosh.compat import load, xrange
+from whoosh.compat import load, xrange, array_frombytes
 from whoosh.codec import base
 from whoosh.codec.base import (deminimize_ids, deminimize_weights,
                                deminimize_values)
             newoffset = postfile.tell()
         elif self.idslen:
             ids = array("I")
-            ids.fromstring(decompress(postfile.read(self.idslen)))
+            array_frombytes(ids, decompress(postfile.read(self.idslen)))
             if IS_LITTLE:
                 ids.byteswap()
             newoffset = offset + self.idslen
             newoffset = offset
         elif weightslen:
             weights = array("f")
-            weights.fromstring(decompress(postfile.read(weightslen)))
+            array_frombytes(weights, decompress(postfile.read(weightslen)))
             if IS_LITTLE:
                 weights.byteswap()
             newoffset = offset + weightslen
             if postingsize < 0:
                 # Pull the array of value lengths off the front of the string
                 lengths = array("i")
-                lengths.fromstring(values_string[:_INT_SIZE * postcount])
+                array_frombytes(lengths, values_string[:_INT_SIZE * postcount])
                 values_string = values_string[_INT_SIZE * postcount:]
 
             # Chop up the block string into individual valuestrings

src/whoosh/compat.py

-import sys
+import array, sys
 
 
 # Run time aliasing of Python2/3 differences
         pass
 
 
+if hasattr(array.array, "tobytes"):
+    def array_tobytes(arry):
+        return arry.tobytes()
+
+    def array_frombytes(arry, bs):
+        return arry.frombytes(bs)
+else:
+    def array_tobytes(arry):
+        return arry.tostring()
+
+    def array_frombytes(arry, bs):
+        return arry.fromstring(bs)
+
+
 # Implementations missing from older versions of Python
 
 try:

src/whoosh/filedb/structfile.py

 
 from whoosh.compat import dump as dump_pickle
 from whoosh.compat import load as load_pickle
-from whoosh.compat import PY3, integer_types, b
+from whoosh.compat import array_frombytes, array_tobytes
 from whoosh.system import (_INT_SIZE, _SHORT_SIZE, _FLOAT_SIZE, _LONG_SIZE,
                            pack_byte, pack_sbyte, pack_ushort, pack_int,
                            pack_uint, pack_long, pack_float, unpack_byte,
         if self.is_real:
             arry.tofile(self.file)
         else:
-            self.file.write(arry.tostring())
+            self.file.write(array_tobytes(arry))
 
     def read_sbyte(self):
         return unpack_sbyte(self.file.read(1))[0]
         if self.is_real:
             a.fromfile(self.file, length)
         else:
-            a.fromstring(self.file.read(length * _SIZEMAP[typecode]))
+            array_frombytes(a, self.file.read(length * _SIZEMAP[typecode]))
         if IS_LITTLE:
             a.byteswap()
         return a

src/whoosh/support/dawg.py

 from hashlib import sha1  #@UnresolvedImport
 
 from whoosh.compat import (b, u, BytesIO, xrange, iteritems, iterkeys,
-                           bytes_type, text_type, izip)
+                           bytes_type, text_type, izip, array_tobytes)
 from whoosh.filedb.structfile import StructFile
 from whoosh.system import (_INT_SIZE, pack_byte, pack_int, pack_uint,
                            pack_long, emptybytes)
 
     @staticmethod
     def to_bytes(v):
-        return v.tostring()
+        return array_tobytes(v)
 
 
 class IntListValues(SequenceValues):

src/whoosh/support/numeric.py

 import struct
 from array import array
 
-from whoosh.compat import long_type, PY3
+from whoosh.compat import long_type, PY3, array_tobytes
 from whoosh.support.base85 import to_base85, from_base85
 
 _istruct = struct.Struct(">i")
         buffer[nchars - 1] = chr(x & 0x7f)
         x >>= 7
         nchars -= 1
-    return buffer.tostring()
+    return array_tobytes(buffer)
 
 
 def from_7bit(text):

src/whoosh/util.py

 from threading import Lock
 
 from whoosh.compat import xrange, u, b, string_type
-from whoosh.system import IS_LITTLE
+from whoosh.compat import array_tobytes
 from whoosh.system import pack_ushort_le, pack_uint_le
 from whoosh.system import unpack_ushort_le, unpack_uint_le
 
 
 # Functions
 
-def array_to_string(a):
-    if IS_LITTLE:
-        a = copy(a)
-        a.byteswap()
-    return a.tostring()
-
-
-def string_to_array(typecode, s):
-    a = array(typecode)
-    a.fromstring(s)
-    if IS_LITTLE:
-        a.byteswap()
-    return a
-
-
 def make_binary_tree(fn, args, **kwargs):
     """Takes a function/class that takes two positional arguments and a list of
     arguments and returns a binary tree of results/instances.
         a.append((i & 0x7F) | 0x80)
         i = i >> 7
     a.append(i)
-    return a.tostring()
+    return array_tobytes(a)
 
 
 _varint_cache_size = 512
         if count == 4:
             #print bin(key), repr(buf)
             dbfile.write(chr(key))
-            dbfile.write(buf.tostring())
+            dbfile.write(array_tobytes(buf))
             count = 0
             key = 0
             del buf[0:len(buf)]
 
     if count:
         dbfile.write(chr(key))
-        dbfile.write(buf.tostring())
+        dbfile.write(array_tobytes(buf))
 
 
 def read_gints(dbfile, n):

tests/test_codecs.py

 
 from whoosh import fields, formats
 from whoosh.compat import u, b, xrange, iteritems
+from whoosh.compat import array_tobytes
 from whoosh.codec.base import FileTermInfo
 from whoosh.codec import default_codec
 from whoosh.filedb.filestore import RamStorage
 
     def random_token():
         a = array("H", (random.randint(0, 0xd7ff) for _ in xrange(1, 20)))
-        return a.tostring().decode("utf-16")
+        return array_tobytes(a).decode("utf-16")
 
     domain = sorted(set([(random_fieldname(), random_token())
                          for _ in xrange(1000)]))

tests/test_dawg.py

 from __future__ import with_statement
 
-from nose.tools import assert_equal, assert_not_equal, assert_raises  #@UnresolvedImport
+from nose.tools import assert_equal, assert_raises  #@UnresolvedImport
 
 import random
 from array import array
 
-from whoosh.compat import b, u, xrange
+from whoosh.compat import b, u, xrange, array_tobytes
 from whoosh.filedb.filestore import RamStorage
 from whoosh.support import dawg
 from whoosh.support.testing import TempStorage
     def randstring():
         length = random.randint(1, 10)
         a = array("B", (random.randint(0, 255) for _ in xrange(length)))
-        return a.tostring()
+        return array_tobytes(a)
     keys = sorted(randstring() for _ in xrange(1000))
 
     with TempStorage() as st: