Source

pypy / pypy / objspace / std / marshal_impl.py

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
# implementation of marshalling by multimethods

"""
The idea is to have an effective but flexible
way to implement marshalling for the native types.

The marshal_w operation is called with an object,
a callback and a state variable.
"""

from pypy.interpreter.error import OperationError
from pypy.objspace.std.register_all import register_all
from pypy.rlib.rarithmetic import LONG_BIT, r_longlong, r_uint, intmask
from pypy.objspace.std import model
from pypy.interpreter.special import Ellipsis
from pypy.interpreter.pycode import PyCode
from pypy.interpreter import gateway, unicodehelper
from pypy.rlib.rstruct import ieee
from pypy.rlib.rstring import StringBuilder

from pypy.objspace.std.boolobject    import W_BoolObject
from pypy.objspace.std.complexobject import W_ComplexObject
from pypy.objspace.std.intobject     import W_IntObject
from pypy.objspace.std.floatobject   import W_FloatObject
from pypy.objspace.std.tupleobject   import W_TupleObject
from pypy.objspace.std.listobject    import W_ListObject
from pypy.objspace.std.dictmultiobject    import W_DictMultiObject
from pypy.objspace.std.stringobject  import W_StringObject
from pypy.objspace.std.ropeobject    import W_RopeObject
from pypy.objspace.std.typeobject    import W_TypeObject
from pypy.objspace.std.longobject    import W_LongObject, newlong
from pypy.objspace.std.noneobject    import W_NoneObject
from pypy.objspace.std.unicodeobject import W_UnicodeObject

from pypy.module.marshal.interp_marshal import register

TYPE_NULL      = '0'
TYPE_NONE      = 'N'
TYPE_FALSE     = 'F'
TYPE_TRUE      = 'T'
TYPE_STOPITER  = 'S'
TYPE_ELLIPSIS  = '.'
TYPE_INT       = 'i'
TYPE_INT64     = 'I'
TYPE_FLOAT     = 'f'
TYPE_BINARY_FLOAT = 'g'
TYPE_COMPLEX   = 'x'
TYPE_BINARY_COMPLEX = 'y'
TYPE_LONG      = 'l'
TYPE_STRING    = 's'
TYPE_STRINGREF = 'R'
TYPE_TUPLE     = '('
TYPE_LIST      = '['
TYPE_DICT      = '{'
TYPE_CODE      = 'c'
TYPE_UNICODE   = 'u'
TYPE_UNKNOWN   = '?'
TYPE_SET       = '<'
TYPE_FROZENSET = '>'

"""
simple approach:
a call to marshal_w has the following semantics:
marshal_w receives a marshaller object which contains
state and several methods.


atomic types including typecode:

atom(tc)                    puts single typecode
atom_int(tc, int)           puts code and int
atom_int64(tc, int64)       puts code and int64
atom_str(tc, str)           puts code, len and string
atom_strlist(tc, strlist)   puts code, len and list of strings

building blocks for compound types:

start(typecode)             sets the type character
put(s)                      puts a string with fixed length
put_short(int)              puts a short integer
put_int(int)                puts an integer
put_pascal(s)               puts a short string
put_w_obj(w_obj)            puts a wrapped object
put_tuple_w(TYPE, tuple_w)  puts tuple_w, an unwrapped list of wrapped objects
"""

handled_by_any = []

def raise_exception(space, msg):
    raise OperationError(space.w_ValueError, space.wrap(msg))

def marshal_w__None(space, w_none, m):
    m.atom(TYPE_NONE)

def unmarshal_None(space, u, tc):
    return space.w_None
register(TYPE_NONE, unmarshal_None)

def marshal_w__Bool(space, w_bool, m):
    if w_bool.boolval:
        m.atom(TYPE_TRUE)
    else:
        m.atom(TYPE_FALSE)

def unmarshal_Bool(space, u, tc):
    if tc == TYPE_TRUE:
        return space.w_True
    else:
        return space.w_False
register(TYPE_TRUE + TYPE_FALSE, unmarshal_Bool)

def marshal_w__Type(space, w_type, m):
    if not space.is_w(w_type, space.w_StopIteration):
        raise_exception(space, "unmarshallable object")
    m.atom(TYPE_STOPITER)

def unmarshal_Type(space, u, tc):
    return space.w_StopIteration
register(TYPE_STOPITER, unmarshal_Type)

# not directly supported:
def marshal_w_Ellipsis(space, w_ellipsis, m):
    m.atom(TYPE_ELLIPSIS)

model.MM.marshal_w.register(marshal_w_Ellipsis, Ellipsis)

def unmarshal_Ellipsis(space, u, tc):
    return space.w_Ellipsis
register(TYPE_ELLIPSIS, unmarshal_Ellipsis)

def marshal_w__Int(space, w_int, m):
    if LONG_BIT == 32:
        m.atom_int(TYPE_INT, w_int.intval)
    else:
        y = w_int.intval >> 31
        if y and y != -1:
            m.atom_int64(TYPE_INT64, w_int.intval)
        else:
            m.atom_int(TYPE_INT, w_int.intval)

def unmarshal_Int(space, u, tc):
    return space.newint(u.get_int())
register(TYPE_INT, unmarshal_Int)

def unmarshal_Int64(space, u, tc):
    lo = u.get_int()    # get the first 32 bits
    hi = u.get_int()    # get the next 32 bits
    if LONG_BIT >= 64:
        x = (hi << 32) | (lo & (2**32-1))    # result fits in an int
    else:
        x = (r_longlong(hi) << 32) | r_longlong(r_uint(lo))  # get a r_longlong
    return space.wrap(x)
register(TYPE_INT64, unmarshal_Int64)

def pack_float(f):
    result = StringBuilder(8)
    ieee.pack_float(result, f, 8, False)
    return result.build()

def unpack_float(s):
    return ieee.unpack_float(s, False)

def marshal_w__Float(space, w_float, m):
    if m.version > 1:
        m.start(TYPE_BINARY_FLOAT)
        m.put(pack_float(w_float.floatval))
    else:
        m.start(TYPE_FLOAT)
        m.put_pascal(space.str_w(space.repr(w_float)))

def unmarshal_Float(space, u, tc):
    return space.call_function(space.builtin.get('float'),
                               space.wrap(u.get_pascal()))
register(TYPE_FLOAT, unmarshal_Float)

def unmarshal_Float_bin(space, u, tc):
    return space.newfloat(unpack_float(u.get(8)))
register(TYPE_BINARY_FLOAT, unmarshal_Float_bin)

def marshal_w__Complex(space, w_complex, m):
    if m.version > 1:
        m.start(TYPE_BINARY_COMPLEX)
        m.put(pack_float(w_complex.realval))
        m.put(pack_float(w_complex.imagval))
    else:
        # XXX a bit too wrap-happy
        w_real = space.wrap(w_complex.realval)
        w_imag = space.wrap(w_complex.imagval)
        m.start(TYPE_COMPLEX)
        m.put_pascal(space.str_w(space.repr(w_real)))
        m.put_pascal(space.str_w(space.repr(w_imag)))

def unmarshal_Complex(space, u, tc):
    w_real = space.call_function(space.builtin.get('float'),
                                 space.wrap(u.get_pascal()))
    w_imag = space.call_function(space.builtin.get('float'),
                                 space.wrap(u.get_pascal()))
    w_t = space.builtin.get('complex')
    return space.call_function(w_t, w_real, w_imag)
register(TYPE_COMPLEX, unmarshal_Complex)

def unmarshal_Complex_bin(space, u, tc):
    real = unpack_float(u.get(8))
    imag = unpack_float(u.get(8))
    return space.newcomplex(real, imag)
register(TYPE_BINARY_COMPLEX, unmarshal_Complex_bin)

def marshal_w__Long(space, w_long, m):
    from pypy.rlib.rbigint import rbigint
    m.start(TYPE_LONG)
    SHIFT = 15
    MASK = (1 << SHIFT) - 1
    num = w_long.num
    sign = num.sign
    num = num.abs()
    ints = []
    while num.tobool():
        next = intmask(num.uintmask() & MASK)
        ints.append(next)
        num = num.rshift(SHIFT)
    m.put_int(len(ints) * sign)
    for i in ints:
        m.put_short(i)

def unmarshal_Long(space, u, tc):
    from pypy.rlib.rbigint import rbigint
    lng = u.get_int()
    if lng < 0:
        negative = True
        lng = -lng
    else:
        negative = False
    SHIFT = 15
    result = rbigint.fromint(0)
    for i in range(lng):
        shift = i * SHIFT
        result = result.or_(rbigint.fromint(u.get_short()).lshift(shift))
    if lng and not result.tobool():
        raise_exception(space, 'bad marshal data')
    if negative:
        result = result.neg()
    w_long = newlong(space, result)
    return w_long
register(TYPE_LONG, unmarshal_Long)

def marshal_w__String(space, w_str, m):
    s = w_str.unwrap(space)
    m.atom_str(TYPE_STRING, s)

marshal_w__Rope = marshal_w__String

def unmarshal_String(space, u, tc):
    return space.wrapbytes(u.get_str())
register(TYPE_STRING, unmarshal_String)

def unmarshal_stringref(space, u, tc):
    idx = u.get_int()
    try:
        return u.stringtable_w[idx]
    except IndexError:
        raise_exception(space, 'bad marshal data')
register(TYPE_STRINGREF, unmarshal_stringref)

def marshal_w__Tuple(space, w_tuple, m):
    items = w_tuple.wrappeditems
    m.put_tuple_w(TYPE_TUPLE, items)

def unmarshal_Tuple(space, u, tc):
    items_w = u.get_tuple_w()
    return space.newtuple(items_w)
register(TYPE_TUPLE, unmarshal_Tuple)

def marshal_w__List(space, w_list, m):
    items = w_list.getitems()[:]
    m.put_tuple_w(TYPE_LIST, items)

def unmarshal_List(space, u, tc):
    items_w = u.get_list_w()
    return space.newlist(items_w)

def finish_List(space, items_w, typecode):
    return space.newlist(items_w)
register(TYPE_LIST, unmarshal_List)

def marshal_w__DictMulti(space, w_dict, m):
    m.start(TYPE_DICT)
    for w_tuple in w_dict.items():
        w_key, w_value = space.fixedview(w_tuple, 2)
        m.put_w_obj(w_key)
        m.put_w_obj(w_value)
    m.atom(TYPE_NULL)

def unmarshal_DictMulti(space, u, tc):
    # since primitive lists are not optimized and we don't know
    # the dict size in advance, use the dict's setitem instead
    # of building a list of tuples.
    w_dic = space.newdict()
    while 1:
        w_key = u.get_w_obj(allow_null=True)
        if w_key is None:
            break
        w_value = u.get_w_obj()
        space.setitem(w_dic, w_key, w_value)
    return w_dic
register(TYPE_DICT, unmarshal_DictMulti)

def unmarshal_NULL(self, u, tc):
    return None
register(TYPE_NULL, unmarshal_NULL)

# this one is registered by hand:
def marshal_w_pycode(space, w_pycode, m):
    m.start(TYPE_CODE)
    # see pypy.interpreter.pycode for the layout
    x = space.interp_w(PyCode, w_pycode)
    m.put_int(x.co_argcount)
    m.put_int(x.co_kwonlyargcount)
    m.put_int(x.co_nlocals)
    m.put_int(x.co_stacksize)
    m.put_int(x.co_flags)
    m.atom_str(TYPE_STRING, x.co_code)
    m.put_tuple_w(TYPE_TUPLE, x.co_consts_w[:])
    m.atom_strlist(TYPE_TUPLE, TYPE_STRING, [space.str_w(w_name) for w_name in x.co_names_w])
    m.atom_strlist(TYPE_TUPLE, TYPE_STRING, x.co_varnames)
    m.atom_strlist(TYPE_TUPLE, TYPE_STRING, x.co_freevars)
    m.atom_strlist(TYPE_TUPLE, TYPE_STRING, x.co_cellvars)
    m.atom_str(TYPE_STRING, x.co_filename)
    m.atom_str(TYPE_STRING, x.co_name)
    m.put_int(x.co_firstlineno)
    m.atom_str(TYPE_STRING, x.co_lnotab)

model.MM.marshal_w.register(marshal_w_pycode, PyCode)

# helper for unmarshalling string lists of code objects.
# unfortunately they now can be interned or referenced,
# so we no longer can handle it in interp_marshal.atom_strlist

def unmarshal_str(u):
    w_obj = u.get_w_obj()
    try:
        return u.space.bytes_w(w_obj)
    except OperationError, e:
        if e.match(u.space, u.space.w_TypeError):
            u.raise_exc('invalid marshal data for code object')
        else:
            raise

def unmarshal_str0(u):
    w_obj = u.get_w_obj()
    try:
        return u.space.bytes0_w(w_obj)
    except OperationError, e:
        if e.match(u.space, u.space.w_TypeError):
            u.raise_exc('invalid marshal data for code object')
        else:
            raise

def unmarshal_strlist(u, tc):
    lng = u.atom_lng(tc)
    res = [None] * lng
    idx = 0
    space = u.space
    while idx < lng:
        res[idx] = unmarshal_str(u)
        idx += 1
    return res

def unmarshal_pycode(space, u, tc):
    argcount    = u.get_int()
    kwonlyargcount = u.get_int()
    nlocals     = u.get_int()
    stacksize   = u.get_int()
    flags       = u.get_int()
    code        = unmarshal_str(u)
    u.start(TYPE_TUPLE)
    consts_w    = u.get_tuple_w()
    # copy in order not to merge it with anything else
    names       = unmarshal_strlist(u, TYPE_TUPLE)
    varnames    = unmarshal_strlist(u, TYPE_TUPLE)
    freevars    = unmarshal_strlist(u, TYPE_TUPLE)
    cellvars    = unmarshal_strlist(u, TYPE_TUPLE)
    filename    = unmarshal_str0(u)
    name        = unmarshal_str(u)
    firstlineno = u.get_int()
    lnotab      = unmarshal_str(u)
    code = PyCode(space, argcount, kwonlyargcount, nlocals, stacksize, flags,
                  code, consts_w[:], names, varnames, filename,
                  name, firstlineno, lnotab, freevars, cellvars)
    return space.wrap(code)
register(TYPE_CODE, unmarshal_pycode)

def marshal_w__Unicode(space, w_unicode, m):
    s = unicodehelper.PyUnicode_EncodeUTF8(space, space.unicode_w(w_unicode),
                                           allow_surrogates=True)
    m.atom_str(TYPE_UNICODE, s)

def unmarshal_Unicode(space, u, tc):
    return space.wrap(
        unicodehelper.PyUnicode_DecodeUTF8(space, u.get_str(),
                                           allow_surrogates=True))
register(TYPE_UNICODE, unmarshal_Unicode)

app = gateway.applevel(r'''
    def tuple_to_set(datalist, frozen=False):
        if frozen:
            return frozenset(datalist)
        return set(datalist)
''')

tuple_to_set = app.interphook('tuple_to_set')

# not directly supported:
def marshal_w_set(space, w_set, m):
    # cannot access this list directly, because it's
    # type is not exactly known through applevel.
    lis_w = space.fixedview(w_set)
    m.put_tuple_w(TYPE_SET, lis_w)

handled_by_any.append( ('set', marshal_w_set) )

# not directly supported:
def marshal_w_frozenset(space, w_frozenset, m):
    lis_w = space.fixedview(w_frozenset)
    m.put_tuple_w(TYPE_FROZENSET, lis_w)

handled_by_any.append( ('frozenset', marshal_w_frozenset) )

def unmarshal_set_frozenset(space, u, tc):
    items_w = u.get_tuple_w()
    if tc == TYPE_SET:
        w_frozen = space.w_False
    else:
        w_frozen = space.w_True
    w_tup = space.newtuple(items_w)
    return tuple_to_set(space, w_tup, w_frozen)
register(TYPE_SET + TYPE_FROZENSET, unmarshal_set_frozenset)

# dispatching for all not directly dispatched types
def marshal_w__ANY(space, w_obj, m):
    w_type = space.type(w_obj)
    for name, func in handled_by_any:
        w_t = space.builtin.get(name)
        if space.is_true(space.issubtype(w_type, w_t)):
            func(space, w_obj, m)
            return

    # any unknown object implementing the buffer protocol is
    # accepted and encoded as a plain string
    try:
        s = space.bufferstr_w(w_obj)
    except OperationError, e:
        if not e.match(space, space.w_TypeError):
            raise
    else:
        m.atom_str(TYPE_STRING, s)
        return

    raise_exception(space, "unmarshallable object")

register_all(vars())
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.