Commits

Amaury Forgeot d'Arc  committed ef504fc

space.wrap("xxx") now returns a unicode string!
space.str_w(w_someunicode) return a RPython bytestring!
(use space.wrapbytes and space.bytes_w to get the previous behavior)

  • Participants
  • Parent commits 2693bfc
  • Branches py3k

Comments (0)

Files changed (19)

File pypy/interpreter/astcompiler/ast.py

     _col_offset_mask = 8
 
     def __init__(self, id, ctx, lineno, col_offset):
+        assert isinstance(id, str)
         self.id = id
         self.ctx = ctx
         expr.__init__(self, lineno, col_offset)

File pypy/interpreter/astcompiler/astbuilder.py

         return name
 
     def handle_arguments(self, arguments_node):
+        # This function handles both typedargslist (function definition)
+        # and varargslist (lambda definition).
         if arguments_node.type == syms.parameters:
             if len(arguments_node.children) == 2:
                 return ast.arguments(None, None, None, None)
         while i < child_count:
             argument = arguments_node.children[i]
             arg_type = argument.type
-            if arg_type == syms.tfpdef:
+            if arg_type == syms.tfpdef or arg_type == syms.vfpdef:
                 parenthesized = False
                 complex_args = False
                 while True:

File pypy/interpreter/baseobjspace.py

 
     # -------------------------------------------------------------------
 
-    def str_w(self, space):
-        w_msg = typed_unwrap_error_msg(space, "string", self)
+    def bytes_w(self, space):
+        w_msg = typed_unwrap_error_msg(space, "bytes", self)
         raise OperationError(space.w_TypeError, w_msg)
 
     def unicode_w(self, space):
         raise OperationError(space.w_TypeError,
-                             typed_unwrap_error_msg(space, "unicode", self))
+                             typed_unwrap_error_msg(space, "string", self))
 
     def int_w(self, space):
         raise OperationError(space.w_TypeError,
         return self.str_w(w_obj)
 
     def str_w(self, w_obj):
-        return w_obj.str_w(self)
+        return self.unicode_w(w_obj).encode('ascii')
+
+    def bytes_w(self, w_obj):
+        return w_obj.bytes_w(self)
 
     def int_w(self, w_obj):
         return w_obj.int_w(self)
 
 ObjSpace.IrregularOpTable = [
     'wrap',
-    'str_w',
+    'bytes_w',
     'int_w',
     'float_w',
     'uint_w',

File pypy/interpreter/error.py

             exc_value    = str(w_value)
         else:
             w = space.wrap
-            if space.is_w(space.type(self.w_type), space.w_str):
+            if space.is_w(space.type(self.w_type), space.w_text):
                 exc_typename = space.str_w(self.w_type)
             else:
                 exc_typename = space.str_w(

File pypy/interpreter/pyopcode.py

     def IMPORT_NAME(self, nameindex, next_instr):
         space = self.space
         w_modulename = self.getname_w(nameindex)
-        modulename = self.space.str_w(w_modulename)
         w_fromlist = self.popvalue()
 
         w_flag = self.popvalue()
         w_locals = self.w_locals
         if w_locals is None:            # CPython does this
             w_locals = space.w_None
-        w_modulename = space.wrap(modulename)
         w_globals = self.w_globals
         if w_flag is None:
             w_obj = space.call_function(w_import, w_modulename, w_globals,

File pypy/interpreter/pyparser/parsestring.py

     rawmode = False
     unicode = True
 
+    assert isinstance(s, str)
+
     # string decoration handling
     o = ord(quote)
     isalpha = (o>=97 and o<=122) or (o>=65 and o<=90)
             w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
             return w_v
         else:
-            return space.wrap(substr)
+            return space.wrapbytes(substr)
 
     enc = None
     if need_encoding:
          enc = encoding
     v = PyString_DecodeEscape(space, substr, enc)
-    return space.wrap(v)
+    return space.wrapbytes(v)
 
 def hexbyte(val):
     result = "%x" % val

File pypy/interpreter/pyparser/pyparse.py

 from pypy.interpreter.astcompiler import consts
 
 
-_recode_to_utf8 = gateway.applevel(r'''
-    def _recode_to_utf8(text, encoding):
-        return unicode(text, encoding).encode("utf-8")
-''').interphook('_recode_to_utf8')
+def decode_source(space, bytes, encoding=None):
+    if encoding is None:
+        encoding = 'utf-8'
+    if encoding == 'utf-8':
+        return bytes
+    text = space.unicode_w(space.call_function(space.w_unicode,
+                                               space.wrapbytes(bytes),
+                                               space.wrap(encoding)))
+    return text.encode('utf-8')
 
-def recode_to_utf8(space, text, encoding):
-    return space.str_w(_recode_to_utf8(space, space.wrap(text),
-                                          space.wrap(encoding)))
 def _normalize_encoding(encoding):
     """returns normalized name for <encoding>
 
         self.space = space
         self.future_flags = future_flags
 
-    def parse_source(self, textsrc, compile_info):
+    def parse_source(self, bytessrc, compile_info):
         """Main entry point for parsing Python source.
 
         Everything from decoding the source to tokenizing to building the parse
         """
         # Detect source encoding.
         enc = None
-        if textsrc.startswith("\xEF\xBB\xBF"):
-            textsrc = textsrc[3:]
+        if bytessrc.startswith("\xEF\xBB\xBF"):
+            bytessrc = bytessrc[3:]
             enc = 'utf-8'
             # If an encoding is explicitly given check that it is utf-8.
-            decl_enc = _check_for_encoding(textsrc)
+            decl_enc = _check_for_encoding(bytessrc)
             if decl_enc and decl_enc != "utf-8":
                 raise error.SyntaxError("UTF-8 BOM with non-utf8 coding cookie",
                                         filename=compile_info.filename)
+            textsrc = decode_source(self.space, bytessrc, enc)
         elif compile_info.flags & consts.PyCF_SOURCE_IS_UTF8:
             enc = 'utf-8'
-            if _check_for_encoding(textsrc) is not None:
+            if _check_for_encoding(bytessrc) is not None:
                 raise error.SyntaxError("coding declaration in unicode string",
                                         filename=compile_info.filename)
+            textsrc = decode_source(self.space, bytessrc, enc)
         else:
-            enc = _normalize_encoding(_check_for_encoding(textsrc))
-            if enc is not None and enc not in ('utf-8', 'iso-8859-1'):
-                try:
-                    textsrc = recode_to_utf8(self.space, textsrc, enc)
-                except OperationError, e:
-                    # if the codec is not found, LookupError is raised.  we
-                    # check using 'is_w' not to mask potential IndexError or
-                    # KeyError
-                    space = self.space
-                    if e.match(space, space.w_LookupError):
-                        raise error.SyntaxError("Unknown encoding: %s" % enc,
-                                                filename=compile_info.filename)
-                    # Transform unicode errors into SyntaxError
-                    if e.match(space, space.w_UnicodeDecodeError):
-                        e.normalize_exception(space)
-                        w_message = space.str(e.get_w_value(space))
-                        raise error.SyntaxError(space.str_w(w_message))
-                    raise
+            enc = _normalize_encoding(_check_for_encoding(bytessrc))
+            try:
+                textsrc = decode_source(self.space, bytessrc, enc)
+            except OperationError, e:
+                # if the codec is not found, LookupError is raised.  we
+                # check using 'is_w' not to mask potential IndexError or
+                # KeyError
+                space = self.space
+                if e.match(space, space.w_LookupError):
+                    raise error.SyntaxError("Unknown encoding: %s" % enc,
+                                            filename=compile_info.filename)
+                # Transform unicode errors into SyntaxError
+                if e.match(space, space.w_UnicodeDecodeError):
+                    e.normalize_exception(space)
+                    w_message = space.str(e.get_w_value(space))
+                    raise error.SyntaxError(space.text_w(w_message))
+                raise
 
         f_flags, future_info = future.get_futures(self.future_flags, textsrc)
         compile_info.last_future_import = future_info

File pypy/module/__builtin__/compiling.py

 
     ast_node = None
     w_ast_type = space.gettypeobject(ast.AST.typedef)
-    str_ = None
+    source_str = None
     if space.is_true(space.isinstance(w_source, w_ast_type)):
         ast_node = space.interp_w(ast.mod, w_source)
         ast_node.sync_app_attrs(space)
-    elif space.is_true(space.isinstance(w_source, space.w_unicode)):
-        w_utf_8_source = space.call_method(w_source, "encode",
-                                           space.wrap("utf-8"))
-        str_ = space.str_w(w_utf_8_source)
+    else:
+        source_str = space.str_w(w_source)
         # This flag tells the parser to reject any coding cookies it sees.
         flags |= consts.PyCF_SOURCE_IS_UTF8
-    else:
-        str_ = space.str_w(w_source)
 
     ec = space.getexecutioncontext()
     if flags & ~(ec.compiler.compiler_flags | consts.PyCF_ONLY_AST |
 
     if ast_node is None:
         if flags & consts.PyCF_ONLY_AST:
-            mod = ec.compiler.compile_to_ast(str_, filename, mode, flags)
+            mod = ec.compiler.compile_to_ast(source_str, filename, mode, flags)
             return space.wrap(mod)
         else:
-            code = ec.compiler.compile(str_, filename, mode, flags)
+            code = ec.compiler.compile(source_str, filename, mode, flags)
     else:
         code = ec.compiler.compile_ast(ast_node, filename, mode, flags)
     return space.wrap(code)

File pypy/module/__builtin__/operation.py

     # space.{get,set,del}attr()...
     # Note that if w_name is already a string (or a subclass of str),
     # it must be returned unmodified (and not e.g. unwrapped-rewrapped).
-    if not space.is_w(space.type(w_name), space.w_str):
-        name = space.str_w(w_name)    # typecheck
+    if not space.is_w(space.type(w_name), space.w_text):
+        name = space.text_w(w_name)   # typecheck
         w_name = space.wrap(name)     # rewrap as a real string
     return w_name
 

File pypy/module/_codecs/interp_codecs.py

         state = space.fromcache(CodecState)
         func = getattr(runicode, rname)
         result = func(uni, len(uni), errors, state.encode_error_handler)
-        return space.newtuple([space.wrap(result), space.wrap(len(uni))])
+        return space.newtuple([space.wrapbytes(result), space.wrap(len(uni))])
     wrap_encoder.func_name = rname
     globals()[name] = wrap_encoder
 

File pypy/module/exceptions/interp_exceptions.py

 
     def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason):
         # typechecking
-        space.realstr_w(w_encoding)
-        space.realstr_w(w_object)
+        space.text_w(w_encoding)
+        space.str_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.realstr_w(w_reason)
+        space.text_w(w_reason)
         # assign attributes
         self.w_encoding = w_encoding
         self.w_object = w_object

File pypy/module/marshal/interp_marshal.py

 by dump(data, file)."""
     m = StringMarshaller(space, space.int_w(w_version))
     m.dump_w_obj(w_data)
-    return space.wrap(m.get_value())
+    return space.wrapbytes(m.get_value())
 
 def load(space, w_f):
     """Read one value from the file 'f' and return it."""

File pypy/module/sys/__init__.py

         'argv'                  : 'state.get(space).w_argv',
         'py3kwarning'           : 'space.w_False',
         'warnoptions'           : 'state.get(space).w_warnoptions', 
-        'builtin_module_names'  : 'state.w_None',
+        'builtin_module_names'  : 'space.w_None',
         'pypy_getudir'          : 'state.pypy_getudir',    # not translated
         'pypy_initial_path'     : 'state.pypy_initial_path',
 

File pypy/objspace/std/dictmultiobject.py

 
     def is_correct_type(self, w_obj):
         space = self.space
-        return space.is_w(space.type(w_obj), space.w_str)
+        return space.is_w(space.type(w_obj), space.w_text)
 
     def get_empty_storage(self):
         res = {}

File pypy/objspace/std/objspace.py

             w_type = self.gettypeobject(typedef)
             self.builtin_types[typedef.name] = w_type
             setattr(self, 'w_' + typedef.name, w_type)
+        self.w_text = self.w_unicode
         self.builtin_types["NotImplemented"] = self.w_NotImplemented
         self.builtin_types["Ellipsis"] = self.w_Ellipsis
 
         assert typedef is not None
         return self.fromcache(stdtypedef.TypeCache).getorbuild(typedef)
 
+    def wrapbytes(self, bytes):
+        return wrapstr(self, bytes)
+
     def wrap(self, x):
         "Wraps the Python value 'x' into one of the wrapper classes."
         # You might notice that this function is rather conspicuously
             else:
                 return self.newint(x)
         if isinstance(x, str):
-            return wrapstr(self, x)
+            return wrapunicode(self, x.decode('ascii'))
         if isinstance(x, unicode):
             return wrapunicode(self, x)
         if isinstance(x, float):

File pypy/objspace/std/ropeunicodeobject.py

         # for testing
         return w_self._node.flatten_unicode()
 
-    def str_w(w_self, space):
-        return space.str_w(space.str(w_self))
-
     def create_if_subclassed(w_self):
         if type(w_self) is W_RopeUnicodeObject:
             return w_self

File pypy/objspace/std/strsliceobject.py

         w_self.stop = len(str)
         return str
 
-    def str_w(w_self, space):
+    def bytes_w(w_self, space):
         return w_self.force()
 
     def __repr__(w_self):

File pypy/objspace/std/unicodeobject.py

             return w_self
         return W_UnicodeObject(w_self._value)
 
-    def str_w(self, space):
-        return space.str_w(space.str(self))
-
     def unicode_w(self, space):
         return self._value
 

File pypy/objspace/std/unicodetype.py

                     space.w_UnicodeDecodeError,
                     space.newtuple([
                     space.wrap('ascii'),
-                    space.wrap(s),
+                    space.wrapbytes(s),
                     space.wrap(i),
                     space.wrap(i+1),
                     space.wrap("ordinal not in range(128)")]))
                                        startingpos, endingpos):
         raise OperationError(space.w_UnicodeDecodeError,
                              space.newtuple([space.wrap(encoding),
-                                             space.wrap(s),
+                                             space.wrapbytes(s),
                                              space.wrap(startingpos),
                                              space.wrap(endingpos),
                                              space.wrap(msg)]))