Ned Batchelder committed 8e297f7

Add a test case for utf-8 string contents.

Comments (0)

Files changed (2)

                                 void while with
                                 """, suffix=r"\b"), next='reg'),
         Tok("reserved",     literals("null true false", suffix=r"\b"), next='div'),
-        Tok("id",           r"([a-zA-Z_$]|\\u[0-9a-fA-Z]{4})([a-zA-Z_$0-9]|\\u[0-9a-fA-F]{4})*", next='div'),
+        Tok("id",           r"""
+                            ([a-zA-Z_$   ]|\\u[0-9a-fA-Z]{4})       # first char
+                            ([a-zA-Z_$0-9]|\\u[0-9a-fA-F]{4})*      # rest chars
+                            """, next='div'),
         Tok("hnum",         r"0[xX][0-9a-fA-F]+", next='div'),
         Tok("onum",         r"0[0-7]+"),
         Tok("dnum",         r"""
 # Tests for jslex
+# encoding: utf-8
 import difflib, unittest
 from jslex import JsLexer, js_to_c_for_gettext
   "Multi-line strings are unequal:\n" + message)
     def assertListsEqual(self, first, second):
+        """Assert that two lists are equal, with a nice diff output if not."""
         if first != second:
             lines1 = [repr(e) for e in first]
             lines2 = [repr(e) for e in second]
   "Lists are unequal:\n" + message)
 class JsTokensTest(JsLexTestCase):
     LEX_CASES = [
         # ids
         (''' 'hello' "hello" ''', ["string 'hello'", 'string "hello"']),
         (r""" 'don\'t' "don\"t" '"' "'" '\'' "\"" """, 
          [r"""string 'don\'t'""", r'''string "don\"t"''', r"""string '"'""", r'''string "'"''', r"""string '\''""", r'''string "\""''']),
+        (ur'"ƃuıxǝ⅂ ʇdıɹɔsɐʌɐſ\""', [ur'string "ƃuıxǝ⅂ ʇdıɹɔsɐʌɐſ\""']),
         # comments
         ("a//b", ["id a", "linecomment //b"]),
         ("/****/a/=2//hello", ["comment /****/", "id a", "punct /=", "dnum 2", "linecomment //hello"]),
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.