Commits

masklinn committed 801b0ed

Fix issue with tokenizer failing on trailing spaces

Comments (0)

Files changed (2)

             while(pseudoprog.lastIndex < max) {
                 var pseudomatch = pseudoprog.exec(s);
                 if (!pseudomatch) {
-                    throw new Error('Failed to tokenize ' + s
-                                    + ' at index ' + (end || 0)
+                    // if match failed on trailing whitespace, end tokenizing
+                    if (/^\s+$/.test(s.slice(end))) {
+                        break;
+                    }
+                    throw new Error('Failed to tokenize <<' + s
+                                    + '>> at index ' + (end || 0)
                                     + '; parsed so far: ' + tokens);
                 }
 
             expect(toks).to.have.tokens(1);
             expect(toks[0]).to.be.constant('False');
         });
+        it('does not fuck up on trailing spaces', function () {
+            var toks = py.tokenize('None ');
+            expect(toks).to.have.tokens(1);
+            expect(toks[0]).to.be.constant('None');
+        });
     });
     describe('collections', function () {
         it('tokenizes opening and closing symbols', function () {