Commits

Anonymous committed 1a9cd60

Accept form feeds as whitespace.

  • Participants
  • Parent commits 947b2d1

Comments (0)

Files changed (2)

src/google/protobuf/io/tokenizer.cc

   }
 
 CHARACTER_CLASS(Whitespace, c == ' ' || c == '\n' || c == '\t' ||
-                            c == '\r' || c == '\v');
+                            c == '\r' || c == '\v' || c == '\f');
 
 CHARACTER_CLASS(Unprintable, c < ' ' && c > '\0');
 

src/google/protobuf/io/tokenizer_unittest.cc

     { Tokenizer::TYPE_SYMBOL, "\300", 0, 0 },
     { Tokenizer::TYPE_END   , ""    , 0, 1 },
   }},
+
+  // Test all whitespace chars
+  { "foo\n\t\r\v\fbar", {
+    { Tokenizer::TYPE_IDENTIFIER, "foo", 0,  0 },
+    { Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11 },
+    { Tokenizer::TYPE_END       , ""   , 1, 14 },
+  }},
 };
 
 TEST_2D(TokenizerTest, MultipleTokens, kMultiTokenCases, kBlockSizes) {