|
@@ -403,6 +403,13 @@ MultiTokenCase kMultiTokenCases[] = {
|
|
|
{ Tokenizer::TYPE_SYMBOL, "\300", 0, 0 },
|
|
|
{ Tokenizer::TYPE_END , "" , 0, 1 },
|
|
|
}},
|
|
|
+
|
|
|
+ // Test all whitespace chars
|
|
|
+ { "foo\n\t\r\v\fbar", {
|
|
|
+ { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0 },
|
|
|
+ { Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11 },
|
|
|
+ { Tokenizer::TYPE_END , "" , 1, 14 },
|
|
|
+ }},
|
|
|
};
|
|
|
|
|
|
TEST_2D(TokenizerTest, MultipleTokens, kMultiTokenCases, kBlockSizes) {
|