Commits

SirAlaran committed 7481c0a Merge

Merged changes

  • Participants
  • Parent commits 49c43c8, 9a57e09

Comments (0)

Files changed (3)

 import tokenrange;
 
 /**
- * Returns: true if  input is a access attribute
+ * Returns: true if input is a access attribute
  */
 pure bool isAccessAttribute(const TokenType input)
 {
 		"function" : TokenType.tFunction,
 		"goto" : TokenType.tGoto,
 		"idouble" : TokenType.tIdouble,
-		"if " : TokenType.tIf,
-		"if loat" : TokenType.tIfloat,
+		"if" : TokenType.tIf,
+		"ifloat" : TokenType.tIfloat,
 		"immutable" : TokenType.tImmutable,
 		"import" : TokenType.tImport,
 		"in" : TokenType.tIn,
 		"struct" : TokenType.tStruct,
 		"string" : TokenType.tString,
 		"super" : TokenType.tSuper,
-		"switch " : TokenType.tSwitch,
+		"switch" : TokenType.tSwitch,
 		"synchronized" : TokenType.tSynchronized,
 		"template" : TokenType.tTemplate,
 		"this" : TokenType.tThis,
 		"void" : TokenType.tVoid,
 		"volatile" : TokenType.tVolatile,
 		"wchar" : TokenType.tWchar,
-		"while " : TokenType.tWhile,
+		"while" : TokenType.tWhile,
 		"with" : TokenType.tWith,
 		"__FILE__" : TokenType.t__FILE__,
 		"__LINE__" : TokenType.t__LINE__,
 				continue;
 			case TokenType.tConst:
 			case TokenType.tAlign:
+			case TokenType.tShared:
 			case TokenType.tImmutable:
 			case TokenType.tInout:
 				tokens ~= range.popFront();
 				currentToken.type = lookupTokenType(currentToken.value);
 		}
 
-		while (finish < str.length)
+		switch (str[finish])
 		{
-			switch (str[finish])
-			{
-				mixin(singleDoubleOrAssign!('+', TokenType.plus, TokenType.uPlus, TokenType.plusEquals));
-				mixin(singleDoubleOrAssign!('-', TokenType.minus, TokenType.uMinus, TokenType.minusEquals));
-				mixin(singleDoubleOrAssign!('&', TokenType.bitAnd, TokenType.lAnd, TokenType.bitAndEquals));
-				mixin(singleDoubleOrAssign!('|', TokenType.bitOr, TokenType.lOr, TokenType.bitOrEquals));
-				mixin(singleOrAssign!('%', TokenType.mod, TokenType.modEquals));
-				mixin(singleOrAssign!('*', TokenType.star, TokenType.mulEquals));
-				mixin(singleOrAssign!('~', TokenType.tilde, TokenType.catEquals));
-				mixin(singleCharToken!('$', TokenType.dollar));
-				mixin(singleCharToken!('#', TokenType.hash));
-				mixin(singleCharToken!(';', TokenType.semicolon));
-				mixin(singleCharToken!(':', TokenType.colon));
-				mixin(singleCharToken!(',', TokenType.comma));
-				mixin(singleCharToken!('(', TokenType.lParen));
-				mixin(singleCharToken!(')', TokenType.rParen));
-				mixin(singleCharToken!('{', TokenType.lBrace));
-				mixin(singleCharToken!('}', TokenType.rBrace));
-				mixin(singleCharToken!('[', TokenType.lBracket));
-				mixin(singleCharToken!(']', TokenType.rBracket));
-				case '0': .. case '9':
+			mixin(singleDoubleOrAssign!('+', TokenType.plus, TokenType.uPlus, TokenType.plusEquals));
+			mixin(singleDoubleOrAssign!('-', TokenType.minus, TokenType.uMinus, TokenType.minusEquals));
+			mixin(singleDoubleOrAssign!('&', TokenType.bitAnd, TokenType.lAnd, TokenType.bitAndEquals));
+			mixin(singleDoubleOrAssign!('|', TokenType.bitOr, TokenType.lOr, TokenType.bitOrEquals));
+			mixin(singleOrAssign!('%', TokenType.mod, TokenType.modEquals));
+			mixin(singleOrAssign!('*', TokenType.star, TokenType.mulEquals));
+			mixin(singleOrAssign!('~', TokenType.tilde, TokenType.catEquals));
+			mixin(singleCharToken!('$', TokenType.dollar));
+			mixin(singleCharToken!('#', TokenType.hash));
+			mixin(singleCharToken!(';', TokenType.semicolon));
+			mixin(singleCharToken!(':', TokenType.colon));
+			mixin(singleCharToken!(',', TokenType.comma));
+			mixin(singleCharToken!('(', TokenType.lParen));
+			mixin(singleCharToken!(')', TokenType.rParen));
+			mixin(singleCharToken!('{', TokenType.lBrace));
+			mixin(singleCharToken!('}', TokenType.rBrace));
+			mixin(singleCharToken!('[', TokenType.lBracket));
+			mixin(singleCharToken!(']', TokenType.rBracket));
+			case '0': .. case '9':
 
-					// If this digit is part of an identifier, continue treating
-					// it as an identifier
-					if (finish > start)
-						goto default;
+				// If this digit is part of an identifier, continue treating
+				// it as an identifier
+				if (finish > start)
+					goto default;
 
-					bool foundDot = false;
-					bool foundX = false;
-					// Treat this as a number
-					numberLoop: while (true)
+				bool foundDot = false;
+				bool foundX = false;
+				// Treat this as a number
+				numberLoop: while (true)
+				{
+					switch (str[finish])
 					{
-						switch (str[finish])
-						{
-							case '0':
-								if (!foundX)
+						case '0':
+							if (!foundX)
+							{
+								if (charAtEquals(finish+1, 'x'))
 								{
-									if (charAtEquals(finish+1, 'x'))
-									{
-										finish++;
-										foundX = true;
-									}
-									break;
+									finish++;
+									foundX = true;
 								}
-								else
-									break;
-							case '.':
-								if (foundDot)
-									goto default;
-								else
-								{
-									foundDot = true;
-									break;
-								}
-							case '1': .. case '9':
-							case '_':
+							}
+							break;
+						case '.':
+							if (foundDot)
+								break numberLoop;
+							else
+							{
+								foundDot = true;
 								break;
-							case 'f':
-							case 'l':
-							default:
-								break numberLoop;
-						}
-						++finish;
+							}
+						case '1': .. case '9':
+						case '_':
+							break;
+						case 'f':
+						case 'l':
+						default:
+							break numberLoop;
 					}
-					currentToken.type = TokenType.numberLiteral;
-					return;
-				case '@':
-					if (str[start..$].startsWith("@disable"))
+					++finish;
+				}
+				currentToken.type = TokenType.numberLiteral;
+				return;
+			case '=':
+				if (charAtEquals(finish + 1, '='))
+				{
+					currentToken.type = TokenType.equals;
+					finish += 2;
+				}
+				else if (charAtEquals(finish + 1, '>'))
+				{
+					currentToken.type = TokenType.goesTo;
+					finish +=2;
+				}
+				else
+				{
+					currentToken.type = TokenType.assign;
+					finish++;
+				}
+				return;
+			case '^':
+				if (charAtEquals(finish + 1, '^'))
+				{
+					if (charAtEquals(finish + 2, '='))
 					{
-						finish += 6;
-						currentToken.type = TokenType.atDisable;
-					}
-					else if (str[start..$].startsWith("@safe"))
-					{
+						currentToken.type = TokenType.powEquals;
 						finish += 3;
-						currentToken.type = TokenType.atSafe;
-					}
-					else if (str[start..$].startsWith("@trusted"))
-					{
-						finish += 4;
-						currentToken.type = TokenType.atTrusted;
-					}
-					else if (str[start..$].startsWith("@property"))
-					{
-						finish += 7;
-						currentToken.type = TokenType.atProperty;
-					}
-					else if (str[start..$].startsWith("@system"))
-					{
-						finish += 5;
-						currentToken.type = TokenType.atSystem;
 					}
 					else
 					{
-						while (!isSeparating(str[finish + 1]) && finish < str.length)
-							finish++;
-						currentToken.type = TokenType.identifier;
-					}
-					return;
-				case '=':
-					if (charAtEquals(finish + 1, '='))
-					{
-						currentToken.type = TokenType.equals;
+						currentToken.type = TokenType.pow;
 						finish += 2;
 					}
-					else if (charAtEquals(finish + 1, '>'))
+				}
+				else if  (charAtEquals(finish + 1, '='))
+				{
+					currentToken.type = TokenType.xorEquals;
+					finish += 2;
+				}
+				else
+				{
+					currentToken.type = TokenType.xor;
+					finish += 1;
+				}
+				return;
+			case '/':
+				if (charAtEquals(finish + 1, '/') || charAtEquals(finish + 1, '*')
+					|| charAtEquals(finish + 1, '+'))
+				{
+					if (iterationStyle == IterationStyle.CODE_ONLY)
 					{
-						currentToken.type = TokenType.goesTo;
-						finish +=2;
+						skipComments(finish, str, lineNumber);
+						skipWhitespace(finish, str, lineNumber);
+						advance();
+						return;
 					}
 					else
 					{
-						currentToken.type = TokenType.assign;
-						finish++;
+						skipComments(finish, str, lineNumber);
+						currentToken.type = TokenType.comment;
+						return;
 					}
+				}
+				else if (charAtEquals(finish + 1, '='))
+				{
+					currentToken.type = TokenType.divEquals;
+					finish += 2;
 					return;
-				case '^':
-					if (charAtEquals(finish + 1, '^'))
+				}
+				else
+				{
+					currentToken.type = TokenType.div;
+					finish += 1;
+					return;
+				}
+			case '`':
+			case '\'':
+			case '"':
+				skipString(finish, str, lineNumber, str[finish]);
+				currentToken.type = TokenType.stringLiteral;
+				return;
+			case '>':
+				if (charAtEquals(finish + 1, '>'))
+				{
+					if  (charAtEquals(finish + 2, '>'))
 					{
-						if (charAtEquals(finish + 2, '='))
+						if (charAtEquals(finish + 3, '='))
 						{
-							currentToken.type = TokenType.powEquals;
+							currentToken.type = TokenType.unsignedShiftRightEqual;
 							finish += 3;
 						}
 						else
 						{
-							currentToken.type = TokenType.pow;
+							currentToken.type = TokenType.unsignedShiftRight;
 							finish += 2;
 						}
 					}
-					else if  (charAtEquals(finish + 1, '='))
+					else if (charAtEquals(finish + 2, '='))
 					{
-						currentToken.type = TokenType.xorEquals;
+						currentToken.type = TokenType.shiftRightEqual;
 						finish += 2;
 					}
 					else
 					{
-						currentToken.type = TokenType.xor;
+						currentToken.type = TokenType.shiftRight;
 						finish += 1;
 					}
-					break;
-				case '/':
-					if (charAtEquals(finish + 1, '/') || charAtEquals(finish + 1, '*')
-						|| charAtEquals(finish + 1, '+'))
+				}
+				else if (charAtEquals(finish + 1, '='))
+					currentToken.type = TokenType.greaterEqual;
+				else
+					currentToken.type = TokenType.greater;
+				finish++;
+				return;
+			case '<':
+				if (charAtEquals(finish + 1, '<'))
+				{
+					if (charAtEquals(finish + 2, '='))
 					{
-						if (iterationStyle == IterationStyle.CODE_ONLY)
+						currentToken.type = TokenType.shiftLeftEqual;
+						finish += 3;
+					}
+					else
+					{
+						currentToken.type = TokenType.shiftLeft;
+						finish += 2;
+					}
+				}
+				else if (charAtEquals(finish + 1, '>') && charAtEquals(finish + 2, '='))
+				{
+					currentToken.type = TokenType.lessEqualGreater;
+					finish += 3;
+				}
+				else if (charAtEquals(finish + 1, '='))
+				{
+					currentToken.type = TokenType.lessEqual;
+					finish += 2;
+				}
+				else
+				{
+					currentToken.type = TokenType.less;
+					finish += 1;
+				}
+				return;
+			case '!':
+				if (charAtEquals(finish + 1, '<'))
+				{
+					if (charAtEquals(finish + 2, '>'))
+					{
+						if (charAtEquals(finish + 3, '='))
 						{
-							skipComments(finish, str, lineNumber);
-							skipWhitespace(finish, str, lineNumber);
+							currentToken.type = TokenType.notLessEqualGreater;
+							finish += 4;
 						}
 						else
 						{
-							skipComments(finish, str, lineNumber);
-							currentToken.type = TokenType.comment;
-							return;
-						}
-						start = finish;
-						continue;
-					} else if (charAtEquals(finish + 1, '=')) {
-						currentToken.type = TokenType.divEquals;
-						finish += 2;
-						return;
-					} else {
-						currentToken.type = TokenType.div;
-						finish += 1;
-						return;
-					}
-				case '`':
-				case '\'':
-				case '"':
-					skipString(finish, str, lineNumber, str[finish]);
-					currentToken.type = TokenType.stringLiteral;
-					return;
-				case '>':
-					if (charAtEquals(finish + 1, '>'))
-					{
-						if  (charAtEquals(finish + 2, '>'))
-						{
-							if (charAtEquals(finish + 3, '='))
-							{
-								currentToken.type = TokenType.unsignedShiftRightEqual;
-								finish += 4;
-							}
-							else
-							{
-								currentToken.type = TokenType.unsignedShiftRight;
-								finish += 3;
-							}
-						}
-						else if (charAtEquals(finish + 2, '='))
-						{
-							currentToken.type = TokenType.shiftRightEqual;
+							currentToken.type = TokenType.notLessEqual;
 							finish += 3;
 						}
-						else
-						{
-							currentToken.type = TokenType.shiftRight;
-							finish += 2;
-						}
 					}
-					else if (charAtEquals(finish + 1, '='))
+					else if  (charAtEquals(finish + 2, '='))
 					{
-						currentToken.type = TokenType.greaterEqual;
-						finish += 1;
+						currentToken.type = TokenType.notLessEqual;
+						finish += 3;
 					}
 					else
 					{
-						currentToken.type = TokenType.greater;
-						finish += 1;
+						currentToken.type = TokenType.notLess; // "!<"
+						finish += 2;
 					}
-					return;
-				case '<':
-					if (charAtEquals(finish + 1, '<'))
+				}
+				else if (charAtEquals(finish + 1, '>'))
+				{
+					if (charAtEquals(finish + 2, '='))
 					{
-						if (charAtEquals(finish + 2, '='))
-						{
-							currentToken.type = TokenType.shiftLeftEqual;
-							finish += 3;
-						}
-						else
-						{
-							currentToken.type = TokenType.shiftLeft;
-							finish += 2;
-						}
-					}
-					else if (charAtEquals(finish + 1, '>') && charAtEquals(finish + 2, '='))
-					{
-						currentToken.type = TokenType.lessEqualGreater;
+						currentToken.type = TokenType.notGreaterEqual;
 						finish += 3;
 					}
-					else if (charAtEquals(finish + 1, '='))
+					else
 					{
-						currentToken.type = TokenType.lessEqual;
+						currentToken.type = TokenType.notGreater;
+						finish += 2;
+					}
+				}
+				else if (charAtEquals(finish + 1, '='))
+				{
+					currentToken.type = TokenType.notEquals;
+					finish += 2;
+				}
+				else
+				{
+					currentToken.type = TokenType.not;
+					finish += 1;
+				}
+				return;
+			case '.':
+				if (charAtEquals(finish + 1, '.'))
+				{
+					if (charAtEquals(finish + 2, '.'))
+					{
+						currentToken.type = TokenType.vararg;
 						finish += 2;
 					}
 					else
 					{
-						currentToken.type = TokenType.less;
+						currentToken.type = TokenType.slice;
 						finish += 1;
 					}
+				}
+				else
+					currentToken.type = TokenType.dot;
+				++finish;
+				return;
+			case 'q':
+				if (charAtEquals(finish + 1, '\"'))
+				{
+					finish++;
+					skipString(finish, str, lineNumber);
+					currentToken.type = TokenType.stringLiteral;
 					return;
-				case '!':
-					if (charAtEquals(finish + 1, '<'))
-					{
-						if (charAtEquals(finish + 2, '>'))
-						{
-							if (charAtEquals(finish + 3, '='))
-							{
-								currentToken.type = TokenType.notLessEqualGreater;
-								finish += 4;
-							}
-							else
-							{
-								currentToken.type = TokenType.notLessEqual;
-								finish += 3;
-							}
-						}
-						else if  (charAtEquals(finish + 2, '='))
-						{
-							currentToken.type = TokenType.notLessEqual;
-							finish += 3;
-						}
-						else
-						{
-							currentToken.type = TokenType.notLess; // "!<"
-							finish += 2;
-						}
-					}
-					else if (charAtEquals(finish + 1, '>'))
-					{
-						if (charAtEquals(finish + 2, '='))
-						{
-							currentToken.type = TokenType.notGreaterEqual;
-							finish += 3;
-						}
-						else
-						{
-							currentToken.type = TokenType.notGreater;
-							finish += 2;
-						}
-					}
-					else if (charAtEquals(finish + 1, '='))
-					{
-						currentToken.type = TokenType.notEquals;
-						finish += 2;
-					}
-					else
-					{
-						currentToken.type = TokenType.not;
-						finish += 1;
-					}
-					return;
-				case '.':
-					if (charAtEquals(finish + 1, '.'))
-					{
-						if (charAtEquals(finish + 2, '.'))
-						{
-							currentToken.type = TokenType.vararg;
-							finish += 2;
-						}
-						else
-						{
-							currentToken.type = TokenType.slice;
-							finish += 1;
-						}
-					}
-					else
-						currentToken.type = TokenType.dot;
+				}
+				goto default;
+			case '@':
+				++finish;
+				goto default;
+			default:
+				currentToken.type = TokenType.blank;
+				while (finish + 1 < str.length && !isSeparating(str[finish + 1]))
 					++finish;
-					return;
-				case 'q':
-					if (charAtEquals(finish + 1, '\"'))
-					{
-						finish++;
-						skipString(finish, str, lineNumber);
-						currentToken.type = TokenType.stringLiteral;
-						return;
-					}
-					goto default;
-				default:
-					++finish;
-					if (finish >= str.length)
-					{
-						done = true;
-						return;
-					}
-					if (isSeparating(str[finish]))
-						return;
-					break;
-			}
+				++finish;
+				return;
 		}
-		done = true;
 	}
 
 	string str;
 
 unittest
 {
-	auto t = byDToken("test1test @safe.test1234\n<<=>>>=(+/-=/=options...)=>===");
+	auto t = byDToken("test1test @safe.test1234\n<<=>>>=(+/-=/=options...)=>===if*while");
 	assert(t.front().value == "test1test" && t.popFront() == TokenType.identifier);
 	assert(t.front().value == "@safe" && t.popFront() == TokenType.atSafe);
 	assert(t.front().value == "." && t.popFront() == TokenType.dot, t.front().value);
 	assert(t.popFront() == TokenType.goesTo, t.front().value);
 	assert(t.popFront() == TokenType.equals, t.front().value);
 	assert(t.popFront() == TokenType.assign, t.front().value);
+	assert(t.popFront() == TokenType.tIf);
+	assert(t.popFront() == TokenType.star);
+	assert(t.popFront() == TokenType.tWhile);
 }
 
 private class BalancedRange : TokenRange