Commits

Stephen McKamey committed 752c0c8

- porting rough cut of unparsed block tokenization logic

  • Participants
  • Parent commits a49de00

Comments (0)

Files changed (3)

File src/org/duelengine/duel/parsing/DuelGrammar.java

 	public static final char OP_ENTITY_HEX = 'x';
 	public static final char OP_ENTITY_HEX_ALT = 'X';
 	public static final char OP_ENTITY_END = ';';
-
-	public static final char OP_COMMENT = '!';
-	public static final char OP_COMMENT_DELIM = '-';
-	public static final String OP_COMMENT_BEGIN = "--";
-	public static final String OP_COMMENT_END = "--";
-	public static final String OP_CDATA_BEGIN = "[CDATA[";
-	public static final String OP_CDATA_END = "]]";
 }

File src/org/duelengine/duel/parsing/DuelLexer.java

 		}
 	}
 
+	/**
+	 * Gets the current line within the input
+	 * @return
+	 */
 	public int getLine() {
 		return this.line;
 	}
 
+	/**
+	 * Gets the current column within the input
+	 * @return
+	 */
 	public int getColumn() {
 		return this.column;
 	}
 
+	/**
+	 * Gets the current index within the input 
+	 * @return
+	 */
 	public int getIndex() {
 		return this.index;
 	}
 					case UNPARSED:
 						switch (this.ch) {
 							case DuelGrammar.OP_ELEM_BEGIN:
-								if (this.tryScanUnparsedBlock(false) || this.tryScanTag()) {
+								if (this.tryScanBlock(false) || this.tryScanTag()) {
 									return this.token;
 								}
 								break;
 
 
 	/**
-
-	 * Tries to scan the next token as an unparsed block
-	 * @return true if block token was found
-	 * @throws IOException
-	 */
-	private boolean tryScanUnparsedBlock(boolean asAttr)
-		throws IOException {
-
-		this.setMark(3);
-
-		// TODO
-		this.resetMark();
-		return false;
-	}
-
-	/**
 	 * Tries to scan the next token as a tag
 	 * @return true if tag token was found
 	 * @throws IOException
 				delim = DuelGrammar.OP_ATTR_DELIM;
 		}
 
-		if (!this.tryScanUnparsedBlock(true)) {
+		if (this.ch != DuelGrammar.OP_ELEM_BEGIN || !this.tryScanBlock(true)) {
 			this.scanAttrLiteral(delim);
 		}
 
 	}
 
 	/**
+	 * Tries to scan the next token as an unparsed block
+	 * @return true if block token was found
+	 * @throws IOException
+	 */
+	private boolean tryScanBlock(boolean asAttr)
+		throws IOException {
+
+		// mark current position with capacity to check start delims
+		final int CAPACITY = 16;
+		this.setMark(CAPACITY);
+
+		String value = null;
+		String begin = null;
+		String end = null;
+
+		// '<'
+		switch (this.nextChar()) {
+
+			case '%':
+				switch (this.nextChar()) {
+					case '-':	// "<%--", "--%>"		ASP/PSP/JSP-style code comment
+						begin = "<%--";
+						end = "--%>";
+						value = this.scanBlockValue("--", end);
+						break;
+
+					case '@':	// "<%@",  "%>"			ASP/PSP/JSP directive
+					case '=':	// "<%=",  "%>"			ASP/PSP/JSP/Duel expression
+					case '!':	// "<%!",  "%>"			JSP/JBST declaration
+					case '#':	// "<%#",  "%>"			ASP.NET/Duel databind expression
+					case '$':	// "<%$",  "%>"			ASP.NET/Duel extension
+					case ':':	// "<%:",  "%>"			ASP.NET 4 HTML-encoded expression
+
+						begin = "<%"+(char)this.ch;
+						end = "%>";
+						value = this.scanBlockValue(""+(char)this.ch, end);
+						break;
+
+					default:
+						begin = "<%";
+						end = "%>";
+						value = this.scanBlockValue("", end);
+						break;
+				}
+				break;
+
+			case '!':
+				switch (this.nextChar()) {
+					case '-':	// "<!--", "-->"		XML/HTML/SGML comment
+						begin = "<!--";
+						end = "-->";
+						value = this.scanBlockValue("--", end);
+						break;
+
+					case '[':	// "<![CDATA[", "]]>"	CDATA section
+						value = this.scanBlockValue("[CDATA[", "]]>");
+						if (value != null) {
+							// unwrap CDATA as literal text
+							this.token = asAttr ? DuelToken.AttrValue(value) : DuelToken.Literal(value);
+							return true;
+						}
+						break;
+
+					default:	// "<!", ">"			SGML declaration (e.g. DOCTYPE or server-side include)
+						begin = "<!";
+						end = ">";
+						value = this.scanBlockValue("!", ">");
+						break;
+				}
+				break;
+		}
+
+		if (value == null) {
+			// NOTE: this may throw an exception if block was unterminated
+			this.resetMark();
+			return false;
+		}
+
+		if (this.ch == DuelGrammar.OP_ELEM_END) {
+			this.nextChar();
+		}
+
+		UnparsedBlock block = new UnparsedBlock(begin, end, value);
+		this.token = asAttr ? DuelToken.AttrValue(block) : DuelToken.Unparsed(block);
+		return true;
+	}
+
+	private String scanBlockValue(String begin, String end)
+		throws IOException {
+
+		for (int i=0, length=begin.length(); i<length; i++) {
+			if (this.ch != begin.charAt(i)) {
+				// didn't match begin delim
+				return null;
+			}
+
+			this.nextChar();
+		}
+
+		// reset the buffer, mark start
+		this.buffer.setLength(0);
+
+		for (int i=0, length=end.length(); this.ch != DuelGrammar.EOF; ) {
+			// check each char
+			if (this.ch == end.charAt(i)) {
+				// move to next char
+				i++;
+				if (i >= length) {
+					length--;
+
+					// trim ending delim from buffer
+					this.buffer.setLength(this.buffer.length() - length);
+					return this.buffer.toString();
+				}
+			} else {
+				// reset to start of delim
+				i = 0;
+			}
+
+			this.buffer.append((char)this.ch);
+			this.nextChar();
+		}
+
+		// TODO: determine better exception type
+		throw new IOException("Unterminated block");
+	}
+
+	/**
 	 * Gets the next character in the input and updates statistics
 	 * @return
 	 * @throws IOException

File test/org/duelengine/duel/parsing/DuelLexerTests.java

 		assertArrayEquals(expected, actual);
 	}
 
+	@Test
+	public void codeSimpleTest() {
+
+		String input = "<% code block %>";
+
+		Object[] expected = {
+				DuelToken.Unparsed(new UnparsedBlock("<%", "%>", " code block ")),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+
+		assertArrayEquals(expected, actual);
+	}
+
+	@Test
+	public void codeAttributeTest() {
+
+		String input = "<a href=\"<%= simple expr %>\">foo</a>";
+
+		Object[] expected = {
+				DuelToken.ElemBegin("a"),
+				DuelToken.AttrName("href"),
+				DuelToken.AttrValue(new UnparsedBlock("<%=", "%>", " simple expr ")),
+				DuelToken.Literal("foo"),
+				DuelToken.ElemEnd("a"),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+
+		assertArrayEquals(expected, actual);
+	}
+
 	private void dumpList(String label, Object[] tokens) {
 		System.out.println();
 		System.out.print(label+":");