1. Stephen McKamey
  2. duel

Commits

Stephen McKamey  committed a49de00

- porting literal attribute tokenizing logic

  • Participants
  • Parent commits 9a03f41
  • Branches default

Comments (0)

Files changed (3)

File src/org/duelengine/duel/parsing/DuelGrammar.java

View file
 	public static final char OP_ELEM_BEGIN = '<';
 	public static final char OP_ELEM_END = '>';
 	public static final char OP_ELEM_CLOSE = '/';
-	public static final char OP_VALUE_DELIM = ' ';
+	public static final char OP_ATTR_DELIM = ' ';
 	public static final char OP_PAIR_DELIM = '=';
 	public static final char OP_PREFIX_DELIM = ':';
 

File src/org/duelengine/duel/parsing/DuelLexer.java

View file
 					case UNPARSED:
 						switch (this.ch) {
 							case DuelGrammar.OP_ELEM_BEGIN:
-								if (this.tryScanUnparsedBlock() || this.tryScanTag()) {
+								if (this.tryScanUnparsedBlock(false) || this.tryScanTag()) {
 									return this.token;
 								}
 								break;
 							return this.token;
 						}
 
-						return this.scanLiteral();
+						boolean skip = true;
+						while (skip) {
+							switch (this.ch) {
+								case DuelGrammar.OP_ELEM_CLOSE:
+								case DuelGrammar.OP_ELEM_END:
+								case DuelGrammar.EOF:
+									skip = false;
+									break;
+								default:
+									this.nextChar();
+									break;
+							}
+						}
+						continue;
+
 					case ATTR_NAME:
 						if (this.tryScanAttrValue()) {
 							return this.token;
 						}
 
-						// reset to elem state
+						// no value, reset to elem state
 						this.token = DuelToken.ElemBegin(this.lastTag);
 						continue;
 
 	 * @return true if block token was found
 	 * @throws IOException
 	 */
-	private boolean tryScanUnparsedBlock()
+	private boolean tryScanUnparsedBlock(boolean asAttr)
 		throws IOException {
 
 		this.setMark(3);
 			return false;
 		}
 
-		this.token = DuelToken.AttrName(this.buffer.toString().toLowerCase());
+		this.token = DuelToken.AttrName(this.buffer.toString());
 		return true;
 	}
 
 	private boolean tryScanAttrValue()
 		throws IOException {
 
-		// TODO
-		return false;
+		// skip whitespace
+		while (CharUtility.isWhiteSpace(this.ch)) {
+			this.nextChar();
+		}
+
+		if (this.ch != DuelGrammar.OP_PAIR_DELIM) {
+			return false;
+		}
+
+		int delim;
+		switch (this.nextChar()) {
+			case DuelGrammar.OP_STRING_DELIM:
+			case DuelGrammar.OP_STRING_DELIM_ALT:
+				delim = this.ch;
+				this.nextChar();
+				break;
+			default:
+				delim = DuelGrammar.OP_ATTR_DELIM;
+		}
+
+		if (!this.tryScanUnparsedBlock(true)) {
+			this.scanAttrLiteral(delim);
+		}
+
+		if (this.ch == delim) {
+			this.nextChar();
+		}
+
+		return true;
+	}
+
+	/**
+	 * Scans the next token as a literal attribute value
+	 * @return
+	 * @throws IOException
+	 */
+	private DuelToken scanAttrLiteral(int delim)
+		throws IOException {
+
+		// reset the buffer
+		this.buffer.setLength(0);
+
+		while (true) {
+			switch (this.ch) {
+				case DuelGrammar.EOF:
+				case DuelGrammar.OP_ELEM_END:
+					// flush the buffer
+					return (this.token = DuelToken.AttrValue(this.buffer.toString()));
+
+				case DuelGrammar.OP_ENTITY_BEGIN:
+					// attempt to decode entities
+					this.decodeEntity();
+					continue;
+
+				default:
+					if (this.ch == delim) {
+						// flush the buffer
+						return (this.token = DuelToken.AttrValue(this.buffer.toString()));
+					}
+
+					// consume until reach a special char
+					this.buffer.append((char)this.ch);
+					this.nextChar();
+					continue;
+			}
+		}
 	}
 
 	/**

File test/org/duelengine/duel/parsing/DuelLexerTests.java

View file
 	}
 
 	@Test
+	public void elemBeginEndTest() {
+
+		String input = "<div></div>";
+
+		Object[] expected = {
+				DuelToken.ElemBegin("div"),
+				DuelToken.ElemEnd("div"),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+
+		assertArrayEquals(expected, actual);
+	}
+
+	@Test
 	public void elemVoidTest() {
 
 		String input = "<div/>";
 
 		assertArrayEquals(expected, actual);
 	}
+
+	@Test
+	public void attrNoValueTest() {
+
+		String input = "<div noValue/>";
+
+		Object[] expected = {
+				DuelToken.ElemBegin("div"),
+				DuelToken.AttrName("noValue"),
+				DuelToken.ElemEnd("div"),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+
+		assertArrayEquals(expected, actual);
+	}
+
+	@Test
+	public void attrNoValueWhitespaceTest() {
+
+		String input = "<div noValue      ></div>";
+
+		Object[] expected = {
+				DuelToken.ElemBegin("div"),
+				DuelToken.AttrName("noValue"),
+				DuelToken.ElemEnd("div"),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+
+		assertArrayEquals(expected, actual);
+	}
+
+	@Test
+	public void attrEmptyTest() {
+
+		String input = "<div emptyValue=\"\" />";
+
+		Object[] expected = {
+				DuelToken.ElemBegin("div"),
+				DuelToken.AttrName("emptyValue"),
+				DuelToken.AttrValue(""),
+				DuelToken.ElemEnd("div"),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+
+		assertArrayEquals(expected, actual);
+	}
+
+	@Test
+	public void attrEmptyAltDelimTest() {
+
+		String input = "<div emptyValue=''></div>";
+
+		Object[] expected = {
+				DuelToken.ElemBegin("div"),
+				DuelToken.AttrName("emptyValue"),
+				DuelToken.AttrValue(""),
+				DuelToken.ElemEnd("div"),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+
+		assertArrayEquals(expected, actual);
+	}
+
+	@Test
+	public void attrSimpleTest() {
+
+		String input = "<div simpleValue=\" this is the 'value' \"></div>";
+
+		Object[] expected = {
+				DuelToken.ElemBegin("div"),
+				DuelToken.AttrName("simpleValue"),
+				DuelToken.AttrValue(" this is the 'value' "),
+				DuelToken.ElemEnd("div"),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+
+		assertArrayEquals(expected, actual);
+	}
+
+	@Test
+	public void attrSimpleAltDelimTest() {
+
+		String input = "<div simpleValue=' this is the \"value\" '></div>";
+
+		Object[] expected = {
+				DuelToken.ElemBegin("div"),
+				DuelToken.AttrName("simpleValue"),
+				DuelToken.AttrValue(" this is the \"value\" "),
+				DuelToken.ElemEnd("div"),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+		
+		assertArrayEquals(expected, actual);
+	}
+
+	@Test
+	public void attrNoDelimTest() {
+
+		String input = "<div simpleValue=this_is_the_value another></div>";
+
+		Object[] expected = {
+				DuelToken.ElemBegin("div"),
+				DuelToken.AttrName("simpleValue"),
+				DuelToken.AttrValue("this_is_the_value"),
+				DuelToken.AttrName("another"),
+				DuelToken.ElemEnd("div"),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+
+		assertArrayEquals(expected, actual);
+	}
+
+	@Test
+	public void attrOnlyEntitiesTest() {
+
+		String input = "<div simpleValue='&vert;&semi;&comma;'></div>";
+
+		Object[] expected = {
+				DuelToken.ElemBegin("div"),
+				DuelToken.AttrName("simpleValue"),
+				DuelToken.AttrValue("|;,"),
+				DuelToken.ElemEnd("div"),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+
+		assertArrayEquals(expected, actual);
+	}
+
+	@Test
+	public void attrQuotEntityTest() {
+
+		String input = "<div simpleValue=\"the &quot;quote&quot;ed value &nothin\"></div>";
+
+		Object[] expected = {
+				DuelToken.ElemBegin("div"),
+				DuelToken.AttrName("simpleValue"),
+				DuelToken.AttrValue("the \"quote\"ed value &nothin"),
+				DuelToken.ElemEnd("div"),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+
+		assertArrayEquals(expected, actual);
+	}
+
+	@Test
+	public void attrAposEntityTest() {
+
+		String input = "<div simpleValue='the attribute&apos;s apos &nothin'></div>";
+
+		Object[] expected = {
+				DuelToken.ElemBegin("div"),
+				DuelToken.AttrName("simpleValue"),
+				DuelToken.AttrValue("the attribute's apos &nothin"),
+				DuelToken.ElemEnd("div"),
+				DuelToken.End
+			};
+
+		Object[] actual = new DuelLexer(input).toList().toArray();
+
+		assertArrayEquals(expected, actual);
+	}
+
+	private void dumpList(String label, Object[] tokens) {
+		System.out.println();
+		System.out.print(label+":");
+		for (Object token : tokens) {
+			System.out.print("\n\t"+token);
+		}
+		System.out.println();
+	}
 }