Anonymous avatar Anonymous committed 6a3a09e

(no commit message)

Comments (0)

Files changed (55)

+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
+	<classpathentry kind="src" path="src"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>org.lispdev.parser</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.pde.ManifestBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.pde.SchemaBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>net.sourceforge.metrics.builder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.pde.PluginNature</nature>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+		<nature>net.sourceforge.metrics.nature</nature>
+	</natures>
+</projectDescription>

.settings/org.eclipse.jdt.core.prefs

+#Mon Jun 22 09:44:54 BST 2009
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
+org.eclipse.jdt.core.compiler.compliance=1.6
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.source=1.6

META-INF/MANIFEST.MF

+Manifest-Version: 1.0
+Bundle-ManifestVersion: 2
+Bundle-Name: Lisp Parser
+Bundle-SymbolicName: org.lispdev.parser
+Bundle-Version: 0.0.1
+Bundle-RequiredExecutionEnvironment: JavaSE-1.6
+Require-Bundle: org.eclipse.core.runtime;bundle-version="3.5.0"
+Export-Package: org.lispdev.parser,
+ org.lispdev.parser.nodes,
+ org.lispdev.parser.tokens
+source.. = src/
+output.. = bin/
+bin.includes = META-INF/,\
+               .

src/org/lispdev/parser/ICharStream.java

+package org.lispdev.parser;
+
+
+public interface ICharStream
+{
+  static public final char EOF = '\uFFFF';
+
+  /**
+   * @return next character in stream (also increments position)
+   */
+  char read();
+
+  /**
+   * marks current position
+   */
+  void mark();
+
+  /**
+   * marks current position - delta
+   * @param delta
+   */
+  void mark(int delta);
+
+  /**
+   * resets stream to mark
+   */
+  void resetToMark();
+  /**
+   * reset to beginning of the stream
+   */
+  void reset();
+  /**
+   * @return current position (i.e. position of last read character)
+   */
+  int pos();
+  /**
+   * @return total number of characters
+   */
+  int size();
+}

src/org/lispdev/parser/ILispCharStream.java

+package org.lispdev.parser;
+
+
+public interface ILispCharStream
+{
+  static public final char EOF = '\uFFFF';
+
+  /**
+   * @return next character in stream
+   */
+  char read();
+
+  /**
+   * marks current position
+   */
+  void mark();
+
+  /**
+   * resets stream to mark
+   */
+  void resetToMark();
+  /**
+   * reset to beginning of the stream
+   */
+  void reset();
+  /**
+   * @return current position
+   */
+  int pos();
+  /**
+   * @return total number of characters
+   */
+  int size();
+}

src/org/lispdev/parser/ITokenStream.java

+package org.lispdev.parser;
+
+import org.lispdev.parser.tokens.Token;
+
+
+public interface ITokenStream
+{
+  /**
+   * @return next token in stream
+   */
+  Token read();
+
+  /**
+   * marks current position
+   */
+  void mark();
+
+  /**
+   * resets stream to mark
+   */
+  void resetToMark();
+  /**
+   * reset to beginning of the stream
+   */
+  void reset();
+}

src/org/lispdev/parser/LispCharStream.java

+package org.lispdev.parser;
+
+public class LispCharStream implements ICharStream
+{
+  CharSequence s;
+  int mark;
+  int pos;
+
+  @Override
+  public void mark()
+  {
+    mark = pos;
+  }
+
+  @Override
+  public void mark(int delta)
+  {
+    mark = pos - delta;
+  }
+
+  @Override
+  public int pos()
+  {
+    return pos;
+  }
+
+  @Override
+  public char read()
+  {
+    if( pos == s.length() )
+    {
+      return ICharStream.EOF;
+    }
+    else
+    {
+      ++pos;
+      if( pos == s.length() )
+      {
+        return ICharStream.EOF;
+      }
+      else
+      {
+        return s.charAt(pos);
+      }
+    }
+  }
+
+  @Override
+  public void reset()
+  {
+    pos = -1;
+    mark = -1;
+  }
+
+  @Override
+  public void resetToMark()
+  {
+    pos = mark;
+  }
+
+  @Override
+  public int size()
+  {
+    return s.length();
+  }
+
+  public LispCharStream(CharSequence str)
+  {
+    s = str;
+    reset();
+  }
+
+  @Override
+  public String toString()
+  {
+    return s.toString();
+  }
+
+
+}

src/org/lispdev/parser/LispLexer.java

+package org.lispdev.parser;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.regex.Pattern;
+
+import org.eclipse.core.runtime.Assert;
+import org.lispdev.parser.tokens.Token;
+import org.lispdev.parser.tokens.TokenAt;
+import org.lispdev.parser.tokens.TokenBQuote;
+import org.lispdev.parser.tokens.TokenBad;
+import org.lispdev.parser.tokens.TokenBitVector;
+import org.lispdev.parser.tokens.TokenChar;
+import org.lispdev.parser.tokens.TokenCloseParen;
+import org.lispdev.parser.tokens.TokenComma;
+import org.lispdev.parser.tokens.TokenComplexHeader;
+import org.lispdev.parser.tokens.TokenEOF;
+import org.lispdev.parser.tokens.TokenFuncNamespace;
+import org.lispdev.parser.tokens.TokenKey;
+import org.lispdev.parser.tokens.TokenKeyword;
+import org.lispdev.parser.tokens.TokenMLC;
+import org.lispdev.parser.tokens.TokenNumber;
+import org.lispdev.parser.tokens.TokenOpenParen;
+import org.lispdev.parser.tokens.TokenOutline;
+import org.lispdev.parser.tokens.TokenParam;
+import org.lispdev.parser.tokens.TokenPeriod;
+import org.lispdev.parser.tokens.TokenQuote;
+import org.lispdev.parser.tokens.TokenSLC;
+import org.lispdev.parser.tokens.TokenString;
+import org.lispdev.parser.tokens.TokenStructHeader;
+import org.lispdev.parser.tokens.TokenSymbol;
+import org.lispdev.parser.tokens.TokenTLParen;
+import org.lispdev.parser.tokens.TokenUnreadable;
+import org.lispdev.parser.tokens.TokenVectorHeader;
+import org.lispdev.parser.tokens.TokenWS;
+import org.lispdev.parser.utils.LispCharBuffer;
+
+/**
+ * Takes lisp stream and creates array of tokens.
+ */
+public class LispLexer
+{
+  private ArrayList<Token> tokens = new ArrayList<Token>();
+  private LispCharBuffer buffer;
+  private ICharStream chars;
+  private SortedSet<String> keywords;
+
+  private int maxSymbolLines = 1;
+  /**
+   * Symbols between || can extend maximum this number of lines.
+   * Usually symbols do not extend more than one line. But if they
+   * do in your code, change this number.
+   *
+   * Lower settings allow for better error recovery. Default = 1.
+   */
+  public void setMaxSymbolLines(int i)
+  {
+    maxSymbolLines = i;
+  }
+
+  private boolean ignoreOutline = false;
+  public void setIgnoreOutline(boolean b)
+  {
+    ignoreOutline = b;
+  }
+
+  // state variables
+  // are not used now for setting properties of tokens,
+  // but potentially can be used to
+  // track line and column
+  private int currentLine = 0;
+  private int currentLineOffset = 0;
+  private int line0 = 0; //line at which token started
+
+
+  public LispLexer(ICharStream c)
+  {
+    chars = c;
+    buffer = new LispCharBuffer(80);
+  }
+
+  public LispLexer(ICharStream c, LispCharBuffer buffer)
+  {
+    chars = c;
+    this.buffer = buffer;
+    buffer.reset();
+  }
+
+  public void setKeywords(SortedSet<String> keywords)
+  {
+    this.keywords = keywords;
+  }
+
+  @Override
+  public String toString()
+  {
+    return String.valueOf(tokens);
+  }
+
+  /* potential problems that I need to handle:
+   * did not close comment, symbol bar, quote -
+   * need to find when to stop.
+   * Strategy:
+   * - symbol bar -> run until next stop character
+   * - comment -> run until end of line
+   * - string -> run until end of line
+   * Anything else does not require backtracking.
+   * So once in one of these states, mark buffer for backtracking.
+   * If backtracking is required create BAD_SYMBOL, BAD_COMMENT or BAD_STRING
+   * tokens.
+   *
+   * How to handle multilevel bars or comments?
+   * bars - same, but count levels, if not matched, assume first is stopped as
+   *  usual
+   * comments - also same
+   *
+   */
+
+  /**
+   * @param c should be space character
+   * @return character at which reading has stopped
+   * Also adds white space to list of tokens
+   */
+  private char readWS(char c)
+  {
+    final int pos = chars.pos();
+    buffer.reset();
+    while( isSpace(c) )
+    {
+      if( isEOL(c) )
+      {
+        c = readEOL(c,buffer);
+      }
+      else
+      {
+        buffer.append(c);
+        c = chars.read();
+      }
+    }
+    tokens.add(new TokenWS(pos,buffer.toString(),EOLcount()));
+    return c;
+  }
+
+  private char readSLC()
+  {
+    final int pos = chars.pos(); // starting position
+    int lvl = 1; // first ; is found
+    buffer.reset().append(';');
+    char c = chars.read();
+    while( c == ';' )
+    {
+      ++lvl;
+      buffer.append(';');
+      c = chars.read();
+    }
+    while( !isEOL(c) && !isEOF(c) )
+    {
+      buffer.append(c);
+      c = chars.read();
+    }
+    if( pos == currentLineOffset && lvl >= 4 )
+    {
+      tokens.add(new TokenOutline(pos, buffer.toString(),
+          lvl, ignoreOutline, EOLcount()));
+    }
+    else
+    {
+      tokens.add(new TokenSLC(pos, buffer.toString(), lvl, EOLcount()));
+    }
+
+    return c;
+  }
+
+  /**
+   * state flag used only for symbol parsing
+   */
+  boolean hasError;
+  int stopSize;
+  String expected;
+
+  /**
+   * Helper of readSymbol(LispCharBuffer sb, char c).
+   * start correspond to offset of first character in sb
+   */
+  private char readBarSymbol(LispCharBuffer sb)
+  {
+    sb.append('|');
+    boolean escaped = false;
+    int line = 1;
+    char c = chars.read();
+    stopSize = -1;
+    char stopChar = c;
+    hasError = true;
+    expected = "|";
+    int markLine = -1;
+
+    while(true)
+    {
+      // if has not found any stop yet and this is a terminator
+      // reset to be able to fall back to this char
+      if( stopSize < 0 && isTerminator(c) && !escaped )
+      {
+        stopSize = sb.length();
+        stopChar = c;
+        chars.mark();
+        markLine = currentLine;
+      }
+
+      // if end of line, check if should stop parsing
+      if( isEOL(c) )
+      {
+        if( line > maxSymbolLines )
+        {
+          // fall back to marked position
+          if( stopSize < 1 ) //no stop found, so stop here - should never happen
+          {
+            stopSize = sb.length();
+            stopChar = c;
+            return c;
+          }
+          else // fall back
+          {
+            chars.resetToMark();
+            currentLine = markLine;
+            return stopChar;
+          }
+        }
+        else
+        {
+          ++line;
+          c = readEOL(c,sb);
+          escaped = false;
+          continue; // got new c, so start processing over
+        }
+      }
+
+      if( isEOF(c) ) // no matching closing '|'
+      {
+        // fall back to marked position
+        if( stopSize < 1 ) // no stop found, so stop here
+        {
+          stopSize = sb.length();
+          return c;
+        }
+        else // fall back
+        {
+          chars.resetToMark();
+          currentLine = markLine;
+          return stopChar;
+        }
+      }
+
+      sb.append(c);
+      if( escaped )
+      {
+        escaped = false;
+      }
+      else
+      {
+        if( c == '|' ) // found closing
+        {
+          hasError = false;
+          return chars.read();
+        }
+        else if( c == '\\' ) //escape
+        {
+          escaped = true;
+        }
+      }
+      c = chars.read();
+    }
+  }
+
+  /**
+   * helper of readSymbol(char c), readChar(), readBitVector(), readPeriod(),
+   * readRadixNumber().
+   * Reads symbol characters until break character is found. Then corresponding
+   * function find out if specific token is valid.
+   */
+  private char readSymbol(LispCharBuffer sb, char c)
+  {
+    boolean escaped = false;
+
+    while(true)
+    {
+      if( isEOF(c) )
+      {
+        expected = "any char";
+        stopSize = sb.length();
+        hasError = escaped;
+        return c;
+      }
+      if( escaped )
+      {
+        escaped = false;
+        // attach all end of line characters of single line to escaped
+        if( isEOL(c) )
+        {
+          c = readEOL(c,sb);
+        }
+        else
+        {
+          sb.append(c);
+          c = chars.read();
+        }
+        continue; // got new c, so start processing over
+      }
+      else // not escaped
+      {
+        if( c == '|' )
+        {
+          c = readBarSymbol(sb);
+          if( hasError ) // token is created by readBarSymbol
+          {
+            return c;
+          }
+          else
+          {
+            continue;
+          }
+        }
+        else if( isTerminator(c) )
+        {
+          hasError = false;
+          stopSize = sb.length();
+          return c;
+        }
+        else
+        {
+          sb.append(c);
+          if( c == '\\' )
+          {
+            escaped = true;
+          }
+          c = chars.read();
+        }
+      }
+    }
+  }
+
+  private void emmitSymbol(int offset, ParseError error)
+  {
+    buffer.trim(stopSize);
+    String text = buffer.toString();
+    switch(buffer.charAt(0) )
+    {
+      case ':': //key
+        String ss[] = text.split(":");
+        if( ss.length > 2 )
+        {
+          error = new ParseError( 1 + ss[1].length(), 1, ":symbol", "extra :");
+        }
+        else if( ss.length == 2 && (ss[1] == null || ss[1].length() == 0) )
+        {
+          error = new ParseError( 2, 1, ":symbol", "no symbol after :" );
+        }
+        tokens.add(new TokenKey(offset, text, error, EOLcount()));
+        break;
+      case '&': //amp key
+        int pos = text.indexOf(":");
+        if( pos >= 0 )
+        {
+          error = new ParseError( offset + pos, 1, "&symbol", "invalid :");
+        }
+        tokens.add(new TokenParam(offset, text, error, EOLcount()));
+        break;
+      default:
+        ss = null;
+        if( error == null )
+        {
+          ss = text.split(":");
+          if( ss.length > 3 // a:b:c:d or a::b:c
+              || (ss.length == 3 && (ss[1] != null && ss[1].length() > 0))) //a:b:c
+          {
+            pos = 0;
+            for( int i = 0; i < ss.length - 1; ++i )
+            {
+              pos += ss[i].length() + 1;
+            }
+            error = new ParseError( offset + pos - 1, 1, "symbol", "too many :");
+          }
+          else if( ss.length > 1 )
+          {
+            if(ss[ss.length-1] == null || ss[ss.length-1].length() == 0) //a:
+            {
+              error = new ParseError(offset+text.length()-1, 1, "symbol", ":");
+            }
+            else if( isDecimal(ss[0]) )
+            {
+              error = new ParseError(offset, ss[0].length(),
+                  "package", "number");
+            }
+            else if( isDecimal(ss[ss.length-1]) )
+            {
+              pos = offset;
+              for( int i = 0; i < ss.length-1; ++i )
+              {
+                pos += ss[0].length()+1;
+              }
+              error = new ParseError(pos, ss[ss.length-1].length(),
+                  "symbol", "number");
+            }
+          }
+        }
+        if( error == null && isDecimal(buffer) )
+        {
+          tokens.add(new TokenNumber(offset, text, error, EOLcount()));
+        }
+        else if ( ss == null )
+        {
+            tokens.add(
+                new TokenSymbol(offset, text, error, text, null,
+                    false, EOLcount() ));
+        }
+        else
+        {
+          if( error == null && isKeyword(text) )
+          {
+            tokens.add( new TokenKeyword(offset, text, error, EOLcount()));
+          }
+          else
+          {
+            tokens.add(
+                new TokenSymbol(offset, text, error,
+                    ss[ss.length-1], ss[0], ss.length != 2, EOLcount() ));
+          }
+        }
+    }
+
+  }
+
+  private char readSymbol(char c)
+  {
+    final int offset = chars.pos();
+    buffer.reset();
+    c = readSymbol(buffer, c);
+    ParseError error = null;
+    if( hasError )
+    {
+      error =
+        new ParseError(offset+stopSize,1,expected,
+            (isEOF(c)?"EOF":String.valueOf(c)));
+    }
+    emmitSymbol(offset, error);
+    return c;
+  }
+
+  private char readString()
+  {
+    final int offset = chars.pos();
+    buffer.reset().append('"');
+    char c = chars.read();
+    boolean escaped = false;
+    int stopSize = -1;
+    char stopChar = '"';
+    int markLine = -1;
+
+    while(true)
+    {
+      if( isEOL(c) )
+      {
+        // if has not found any end of line yet
+        // reset to be able to fall back to this char
+        if( stopSize < 0 )
+        {
+          stopSize = buffer.length();
+          stopChar = c;
+          chars.mark();
+          markLine = currentLine;
+        }
+        c = readEOL(c,buffer);
+        escaped = false;
+        continue;
+      }
+
+      if( isEOF(c) ) // no matching closing '"'
+      {
+        // fall back to marked position
+        if( stopSize < 1 ) // no stop found, so stop here
+        {
+          tokens.add(new TokenString(offset, buffer.toString(),
+              new ParseError(chars.pos(), 1, "\"", "EOF"), EOLcount()));
+          return c;
+        }
+        else // fall back
+        {
+          chars.resetToMark();
+          currentLine = markLine;
+          tokens.add(new TokenString(offset, buffer.substring(0, stopSize),
+              new ParseError(offset+stopSize, 1, "\"",
+                  String.valueOf(stopChar)), EOLcount()));
+          return stopChar;
+        }
+      }
+
+      buffer.append(c);
+      if( escaped )
+      {
+        escaped = false;
+      }
+      else
+      {
+        if( c == '"' ) // found closing
+        {
+          tokens.add(new TokenString(offset, buffer.toString(), null,
+              EOLcount()));
+          return chars.read();
+        }
+        else if( c == '\\' ) //escape
+        {
+          escaped = true;
+        }
+      }
+      c = chars.read();
+    }
+  }
+
+  /**
+   * Assume current character is | (in first #)
+   * Error recovery strategy: if no other comment - end of first line,
+   * if there is another comment smaller of the two: end of line or start
+   * of the second comment (comments are MLC)
+   * @return
+   */
+  private char readMLC()
+  {
+    final int offset = chars.pos()-1;
+    buffer.reset().append('#').append('|');
+    char c = chars.read();
+    int stopSize = -1;
+    char stopChar = '\n';
+    int lvl = 0; // can have comment inside comment
+    boolean gotbar = false; // previous character was bar unmatched to pound
+    boolean gotpound = false; // previous character was pound unmatched to bar
+    int markLine = -1;
+
+    while(true)
+    {
+      if( isEOL(c) )
+      {
+        // if has not found any end stop yet
+        // reset to be able to fall back to this char
+        if( stopSize < 0 )
+        {
+          stopSize = buffer.length();
+          stopChar = c;
+          chars.mark();
+          markLine = currentLine;
+        }
+        c = readEOL(c,buffer);
+        continue;
+      }
+
+      if( isEOF(c) ) // no matching closing '|#'
+      {
+        // fall back to marked position
+        if( stopSize < 1 ) // no stop found, so stop here
+        {
+          tokens.add(new TokenMLC(offset, buffer.toString(),
+              new ParseError(chars.pos(), 1, "|#", "EOF"), EOLcount()));
+          return c;
+        }
+        else // fall back
+        {
+          chars.resetToMark();
+          currentLine = markLine;
+          tokens.add(new TokenMLC(offset, buffer.substring(0, stopSize),
+              new ParseError(offset+stopSize, 1, "|#",
+                  String.valueOf(stopChar)), EOLcount()));
+          return stopChar;
+        }
+      }
+
+      buffer.append(c);
+      if( c == '#' )
+      {
+        c = chars.read();
+        if( gotbar ) //found matching of lvl
+        {
+          if( lvl == 0 )
+          {
+            tokens.add(new TokenMLC(offset, buffer.toString(), null,
+                EOLcount()));
+            return c;
+          }
+          else
+          {
+            --lvl;
+            gotbar = false;
+            continue;
+          }
+        }
+        else
+        {
+          gotpound = true;
+          continue;
+        }
+      }
+      else if( c == '|' )
+      {
+        c = chars.read();
+        if( gotpound ) //found opening of next lvl
+        {
+          // if has not found any end stop yet
+          // reset to be able to fall back to char before
+          if( stopSize < 0 )
+          {
+            stopSize = buffer.length()-2;
+            stopChar = '#';
+            chars.mark(2);
+          }
+          ++lvl;
+          gotpound = false;
+          continue;
+        }
+        else
+        {
+          gotbar = true;
+          continue;
+        }
+      }
+      else
+      {
+        gotbar = false;
+        gotpound = false;
+        c = chars.read();
+      }
+    }
+  }
+
+  private char readBitVector()
+  {
+    final int offset = chars.pos()-1;
+    buffer.reset().append('#').append('*');
+    char c = chars.read();
+    if( isTerminator(c) )
+    {
+      tokens.add(new TokenBitVector(offset, "#*",
+          new ParseError(offset+2,1,"0 or 1",
+              (isEOF(c)?"EOF":String.valueOf(c))), EOLcount()));
+      return c;
+    }
+    c = readSymbol(buffer, c);
+    ParseError error = null;
+    if( hasError )
+    {
+      error =
+        new ParseError(offset+stopSize,1,expected,
+            (isEOF(c)?"EOF":String.valueOf(c)));
+    }
+    tokens.add(
+        new TokenBitVector(offset, buffer.substring(0, stopSize), error,
+            EOLcount()));
+    return c;
+  }
+
+  private char readChar()
+  {
+    final int offset = chars.pos()-1;
+    char c0 = chars.read(); //character (escaped)
+    if( isEOF(c0) )
+    {
+      tokens.add(new TokenChar(offset, "#\\",
+          new ParseError(offset+1,1,"any char","EOF"), EOLcount()));
+      return c0;
+    }
+    char c = c0;
+    if( isEOL(c0) )
+    {
+      buffer.reset().append('#').append('\\');
+      c = readEOL(c0,buffer);
+      if( isTerminator(c) )
+      {
+        tokens.add(new TokenChar(offset,buffer.toString(),null,EOLcount()));
+        return c;
+      }
+    }
+    else
+    {
+      c = chars.read(); //let's see what comes next
+      if( isTerminator(c) || isEOF(c) ) //most common case
+      {
+        tokens.add(new TokenChar(offset, "#\\"+String.valueOf(c0), null,
+            EOLcount()));
+        return c;
+      }
+      buffer.reset().append('#').append('\\').append(c0);
+    }
+    c = readSymbol(buffer, c);
+    ParseError error = null;
+    if( hasError )
+    {
+      error =
+        new ParseError(offset+stopSize,1,expected,
+            (isEOF(c)?"EOF":String.valueOf(c)));
+    }
+    tokens.add(
+        new TokenChar(offset, buffer.substring(0, stopSize),error,EOLcount()));
+    return c;
+  }
+
+  private char readUnreadable()
+  {
+    final int offset = chars.pos()-1;
+    //StringBuilder sb = new StringBuilder(80);
+    buffer.reset().append('#').append('<');
+    char c = chars.read();
+    while(true)
+    {
+      if( c == '>' )
+      {
+        buffer.append(c);
+        tokens.add(
+            new TokenUnreadable(offset, buffer.toString(), null, EOLcount()));
+        return chars.read();
+      }
+      else if( isEOL(c) || isEOF(c) )
+      {
+        tokens.add(new TokenUnreadable(offset, buffer.toString(),
+            new ParseError(offset+buffer.length(), 1, ">",
+                (isEOF(c)?"EOF":String.valueOf(c))), EOLcount()));
+        return c;
+      }
+      else
+      {
+        buffer.append(c);
+        c = chars.read();
+      }
+    }
+  }
+
+  private char readRadixNumber(char c)
+  {
+    final int offset = chars.pos()-1;
+    buffer.reset().append("#");
+    c = readSymbol(buffer, c);
+    ParseError error = null;
+    if( hasError )
+    {
+      error =
+        new ParseError(offset+stopSize,1,expected,
+            (isEOF(c)?"EOF":String.valueOf(c)));
+    }
+    tokens.add(
+        new TokenNumber(offset, buffer.substring(0, stopSize), error,
+            EOLcount()));
+    return c;
+  }
+
+  /**
+   * Current character is assumed to be #
+   * @return
+   */
+  private char readMacro()
+  {
+    char c = chars.read();
+    final int offset = chars.pos()-1;
+    switch(c)
+    {
+      case ICharStream.EOF:
+        tokens.add(
+            new TokenBad(offset, "#", new ParseError(offset, 1, null, null),
+                EOLcount()));
+        return c;
+      case '|': //MLC
+        return readMLC();
+      case '*': //bit vector - the rest as in symbol
+        return readBitVector();
+      case '\\': //character
+        return readChar();
+      case '(': //vector head
+        tokens.add(new TokenVectorHeader(offset));
+        return chars.read();
+      case 's':
+        tokens.add(new TokenStructHeader(offset, true));
+        return chars.read();
+      case 'S': //structure head
+        tokens.add(new TokenStructHeader(offset, false));
+        return chars.read();
+      case 'c':
+        tokens.add(new TokenComplexHeader(offset, true));
+        return chars.read();
+      case 'C': //structure head
+        tokens.add(new TokenComplexHeader(offset, false));
+        return chars.read();
+      case '\'': //function namespace
+        tokens.add(new TokenFuncNamespace(offset));
+        return chars.read();
+      case '<': //unreadable
+        return readUnreadable();
+      default: // the rest are potentially numbers
+        if( isTerminator(c) )
+        {
+          tokens.add(
+              new TokenBad(offset,"#",new ParseError(offset, 1, null, null),
+                  EOLcount()));
+          return c;
+        }
+        else
+        {
+          return readRadixNumber(c);
+        }
+    }
+  }
+
+  private char readPeriod()
+  {
+    char c = chars.read();
+    final int offset = chars.pos()-1;
+    if( isTerminator(c) )
+    {
+      tokens.add(new TokenPeriod(offset));
+      return c;
+    }
+
+    buffer.reset().append('.');
+    c = readSymbol(buffer, c);
+    ParseError error = null;
+    if( hasError )
+    {
+      error =
+        new ParseError(offset+stopSize,1,expected,
+            (isEOF(c)?"EOF":String.valueOf(c)));
+    }
+    else
+    {
+      // check if sb contains only '.'
+      boolean err = true;
+      for( int i = 0; i < buffer.length(); ++i )
+      {
+        if( buffer.charAt(i) != '.' )
+        {
+          err = false;
+          break;
+        }
+      }
+      if( err )
+      {
+        error = new ParseError(offset,stopSize,null,"too many dots");
+      }
+    }
+    emmitSymbol(offset,error);
+    return c;
+
+  }
+
+  public List<Token> parse()
+  {
+    line0 = 0;
+    currentLine = 0;
+    currentLineOffset = 0;
+    char c = chars.read(); //read first character
+    while( !isEOF(c) )
+    {
+      c = read(c);
+    }
+    tokens.add(new TokenEOF(chars.pos()));
+    return tokens;
+  }
+
+  private char readComma()
+  {
+    final int offset = chars.pos();
+    char c = chars.read();
+    if( c == '@' )
+    {
+      tokens.add(new TokenAt(offset));
+      return chars.read();
+    }
+    else
+    {
+      tokens.add(new TokenComma(offset));
+      return c;
+    }
+  }
+
+  /**
+   * Returns EOL count and resets counter.
+   * @return number of EOL encountered from last time setEOLcount was called
+   */
+  private int EOLcount()
+  {
+    final int n = currentLine - line0;
+    line0 = currentLine;
+    return n;
+  }
+
+  /**
+   * The function reads next token from chars stream. The token starts with
+   * character c. The resulting token is added to tokens list and function
+   * returns first character after token.
+   */
+  private char read(char c)
+  {
+    final int offset = chars.pos();
+    switch(c)
+    {
+      case ICharStream.EOF : //should never be here
+        Assert.isLegal(false,"EOF encountered");
+      case '(':
+        if( offset == currentLineOffset )
+        {
+          tokens.add(new TokenTLParen(offset));
+          return chars.read();
+        }
+        else
+        {
+          tokens.add(new TokenOpenParen(offset));
+          return chars.read();
+        }
+      case ')':
+        tokens.add(new TokenCloseParen(offset));
+        return chars.read();
+      case '\'':
+        tokens.add(new TokenQuote(offset));
+        return chars.read();
+      case '`':
+        tokens.add(new TokenBQuote(offset));
+        return chars.read();
+      case ',':
+        return readComma();
+      case '.':
+        return readPeriod();
+      case ';':
+        return readSLC();
+      case '"':
+        return readString();
+      case '#':
+        return readMacro();
+      default :
+        if( isSpace(c) )
+        {
+          return readWS(c);
+        }
+        else
+        {
+          return readSymbol(c);
+        }
+    }
+  }
+
+  // c should be either \n or \r
+  private char readEOL(char c, LispCharBuffer s)
+  {
+    ++currentLine;
+    s.append(c); // \n or \r
+    char res = chars.read();
+    if( c == '\r' )
+    {
+      if( res == '\n' ) //windows \r\n
+      {
+        s.append(res);
+        c = chars.read();
+        currentLineOffset = chars.pos();
+        return c;
+      }
+      else //MacOSX \r
+      {
+        currentLineOffset = chars.pos();
+        return res;
+      }
+    }
+    else //c == \n unix
+    {
+      currentLineOffset = chars.pos();
+      return res;
+    }
+  }
+
+  // ----  Utility functions  ----------------------------------
+
+  static boolean isEOF(char x)
+  {
+    return (x == ICharStream.EOF);
+  };
+
+  static boolean isEOL(char x)
+  {
+    return (x == '\n' || x == '\r');
+  }
+
+  /*
+   * Lookup tables to speedup comparisons.
+   */
+  private static final char[] spaceChars =
+    new char[] {' ', '\n', '\r', '\t', '\f', '\b'};
+
+  /**
+   * @deprecated Used for setting up lookup tables.
+   */
+  @Deprecated
+  static private boolean isSpaceChar(char c)
+  {
+    for( char x : spaceChars )
+    {
+      if( c == x ) return true;
+    }
+    return false;
+  }
+  static private int maxChar(char[] chars)
+  {
+    int res = chars[0];
+    for( char c : chars )
+    {
+      res = ( c > res ? c : res );
+    }
+    return res;
+  }
+
+  private static final char[] stopChars =
+    new char[]{'(', ')', '\'', ';', '"', ',', '`'};
+  /**
+   * @deprecated Used for setting up lookup tables.
+   */
+  @Deprecated
+  static private boolean isStopChar(char c)
+  {
+    if( isSpaceChar(c) ) return true;
+    for( char x : stopChars )
+    {
+      if( c == x ) return true;
+    }
+    return false;
+  }
+
+  private static final boolean[] terminatorLookupTable;
+  private static final boolean[] spaceLookupTable;
+  static
+  {
+    final int maxSpace = maxChar(spaceChars);
+    terminatorLookupTable =
+      new boolean[Math.max(maxSpace, maxChar(stopChars))+1];
+    spaceLookupTable = new boolean[maxSpace + 1];
+    for(int i = 0; i < terminatorLookupTable.length; ++i)
+    {
+      terminatorLookupTable[i] = isStopChar((char)i);
+    }
+    for(int i = 0; i < spaceLookupTable.length; ++i)
+    {
+      spaceLookupTable[i] = isSpaceChar((char)i);
+    }
+  }
+
+  static boolean isTerminator(char x)
+  {
+    return ( x < terminatorLookupTable.length && terminatorLookupTable[x] ) ;
+  }
+
+  static boolean isSpace(char x)
+  {
+    return ( x < spaceLookupTable.length && spaceLookupTable[x] );
+  }
+
+//  static boolean isDigit(char x)
+//  {
+//    return (x >= '0' && x <= '9');
+//  }
+
+
+  // ====== matchers ==========
+
+  // decimal number matcher
+  static final Pattern decimalMatcher = Pattern.compile("("
+      + "([-\\+]?((\\d+(\\.\\d*)?)|(\\.\\d+))(([esfdlESFDL])[-\\+]?\\d+)?)"
+      + "|" + "([-\\+]?\\d+(/\\d+)?)" + ")");
+
+  static boolean isDecimal(CharSequence s)
+  {
+    return decimalMatcher.matcher(s).matches();
+  }
+
+  // constant symbol matcher, e.g. +this+
+  static boolean isConst(CharSequence s)
+  {
+    return s.length() > 1 && s.charAt(0) == '+'
+        && s.charAt(s.length() - 1) == '+';
+  }
+
+  // global symbol matcher, e.g. *this*
+  static boolean isGlobal(CharSequence s)
+  {
+    return s.length() > 1 && s.charAt(0) == '*'
+        && s.charAt(s.length() - 1) == '*';
+  }
+
+  // ampersand keyword matcher, e.g. &this
+  static boolean isParam(CharSequence s)
+  {
+    return s.length() > 1 && s.charAt(0) == '&';
+  }
+
+  // binary matcher, e.g. #b-00101/0111
+  static final Pattern binaryMatcher = Pattern
+      .compile("#[bB][-\\+]?[01]+(/[01]+)?");
+
+  static boolean isBinary(CharSequence s)
+  {
+    return binaryMatcher.matcher(s).matches();
+  }
+
+  // octal matcher, e.g. #o+00701/07611
+  static final Pattern octalMatcher = Pattern
+      .compile("#[oO][-\\+]?[0-7]+(/[0-7]+)?");
+
+  static boolean isOctal(CharSequence s)
+  {
+    return octalMatcher.matcher(s).matches();
+  }
+
+  // hex matcher, e.g. #x+a0701/b7611
+  static final Pattern hexMatcher = Pattern
+      .compile("#[xX][-\\+]?[A-Fa-f0-9]+(/[A-Fa-f0-9]+)?");
+
+  static boolean isHex(CharSequence s)
+  {
+    return hexMatcher.matcher(s).matches();
+  }
+
+  // radix matchers, e.g. #03r012
+  static final Pattern[] radixMatchers = {
+      Pattern.compile("#0*2[rR][-\\+]?[01]+(/[01]+)?"),
+      Pattern.compile("#0*3[rR][-\\+]?[0-2]+(/[0-2]+)?"),
+      Pattern.compile("#0*4[rR][-\\+]?[0-3]+(/[0-3]+)?"),
+      Pattern.compile("#0*5[rR][-\\+]?[0-4]+(/[0-4]+)?"),
+      Pattern.compile("#0*6[rR][-\\+]?[0-5]+(/[0-5]+)?"),
+      Pattern.compile("#0*7[rR][-\\+]?[0-6]+(/[0-6]+)?"),
+      Pattern.compile("#0*8[rR][-\\+]?[0-7]+(/[0-7]+)?"),
+      Pattern.compile("#0*9[rR][-\\+]?[0-8]+(/[0-8]+)?"),
+      Pattern.compile("#0*10[rR][-\\+]?[0-9]+(/[0-9]+)?"),
+      Pattern.compile("#0*11[rR][-\\+]?[Aa0-9]+(/[Aa0-9]+)?"),
+      Pattern.compile("#0*12[rR][-\\+]?[ABab0-9]+(/[ABab0-9]+)?"),
+      Pattern.compile("#0*13[rR][-\\+]?[A-Ca-c0-9]+(/[A-Ca-c0-9]+)?"),
+      Pattern.compile("#0*14[rR][-\\+]?[A-Da-d0-9]+(/[A-Da-d0-9]+)?"),
+      Pattern.compile("#0*15[rR][-\\+]?[A-Ea-e0-9]+(/[A-Ea-e0-9]+)?"),
+      Pattern.compile("#0*16[rR][-\\+]?[A-Fa-f0-9]+(/[A-Fa-f0-9]+)?"),
+      Pattern.compile("#0*17[rR][-\\+]?[A-Ga-g0-9]+(/[A-Ga-g0-9]+)?"),
+      Pattern.compile("#0*18[rR][-\\+]?[A-Ha-h0-9]+(/[A-Ha-h0-9]+)?"),
+      Pattern.compile("#0*19[rR][-\\+]?[A-Ia-i0-9]+(/[A-Ia-i0-9]+)?"),
+      Pattern.compile("#0*20[rR][-\\+]?[A-Ja-j0-9]+(/[A-Ja-j0-9]+)?"),
+      Pattern.compile("#0*21[rR][-\\+]?[A-Ka-k0-9]+(/[A-Ka-k0-9]+)?"),
+      Pattern.compile("#0*22[rR][-\\+]?[A-La-l0-9]+(/[A-La-l0-9]+)?"),
+      Pattern.compile("#0*23[rR][-\\+]?[A-Ma-m0-9]+(/[A-Ma-m0-9]+)?"),
+      Pattern.compile("#0*24[rR][-\\+]?[A-Na-n0-9]+(/[A-Na-n0-9]+)?"),
+      Pattern.compile("#0*25[rR][-\\+]?[A-Oa-o0-9]+(/[A-Oa-o0-9]+)?"),
+      Pattern.compile("#0*26[rR][-\\+]?[A-Pa-p0-9]+(/[A-Pa-p0-9]+)?"),
+      Pattern.compile("#0*27[rR][-\\+]?[A-Qa-q0-9]+(/[A-Qa-q0-9]+)?"),
+      Pattern.compile("#0*28[rR][-\\+]?[A-Ra-r0-9]+(/[A-Ra-r0-9]+)?"),
+      Pattern.compile("#0*29[rR][-\\+]?[A-Sa-s0-9]+(/[A-Sa-s0-9]+)?"),
+      Pattern.compile("#0*30[rR][-\\+]?[A-Ta-t0-9]+(/[A-Ta-t0-9]+)?"),
+      Pattern.compile("#0*31[rR][-\\+]?[A-Ua-u0-9]+(/[A-Ua-u0-9]+)?"),
+      Pattern.compile("#0*32[rR][-\\+]?[A-Va-v0-9]+(/[A-Va-v0-9]+)?"),
+      Pattern.compile("#0*33[rR][-\\+]?[A-Wa-w0-9]+(/[A-Wa-w0-9]+)?"),
+      Pattern.compile("#0*34[rR][-\\+]?[A-Xa-x0-9]+(/[A-Xa-x0-9]+)?"),
+      Pattern.compile("#0*35[rR][-\\+]?[A-Ya-y0-9]+(/[A-Ya-y0-9]+)?"),
+      Pattern.compile("#0*36[rR][-\\+]?[A-Za-z0-9]+(/[A-Za-z0-9]+)?")};
+
+  static boolean isRadix(CharSequence s)
+  {
+    for(Pattern p : radixMatchers)
+    {
+      if( p.matcher(s).matches() ) return true;
+    }
+    return false;
+  }
+
+  static final Pattern[] charMatchers = {
+      Pattern.compile("#\\\\\\w"),
+      Pattern.compile("#\\\\[\\+\\-\\(\\)\\!\\@\\`\\~\\.\\%\\&\\^\\:\\;\\'\\|\\{\\}\\[\\]\\?\\,\\<\\>\\=\\#\\*\\/\\$\\\"\\\\]"),
+      Pattern.compile("#\\\\space", Pattern.CASE_INSENSITIVE),
+      Pattern.compile("#\\\\newline", Pattern.CASE_INSENSITIVE),
+      Pattern.compile("#\\\\return", Pattern.CASE_INSENSITIVE),
+      Pattern.compile("#\\\\backspace", Pattern.CASE_INSENSITIVE),
+      Pattern.compile("#\\\\tab", Pattern.CASE_INSENSITIVE),
+      Pattern.compile("#\\\\linefeed", Pattern.CASE_INSENSITIVE),
+      Pattern.compile("#\\\\page", Pattern.CASE_INSENSITIVE),
+      Pattern.compile("#\\\\rubout", Pattern.CASE_INSENSITIVE),
+      Pattern.compile("#\\\\u[a-f0-9][a-f0-9][a-f0-9][a-f0-9]",
+          Pattern.CASE_INSENSITIVE)};
+
+  static boolean isChar(CharSequence s)
+  {
+    for(Pattern p : charMatchers)
+    {
+      if( p.matcher(s).matches() ) return true;
+    }
+    return false;
+  }
+
+  static final Pattern bitVectorMatcher = Pattern.compile("#\\*[01]*");
+
+  static boolean isBitVector(CharSequence s)
+  {
+    return bitVectorMatcher.matcher(s).matches();
+  }
+
+  boolean isKeyword(String text)
+  {
+    if( keywords != null && keywords.contains(text) )
+    {
+      return true;
+    }
+    else
+    {
+      return false;
+    }
+  }
+
+
+  // ========== end matchers ========
+
+}

src/org/lispdev/parser/LispParser.java

+package org.lispdev.parser;
+
+import java.util.List;
+import java.util.ListIterator;
+
+import org.eclipse.core.runtime.Assert;
+import org.lispdev.parser.nodes.Node;
+import org.lispdev.parser.nodes.OperatorNode;
+import org.lispdev.parser.nodes.Sexp;
+import org.lispdev.parser.nodes.TreeRoot;
+import org.lispdev.parser.tokens.ParseType;
+import org.lispdev.parser.tokens.Token;
+import org.lispdev.parser.tokens.TokenEOF;
+
+
+public class LispParser
+{
+  List<Token> tokens;
+  ListIterator<Token> iterator;
+
+  public LispParser(List<Token> t)
+  {
+    tokens = t;
+    iterator = tokens.listIterator();
+  }
+
+  Node newNode;
+
+  public TreeRoot parse()
+  {
+    TreeRoot res = new TreeRoot();
+    iterator = tokens.listIterator();
+    Token t = skipHiddens(res);
+    while(!( t instanceof TokenEOF ) )
+    {
+      t = parse(t);
+      res.add(newNode);
+      t = skipHiddens(t, res);
+    }
+    newNode = res;
+    return res;
+  }
+
+  /**
+   * General parsing starting with token t. At return puts resulting node
+   * into newNode.
+   * @param t should not be hidden
+   * @return next token to parse
+   */
+  private Token parse(Token t)
+  {
+    switch( t.parseType() )
+    {
+      case ATOM :
+        Node n = new Node(t);
+        newNode = n;
+        return iterator.next();
+      case OPERATOR :
+        return parseOperator(t);
+      case OPEN_PAREN :
+        return parseOpenParen(t);
+      default:
+        Assert.isLegal(true,"Parsing is not defined for: "+t.toString());
+        return null;
+    }
+  }
+
+  /**
+   * Can return same token, if it is not hidden.
+   * @param t token to start with
+   * @param n put hiddens into this node
+   * @return next not hidden token (including t)
+   */
+  private Token skipHiddens(Token t, Node n)
+  {
+    while( t.parseType() == ParseType.HIDDEN )
+    {
+      n.addHidden(t);
+      t = iterator.next();
+    }
+    return t;
+  }
+
+  /**
+   * Never returns same token.
+   * @param n put hiddens into this node
+   * @return next not hidden token (excluding t)
+   */
+  private Token skipHiddens(Node n)
+  {
+    return skipHiddens(iterator.next(), n);
+  }
+
+  /**
+   * @param t token having the property that t.isOpenParen() == true
+   */
+  private Token parseOpenParen(Token t)
+  {
+    Sexp res = new Sexp(t);
+    t = iterator.next();
+    while(true)
+    {
+      t = skipHiddens(t, res);
+      // closing paren - finished parsing sexp
+      if( t.parseType() == ParseType.CLOSE_PAREN )
+      {
+        res.add(new Node(t));
+        newNode = res;
+        return iterator.next();
+      }
+      String breakStr = t.isBraking();
+      if( breakStr != null )
+      {
+        res.setError(new ParseError(t.offset(), 1, ")", breakStr));
+        newNode = res;
+        return t;
+      }
+      newNode = null;
+      t = parse(t);
+      Assert.isNotNull(newNode);
+      res.add(newNode);
+    }
+  }
+
+  private Token parseOperator(Token t)
+  {
+    OperatorNode res = new OperatorNode(t);
+    t = skipHiddens(res);
+    String bs = t.isBraking();
+    if( bs != null ) //error
+    {
+      if( bs.equals("(") )
+      {
+        res.setError(
+            new ParseError(t.offset(),1,"expression","new top level"));
+      }
+      else
+      {
+        res.setError(
+            new ParseError(t.offset(),1,"expression",bs));
+      }
+      newNode = res;
+      return t;
+    }
+    newNode = null;
+    t = parse(t);
+    Assert.isNotNull(newNode);
+    res.add(newNode);
+    newNode = res;
+    return t;
+  }
+
+}

src/org/lispdev/parser/ParseError.java

+package org.lispdev.parser;
+
+import org.lispdev.parser.utils.StringUtils;
+
+public class ParseError
+{
+  public final int offset;
+  public final int length;
+  public final String expected;
+  public final String found;
+
+  // if both expected and found are nulls, message: bad token
+
+
+  public ParseError(int offset, int length, String expected, String found)
+  {
+    this.offset = offset;
+    this.length = length;
+    this.expected = (expected == null ? null :
+      StringUtils.escapeSpaces(expected));
+    this.found = (found == null ? null :
+      StringUtils.escapeSpaces(found));
+  }
+
+  @Override
+  public String toString()
+  {
+    return "E("+offset+"+"+length+":"+
+      String.valueOf(expected)+"<-"+String.valueOf(found)+")";
+  }
+}

src/org/lispdev/parser/nodes/AbstractTreeNode.java

+package org.lispdev.parser.nodes;
+
+import java.util.List;
+import java.util.Map;
+
+import org.eclipse.core.runtime.Assert;
+import org.lispdev.parser.ParseError;
+import org.lispdev.parser.tokens.Token;
+import org.lispdev.parser.utils.LispCharBuffer;
+
+public class AbstractTreeNode extends Node
+{
+  //starts and ends with nodes that have tokens
+  protected NodeList children;
+  protected int endOffset; //local
+  protected int startOffset; //local
+  protected int EOLcount; //local
+
+  /**
+   * @param capacity number of regular nodes expected (ignoring hidden)
+   */
+  public AbstractTreeNode(int capacity, int offset)
+  {
+    children = new NodeList(2*capacity-1);
+  }
+
+  @Override
+  public int length()
+  {
+    return endOffset - startOffset;
+  }
+
+  @Override
+  public int EOLcount()
+  {
+    return EOLcount;
+  }
+
+  @Override
+  protected int line(Node n)
+  {
+    if( parent == null )
+      Assert.isLegal(false, "Null parent can only be in TreeRoot");
+
+    int l = parent.line(this);
+    for( Node ni : children )
+    {
+      l += ni.EOLcount();
+      if( ni == n )
+      {
+        return l;
+      }
+    }
+    Assert.isLegal(false, "Node is not a child");
+    return -1;
+  }
+
+  @Override
+  public int localOffset()
+  {
+    return startOffset;
+  }
+
+  @Override
+  public void addHidden(Token c)
+  {
+    if( children.size() == 0
+        || !(children.get(children.size()-1) instanceof Hidden) )
+    {
+      Hidden h = new Hidden(c, this);
+      children.add(h);
+    }
+    else
+    {
+      Assert.isTrue(endOffset == c.localOffset(),
+          "Adding token with missmatched offset");
+      children.get(children.size()-1).addHidden(c);
+    }
+    endOffset += c.length();
+    EOLcount += c.EOLcount();
+  }
+
+  @Override
+  public void add(Node n)
+  {
+    if( children.size() == 0 )
+    {
+      endOffset = n.localOffset();
+      startOffset = endOffset;
+      EOLcount = 0;
+    }
+    else
+    {
+      Assert.isTrue(endOffset == n.localOffset(),
+          "Adding node with missmatched offset");
+    }
+    children.add(n);
+    n.setParent(this);
+    endOffset += n.length();
+    EOLcount = n.EOLcount();
+  }
+
+  @Override
+  public Token firstToken()
+  {
+    return children.get(0).firstToken();
+  }
+
+  @Override
+  public Token lastToken()
+  {
+    return children.get(children.size()-1).lastToken();
+  }
+
+  @Override
+  public void initTokenIterator(boolean atStart)
+  {
+    iteratorPosition = (atStart ? 0 : children.size()-1);
+    children.get(iteratorPosition).initTokenIterator(atStart);
+  }
+
+  /**
+  * Assumes that offset is inside this node
+  * <p>
+  * @param pos assume that binary search returns this position
+  */
+  protected int getNodeWithOffset(int pos)
+  {
+    if( pos < 0 )
+    {
+      //ip is an insertion point, e.g. child[ip] is after offset
+      //so child[ip-1] contains offset
+      final int ip = -pos - 1;
+      Assert.isLegal(ip > 0 && ip <= children.size(),
+          "Contradicts assumption that offset is inside node");
+      return ip - 1;
+    }
+    else
+    {
+      return pos;
+    }
+  }
+
+  @Override
+  public boolean initTokenIterator(int localOffset, boolean next)
+  {
+    if( startOffset > localOffset || endOffset <= localOffset )
+    {
+      iteratorPosition = -1;
+      return false;
+    }
+    final int pos = NodeUtilities.search(children, localOffset);
+    iteratorPosition = getNodeWithOffset(pos);
+    children.get(iteratorPosition).initTokenIterator(localOffset, next);
+    return true;
+  }
+
+  @Override
+  public Token nextToken()
+  {
+    if( iteratorPosition < 0 || iteratorPosition >= children.size() )
+    {
+      iteratorPosition = -1;
+      return null;
+    }
+    Token t = children.get(iteratorPosition).nextToken();
+    if( t != null ) return t;
+    // try next interval
+    ++iteratorPosition;
+    if( iteratorPosition >= children.size() )
+    {
+      iteratorPosition = -1;
+      return null;
+    }
+
+    Node n = children.get(iteratorPosition);
+    n.initTokenIterator(true);
+    return n.nextToken();
+  }
+
+  @Override
+  public Token prevToken()
+  {
+    if( iteratorPosition < 0 || iteratorPosition >= children.size() )
+    {
+      iteratorPosition = -1;
+      return null;
+    }
+    Token t = children.get(iteratorPosition).prevToken();
+    if( t != null ) return t;
+    // try prev interval
+    --iteratorPosition;
+    if( iteratorPosition < 0 ) return null;
+
+    Node n = children.get(iteratorPosition);
+    n.initTokenIterator(true);
+    return n.prevToken();
+  }
+
+  @Override
+  public void appendToStringBuilder(LispCharBuffer sb)
+  {
+    for(Node n : children )
+    {
+      n.appendToStringBuilder(sb);
+    }
+  }
+
+  @Override
+  public String text()
+  {
+    LispCharBuffer sb = new LispCharBuffer(80);
+    appendToStringBuilder(sb);
+    return sb.toString();
+  }
+
+  @Override
+  public void copyTo(Node to)
+  {
+    Assert.isLegal(this.getClass() == to.getClass());
+    AbstractTreeNode t = (AbstractTreeNode)to;
+    t.token = token;
+    t.error = error;
+    t.children = children;
+    for( Node n : children )
+    {
+      n.setParent(t);
+    }
+    t.startOffset = startOffset;
+    t.endOffset = endOffset;
+    t.EOLcount = EOLcount;
+  }
+
+  @Override
+  protected void addToPrintTree(StringBuilder sb, String indent)
+  {
+    sb.append(indent);
+    sb.append(this.getClass().getSimpleName());
+    sb.append(":");
+    String newIndent = indent + AbstractTreeNode.indent;
+    for( Node n : children )
+    {
+      sb.append("\n");
+      n.addToPrintTree(sb, newIndent);
+    }
+  }
+
+  @Override
+  public String toString()
+  {
+    StringBuilder sb = new StringBuilder();
+    addToPrintTree(sb, "");
+    return sb.toString();
+  }
+
+  @Override
+  public boolean hasError()
+  {
+    if( error != null ) return true;
+    for( Node n : children )
+    {
+      if( n.hasError() ) return true;
+    }
+    return false;
+  }
+
+  @Override
+  protected void collectErrors(Map<ParseError,int[]> errors, Integer delta)
+  {
+    super.collectErrors(errors,delta);
+    for( Node n : children )
+    {
+      n.collectErrors(errors,delta);
+    }
+  }
+
+  @Override
+  public List<Node> children()
+  {
+    return children;
+  }
+
+}

src/org/lispdev/parser/nodes/Hidden.java

+package org.lispdev.parser.nodes;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.eclipse.core.runtime.Assert;
+import org.lispdev.parser.ParseError;
+import org.lispdev.parser.tokens.ParseType;
+import org.lispdev.parser.tokens.Token;
+import org.lispdev.parser.tokens.TokenUtils;
+import org.lispdev.parser.utils.LispCharBuffer;
+
+
+public class Hidden extends Node
+{
+  ArrayList<Token> tokens;
+
+  public Hidden(Token t, Node parent)
+  {
+    Assert.isLegal(t.parseType() == ParseType.HIDDEN);
+    token = t; //last token
+    tokens = new ArrayList<Token>(1);
+    setParent(parent);
+    tokens.add(t);
+    t.setParent(this);
+  }
+
+  @Override
+  public String toString()
+  {
+    return "H:"+(error != null ? error : "")+tokens.toString();
+  }
+
+  @Override
+  public Token firstToken()
+  {
+    return tokens.get(0);
+  }
+
+  @Override
+  public Token lastToken()
+  {
+    return token;
+  }
+
+  @Override
+  public int EOLcount()
+  {
+    int res = 0;
+    for( Token t : tokens )
+    {
+      res += t.EOLcount();
+    }
+    return res;
+  }
+
+  @Override
+  public void add(Node o)
+  {
+    Assert.isLegal(false,"Hidden cannot contain nodes");
+  }
+
+  @Override
+  public void addHidden(Token c)
+  {
+    Assert.isLegal(c.parseType() == ParseType.HIDDEN);
+    Assert.isTrue(token.localOffset() + token.length() == c.localOffset(),
+        "Adding token with missmatched offset");
+    tokens.add(c);
+    c.setParent(this);
+    token = c;
+  }
+
+  @Override
+  public int length()
+  {
+    return token.localOffset() + token.length() - localOffset();
+  }
+
+  @Override
+  public int localOffset()
+  {
+    return firstToken().localOffset();
+  }
+
+  @Override
+  public void initTokenIterator(boolean atStart)
+  {
+    iteratorPosition = (atStart ? 0 : tokens.size());
+  }
+
+  @Override
+  public boolean initTokenIterator(int localOffset, boolean next)
+  {
+    final int pos = TokenUtils.search(tokens, localOffset);
+    if( pos < 0 )
+    {
+      //insertion point, child[ip] is after position
+      //so child[ip-1] contains offset
+      final int ip = -pos - 1;
+      if( ip == 0 )
+      {
+        iteratorPosition = -1;
+        return false;
+      }
+      if( ip == tokens.size() )
+      {
+        if( lastToken().containsLocalOffset(localOffset) )
+        {
+          iteratorPosition = (next ? ip - 1 : ip );
+          return true;
+        }
+        else
+        {
+          iteratorPosition = -1;
+          return false;
+        }
+      }
+      iteratorPosition = (next ? ip - 1 : ip );
+      return true;
+    }
+    else
+    {
+      iteratorPosition = (next ? pos : pos + 1 );