Commits

Jan Lahoda  committed 562a96e

Improving the editor support for the hints files.

  • Participants
  • Parent commits 5eff359

Comments (0)

Files changed (6)

File file/src/org/netbeans/modules/jackpot30/file/DeclarativeHintLexer.java

 
 package org.netbeans.modules.jackpot30.file;
 
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.netbeans.api.lexer.PartType;
 import org.netbeans.api.lexer.Token;
 import org.netbeans.spi.lexer.Lexer;
 import org.netbeans.spi.lexer.LexerInput;
     private final TokenFactory<DeclarativeHintTokenId> fact;
 
     private int state;
+    private boolean wasSnippetPart;
 
     public DeclarativeHintLexer(LexerRestartInfo<DeclarativeHintTokenId> info) {
         input = info.input();
         fact  = info.tokenFactory();
 
         if (info.state() != null) {
-            state = (Integer) info.state();
+            State s = (State) info.state();
+            
+            state = s.state;
+            wasSnippetPart = s.wasSnippetPart;
         }
     }
 
 
         input.backup(1);
 
-        Token<DeclarativeHintTokenId> readWhiteSpace = readWhiteSpace();
+        if (state == 0 || state == 2) {
+            Token<DeclarativeHintTokenId> readWhiteSpace = readWhiteSpace();
 
-        if (readWhiteSpace != null) {
-            return readWhiteSpace;
+            if (readWhiteSpace != null) {
+                return readWhiteSpace;
+            }
         }
 
         int read;
                 if (read == ':') {
                     state = 1;
                     return fact.createToken(DeclarativeHintTokenId.COLON);
-                } else {
-                    return fact.createToken(DeclarativeHintTokenId.ERROR);
                 }
             case 1:
-                state = 2;
-                return readSnippet();
+                s = readSnippet();
+
+                if (wasSnippetPart) {
+                    state = 4;
+                } else {
+                    state = 2;
+                }
+                return s;
             case 2:
                 read = input.read();
 
                 }
 
                 if (read == ';') {
-                    state = 0;
-                    return fact.createToken(DeclarativeHintTokenId.SEMICOLON);
+                    int next = input.read();
+
+                    if (next == ';') {
+                        state = 0;
+                        return fact.createToken(DeclarativeHintTokenId.DOUBLE_SEMICOLON);
+                    }
+
+                    if (next != LexerInput.EOF) {
+                        input.backup(1);
+                    }
+
+                    return fact.createToken(DeclarativeHintTokenId.ERROR);
                 }
 
                 if (read == '"') {
                         return s;
                     }
                 }
-
-                return fact.createToken(DeclarativeHintTokenId.ERROR);
             case 3:
+                s = readSnippet();
+                if (wasSnippetPart) {
+                    state = 4;
+                } else {
+                    state = 2;
+                }
+                return s;
+            case 4:
                 state = 2;
-                return readSnippet();
+                return readType();
         }
 
         throw new IllegalStateException("" + state);
     }
 
     public Object state() {
-        return state;
+        return new State(state, wasSnippetPart);
     }
 
-    public void release() {
-    }
+    public void release() {}
 
     private Token<DeclarativeHintTokenId> readWhiteSpace() {
         int read = input.read();
             create = true;
         }
 
-        input.backup(1);
-        
+        if (read != LexerInput.EOF) {
+            input.backup(1);
+        }
+
+        if (read != '"' && read != ':' && read != LexerInput.EOF) {
+            input.backup(input.readLength());
+            return null;
+        }
+
         if (create) {
             return fact.createToken(DeclarativeHintTokenId.WHITESPACE);
         }
     }
 
     private Token<DeclarativeHintTokenId> readSnippet() {
+        boolean wasSnippetPartCopy = this.wasSnippetPart;
+        
         while (input.read() != LexerInput.EOF) {
-            if (input.readText().toString().endsWith("=>") || input.readText().toString().endsWith(";")) {
-                input.backup(input.readText().toString().endsWith("=>") ? 2 : 1);
+            if (input.readText().toString().endsWith("=>") || input.readText().toString().endsWith(";;")) {
+                input.backup(2);
 
-                while (Character.isWhitespace(input.readText().charAt(input.readText().length() - 1))) {
-                    input.backup(1);
-                }
+//                while (Character.isWhitespace(input.readText().charAt(input.readText().length() - 1))) {
+//                    input.backup(1);
+//                }
 
-                return fact.createToken(DeclarativeHintTokenId.PATTERN);
+                this.wasSnippetPart = false;
+
+                return fact.createToken(DeclarativeHintTokenId.PATTERN, input.readLength(), wasSnippetPartCopy ? PartType.END : PartType.COMPLETE);
+            }
+
+            Matcher m = VARIABLE_WITH_TYPE_RE.matcher(input.readText());
+
+            if (m.find() && m.end() == input.readLength()) {
+                input.backup(1);
+                this.wasSnippetPart = true;
+                
+                return fact.createToken(DeclarativeHintTokenId.PATTERN, input.readLength(), wasSnippetPartCopy ? PartType.MIDDLE : PartType.START);
             }
         }
 
         if (input.read() != LexerInput.EOF) {
-            return fact.createToken(DeclarativeHintTokenId.PATTERN);
+            this.wasSnippetPart = false;
+            return fact.createToken(DeclarativeHintTokenId.PATTERN, input.readLength(), wasSnippetPartCopy ? PartType.END : PartType.COMPLETE);
         }
 
         return null;
     }
+
+    private Token<DeclarativeHintTokenId> readType() {
+        int read = input.read();
+
+        assert read == '{';
+
+        while ((read != LexerInput.EOF) && read != '}') {
+            read = input.read();
+        }
+
+        return fact.createToken(DeclarativeHintTokenId.TYPE);
+    }
+
+    private static final Pattern VARIABLE_WITH_TYPE_RE = Pattern.compile("\\$[A-Za-z0-9_]\\{");
+
+    private static final class State {
+        private final int state;
+        private final boolean wasSnippetPart;
+
+        public State(int state, boolean wasSnippetPart) {
+            this.state = state;
+            this.wasSnippetPart = wasSnippetPart;
+        }
+
+    }
 }

File file/src/org/netbeans/modules/jackpot30/file/DeclarativeHintTokenId.java

 
     DISPLAY_NAME("string"),
     COLON("operator"),
+    TYPE("pattern"),
     PATTERN("pattern"),
     LEADS_TO("operator"),
     WHITESPACE("whitespace"),
-    SEMICOLON("operator"),
+    DOUBLE_SEMICOLON("operator"),
     ERROR("error");
 
     private final String cat;
             switch (token.id()) {
                 case PATTERN:
                     return LanguageEmbedding.create(Language.find("text/x-java"), 0, 0);
+                case TYPE:
+                    return LanguageEmbedding.create(Language.find("text/x-java"), 1, 1);
                 default:
                     return null;
             }

File file/src/org/netbeans/modules/jackpot30/file/EmbeddingProviderImpl.java

+package org.netbeans.modules.jackpot30.file;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.netbeans.api.lexer.PartType;
+import org.netbeans.api.lexer.Token;
+import org.netbeans.api.lexer.TokenSequence;
+import org.netbeans.modules.parsing.api.Embedding;
+import org.netbeans.modules.parsing.api.Snapshot;
+import org.netbeans.modules.parsing.spi.EmbeddingProvider;
+import org.netbeans.modules.parsing.spi.SchedulerTask;
+import org.netbeans.modules.parsing.spi.TaskFactory;
+
+/**
+ *
+ * @author lahvac
+ */
+public class EmbeddingProviderImpl extends EmbeddingProvider {
+
+    @Override
+    public List<Embedding> getEmbeddings(Snapshot snapshot) {
+        int index = 0;
+        List<Embedding> result = new LinkedList<Embedding>();
+        TokenSequence<DeclarativeHintTokenId> ts = snapshot.getTokenHierarchy().tokenSequence(DeclarativeHintTokenId.language());
+
+        List<Token<DeclarativeHintTokenId>> parts = new LinkedList<Token<DeclarativeHintTokenId>>();
+        List<Embedding> declarations = new LinkedList<Embedding>();
+        Token<DeclarativeHintTokenId> previous = null;
+
+        while (ts.moveNext()) {
+            Token<DeclarativeHintTokenId> t = ts.token();
+            
+            if (t.id() == DeclarativeHintTokenId.PATTERN) {
+                parts.add(t);
+
+                if (t.partType() != PartType.END && t.partType() != PartType.COMPLETE) {
+                    previous = t;
+                    continue;
+                }
+
+                List<Embedding> embeddingParts = new LinkedList<Embedding>();
+
+                String prefix = SNIPPET_PATTERN_PREFIX_PART1;
+
+                prefix = prefix.replaceAll("\\{0\\}", "" + (index++));
+
+                embeddingParts.add(snapshot.create(prefix, "text/x-java"));
+
+                boolean first = true;
+
+                for (Embedding d : declarations) {
+                    if (!first) {
+                        embeddingParts.add(snapshot.create(", ", "text/x-java"));
+                    }
+                    first = false;
+                    embeddingParts.add(d);
+                }
+
+                embeddingParts.add(snapshot.create(SNIPPET_PATTERN_PREFIX_PART2, "text/x-java"));
+
+                for (Token<DeclarativeHintTokenId> tokenPart : parts) {
+                    embeddingParts.add(snapshot.create(tokenPart.offset(null), tokenPart.length(), "text/x-java"));
+                }
+                
+                embeddingParts.add(snapshot.create(SNIPPET_PATTERN_SUFFIX, "text/x-java"));
+
+                Embedding e = Embedding.create(embeddingParts);
+
+                result.add(e);
+
+                parts.clear();
+            }
+
+            if (t.id() == DeclarativeHintTokenId.TYPE && previous != null && previous.id() == DeclarativeHintTokenId.PATTERN) {
+                Matcher m = VARIABLE_RE.matcher(previous.text().toString());
+
+                if (m.matches()) {
+                    String text = t.text().toString();
+
+                    Embedding e1 = snapshot.create(ts.offset() + 1, text.length() - 2, "text/x-java");
+                    Embedding e2 = snapshot.create(" " + m.group(1), "text/x-java");
+
+                    declarations.add(Embedding.create(Arrays.asList(e1, e2)));
+                }
+            }
+
+            if (t.id() == DeclarativeHintTokenId.DOUBLE_SEMICOLON) {
+                declarations.clear();
+            }
+
+            previous = t;
+        }
+
+        if (result.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        result.add(0, snapshot.create(GLOBAL_PATTERN_PREFIX, "text/x-java"));
+        result.add(snapshot.create(GLOBAL_PATTERN_SUFFIX, "text/x-java"));
+        
+        return Collections.singletonList(Embedding.create(result));
+    }
+
+    private static final Pattern VARIABLE_RE = Pattern.compile(".*(\\$[A-Za-z0-9_]+)");
+
+    @Override
+    public int getPriority() {
+        return 100;
+    }
+
+    @Override
+    public void cancel() {}
+
+    private static final String GLOBAL_PATTERN_PREFIX = "package $; class $ {\n";
+    private static final String GLOBAL_PATTERN_SUFFIX = "\n}\n";
+    private static final String SNIPPET_PATTERN_PREFIX_PART1 = "private void ${0}(";
+    private static final String SNIPPET_PATTERN_PREFIX_PART2 = ") throws Throwable {\n";
+    private static final String SNIPPET_PATTERN_SUFFIX = " ;\n}\n";
+
+    public static final class FactoryImpl extends TaskFactory {
+
+        @Override
+        public Collection<? extends SchedulerTask> create(Snapshot snapshot) {
+            return Collections.singletonList(new EmbeddingProviderImpl());
+        }
+        
+    }
+
+}

File file/src/org/netbeans/modules/jackpot30/file/resources/layer.xml

                         </folder>
                     </folder>
                 </folder>
+                <file name="org-netbeans-modules-jackpot30-file-EmbeddingProviderImpl$FactoryImpl.instance" />
+                <file name="org-netbeans-modules-java-editor-semantic-HighlightsLayerFactoryImpl.instance" />
+                <folder name="CompletionProviders">
+                    <file name="org-netbeans-modules-editor-java-JavaCompletionProvider.instance"/>
+                </folder>
+                <folder name="HyperlinkProviders">
+                    <file name="JavaHyperlinkProvider.instance">
+                        <attr name="instanceClass" stringvalue="org.netbeans.modules.java.editor.hyperlink.JavaHyperlinkProvider"/>
+                        <attr name="instanceOf" stringvalue="org.netbeans.lib.editor.hyperlink.spi.HyperlinkProviderExt"/>
+                    </file>
+                </folder>
+
             </folder>
         </folder>
     </folder>

File file/test/unit/src/org/netbeans/modules/jackpot30/file/DeclarativeHintLexerTest.java

+package org.netbeans.modules.jackpot30.file;
+
+import org.junit.Test;
+import org.netbeans.api.lexer.PartType;
+import static org.junit.Assert.*;
+import org.netbeans.api.lexer.Token;
+import org.netbeans.api.lexer.TokenHierarchy;
+import org.netbeans.api.lexer.TokenSequence;
+
+import static org.netbeans.modules.jackpot30.file.DeclarativeHintTokenId.*;
+
+/**
+ *
+ * @author lahvac
+ */
+public class DeclarativeHintLexerTest {
+
+    public DeclarativeHintLexerTest() {
+    }
+
+    @Test
+    public void testSimple() {
+        String text = " \"test\": 1 + 1 => \"fix\": 1 + 1;;";
+        TokenHierarchy<?> hi = TokenHierarchy.create(text, language());
+        TokenSequence<?> ts = hi.tokenSequence();
+        assertNextTokenEquals(ts, WHITESPACE, " ");
+        assertNextTokenEquals(ts, DISPLAY_NAME, "\"test\"");
+        assertNextTokenEquals(ts, COLON, ":");
+        assertNextTokenEquals(ts, PATTERN, " 1 + 1 ");
+        assertNextTokenEquals(ts, LEADS_TO, "=>");
+        assertNextTokenEquals(ts, WHITESPACE, " ");
+        assertNextTokenEquals(ts, DISPLAY_NAME, "\"fix\"");
+        assertNextTokenEquals(ts, COLON, ":");
+        assertNextTokenEquals(ts, PATTERN, " 1 + 1");
+        assertNextTokenEquals(ts, DOUBLE_SEMICOLON, ";;");
+
+        assertFalse(ts.moveNext());
+    }
+
+    @Test
+    public void testSimpleNoDisplayNames() {
+        String text = " 1 + 1 => 1 + 1;;";
+        TokenHierarchy<?> hi = TokenHierarchy.create(text, language());
+        TokenSequence<?> ts = hi.tokenSequence();
+        assertNextTokenEquals(ts, PATTERN, " 1 + 1 ");
+        assertNextTokenEquals(ts, LEADS_TO, "=>");
+        assertNextTokenEquals(ts, PATTERN, " 1 + 1");
+        assertNextTokenEquals(ts, DOUBLE_SEMICOLON, ";;");
+
+        assertFalse(ts.moveNext());
+    }
+    
+    @Test
+    public void testWhitespaceAtTheEnd() {
+        String text = " 1 + 1 => 1 + 1;; ";
+        TokenHierarchy<?> hi = TokenHierarchy.create(text, language());
+        TokenSequence<?> ts = hi.tokenSequence();
+        assertNextTokenEquals(ts, PATTERN, " 1 + 1 ");
+        assertNextTokenEquals(ts, LEADS_TO, "=>");
+        assertNextTokenEquals(ts, PATTERN, " 1 + 1");
+        assertNextTokenEquals(ts, DOUBLE_SEMICOLON, ";;");
+        assertNextTokenEquals(ts, WHITESPACE, " ");
+
+        assertFalse(ts.moveNext());
+    }
+
+    @Test
+    public void testVariablesWithTypes1() {
+        String text = " $1{int} + $2{int} => 1 + 1;; ";
+        TokenHierarchy<?> hi = TokenHierarchy.create(text, language());
+        TokenSequence<?> ts = hi.tokenSequence();
+        assertNextTokenEquals(ts, PATTERN, PartType.START, " $1");
+        assertNextTokenEquals(ts, TYPE,  "{int}");
+        assertNextTokenEquals(ts, PATTERN, PartType.MIDDLE, " + $2");
+        assertNextTokenEquals(ts, TYPE,  "{int}");
+        assertNextTokenEquals(ts, PATTERN, PartType.END, " ");
+        assertNextTokenEquals(ts, LEADS_TO, "=>");
+        assertNextTokenEquals(ts, PATTERN, " 1 + 1");
+        assertNextTokenEquals(ts, DOUBLE_SEMICOLON, ";;");
+        assertNextTokenEquals(ts, WHITESPACE, " ");
+
+        assertFalse(ts.moveNext());
+    }
+
+    public static void assertNextTokenEquals(TokenSequence<?> ts, DeclarativeHintTokenId id, String text) {
+        assertNextTokenEquals(ts, id, PartType.COMPLETE, text);
+    }
+
+    public static void assertNextTokenEquals(TokenSequence<?> ts, DeclarativeHintTokenId id, PartType pt, String text) {
+        assertTrue(ts.moveNext());
+
+        Token t = ts.token();
+
+        assertNotNull(t);
+        assertEquals(id, t.id());
+        assertEquals(pt, t.partType());
+        assertEquals(text, t.text().toString());
+    }
+
+}

File file/test/unit/src/org/netbeans/modules/jackpot30/file/EmbeddingProviderImplTest.java

+package org.netbeans.modules.jackpot30.file;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import javax.swing.text.Document;
+import org.netbeans.api.java.source.SourceUtilsTestUtil;
+import org.netbeans.api.java.source.TestUtilities;
+import org.netbeans.api.lexer.InputAttributes;
+import org.netbeans.api.lexer.Language;
+import org.netbeans.api.lexer.LanguagePath;
+import org.netbeans.api.lexer.Token;
+import org.netbeans.junit.NbTestCase;
+import org.netbeans.modules.parsing.api.Embedding;
+import org.netbeans.modules.parsing.api.ParserManager;
+import org.netbeans.modules.parsing.api.ResultIterator;
+import org.netbeans.modules.parsing.api.Source;
+import org.netbeans.modules.parsing.api.UserTask;
+import org.netbeans.spi.lexer.LanguageEmbedding;
+import org.netbeans.spi.lexer.LanguageProvider;
+import org.openide.cookies.EditorCookie;
+import org.openide.filesystems.FileObject;
+import org.openide.filesystems.FileUtil;
+import org.openide.loaders.DataObject;
+import org.openide.util.lookup.ServiceProvider;
+
+
+/**
+ *
+ * @author lahvac
+ */
+public class EmbeddingProviderImplTest extends NbTestCase {
+
+    public EmbeddingProviderImplTest(String name) {
+        super(name);
+    }
+
+    public void testSimpleEmbedding() throws Exception {
+        performEmbeddingTest("\"\": 1 + 1 => \"\":1 + 1;;",
+                             "package $; class $ {\n" +
+                             "     private void $0() throws Throwable {\n" +
+                             "         1 + 1 ;\n" +
+                             "     }\n" +
+                             "     private void $1() throws Throwable {\n" +
+                             "         1 + 1 ;\n" +
+                             "     }\n" +
+                             "}\n");
+    }
+
+    public void testEmbeddingWithVariables1() throws Exception {
+        performEmbeddingTest("\"\": $1{int} + $2{double} => \"\":1 + 1;;",
+                             "package $; class $ {\n" +
+                             "     private void $0(int $1, double $2) throws Throwable {\n" +
+                             "         $1 + $2 ;\n" +
+                             "     }\n" +
+                             "     private void $1(int $1, double $2) throws Throwable {\n" +
+                             "         1 + 1 ;\n" +
+                             "     }\n" +
+                             "}\n");
+    }
+
+    public void testEmbeddingWithVariables2() throws Exception {
+        performEmbeddingTest("\"\": $1{int} + $2{double} => \"\":1 + 1;;\"\": 1 + 1 => \"\":1 + 1;;",
+                             "package $; class $ {\n" +
+                             "     private void $0(int $1, double $2) throws Throwable {\n" +
+                             "         $1 + $2 ;\n" +
+                             "     }\n" +
+                             "     private void $1(int $1, double $2) throws Throwable {\n" +
+                             "         1 + 1 ;\n" +
+                             "     }\n" +
+                             "     private void $2() throws Throwable {\n" +
+                             "         1 + 1 ;\n" +
+                             "     }\n" +
+                             "     private void $3() throws Throwable {\n" +
+                             "         1 + 1 ;\n" +
+                             "     }\n" +
+                             "}\n");
+    }
+
+    private void performEmbeddingTest(String code, String... golden) throws Exception {
+        prepareTest(code, -1);
+
+        Source s = Source.create(doc);
+        final List<String> output = new LinkedList<String>();
+
+        ParserManager.parse(Collections.singletonList(s), new UserTask() {
+            @Override
+            public void run(ResultIterator resultIterator) throws Exception {
+                for (Embedding e : new EmbeddingProviderImpl().getEmbeddings(resultIterator.getSnapshot())) {
+                    output.add(e.getSnapshot().getText().toString());
+                }
+            }
+        });
+
+        Iterator<String> goldenIt = Arrays.asList(golden).iterator();
+        Iterator<String> outputIt = output.iterator();
+
+        while (goldenIt.hasNext() && outputIt.hasNext()) {
+            assertEquals(goldenIt.next().replaceAll("[ \t\n]+", " "), outputIt.next().replaceAll("[ \t\n]+", " "));
+        }
+
+        assertEquals(output.toString(), goldenIt.hasNext(), outputIt.hasNext());
+    }
+    
+    @Override
+    protected void setUp() throws Exception {
+        SourceUtilsTestUtil.prepareTest(new String[0], new Object[0]);
+        super.setUp();
+    }
+
+    protected void prepareTest(String code, int testIndex) throws Exception {
+        File workDirWithIndexFile = testIndex != (-1) ? new File(getWorkDir(), Integer.toString(testIndex)) : getWorkDir();
+        FileObject workDirWithIndex = FileUtil.toFileObject(workDirWithIndexFile);
+
+        if (workDirWithIndex != null) {
+            workDirWithIndex.delete();
+        }
+
+        workDirWithIndex = FileUtil.createFolder(workDirWithIndexFile);
+
+        assertNotNull(workDirWithIndexFile);
+
+        FileUtil.setMIMEType("hint", "text/x-javahints");
+
+        FileObject sourceRoot = workDirWithIndex.createFolder("src");
+//        FileObject buildRoot  = workDirWithIndex.createFolder("build");
+//        FileObject cache = workDirWithIndex.createFolder("cache");
+
+        FileObject data = FileUtil.createData(sourceRoot, "rule.hint");
+
+        TestUtilities.copyStringToFile(data, code);
+
+        data.refresh();
+
+//        SourceUtilsTestUtil.prepareTest(sourceRoot, buildRoot, cache);
+
+        DataObject od = DataObject.find(data);
+        EditorCookie ec = od.getLookup().lookup(EditorCookie.class);
+
+        assertNotNull(ec);
+
+        doc = ec.openDocument();
+
+        doc.putProperty(Language.class, DeclarativeHintTokenId.language());
+        doc.putProperty("mimeType", "text/x-javahints");
+    }
+
+    private Document doc;
+
+    @ServiceProvider(service=LanguageProvider.class)
+    public static final class JavacParserProvider extends LanguageProvider {
+
+        @Override
+        public Language<?> findLanguage(String mimeType) {
+            if (mimeType.equals("text/x-javahints")) {
+                return DeclarativeHintTokenId.language();
+            }
+
+            return null;
+        }
+
+        @Override
+        public LanguageEmbedding<?> findLanguageEmbedding(Token<?> token, LanguagePath languagePath, InputAttributes inputAttributes) {
+            return null;
+        }
+
+    }
+    
+}