Commits

Jan Lahoda committed 439ad54

Generating index statistics, source indexer should be using the correct encoding now.

Comments (0)

Files changed (6)

remoting/server/indexer/impl/src/org/netbeans/modules/jackpot30/backend/impl/OptionProcessorImpl.java

 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.URL;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.Set;
 import java.util.jar.JarOutputStream;
 import java.util.zip.ZipEntry;
 import org.apache.lucene.analysis.KeywordAnalyzer;
 import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.FSDirectory;
 import org.netbeans.api.java.classpath.ClassPath;
 import org.netbeans.api.project.ui.OpenProjects;
 import org.netbeans.api.sendopts.CommandException;
 import org.netbeans.modules.jackpot30.backend.impl.spi.IndexAccessor;
+import org.netbeans.modules.jackpot30.backend.impl.spi.StatisticsGenerator;
 import org.netbeans.modules.parsing.impl.indexing.CacheFolder;
 import org.netbeans.spi.java.classpath.support.ClassPathSupport;
 import org.netbeans.spi.project.support.ant.PropertyUtils;
             }
         }
 
+        Map<String, Long> statistics = Collections.emptyMap();
+        IndexReader r = null;
+
+        try {
+            r = IndexReader.open(FSDirectory.open(FileUtil.toFile(cacheTemp)), true);
+
+            statistics = StatisticsGenerator.generateStatistics(r);
+        } catch (CorruptIndexException ex) {
+            Exceptions.printStackTrace(ex);
+        } catch (IOException ex) {
+            Exceptions.printStackTrace(ex);
+        } finally {
+            if (r != null) {
+                try {
+                    r.close();
+                } catch (IOException ex) {
+                    Exceptions.printStackTrace(ex);
+                }
+            }
+        }
+
         JarOutputStream out = null;
         InputStream segments = null;
 
                     out.write((",\n\"" + infoValue.substring(0, eqSign) + "\": \"" + infoValue.substring(eqSign + 1) + "\"").getBytes("UTF-8"));
                 }
             }
+            out.write(",\n \"statistics\" : {\n".getBytes("UTF-8"));
+            boolean wasEntry = false;
+            for (Entry<String, Long> e : statistics.entrySet()) {
+                if (wasEntry) out.write(", \n".getBytes("UTF-8"));
+                out.write(("\"" + e.getKey() + "\" : " + e.getValue()).getBytes("UTF-8"));
+                wasEntry = true;
+            }
+            out.write("\n}\n".getBytes("UTF-8"));
             out.write("\n}\n".getBytes("UTF-8"));
 
             for (FileObject s : cacheFolder.getChildren()) {

remoting/server/indexer/impl/src/org/netbeans/modules/jackpot30/backend/impl/spi/StatisticsGenerator.java

+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2012 Oracle and/or its affiliates. All rights reserved.
+ *
+ * Oracle and Java are registered trademarks of Oracle and/or its affiliates.
+ * Other names may be trademarks of their respective owners.
+ *
+ * The contents of this file are subject to the terms of either the GNU
+ * General Public License Version 2 only ("GPL") or the Common
+ * Development and Distribution License("CDDL") (collectively, the
+ * "License"). You may not use this file except in compliance with the
+ * License. You can obtain a copy of the License at
+ * http://www.netbeans.org/cddl-gplv2.html
+ * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+ * specific language governing permissions and limitations under the
+ * License.  When distributing the software, include this License Header
+ * Notice in each file and include the License file at
+ * nbbuild/licenses/CDDL-GPL-2-CP.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the GPL Version 2 section of the License file that
+ * accompanied this code. If applicable, add the following below the
+ * License Header, with the fields enclosed by brackets [] replaced by
+ * your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ *
+ * If you wish your version of this file to be governed by only the CDDL
+ * or only the GPL Version 2, indicate your decision by adding
+ * "[Contributor] elects to include this software in this distribution
+ * under the [CDDL or GPL Version 2] license." If you do not indicate a
+ * single choice of license, a recipient has the option to distribute
+ * your version of this file under either the CDDL, the GPL Version 2 or
+ * to extend the choice of license to its licensees as provided above.
+ * However, if you add GPL Version 2 code and therefore, elected the GPL
+ * Version 2 license, then the option applies only if the new code is
+ * made subject to such option by the copyright holder.
+ *
+ * Contributor(s):
+ *
+ * Portions Copyrighted 2012 Sun Microsystems, Inc.
+ */
+package org.netbeans.modules.jackpot30.backend.impl.spi;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.FieldSelectorResult;
+import org.apache.lucene.index.IndexReader;
+import org.openide.util.Lookup;
+
+/**
+ *
+ * @author lahvac
+ */
+public abstract class StatisticsGenerator {
+
+    protected abstract void amendStatistics(IndexReader r, Document doc) throws IOException;
+
+    public static Map<String, Long> generateStatistics(IndexReader r) throws IOException {
+        statistics = new HashMap<String, Long>();
+
+        Collection<? extends StatisticsGenerator> generators = Lookup.getDefault().lookupAll(StatisticsGenerator.class);
+
+        int maxDocs = r.maxDoc();
+
+        for (int d = 0; d < maxDocs; d++) {
+            Document doc = r.document(d, new FieldSelector() {
+                @Override public FieldSelectorResult accept(String string) {
+                    return FieldSelectorResult.LAZY_LOAD;
+                }
+            });
+
+            for (StatisticsGenerator sg : generators) {
+                sg.amendStatistics(r, doc);
+            }
+        }
+        
+        Map<String, Long> result = statistics;
+
+        statistics = null;
+
+        return result;
+    }
+
+    private static Map<String, Long> statistics;
+
+    protected final void increment(String key) {
+        add(key, 1);
+    }
+
+    protected void add(String key, long count) {
+        Long val = statistics.get(key);
+
+        if (val == null) val = 0L;
+
+        statistics.put(key, val + count);
+    }
+
+}

remoting/server/indexer/source/nbproject/project.xml

                     </run-dependency>
                 </dependency>
                 <dependency>
+                    <code-name-base>org.netbeans.modules.queries</code-name-base>
+                    <build-prerequisite/>
+                    <compile-dependency/>
+                    <run-dependency>
+                        <release-version>1</release-version>
+                        <specification-version>1.29</specification-version>
+                    </run-dependency>
+                </dependency>
+                <dependency>
                     <code-name-base>org.openide.filesystems</code-name-base>
                     <build-prerequisite/>
                     <compile-dependency/>
                         <specification-version>7.48</specification-version>
                     </run-dependency>
                 </dependency>
+                <dependency>
+                    <code-name-base>org.openide.util.lookup</code-name-base>
+                    <build-prerequisite/>
+                    <compile-dependency/>
+                    <run-dependency>
+                        <specification-version>8.16</specification-version>
+                    </run-dependency>
+                </dependency>
             </module-dependencies>
             <public-packages/>
         </data>

remoting/server/indexer/source/src/org/netbeans/modules/jackpot30/indexer/source/SourceIndexer.java

 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
-import java.net.URL;
+import java.nio.charset.Charset;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.apache.lucene.document.CompressionTools;
 import org.apache.lucene.document.Field.Index;
 import org.apache.lucene.document.Field.Store;
 import org.netbeans.api.editor.mimelookup.MimeRegistration;
+import org.netbeans.api.queries.FileEncodingQuery;
 import org.netbeans.modules.jackpot30.backend.impl.spi.IndexAccessor;
 import org.netbeans.modules.parsing.spi.indexing.Context;
 import org.netbeans.modules.parsing.spi.indexing.CustomIndexer;
 import org.netbeans.modules.parsing.spi.indexing.CustomIndexerFactory;
 import org.netbeans.modules.parsing.spi.indexing.Indexable;
+import org.openide.filesystems.FileObject;
+import org.openide.filesystems.URLMapper;
 
 /**
  *
                 String relPath = IndexAccessor.getCurrent().getPath(i.getURL());
 
                 if (relPath == null) continue;
+
+                FileObject file = URLMapper.findFileObject(i.getURL());
+
+                if (file == null) {
+                    //TODO: log
+                    continue;
+                }
                 
                 Document doc = new Document();
 
                 doc.add(new Field("relativePath", relPath, Store.YES, Index.NOT_ANALYZED));
-                doc.add(new Field(KEY_CONTENT, CompressionTools.compressString(readFully(i.getURL())), Store.YES));
+                doc.add(new Field(KEY_CONTENT, CompressionTools.compressString(readFully(file)), Store.YES));
+                doc.add(new Field("fileMimeType", file.getMIMEType(), Store.YES, Index.NO));
+                doc.add(new Field("sizeInBytes", Long.toString(file.getSize()), Store.YES, Index.NO));
 
                 IndexAccessor.getCurrent().getIndexWriter().addDocument(doc);
             }
         }
     }
 
-    private static String readFully(URL source) throws IOException {
+    private static String readFully(FileObject source) throws IOException {
         Reader in = null;
         StringBuilder result = new StringBuilder();
 
         try {
-            in = new BufferedReader(new InputStreamReader(source.openStream(), "UTF-8"));
+            Charset charset = FileEncodingQuery.getEncoding(source);
+            in = new BufferedReader(new InputStreamReader(source.getInputStream(), charset));
 
             int read;
 

remoting/server/indexer/source/src/org/netbeans/modules/jackpot30/indexer/source/SourceStatisticsGenerator.java

+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2012 Oracle and/or its affiliates. All rights reserved.
+ *
+ * Oracle and Java are registered trademarks of Oracle and/or its affiliates.
+ * Other names may be trademarks of their respective owners.
+ *
+ * The contents of this file are subject to the terms of either the GNU
+ * General Public License Version 2 only ("GPL") or the Common
+ * Development and Distribution License("CDDL") (collectively, the
+ * "License"). You may not use this file except in compliance with the
+ * License. You can obtain a copy of the License at
+ * http://www.netbeans.org/cddl-gplv2.html
+ * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+ * specific language governing permissions and limitations under the
+ * License.  When distributing the software, include this License Header
+ * Notice in each file and include the License file at
+ * nbbuild/licenses/CDDL-GPL-2-CP.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the GPL Version 2 section of the License file that
+ * accompanied this code. If applicable, add the following below the
+ * License Header, with the fields enclosed by brackets [] replaced by
+ * your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ *
+ * If you wish your version of this file to be governed by only the CDDL
+ * or only the GPL Version 2, indicate your decision by adding
+ * "[Contributor] elects to include this software in this distribution
+ * under the [CDDL or GPL Version 2] license." If you do not indicate a
+ * single choice of license, a recipient has the option to distribute
+ * your version of this file under either the CDDL, the GPL Version 2 or
+ * to extend the choice of license to its licensees as provided above.
+ * However, if you add GPL Version 2 code and therefore, elected the GPL
+ * Version 2 license, then the option applies only if the new code is
+ * made subject to such option by the copyright holder.
+ *
+ * Contributor(s):
+ *
+ * Portions Copyrighted 2012 Sun Microsystems, Inc.
+ */
+package org.netbeans.modules.jackpot30.indexer.source;
+
+import java.io.IOException;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.IndexReader;
+import org.netbeans.modules.jackpot30.backend.impl.spi.StatisticsGenerator;
+import org.openide.util.lookup.ServiceProvider;
+
+/**
+ *
+ * @author lahvac
+ */
+@ServiceProvider(service=StatisticsGenerator.class)
+public class SourceStatisticsGenerator extends StatisticsGenerator {
+
+    @Override
+    protected void amendStatistics(IndexReader r, Document doc) throws IOException {
+        if (doc.getFieldable("relativePath") != null) {
+            increment("files/total");
+            Fieldable mimeType = doc.getFieldable("fileMimeType");
+            if (mimeType != null) {
+                increment("files/" + mimeType.stringValue());
+            }
+            Fieldable sizeInBytes = doc.getFieldable("sizeInBytes");
+            if (sizeInBytes != null) {
+                long size = Long.parseLong(sizeInBytes.stringValue());
+                add("fileSize/total", size);
+                if (mimeType != null) {
+                    add("fileSize/" + mimeType.stringValue(), size);
+                }
+            }
+        }
+    }
+
+}

remoting/server/indexer/usages/src/org/netbeans/modules/jackpot30/indexer/usages/UsagesStatisticsGenerator.java

+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2012 Oracle and/or its affiliates. All rights reserved.
+ *
+ * Oracle and Java are registered trademarks of Oracle and/or its affiliates.
+ * Other names may be trademarks of their respective owners.
+ *
+ * The contents of this file are subject to the terms of either the GNU
+ * General Public License Version 2 only ("GPL") or the Common
+ * Development and Distribution License("CDDL") (collectively, the
+ * "License"). You may not use this file except in compliance with the
+ * License. You can obtain a copy of the License at
+ * http://www.netbeans.org/cddl-gplv2.html
+ * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+ * specific language governing permissions and limitations under the
+ * License.  When distributing the software, include this License Header
+ * Notice in each file and include the License file at
+ * nbbuild/licenses/CDDL-GPL-2-CP.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the GPL Version 2 section of the License file that
+ * accompanied this code. If applicable, add the following below the
+ * License Header, with the fields enclosed by brackets [] replaced by
+ * your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ *
+ * If you wish your version of this file to be governed by only the CDDL
+ * or only the GPL Version 2, indicate your decision by adding
+ * "[Contributor] elects to include this software in this distribution
+ * under the [CDDL or GPL Version 2] license." If you do not indicate a
+ * single choice of license, a recipient has the option to distribute
+ * your version of this file under either the CDDL, the GPL Version 2 or
+ * to extend the choice of license to its licensees as provided above.
+ * However, if you add GPL Version 2 code and therefore, elected the GPL
+ * Version 2 license, then the option applies only if the new code is
+ * made subject to such option by the copyright holder.
+ *
+ * Contributor(s):
+ *
+ * Portions Copyrighted 2012 Sun Microsystems, Inc.
+ */
+package org.netbeans.modules.jackpot30.indexer.usages;
+
+import java.io.IOException;
+import javax.lang.model.element.ElementKind;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.netbeans.modules.jackpot30.backend.impl.spi.StatisticsGenerator;
+import org.openide.util.lookup.ServiceProvider;
+
+/**
+ *
+ * @author lahvac
+ */
+@ServiceProvider(service=StatisticsGenerator.class)
+public class UsagesStatisticsGenerator extends StatisticsGenerator {
+
+    @Override
+    protected void amendStatistics(IndexReader r, Document doc) throws IOException {
+        if (doc.getFieldable("classFQN") != null) increment("java-classes");
+        else if (doc.getFieldable("featureClassFQN") != null) {
+            ElementKind kind = ElementKind.valueOf(doc.getFieldable("featureKind").stringValue());
+
+            if (kind.isField()) {
+                increment("java-fields");
+            } else {
+                increment("java-methods");
+            }
+        }
+    }
+
+}