Commits

Jonathan Simms  committed 455b13b

initial import

  • Participants

Comments (0)

Files changed (3)

+# use glob syntax.
+syntax: glob
+*.ser
+*.class
+*~
+*.bak
+#*.off
+*.old
+
+# eclipse conf file
+.settings
+.classpath
+.project
+.manager
+.scala_dependencies
+
+# idea
+.idea
+*.iml
+
+# building
+target
+build
+null
+tmp*
+temp*
+dist
+test-output
+build.log
+
+# other scm
+.svn
+.CVS
+.hg*
+
+# switch to regexp syntax.
+#  syntax: regexp
+#  ^\.pc/
+
+#SHITTY output not in target directory
+build.log
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>net.freenose</groupId>
+  <artifactId>lucener</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <name>${project.artifactId}</name>
+  <description>My wonderfull scala app</description>
+  <inceptionYear>2010</inceptionYear>
+  <licenses>
+    <license>
+      <name>My License</name>
+      <url>http://....</url>
+      <distribution>repo</distribution>
+    </license>
+  </licenses>
+
+  <properties>
+    <maven.compiler.source>1.5</maven.compiler.source>
+    <maven.compiler.target>1.5</maven.compiler.target>
+    <encoding>UTF-8</encoding>
+    <scala.version>2.8.0</scala.version>
+  </properties>
+
+<!--
+  <repositories>
+    <repository>
+      <id>scala-tools.org</id>
+      <name>Scala-Tools Maven2 Repository</name>
+      <url>http://scala-tools.org/repo-releases</url>
+    </repository>
+  </repositories>
+
+  <pluginRepositories>
+    <pluginRepository>
+      <id>scala-tools.org</id>
+      <name>Scala-Tools Maven2 Repository</name>
+      <url>http://scala-tools.org/repo-releases</url>
+    </pluginRepository>
+  </pluginRepositories>
+-->
+  <dependencies>
+    <dependency>
+      <groupId>org.scala-lang</groupId>
+      <artifactId>scala-library</artifactId>
+      <version>${scala.version}</version>
+    </dependency>
+
+    <!-- Test -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.8.1</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.scala-tools.testing</groupId>
+      <artifactId>specs_${scala.version}</artifactId>
+      <version>1.6.5</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.scalatest</groupId>
+      <artifactId>scalatest</artifactId>
+      <version>1.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-core</artifactId>
+      <version>4.0.0-BETA</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-analyzers-common</artifactId>
+      <version>4.0.0-BETA</version>
+    </dependency>
+    <!--<dependency>-->
+      <!--<groupId>org.apache.lucene</groupId>-->
+      <!--<artifactId>lucene-demo</artifactId>-->
+      <!--<version>4.0.0-BETA</version>-->
+    <!--</dependency>-->
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-queryparser</artifactId>
+      <version>4.0.0-BETA</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <sourceDirectory>src/main/scala</sourceDirectory>
+    <testSourceDirectory>src/test/scala</testSourceDirectory>
+    <plugins>
+      <plugin>
+        <groupId>org.scala-tools</groupId>
+        <artifactId>maven-scala-plugin</artifactId>
+        <version>2.15.0</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>compile</goal>
+              <goal>testCompile</goal>
+            </goals>
+            <configuration>
+              <args>
+                <arg>-make:transitive</arg>
+                <arg>-dependencyfile</arg>
+                <arg>${project.build.directory}/.scala_dependencies</arg>
+              </args>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>2.6</version>
+        <configuration>
+          <useFile>false</useFile>
+          <disableXmlReport>true</disableXmlReport>
+          <!-- If you have classpath issue like NoDefClassError,... -->
+          <!-- useManifestOnlyJar>false</useManifestOnlyJar -->
+          <includes>
+            <include>**/*Test.*</include>
+            <include>**/*Suite.*</include>
+          </includes>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>

File src/net/freenose/lucener/IndexFiles.scala

+package net.freenose.lucener
+
+import java.io.{FileInputStream, InputStreamReader, BufferedReader, File}
+import java.util.Date
+import org.apache.lucene.store.FSDirectory
+import org.apache.lucene.analysis.standard.StandardAnalyzer
+import org.apache.lucene.util.Version
+import org.apache.lucene.index.{IndexWriter, IndexWriterConfig}
+import org.apache.lucene.index.IndexWriterConfig.OpenMode
+import org.apache.lucene.document._
+
+class IndexFiles(indexPath: String, docsPath: String) {
+  private val docDir = new File(docsPath)
+  private val indexDir = new File(indexPath)
+
+  def run() { run(true) }
+
+  def run(create: Boolean) {
+    val dir = FSDirectory.open(indexDir)
+    val analyzer = new StandardAnalyzer(Version.LUCENE_40)
+    val iwc = new IndexWriterConfig(Version.LUCENE_40, analyzer)
+
+    val start = new Date()
+
+    if (create) {
+      iwc.setOpenMode(OpenMode.CREATE)
+    } else {
+      iwc.setOpenMode(OpenMode.CREATE_OR_APPEND)
+    }
+
+    val writer = new IndexWriter(dir, iwc)
+
+    try {
+      indexDocs(writer, docDir)
+    } finally {
+      writer.close()
+    }
+
+    val end = new Date()
+    println((end.getTime - start.getTime).toString + " total milliseconds")
+
+  }
+
+  private def indexDocs(writer: IndexWriter, file: File) {
+    if (!file.canRead) return
+
+    if (file.isDirectory) {
+      for (f <- file.listFiles())
+        indexDocs(writer, f)
+    } else {
+      val doc = new Document()
+
+      // add a path field, index the field but don't tokenize it
+      doc.add(new StringField("path", file.getPath, Field.Store.YES))
+
+      // add the last modified date, use a LongField that's indexed
+      doc.add(new LongField("modified", file.lastModified, Field.Store.NO))
+
+      val reader = new BufferedReader(new InputStreamReader(new FileInputStream(file)))
+
+      try {
+        // add the contents, specify a reader so the text is tokenized and indexed
+        doc.add(new TextField("contents", reader))
+      } finally {
+        reader.close()
+      }
+    }
+  }
+}
+
+object IndexFiles {
+  val UsageString =
+    """
+      |java net.freenose.lucener.IndexFiles
+      |  [-index INDEX_PATH] [-docs DOCS_PATH] [-update]
+      |
+      |This indexes the documents in DOCS_PATH, creating a Lucene index
+      |in INDEX_PATH that can be searched with SearchFiles
+    """.stripMargin
+
+  val DefaultIndexDir = "/tmp/lucener-index"
+  val DefaultDocsPath = "/Users/slyphon/.weechat/logs"
+
+  def main(args: Array[String]) {
+//    val indx = new IndexFiles(index)
+  }
+}
+
+