Commits

Anonymous committed 75ee55e

ENCUESTAME-136 Create a job to Reindex Attachment and Index Directory

Comments (0)

Files changed (6)

encuestame-business/src/main/java/org/encuestame/business/search/ReIndexAttachmentsJob.java

+package org.encuestame.business.search;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.encuestame.business.setup.DirectorySetupOperations;
+import org.encuestame.core.cron.IndexRebuilder;
+import org.encuestame.core.cron.ReIndexJob;
+import org.encuestame.search.IndexerManager;
+
+public class ReIndexAttachmentsJob {
+
+    /**
+     * Log.
+     */
+    private static final Log log = LogFactory.getLog(ReIndexAttachmentsJob.class);
+
+
+    /**
+     *
+     */
+    private String indexStorePath;
+
+    /**
+     *
+     */
+    private String attachmentsPlacesHolder = "/home/dmorales/encuestame/profiles/1/";
+
+    private String attachmentsPlacesHolder2 = "/home/dmorales/encuestame/profiles/2/";
+
+    private IndexerManager indexerManager;
+
+
+    /**
+     * Constructor.
+     */
+    public ReIndexAttachmentsJob() {    	//
+        this.indexStorePath = DirectorySetupOperations.getIndexesDirectory();
+        log.debug("Index path location "+this.indexStorePath);
+
+    }
+
+
+    /**
+     * Reindex.
+     */
+    public void reindexAttachments(){
+        log.debug("reindexAttachments");
+        final List<File> userDomainAttachmentsLocation = new ArrayList<File>();
+        userDomainAttachmentsLocation.add(new File(this.attachmentsPlacesHolder));
+        log.debug("Location size"+userDomainAttachmentsLocation.size());
+        try {
+            log.debug("IndexManager initialize....");
+            this.indexerManager = new IndexerManager(userDomainAttachmentsLocation, this.indexStorePath);
+            log.debug("Initialize Index Starting...");
+            this.indexerManager.initializeIndex();
+        } catch (Exception e) {
+            log.fatal("Error al indexar "+e);
+            e.printStackTrace();
+        }
+    }
+}

encuestame-search/src/main/java/org/encuestame/search/IndexerManager.java

+package org.encuestame.search;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.util.Version;
+
+
+/**
+ * Directory Operations.
+ * @author Morales Diana, Juan juanATencuestame.org
+ * @since Mar 20, 2011
+ */
+public class IndexerManager {
+
+
+    private static final Log log = LogFactory.getLog(IndexerManager.class);
+
+    private List<File> directoriesToIndex = new ArrayList<File>();
+
+    private String indexesLocation;
+    private IndexWriter indexWriter;
+
+    /**
+    *
+    * @param files
+    * @throws Exception
+    */
+    public IndexerManager(final List<File> files, final String indexesLocation) {
+        this.indexesLocation = indexesLocation;
+        this.directoriesToIndex = files;
+    }
+
+    /** Constructor. **/
+    public IndexerManager() {
+    }
+
+    /**
+    * @throws IOException
+    * @throws LockObtainFailedException
+    * @throws CorruptIndexException
+    *
+    */
+    private void startIndexWriter() throws CorruptIndexException,
+            LockObtainFailedException, IOException {
+        final Directory dir = FSDirectory.open(new java.io.File(
+                this.indexesLocation));
+        this.indexWriter = new IndexWriter(dir, new StandardAnalyzer(
+                Version.LUCENE_29), true, IndexWriter.MaxFieldLength.UNLIMITED);
+    }
+
+    /**
+    *
+    * @param dataDir
+    * @param filter
+    * @return
+    * @throws Exception
+    */
+    public int index(final File dataDir, FileFilter filter) throws Exception {
+        log.debug("Index file is directory: "+dataDir.isDirectory());
+        File[] files = dataDir.listFiles();
+        for (File f : files) {
+            if (!f.isDirectory() && !f.isHidden() && f.exists() && f.canRead()
+                    && (filter == null || filter.accept(f))) {
+                this.indexFile(f);
+            }
+        }
+        return this.indexWriter.numDocs();
+    }
+
+    /**
+    *
+    * @param f
+    * @return
+    * @throws Exception
+    */
+    protected Document getDocument(File f) throws Exception {
+        Document doc = new Document();
+        doc.add(new Field("contents", new FileReader(f))); // Index file
+                                                            // contents
+        doc.add(new Field("filename", f.getName(), Field.Store.YES,
+                Field.Index.NOT_ANALYZED)); // Index File Name.
+        doc.add(new Field("fullpath", f.getCanonicalPath(), Field.Store.YES,
+                Field.Index.NOT_ANALYZED)); // Index Full Path.
+        return doc;
+    }
+
+    /**
+    *
+    * @param f
+    * @throws Exception
+    */
+    private void indexFile(File f) throws Exception {
+        log.debug("Indexing " + f.getCanonicalPath());
+        Document doc = getDocument(f);
+        log.debug("Adding document..."+doc);
+        this.indexWriter.addDocument(doc); // Add Document to Lucene Index.
+    }
+
+    /**
+    * Initialize index process.
+    *
+    * @throws Exception
+    */
+    public void initializeIndex() throws Exception {
+        log.debug("Initialize");
+        this.startIndexWriter();
+        for (File file : this.directoriesToIndex) {
+            long start = System.currentTimeMillis();
+            int numIndexed;
+            try {
+                numIndexed = this.index(file, new TextFilesFilter());
+            } finally {
+                this.close();
+            }
+            long end = System.currentTimeMillis();
+            log.debug("Indexing " + numIndexed + " files took "
+                    + (end - start) + " milliseconds");
+        }
+    }
+
+    /**
+    * Close Lucene IndexWriter.
+    *
+    * @throws IOException
+    */
+    public void close() throws IOException {
+        this.indexWriter.close();
+    }
+
+    private static class TextFilesFilter implements FileFilter {
+        public boolean accept(File path) {
+            return path.getName().toLowerCase().endsWith(".txt");
+        }
+    }
+}

encuestame-war/src/main/webapp/WEB-INF/spring/encuestame-quartz-context.xml

         <property name="indexRebuilder" ref="indexRebuilder"/>
     </bean>
 
+    <bean id="reindexAttachmentJob" class="org.encuestame.business.search.ReIndexAttachmentsJob"/>
+
     <bean id="indexRebuilder" class="org.encuestame.core.cron.IndexRebuilder"/>
 
     <bean id="reindexingInvokingJobDetail" class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean">
         </property>
     </bean>
 
+    <!-- Attachments process indexer -->
+    <bean id="reindexingAttachmentsJobDetail" class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean">
+        <property name="targetObject" ref="reindexAttachmentJob"/>
+        <property name="targetMethod" value="reindexAttachments"/>
+        <property name="concurrent">
+            <value>false</value>
+        </property>
+    </bean>
+
     <bean id="schedulerFactoryBean"
         class="org.springframework.scheduling.quartz.SchedulerFactoryBean">
         <property name="triggers">
             <list>
+                 <!--  Reindex domains -->
                  <bean id="cronReindex" class="org.springframework.scheduling.quartz.CronTriggerBean">
                     <property name="jobDetail" ref="reindexingInvokingJobDetail" />
                     <property name="cronExpression" value="0 0/15 * * * ?" />
                     <!-- Every 5 minutes 0 0/5 * * * ? -->
                     <!-- 0 0 12 1/5 * ? Fire at 12pm (noon) every 5 days every month, starting on the first day of the month. -->
                 </bean>
+                <!-- Reindex attachments -->
+                <bean id="attachmentsReindex" class="org.springframework.scheduling.quartz.CronTriggerBean">
+                    <property name="jobDetail" ref="reindexingAttachmentsJobDetail" />
+                    <property name="cronExpression" value="0 0/1 * * * ?" />
+                    <!-- Every 5 minutes 0 0/5 * * * ? -->
+                    <!-- 0 0 12 1/5 * ? Fire at 12pm (noon) every 5 days every month, starting on the first day of the month. -->
+                </bean>
             </list>
         </property>
     </bean>

encuestame-war/src/main/webapp/WEB-INF/spring/encuestame-task-search.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<beans:beans xmlns="http://www.springframework.org/schema/task"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xmlns:task="http://www.springframework.org/schema/task"
+    xmlns:beans="http://www.springframework.org/schema/beans"
+    xsi:schemaLocation="http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task-3.0.xsd
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd">
+
+
+</beans:beans>

encuestame-war/src/main/webapp/WEB-INF/spring/encuestame-task.xml

     <executor id="taskExecutor" />
 
     <beans:import resource="encuestame-quartz-context.xml"/>
+    <beans:import resource="encuestame-task-search.xml"/>
 
 </beans:beans>
            <version>${lucene.version}</version>
            <scope>compile</scope>
         </dependency>
+  	<dependency>
+	   <groupId>org.apache.lucene</groupId>
+   	   <artifactId>lucene-highlighter</artifactId>
+  	   <version>${lucene.version}</version>
+        </dependency>
+        <dependency>
+           <groupId>org.apache.lucene</groupId>
+    	   <artifactId>lucene-fast-vector-highlighter</artifactId>
+    	   <version>${lucene.version}</version>
+       </dependency>
+       <dependency>
+    	   <groupId>org.apache.lucene</groupId>
+    	   <artifactId>lucene-spellchecker</artifactId>
+    	   <version>${lucene.version}</version>
+      </dependency>
         <!-- Apache Tiles -->
         <dependency>
             <groupId>org.apache.tiles</groupId>
              <groupId>com.google.api.client</groupId>
              <artifactId>google-api-client</artifactId>
              <version>1.2.2-alpha</version>
-        </dependency>
-
-    <dependency>
-	 <groupId>org.apache.lucene</groupId>
-   	 <artifactId>lucene-highlighter</artifactId>
-  	 <version>2.9.3</version>
-     </dependency>
-     <dependency>
-    	<groupId>org.apache.lucene</groupId>
-    	<artifactId>lucene-fast-vector-highlighter</artifactId>
-    	<version>2.9.1</version>
-     </dependency>
-    <dependency>
-    	<groupId>org.apache.lucene</groupId>
-    	<artifactId>lucene-spellchecker</artifactId>
-    	<version>3.0.3</version>
-    </dependency>
+        </dependency>  
   	<!-- CommetD -->
         <dependency>
             <groupId>org.cometd.java</groupId>
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.