Commits

Anonymous committed 6e04584 Merge

mimetype (Tika) support

Comments (0)

Files changed (10)

ingest-src/pom.xml

           <artifactId>asm</artifactId>
           <version>3.1</version>
       </dependency>
-    
+      <dependency>
+          <groupId>org.apache.commons</groupId>
+          <artifactId>commons-compress</artifactId>
+          <version>1.4.1</version>
+      </dependency>
   </dependencies>
 
 </project>

ingest-src/src/main/java/org/cdlib/mrt/ingest/BatchState.java

     private String batchStatusMessage = null;
     private String queueConnectionString = null;
     private String queueNode = null;
+    private String inventoryNode = null;
     private Map<String, JobState> jobStates = new HashMap<String, JobState>();
     private static Map<String, BatchState> batchStates = new HashMap<String, BatchState>();
     private static Map<String, Integer> batchReadiness = new HashMap<String, Integer>();
         return this.queueNode;
     }
 
+    public String grabTargetInventoryNode()
+    {
+        return this.inventoryNode;
+    }
+
     /**
      * Set queue target node
      * @param String target queue node
         this.queueNode = queueNode;
     }
 
+    /**
+     * Set inventory target node
+     * @param String target inventory node
+     */
+    public void setTargetInventoryNode(String inventoryNode) {
+        this.inventoryNode = inventoryNode;
+    }
+
     public ProfileState grabBatchProfile()
     {
         return batchProfile;

ingest-src/src/main/java/org/cdlib/mrt/ingest/IngestManager.java

                     if (handler.getClass() == org.cdlib.mrt.ingest.handlers.HandlerInventoryQueue.class) {
 	    		if ( ! batchState.getBatchID().getValue().equalsIgnoreCase(ProfileUtil.DEFAULT_BATCH_ID)) {
 			    jobState.setMisc(batchState.grabTargetQueue());
+			    jobState.setExtra(batchState.grabTargetInventoryNode());
 			} else {
 			    // not a batch
 			    jobState.setMisc(getProps(ingestRequest, "queue.txt").getProperty("QueueService"));
+			    jobState.setExtra(getProps(ingestRequest, "queue.txt").getProperty("InventoryName"));
 			}
                     }
 

ingest-src/src/main/java/org/cdlib/mrt/ingest/JobState.java

 	private String jobStatusMessage = null;
 	private String objectType = null;
 	private String misc = null;
+	private String extra = null;
 	private String note = null;
 	private String queuePriority = null;
 	private String metacatStatus = null;		// only if dataONE handler active
     }
 
     /**
+     * Set extra data
+     * @param String extra
+     */
+    public void setExtra(String extra) {
+        this.extra = extra;
+    }
+
+    /**
+     * Get extra data
+     * @return String extra
+     */
+    public String grabExtra() {
+        return extra;
+    }
+
+    /**
      * Set DataCite metadata
      * @param String DataCite metadata
      */

ingest-src/src/main/java/org/cdlib/mrt/ingest/QueueManager.java

     private Properties queueProperties = null;
     private String queueConnectionString = null;
     private String queueNode = null;
+    private String inventoryNode = "/inv";	// default
     private ArrayList<String> m_admin = new ArrayList<String>(20);
 
     private boolean debugDump = false;
             String matchIngest = "ingestServicePath";
             String matchQueueService = "QueueService";
             String matchQueueNode = "QueueName";
+            String matchInventoryNode = "InventoryName";
             String matchAdmin = "admin";
             String defaultIDKey = "IDDefault";
 	    Integer storageID = null;
             while( e.hasMoreElements() ) {
                 key = (String) e.nextElement();
                 value = ingestProperties.getProperty(key);
-                if (key.equals(matchQueueService)) {
-		    this.queueConnectionString = value;
-                }
-                if (key.equals(matchQueueNode)) {
-		    this.queueNode = value;
-                }
+                if (key.equals(matchQueueService)) this.queueConnectionString = value;
+                if (key.equals(matchQueueNode) )this.queueNode = value;
+                if (key.equals(matchInventoryNode)) this.inventoryNode = value;
 
                 // admin notification
                 if (key.startsWith(matchAdmin)) {
 	    batchState.clear();
 	    batchState.setTargetQueue(queueConnectionString);
 	    batchState.setTargetQueueNode(queueNode);
+	    batchState.setTargetInventoryNode(inventoryNode);
 	    batchState.setUserAgent(ingestRequest.getJob().grabUserAgent());
 	    batchState.setSubmissionDate(new DateState(DateUtil.getCurrentDate()));
 	    batchState.setBatchStatus(BatchStatusEnum.QUEUED);

ingest-src/src/main/java/org/cdlib/mrt/ingest/handlers/HandlerDigest.java

     {
 
 	try {
+            File manifest = new File(ingestRequest.getQueuePath().getAbsolutePath() + "/system/mrt-manifest.txt");
+            manifest.createNewFile();
 
-	    File manifest = new File(ingestRequest.getQueuePath().getAbsolutePath() + "/system/mrt-manifest.txt");
-	    manifest.createNewFile();
 
 	    // requires symlink from webapps/ingestqueue to home ingest queue directory
 	    URL link = new URL(ingestRequest.getLink());	
 	    // build manifest
 	    ManifestBuild.getPostManifest(baseURL, ingestRequest.getQueuePath(), manifest);
 
-	    FileUtil.removeLineFromFile(manifest.getAbsolutePath(), "mrt-manifest.txt", "END");
+	    FileUtil.removeLineFromFile(manifest.getAbsolutePath(), "system/mrt-manifest.txt", "CONTAIN");
 
 	    return new HandlerResult(true, "SUCCESS: " + NAME + " created manifest");
 	} catch (TException te) {

ingest-src/src/main/java/org/cdlib/mrt/ingest/handlers/HandlerInventoryQueue.java

     public HandlerResult handle(ProfileState profileState, IngestRequest ingestRequest, 
                                 JobState jobState) throws TException {
         try {
-	    // 4Store
+	    // 4Store & MySQL (minimum schema)
             zooKeeper = 
                 new ZooKeeper(jobState.grabMisc(), 10000, new Ignorer());
             distributedQueue = 
 	    System.out.println("HandlerInventoryQueue submitting URL [4Store]: " + jobState.grabObjectState());
             submit(jobState.grabObjectState().getBytes());
 
-	    // MySQL
+	    // MySQL (full schema)
             distributedQueue = 
-                new DistributedQueue(zooKeeper, "/inv", null);
+                new DistributedQueue(zooKeeper, jobState.grabExtra(), null);
 	    prop = getInventoryProps(profileState, jobState);
 	    System.out.println("HandlerInventoryQueue submitting properties [MySQL]: " + prop.toString());
             submit(ZooCodeUtil.encodeItem(prop));
     {
         Properties prop = new Properties();
 
-        prop.setProperty("profile", profileState.getProfileID().getValue());
-        prop.setProperty("batch_id", jobState.grabBatchID().getValue());
-        prop.setProperty("job_id", jobState.getJobID().getValue());
-        prop.setProperty("user_agent", "unknown");
-        prop.setProperty("submitted_date_time", jobState.getSubmissionDate().toString());
-        prop.setProperty("current_version_number", jobState.getVersionID().toString());
-        prop.setProperty("fileid", jobState.getPackageName());
-        prop.setProperty("storage_url", jobState.grabObjectState());
+        String manifestURL = jobState.grabObjectState().replace("/state/", "/manifest/");
+        prop.setProperty("manifestURL", manifestURL);
 
         return prop;
     }

ingest-src/src/main/java/org/cdlib/mrt/ingest/handlers/HandlerNotification.java

File contents unchanged.

ingest-src/src/main/java/org/cdlib/mrt/ingest/handlers/HandlerTransfer.java

 	    WebResource webResource = client.resource(url);
 
 	    // convert manifest to encoded string
-	    String manifest = getManifest(new File(ingestRequest.getQueuePath().getAbsolutePath() + "/system/mrt-manifest.txt"));
+            String manifest = getManifest(new File(ingestRequest.getQueuePath().getAbsolutePath() + "/system/mrt-manifest.txt"));
 	    if (DEBUG) System.out.println("[debug] " + MESSAGE + " manifest: " + manifest);
 
 	    Form formData = new Form();

ingest-src/src/main/java/org/cdlib/mrt/ingest/handlers/queue/HandlerNotification.java

File contents unchanged.