Commits

dloy  committed 73c58e4 Draft

Add S3 for bitbucket

  • Participants

Comments (0)

Files changed (18)

+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>mrt-s3</artifactId>
+    <groupId>org.cdlib.mrt</groupId>
+
+    <packaging>pom</packaging>
+    <version>1.0-SNAPSHOT</version>
+    <name>UC3-mrtS3</name>
+    <description>UC3 Merritt Micro-Services</description>
+    <url>http://uc3.cdlib.org</url>
+    <modules>
+        <module>s3-src</module>
+    </modules>
+
+</project>
+

File s3-src/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>org.cdlib.mrt</groupId>
+    <artifactId>mrt-s3src</artifactId>
+    <packaging>jar</packaging>
+    <version>1.0-SNAPSHOT</version>
+    <name>UC3-mrtS3Src</name>
+    <url>http://uc3.cdlib.org</url>
+    <repositories>
+        <repository>
+            <name>jets3t</name>
+            <id>jets3t</id>
+            <url>http://www.jets3t.org/maven2</url>
+        </repository>
+    </repositories>
+
+  <distributionManagement>
+    <repository>
+      <id>cdl-releases</id>
+      <name>CDL Snapshot Repository</name>
+      <url>http://builds.cdlib.org/repo/content/repositories/cdl-releases/</url>
+    </repository>
+    <snapshotRepository>
+      <id>cdl-snapshots</id>
+      <name>CDL Snapshot Repository</name>
+      <url>http://builds.cdlib.org/repo/content/repositories/cdl-snapshots/</url>
+    </snapshotRepository>
+  </distributionManagement>
+
+    <!-- force UTF-8 -->
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>2.0.2</version>
+                <configuration>
+                    <source>1.6</source>
+                    <target>1.6</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+    
+    <dependencies>
+        <dependency>
+            <groupId>org.cdlib.mrt</groupId>
+            <artifactId>mrt-core</artifactId>
+            <version>1.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.cdlib.mrt</groupId>
+            <artifactId>mrt-jena</artifactId>
+            <version>1.0-SNAPSHOT</version>
+        </dependency>
+
+        <dependency>
+            <groupId>net.java.dev.jets3t</groupId>
+            <artifactId>jets3t</artifactId>
+            <version>0.8.1</version>
+        </dependency>
+        
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.5</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpcore</artifactId>
+            <version>4.2-alpha2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpmime</artifactId>
+            <version>4.0.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.0.1</version>
+        </dependency>
+        <dependency>
+            <groupId>xalan</groupId>
+            <artifactId>xalan</artifactId>
+            <version>2.7.1</version>
+        </dependency>
+        <dependency>
+            <groupId>javax.servlet</groupId>
+            <artifactId>servlet-api</artifactId>
+            <version>2.5</version>
+        </dependency>
+        <dependency>
+            <groupId>ant</groupId>
+            <artifactId>ant</artifactId>
+            <version>1.7.0</version>
+        </dependency>
+        <dependency>
+            <groupId>jdom</groupId>
+            <artifactId>jdom</artifactId>
+            <version>1.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpmime</artifactId>
+            <version>4.0.1</version>
+        </dependency>
+        <dependency>
+            <groupId>xalan</groupId>
+            <artifactId>xalan</artifactId>
+            <version>2.7.1</version>
+        </dependency>
+
+    </dependencies>
+
+</project>

File s3-src/src/main/java/org/cdlib/mrt/s3/IdList.properties

+# Fill in your AWS Access Key ID and Secret Access Key
+# http://aws.amazon.com/security-credentials
+accessKey =xxxx
+secretKey =xxxx
+access_key=xxxx
+#secret_key=xxxx
+secret_key=xxx
+
+host=cloud.edu

File s3-src/src/main/java/org/cdlib/mrt/s3/pairtree/PairtreeCloud.java

+/*
+Copyright (c) 2005-2010, Regents of the University of California
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+ *
+- Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+- Neither the name of the University of California nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
+*********************************************************************/
+package org.cdlib.mrt.s3.pairtree;
+//import org.cdlib.mrt.s3.service.*;
+
+
+import org.cdlib.mrt.s3.sdsc.*;
+
+import org.cdlib.mrt.s3.service.*;
+
+import org.cdlib.mrt.core.Identifier;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import org.jets3t.service.S3Service;
+
+import org.cdlib.mrt.utility.FixityTests;
+import org.cdlib.mrt.utility.StringUtil;
+import org.cdlib.mrt.utility.LoggerInf;
+import org.cdlib.mrt.utility.TException;
+
+import org.cdlib.mrt.s3.service.CloudStoreAbs;
+import org.cdlib.mrt.s3.service.CloudUtil;
+import org.cdlib.mrt.utility.FileUtil;
+import org.cdlib.mrt.utility.PairtreeUtil;
+import org.cdlib.mrt.utility.PropertiesUtil;
+
+/**
+ * Specific SDSC Storage Cloud handling
+ * @author dloy
+ */
+public class PairtreeCloud
+    extends CloudStoreAbs
+    implements CloudStoreInf
+{
+    private static final boolean DEBUG = false;
+    private static final boolean ALPHANUMERIC = false;
+    private static final String FILEPROPERTIES = "component.properties";
+    private static final String DIGESTNAME = "digest";
+    private boolean pairPath = false;
+    
+    public static PairtreeCloud getPairtreeCloud(
+            boolean pairPath,
+            LoggerInf logger)
+        throws TException
+    {
+        return new PairtreeCloud(pairPath, logger);
+    }
+    
+    protected PairtreeCloud(
+            boolean pairPath,
+            LoggerInf logger)
+        throws TException
+    {
+        super(logger);
+        this.pairPath = pairPath;
+    }
+    
+    public S3Service getStorageService()
+    {
+        return null;
+    }
+
+    protected void setStorageService()
+        throws TException
+    {
+        
+    }
+    
+    protected File getBase(String filePath)
+        throws TException
+    {
+        try {
+            if (StringUtil.isEmpty(filePath)) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "getBase missing filePath");
+            }
+            File baseDir = new File(filePath);
+            if (!baseDir.exists()) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "getBase file does not exist:" + filePath);
+                
+            }
+            return baseDir;
+            
+        } catch (TException tex) {
+            throw tex;
+            
+        } catch (Exception ex) {
+            throw new TException (ex);
+        }
+    }
+    
+    protected File getDir(File baseDir, String key)
+        throws TException
+    {
+        try {
+            if (pairPath) {
+                return PairtreeUtil.buildPairDirectory(baseDir, key);
+            } else {
+                String keyName = PairtreeUtil.getPairName(key);
+                File retFile = new File(baseDir, keyName);
+                retFile.mkdir();
+                return retFile;
+            }
+            
+        } catch (TException tex) {
+            throw tex;
+            
+        } catch (Exception ex) {
+            throw new TException (ex);
+        }
+    }
+    
+    public CloudResponse putObject(
+            CloudResponse response,
+            File inputFile)
+        throws TException
+    {        
+        try {
+            if (!isValidFile(inputFile)) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "putObject - file not valid");
+            }
+            String bucketName = response.getBucketName();
+            File baseDir = getBase(bucketName);
+            String key = response.getStorageKey();
+            File pairDir = getDir(baseDir, key);
+            File component = new File(pairDir, "component.txt");
+            InputStream iStream = new FileInputStream(inputFile);
+            FileUtil.stream2File(iStream, component);
+            
+            String md5HexIn = CloudUtil.getDigestValue(inputFile, logger);
+            response.setMd5(md5HexIn);
+            String md5HexCopy = CloudUtil.getDigestValue(component, logger);
+            if ((inputFile.length() != component.length()) || !md5HexIn.equals(md5HexCopy)) {
+                throw new TException.FIXITY_CHECK_FAILS(MESSAGE + "Component copy fails "
+                        + " - inputFile length=" + inputFile.length()
+                        + " - component length=" + component.length()
+                        + " - inputFile md5=" + md5HexIn
+                        + " - copyFile md5=" + md5HexCopy
+                        );
+            }
+            response.setFileMetaProperty("key", key);
+            response.setFileMetaProperty("digestType", "md5");
+            response.setFileMetaProperty(DIGESTNAME, md5HexCopy);
+            response.setFileMetaProperty("size", "" + component.length());
+            Properties prop = response.getFileMetaProperties();
+            String loadString = PropertiesUtil.buildLoadProperties(prop);
+            File propFile = new File(pairDir, FILEPROPERTIES);
+            FileUtil.string2File(propFile, loadString);
+                        
+        } catch (Exception ex) {
+            handleException(response, ex);
+        }
+        return response;
+    }
+    public CloudResponse putObject(
+            String bucketName,
+            String key,
+            File inputFile)
+        throws TException
+    { 
+        if (StringUtil.isEmpty(bucketName)) {
+            throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "putObject - bucketName not valid");
+        }
+        if (StringUtil.isEmpty(key)) {
+            throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "putObject - key not valid");
+        }
+        CloudResponse response = new CloudResponse(bucketName, key);
+        return putObject(response, inputFile);
+    }
+    
+    public CloudResponse putObject(
+            String bucketName,
+            Identifier objectID,
+            int versionID,
+            String fileID,
+            File inputFile)
+        throws TException
+    {
+        CloudResponse response = new CloudResponse(bucketName, objectID, versionID, fileID);
+        String key = CloudUtil.getKey(objectID, versionID, fileID, ALPHANUMERIC);
+        response.setStorageKey(key);
+        return putObject(response, inputFile);
+    }
+    
+    public CloudResponse putObject(
+            String bucketName,
+            String key,
+            File inputFile,
+            Properties cloudProperties)
+        throws TException
+    {
+        if (StringUtil.isEmpty(bucketName)) {
+            throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "putObject - bucketName not valid");
+        }
+        if (StringUtil.isEmpty(key)) {
+            throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "putObject - key not valid");
+        }
+        CloudResponse response = new CloudResponse(bucketName, key);
+        response.setFileMeta(cloudProperties);
+        return putObject(response, inputFile);
+        
+    }
+    
+    public CloudResponse putObject(
+            String bucketName,
+            Identifier objectID,
+            int versionID,
+            String fileID,
+            File inputFile,
+            Properties cloudProperties)
+        throws TException
+    {
+        if (DEBUG) System.out.println(PropertiesUtil.dumpProperties("putObject", cloudProperties));
+        CloudResponse response = new CloudResponse(bucketName, objectID, versionID, fileID);
+        String key = CloudUtil.getKey(objectID, versionID, fileID, ALPHANUMERIC);
+        response.setStorageKey(key);
+        response.setFileMeta(cloudProperties);
+        return putObject(response, inputFile);
+    }
+    
+    public CloudResponse putManifest(
+            String bucketName,
+            Identifier objectID,
+            File inputFile)
+        throws TException
+    {
+        CloudResponse response = new CloudResponse(bucketName, objectID, null, null);
+        String key = CloudUtil.getManifestKey(objectID, ALPHANUMERIC);
+        response.setStorageKey(key);
+        return putObject(response, inputFile);
+    }
+
+    public CloudResponse deleteObject (
+            String bucketName,
+            String key)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            response = new CloudResponse(bucketName, key);
+            response.setStorageKey(key);
+            
+            File baseDir = getBase(bucketName);
+            File pairDir = getDir(baseDir, key);
+            File [] files = pairDir.listFiles();
+            if ((files == null) || (files.length == 0)) {
+                throw new TException.REQUESTED_ITEM_NOT_FOUND("Unable to delete object - object not found:" 
+                        + pairDir.getCanonicalPath());
+            }
+            PairtreeUtil.removePairDirectory(pairDir);
+            return response;
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+        }
+        return response;
+        
+    }
+
+    public CloudResponse deleteObject (
+            String bucketName,
+            Identifier objectID,
+            int versionID,
+            String fileID)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            response = new CloudResponse(bucketName, objectID, versionID, fileID);
+            String key = CloudUtil.getKey(objectID, versionID, fileID, ALPHANUMERIC);
+            response.setStorageKey(key);
+            deleteObject(bucketName, key);
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+        }
+        return response;
+        
+    }
+
+    public CloudResponse deleteManifest (
+            String bucketName,
+            Identifier objectID)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            response = new CloudResponse(bucketName, objectID, null, null);
+            String key = CloudUtil.getManifestKey(objectID, ALPHANUMERIC);
+            response.setStorageKey(key);
+            deleteObject(bucketName, key);
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+        }
+        return response;
+        
+    }
+
+    public InputStream getObject(
+            String bucketName,
+            Identifier objectID,
+            int versionID,
+            String fileID,
+            CloudResponse response)
+        throws TException
+    {
+        try {
+            response.set(bucketName, objectID, versionID, fileID);
+            String key = CloudUtil.getKey(objectID, versionID, fileID, ALPHANUMERIC);
+            return getObject(bucketName, key, response);
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+
+    public InputStream getObject(
+            String bucketName,
+            String key,
+            CloudResponse response)
+        throws TException
+    {
+        try {
+            response.setBucketName(bucketName);
+            response.setStorageKey(key);
+            
+            File baseDir = getBase(bucketName);
+            File pairDir = getDir(baseDir, key);
+            File component = new File(pairDir, "component.txt");
+            if (!component.exists()) {
+                throw new TException.REQUESTED_ITEM_NOT_FOUND("Component not found for:" + key);
+            }
+            InputStream respStream = new FileInputStream(component);
+            
+            File propFile = new File(pairDir, FILEPROPERTIES);
+            if (!propFile.exists()) {
+                throw new TException.INVALID_ARCHITECTURE(MESSAGE + "getObject - " + FILEPROPERTIES + " missing");
+            }
+            Properties fileProp = PropertiesUtil.loadFileProperties(propFile);
+            response.setFileMeta(fileProp);
+            response.setFromProp();
+            return respStream;
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+
+    public InputStream getManifest(
+            String bucketName,
+            Identifier objectID)
+        throws TException
+    {
+        CloudResponse response = null;
+        response = new CloudResponse(bucketName, objectID, null, null);
+        return getManifest(bucketName, objectID,  response);
+    }
+
+    public InputStream getManifest(
+            String bucketName,
+            Identifier objectID,
+            CloudResponse response)
+        throws TException
+    {
+        try {
+            response.set(bucketName, objectID, null, null);
+            String key = CloudUtil.getManifestKey(objectID, ALPHANUMERIC);
+            response.setStorageKey(key);
+            return getObject(bucketName, key, response);
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+   
+    public InputStream getObject(
+            String bucketName,
+            Identifier objectID,
+            int versionID,
+            String fileID)
+        throws TException
+    {
+        CloudResponse response = null;
+        response = new CloudResponse(bucketName, objectID, versionID, fileID);
+        return getObject(bucketName, objectID, versionID, fileID, response);
+    }
+
+
+    public CloudResponse getObjectList (
+            String bucketName,
+            Identifier objectID,
+            Integer versionID,
+            String fileID)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            throw new TException.UNIMPLEMENTED_CODE(MESSAGE + "getObjectList: this repository form does not support this function");
+            
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+    
+    public CloudResponse getObjectList (
+            String bucketName,
+            String key)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            throw new TException.UNIMPLEMENTED_CODE(MESSAGE + "getObjectList: this repository form does not support this function");
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+
+    public CloudResponse getObjectList (
+            String bucketName)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            throw new TException.UNIMPLEMENTED_CODE(MESSAGE + "getObjectList: this repository form does not support this function");
+            
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+    
+    public CloudResponse validateMd5(String bucketName, String key, String inMd5)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            response = new CloudResponse(bucketName, key);
+            InputStream componentStream = getObject(bucketName, key, response);
+            if (response.getException() != null) {
+                response.setMatch(false);
+                return response;
+            }
+            String retMd5 = response.getFileMetaProperty(DIGESTNAME);
+            if (DEBUG) System.out.println("RETMD5=" + retMd5);            
+            if (retMd5.equals(inMd5)) response.setMatch(true);
+            else response.setMatch(false);
+            response.setMd5(retMd5);
+            return response;
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+    
+    public boolean isAlphaNumericKey() 
+    {
+        return ALPHANUMERIC;
+    }
+    
+    
+}
+

File s3-src/src/main/java/org/cdlib/mrt/s3/sdsc/SDSCCloud.java

+/*
+Copyright (c) 2005-2010, Regents of the University of California
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+ *
+- Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+- Neither the name of the University of California nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
+*********************************************************************/
+package org.cdlib.mrt.s3.sdsc;
+//import org.cdlib.mrt.s3.service.*;
+
+
+
+import org.cdlib.mrt.s3.service.*;
+
+import org.cdlib.mrt.core.Identifier;
+import java.io.File;
+import java.io.InputStream;
+import java.util.Enumeration;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import org.jets3t.service.Jets3tProperties;
+import org.jets3t.service.impl.rest.httpclient.RestS3Service;
+import org.jets3t.service.model.S3Object;
+import org.jets3t.service.model.StorageObject;
+import org.jets3t.service.S3Service;
+import org.jets3t.service.model.S3Bucket;
+
+import org.cdlib.mrt.utility.FixityTests;
+import org.cdlib.mrt.utility.StringUtil;
+import org.cdlib.mrt.utility.LoggerInf;
+import org.cdlib.mrt.utility.TException;
+
+import org.cdlib.mrt.s3.service.CloudStoreAbs;
+import org.cdlib.mrt.s3.service.CloudUtil;
+
+/**
+ * Specific SDSC Storage Cloud handling
+ * @author dloy
+ */
+public class SDSCCloud
+    extends CloudStoreAbs
+    implements CloudStoreInf
+{
+    private static final boolean DEBUG = false;
+    private static final boolean ALPHANUMERIC = false;
+    
+    public static SDSCCloud getSDSC(InputStream propStream, LoggerInf logger)
+        throws TException
+    {
+        return new SDSCCloud(propStream, logger);
+    }
+    
+    protected SDSCCloud(
+            InputStream propStream,
+            LoggerInf logger)
+        throws TException
+    {
+        super(propStream, logger);
+        setStorageService();
+    }
+    public static SDSCCloud getSDSC(Properties prop, LoggerInf logger)
+        throws TException
+    {
+        return new SDSCCloud(prop, logger);
+    }
+    
+    protected SDSCCloud(
+            Properties prop,
+            LoggerInf logger)
+        throws TException
+    {
+        super(prop, logger);
+        setStorageService();
+    }
+
+    protected void setStorageService()
+        throws TException
+    {
+        setAWSCredentials();
+        Jets3tProperties properties = new Jets3tProperties();
+        properties.setProperty("s3service.s3-endpoint", endpointHostname);
+        properties.setProperty("s3service.disable-dns-buckets", "true");
+        try {
+            service = new RestS3Service(credentials, null, null, properties);
+        } catch (Exception ex) {
+            throw new TException(ex);
+        }
+        
+    }
+    
+    public S3Service getStorageService()
+    {
+        return service;
+    }
+    
+    public CloudResponse putObject(
+            CloudResponse response,
+            File inputFile)
+        throws TException
+    {        
+        try {
+            if (!isValidFile(inputFile)) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "putObject - file not valid");
+            }
+            String bucketName = response.getBucketName();
+            String key = response.getStorageKey();
+            S3Bucket bucket = new S3Bucket(bucketName);
+            if (DEBUG)System.out.println("FILE size:" + inputFile.length());
+            String md5Hex = CloudUtil.getDigestValue(inputFile, logger);
+            byte[] md5Bytes = FixityTests.toByteArray(md5Hex);
+            response.setMd5(md5Hex);
+            
+            
+            S3Object object = new S3Object(bucket, inputFile);
+            object.setKey(key);
+            object.setDataInputFile(inputFile);
+            object.setContentLength(inputFile.length());
+            object.setContentType("application/octet-stream");
+            object.setMd5Hash(md5Bytes);
+            if (response.getFileMetaSize() > 0) {
+                Properties prop = response.getFileMetaProperties();
+                setObjectMeta(prop, object);
+            }
+            
+            service.putObject(bucket, object);
+                        
+        } catch (Exception ex) {
+            handleException(response, ex);
+        }
+        return response;
+    }
+    
+    protected void setObjectMeta(Properties prop, S3Object object)
+    {        if (prop == null) return;
+        try {
+            Enumeration e = prop.propertyNames();
+            String key = null;
+            String value = null;
+
+            while( e.hasMoreElements() )
+            {
+                key = (String)e.nextElement();
+                value = prop.getProperty(key);
+                object.addMetadata(key, value);;
+            }
+            
+        } catch (Exception ex) {
+            throw new RuntimeException("setFilePrope:" + ex);
+        }
+    }
+    public CloudResponse putObject(
+            String bucketName,
+            String key,
+            File inputFile)
+        throws TException
+    { 
+        if (StringUtil.isEmpty(bucketName)) {
+            throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "putObject - bucketName not valid");
+        }
+        if (StringUtil.isEmpty(key)) {
+            throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "putObject - key not valid");
+        }
+        CloudResponse response = new CloudResponse(bucketName, key);
+        return putObject(response, inputFile);
+    }
+    
+    public CloudResponse putObject(
+            String bucketName,
+            Identifier objectID,
+            int versionID,
+            String fileID,
+            File inputFile)
+        throws TException
+    {
+        CloudResponse response = new CloudResponse(bucketName, objectID, versionID, fileID);
+        String key = CloudUtil.getKey(objectID, versionID, fileID, ALPHANUMERIC);
+        response.setStorageKey(key);
+        return putObject(response, inputFile);
+    }
+    
+    public CloudResponse putObject(
+            String bucketName,
+            String key,
+            File inputFile,
+            Properties fileMeta)
+        throws TException
+    { 
+        if (StringUtil.isEmpty(bucketName)) {
+            throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "putObject - bucketName not valid");
+        }
+        if (StringUtil.isEmpty(key)) {
+            throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "putObject - key not valid");
+        }
+        CloudResponse response = new CloudResponse(bucketName, key);
+        response.setFileMeta(fileMeta);
+        return putObject(response, inputFile);
+    }
+    
+    public CloudResponse putObject(
+            String bucketName,
+            Identifier objectID,
+            int versionID,
+            String fileID,
+            File inputFile,
+            Properties fileMeta)
+        throws TException
+    {
+        CloudResponse response = new CloudResponse(bucketName, objectID, versionID, fileID);
+        String key = CloudUtil.getKey(objectID, versionID, fileID, ALPHANUMERIC);
+        response.setStorageKey(key);
+        response.setFileMeta(fileMeta);
+        return putObject(response, inputFile);
+    }
+    
+    public CloudResponse putManifest(
+            String bucketName,
+            Identifier objectID,
+            File inputFile)
+        throws TException
+    {
+        CloudResponse response = new CloudResponse(bucketName, objectID, null, null);
+        String key = CloudUtil.getManifestKey(objectID, ALPHANUMERIC);
+        response.setStorageKey(key);
+        return putObject(response, inputFile);
+    }
+
+    public CloudResponse deleteObject (
+            String bucketName,
+            String key)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            response = new CloudResponse(bucketName, key);
+            response.setStorageKey(key);
+            
+            S3Bucket bucket = new S3Bucket(bucketName);
+            service.deleteObject(bucket, key);
+            return response;
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+        }
+        return response;
+        
+    }
+
+    public CloudResponse deleteObject (
+            String bucketName,
+            Identifier objectID,
+            int versionID,
+            String fileID)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            response = new CloudResponse(bucketName, objectID, versionID, fileID);
+            String key = CloudUtil.getKey(objectID, versionID, fileID, ALPHANUMERIC);
+            response.setStorageKey(key);
+            
+            S3Bucket bucket = new S3Bucket(bucketName);
+            service.deleteObject(bucket, key);
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+        }
+        return response;
+        
+    }
+
+    public CloudResponse deleteManifest (
+            String bucketName,
+            Identifier objectID)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            response = new CloudResponse(bucketName, objectID, null, null);
+            String key = CloudUtil.getManifestKey(objectID, ALPHANUMERIC);
+            response.setStorageKey(key);
+            
+            S3Bucket bucket = new S3Bucket(bucketName);
+            service.deleteObject(bucket, key);
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+        }
+        return response;
+        
+    }
+
+    public InputStream getObject(
+            String bucketName,
+            Identifier objectID,
+            int versionID,
+            String fileID,
+            CloudResponse response)
+        throws TException
+    {
+        try {
+            response.set(bucketName, objectID, versionID, fileID);
+            String key = CloudUtil.getKey(objectID, versionID, fileID, ALPHANUMERIC);
+            return getObject(bucketName, key, response);
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+
+    public InputStream getObject(
+            String bucketName,
+            String key,
+            CloudResponse response)
+        throws TException
+    {
+        try {
+            
+            S3Object outObject = service.getObject(bucketName, key);
+            response.setExtMetadata(outObject);
+            response.setBucketName(bucketName);
+            response.setStorageKey(key);
+            return outObject.getDataInputStream();
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+
+    public InputStream getManifest(
+            String bucketName,
+            Identifier objectID)
+        throws TException
+    {
+        CloudResponse response = null;
+        response = new CloudResponse(bucketName, objectID, null, null);
+        return getManifest(bucketName, objectID,  response);
+    }
+
+    public InputStream getManifest(
+            String bucketName,
+            Identifier objectID,
+            CloudResponse response)
+        throws TException
+    {
+        try {
+            response.set(bucketName, objectID, null, null);
+            String key = CloudUtil.getManifestKey(objectID, ALPHANUMERIC);
+            response.setStorageKey(key);
+            
+            S3Object outObject = service.getObject(bucketName, key);
+            if (outObject == null) {
+                if (DEBUG) System.out.println("null outObject");
+            } else {
+                if (outObject.getDataInputFile() == null) {
+                    if (DEBUG) System.out.println("null outObject.getDataInputFile()");
+                }
+            }
+            response.setExtObject(outObject);
+            return outObject.getDataInputStream();
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+   
+    public InputStream getObject(
+            String bucketName,
+            Identifier objectID,
+            int versionID,
+            String fileID)
+        throws TException
+    {
+        CloudResponse response = null;
+        response = new CloudResponse(bucketName, objectID, versionID, fileID);
+        return getObject(bucketName, objectID, versionID, fileID, response);
+    }
+
+
+    public CloudResponse getObjectList (
+            String bucketName,
+            Identifier objectID,
+            Integer versionID,
+            String fileID)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            response = new CloudResponse(bucketName, objectID, versionID, fileID);
+            String key = CloudUtil.getKey(objectID, versionID, fileID, ALPHANUMERIC);
+            response.setStorageKey(key);
+            
+            S3Object[] objects = service.listObjects(bucketName, key, null);
+            response.setObjectList(objects);
+            return response;
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+    
+    public CloudResponse getObjectList (
+            String bucketName,
+            String key)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            response = new CloudResponse(bucketName, key);
+            response.setStorageKey(key);
+            
+            S3Object[] objects = service.listObjects(bucketName, key, null);
+            response.setObjectList(objects);
+            return response;
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+
+    public CloudResponse getObjectList (
+            String bucketName)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            response = new CloudResponse();
+            response.setBucketName(bucketName);
+            S3Object[] objects = service.listObjects(bucketName);
+            
+            response.setObjectList(objects);
+            return response;
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+    
+    public CloudResponse validateMd5(String bucketName, String key, String inMd5)
+        throws TException
+    {
+        CloudResponse response = null;
+        try {
+            response = getObjectList(bucketName, key);
+            if (response.getException() != null) {
+                response.setMatch(false);
+                return response;
+            }
+            StorageObject[] objects = response.getObjectList();
+            if ((objects == null) || (objects.length == 0)) {
+                throw new TException.REQUESTED_ITEM_NOT_FOUND("item not found - key=" + key);
+            }
+            if (objects.length != 1) {
+                throw new TException.INVALID_OR_MISSING_PARM("key returns multiple objects - key=" + key
+                        + " - count=" + objects.length);
+            }
+            StorageObject object = objects[0];
+            String retMd5 = object.getETag();
+            if (DEBUG) System.out.println("RETMD5=" + retMd5);
+            if (retMd5.equals(inMd5)) response.setMatch(true);
+            else response.setMatch(false);
+            response.setMd5(retMd5);
+            return response;
+            
+        } catch (Exception ex) {
+            handleException(response, ex);
+            return null;
+        }
+    }
+    
+    public boolean isAlphaNumericKey() 
+    {
+        return ALPHANUMERIC;
+    }
+}
+

File s3-src/src/main/java/org/cdlib/mrt/s3/sdsc/SDSCCloudEnum.java

+/*
+Copyright (c) 2005-2010, Regents of the University of California
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+ *
+- Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+- Neither the name of the University of California nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
+*********************************************************************/
+package org.cdlib.mrt.s3.sdsc;
+//import org.cdlib.mrt.s3.service.*;
+
+
+
+import org.cdlib.mrt.s3.service.*;
+
+import java.util.Enumeration;
+import java.util.Properties;
+
+import org.jets3t.service.Jets3tProperties;
+import org.jets3t.service.impl.rest.httpclient.RestS3Service;
+import org.jets3t.service.S3Service;
+import org.jets3t.service.model.StorageObject;
+import org.jets3t.service.StorageObjectsChunk;
+
+import org.cdlib.mrt.utility.StringUtil;
+import org.cdlib.mrt.utility.LoggerInf;
+import org.cdlib.mrt.utility.TException;
+
+import org.cdlib.mrt.s3.service.CloudStoreAbs;
+
+/**
+ * Specific SDSC Storage Cloud handling
+ * @author dloy
+ */
+public class SDSCCloudEnum
+    extends CloudStoreAbs
+    implements Enumeration
+{
+    private static final boolean DEBUG = false;
+    private static final boolean ALPHANUMERIC = false;
+    private CloudResponse current = null;
+    private String startKey = null;
+    private String bucketName = null;
+    private long maxListingLength = 0;
+    private String priorLastKey = null;
+    private StorageObjectsChunk soc = null;
+    private boolean listingComplete = false;
+    private StorageObject [] storageObjects = null;
+    private long cnt=0;
+    
+    public static SDSCCloudEnum getSDSCCloudEnum(
+            Properties prop,
+            String bucketName,
+            String startKey, 
+            long maxListingLength,
+            LoggerInf logger)
+        throws TException
+    {
+        return new SDSCCloudEnum(prop, bucketName, startKey, maxListingLength, logger);
+    }
+    
+    protected SDSCCloudEnum(
+            Properties prop,
+            String bucketName,
+            String startKey, 
+            long maxListingLength,
+            LoggerInf logger)
+        throws TException
+    {
+        super(prop, logger);
+        
+        this.bucketName = bucketName;
+        this.startKey = startKey;
+        this.maxListingLength = maxListingLength;
+        validate();
+        setStorageService();
+    }
+    
+    public static SDSCCloudEnum SDSCCloudEnum(
+            CloudStoreAbs cloudStoreAbs,
+            String bucketName,
+            String startKey, 
+            long maxListingLength)
+        throws TException
+    {
+        return new SDSCCloudEnum(cloudStoreAbs, bucketName, startKey, maxListingLength);
+    }
+    
+    protected SDSCCloudEnum(
+            CloudStoreAbs cloudStoreAbs,
+            String bucketName,
+            String startKey, 
+            long maxListingLength)
+        throws TException
+    {
+        super(cloudStoreAbs);
+        this.bucketName = bucketName;
+        this.startKey = startKey;
+        this.maxListingLength = maxListingLength;
+        validate();
+    }
+
+            
+    private void validate()
+        throws  TException
+    {
+        if (StringUtil.isAllBlank(bucketName)) {
+            throw new TException.INVALID_OR_MISSING_PARM(
+                    "bucketName not supplied");
+        }
+        if (maxListingLength <= 0) maxListingLength = 1000;
+    }
+    
+    public Enumeration<SDSCCloudEnum> getListEnumeration()
+        throws TException
+    {
+        try {
+            return this;
+                    
+        } catch (Exception ex) {
+            throw new TException(ex);
+        }
+    }
+    
+    public boolean hasMoreElements()
+    {
+        try {
+            if (DEBUG) System.out.println(">>>>>>>>>>>>>>>>>>hasMoreElements - cnt=" + cnt
+                    + " - listingComplete=" + listingComplete
+                    + " - bucketName=" + bucketName
+                    + " - priorLastKey=" + priorLastKey
+                    + " - maxListingLength=" + maxListingLength
+                    + " - storageObjects=" + ((storageObjects == null) ? "null" : storageObjects.length)
+                    );
+            if (storageObjects != null) return true;
+            if (listingComplete) return false;
+            soc = service.listObjectsChunked(
+                    bucketName,
+                    null, null, 
+                    maxListingLength, 
+                    priorLastKey);
+            priorLastKey = soc.getPriorLastKey();
+            if (DEBUG) System.out.println("After:" + soc.isListingComplete()
+                    + " - length:" + soc.getObjects().length
+                    + " - priorLastKey:" + soc.getPriorLastKey()
+                    );
+            listingComplete = soc.isListingComplete();
+            storageObjects = soc.getObjects();
+            storageObjects 
+                    = (storageObjects == null) || (storageObjects.length == 0) 
+                    ? null : storageObjects;
+            if (storageObjects != null) return true;
+            return false;
+                    
+        } catch (Exception ex) {
+            return false;
+        }
+    }
+
+    public CloudResponse nextElement()
+    {
+        if (DEBUG) System.out.println(">>>>>>>>>>>>>>>>>>nextElement "
+                    + " - storageObjects=" + ((storageObjects == null) ? "null" : storageObjects.length)
+                    );
+        if (storageObjects == null) {
+            throw new RuntimeException("nextElement not found");
+        }
+        CloudResponse response = new CloudResponse();
+        response.setObjectList(storageObjects);
+        response.setBucketName(bucketName);
+        cnt += storageObjects.length;
+        storageObjects = null;
+        return response;
+    }
+    
+    protected void setStorageService()
+        throws TException
+    {
+        setAWSCredentials();
+        Jets3tProperties properties = new Jets3tProperties();
+        properties.setProperty("s3service.s3-endpoint", endpointHostname);
+        properties.setProperty("s3service.disable-dns-buckets", "true");
+        try {
+            service = new RestS3Service(credentials, null, null, properties);
+        } catch (Exception ex) {
+            throw new TException(ex);
+        }
+        
+    }
+    
+    public S3Service getStorageService()
+    {
+        return service;
+    }
+}
+

File s3-src/src/main/java/org/cdlib/mrt/s3/service/CloudProperties.java

+/*
+Copyright (c) 2005-2010, Regents of the University of California
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+ *
+- Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+- Neither the name of the University of California nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
+*********************************************************************/
+package org.cdlib.mrt.s3.service;
+import org.cdlib.mrt.core.Identifier;
+import org.cdlib.mrt.utility.TException;
+import org.cdlib.mrt.utility.StringUtil;
+
+import org.jets3t.service.model.S3Object;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Vector;
+import org.cdlib.mrt.utility.PropertiesUtil;
+
+
+
+
+/**
+ * Cloud Properties - case insensitive Properties
+ * @author dloy
+ */
+public class CloudProperties
+{
+    private Properties prop = new Properties();
+    
+    public void setProperty(String key, String value)
+    {
+        if (StringUtil.isEmpty(key)) return;
+        if (StringUtil.isEmpty(value)) return;
+        prop.setProperty(key.toLowerCase(), value);
+    }
+    
+    public String getProperty(String key)
+    {
+        if (StringUtil.isEmpty(key)) return null;
+        return prop.getProperty(key.toLowerCase());
+    }
+    
+    public Properties getProperties()
+    {
+        return prop;
+    }
+    
+    public int size()
+    {
+        return prop.size();
+    }
+    
+    public void clear()
+    {
+        prop.clear();
+    }
+    
+    public String toString()
+    {
+        return prop.toString();
+    }
+    
+    public String dump(String header)
+    {
+        return PropertiesUtil.dumpProperties(header, prop);
+    }
+}
+

File s3-src/src/main/java/org/cdlib/mrt/s3/service/CloudResponse.java

+/*
+Copyright (c) 2005-2010, Regents of the University of California
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+ *
+- Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+- Neither the name of the University of California nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
+*********************************************************************/
+package org.cdlib.mrt.s3.service;
+import org.cdlib.mrt.core.Identifier;
+import org.cdlib.mrt.utility.StringUtil;
+
+import org.jets3t.service.model.StorageObject;
+
+import java.util.Enumeration;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Vector;
+
+
+
+
+/**
+ * Cloud response to a cloud storage request
+ * @author dloy
+ */
+public class CloudResponse
+{
+    private static final boolean DEBUG = false;
+    public enum ResponseStatus{ok, fail, unknown};
+    private int httpStatus = 0;
+    private ResponseStatus status = ResponseStatus.ok;
+    private StorageObject[] objectList = null;
+    private StorageObject extObject = null;
+    private Exception exception = null;
+
+    private String bucketName = null;
+    private Identifier objectID = null;
+    private Integer versionID = null;
+    private String fileID = null;
+    private String storageKey = null;
+    private String errMsg = null;
+    private String md5 = null;
+    private String mimeType = null;
+    private long storageSize = 0;
+    private boolean match = false;
+    private CloudProperties fileMeta = new CloudProperties();
+    
+    public CloudResponse() {} 
+    
+    public CloudResponse(
+            String bucketName,
+            Identifier objectID,
+            Integer versionID,
+            String fileID)
+    {
+        set(bucketName, objectID, versionID, fileID);
+    }
+    
+    public CloudResponse(
+            String bucketName,
+            String storageKey)
+    {
+        this.bucketName = bucketName;
+        this.storageKey = storageKey;
+    }
+    
+    public void set(
+            String bucketName,
+            Identifier objectID,
+            Integer versionID,
+            String fileID)
+    {
+        this.bucketName = bucketName;
+        this.objectID = objectID;
+        this.versionID = versionID;
+        this.fileID = fileID;
+    }
+
+    public String getBucketName() {
+        return bucketName;
+    }
+
+    public void setBucketName(String bucketName) {
+        this.bucketName = bucketName;
+    }
+
+    public Exception getException() {
+        return exception;
+    }
+
+    public void setException(Exception exception) {
+        this.exception = exception;
+    }
+
+    public String getFileID() {
+        return fileID;
+    }
+
+    public void setFileID(String fileID) {
+        this.fileID = fileID;
+    }
+
+    public int getHttpStatus() {
+        return httpStatus;
+    }
+
+    public void setHttpStatus(int httpStatus) {
+        this.httpStatus = httpStatus;
+    }
+
+    public Identifier getObjectID() {
+        return objectID;
+    }
+
+    public void setObjectID(Identifier objectID) {
+        this.objectID = objectID;
+    }
+
+    public StorageObject[] getObjectList() {
+        return objectList;
+    }
+
+    public void setObjectList(StorageObject[] objectList) {
+        this.objectList = objectList;
+    }
+
+    public ResponseStatus getStatus() {
+        return status;
+    }
+
+    public void setStatus(ResponseStatus status) {
+        this.status = status;
+    }
+
+    public String getStorageKey() {
+        return storageKey;
+    }
+
+    public void setStorageKey(String storageKey) {
+        this.storageKey = storageKey;
+    }
+
+    public Integer getVersionID() {
+        return versionID;
+    }
+
+    public void setVersionID(Integer versionID) {
+        this.versionID = versionID;
+    }
+
+    public String getErrMsg() {
+        return errMsg;
+    }
+
+    public void setErrMsg(String errMsg) {
+        this.errMsg = errMsg;
+    }
+
+    public StorageObject getExtObject() {
+        return extObject;
+    }
+
+    public void setExtObject(StorageObject extObject) {
+        this.extObject = extObject;
+    }
+
+    public void setExtMetadata(StorageObject extObject) {
+        this.extObject = extObject;
+        setStorageSize(extObject.getContentLength());
+        Map<String, Object> map = extObject.getModifiableMetadata();
+        Set<String> keys = map.keySet();
+        for (String key: keys) {
+            Object value = map.get(key);
+            if (value instanceof String) {
+                if (DEBUG) System.out.println("setExtMetadata key=" + key + " - value=" + value);
+                fileMeta.setProperty(key, (String)value);
+            }
+        }
+        String etag = fileMeta.getProperty("etag");
+        setMd5(etag);
+        if (DEBUG) System.out.println("setExtMetadata3 etag=" + getMd5());
+        if (StringUtil.isNotEmpty(storageKey)) {
+            try {
+                CloudUtil.KeyElements element = CloudUtil.getKeyElements(storageKey);
+                setObjectID(element.objectID);
+                setVersionID(element.versionID);
+                setFileID(element.fileID);
+            } catch (Exception ex) { }
+            
+        }
+        setFromProp();
+    }
+
+    public CloudProperties getFileMeta() {
+        return fileMeta;
+    }
+
+    public Properties getFileMetaProperties() {
+        if ((fileMeta == null) || (fileMeta.size() == 0)) return null;
+        return fileMeta.getProperties();
+    }
+
+    public void setFileMeta(CloudProperties fileProp)
+           
+    {
+        this.fileMeta = fileProp;
+        if (fileProp != null) {
+            setFromProp();
+        }
+    }
+
+    public void addFileMeta(Properties prop) {
+        if (prop == null) return;
+        
+        if (DEBUG) System.out.println("***addFileMeta storageKey1=" + storageKey);
+        try {
+            Enumeration e = prop.propertyNames();
+            String key = null;
+            String value = null;
+
+            while( e.hasMoreElements() )
+            {
+                key = (String)e.nextElement();
+                value = prop.getProperty(key);
+                String testValue = fileMeta.getProperty(key);
+                if ((testValue != null) && !value.equals(testValue))
+                    throw new RuntimeException("setFileProp duplicate key exists:" + key
+                            + " - value=" + value
+                            + " - testValue=" + testValue
+                            );
+                fileMeta.setProperty(key, value);
+                if (DEBUG) System.out.println("**addFileMeta - key=" + key + " - value=" + value);
+            }
+            
+        } catch (Exception ex) {
+            throw new RuntimeException("setFilePrope:" + ex);
+        }
+    }
+
+    public void setFileMeta(Properties prop) {
+        if (prop == null) return;
+        setFileMetaClear();
+        addFileMeta(prop);
+    }
+    
+    public void setFileMetaClear()
+    {
+        fileMeta.clear();
+    }
+    
+    public int getFileMetaSize()
+    {
+        return fileMeta.size();
+    }
+    
+    public String getFileMetaProperty(String key)
+    {
+        if ((fileMeta == null) || (fileMeta.size() == 0)) return null;
+        return fileMeta.getProperty(key);
+    }
+    
+    public void setFileMetaProperty(String key, String value)
+    {
+        fileMeta.setProperty(key, value);
+    }
+
+    public boolean err()
+    {
+        if (status == ResponseStatus.ok) return false;
+        return true;
+    }
+
+    public String getMd5() {
+        return md5;
+    }
+
+    public void setMd5(String md5) {
+        this.md5 = md5;
+    }
+
+    public boolean isMatch() {
+        return match;
+    }
+
+    public void setMatch(boolean match) {
+        this.match = match;
+    }
+
+    public long getStorageSize() {
+        return storageSize;
+    }
+
+    public void setStorageSize(long storageSize) {
+        this.storageSize = storageSize;
+    }
+
+    public void setStorageSize(String storageSizeS) {
+        if (StringUtil.isEmpty(storageSizeS)) {
+            this.storageSize = 0;
+        } else {
+            try {
+                this.storageSize = Long.parseLong(storageSizeS);
+            } catch (Exception ex) {
+                this.storageSize = 0;
+            }
+        }
+    }
+
+    public String getMimeType() {
+        return mimeType;
+    }
+
+    public void setMimeType(String mimeType) {
+        this.mimeType = mimeType;
+    }
+    
+    public String dump(String header) {
+        StringBuffer buf = new StringBuffer();
+        buf.append("CloudResponse **" + header + "**");
+        if (objectID == null) {
+            buf.append("EMPTY element");
+        }
+        if (objectID != null) {
+            buf.append(" - objectID=" + objectID.toString());
+        }
+        if (versionID != null) {
+            buf.append(" - versionID=" + versionID);
+        }
+        if (fileID != null) {
+            buf.append(" - fileID=" + fileID);
+