Commits

dloy committed e2f6843

Changes from old fixity version

  • Participants
  • Parent commits dede01b

Comments (0)

Files changed (51)

     <version>1.0-SNAPSHOT</version>
     <name>UC3-mrtcore</name>
     <url>http://uc3.cdlib.org</url>
-
     <!-- force UTF-8 -->
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
             <version>1.7.0</version>
         </dependency>
         <dependency>
+            <groupId>jaxen</groupId>
+            <artifactId>jaxen</artifactId>
+            <version>1.1.1</version>
+        </dependency>
+        <dependency>
             <groupId>jdom</groupId>
             <artifactId>jdom</artifactId>
             <version>1.1</version>
         </dependency>
+        <dependency>
+            <groupId>com.jolbox</groupId>
+            <artifactId>bonecp</artifactId>
+            <!--version>0.7.2-SNAPSHOT</version-->
+            <version>0.7.1-rc3</version>
+        </dependency>
+  
+        <dependency>
+            <groupId>org.apache.tika</groupId>
+            <artifactId>tika-app</artifactId>
+            <version>1.2</version>
+        </dependency>
         <!--dependency>
             <groupId>net.sf</groupId>
             <artifactId>jargs</artifactId>

core/src/main/java/org/cdlib/mrt/cloud/ManInfo.java

+/*
+Copyright (c) 2005-2012, Regents of the University of California
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+ *
+- Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+- Neither the name of the University of California nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************/
+package org.cdlib.mrt.cloud;
+
+import java.util.List;
+
+
+import org.jdom.Element;
+
+import org.cdlib.mrt.core.DateState;
+import org.cdlib.mrt.core.FileComponent;
+import org.cdlib.mrt.core.ComponentContent;
+import org.cdlib.mrt.utility.TException;
+
+
+    
+public class ManInfo
+{
+    public int versionID = 0;
+    public int count=0; 
+    public long size=0;
+    public DateState created = null;
+    public Element elem = null;
+    public ComponentContent components = null;
+
+    public String dump(String header) {
+        StringBuffer buf = new StringBuffer();
+        buf.append(header
+                + " - versionID=" + versionID + "\n"
+                + " - count=" + count + "\n"
+                + " - size=" + size + "\n"
+                + " - created=" + created.getIsoDate() + "\n"
+                );
+        return buf.toString();
+    }
+    public void setComponents(int versionID, List<FileComponent> componentList)
+        throws TException
+    {
+        components = new ComponentContent(componentList); 
+        components.setVersionID(versionID);
+    }
+}

core/src/main/java/org/cdlib/mrt/cloud/ManifestXML.java

+/*
+Copyright (c) 2005-2012, Regents of the University of California
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+ *
+- Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+- Neither the name of the University of California nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************/
+package org.cdlib.mrt.cloud;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+import java.util.ArrayList;
+
+
+import org.jdom.Attribute;
+import org.jdom.Document;
+import org.jdom.Element;
+import org.jdom.input.SAXBuilder;
+import org.jdom.output.XMLOutputter;
+import org.jdom.xpath.XPath;
+
+import org.cdlib.mrt.cloud.ManInfo;
+import org.cdlib.mrt.cloud.VersionMap;
+import org.cdlib.mrt.core.ComponentContent;
+import org.cdlib.mrt.core.DateState;
+import org.cdlib.mrt.core.FileComponent;
+import org.cdlib.mrt.core.Identifier;
+import org.cdlib.mrt.core.MessageDigest;
+import org.cdlib.mrt.utility.LoggerInf;
+import org.cdlib.mrt.utility.StringUtil;
+import org.cdlib.mrt.utility.TException;
+import org.cdlib.mrt.utility.TFileLogger;
+import org.jdom.Namespace;
+import org.jdom.output.Format;
+
+/**
+ * This object imports the formatTypes.xml and builds a local table of supported format types.
+ * Note, that the ObjectFormat is being deprecated and replaced by a single format id (fmtid).
+ * This change is happening because formatName is strictly a description and has no functional
+ * use. The scienceMetadata flag is being dropped because the ORE Resource Map is more flexible
+ * and allows for a broader set of data type.
+ * 
+ * @author dloy
+ */
+public class ManifestXML
+{
+    private static final String NAME = "ObjectFormatXML";
+    private static final String MESSAGE = NAME + ": ";
+
+    private static final String NL = System.getProperty("line.separator");
+    private static final boolean DEBUG = false;
+    
+    protected VersionMap versionMap = null;
+    //protected Document doc = null;
+    protected LoggerInf logger = null;
+
+    public static final Namespace uc3Name = Namespace.getNamespace("http://uc3.cdlib.org/ontology/mrt/manifest");
+    public static VersionMap getVersionMap(Identifier objectID, LoggerInf logger, InputStream xmlStream)
+           throws TException
+    {
+        VersionMap versionMap = new VersionMap(objectID, logger);
+        
+        try {
+            if (xmlStream == null) return versionMap;
+            Document doc =  getDocument(xmlStream);
+            if (DEBUG) System.out.println("Doc built");
+            extractObjectInfo(doc, versionMap);
+            if (versionMap.getCurrent() > 0) {
+                extractManifests(doc, versionMap);
+                versionMap.rebuildHash();
+            }
+            return versionMap;
+
+        } catch (Exception ex) {
+            System.out.println("ObjectFormatXML: Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+
+        }
+    }
+    
+    /**
+     * Input the formatTypes.xml and build a Document
+     * @throws TException
+     */
+    protected static Document getDocument(InputStream inStream)
+        throws TException
+    {
+        try {
+            SAXBuilder builder = new SAXBuilder();
+            if (DEBUG) System.out.println("Doc built");
+            return builder.build(inStream);
+
+        } catch (Exception ex) {
+            System.out.println("ObjectFormatXML: Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+
+        } finally {
+            if (inStream != null) {
+                try {
+                    inStream.close();
+                } catch (Exception e) { }
+            }
+        }
+    }
+    
+    protected static void extractObjectInfo(Document doc, VersionMap versionMap)
+        throws TException
+    {
+        
+        try {
+            Element root = doc.getRootElement();
+            Element objectE = root.getChild("object", uc3Name);           
+            if (objectE == null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "object element missing");
+            }
+            Attribute idA = objectE.getAttribute("id");
+            if (idA == null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "object attribute id missing");
+            }
+            String idS = idA.getValue();
+            
+            Element currentE = objectE.getChild("current", uc3Name);
+            Element fileCountE = objectE.getChild("fileCount", uc3Name);
+            Element totalSizeE = objectE.getChild("totalSize", uc3Name);
+            Element actualCountE = objectE.getChild("actualCount", uc3Name);
+            Element actualSizeE = objectE.getChild("actualSize", uc3Name);
+            Element versionCountE = objectE.getChild("versionCount", uc3Name);
+            Element lastAddVersionE = objectE.getChild("lastAddVersion", uc3Name);
+            Element lastDeleteVersionE = objectE.getChild("lastDeleteVersion", uc3Name);
+            
+            if (currentE == null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "current version missing");
+            }
+            
+            if (fileCountE == null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "fileCount missing");
+            }
+            if (totalSizeE == null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "totalSize missing");
+            }
+            if (actualCountE == null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "actualCount missing");
+            }
+            if (actualSizeE == null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "actualSize missing");
+            }
+            if (versionCountE == null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "versionCount missing");
+            }
+            if (lastAddVersionE == null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "lastAddVersionE missing");
+            }
+            int current = getValue(currentE);
+            int fileCount = getValue(fileCountE);
+            long totalSize = getValueLong(totalSizeE);
+            int actualCount = getValue(actualCountE);
+            long actualSize = getValueLong(actualSizeE);
+            int versionCount = getValue(versionCountE);
+            
+            versionMap.setCurrent(current);
+            versionMap.setOriginalActualCount(actualCount);            
+            versionMap.setOriginalFileCount(fileCount);            
+            versionMap.setOriginalActualSize(actualSize);            
+            versionMap.setOriginalTotalSize(totalSize);
+            versionMap.setOriginalVersionCnt(versionCount);
+            versionMap.setOriginalVersionCnt(versionCount);
+            
+            String lastAddVersionS = lastAddVersionE.getValue();
+            DateState lastAddVersion = new DateState(lastAddVersionS);
+            versionMap.setOriginalLastAddVersion(lastAddVersion);
+            versionMap.setLastAddVersion(lastAddVersion);
+            
+            if (lastDeleteVersionE != null) {
+                String lastDeleteVersionS = lastAddVersionE.getValue();
+                DateState lastDeleteVersion = new DateState(lastDeleteVersionS);
+                versionMap.setOriginalLastDeleteVersion(lastDeleteVersion);
+                versionMap.setLastDeleteVersion(lastDeleteVersion);
+            }
+            
+
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    
+    protected static int getValue(Element elem)
+        throws Exception
+    {
+        String valS = elem.getValue();
+        return Integer.parseInt(valS);
+    }
+    
+    protected static long getValueLong(Element elem)
+        throws Exception
+    {
+        String valS = elem.getValue();
+        return Long.parseLong(valS);
+    }
+    
+    public String dump(String header) {
+        StringBuffer buf = new StringBuffer();
+        versionMap.dump("ManifestXml");
+        buf.append(versionMap.dump("versionMap"));
+        return buf.toString();
+    }
+    
+    public String dumpVersionMap(String header) {
+        return versionMap.dump(header);
+    }
+        
+    /**
+     * Extract the 3 elements of the FormatObject from this xml and build table
+     * @throws TException
+     */
+    protected static void extractManifests(Document doc, VersionMap versionMap)
+        throws TException
+    {
+        try {
+            XPath xpath = XPath.newInstance("//x:manifest");
+            xpath.addNamespace("x", uc3Name.getURI());
+            List list = xpath.selectNodes(doc);
+            for (Iterator iter = list.iterator(); iter.hasNext();) {
+                Element elem = (Element) iter.next();
+                ManInfo info = new ManInfo();
+                Attribute countA = elem.getAttribute("count");
+                Attribute sizeA = elem.getAttribute("size");
+                Attribute createdA = elem.getAttribute("created");               
+                info.elem = elem;
+                if (countA == null) {
+                    throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "manifest missing count");
+                }
+                if (sizeA == null) {
+                    throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "manifest missing size");
+                }
+                if (createdA == null) {
+                    throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "manifest missing created");
+                }
+                info.size = sizeA.getLongValue();
+                info.count = countA.getIntValue();
+                
+                String createdS = createdA.getValue();
+                if (StringUtil.isNotEmpty(createdS)) {
+                    info.created = new DateState(createdS);
+                }
+                info.count = countA.getIntValue();
+                Element version = elem.getParentElement();
+                if (version == null) {
+                    throw new TException.INVALID_ARCHITECTURE("version doesn't exist");
+                }
+                Attribute versionIDA = version.getAttribute("id");
+                if (versionIDA == null) {
+                    throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "manifest missing count");
+                }
+                info.versionID = versionIDA.getIntValue();
+                info.components = getVersionContent(info.versionID, info);
+                if (DEBUG) System.out.println("SIZE=" + info.components.size()); //!!!
+                versionMap.add(info);
+                //System.out.println(info.dump("manifest"));
+            }
+            if (versionMap.getVersionCount() == 0) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "No manifest content found");
+                
+            }
+        } catch (TException tex) {
+            System.out.println(MESSAGE + "Exception:" + tex);
+            tex.printStackTrace();
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    
+    protected static ComponentContent getVersionContent(int versionID, ManInfo info)
+         throws TException
+    {
+        ArrayList<FileComponent> components = new ArrayList<FileComponent>();
+        
+        try {
+            if (info == null) return null;
+            Element manE = info.elem;
+           
+            XPath xpath = XPath.newInstance("x:file");
+            xpath.addNamespace("x", uc3Name.getURI());
+            List list = xpath.selectNodes(manE);
+            for (Iterator iter = list.iterator(); iter.hasNext();) {
+                Element fileElem = (Element) iter.next();
+                FileComponent component = getComponent(fileElem);
+                components.add(component);
+                if (DEBUG) System.out.println("getVesionContent:" + component.getIdentifier());
+            }
+            ComponentContent content = new ComponentContent(components);
+            content.setVersionID(versionID);
+            return content;
+            
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    
+    public static FileComponent getComponent(Element fileElem)
+         throws TException
+    {
+        
+        try {
+            if (false) throw new TException.UNIMPLEMENTED_CODE("test");
+            Attribute idA = fileElem.getAttribute("id");
+            String id = idA.getValue();
+            if (id.substring(0,1).equals("/") || id.substring(0,1).equals("\\")) {
+                id = id.substring(1);
+            }
+            FileComponent component = new FileComponent();
+            component.setIdentifier(id);
+            if (DEBUG) System.out.println("Call getComponent:" + id);
+            String digestType = getValue(fileElem, "digestType");
+            String digest = getValue(fileElem, "digest");
+            String sizeS = getValue(fileElem, "size");
+            String creationDateS = getValue(fileElem, "creationDate");
+            String key = getValue(fileElem, "key");
+            component.addMessageDigest(digest, digestType);
+            component.setSize(sizeS);
+            component.setLastModifiedDate(creationDateS);
+            component.setLocalID(key);
+            Element mimeTypeE = fileElem.getChild("mimeType", uc3Name);
+            if (mimeTypeE != null) {
+                component.setMimeType(mimeTypeE.getValue());
+            }
+            
+            
+            return component;
+            
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    
+    protected static String getValue(Element fileElem, String name)
+        throws Exception
+    {
+        Element elem = fileElem.getChild(name, uc3Name);
+        if (elem == null) {
+            throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "element not found for key:" + name);
+        }
+        return elem.getValue();
+        
+    }
+
+    
+    public static Element buildManifestEntry(ManInfo info)
+        throws TException
+    {
+        try {
+            List<FileComponent> components = info.components.getFileComponents();
+            Element manifest = new Element("manifest", uc3Name);
+            manifest.setAttribute("count", "" + info.count);
+            manifest.setAttribute("size", "" + info.size);
+            manifest.setAttribute("created", "" + info.created.getIsoDate());
+            for (FileComponent component : components) {
+                Element componentElem = getComponentElem(component);
+                manifest.addContent(componentElem);
+            }
+            return manifest;
+            
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    
+    public static Element getComponentElem(FileComponent component)
+        throws TException
+    {
+        try {
+            Element fileElem = new Element("file", uc3Name);
+            String id = component.getIdentifier();
+            if (id.substring(0,1).equals("/") || id.substring(0,1).equals("\\")) {
+                id = id.substring(1);
+            }
+            if (StringUtil.isAllBlank(id)) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "getComponentElem - identifier is empty");
+            }
+            fileElem.setAttribute("id", id);
+            
+            MessageDigest digest = component.getMessageDigest();
+            if (digest == null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "getComponentElem - digest missing");
+            }
+            String digestType = digest.getJavaAlgorithm();
+            String digestValue = digest.getValue();
+            Element digestTypeE = new Element("digestType", uc3Name);
+            digestTypeE.addContent(digestType);
+            fileElem.addContent(digestTypeE);
+            Element digestValueE = new Element("digest", uc3Name);
+            digestValueE.addContent(digestValue);
+            fileElem.addContent(digestValueE);
+            
+            long size = component.getSize();
+            if (size < 0 ) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "getComponentElem - size missing");
+            }
+            Element sizeE = new Element("size", uc3Name);
+            sizeE.addContent("" + size);
+            fileElem.addContent(sizeE);
+            
+            DateState creation = component.getCreated();
+            if (creation == null ) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "getComponentElem - creation date missing");
+            }
+            Element creationE = new Element("creationDate", uc3Name);
+            creationE.addContent(creation.getIsoDate());
+            fileElem.addContent(creationE);
+            
+            String mimeType = component.getMimeType();
+            if (StringUtil.isNotEmpty(mimeType)) {
+                Element mimeTypeE = new Element("mimeType", uc3Name);
+                mimeTypeE.addContent(mimeType);
+                fileElem.addContent(mimeTypeE);
+            }
+            
+            String key = component.getLocalID();
+            if (StringUtil.isAllBlank(key)) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "getComponentElem - key missing");
+            }
+            Element keyE = new Element("key", uc3Name);
+            keyE.addContent(key);
+            fileElem.addContent(keyE);
+            
+            return fileElem;
+            
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    
+    public static String elementToString(Element elem)
+        throws TException
+    {
+        try {
+            Format format = Format.getCompactFormat().setIndent("  ");
+            ByteArrayOutputStream bos = new ByteArrayOutputStream();
+            XMLOutputter outputter = new XMLOutputter(format);
+            outputter.output(elem, bos);
+            return bos.toString("utf-8");
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    
+    public static void buildOut(VersionMap versionMap,OutputStream out)
+        throws TException
+    {
+        try {
+            Document outDoc = new Document();
+            Namespace name = Namespace.getNamespace("http://uc3.cdlib.org/ontology/mrt/manifest");
+            Element outEle = new Element("objectInfo", uc3Name);
+            outDoc.setRootElement(outEle);
+            Element objectE = getObjectElem(versionMap);
+            outEle.addContent(objectE);
+            Element versionsE = getVersionsElem(versionMap);
+            outEle.addContent(versionsE);
+            String outS = elementToString(outEle);
+            if (DEBUG) System.out.println("\nOUTPUT BuildOut:\n" + outS);
+            byte[] outB = outS.getBytes("utf-8");
+            out.write(outB);
+            
+            
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+            
+        } finally {
+            try {
+                out.close();
+            } catch (Exception ex) { }
+        }
+    }
+    
+    protected static Element getObjectElem(VersionMap versionMap)
+        throws TException
+    {
+        try {
+            Element objectO = new Element("object", uc3Name);
+            objectO.setAttribute("id", versionMap.getObjectID().getValue());
+                
+            Element currentO = new Element("current", uc3Name);
+            currentO.addContent("" + versionMap.getCurrent());
+            
+            Element fileCountO = new Element("fileCount", uc3Name);
+            fileCountO.addContent("" + versionMap.getTotalCnt());
+            
+            Element totalSizeO = new Element("totalSize", uc3Name);
+            totalSizeO.addContent("" + versionMap.getTotalSize());
+            
+            Element actualCountO = new Element("actualCount", uc3Name);
+            actualCountO.addContent("" + versionMap.getActualCnt());
+            
+            Element actualSizeO = new Element("actualSize", uc3Name);
+            actualSizeO.addContent("" + versionMap.getActualSize());
+            
+            Element versionCountO = new Element("versionCount", uc3Name);
+            versionCountO.addContent("" + versionMap.getVersionCount());
+            
+            Element lastAddVersionO = null;
+            if (versionMap.getLastAddVersion() != null) {
+                lastAddVersionO = new Element("lastAddVersion", uc3Name);
+                lastAddVersionO.addContent(versionMap.getLastAddVersion().getIsoDate());
+            }
+            
+            Element lastDeleteVersionO = null;
+            if (versionMap.getLastDeleteVersion() != null) {
+                System.out.println(MESSAGE + "lastDeleteVersion not null");
+                lastDeleteVersionO = new Element("lastDeleteVersion", uc3Name);
+                lastDeleteVersionO.addContent(versionMap.getLastDeleteVersion().getIsoDate());
+            }
+            
+            objectO.addContent(currentO);
+            objectO.addContent(fileCountO);
+            objectO.addContent(totalSizeO);
+            objectO.addContent(actualCountO);
+            objectO.addContent(actualSizeO);
+            objectO.addContent(versionCountO);
+            if (lastAddVersionO != null) objectO.addContent(lastAddVersionO);
+            if (lastDeleteVersionO != null) objectO.addContent(lastDeleteVersionO);
+            return objectO;
+            
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    
+    protected static Element getVersionsElem(VersionMap versionMap)
+        throws TException
+    {
+        try {
+            Element versionsE = new Element("versions", uc3Name);
+            for (int iv = 1; true; iv++) {
+                ManInfo info = versionMap.getVersionInfo(iv);
+                if (info == null) break;
+                Element versionE = new Element("version", uc3Name);
+                versionE.setAttribute("id", "" + iv);
+                versionsE.addContent(versionE);
+                //Element manifestE = getManifestElem(info);
+                Element manifestE = buildManifestEntry(info);
+                versionE.setContent(manifestE);
+            }
+            return versionsE;
+            
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    
+    protected static void addComponent(Element fileE, String name, String value)
+        throws TException
+    {
+        try {
+            Element nameE = new Element(name, uc3Name);
+            fileE.setContent(nameE);
+            nameE.addContent(value);
+            
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    
+       
+    public ManInfo getManInfo(int inx)
+    {
+        return versionMap.getManInfo(inx);
+    }
+       
+    public int getVersionCnt()
+    {
+        return versionMap.size();
+    }
+    
+    public static void main(String[] args) throws TException {
+        /*
+         * Important: Be sure to fill in your AWS access credentials in the
+         *            AwsCredentials.properties file before you try to run this
+         *            sample.
+         * http://aws.amazon.com/security-credentials
+         */
+        ManInfo test = new ManInfo();
+        InputStream propStream =  test.getClass().getClassLoader().
+                getResourceAsStream("resources/TestProperties.properties");
+        if (propStream == null) {
+            System.out.println("Unable to find resource");
+            return;
+        }
+        Properties xmlProp = new Properties();
+        try {
+            xmlProp.load(propStream);
+            String xmlFileS = xmlProp.getProperty("xmlfile");
+            System.out.println("Test file:" + xmlFileS);
+            LoggerInf logger = new TFileLogger(NAME, 50, 50);
+            File file = new File(xmlFileS);
+            InputStream inStream = new FileInputStream(file);
+            Identifier manID = new Identifier("ark:/13030/ghijk");
+            VersionMap map = ManifestXML.getVersionMap(manID, logger, inStream);
+            System.out.println("cnt=" + map.getVersionCount());
+            System.out.println(map.dump("routine dump"));
+            buildOut(map, null);
+            
+            testOut(map, 1, 3);
+            testOut(map, 2, 4);
+            
+            System.out.println("**************************Test Delete**********************");
+            map.deleteCurrent();
+            buildOut(map, null);
+            
+            try {
+                System.out.println("Test bad current");
+                testOut(map,1,5);
+            } catch (Exception ex) {
+                System.out.println("TEST Exception:" + ex);
+            }
+            
+            
+            Identifier manNewID = new Identifier("ark:/13030/abcde");
+            VersionMap newMap = getVersionMap(manNewID, logger, null);
+            ManInfo addInfo = map.getVersionInfo(1);
+            newMap.addVersion(addInfo.components.getFileComponents());
+            buildOut(newMap, null);
+/*          
+            for (int i=0; true; i++) {
+                ManInfo info = manXML.getManInfo(i);
+                if (info == null) break;
+                System.out.println(info.dump("dump[" + i + "]"));
+            }
+            for (int i=1; i<=manXML.getCurrent(); i++) {
+                ArrayList<FileComponent> components = manXML.getVersionContent(i);
+                dumpComponents(i, components);
+            }
+            
+            //***********output
+            manXML.testOut(1);
+            manXML.testOut(2);
+            
+            System.out.println("****OUTPUT original****");
+            manXML.buildOut(null);
+            
+            ArrayList<FileComponent> testContent = manXML.getVersionContent(1);
+            manXML.addVersion(3, new DateState(), testContent);
+            System.out.println("****OUTPUT add 3****");
+            System.out.println(manXML.dumpVersionMap("versionMap"));
+            manXML.buildOut(null);
+            
+            ArrayList<FileComponent> testContent2 = manXML.getVersionContent(2);
+            manXML.addVersion(4, new DateState(), testContent2);
+            System.out.println("****OUTPUT add 4****");
+            System.out.println(manXML.dumpVersionMap("versionMap"));
+            manXML.buildOut(null);
+
+            manXML.deleteCurrent();
+            System.out.println("****OUTPUT delete 4****");
+            System.out.println(manXML.dumpVersionMap("versionMap"));
+            manXML.buildOut(null);
+            
+            Identifier objectIDLocal = new Identifier("ark:/13030/ghijk");
+            ManifestStaticXML manNew = new ManifestStaticXML(objectIDLocal, logger);
+            ArrayList<FileComponent> testContentNew = manXML.getVersionContent(1);
+            manNew.addVersion(1, new DateState(), testContentNew);
+            System.out.println("****OUTPUT NEW****");
+            System.out.println(manXML.dumpVersionMap("versionMap"));
+            manNew.buildOut(null);
+            
+            try {
+                manNew.deleteCurrent();
+            } catch (Exception ex) {
+                System.out.println("Delete a single version item:" + ex.toString());
+            }
+            */
+        } catch (TException tex) {
+            throw tex;
+            
+        } catch (Exception ex) {
+            ex.printStackTrace();
+            System.out.println("NAME=" + ex.getClass().getName());
+            System.out.println("Exception:" + ex);
+            System.out.println("Caught an AmazonServiceException, which means your request made it "
+                    + "to Amazon S3, but was rejected with an error response for some reason.");
+            
+        }
+        
+    }
+    
+    public static void testOut(VersionMap map, int versionID, int versionOutID)
+        throws TException
+    {
+        System.out.println("*************testOut:"
+                + " - versionID=" + versionID
+                + " - versionOutID=" + versionOutID
+                );
+        map.addTest(versionID, versionOutID);
+        buildOut(map, null);
+    }
+    
+    public static void dumpComponents(int versionID, ArrayList<FileComponent> components)
+    {
+        System.out.println("***Dump version:" + versionID);
+        for (FileComponent component : components) {
+            System.out.println(component.dump(component.getIdentifier()));
+        }
+        System.out.println("****************\n");
+    }
+}

core/src/main/java/org/cdlib/mrt/cloud/VersionMap.java

+/*
+Copyright (c) 2005-2012, Regents of the University of California
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+ *
+- Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+- Neither the name of the University of California nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************/
+package org.cdlib.mrt.cloud;
+
+import java.io.File;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.ArrayList;
+
+import org.cdlib.mrt.cloud.ManInfo;
+
+import org.cdlib.mrt.core.ComponentContent;
+import org.cdlib.mrt.core.DateState;
+import org.cdlib.mrt.core.FileComponent;
+import org.cdlib.mrt.core.Identifier;
+import org.cdlib.mrt.core.MessageDigest;
+import org.cdlib.mrt.utility.FileUtil;
+import org.cdlib.mrt.utility.FixityTests;
+import org.cdlib.mrt.utility.LinkedHashList;
+import org.cdlib.mrt.utility.LoggerInf;
+import org.cdlib.mrt.utility.StringUtil;
+import org.cdlib.mrt.utility.TException;
+
+/**
+ * This object imports the formatTypes.xml and builds a local table of supported format types.
+ * Note, that the ObjectFormat is being deprecated and replaced by a single format id (fmtid).
+ * This change is happening because formatName is strictly a description and has no functional
+ * use. The scienceMetadata flag is being dropped because the ORE Resource Map is more flexible
+ * and allows for a broader set of data type.
+ * 
+ * @author dloy
+ */
+public class VersionMap
+{
+    private static final String NAME = "VersionMap";
+    private static final String MESSAGE = NAME + ": ";
+
+    private static final String NL = System.getProperty("line.separator");
+    private static final boolean DEBUG = false;
+    
+    protected Identifier objectID = null;
+    protected int current = -1;
+    protected long actualSize = 0;
+    protected long totalSize = 0;
+    protected int totalCnt = 0;
+    protected int actualCnt = 0;
+    protected int originalFileCount = 0;
+    protected long originalTotalSize = 0;
+    protected int originalActualCount = 0;
+    protected long originalActualSize = 0;
+    protected DateState lastAddVersion = null;
+    protected DateState lastDeleteObject = null;
+    protected DateState lastDeleteVersion = null;
+    protected DateState originalLastAddVersion = null;
+    protected DateState originalLastDeleteVersion = null;
+    protected int originalVersionCnt = 0;
+    protected LoggerInf logger = null;
+    
+    protected ArrayList<ManInfo> manList = new ArrayList<ManInfo>();
+    protected HashMap<String, VersionFileComponent> keyHash = new HashMap<String, VersionFileComponent>();
+    protected LinkedHashList<String, VersionFileComponent> fileHashList = new LinkedHashList<String, VersionFileComponent>();
+
+    public VersionMap(Identifier objectID, LoggerInf logger)
+        throws TException
+    {
+        this.objectID = objectID;
+        this.logger = logger;
+        
+    }
+
+    public VersionMap(VersionMap inMap)
+        throws TException
+    {
+        this.objectID = inMap.objectID;
+        for (ManInfo man : manList) {
+            inMap.add(man);
+        }
+    }
+
+    public int size()
+    {
+        return manList.size();
+    }
+    public int getVersionCount()
+    {
+        return manList.size();
+    }
+       
+    public ManInfo getManInfo(int inx)
+    {
+        if (inx >= manList.size()) return null;
+        if (inx < 0) return null;
+        return manList.get(inx);
+    }
+       
+    public ManInfo getVersionInfo(int versionID)
+    {
+        if (versionID == 0) versionID = current;
+        if (versionID < 0) return null;
+        for (ManInfo manInfo : manList) {
+            if (manInfo.versionID == versionID) return manInfo;
+            
+        }
+        return null;
+    }
+       
+    public List<FileComponent> getVersionComponents(int versionID)
+    {
+        ManInfo manInfo = getVersionInfo(versionID);
+        if (manInfo == null) return null;
+        if (versionID < 0) return null;
+        return manInfo.components.getFileComponents();
+    }
+       
+    public ComponentContent getVersionContent(int versionID)
+    {
+        ManInfo manInfo = getVersionInfo(versionID);
+        if (manInfo == null) return null;
+        if (versionID < 0) return null;
+        return manInfo.components;
+    }
+       
+    public int getVersionListInx(int versionID)
+    {
+        if (versionID < 0) return 0;
+        for (int inx = 0; inx < manList.size(); inx++) {
+            ManInfo manInfo = manList.get(inx);
+            if (DEBUG) System.out.println("*****getVersionListInx - versionID=" + versionID + " - manInfo.versionID=" + manInfo.versionID);
+            if (manInfo.versionID == versionID) return inx;
+        }
+        return -1;
+    }
+       
+    public int getMaxVersion()
+    {
+        int maxVersion = -1;
+        for (int inx = 0; inx < manList.size(); inx++) {
+            ManInfo manInfo = manList.get(inx);
+            if (manInfo.versionID > maxVersion) maxVersion = manInfo.versionID;
+        }
+        return maxVersion;
+    }
+       
+    public void validateVersion()
+        throws TException
+    {
+        if (DEBUG) System.out.println("**************************VALIDATEVERSION***********************");
+        if (manList.size() == 0) {
+            if (current > 0) {
+                throw new TException.INVALID_ARCHITECTURE(MESSAGE + "validateVersion - current set but no version content - current=" + current);
+            } else return;
+        }
+        
+        int maxVersion = getMaxVersion();
+        int manListSize = manList.size();
+        if (DEBUG) System.out.println("validateVersion:"
+                + " - maxVersion=" + maxVersion
+                + " - manListSize=" + manListSize
+                + " - current=" + current
+                );
+        for (int vix = 1; vix <= manListSize; vix++) {
+            ManInfo vInfo = getVersionInfo(vix);
+            if (vInfo == null) {
+                throw new TException.INVALID_ARCHITECTURE(MESSAGE + "validateVersion - hole in versions:" + vix);
+            }
+        }
+        if (maxVersion != manListSize) {
+            throw new TException.INVALID_ARCHITECTURE(MESSAGE + "validateVersion - maxVersion != version list size "
+                    + " - maxVersion=" + maxVersion
+                    + " - manListSize=" + manListSize
+                    );
+        }
+        if (current != manListSize) {
+            throw new TException.INVALID_ARCHITECTURE(MESSAGE + "validateVersion - current != version list size "
+                    + " - current=" + current
+                    + " - manListSize=" + manListSize
+                    );
+        }
+    }
+    
+    public void deleteFromManList(int delInx)
+    {
+        manList.remove(delInx);
+        lastDeleteVersion = new DateState();
+        rebuildHash();
+    }
+    
+    public void rebuildHash()
+    {
+        keyHash.clear();
+        fileHashList.clear();
+        actualSize = 0;
+        totalSize=0;
+        totalCnt = 0;
+        actualCnt = 0;
+        for (ManInfo manInfo : manList) {
+            addHash(manInfo);
+        }
+    }
+
+    public void addVersion(
+            List<FileComponent> components)
+        throws TException
+    {
+        try {
+            if (DEBUG) System.out.println("*******************ADDVERSION*********************");
+            int versionID = getNextVersion();
+            DateState created = new DateState();
+            ManInfo testInfo = getVersionInfo(versionID);
+            if (testInfo != null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "addVersion - versionID already exists:" + versionID);
+            }
+            validateVersion();
+            ManInfo addMan = buildManInfo(versionID, created, components);
+            current = versionID;
+            add(addMan);
+            validateVersion();
+            lastAddVersion = created;
+            
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+        
+    }
+
+    public void updateVersion(
+            List<FileComponent> components,
+            String [] deleteList)
+        throws TException
+    {
+        try {
+            if (DEBUG) System.out.println("*******************updateVersion*********************");
+            int versionID = getNextVersion();
+            DateState created = new DateState();
+            ManInfo testInfo = getVersionInfo(versionID);
+            if (testInfo != null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "addVersion - versionID already exists:" + versionID);
+            }
+            ManInfo updateInfo = buildManInfo(versionID, created, components);
+            ManInfo addInfo = getMergeInfo(updateInfo, deleteList);
+            current = versionID;
+            addInfo.versionID = versionID;
+            add(addInfo);
+            validateVersion();
+            lastAddVersion = created;
+            
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+        
+    }
+    
+    
+    public void deleteCurrent()
+        throws TException
+    {
+        try {
+            if (DEBUG) System.out.println("****************************deleteCurrent***********************");
+            if (false && current == 1) {
+                throw new TException.REQUEST_INVALID(MESSAGE + "deleteLocalVersion - only one version - use cloud delete");
+            }
+            int delInx = getVersionListInx(current);
+            if (delInx < 0) { //!!!!
+                throw new TException.INVALID_ARCHITECTURE(MESSAGE + "deleteLocalVersion - current not found:" + current);
+            }
+            
+            deleteFromManList(delInx);
+            current = manList.size();
+            lastDeleteVersion = new DateState();
+            validateVersion();
+            
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+        
+    }
+
+    public static ManInfo buildManInfo(
+            int versionID, 
+            DateState created, 
+            List<FileComponent> components)
+        throws TException
+    {
+        try {
+            if (versionID <= 0) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "buildManInfo - versionID not valid:" + versionID);
+            }
+            if (created == null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "buildManInfo - created date not supplied");
+            }
+            ManInfo info = new ManInfo();
+            info.versionID = versionID;
+            info.created = created;
+            if (components == null) return info;
+            info.setComponents(versionID, components);
+            for (FileComponent component : components) {
+                info.size += component.getSize();
+                info.count++;
+            }
+            return info;
+            
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+        
+    }
+    
+    public void add(ManInfo info)
+        throws TException
+    {
+        int versionID = info.versionID;
+        validateManInfo(info);
+        manList.add(info);
+        //lastAddVersion = new DateState();
+        addHash(info);
+    }
+    
+    public ManInfo getMergeInfoOriginal(
+            ManInfo update,
+            String [] deleteList)
+        throws TException
+    {
+        try {
+            int currentVersion = current;
+            if (currentVersion == 0) {
+                if ((deleteList != null) && (deleteList.length > 0)) {
+                    throw new TException.REQUEST_INVALID(MESSAGE + "Delete list provided and no version exists");
+                }
+            }
+            List<FileComponent> currentComponents = getVersionComponents(currentVersion);
+            HashMap<String, FileComponent> hashMerge = new HashMap<String, FileComponent>();
+            
+            // build hash list
+            if (currentComponents != null) {
+                for (FileComponent currentComponent : currentComponents) {
+                    hashMerge.put(currentComponent.getIdentifier(), currentComponent);
+                }
+            }
+            
+            // delete items from hash using delete list
+            if ((deleteList != null) && (deleteList.length > 0)) {
+                for (String fileID : deleteList) {
+                    FileComponent deleteComponent = hashMerge.get(fileID);
+                    if (deleteComponent == null) {
+                        throw new TException.REQUEST_INVALID(MESSAGE + "Delete list component not found:" + fileID);
+                    }
+                    hashMerge.remove(fileID);
+                }
+            }
+            
+            // add updates
+            if (update.components != null) {
+                List<FileComponent> updateComponents = update.components.getFileComponents();
+                for (FileComponent updateComponent : updateComponents) {
+                    hashMerge.put(updateComponent.getIdentifier(), updateComponent);
+                }
+            }
+            
+            //convert from hash to ManInfo
+            ArrayList<FileComponent> addComponents = new ArrayList<FileComponent>();
+            Set<String> keys = hashMerge.keySet();
+            for (String key : keys) {
+                FileComponent addComponent = hashMerge.get(key);
+                addComponents.add(addComponent);
+                if (DEBUG) System.out.println(addComponent.dump("getMergeInfo - " + key));
+            }
+            ManInfo addInfo = new ManInfo();
+            addInfo.components = new ComponentContent(addComponents);
+            addInfo.size = getManifestSize(addInfo);
+            addInfo.created = new DateState();
+            addInfo.count = addComponents.size();
+            
+            return addInfo;
+            
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    public ManInfo getMergeInfo(
+            ManInfo update,
+            String [] deleteList)
+        throws TException
+    {
+        try {
+            int currentVersion = current;
+            if (currentVersion == 0) {
+                if ((deleteList != null) && (deleteList.length > 0)) {
+                    throw new TException.REQUEST_INVALID(MESSAGE + "Delete list provided and no version exists");
+                }
+            }
+            List<FileComponent> currentComponents = getVersionComponents(currentVersion);
+            List<FileComponent> updateComponents = null;
+            
+            // add updates
+            if (update.components != null) {
+                updateComponents = update.components.getFileComponents();
+            }
+            List<FileComponent> addComponents = getMergeComponents(
+                    updateComponents,
+                    currentComponents,
+                    deleteList);
+            ManInfo addInfo = new ManInfo();
+            addInfo.components = new ComponentContent(addComponents);
+            addInfo.size = getManifestSize(addInfo);
+            addInfo.created = new DateState();
+            addInfo.count = addComponents.size();
+            
+            return addInfo;
+            
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    
+    public static ArrayList<FileComponent> getMergeComponents(
+            List<FileComponent> updateComponents,
+            List<FileComponent> currentComponents,
+            String [] deleteList)
+        throws TException
+    {
+        try {
+            HashMap<String, FileComponent> hashMerge = new HashMap<String, FileComponent>();
+            
+            // build hash list
+            if (currentComponents != null) {
+                for (FileComponent currentComponent : currentComponents) {
+                    if (DEBUG) System.out.println("getMergeComponents - current fileID:" + currentComponent.getIdentifier());
+                    hashMerge.put(currentComponent.getIdentifier(), currentComponent);
+                }
+            }
+            
+            // delete items from hash using delete list
+            if ((deleteList != null) && (deleteList.length > 0)) {
+                for (String fileID : deleteList) {
+                    if (DEBUG) System.out.println("getMergeComponents - delete fileID:" + fileID);
+                    FileComponent deleteComponent = hashMerge.get(fileID);
+                    if (deleteComponent == null) {
+                        throw new TException.REQUEST_INVALID(MESSAGE + "Delete list component not found:" + fileID);
+                    }
+                    hashMerge.remove(fileID);
+                }
+            }
+            
+            // add updates
+            if (updateComponents != null) {
+                for (FileComponent updateComponent : updateComponents) {
+                    hashMerge.put(updateComponent.getIdentifier(), updateComponent);
+                }
+            }
+            
+            //convert from hash to ManInfo
+            ArrayList<FileComponent> addComponents = new ArrayList<FileComponent>();
+            Set<String> keys = hashMerge.keySet();
+            for (String key : keys) {
+                FileComponent addComponent = hashMerge.get(key);
+                addComponents.add(addComponent);
+                if (DEBUG) System.out.println(addComponent.dump("getMergeInfo - " + key));
+            }
+            return addComponents;
+            
+        } catch (TException tex) {
+            throw tex;
+
+        } catch (Exception ex) {
+            System.out.println(MESSAGE + "Exception:" + ex);
+            ex.printStackTrace();
+            throw new TException(ex);
+        }
+    }
+    
+    public void addHash(ManInfo info)
+    {
+        int versionID = info.versionID;
+        for (FileComponent component : info.components.getFileComponents()) {
+            String key = component.getLocalID();
+            long fileSize = component.getSize();
+            totalSize += fileSize;
+            totalCnt++;
+            VersionFileComponent vfc = keyHash.get(key);
+            if (vfc != null) continue;
+            actualCnt++;
+            actualSize += fileSize;
+            vfc = new VersionFileComponent();
+            vfc.versionID = versionID;
+            vfc.key = component.getLocalID();
+            vfc.component = component;
+            keyHash.put(key, vfc);
+            fileHashList.put(component.getIdentifier(), vfc);
+            
+        }
+    }
+    
+    protected void validateManInfo(ManInfo info)
+        throws TException
+    {
+        int manCnt = info.components.size();
+        long manSize = getManifestSize(info);
+        if (info.size <= 0) info.size = manSize;
+        else if (info.size != manSize) {
+            throw new TException.INVALID_OR_MISSING_PARM("validateManInfo"
+                    + " - mismatch version=" + info.versionID
+                    + " - info.size=" + info.size
+                    + " - manSize=" + manSize
+                    );
+        }
+    }
+    
+    /**
+     * 
+     * @return count of unique fileComponents
+     */
+    public int getUniqueCnt()
+    {
+        return keyHash.size();
+    }
+    
+    public long getUniqueSize()
+    {
+        long size = 0;
+        for (String key : keyHash.keySet()) {
+            VersionFileComponent vfc = keyHash.get(key);
+            size += vfc.component.getSize();
+        }
+        return size;
+    }
+    
+    public long getManifestSize(ManInfo info)
+    {
+        long size = 0;
+        List<FileComponent> components = info.components.getFileComponents();
+        for (FileComponent component : components) {
+            size += component.getSize();
+        }
+        return size;
+    }
+    
+    
+    
+    public void setCloudComponent(FileComponent manifestComponent, boolean doFill)
+        throws TException
+    {
+        try {
+            List<VersionFileComponent> matchComponents = fileHashList.get(manifestComponent.getIdentifier());
+            if (matchComponents == null) {
+                if (doFill) fillComponent(manifestComponent);
+                return;
+            }
+            for (VersionFileComponent matchVFC : matchComponents) {
+                FileComponent matchComponent = matchVFC.component;
+                if (isMatch(matchComponent, manifestComponent)) {
+                    manifestComponent.setLocalID(matchVFC.key);
+                    return;
+                }
+            }
+            
+            // keep key empty
+            if (doFill) fillComponent(manifestComponent);
+            return;
+            
+        } catch (TException tex) {
+            throw tex;
+            
+        } catch (Exception ex) {
+            String msg = MESSAGE + "buildCloudComponent - Exception:" + ex;
+            logger.logError(msg, 2);
+            logger.logError(StringUtil.stackTrace(ex), 10);
+            throw new TException.GENERAL_EXCEPTION(msg);
+        }
+        
+    }    
+    
+    public void fillComponent(FileComponent manifestComponent)
+        throws TException
+    {
+        try {
+            if (manifestComponent.getComponentFile() != null) return;
+            URL url = manifestComponent.getURL();
+            if (url == null) {
+                throw new TException.INVALID_OR_MISSING_PARM(MESSAGE + "fillComponent - component URL missing");
+            }
+            File tmpFile = FileUtil.getTempFile("tmp", ".txt");
+            FileUtil.url2File(logger, url, tmpFile);
+            manifestComponent.setComponentFile(tmpFile);
+            
+            
+        } catch (TException tex) {
+            throw tex;
+            
+        } catch (Exception ex) {
+            String msg = MESSAGE + "buildCloudComponent - Exception:" + ex;
+            logger.logError(msg, 2);
+            logger.logError(StringUtil.stackTrace(ex), 10);
+            throw new TException.GENERAL_EXCEPTION(msg);
+        }
+        
+    }
+    
+    public boolean isMatch(FileComponent matchComponent, FileComponent manifestComponent)
+        throws TException
+    {
+        try {
+            
+            MessageDigest manifestDigest = manifestComponent.getMessageDigest();
+            String manifestAlgorithm = manifestDigest.getJavaAlgorithm();
+            String manifestValue = manifestDigest.getValue();
+            long manifestSize = manifestComponent.getSize();
+            
+            
+            MessageDigest matchDigest = matchComponent.getMessageDigest();
+            String matchAlgorithm = matchDigest.getJavaAlgorithm();
+            String matchValue = matchDigest.getValue();
+            long matchSize = matchComponent.getSize();
+            
+            if (manifestSize != matchSize) return false;
+            
+            if (manifestAlgorithm.equals(matchAlgorithm)) {
+                if (manifestValue.equals(matchValue)) return true;
+                else return false;
+            }
+            fillComponent(manifestComponent);
+            FixityTests manifestTest = new FixityTests(manifestComponent.getComponentFile(), matchAlgorithm, logger);
+            FixityTests.FixityResult fixityTest = manifestTest.validateSizeChecksum(matchValue, matchValue, matchSize);
+            if (fixityTest.checksumMatch) return true;
+            else return false;
+            
+            
+        } catch (TException tex) {
+            throw tex;
+            
+        } catch (Exception ex) {
+            String msg = MESSAGE + "buildCloudComponent - Exception:" + ex;
+            logger.logError(msg, 2);
+            logger.logError(StringUtil.stackTrace(ex), 10);
+            throw new TException.GENERAL_EXCEPTION(msg);
+        }
+        
+    }
+
+    public int getActualCnt() {
+        return actualCnt;
+    }
+
+    public long getActualSize() {
+        return actualSize;
+    }
+
+    public int getTotalCnt() {
+        return totalCnt;
+    }
+
+    public long getTotalSize() {
+        return totalSize;
+    }
+    
+
+    public Identifier getObjectID() {
+        return objectID;
+    }
+
+    public DateState getLastAddVersion() {
+        return lastAddVersion;
+    }
+
+    public DateState getLastDeleteVersion() {
+        return lastDeleteVersion;
+    }
+
+    public DateState getOriginalLastAddVersion() {
+        return originalLastAddVersion;
+    }
+
+    public void setActualCnt(int actualCnt) {
+        this.actualCnt = actualCnt;
+    }
+
+    public void setActualSize(long actualSize) {
+        this.actualSize = actualSize;
+    }
+
+    public void setLastAddVersion(DateState lastAddVersion) {
+        this.lastAddVersion = lastAddVersion;
+    }
+
+    public void setLastDeleteVersion(DateState lastDeleteVersion) {
+        this.lastDeleteVersion = lastDeleteVersion;
+    }
+
+    public void setObjectID(Identifier objectID) {
+        this.objectID = objectID;
+    }
+
+    public void setTotalCnt(int totalCnt) {
+        this.totalCnt = totalCnt;
+    }
+
+    public void setTotalSize(long totalSize) {
+        this.totalSize = totalSize;
+    }
+
+    public void setOriginalLastAddVersion(DateState originalLastAddVersion) {
+        this.originalLastAddVersion = originalLastAddVersion;
+    }
+
+    public DateState getOriginalLastDeleteVersion() {
+        return originalLastDeleteVersion;
+    }
+
+    public void setOriginalLastDeleteVersion(DateState originalLastDeleteVersion) {
+        this.originalLastDeleteVersion = originalLastDeleteVersion;
+    }
+
+    public int getOriginalActualCount() {
+        return originalActualCount;
+    }
+
+    public void setOriginalActualCount(int originalActualCount) {
+        this.originalActualCount = originalActualCount;
+    }
+
+    public long getOriginalActualSize() {
+        return originalActualSize;
+    }
+
+    public void setOriginalActualSize(long originalActualSize) {
+        this.originalActualSize = originalActualSize;
+    }
+
+    public int getOriginalFileCount() {
+        return originalFileCount;
+    }
+
+    public void setOriginalFileCount(int originalFileCount) {
+        this.originalFileCount = originalFileCount;
+    }
+
+    public long getOriginalTotalSize() {
+        return originalTotalSize;
+    }
+
+    public void setOriginalTotalSize(long originalTotalSize) {
+        this.originalTotalSize = originalTotalSize;
+    }
+
+    public int getNextVersion() {
+        if (current <= 0) return 1;
+        return current + 1;
+    }
+
+    public int getCurrent() {
+        return current;
+    }
+
+    public void setCurrent(int current) {
+        this.current = current;
+    }
+
+    public int getOriginalVersionCnt() {
+        return originalVersionCnt;
+    }
+
+    public void setOriginalVersionCnt(int originalVersionCnt) {
+        this.originalVersionCnt = originalVersionCnt;
+    }
+
+    public DateState getLastDeleteObject() {
+        return lastDeleteObject;
+    }
+
+    public void setLastDeleteObject(DateState lastDeleteObject) {
+        this.lastDeleteObject = lastDeleteObject;
+    }
+    
+    public DeltaStats getDeltaStats()
+    {
+        DeltaStats delta = new DeltaStats();
+        delta.deltaActualCount = actualCnt - originalActualCount;
+        delta.deltaActualSize = actualSize - originalActualSize;
+        delta.deltaFileCount = totalCnt - originalFileCount;
+        delta.deltaTotalSize = totalSize - originalTotalSize;
+        return delta;
+    }
+    
+    
+    public String dump(String header) {
+        StringBuffer buf = new StringBuffer();
+        buf.append(MESSAGE + header + "\n"
+                + " - actualSize=" + actualSize + "\n"
+                + " - totalSize=" + totalSize + "\n"
+                + " - totalCnt=" + totalCnt + "\n"
+                + " - actualCnt=" + actualCnt + "\n"
+                + " - maxVersion=" + getMaxVersion() + "\n"
+                );
+        
+        if (lastAddVersion != null) {
+            buf.append(
+                " - lastAddVersion=" + lastAddVersion.getIsoDate() + "\n"
+                );
+        }
+        if (lastDeleteVersion != null) {
+            buf.append(
+                " - lastDeleteVersion=" + lastDeleteVersion.getIsoDate() + "\n"
+                );
+        }
+        buf.append(""
+                + " - current=" + current + "\n"
+                + " - originalFileCount=" + originalFileCount + "\n"
+                + " - originalTotalSize=" + originalTotalSize + "\n"
+                + " - originalActualCount=" + originalActualCount + "\n"
+                + " - originalActualSize=" + originalActualSize + "\n"
+                );
+        if (originalLastAddVersion != null) {
+            buf.append(
+                " - originalLastAddVersion=" + originalLastAddVersion.getIsoDate() + "\n"
+                );
+        }
+        if (originalLastDeleteVersion != null) {
+            buf.append(
+                " - originalLastDeleteVersion=" + originalLastDeleteVersion.getIsoDate() + "\n"
+                );
+        }
+        return buf.toString();
+    }
+    
+    public ManInfo addTest(int versionID, int outVersionID)
+        throws TException
+    {
+            ComponentContent testContent = getVersionContent(versionID);
+            ManInfo info = getVersionInfo(versionID);
+            info.components = testContent;
+            addVersion(testContent.getFileComponents());
+            return info;
+    }
+    
+    public FileComponent getFileComponent(int versionID, String fileID)
+        throws TException
+    {
+        ComponentContent content = getVersionContent(versionID);
+        if (content == null) {
+            throw new TException.REQUESTED_ITEM_NOT_FOUND("versionID not found:" + versionID);
+        }
+        return content.getFileComponent(fileID);
+        
+    }
+    
+    public VersionStats getVersionStats(int versionID)
+        throws TException
+    {
+        ComponentContent content = getVersionContent(versionID);
+        if (content == null) {
+            throw new TException.REQUESTED_ITEM_NOT_FOUND(MESSAGE + "getVersionStats: versionID not found:" + versionID);
+        }
+        VersionStats stats = new VersionStats();
+        List<FileComponent> components = content.getFileComponents();
+        for (FileComponent component : components) {
+            stats.totalSize += component.getSize();
+            stats.numFiles++;
+            VersionFileComponent vfc = keyHash.get(component.getLocalID());
+            if (vfc.versionID == versionID) {
+                stats.totalActualSize += component.getSize();
+                stats.numActualFiles++;
+            }
+        }
+        return stats;
+    }
+    
+    public LoggerInf getLogger() {