Commits

Sam Adams committed 8ca502c

Refactored deposit processing

Comments (0)

Files changed (11)

crystallography-common/src/main/java/net/chempound/crystal/rdf/CrystalPersonalities.java

 import com.hp.hpl.jena.enhanced.BuiltinPersonalities;
 import net.chempound.crystal.rdf.impl.CrystalEyeEntryImpl;
 import net.chempound.crystal.rdf.impl.CrystalStructureImpl;
+import net.chempound.rdf.ChempoundPersonalities;
 
 /**
  * @author Sam Adams
 public class CrystalPersonalities extends BuiltinPersonalities {
 
     static {
+        ChempoundPersonalities.init();
+
         model.add(CrystalStructure.class, CrystalStructureImpl.factory);
         model.add(CrystalEyeEntry.class, CrystalEyeEntryImpl.factory);
     }

crystallography-common/src/main/java/net/chempound/crystal/rdf/ont/CrystalEye.java

     /** <p>The namespace of the vocabulary as a string</p> */
     public static final String NS = "http://crystaleye.ch.cam.ac.uk/-/schema/";
 
+    public static final Resource CrystalEyeIndex = m_model.createResource(NS+"Index");
     public static final Resource CrystalEyeEntry = m_model.createResource(NS+"Entry");
     public static final Resource CrystalEyeIssue = m_model.createResource(NS+"Issue");
     public static final Resource CrystalEyeVolume = m_model.createResource(NS+"Volume");

crystallography-handler/src/main/resources/net/chempound/crystal/includes/style.css

         padding: 5px;
     }
 
+    .data h2 {
+        clear:left;
+    }
+
 
     dl.identifier dt {
         clear: left;

crystallography-importer/src/main/java/net/chempound/crystal/importer/CifStringParser.java

         for (int i = 0; i < s.length(); i++) {
 
             final char c = s.charAt(i);
-            if (c == '\\') {
-                i += readChar(s, i+1, html);
-            }
-            else if (c == '+') {
-                if ("+-".equals(s.substring(i, i+2))) {
-                    html.append(Latin1Supplement.Plus_Minus_Sign);
-                }
-                i += 1;
-            }
-            else if (c == '-') {
-                if ("---".equals(s.substring(i, i+3))) {
-                    // TODO single bond
-                    html.append(GeneralPunctuation.Em_Dash);
-                    i += 2;
-                }
-                else if ("-+".equals(s.substring(i, i+2))) {
-                    html.append(MathematicalOperators.Minus_Or_Plus_Sign);
+
+            switch (c) {
+                case '\\':
+                    i += readChar(s, i+1, html);
+                    break;
+
+                case '+':
+                    if ("+-".equals(s.substring(i, i+2))) {
+                        html.append(Latin1Supplement.Plus_Minus_Sign);
+                    }
                     i += 1;
-                }
-                else if ("--".equals(s.substring(i, i+2))) {
-                    html.append(GeneralPunctuation.En_Dash);
-                    i += 1;
-                } else {
+                    break;
+
+                case '-':
+                    if ("---".equals(s.substring(i, i+3))) {
+                        // TODO single bond
+                        html.append(GeneralPunctuation.Em_Dash);
+                        i += 2;
+                    }
+                    else if ("-+".equals(s.substring(i, i+2))) {
+                        html.append(MathematicalOperators.Minus_Or_Plus_Sign);
+                        i += 1;
+                    }
+                    else if ("--".equals(s.substring(i, i+2))) {
+                        html.append(GeneralPunctuation.En_Dash);
+                        i += 1;
+                    } else {
+                        html.append(c);
+                    }
+                    break;
+
+                case '<':
+                    if (
+                            "<i>".equals(s.substring(i, i+3))
+                                    || "<b>".equals(s.substring(i, i+3))) {
+                        html.append(s.substring(i, i+3));
+                        i += 2;
+                    } else if (
+                            "</i>".equals(s.substring(i, i+4))
+                                    || "</b>".equals(s.substring(i, i+4))) {
+                        html.append(s.substring(i, i+4));
+                        i += 3;
+                    } else {
+                        throw new IllegalArgumentException("Bad string: "+s);
+                    }
+                    break;
+
+                case '^':    // superscript
+                    if (sup) {
+                        html.append("</sup>");
+                    } else {
+                        html.append("<sub>");
+                    }
+                    sup = !sup;
+                    break;
+
+                case '~':   // subscript
+                    if (sub) {
+                        html.append("</sub>");
+                    } else {
+                        html.append("<sub>");
+                    }
+                    sub = !sub;
+                    break;
+
+                default:
                     html.append(c);
-                }
-            }
-            else if (c == '<') {
-                if (
-                        "<i>".equals(s.substring(i, i+3))
-                                || "<b>".equals(s.substring(i, i+3))) {
-                    html.append(s.substring(i, i+3));
-                    i += 2;
-                } else if (
-                        "</i>".equals(s.substring(i, i+4))
-                                || "</b>".equals(s.substring(i, i+4))) {
-                    html.append(s.substring(i, i+4));
-                    i += 3;
-                } else {
-                    throw new IllegalArgumentException("Bad string: "+s);
-                }
-            }
-            else if (c == '^') { // superscript
-                if (sup) {
-                    html.append("</sup>");
-                } else {
-                    html.append("<sub>");
-                }
-                sup = !sup;
-            }
-            else if (c == '~') { // subscript
-                if (sub) {
-                    html.append("</sub>");
-                } else {
-                    html.append("<sub>");
-                }
-                sub = !sub;
-            }
-            else {
-                html.append(c);
             }
         }
         return html.toString();
 
         }
 
-        throw new IllegalArgumentException("Unsupported: ["+c+"]");
-    }
-
-    public static void main(final String[] args) {
-        System.out.println(toHtml("\\q2\\q +- foo -+ bar -- x --- xx 54\\% 2.2\\%A !! \\?i"));
-        System.out.println(toHtml("Incorporation of vanadium(V) into the rutile-type phase of GeO~2~: the solid solution Ge~0.74~V~0.21~\\\\square~0.05~O~2~"));
-
+        throw new IllegalArgumentException("Unsupported character: ["+c+"]");
     }
 
 }

crystallography-importer/src/main/java/net/chempound/crystal/importer/CrystalCml2RdfConverter.java

 import com.hp.hpl.jena.vocabulary.RDF;
 import com.hp.hpl.jena.vocabulary.RDFS;
 import com.hp.hpl.jena.vocabulary.XSD;
-import nu.xom.*;
-import org.apache.log4j.Logger;
-import org.xmlcml.cml.base.CMLElement;
-import org.xmlcml.cml.base.CMLUtil;
-import org.xmlcml.cml.converters.cif.dict.UnitsDictionary;
-import org.xmlcml.cml.element.*;
 import net.chempound.chemistry.Cml2RdfConverter;
 import net.chempound.crystal.rdf.ont.ChemAxiom;
 import net.chempound.crystal.rdf.ont.CifDict;
 import net.chempound.rdf.cml.CmlRdf;
 import net.chempound.rdf.foaf.FoafUtil;
 import net.chempound.rdf.foaf.Person;
+import nu.xom.Elements;
+import nu.xom.Node;
+import nu.xom.Nodes;
+import nu.xom.XPathContext;
+import org.xmlcml.cml.base.CMLElement;
+import org.xmlcml.cml.base.CMLUtil;
+import org.xmlcml.cml.converters.cif.dict.UnitsDictionary;
+import org.xmlcml.cml.element.*;
 
+import javax.xml.namespace.QName;
 import java.net.URI;
 import java.util.*;
 
+import static net.chempound.chemistry.cml.CmlUtils.findModuleByConvention;
 import static org.xmlcml.cml.base.CMLConstants.CML_XPATH;
 
 /**
  */
 public class CrystalCml2RdfConverter extends Cml2RdfConverter {
 
-    private static final Logger LOG = Logger.getLogger(CrystalCml2RdfConverter.class);
+    private static final Map<String,String> DEPRECATED_MAP;
 
-    private static final Map<String,String> DEPRECATED_MAP;
+    private static final QName CRYSTALLOGRAPHY_EXPERIMENT = new QName("http://www.xml-cml.org/convention/", "crystallographyExperiment");
+    private static final QName CRYSTAL_STRUCTURE = new QName("http://www.xml-cml.org/convention/", "crystalStructure");
+
+    public static final String CELL_LENGTH_A = "iucr:cell_length_a";
+    public static final String CELL_LENGTH_B = "iucr:cell_length_b";
+    public static final String CELL_LENGTH_C = "iucr:cell_length_c";
+    public static final String CELL_ANGLE_ALPHA = "iucr:cell_angle_alpha";
+    public static final String CELL_ANGLE_BETA = "iucr:cell_angle_beta";
+    public static final String CELL_ANGLE_GAMMA = "iucr:cell_angle_gamma";
 
     static {
         final Map<String,String> map = new HashMap<String, String>();
         DEPRECATED_MAP = Collections.unmodifiableMap(map);
     }
 
+    private boolean normaliseUris = true;
+
 
     public Model createModel(final CMLCml cml, final URI baseUri) {
         final CMLModule module = findCrystallographyExperimentModule(cml);
     }
 
     private CMLModule findCrystallographyExperimentModule(final CMLElement cml) {
-        return findModule(cml, "http://www.xml-cml.org/convention/crystallographyExperiment");
+        return findModuleByConvention(cml, CRYSTALLOGRAPHY_EXPERIMENT);
     }
 
-    private CMLModule findModule(final CMLElement cml, final String uri) {
-        final Elements list = cml.getChildCMLElements("module");
-        for (int i = 0; i < list.size(); i++) {
-            final CMLModule module = (CMLModule) list.get(i);
-            final String convention = module.getConvention();
-            if (convention != null) {
-                final int ix = convention.indexOf(':');
-                final String prefix = convention.substring(0, ix);
-                final String name = convention.substring(ix+1);
-                final String ns = module.getNamespaceURIForPrefix(prefix);
-                if (ns != null && uri.equals(ns+name)) {
-                    return module;
-                }
-            }
-        }
-        return null;
-    }
+
 
     public Model createModel(final CMLModule experimentModule, final URI baseUri) {
 
             attachSymmetry(model, thisCryst, crystZ, symmetry);
         }
 
-        final Elements formulaElements = experimentModule.getChildCMLElements(CMLFormula.TAG);
-        for (int i = 0; i < formulaElements.size(); i++) {
-            final CMLFormula formula = (CMLFormula)formulaElements.get(i);
-            final String dictRef = formula.getDictRef();
-            if (dictRef.startsWith("iucr:")) {
-                final Property prop = model.createProperty(CifDict.NS, dictRef.substring(5));
-                thisCryst.addProperty(prop, model.createTypedLiteral(formula.getInline(), XSDDatatype.XSDstring));
+        recordTitleInRdf(thisCryst);
+        recordFormulaeInRdf(experimentModule, model, thisCryst);
+        recordContributorsInRdf(experimentModule, model, thisCryst);
+        createAuthorListInRdf(model, thisCryst);
+
+        return model;
+    }
+
+    private void createAuthorListInRdf(final Model model, final Resource thisCryst) {
+        final List<String> authors = findAuthors(thisCryst);
+        if (authors != null && !authors.isEmpty()) {
+            final Seq authorList = model.createSeq();
+            for (final String name : authors) {
+                final Person person = FoafUtil.createPerson(model);
+                person.setName(name);
+                authorList.add(person);
+
             }
+            thisCryst.addProperty(Bibo.authorList, authorList);
         }
+    }
 
+    private void recordTitleInRdf(final Resource thisCryst) {
+        final String title = findTitle(thisCryst);
+        if (title != null) {
+            thisCryst.addProperty(DCTerms.title, title.trim());
+        }
+    }
+
+    private void recordContributorsInRdf(final CMLModule experimentModule, final Model model, final Resource thisCryst) {
         final List<Node> arrays = CMLUtil.getQueryNodes(experimentModule, ".//cml:array[@dictRef]", CML_XPATH);
         for (final Node node : arrays) {
             final CMLArray array = (CMLArray) node;
                 }
             }
         }
+    }
 
-        // Add dc:title
-        final String title = findTitle(thisCryst);
-        if (title != null) {
-            thisCryst.addProperty(DCTerms.title, title.trim());
+    private void recordFormulaeInRdf(final CMLModule experimentModule, final Model model, final Resource thisCryst) {
+        final Elements formulaElements = experimentModule.getChildCMLElements(CMLFormula.TAG);
+        for (int i = 0; i < formulaElements.size(); i++) {
+            final CMLFormula formula = (CMLFormula)formulaElements.get(i);
+            final String dictRef = formula.getDictRef();
+            if (dictRef.startsWith("iucr:")) {
+                final Property prop = model.createProperty(CifDict.NS, dictRef.substring(5));
+                thisCryst.addProperty(prop, model.createTypedLiteral(formula.getInline(), XSDDatatype.XSDstring));
+            }
         }
-
-        final List<String> authors = findAuthors(thisCryst);
-        if (authors != null && !authors.isEmpty()) {
-            final Seq authorList = model.createSeq();
-            for (final String name : authors) {
-                final Person person = FoafUtil.createPerson(model);
-                person.setName(name);
-                authorList.add(person);
-
-            }
-            thisCryst.addProperty(Bibo.authorList, authorList);
-        }
-
-        return model;
     }
 
     private List<String> findAuthors(final Resource thisCryst) {
         return null;
     }
 
-    private Statement findStatement(final Resource resource, final Property... properties) {
+    private static Statement findStatement(final Resource resource, final Property... properties) {
         for (final Property property : properties) {
             final Statement s = resource.getProperty(property);
             if (s != null) {
     }
 
     private CMLModule findCrystalStructureModule(final CMLModule experimentModule) {
-        return findModule(experimentModule, "http://www.xml-cml.org/convention/crystalStructure");
+        return findModuleByConvention(experimentModule, CRYSTAL_STRUCTURE);
     }
 
     private void attachIdentifiers(final Resource resource, final Model model, final CMLMolecule molecule) {
     }
 
     private void attachCellParameters(final Model model, final Resource thisUnitCell, final CMLCrystal crystal) {
+        final Resource angstroms = model.createResource(UnitsDictionary.U_ANGSTROM);
+        final Resource degrees = model.createResource(UnitsDictionary.U_DEGREE);
+
         for (final CMLScalar cellScalar : crystal.getCellScalars()) {
 
             final Resource paramBNode = model.createResource(CmlRdf.Scalar);
 
             final String dictRef = cellScalar.getDictRef();
             final Property cellParamProp;
-            if ("iucr:cell_length_a".equals(dictRef)) {
+            if (CELL_LENGTH_A.equals(dictRef)) {
                 cellParamProp = Cryst.cellLengthA;
-                paramBNode.addProperty(CmlRdf.units, model.createResource(UnitsDictionary.U_ANGSTROM));
-            } else if ("iucr:cell_length_b".equals(dictRef)) {
+                paramBNode.addProperty(CmlRdf.units, angstroms);
+            } else if (CELL_LENGTH_B.equals(dictRef)) {
                 cellParamProp = Cryst.cellLengthB;
-                paramBNode.addProperty(CmlRdf.units, model.createResource(UnitsDictionary.U_ANGSTROM));
-            } else if ("iucr:cell_length_c".equals(dictRef)) {
+                paramBNode.addProperty(CmlRdf.units, angstroms);
+            } else if (CELL_LENGTH_C.equals(dictRef)) {
                 cellParamProp = Cryst.cellLengthC;
-                paramBNode.addProperty(CmlRdf.units, model.createResource(UnitsDictionary.U_ANGSTROM));
-            } else if ("iucr:cell_angle_alpha".equals(dictRef)) {
+                paramBNode.addProperty(CmlRdf.units, angstroms);
+            } else if (CELL_ANGLE_ALPHA.equals(dictRef)) {
                 cellParamProp = Cryst.cellAngleAlpha;
-                paramBNode.addProperty(CmlRdf.units, model.createResource(UnitsDictionary.U_DEGREE));
-            } else if ("iucr:cell_angle_beta".equals(dictRef)) {
+                paramBNode.addProperty(CmlRdf.units, degrees);
+            } else if (CELL_ANGLE_BETA.equals(dictRef)) {
                 cellParamProp = Cryst.cellAngleBeta;
-                paramBNode.addProperty(CmlRdf.units, model.createResource(UnitsDictionary.U_DEGREE));
-            } else if ("iucr:cell_angle_gamma".equals(dictRef)) {
+                paramBNode.addProperty(CmlRdf.units, degrees);
+            } else if (CELL_ANGLE_GAMMA.equals(dictRef)) {
                 cellParamProp = Cryst.cellAngleGamma;
-                paramBNode.addProperty(CmlRdf.units, model.createResource(UnitsDictionary.U_DEGREE));
+                paramBNode.addProperty(CmlRdf.units, degrees);
             } else {
                 throw new RuntimeException("Unsupported cell parameter: "+dictRef);
             }
     @Override
     protected String getPropertyUri(final String namespaceURI, final String localPart) {
         final String uri = super.getPropertyUri(namespaceURI, localPart);
-        if (DEPRECATED_MAP.containsKey(uri)) {
+        if (normaliseUris && DEPRECATED_MAP.containsKey(uri)) {
             return DEPRECATED_MAP.get(uri);
         }
         return uri;

crystallography-importer/src/main/java/net/chempound/crystal/importer/CrystalStructureImporter.java

 
 import com.hp.hpl.jena.rdf.model.Model;
 import com.hp.hpl.jena.rdf.model.Resource;
+import com.hp.hpl.jena.rdf.model.Statement;
+import com.hp.hpl.jena.rdf.model.StmtIterator;
 import com.hp.hpl.jena.sparql.vocabulary.FOAF;
 import com.hp.hpl.jena.vocabulary.RDFS;
-import nu.xom.*;
+import net.chempound.chemistry.CmlImporter;
+import net.chempound.chemistry.inchi.InchiTool;
+import net.chempound.content.DefaultDepositRequest;
+import net.chempound.content.DepositRequest;
+import net.chempound.crystal.importer.utils.CifIO;
+import net.chempound.crystal.importer.utils.CifSplitter;
+import net.chempound.crystal.rdf.ont.CifDict;
+import net.chempound.crystal.rdf.ont.Cryst;
+import net.chempound.rdf.DCTerms;
+import net.chempound.rdf.chempound.ChempoundAggregation;
+import net.chempound.storage.InMemoryResource;
+import net.chempound.storage.LocalResource;
+import net.chempound.util.MimeType;
+import nu.xom.Element;
+import nu.xom.Elements;
+import nu.xom.Node;
+import nu.xom.ParsingException;
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.xmlcml.cml.element.CMLIdentifier;
 import org.xmlcml.cml.element.CMLMolecule;
 import org.xmlcml.cml.tools.ConnectionTableTool;
-import net.chempound.chemistry.CmlImporter;
-import net.chempound.chemistry.inchi.InchiTool;
-import net.chempound.content.DepositRequest;
-import net.chempound.content.DefaultDepositRequest;
-import net.chempound.crystal.importer.utils.CifIO;
-import net.chempound.crystal.importer.utils.CifSplitter;
-import net.chempound.crystal.rdf.ont.Cryst;
-import net.chempound.rdf.DCTerms;
-import net.chempound.rdf.chempound.ChempoundAggregation;
-import net.chempound.storage.InMemoryResource;
-import net.chempound.storage.LocalResource;
-import net.chempound.util.MimeType;
 
-import java.io.*;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
 import java.net.URI;
+import java.util.ArrayList;
 import java.util.List;
 
 import static org.xmlcml.cml.base.CMLConstants.CML_XPATH;
         LOG.trace("Generating RDF from complete CML");
         final CrystalCml2RdfConverter conv = new CrystalCml2RdfConverter();
         final Model model = conv.createModel(completeCml, URI.create(""));
-        model.add(cifResource.getModel());
-        model.add(cmlResource.getModel());
+
 
         final DepositRequest loadRequest = new DefaultDepositRequest();
         loadRequest.setSlug(id);
         final ChempoundAggregation item = loadRequest.getResourceMetadata();
         addType(item);
 
+        final LocalResource rdfResource = writeRdf(id+".rdf", model);
+
         loadRequest.addResource(cifResource);
         loadRequest.addResource(cmlResource);
-
+        loadRequest.addResource(rdfResource);
 
         if (imageResource != null) {
-            model.add(imageResource.getModel());
             item.addProperty(FOAF.img, imageResource);
             loadRequest.addResource(imageResource);
         }
         if (thumbnailResource != null) {
-            model.add(thumbnailResource.getModel());
             item.addProperty(FOAF.thumbnail, thumbnailResource);
             loadRequest.addResource(thumbnailResource);
         }
 
         final Resource root = model.getResource("");
-//        if (root.hasProperty(CifDict.chemicalNameSystematic)) {
-//            String title = root.getProperty(CifDict.chemicalNameSystematic).getString();
-//            item.setTitle(title);
-//        } else if (root.hasProperty(CifDict.publSectionTitle)) {
-//            String title = root.getProperty(CifDict.publSectionTitle).getString();
-//            item.setTitle(title);
-//        }
+        if (root.hasProperty(CifDict.chemicalNameSystematic)) {
+            String title = root.getProperty(CifDict.chemicalNameSystematic).getString();
+            item.setTitle(title);
+        } else if (root.hasProperty(CifDict.publSectionTitle)) {
+            String title = root.getProperty(CifDict.publSectionTitle).getString();
+            item.setTitle(title);
+        }
 
         // chemical_formula_moiety
 
+        if (root.hasProperty(CifDict.publAuthorName)) {
+            StmtIterator it = root.listProperties(CifDict.publAuthorName);
+            List<String> names = new ArrayList<String>();
+            while (it.hasNext()) {
+                Statement s = it.next();
+                names.add(s.getObject().asLiteral().getString());
+            }
+            for (String name : names) {
+                root.addProperty(DCTerms.contributor, name);
+            }
+        }
 
-//        if (root.hasProperty(CifDict.publAuthorName)) {
-//            StmtIterator it = root.listProperties(CifDict.publAuthorName);
-//            List<String> names = new ArrayList<String>();
-//            while (it.hasNext()) {
-//                Statement s = it.next();
-//                names.add(s.getObject().asLiteral().getString());
-//            }
-//            for (String name : names) {
-//                root.addProperty(DCTerms.contributor, name);
-//            }
-//        }
-
-
-
-        loadRequest.getMetadataModel().add(model);
         return loadRequest;
     }
 

crystallography-importer/src/main/java/net/chempound/crystal/importer/Main.java

-package net.chempound.crystal.importer;
-
-import nu.xom.Document;
-import nu.xom.Serializer;
-import org.xmlcml.cif.CIF;
-import org.xmlcml.cif.CIFException;
-import org.xmlcml.cml.element.CMLCml;
-import sea36.image.jmol.ImageGeneratorJmol;
-import net.chempound.crystal.importer.utils.CifIO;
-import net.chempound.crystal.importer.utils.CifSplitter;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-/**
- * @author Sam Adams
- */
-public class Main {
-
-    ImageGeneratorJmol imageGen = new ImageGeneratorJmol();
-
-    static Queue<String> queue = new ConcurrentLinkedQueue<String>();
-
-    private static final File ROOT = new File("d:/chempound/2008/");
-    private static final File OUT = new File("d:/chempound/cifout/");
-
-    public static void main(final String[] args) throws Exception {
-
-        run(ROOT, "");
-
-        for (int i = 0; i < Runtime.getRuntime().availableProcessors(); i++) {
-            new Thread() {
-                @Override
-                public void run() {
-                    final Main main = new Main();
-                    for (String file = queue.poll(); file != null; file = queue.poll()) {
-                        try {
-                            main.process(file);
-                        } catch (Exception e) {
-                            e.printStackTrace();
-                        }
-                    }
-                    main.imageGen.shutdown();
-                }
-            }.start();
-        }
-
-    }
-
-    private static void run(final File root, final String s) throws Exception {
-        for (final File file : root.listFiles()) {
-            if (file.isDirectory()) {
-                run(file, s+file.getName()+"/");
-            } else if (file.getName().toLowerCase().endsWith(".cif")) {
-                queue.add(s+file.getName());
-            }
-        }
-
-    }
-
-    private void process(final String path) throws IOException, CIFException {
-
-        final File file = new File(ROOT, path);
-
-        String fn = path.substring(path.lastIndexOf('/')+1);
-        final String lc = fn.toLowerCase();
-        if (lc.endsWith(".txt") || lc.endsWith(".cif")) {
-            fn = fn.substring(0, fn.length()-4);
-        }
-
-        System.err.println("**\n** "+fn+"\n**");
-
-        final CIF originalCif = CifIO.readCif(file, "UTF-8");
-        originalCif.setId(fn);
-
-        final List<CIF> splitCifs = CifSplitter.split(originalCif);
-        final String cifId = originalCif.getId();
-        for (final CIF cif : splitCifs) {
-            final String datablockId = CrystalStructureImporter.getDataBlockId(cif);
-            final String id = cifId+"_"+datablockId;
-
-            try {
-                loadCif(cif, id, file.getParentFile(), new File(OUT, path.substring(0, path.lastIndexOf('/'))+id));
-            } catch (Exception e) {
-                e.printStackTrace();
-            }
-        }
-
-    }
-
-    private void loadCif(final CIF cif, final String id, final File root, final File dir) throws IOException {
-        dir.mkdirs();
-
-        final File cifFile = new File(dir, id + ".cif");
-        CifIO.writeCif(cif, cifFile, "UTF-8");
-
-        final CMLCml completeCml = (CMLCml) CrystalStructureImporter.convertCifToCompleteCml(cif);
-        CrystalStructureImporter.attachIdentifiers(completeCml);
-
-        final BufferedOutputStream cmlOut = new BufferedOutputStream(new FileOutputStream(new File(dir, id+".cml")));
-        final Serializer ser = new Serializer(cmlOut);
-        ser.write(new Document(completeCml));
-        cmlOut.close();
-
-
-        imageGen.loadMolecule(cifFile);
-        imageGen.setShowAxes(true);
-        imageGen.setShowUnitCell(true);
-        imageGen.createImage(new File(dir, id+".png"), 400, 400);
-        imageGen.createImage(new File(dir, id+"_big.png"), 800, 800);
-
-        imageGen.setShowAxes(false);
-        imageGen.setShowUnitCell(false);
-        imageGen.setFrankOn(false);
-        imageGen.createImage(new File(dir, id+"_tn.png"), 100, 100);
-
-    }
-
-}

crystallography-importer/src/test/java/net/chempound/crystal/importer/CrystalCml2RdfConverterTest.java

 
     static {
         CrystalPersonalities.init();
-        ChempoundPersonalities.init();
     }
 
     private static CMLCml cml;

crystallography-importer/src/test/java/net/chempound/crystal/importer/CrystalStructureImporterTest.java

 package net.chempound.crystal.importer;
 
+import com.hp.hpl.jena.rdf.arp.JenaReader;
 import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.ModelFactory;
+import com.hp.hpl.jena.rdf.model.RDFReader;
+import net.chempound.rdf.IgnoringEmptyBaseUriErrorHandler;
+import net.chempound.storage.DepositResource;
+import net.chempound.util.MimeType;
 import nu.xom.ParsingException;
+import org.apache.commons.io.IOUtils;
 import org.junit.Test;
 import org.xmlcml.cif.CIF;
 import org.xmlcml.cif.CIFException;
 import net.chempound.content.DepositRequest;
 import net.chempound.crystal.importer.utils.CifIO;
-import net.chempound.crystal.rdf.CrystalEyeUtil;
 import net.chempound.crystal.rdf.CrystalPersonalities;
 import net.chempound.crystal.rdf.CrystalStructure;
 import net.chempound.crystal.rdf.UnitCell;
 import java.io.InputStream;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 /**
 
     @Test
     public void testCifImporter() throws IOException, CIFException, ParsingException {
-        InputStream in = getClass().getResourceAsStream("/wn2432.cif");
-        CIF cif;
-        try {
-            cif = CifIO.readCif(in, "UTF-8");
-        } finally {
-            in.close();
-        }
+        CIF cif = loadCif("/wn2432.cif");
 
         CrystalStructureImporter importer = new CrystalStructureImporter();
         DepositRequest request = importer.generateDepositRequest(cif, "wn2432");
 
-        Model model = request.getMetadataModel();
-        CrystalStructure crystalStructure = CrystalEyeUtil.createCrystalStructure(model, "");
+        Model model = getRdfModel(request);
+        assertNotNull(model);
+
+        CrystalStructure crystalStructure = model.getResource("").as(CrystalStructure.class);
         assertTrue(crystalStructure.listProperties().hasNext());
 
         UnitCell unitCell = crystalStructure.getUnitCell();
         assertEquals("<i>N</i>-(Pyridin-2-ylmethyl)pyridin-2-amine", crystalStructure.getProperty(DCTerms.title).getString());
     }
 
+    private Model getRdfModel(final DepositRequest request) throws IOException {
+        for (final DepositResource resource : request.getAggregatedResources()) {
+            if (MimeType.APPLICATION_RDF_XML.equals(resource.getMimeType())) {
+                return loadRdf(resource);
+            }
+        }
+        return null;
+    }
+
+    private Model loadRdf(final DepositResource resource) throws IOException {
+        final InputStream in = resource.openInputStream();
+        try {
+            final RDFReader reader = new JenaReader();
+            reader.setErrorHandler(new IgnoringEmptyBaseUriErrorHandler());
+
+            final Model model = ModelFactory.createDefaultModel();
+            reader.read(model, in, "");
+
+            return model;
+        } finally {
+            IOUtils.closeQuietly(in);
+        }
+    }
+
+    private CIF loadCif(final String name) throws IOException, CIFException {
+        final InputStream in = getClass().getResourceAsStream(name);
+        try {
+            return CifIO.readCif(in, "UTF-8");
+        } finally {
+            IOUtils.closeQuietly(in);
+        }
+    }
+
 }

crystallography-importer/src/test/resources/log4j.properties

+
+log4j.rootLogger = WARN, A
+
+log4j.appender.A = org.apache.log4j.ConsoleAppender
+log4j.appender.A.layout = org.apache.log4j.PatternLayout
+log4j.appender.A.layout.ConversionPattern = %-4r [%t] %-5p %c %x - %m%n
+log4j.appender.A.target = System.err
+
+log4j.logger.net.chempound = DEBUG

crystallography-test-harness/src/test/java/net/chempound/crystal/CrystalSearchIntegrationTest.java

     private static void loadData() throws Exception {
 
         loadData("ds2101sup1");
+
+        Thread.sleep(120000);
+
         loadData("ff2005sup1");
         loadData("jj2086");
         loadData("wn2432");
         InputStream in = CrystalSearchIntegrationTest.class.getResourceAsStream("/"+id+".cif");
         try {
             CrystalStructureImporter importer = new CrystalStructureImporter();
-//            importer.setId(id);
             importer.setCreateImages(false);
             DepositRequest request = importer.generateDepositRequest(in, id);
             Deposit deposit = SwordUtil.createSwordDeposit(request);