Commits

Sara Magliacane committed 43b9d0d

adding serialization of intermediate results

Comments (0)

Files changed (8)

src/main/java/nl/vu/recoprov/CompletePipeline.java

 	private ProvDMtranslator ProvDMtranslator = new ProvDMtranslator();
 	private DropboxClient client = new DropboxClient();
 	private BackwardTemporalFilter backwardTemporalFilter = new BackwardTemporalFilter();
+	private LuceneIndexer indexer = new LuceneIndexer();
+	private LuceneSimilaritySignal luceneSimilaritySignal = new LuceneSimilaritySignal();
+	private LuceneInverseSimilarity luceneInverseSimilarity = new LuceneInverseSimilarity();
+	private MetadataSimilaritySignal metadataSimilaritySignal = new MetadataSimilaritySignal();
+	private MatchTitleInContentSignal matchTitleInContentSignal = new MatchTitleInContentSignal();
+	private ImageSimilaritySignal imageSimilaritySignal = new ImageSimilaritySignal();
+	
+	private TikaReader tika;
 	
 	public CompletePipeline() {
-		logger = LoggerFactory
-				.getLogger("nl.vu.recoprov.CompletePipeline");
+		this (false, "", null);
 	}
 
 
 		this.params = params;
 		logger = LoggerFactory
 				.getLogger("nl.vu.recoprov.CompletePipeline");
+		tika = new TikaReader(currentDir);
 	}
 	
 	public static void main(String[] args) throws Exception {
-		CompletePipeline pipeline = new CompletePipeline();
 
 		ConfigurationReader confreader = new ConfigurationReader();
-		confreader.readParameters(pipeline);
-
-		// System.out.println("### FIRST PHASE: Fetch the documents from Dropbox.\n");
-		DependencyGraph depGraph = new DependencyGraph();
-
-		depGraph = pipeline.initDependencyGraph();
-
-		pipeline.loadMetadaAndIndexes(depGraph);
-
+		CompletePipeline pipeline = confreader.readParameters();
+		
+		DependencyGraph depGraph = pipeline.initDependencyGraph();
 		pipeline.computeSignals(depGraph);
-
 		pipeline.filterSignals(depGraph);
-
 		pipeline.aggregateSignals(depGraph);
-
 		pipeline.writeToFile(depGraph);
-
-
 		pipeline.translateToPROVDM(depGraph);
-		
-
 
 	}
+	
+	public void translateToPROVDM(DependencyGraph depGraph, String name) throws FileNotFoundException {
+		ProvDMtranslator.translate(depGraph, name);	
+	}
 
 
-	private void translateToPROVDM(DependencyGraph depGraph) throws FileNotFoundException {
-		ProvDMtranslator.translate(depGraph);
-		
+	public void translateToPROVDM(DependencyGraph depGraph) throws FileNotFoundException {
+		ProvDMtranslator.translate(depGraph);	
 	}
 
-	public DependencyGraph initDependencyGraph() throws DropboxException {
+	public DependencyGraph initDependencyGraph() throws DropboxException, IOException {
+		DependencyGraph input = getDropboxMetadata();
+		loadMetadaAndIndexes(input);
+		return input;
+	}
+		
+	public DependencyGraph getDropboxMetadata() throws DropboxException{
 		logger.info("Initializing DependencyGraph for directory: {}", currentDir);
 		DependencyGraph depGraph = new DependencyGraph();
 		
 		}		
 		return depGraph;
 	}
-
+	
 	public void loadMetadaAndIndexes(DependencyGraph depGraph) throws IOException {
-
-		TikaReader tika = new TikaReader(currentDir);
-		depGraph = tika.read(depGraph, params);
-
-		// TODO: stripping of tags
-
-		// System.out.println("### THIRD PHASE: Index the contents with Apache Lucene. \n");
+		tika.read(depGraph, params);
 		indexFiles(depGraph);
 
-		// EXTRACT IMAGES
 		depGraph = ImageReader.read(currentDir, depGraph);
 
 	}
 
 	public void indexFiles(DependencyGraph depGraph) throws IOException{
-		LuceneIndexer indexer = new LuceneIndexer();
 		indexer.indexFiles(depGraph);
 	}
 	
-	public DependencyGraph computeSignals(DependencyGraph depGraph) {
-		// SIGNALS
-		new LuceneSimilaritySignal().computeSignal(depGraph);
-		new LuceneInverseSimilarity().computeSignal(depGraph);
-		// use lucene for better similarity - overlap of words
-		new MetadataSimilaritySignal().computeSignal(depGraph);
-		new MatchTitleInContentSignal().computeSignal(depGraph);
-
-		// TODO: compare nouns, verbs, named entities
-		// preprocessing get rid of the tags
+	// TODO: compare nouns, verbs, named entities
+	// preprocessing get rid of the tags
+	// overlap of words
+	
+	public void computeSignals(DependencyGraph depGraph) {
+		luceneSimilaritySignal(depGraph);
+		luceneInverseSimilarity(depGraph);
+		metadataSimilaritySignal(depGraph);
+		matchTitleInContentSignal(depGraph);
+		imageSimilaritySignal(depGraph);
+	}
+	
+	public void luceneSimilaritySignal(DependencyGraph depGraph){
+		luceneSimilaritySignal.computeSignal(depGraph);
+	}
+	
+	public void luceneInverseSimilarity(DependencyGraph depGraph) {
+		luceneInverseSimilarity.computeSignal(depGraph);
+	}
 
-		// TODO: transitive reduction after transitive closure?
+	public void metadataSimilaritySignal(DependencyGraph depGraph) {
+		metadataSimilaritySignal.computeSignal(depGraph);
+	}
 
-		new ImageSimilaritySignal().computeSignal(depGraph);
+	public void matchTitleInContentSignal(DependencyGraph depGraph) {
+		matchTitleInContentSignal.computeSignal(depGraph);
+	}
 
-		// depGraph = new DiffSignal().computeSignal(depGraph);
-		return depGraph;
+	public void imageSimilaritySignal(DependencyGraph depGraph) {
+		imageSimilaritySignal.computeSignal(depGraph);
 	}
 
-	public void filterSignals(DependencyGraph depGraph) {
-		// FILTERS
+	public void filterBackWards(DependencyGraph depGraph) {
 		backwardTemporalFilter.filterSignals(depGraph);
+	}
 
+	public void filterLuceneThreshold(DependencyGraph depGraph) {
+		luceneThresholdFilter.filterSignals(depGraph);
+	}
+
+	public void filterTextContainment (DependencyGraph depGraph){
+		textContainmentFilter.filterSignals(depGraph);
 	}
 	
+	public void filterPlagiarismCorpus (DependencyGraph depGraph){
+		plagiarismCorpusSpecificFilter.filterSignals(depGraph);
+	}
 	
+	public void filterSignals(DependencyGraph depGraph) {
+		filterBackWards(depGraph);
+		filterLuceneThreshold (depGraph);
+		filterTextContainment (depGraph);	
+	}
+
 	public void aggregateSignals(DependencyGraph depGraph) {
 		aggregator.aggregateSignals(depGraph);
 	}
 		connectToInternet = online;
 	}
 	
+
+	
 }

src/main/java/nl/vu/recoprov/ProvDMtranslator.java

 import org.openprovenance.prov.xml.SpecializationOf;
 import org.openprovenance.prov.xml.Statement;
 import org.openprovenance.prov.xml.WasDerivedFrom;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class ProvDMtranslator {
 
 	private Collection<Statement> listOfAvailableRelations;
 	private boolean useTred = false;
 	private ProvFactory factory;
+	private Logger logger;
 	
 	public String translate(DependencyGraph input) throws FileNotFoundException {
 		System.out.println("Got called...");
 	}
 	 
 	public String translate(DependencyGraph input, String graphfilename) throws FileNotFoundException {
-
-		System.out.println("Translate to PROVDM.");
+		
+		logger = LoggerFactory
+				.getLogger("nl.vu.recoprov.ProvDMTranslator");
+		
+		logger.info("Translate to PROVDM.");
 		
 		factory = new ProvFactory();		
 		listOfAvailableEntities = new HashMap<Integer, Entity> ();
 	
 	public void convertToDot(String graphfilename) throws FileNotFoundException {
 
-		System.out.println("Going to build a dot file");
+		logger.info("Going to build a dot file");
 		ProvToDot provtodot = new ProvToDot();
 
 		Document container = factory.newDocument(new LinkedList<Activity>(),

src/main/java/nl/vu/recoprov/baseclasses/DependencyGraph.java

 package nl.vu.recoprov.baseclasses;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutput;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import nl.vu.recoprov.signalaggregators.WeightedSumAggregator;
 import nl.vu.recoprov.utils.TransitiveClosure;
 
 public class DependencyGraph extends TreeMap<String, DependencyNode>{
 
 
-	private static final long serialVersionUID = 1L;
+	private static final long serialVersionUID = 42L;
 
 	private HashMap<Integer, ArrayList<LabelledEdge>> incidendencyMatrix;
-	
 	public HashMap<Integer, String> translationMap;
-			
 	private Set<String> attributes = new LinkedHashSet<String>();
+	private Logger logger;
 			
 	public DependencyGraph(){
 		super();
 		incidendencyMatrix = new LinkedHashMap<Integer,  ArrayList<LabelledEdge> >();
 		translationMap = new HashMap<Integer, String>();
+		logger = LoggerFactory
+				.getLogger("nl.vu.recoprov.baseclasses.DependencyGraph");
 	}
 
 
 	
 	public SimilarGraphResults similarToGraph(DependencyGraph g){
 		if( this.size()!= g.size()){
-			System.out.println("Different size");
+			logger.info("Different size");
 			return new SimilarGraphResults();
 		}
 		if(!this.keySet().equals(g.keySet())){
-			System.out.println("Different keyset: ");
-			System.out.println(this.keySet());
-			System.out.println(g.keySet());
+			logger.info("Different keyset: {} ", this.keySet());
+			logger.info("Different keyset: {} ", g.keySet());
 			return new SimilarGraphResults();
 		}
 		else{
 						}
 						if(!found){
 							 results.add(i,l1, "FN");
-							 System.out.println("False negative: from "+ i + " -> "+ l1.getId()  + " does not exist in predicted");
+							 logger.trace("False negative: from "+ i + " -> "+ l1.getId()  + " does not exist in predicted");
 						 }	 
 						
 					}
 		
 	}
 	
-	public class LabelledEdge {
+	public class LabelledEdge implements Serializable {
+		/**
+		 * 
+		 */
+		private static final long serialVersionUID = 1L;
 		private int id;
 		private String label ;
 		private double score = 0.0;
 	}
 	
 	
-	public class SimilarGraphResults{
+	public class SimilarGraphResults implements Serializable{
+		/**
+		 * 
+		 */
+		private static final long serialVersionUID = 1L;
 		public LinkedHashMap<Integer, ArrayList<LabelledEdge>> FalsePositives;
 		public LinkedHashMap<Integer, ArrayList<LabelledEdge>> FalseNegatives;
 		public LinkedHashMap<Integer, ArrayList<LabelledEdge>> TruePositives;
 			
 	}
 	
+	public void serialize (String name) throws IOException {		
+	      //use buffering
+	      OutputStream file = new FileOutputStream( name );
+	      OutputStream buffer = new BufferedOutputStream( file );
+	      ObjectOutput output = new ObjectOutputStream( buffer );
+	      output.writeObject(this);
+		  logger.info("Serializing object: {}", name);
+	      output.close();
+	}
+	
+	public static DependencyGraph deserializeDependencyGraph(String name) throws IOException, ClassNotFoundException {
+		InputStream file = new FileInputStream(name);
+		InputStream buffer = new BufferedInputStream(file);
+		ObjectInput input = new ObjectInputStream(buffer);
+		DependencyGraph recovered = (DependencyGraph) input.readObject();
+		input.close();
+		return recovered;
+	}
+	
 }

src/main/java/nl/vu/recoprov/baseclasses/DependencyNode.java

 package nl.vu.recoprov.baseclasses;
 
+import java.io.Serializable;
 import java.util.Set;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.tika.metadata.Metadata;
 import com.dropbox.client2.DropboxAPI.Entry;
 
 
-public class DependencyNode {
+public class DependencyNode implements Serializable {
+
+	/**
+	 * 
+	 */
+	private static final long serialVersionUID = 84L;
 
 	public static final String DELIMITER = "&";
 
 	private String completefilepath;
 	private DependencyGraph depGraph;
-	private ScoreDoc[] luceneSimilarity = null;
+	//private ScoreDoc[] luceneSimilarity = null;
 	private String content = null;
 	private RecoMetadata recoMetadata = new RecoMetadata();
 
 		depGraph = d;
 	}
 
-	public ScoreDoc[] getLuceneSimilarity() {
-		return luceneSimilarity;
-	}
-
-	public void setLuceneSimilarity(ScoreDoc[] luceneSimilarity) {
-		this.luceneSimilarity = luceneSimilarity;
-	}
+//	public ScoreDoc[] getLuceneSimilarity() {
+//		return luceneSimilarity;
+//	}
+//
+//	public void setLuceneSimilarity(ScoreDoc[] luceneSimilarity) {
+//		this.luceneSimilarity = luceneSimilarity;
+//	}
 
 	public String getMimeType() {
 		return this.recoMetadata.getMimeType();
 		d.setContent(this.getContent());
 		d.setCompleteFilepath(this.getCompleteFilepath());
 		d.setDropboxEntry(getDropboxEntry());
-		d.setLuceneSimilarity(getLuceneSimilarity());
+		//d.setLuceneSimilarity(getLuceneSimilarity());
 		d.setLuceneDocNumber(getLuceneDocNumber());
 		d.setMetadata(getMetadata());
 		return d;

src/main/java/nl/vu/recoprov/experiments/Experiment3.java

 import java.io.FileWriter;
 import java.io.IOException;
 import nl.vu.recoprov.CompletePipeline;
-import nl.vu.recoprov.ProvDMtranslator;
 import nl.vu.recoprov.baseclasses.DependencyGraph;
-import nl.vu.recoprov.signalaggregators.WeightedSumAggregator;
-import nl.vu.recoprov.signaldetectors.CompressionDistanceSignal;
-import nl.vu.recoprov.signaldetectors.LuceneInverseSimilarity;
-import nl.vu.recoprov.signaldetectors.LuceneMoreLikeThisSignal;
-import nl.vu.recoprov.signaldetectors.LuceneSimilaritySignal;
-import nl.vu.recoprov.signalfilters.LuceneThresholdFilter;
-import nl.vu.recoprov.signalfilters.PlagiarismCorpusSpecificFilter;
-import nl.vu.recoprov.signalfilters.TextContainmentFilter;
 import nl.vu.recoprov.utils.ConfigurationDefaults;
-import nl.vu.recoprov.utils.TransitiveClosure;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 
 	private static DependencyGraph baselineGraph = null;
 	private static DependencyGraph depGraph = null;
-	private static ProvDMtranslator provtranslator = new ProvDMtranslator();
 	private static Logger logger;
 
 	public static void main(String[] args) throws Exception  {
 				.getLogger("nl.vu.recoprov.experiments.Experiment3");
 		
 		if (!checkInitialParameters()){
-			// if anything is wrong with the two parameters
 			System.exit(0);
 		}
 		
-		String resultsfilename = "results" + System.currentTimeMillis() + ".txt";
-		FileWriter writer;
-		try {
-			writer = createFileResultsWriter(resultsfilename);
-		} catch (IOException e) {
-			e.printStackTrace();
-			logger.error("Could not create results file writer {}", resultsfilename);
-			throw e;
-		}
-
-		DependencyGraph depGraphLucene;
-		try {
-			depGraphLucene = createGraph();
-		} catch (Exception e) {
-			e.printStackTrace();
-			logger.error("Could not create baseline graph.");
-			throw e;
-		}
-		
-		try {
-			new LuceneSimilaritySignal().computeSignal(depGraphLucene);
-			logger.info("Lucene graph created.");
-		} catch (Exception e) {
-			e.printStackTrace();
-			logger.error("Could not create Lucene graph.");
-			throw e;
-		}
-		
-		try {
-			PROVReader provreader = new PROVReader(dirfile, jsonfile);
-			depGraph = provreader.generatePANDepGraph();
-			provtranslator.translate(depGraph, "graphCorpus.gv");
-			logger.info("Reference graph created.");
-		} catch (Exception e) {
-			e.printStackTrace();
-			logger.error("Could not create reference graph.");
-			throw e;
-		}
-	
-		LuceneThresholdFilter luceneThresholdFilter = new LuceneThresholdFilter();
-		WeightedSumAggregator aggregator = new WeightedSumAggregator();
-		TextContainmentFilter textContainmentFilter = new TextContainmentFilter();
-		PlagiarismCorpusSpecificFilter plagiarismCorpusSpecificFilter = new PlagiarismCorpusSpecificFilter();
-
+		CompletePipeline pipeline = new CompletePipeline(false, dirfile);
+		FileWriter writer = createFileResultsWriter();
+		DependencyGraph depGraphLucene = createLuceneGraph(pipeline);		
+		depGraph = createReferenceGraph(pipeline);
+				
 		// useful for small experiments
 		double[] thresholds = { 0.01, 0.05, 0.1, 0.2 };
 		
 
 				DependencyGraph depGraphLuceneThreshold = depGraphLucene
 						.copyGraph();
-				luceneThresholdFilter.filterSignals(depGraphLuceneThreshold);
+				pipeline.filterLuceneThreshold(depGraphLuceneThreshold);
 
 				DependencyGraph depGraph1 = depGraphLuceneThreshold.copyGraph();
-				aggregator.aggregateSignals(depGraph1);
-				writeResults(depGraph1, "Lucene", writer);
+				pipeline.aggregateSignals(depGraph1);
+				writeResults(pipeline, depGraph1, "Lucene", writer);
 
 				depGraph1 = depGraphLuceneThreshold.copyGraph();
-				textContainmentFilter.filterSignals(depGraph1);
-				aggregator.aggregateSignals(depGraph1);
-				writeResults(depGraph1, "LuceneFilter", writer);
+				pipeline.filterPlagiarismCorpus(depGraph1);
+				pipeline.aggregateSignals(depGraph1);
+				writeResults(pipeline, depGraph1, "LuceneFilter", writer);
 
 				depGraph1 = depGraphLuceneThreshold.copyGraph();
-				plagiarismCorpusSpecificFilter.filterSignals(depGraph1);
-				aggregator.aggregateSignals(depGraph1);
-				writeResults(depGraph1, "LucenePANFilter", writer);
+				pipeline.filterPlagiarismCorpus(depGraph1);
+				pipeline.aggregateSignals(depGraph1);
+				writeResults(pipeline, depGraph1, "LucenePANFilter", writer);
 
 				depGraph1 = depGraphLuceneThreshold.copyGraph();
-				textContainmentFilter.filterSignals(depGraph1);
-				plagiarismCorpusSpecificFilter.filterSignals(depGraph1);
-				aggregator.aggregateSignals(depGraph1);
-				writeResults(depGraph1, "LucenePAN2Filters", writer);
+				pipeline.filterTextContainment(depGraph1);
+				pipeline.filterPlagiarismCorpus(depGraph1);
+				pipeline.aggregateSignals(depGraph1);
+				writeResults(pipeline, depGraph1, "LucenePAN2Filters", writer);
 
 			} catch (Exception e) {
 				e.printStackTrace();
 //			writeResults(depGraph1, "LuceneMoreLikeThisPAN2Filters", writer);
 //		}
 
-
 		writer.flush();
 		writer.close();
 
 	}
 
-	private static void writeResults(DependencyGraph predicted, String message,
+	private static void writeResults(CompletePipeline pipeline, DependencyGraph predicted, String message,
 			FileWriter writer) throws FileNotFoundException {
 
 		String temp = depGraph.similarToGraph(predicted).toString();
 		}
 		
 		logger.info("{} graph created, threshold {}", message, ConfigurationDefaults.LUCENE_THRESHOLD );
-
-		// writer.append("\n\n"+ predicted.toString() +" \n");
-
-		provtranslator.translate(predicted, "graph" + message + "_"
+		pipeline.translateToPROVDM(predicted, "graph" + message + "_"
 				+ ConfigurationDefaults.LUCENE_THRESHOLD + ".gv");
 
 
 	}
 
 	public static DependencyGraph createGraph() throws DropboxException, IOException {
+		try {
+			if (baselineGraph == null) {
+				baselineGraph = new DependencyGraph();
+				CompletePipeline pipeline = new CompletePipeline(false, dirfile,
+						ConfigurationDefaults.PLAGIARISMDETECTIONDIRS);
 
-		if (baselineGraph == null) {
-			baselineGraph = new DependencyGraph();
-			CompletePipeline pipeline = new CompletePipeline(false, dirfile,
-					ConfigurationDefaults.PLAGIARISMDETECTIONDIRS);
-
-			baselineGraph = pipeline.initDependencyGraph();
-			pipeline.loadMetadaAndIndexes(baselineGraph);
-			logger.info("Baseline graph created.");
+				baselineGraph = pipeline.initDependencyGraph();
+				logger.info("Baseline graph created.");
+			}
+				
+		} catch (DropboxException e) {
+			e.printStackTrace();
+			logger.error("Could not create baseline graph.");
+			throw e;
 		}
-
+		
 		DependencyGraph copyOfBaseline = baselineGraph.copyGraph();
+		
 		return copyOfBaseline;
 	}
 	
 
 	}
 	
-	public static FileWriter createFileResultsWriter(String resultsfilename) throws IOException {
+	public static DependencyGraph createLuceneGraph(CompletePipeline pipeline) throws Exception{
+		DependencyGraph depGraphLucene = deserializeDependencyGraph("depGraphLucene.ser");
+		
+		if (depGraphLucene == null){
+			depGraphLucene = createGraph();
+			
+			try {
+				pipeline.luceneSimilaritySignal(depGraphLucene);
+				logger.info("Lucene graph created.");
+			} catch (Exception e) {
+				e.printStackTrace();
+				logger.error("Could not create Lucene graph.");
+				throw e;
+			}
+			
+			depGraphLucene.serialize("depGraphLucene.ser");
+		}
+		return depGraphLucene;
+	}
+	
+	
+	public static DependencyGraph createReferenceGraph(CompletePipeline pipeline) throws Exception{
+		
+		DependencyGraph depGraph1 = deserializeDependencyGraph("depGraphReference.ser");
+		
+		if (depGraph1 != null)
+			return depGraph1;
+		
+		PROVReader provreader = new PROVReader(dirfile, jsonfile);
+		try {
+			depGraph1 = provreader.generatePANDepGraph();
+			pipeline.translateToPROVDM(depGraph1, "graphCorpus.gv");
+			depGraph1.serialize("depGraphReference.ser");
+			logger.info("Reference graph created.");
+		} catch (Exception e) {
+			e.printStackTrace();
+			logger.error("Could not create reference graph.");
+			throw e;
+		}
+		
+		return depGraph1;
+	
+	}
+	
+	
+	public static FileWriter createFileResultsWriter() throws IOException {
+		String resultsfilename = "results" + System.currentTimeMillis() + ".txt";
 		FileWriter writer;
 		try {
 			writer = new FileWriter(resultsfilename);
 		return writer;
 	}
 
+
+	public static DependencyGraph deserializeDependencyGraph(String name) {		
+		DependencyGraph recovered = null;
+		
+		try {
+			recovered = DependencyGraph.deserializeDependencyGraph(name);
+		} catch (ClassNotFoundException ex) {
+			logger.error("Cannot perform input. Class not found.", ex);
+		} catch (IOException ex) {
+			logger.error("Cannot perform input.", ex);
+		}
+		
+		return recovered;
+	}
+	
+	
 }

src/main/java/nl/vu/recoprov/signaldetectors/LuceneMoreLikeThisSignal.java

 
 			String key = doc.getField("name").stringValue();
 			DependencyNode d = input.get(key);
-			d.setLuceneSimilarity(hits.scoreDocs);
 
 			for (int j = 0; j < hits.scoreDocs.length; j++) {
 

src/main/java/nl/vu/recoprov/signaldetectors/LuceneSimilaritySignal.java

 			if (tfvs == null) {
 				logger.error("No term vector for doc {}", i);
 			} else {
-				logger.trace("LuceneSimilarity: working on doc {} ", i);
+				logger.info("LuceneSimilarity: working on doc {} ", i);
 				TermsEnum tenum;
 				try {
 					tenum = tfvs.iterator(null);
 			}
 
 			String key = doc.getField("name").stringValue();
-			DependencyNode d = input.get(key);
-			d.setLuceneSimilarity(hits);
 
 			for (int j = 0; j < hits.length; j++) {
 

src/main/java/nl/vu/recoprov/utils/ConfigurationReader.java

 	
 
 	private String configfile = ConfigurationDefaults.CONFIG_FILE;
+	private Boolean online = false;
+	private String currentDir = "";
 	
 	public ConfigurationReader(){}
 	
 	public ConfigurationReader(String filename){
 		configfile = filename;
 	}
+	
+	public CompletePipeline readParameters() {
+		readConfigFile();
+		return new CompletePipeline(online, currentDir);
+	}
+	
 
 	public void readParameters(CompletePipeline pipeline) {
-
+		readConfigFile();
+		pipeline.setCurrentDir(currentDir);
+		pipeline.setConnectToInternet(online);
+		
+	}
+		
+	private void readConfigFile(){
 		System.out.println("Reading config file: " + configfile);
 
-		String currentDir = "";
+
 		String line = null;
 
 		CustomFileReader reader;
 			line = line.toLowerCase().trim();
 
 			if (line.startsWith("dropbox")) {
-				pipeline.setConnectToInternet(true);
+				online = true;
 			}
 
 			if (line.startsWith("local")) {
-				pipeline.setConnectToInternet(false);
+				online = false;
 			}
 
 		}
 
 		reader.close();
-		pipeline.setCurrentDir(currentDir);
+
+
 
 	}
 }