Commits

tbrugz  committed 01563ca

datadump: multiple 'partitionBy' - added DumpSyntax.isStateful()

  • Participants
  • Parent commits 804bae7

Comments (0)

Files changed (4)

File src/tbrugz/sqldump/datadump/DataDump.java

 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 				}
 				
 				String filename = getDynamicFileName(prop, tableOrQueryId, ds.getSyntaxId());
+
+				//TODO: remove when multiple partition-patterns for stateful syntaxes is implemented
+				if(ds.isStateful()) {
+					if(partitionByPatterns.length>1) {
+						log.warn("syntax "+ds.getSyntaxId()+" is stateful and not ready for dumping queries with multiple partition-patterns");
+						continue;
+					}
+				}
 				
 				if(filename==null) {
 					log.warn("no output file defined for syntax '"+ds.getSyntaxId()+"'");
 			}
 			
 			Map<String, String> lastPartitionIdByPartitionPattern = new HashMap<String, String>();
+			//Map<String, DumpSyntax> statefulDumpSyntaxes = new HashMap<String, DumpSyntax>();
+			Set<DumpSyntax> hasWarnedAboutNonimplementedStatefulness = new HashSet<DumpSyntax>();
 			
 			//rows
 			long countInPartition = 0;
 								continue;
 							}
 							
+							if(ds.isStateful()) {
+								//TODO: implement dumping queries with multiple partition-patterns for stateful syntaxes 
+								if(partitionByPatterns.length>1) {
+									if(!hasWarnedAboutNonimplementedStatefulness.contains(ds)) {
+										log.warn("syntax "+ds.getSyntaxId()+" is stateful and not ready for dumping queries with multiple partition-patterns");
+										hasWarnedAboutNonimplementedStatefulness.add(ds);
+									}
+									continue;
+								}
+								/*DumpSyntax ds2 = statefulDumpSyntaxes.get(ds.getSyntaxId()+"$"+partitionByPattern);
+								if(ds2==null) {
+									//create/clone dumper... what abount header line?
+								}*/
+							}
+							
 							String finalFilename = getFinalFilenameForAbstractFilename(filenameList.get(i), partitionByStrId);
 							//Writer w = getWriterForFilename(finalFilename, charset, true);
 							boolean newFilename = isSetNewFilename(writersOpened, finalFilename, partitionByPattern, charset, writeBOM);
 							if(lastPartitionId!=null && !partitionByStrId.equals(lastPartitionId)) {
 								String lastFinalFilename = getFinalFilenameForAbstractFilename(filenameList.get(i), lastPartitionId);
 								//log.info("partid>> "+lastPartitionId+" // "+partitionByStrId+" // "+lastFinalFilename);
-								closeWriter(writersOpened, writersSyntaxes, getWriterMapKey(lastFinalFilename, partitionByPattern));
-								removeWriter(writersOpened, writersSyntaxes, getWriterMapKey(lastFinalFilename, partitionByPattern));
+								String lastWriterMapKey = getWriterMapKey(lastFinalFilename, partitionByPattern);
+								closeWriter(writersOpened, writersSyntaxes, lastWriterMapKey);
+								removeWriter(writersOpened, writersSyntaxes, lastWriterMapKey);
 							}
 							
 							if(newFilename) {

File src/tbrugz/sqldump/datadump/DumpSyntax.java

 	public boolean isWriterIndependent() {
 		return false;
 	}
+
+	/**
+	 * Should return true if dumpsyntax has buffer
+	 * 
+	 * FFCDataDump is stateful
+	 */
+	public boolean isStateful() {
+		return false;
+	}
 	
 	//XXX: method supportResultSetDump()?
 	//XXX> method cloneBaseProperties()? duplicateInstance(DumpSyntax)? clone()?

File src/tbrugz/sqldump/datadump/FFCDataDump.java

 	}
 	
 	@Override
+	public boolean isStateful() {
+		return true;
+	}
+	
+	@Override
 	public String getSyntaxId() {
 		return FFC_SYNTAX_ID;
 	}

File src/tbrugz/sqldump/datadump/SQLQueries.java

 			//String partitionBy = prop.getProperty("sqldump.query."+qid+".partitionby");
 			List<String> partitionsBy = Utils.getStringListFromProp(prop, "sqldump.query."+qid+".partitionby", "\\|");
 			//if(partitionsBy==null) { partitionsBy = new ArrayList<String>(); }
-			log.info("partitions: "+partitionsBy);
+			log.info("partition-patterns: "+partitionsBy); //XXX: move log into DataDump
 
 			List<String> keyCols = Utils.getStringListFromProp(prop, "sqldump.query."+qid+".keycols", ",");