Commits

Dan Wu committed 8b6f486

Refactored NC writer code to separate writing of set variables (including used coordinates) and non-set variables.

Comments (0)

Files changed (8)

src/io/NCWriteGCRM.cpp

   return MB_SUCCESS;
 }
 
-ErrorCode NCWriteGCRM::write_values(std::vector<std::string>& var_names, std::vector<int>& tstep_nums)
+ErrorCode NCWriteGCRM::write_nonset_variables(std::vector<WriteNC::VarData>& vdatas, std::vector<int>& tstep_nums)
 {
   Interface*& mbImpl = _writeNC->mbImpl;
-  std::set<std::string>& usedCoordinates = _writeNC->usedCoordinates;
-  std::set<std::string>& dummyVarNames = _writeNC->dummyVarNames;
-  std::map<std::string, WriteNC::VarData>& varInfo = _writeNC->varInfo;
 
   int success;
   Range* pLocalEntsOwned = NULL;
   // Now look at requested var_names; if they have time, we will have a list, and write one at a time
   // For each variable tag in the indexed lists, write a time step data
   // Assume the first dimension is time (need to check); if not, just write regularly
-  for (size_t i = 0; i < var_names.size(); i++) {
-    std::map<std::string, WriteNC::VarData>::iterator vit = varInfo.find(var_names[i]);
-    if (vit == varInfo.end())
-      ERRORR(MB_FAILURE, "Can't find variable requested.");
-
-    WriteNC::VarData& variableData = vit->second;
+  for (unsigned int i = 0; i < vdatas.size(); i++) {
+    WriteNC::VarData& variableData = vdatas[i];
 
     // Skip edge variables, if there are no edges
     if (localEdgesOwned.empty() && variableData.entLoc == WriteNC::ENTLOCEDGE)
       continue;
 
-    if (variableData.has_tsteps) {
-      // Time should be the first dimension
-      assert(tDim == variableData.varDims[0]);
-
-      // Get local owned entities of this variable
-      switch (variableData.entLoc) {
-        case WriteNC::ENTLOCVERT:
-          // Vertices
-          pLocalEntsOwned = &localVertsOwned;
-          pLocalGidEntsOwned = &localGidVertsOwned;
-          break;
-        case WriteNC::ENTLOCEDGE:
-          // Edges
-          pLocalEntsOwned = &localEdgesOwned;
-          pLocalGidEntsOwned = &localGidEdgesOwned;
-          break;
-        case WriteNC::ENTLOCFACE:
-          // Cells
-          pLocalEntsOwned = &localCellsOwned;
-          pLocalGidEntsOwned = &localGidCellsOwned;
-          break;
-        default:
-          ERRORR(MB_FAILURE, "Unexpected entity location type for GCRM non-set variable.");
-      }
+    // Time should be the first dimension
+    assert(tDim == variableData.varDims[0]);
+
+    // Get local owned entities of this variable
+    switch (variableData.entLoc) {
+      case WriteNC::ENTLOCVERT:
+        // Vertices
+        pLocalEntsOwned = &localVertsOwned;
+        pLocalGidEntsOwned = &localGidVertsOwned;
+        break;
+      case WriteNC::ENTLOCEDGE:
+        // Edges
+        pLocalEntsOwned = &localEdgesOwned;
+        pLocalGidEntsOwned = &localGidEdgesOwned;
+        break;
+      case WriteNC::ENTLOCFACE:
+        // Cells
+        pLocalEntsOwned = &localCellsOwned;
+        pLocalGidEntsOwned = &localGidCellsOwned;
+        break;
+      default:
+        ERRORR(MB_FAILURE, "Unexpected entity location type for MPAS non-set variable.");
+    }
 
-      // A typical variable has 3 dimensions as (Time, nCells, nVertLevels)
-      for (unsigned int t = 0; t < tstep_nums.size(); t++) {
-        // We will write one time step, and count will be one; start will be different
-        // Use tag_get_data instead of tag_iterate to get values, as localEntsOwned
-        // might not be contiguous.
-        variableData.writeStarts[0] = t; // This is time, again
-        std::vector<double> tag_data(pLocalEntsOwned->size() * variableData.numLev);
-        ErrorCode rval = mbImpl->tag_get_data(variableData.varTags[t], *pLocalEntsOwned, &tag_data[0]);
-        ERRORR(rval, "Trouble getting tag data on owned vertices.");
+    // A typical variable has 3 dimensions as (Time, nCells, nVertLevels)
+    for (unsigned int t = 0; t < tstep_nums.size(); t++) {
+      // We will write one time step, and count will be one; start will be different
+      // Use tag_get_data instead of tag_iterate to get values, as localEntsOwned
+      // might not be contiguous.
+      variableData.writeStarts[0] = t; // This is time, again
+      std::vector<double> tag_data(pLocalEntsOwned->size() * variableData.numLev);
+      ErrorCode rval = mbImpl->tag_get_data(variableData.varTags[t], *pLocalEntsOwned, &tag_data[0]);
+      ERRORR(rval, "Trouble getting tag data on owned vertices.");
 
 #ifdef PNETCDF_FILE
-        size_t nb_writes = pLocalGidEntsOwned->psize();
-        std::vector<int> requests(nb_writes), statuss(nb_writes);
-        size_t idxReq = 0;
+      size_t nb_writes = pLocalGidEntsOwned->psize();
+      std::vector<int> requests(nb_writes), statuss(nb_writes);
+      size_t idxReq = 0;
 #endif
 
-        // Now write from memory directly
-        switch (variableData.varDataType) {
-          case NC_DOUBLE: {
-            size_t indexInDoubleArray = 0;
-            size_t ic = 0;
-            for (Range::pair_iterator pair_iter = pLocalGidEntsOwned->pair_begin();
-                pair_iter != pLocalGidEntsOwned->pair_end(); ++pair_iter, ic++) {
-              EntityHandle starth = pair_iter->first;
-              EntityHandle endh = pair_iter->second;
-              variableData.writeStarts[1] = (NCDF_SIZE)(starth - 1);
-              variableData.writeCounts[1] = (NCDF_SIZE)(endh - starth + 1);
-
-              // Do a partial write, in each subrange
+      // Now write from memory directly
+      switch (variableData.varDataType) {
+        case NC_DOUBLE: {
+          size_t indexInDoubleArray = 0;
+          size_t ic = 0;
+          for (Range::pair_iterator pair_iter = pLocalGidEntsOwned->pair_begin();
+              pair_iter != pLocalGidEntsOwned->pair_end(); ++pair_iter, ic++) {
+            EntityHandle starth = pair_iter->first;
+            EntityHandle endh = pair_iter->second;
+            variableData.writeStarts[1] = (NCDF_SIZE)(starth - 1);
+            variableData.writeCounts[1] = (NCDF_SIZE)(endh - starth + 1);
+
+            // Do a partial write, in each subrange
 #ifdef PNETCDF_FILE
-              // Wait outside this loop
-              success = NCFUNCREQP(_vara_double)(_fileId, variableData.varId,
-                  &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
-                             &(tag_data[indexInDoubleArray]), &requests[idxReq++]);
+            // Wait outside this loop
+            success = NCFUNCREQP(_vara_double)(_fileId, variableData.varId,
+                &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
+                           &(tag_data[indexInDoubleArray]), &requests[idxReq++]);
 #else
-              success = NCFUNCAP(_vara_double)(_fileId, variableData.varId,
-                  &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
-                             &(tag_data[indexInDoubleArray]));
-#endif
-              ERRORS(success, "Failed to read double data in loop");
-              // We need to increment the index in double array for the
-              // next subrange
-              indexInDoubleArray += (endh - starth + 1) * variableData.numLev;
-            }
-            assert(ic == pLocalGidEntsOwned->psize());
-#ifdef PNETCDF_FILE
-            success = ncmpi_wait_all(_fileId, requests.size(), &requests[0], &statuss[0]);
-            ERRORS(success, "Failed on wait_all.");
+            success = NCFUNCAP(_vara_double)(_fileId, variableData.varId,
+                &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
+                           &(tag_data[indexInDoubleArray]));
 #endif
-            break;
+            ERRORS(success, "Failed to read double data in loop");
+            // We need to increment the index in double array for the
+            // next subrange
+            indexInDoubleArray += (endh - starth + 1) * variableData.numLev;
           }
-          default:
-            ERRORR(MB_FAILURE, "Not implemented yet.");
-        }
-      }
-    } // if (variableData.has_tsteps)
-    else {
-      switch (variableData.varDataType) {
-        case NC_DOUBLE:
-          success = NCFUNCAP(_vara_double)(_fileId, variableData.varId, &variableData.writeStarts[0],
-                    &variableData.writeCounts[0], (double*)(variableData.memoryHogs[0]));
-          ERRORS(success, "Failed to write double data.");
-          break;
-        default:
-          ERRORR(MB_FAILURE, "Not implemented yet.");
-      }
-    }
-  }
-
-  // Write coordinates used by requested var_names
-  // Use independent I/O mode put, since this write is only for the root processor
-  // CAUTION: if the NetCDF ID is from a previous call to ncmpi_create rather than ncmpi_open,
-  // all processors need to call ncmpi_begin_indep_data(). If only the root processor does so,
-  // ncmpi_begin_indep_data() call will be blocked forever :(
+          assert(ic == pLocalGidEntsOwned->psize());
 #ifdef PNETCDF_FILE
-  // Enter independent I/O mode
-  success = NCFUNC(begin_indep_data)(_fileId);
-  ERRORS(success, "Failed to begin independent I/O mode.");
+          success = ncmpi_wait_all(_fileId, requests.size(), &requests[0], &statuss[0]);
+          ERRORS(success, "Failed on wait_all.");
 #endif
-
-  int rank = 0;
-#ifdef USE_MPI
-  bool& isParallel = _writeNC->isParallel;
-  if (isParallel) {
-    ParallelComm*& myPcomm = _writeNC->myPcomm;
-    rank = myPcomm->proc_config().proc_rank();
-  }
-#endif
-  if (0 == rank) {
-    for (std::set<std::string>::iterator setIt = usedCoordinates.begin();
-        setIt != usedCoordinates.end(); ++setIt) {
-      const std::string& coordName = *setIt;
-
-      // Skip dummy coordinate variables (e.g. ncol)
-      if (dummyVarNames.find(coordName) != dummyVarNames.end())
-        continue;
-
-      std::map<std::string, WriteNC::VarData>::iterator vit = varInfo.find(coordName);
-      if (vit == varInfo.end())
-        ERRORR(MB_FAILURE, "Can't find one coordinate variable.");
-
-      WriteNC::VarData& varCoordData = vit->second;
-
-      switch (varCoordData.varDataType) {
-        case NC_DOUBLE:
-          // Independent I/O mode put
-          success = NCFUNCP(_vara_double)(_fileId, varCoordData.varId, &varCoordData.writeStarts[0],
-                    &varCoordData.writeCounts[0], (double*)(varCoordData.memoryHogs[0]));
-          ERRORS(success, "Failed to write double data.");
-          break;
-        case NC_INT:
-          // Independent I/O mode put
-          success = NCFUNCP(_vara_int)(_fileId, varCoordData.varId, &varCoordData.writeStarts[0],
-                    &varCoordData.writeCounts[0], (int*)(varCoordData.memoryHogs[0]));
-          ERRORS(success, "Failed to write int data.");
           break;
+        }
         default:
-          ERRORR(MB_FAILURE, "Not implemented yet.");
+          ERRORR(MB_NOT_IMPLEMENTED, "Writing with current data type not implemented yet.");
       }
     }
   }
 
-#ifdef PNETCDF_FILE
-  // End independent I/O mode
-  success = NCFUNC(end_indep_data)(_fileId);
-  ERRORS(success, "Failed to end independent I/O mode.");
-#endif
-
   return MB_SUCCESS;
 }
 

src/io/NCWriteGCRM.hpp

   //! Collect data for specified variables
   virtual ErrorCode collect_variable_data(std::vector<std::string>& var_names, std::vector<int>& tstep_nums);
 
-  //! Implementation of NCWriteHelper::write_values()
-  virtual ErrorCode write_values(std::vector<std::string>& var_names, std::vector<int>& tstep_nums);
+  //! Implementation of NCWriteHelper::write_nonset_variables()
+  virtual ErrorCode write_nonset_variables(std::vector<WriteNC::VarData>& vdatas, std::vector<int>& tstep_nums);
 };
 
 } // namespace moab

src/io/NCWriteHOMME.cpp

   return MB_SUCCESS;
 }
 
-ErrorCode NCWriteHOMME::write_values(std::vector<std::string>& var_names, std::vector<int>& tstep_nums)
+ErrorCode NCWriteHOMME::write_nonset_variables(std::vector<WriteNC::VarData>& vdatas, std::vector<int>& tstep_nums)
 {
   Interface*& mbImpl = _writeNC->mbImpl;
-  std::set<std::string>& usedCoordinates = _writeNC->usedCoordinates;
-  std::set<std::string>& dummyVarNames = _writeNC->dummyVarNames;
-  std::map<std::string, WriteNC::VarData>& varInfo = _writeNC->varInfo;
 
   int success;
   int num_local_verts_owned = localVertsOwned.size();
   // Need to transpose from lev dimension
   // For each variable tag in the indexed lists, write a time step data
   // Assume the first dimension is time (need to check); if not, just write regularly
-  for (size_t i = 0; i < var_names.size(); i++) {
-    std::map<std::string, WriteNC::VarData>::iterator vit = varInfo.find(var_names[i]);
-    if (vit == varInfo.end())
-      ERRORR(MB_FAILURE, "Can't find variable requested.");
-
-    WriteNC::VarData& variableData = vit->second;
-
-    if (variableData.has_tsteps) {
-      // Time should be the first dimension
-      assert(tDim == variableData.varDims[0]);
-
-      // Assume this variable is on vertices for the time being
-      switch (variableData.entLoc) {
-        case WriteNC::ENTLOCVERT:
-          // Vertices
-          break;
-        default:
-          ERRORR(MB_FAILURE, "Unexpected entity location type for HOMME non-set variable.");
-      }
+  for (unsigned int i = 0; i < vdatas.size(); i++) {
+    WriteNC::VarData& variableData = vdatas[i];
+
+    // Time should be the first dimension
+    assert(tDim == variableData.varDims[0]);
+
+    // Assume this variable is on vertices for the time being
+    switch (variableData.entLoc) {
+      case WriteNC::ENTLOCVERT:
+        // Vertices
+        break;
+      default:
+        ERRORR(MB_FAILURE, "Unexpected entity location type for HOMME non-set variable.");
+    }
 
-      // A typical variable has 3 dimensions as (time, lev, ncol)
-      // At each timestep, we need to transpose tag format (ncol, lev) back
-      // to NC format (lev, ncol) for writing
-      for (unsigned int t = 0; t < tstep_nums.size(); t++) {
-        // We will write one time step, and count will be one; start will be different
-        // Use tag_get_data instead of tag_iterate to get values, as localVertsOwned
-        // might not be contiguous. We should also transpose for level so that means
-        // deep copy for transpose
-        variableData.writeStarts[0] = t; // This is time, again
-        std::vector<double> tag_data(num_local_verts_owned * variableData.numLev);
-        ErrorCode rval = mbImpl->tag_get_data(variableData.varTags[t], localVertsOwned, &tag_data[0]);
-        ERRORR(rval, "Trouble getting tag data on owned vertices.");
+    // A typical HOMME non-set variable has 3 dimensions as (time, lev, ncol)
+    // At each timestep, we need to transpose tag format (ncol, lev) back
+    // to NC format (lev, ncol) for writing
+    for (unsigned int t = 0; t < tstep_nums.size(); t++) {
+      // We will write one time step, and count will be one; start will be different
+      // Use tag_get_data instead of tag_iterate to get values, as localVertsOwned
+      // might not be contiguous. We should also transpose for level so that means
+      // deep copy for transpose
+      variableData.writeStarts[0] = t; // This is time, again
+      std::vector<double> tag_data(num_local_verts_owned * variableData.numLev);
+      ErrorCode rval = mbImpl->tag_get_data(variableData.varTags[t], localVertsOwned, &tag_data[0]);
+      ERRORR(rval, "Trouble getting tag data on owned vertices.");
 
 #ifdef PNETCDF_FILE
-        size_t nb_writes = localGidVertsOwned.psize();
-        std::vector<int> requests(nb_writes), statuss(nb_writes);
-        size_t idxReq = 0;
+      size_t nb_writes = localGidVertsOwned.psize();
+      std::vector<int> requests(nb_writes), statuss(nb_writes);
+      size_t idxReq = 0;
 #endif
 
-        // Now write from memory directly
-        switch (variableData.varDataType) {
-          case NC_DOUBLE: {
-            std::vector<double> tmpdoubledata(num_local_verts_owned * variableData.numLev);
-            // Transpose (ncol, lev) back to (lev, ncol)
-            jik_to_kji(num_local_verts_owned, 1, variableData.numLev, &tmpdoubledata[0], &tag_data[0]);
-
-            size_t indexInDoubleArray = 0;
-            size_t ic = 0;
-            for (Range::pair_iterator pair_iter = localGidVertsOwned.pair_begin();
-                pair_iter != localGidVertsOwned.pair_end(); ++pair_iter, ic++) {
-              EntityHandle starth = pair_iter->first;
-              EntityHandle endh = pair_iter->second;
-              variableData.writeStarts[2] = (NCDF_SIZE)(starth - 1);
-              variableData.writeCounts[2] = (NCDF_SIZE)(endh - starth + 1);
-
-              // Do a partial write, in each subrange
+      // Now write from memory directly
+      switch (variableData.varDataType) {
+        case NC_DOUBLE: {
+          std::vector<double> tmpdoubledata(num_local_verts_owned * variableData.numLev);
+          // Transpose (ncol, lev) back to (lev, ncol)
+          jik_to_kji(num_local_verts_owned, 1, variableData.numLev, &tmpdoubledata[0], &tag_data[0]);
+
+          size_t indexInDoubleArray = 0;
+          size_t ic = 0;
+          for (Range::pair_iterator pair_iter = localGidVertsOwned.pair_begin();
+              pair_iter != localGidVertsOwned.pair_end(); ++pair_iter, ic++) {
+            EntityHandle starth = pair_iter->first;
+            EntityHandle endh = pair_iter->second;
+            variableData.writeStarts[2] = (NCDF_SIZE)(starth - 1);
+            variableData.writeCounts[2] = (NCDF_SIZE)(endh - starth + 1);
+
+            // Do a partial write, in each subrange
 #ifdef PNETCDF_FILE
-              // Wait outside this loop
-              success = NCFUNCREQP(_vara_double)(_fileId, variableData.varId,
-                  &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
-                             &(tmpdoubledata[indexInDoubleArray]), &requests[idxReq++]);
+            // Wait outside this loop
+            success = NCFUNCREQP(_vara_double)(_fileId, variableData.varId,
+                &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
+                           &(tmpdoubledata[indexInDoubleArray]), &requests[idxReq++]);
 #else
-              success = NCFUNCAP(_vara_double)(_fileId, variableData.varId,
-                  &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
-                             &(tmpdoubledata[indexInDoubleArray]));
+            success = NCFUNCAP(_vara_double)(_fileId, variableData.varId,
+                &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
+                           &(tmpdoubledata[indexInDoubleArray]));
 #endif
-              ERRORS(success, "Failed to read double data in loop");
-              // We need to increment the index in double array for the
-              // next subrange
-              indexInDoubleArray += (endh - starth + 1) * variableData.numLev;
-            }
-            assert(ic == localGidVertsOwned.psize());
-#ifdef PNETCDF_FILE
-            success = ncmpi_wait_all(_fileId, requests.size(), &requests[0], &statuss[0]);
-            ERRORS(success, "Failed on wait_all.");
-#endif
-            break;
+            ERRORS(success, "Failed to read double data in loop");
+            // We need to increment the index in double array for the
+            // next subrange
+            indexInDoubleArray += (endh - starth + 1) * variableData.numLev;
           }
-          default:
-            ERRORR(MB_FAILURE, "Not implemented yet.");
-        }
-      }
-    } // if (variableData.has_tsteps)
-    else {
-      switch (variableData.varDataType) {
-        case NC_DOUBLE:
-          success = NCFUNCAP(_vara_double)(_fileId, variableData.varId, &variableData.writeStarts[0],
-                    &variableData.writeCounts[0], (double*)(variableData.memoryHogs[0]));
-          ERRORS(success, "Failed to write double data.");
-          break;
-        default:
-          ERRORR(MB_FAILURE, "Not implemented yet.");
-      }
-    }
-  }
-
-  // Write coordinates used by requested var_names
-  // Use independent I/O mode put, since this write is only for the root processor
-  // CAUTION: if the NetCDF ID is from a previous call to ncmpi_create rather than ncmpi_open,
-  // all processors need to call ncmpi_begin_indep_data(). If only the root processor does so,
-  // ncmpi_begin_indep_data() call will be blocked forever :(
+          assert(ic == localGidVertsOwned.psize());
 #ifdef PNETCDF_FILE
-  // Enter independent I/O mode
-  success = NCFUNC(begin_indep_data)(_fileId);
-  ERRORS(success, "Failed to begin independent I/O mode.");
+          success = ncmpi_wait_all(_fileId, requests.size(), &requests[0], &statuss[0]);
+          ERRORS(success, "Failed on wait_all.");
 #endif
-
-  int rank = 0;
-#ifdef USE_MPI
-  bool& isParallel = _writeNC->isParallel;
-  if (isParallel) {
-    ParallelComm*& myPcomm = _writeNC->myPcomm;
-    rank = myPcomm->proc_config().proc_rank();
-  }
-#endif
-  if (0 == rank) {
-    for (std::set<std::string>::iterator setIt = usedCoordinates.begin();
-        setIt != usedCoordinates.end(); ++setIt) {
-      const std::string& coordName = *setIt;
-
-      // Skip dummy coordinate variables (e.g. ncol)
-      if (dummyVarNames.find(coordName) != dummyVarNames.end())
-        continue;
-
-      std::map<std::string, WriteNC::VarData>::iterator vit = varInfo.find(coordName);
-      if (vit == varInfo.end())
-        ERRORR(MB_FAILURE, "Can't find one coordinate variable.");
-
-      WriteNC::VarData& varCoordData = vit->second;
-
-      switch (varCoordData.varDataType) {
-        case NC_DOUBLE:
-          // Independent I/O mode put
-          success = NCFUNCP(_vara_double)(_fileId, varCoordData.varId, &varCoordData.writeStarts[0],
-                    &varCoordData.writeCounts[0], (double*)(varCoordData.memoryHogs[0]));
-          ERRORS(success, "Failed to write double data.");
-          break;
-        case NC_INT:
-          // Independent I/O mode put
-          success = NCFUNCP(_vara_int)(_fileId, varCoordData.varId, &varCoordData.writeStarts[0],
-                    &varCoordData.writeCounts[0], (int*)(varCoordData.memoryHogs[0]));
-          ERRORS(success, "Failed to write int data.");
           break;
+        }
         default:
-          ERRORR(MB_FAILURE, "Not implemented yet.");
+          ERRORR(MB_NOT_IMPLEMENTED, "Writing with current data type not implemented yet.");
       }
     }
   }
 
-#ifdef PNETCDF_FILE
-  // End independent I/O mode
-  success = NCFUNC(end_indep_data)(_fileId);
-  ERRORS(success, "Failed to end independent I/O mode.");
-#endif
-
   return MB_SUCCESS;
 }
 

src/io/NCWriteHOMME.hpp

   //! Collect data for specified variables
   virtual ErrorCode collect_variable_data(std::vector<std::string>& var_names, std::vector<int>& tstep_nums);
 
-  //! Implementation of NCWriteHelper::write_values()
-  virtual ErrorCode write_values(std::vector<std::string>& var_names, std::vector<int>& tstep_nums);
+  //! Implementation of NCWriteHelper::write_nonset_variables()
+  virtual ErrorCode write_nonset_variables(std::vector<WriteNC::VarData>& vdatas, std::vector<int>& tstep_nums);
 };
 
 } // namespace moab

src/io/NCWriteHelper.cpp

 
     WriteNC::VarData& currentVarData = vit->second;
 
-    currentVarData.has_tsteps = false;
-    if ((std::find(currentVarData.varDims.begin(), currentVarData.varDims.end(), tDim) != currentVarData.varDims.end())
-        && (currentVarData.varDims.size() > 1)) // So it is not time itself
-      currentVarData.has_tsteps = true;
-
-    currentVarData.numLev = 1;
-    if ((std::find(currentVarData.varDims.begin(), currentVarData.varDims.end(), levDim) != currentVarData.varDims.end()))
-      currentVarData.numLev = nLevels;
-
     dbgOut.tprintf(2, "    for variable %s varDims.size %d \n", varname.c_str(), (int)currentVarData.varDims.size());
     for (size_t j = 0; j < currentVarData.varDims.size(); j++) {
       std::string dimName = dimNames[currentVarData.varDims[j]];
     if (usedCoordinates.find(varname) != usedCoordinates.end())
       continue;
 
-    // Process non-set variables with time steps
+    if (std::find(currentVarData.varDims.begin(), currentVarData.varDims.end(), tDim) != currentVarData.varDims.end())
+      currentVarData.has_tsteps = true;
+
+    if ((std::find(currentVarData.varDims.begin(), currentVarData.varDims.end(), levDim) != currentVarData.varDims.end()))
+      currentVarData.numLev = nLevels;
+
+    // Process non-coordinate variables with time steps
     if (currentVarData.has_tsteps) {
+      // It might be a set variable, e.g. xtime(Time) or xtime(Time, StrLen)
       for (unsigned int t = 0; t < tstep_nums.size(); t++) {
         Tag indexedTag = 0;
         std::stringstream ssTagNameWithIndex;
           currentVarData.varDataType = NC_INT;
       }
     }
-    // Process set variables that are not coordinate variables
+    // Process non-coordinate variables without time steps
     else {
+      // Should be a set variable
+      assert(WriteNC::ENTLOCSET == currentVarData.entLoc);
+
       // Get the tag with varname
       Tag tag = 0;
       rval = mbImpl->tag_get_handle(varname.c_str(), tag);
         currentVarData.writeCounts.push_back(1);
       }
       else {
-        assert(1 == currentVarData.varDims.size());
-        currentVarData.writeStarts.push_back(0);
-        currentVarData.writeCounts.push_back(dimLens[currentVarData.varDims[0]]);
+        for (size_t j = 0; j < currentVarData.varDims.size(); j++) {
+          currentVarData.writeStarts.push_back(0);
+          currentVarData.writeCounts.push_back(dimLens[currentVarData.varDims[j]]);
+        }
       }
     }
   }
 
-  // Process coordinate variables
+  // Process coordinate variables here
   // Check that for used coordinates we have found the tags
   for (std::set<std::string>::iterator setIt = usedCoordinates.begin();
       setIt != usedCoordinates.end(); ++setIt) {
         (int)variableData.varDims.size(), &(variableData.varDims[0]),
         &variableData.varId);
     if (errCode != NC_NOERR)
-      ERRORR(MB_FAILURE, "Failed to create coordinate variable.");
+      ERRORR(MB_FAILURE, "Failed to create requested variable.");
 
     dbgOut.tprintf(2, "    for variable %s with desired name %s variable id is %d \n", var_names[i].c_str(),
         desired_names[i].c_str(), variableData.varId);
   return MB_SUCCESS;
 }
 
+ErrorCode NCWriteHelper::write_values(std::vector<std::string>& var_names, std::vector<int>& tstep_nums)
+{
+  std::set<std::string>& usedCoordinates = _writeNC->usedCoordinates;
+  std::set<std::string>& dummyVarNames = _writeNC->dummyVarNames;
+  std::map<std::string, WriteNC::VarData>& varInfo = _writeNC->varInfo;
+
+  std::vector<WriteNC::VarData> vdatas;
+  std::vector<WriteNC::VarData> vsetdatas;
+
+  // For set variables, include coordinates used by requested var_names
+  for (std::set<std::string>::iterator setIt = usedCoordinates.begin();
+      setIt != usedCoordinates.end(); ++setIt) {
+    const std::string& coordName = *setIt;
+
+    // Skip dummy coordinate variables (if any)
+    if (dummyVarNames.find(coordName) != dummyVarNames.end())
+      continue;
+
+    std::map<std::string, WriteNC::VarData>::iterator vit = varInfo.find(coordName);
+    if (vit == varInfo.end()) {
+      ERRORR(MB_FAILURE, "Can't find one coordinate variable.");
+    }
+
+     vsetdatas.push_back(vit->second);
+  }
+
+  // Collect non-set and set variables from requested var_names
+  for (unsigned int i = 0; i < var_names.size(); i++) {
+    std::map<std::string, WriteNC::VarData>::iterator vit = varInfo.find(var_names[i]);
+    if (vit == varInfo.end()) {
+      ERRORR(MB_FAILURE, "Can't find variable requested.");
+    }
+
+    WriteNC::VarData& variableData = vit->second;
+    if (WriteNC::ENTLOCSET == variableData.entLoc) {
+      // Used coordinates has all ready been included
+      if (usedCoordinates.find(var_names[i]) != usedCoordinates.end())
+        continue;
+
+      vsetdatas.push_back(variableData);
+    }
+    else
+      vdatas.push_back(variableData);
+  }
+
+  // Assume that the data ranges do not overlap across processors
+  // While overlapped writing might still work, we should better not take that risk
+  write_nonset_variables(vdatas, tstep_nums);
+
+  // Use independent I/O mode put, since this write is only for the root processor
+  write_set_variables(vsetdatas, tstep_nums);
+
+  return MB_SUCCESS;
+}
+
+ErrorCode NCWriteHelper::write_set_variables(std::vector<WriteNC::VarData>& vsetdatas, std::vector<int>& /* tstep_nums */)
+{
+  int success;
+
+// CAUTION: if the NetCDF ID is from a previous call to ncmpi_create rather than ncmpi_open,
+// all processors need to call ncmpi_begin_indep_data(). If only the root processor does so,
+// ncmpi_begin_indep_data() call will be blocked forever :(
+ #ifdef PNETCDF_FILE
+   // Enter independent I/O mode
+   success = NCFUNC(begin_indep_data)(_fileId);
+   ERRORS(success, "Failed to begin independent I/O mode.");
+ #endif
+
+   int rank = 0;
+ #ifdef USE_MPI
+   bool& isParallel = _writeNC->isParallel;
+   if (isParallel) {
+     ParallelComm*& myPcomm = _writeNC->myPcomm;
+     rank = myPcomm->proc_config().proc_rank();
+   }
+ #endif
+   if (0 == rank) {
+     for (unsigned int i = 0; i < vsetdatas.size(); i++) {
+       WriteNC::VarData& variableData = vsetdatas[i];
+
+       // Set variables with timesteps, e.g. xtime(Time) or xtime(Time, StrLen)
+       if (variableData.has_tsteps) {
+         ERRORR(MB_NOT_IMPLEMENTED, "Writing set variables with timesteps not implemented yet.");
+       }
+
+       switch (variableData.varDataType) {
+         case NC_DOUBLE:
+           // Independent I/O mode put
+           success = NCFUNCP(_vara_double)(_fileId, variableData.varId, &variableData.writeStarts[0],
+                     &variableData.writeCounts[0], (double*)(variableData.memoryHogs[0]));
+           ERRORS(success, "Failed to write double data.");
+           break;
+         case NC_INT:
+           // Independent I/O mode put
+           success = NCFUNCP(_vara_int)(_fileId, variableData.varId, &variableData.writeStarts[0],
+                     &variableData.writeCounts[0], (int*)(variableData.memoryHogs[0]));
+           ERRORS(success, "Failed to write int data.");
+           break;
+         default:
+           ERRORR(MB_NOT_IMPLEMENTED, "Writing with current data type not implemented yet.");
+       }
+     }
+   }
+
+ #ifdef PNETCDF_FILE
+   // End independent I/O mode
+   success = NCFUNC(end_indep_data)(_fileId);
+   ERRORS(success, "Failed to end independent I/O mode.");
+ #endif
+
+   return MB_SUCCESS;
+}
+
 ErrorCode ScdNCWriteHelper::collect_mesh_info()
 {
   Interface*& mbImpl = _writeNC->mbImpl;
   return MB_SUCCESS;
 }
 
-// For parallel write, we assume that the data ranges do not overlap across processors
-// While overlapped writing might still work, we should better not take that risk
-// For CAM-EUL and CAM-FV variables on non-shared quads (e.g. T), this is not an issue
-// We assume that there are no variables on vertices and we do not support variables
-// on edges (e.g. US in CAM-FV) for the time being
-ErrorCode ScdNCWriteHelper::write_values(std::vector<std::string>& var_names, std::vector<int>& tstep_nums)
+// Write CAM-EUL and CAM-FV non-set variables on non-shared quads (e.g. T)
+// We assume that there are no variables on vertices and we do not support
+// variables on edges (e.g. US in CAM-FV) for the time being
+ErrorCode ScdNCWriteHelper::write_nonset_variables(std::vector<WriteNC::VarData>& vdatas, std::vector<int>& tstep_nums)
 {
   Interface*& mbImpl = _writeNC->mbImpl;
-  std::set<std::string>& usedCoordinates = _writeNC->usedCoordinates;
-  std::set<std::string>& dummyVarNames = _writeNC->dummyVarNames;
-  std::map<std::string, WriteNC::VarData>& varInfo = _writeNC->varInfo;
 
   int success;
 
-  // Now look at requested var_names; if they have time, we will have a list, and write one at a time
-  // If not, just write regularly
-  // Use collective I/O mode put (synchronous write) for the time being, we can try nonblocking put
-  // (request aggregation) later
-  for (size_t i = 0; i < var_names.size(); i++) {
-    std::map<std::string, WriteNC::VarData>::iterator vit = varInfo.find(var_names[i]);
-    if (vit == varInfo.end())
-      ERRORR(MB_FAILURE, "Can't find variable requested.");
+  for (unsigned int i = 0; i < vdatas.size(); i++) {
+    WriteNC::VarData& variableData = vdatas[i];
 
-    WriteNC::VarData& variableData = vit->second;
-    if (variableData.has_tsteps) {
-      // Time should be the first dimension
-      assert(tDim == variableData.varDims[0]);
+    // Time should be the first dimension
+    assert(tDim == variableData.varDims[0]);
 
-      // Assume this variable is on faces for the time being
-      switch (variableData.entLoc) {
-        case WriteNC::ENTLOCFACE:
-          // Faces
-          break;
-        default:
-          ERRORR(MB_FAILURE, "Unexpected entity location type for structured mesh non-set variable.");
-      }
-
-      // A typical variable has 4 dimensions as (time, lev, lat, lon)
-      // At each timestep, we need to transpose tag format (lat, lon, lev) back
-      // to NC format (lev, lat, lon) for writing
-      size_t ni = variableData.writeCounts[3]; // lon
-      size_t nj = variableData.writeCounts[2]; // lat
-      size_t nk = variableData.writeCounts[1]; // lev
-
-      variableData.writeCounts[0] = 1; // We will write one time step
-
-      for (unsigned int t = 0; t < tstep_nums.size(); t++) {
-        // We will write one time step, and count will be one; start will be different
-        // We will write values directly from tag_iterate, but we should also transpose for level
-        // so that means deep copy for transpose
-        variableData.writeStarts[0] = t; // This is time, again
-        int count;
-        void* dataptr;
-        ErrorCode rval = mbImpl->tag_iterate(variableData.varTags[t], localCellsOwned.begin(), localCellsOwned.end(), count, dataptr);
-        ERRORR(rval, "Failed to get tag iterator on owned faces.");
-        assert(count == (int)localCellsOwned.size());
-
-        // Now write from memory directly
-        switch (variableData.varDataType) {
-          case NC_DOUBLE: {
-            std::vector<double> tmpdoubledata(ni*nj*nk);
-            // Transpose (lat, lon, lev) back to (lev, lat, lon)
-            jik_to_kji(ni, nj, nk, &tmpdoubledata[0], (double*)(dataptr));
-            success = NCFUNCAP(_vara_double)(_fileId, variableData.varId,
-                      &variableData.writeStarts[0], &variableData.writeCounts[0],
-                      &tmpdoubledata[0]);
-            ERRORS(success, "Failed to write double data.");
-            break;
-          }
-          default:
-            ERRORR(MB_FAILURE, "Not implemented yet.");
-        }
-      }
-    }
-    else {
-      switch (variableData.varDataType) {
-        case NC_DOUBLE:
-          success = NCFUNCAP(_vara_double)(_fileId, variableData.varId, &variableData.writeStarts[0],
-                    &variableData.writeCounts[0], (double*)(variableData.memoryHogs[0]));
-          ERRORS(success, "Failed to write double data.");
-          break;
-        default:
-          ERRORR(MB_FAILURE, "Not implemented yet.");
-      }
+    // Assume this variable is on faces for the time being
+    switch (variableData.entLoc) {
+      case WriteNC::ENTLOCFACE:
+        // Faces
+        break;
+      default:
+        ERRORR(MB_FAILURE, "Unexpected entity location type for structured mesh non-set variable.");
     }
-  }
-
-  // Write coordinates used by requested var_names
-  // Use independent I/O mode put, since this write is only for the root processor
-  // CAUTION: if the NetCDF ID is from a previous call to ncmpi_create rather than ncmpi_open,
-  // all processors need to call ncmpi_begin_indep_data(). If only the root processor does so,
-  // ncmpi_begin_indep_data() call will be blocked forever :(
-#ifdef PNETCDF_FILE
-  // Enter independent I/O mode
-  success = NCFUNC(begin_indep_data)(_fileId);
-  ERRORS(success, "Failed to begin independent I/O mode.");
-#endif
-
-  int rank = 0;
-#ifdef USE_MPI
-  bool& isParallel = _writeNC->isParallel;
-  if (isParallel) {
-    ParallelComm*& myPcomm = _writeNC->myPcomm;
-    rank = myPcomm->proc_config().proc_rank();
-  }
-#endif
-  if (0 == rank) {
-    for (std::set<std::string>::iterator setIt = usedCoordinates.begin();
-        setIt != usedCoordinates.end(); ++setIt) {
-      const std::string& coordName = *setIt;
-
-      // Skip dummy coordinate variables (if any)
-      if (dummyVarNames.find(coordName) != dummyVarNames.end())
-        continue;
-
-      std::map<std::string, WriteNC::VarData>::iterator vit = varInfo.find(coordName);
-      if (vit == varInfo.end())
-        ERRORR(MB_FAILURE, "Can't find one coordinate variable.");
 
-      WriteNC::VarData& varCoordData = vit->second;
-
-      switch (varCoordData.varDataType) {
-        case NC_DOUBLE:
-          // Independent I/O mode put
-          success = NCFUNCP(_vara_double)(_fileId, varCoordData.varId, &varCoordData.writeStarts[0],
-                    &varCoordData.writeCounts[0], (double*)(varCoordData.memoryHogs[0]));
+    // A typical CAM-EUL or CAM-FV non-set variable has 4 dimensions as (time, lev, lat, lon)
+    // At each timestep, we need to transpose tag format (lat, lon, lev) back
+    // to NC format (lev, lat, lon) for writing
+    size_t ni = variableData.writeCounts[3]; // lon
+    size_t nj = variableData.writeCounts[2]; // lat
+    size_t nk = variableData.writeCounts[1]; // lev
+
+    variableData.writeCounts[0] = 1; // We will write one time step
+
+    for (unsigned int t = 0; t < tstep_nums.size(); t++) {
+      // We will write one time step, and count will be one; start will be different
+      // We should also transpose for level so that means deep copy for transpose
+      variableData.writeStarts[0] = t; // This is start for time
+      int count;
+      void* dataptr;
+      ErrorCode rval = mbImpl->tag_iterate(variableData.varTags[t], localCellsOwned.begin(), localCellsOwned.end(), count, dataptr);
+      ERRORR(rval, "Failed to get tag iterator on owned faces.");
+      assert(count == (int)localCellsOwned.size());
+
+      // Use collective I/O mode put (synchronous write) for the time being, we can try nonblocking put
+      // (request aggregation) later
+      switch (variableData.varDataType) {
+        case NC_DOUBLE: {
+          std::vector<double> tmpdoubledata(ni*nj*nk);
+          // Transpose (lat, lon, lev) back to (lev, lat, lon)
+          jik_to_kji(ni, nj, nk, &tmpdoubledata[0], (double*)(dataptr));
+          success = NCFUNCAP(_vara_double)(_fileId, variableData.varId,
+                    &variableData.writeStarts[0], &variableData.writeCounts[0],
+                    &tmpdoubledata[0]);
           ERRORS(success, "Failed to write double data.");
           break;
-        case NC_INT:
-          // Independent I/O mode put
-          success = NCFUNCP(_vara_int)(_fileId, varCoordData.varId, &varCoordData.writeStarts[0],
-                    &varCoordData.writeCounts[0], (int*)(varCoordData.memoryHogs[0]));
-          ERRORS(success, "Failed to write int data.");
-          break;
+        }
         default:
-          ERRORR(MB_FAILURE, "Not implemented yet.");
+          ERRORR(MB_NOT_IMPLEMENTED, "Writing with current data type not implemented yet.");
       }
     }
   }
 
-#ifdef PNETCDF_FILE
-  // End independent I/O mode
-  success = NCFUNC(end_indep_data)(_fileId);
-  ERRORS(success, "Failed to end independent I/O mode.");
-#endif
-
   return MB_SUCCESS;
 }
 

src/io/NCWriteHelper.hpp

   //! Get appropriate helper instance for WriteNC class based on some info in the file set
   static NCWriteHelper* get_nc_helper(WriteNC* writeNC, int fileId, const FileOptions& opts, EntityHandle fileSet);
 
-  //! Collect necessary info about local mesh
+  //! Collect necessary info about local mesh (implemented in child classes)
   virtual ErrorCode collect_mesh_info() = 0;
 
-  //! Collect data for specified variables
+  //! Collect data for specified variables (partially implemented in child classes)
   virtual ErrorCode collect_variable_data(std::vector<std::string>& var_names, std::vector<int>& tstep_nums);
 
-  //! Take the info from VarData and write first the coordinates, then the actual variables
-  virtual ErrorCode write_values(std::vector<std::string>& var_names, std::vector<int>& tstep_nums) = 0;
-
   //! Initialize file: this is where all defines are done
   //! The VarData dimension ids are filled up after define
   ErrorCode init_file(std::vector<std::string>& var_names, std::vector<std::string>& desired_names, bool _append);
 
+  //! Take the info from VarData and write first non-set variables, then set variables
+  ErrorCode write_values(std::vector<std::string>& var_names, std::vector<int>& tstep_nums);
+
+private:
+  // Write set variables (common to scd mesh and ucd mesh)
+  ErrorCode write_set_variables(std::vector<WriteNC::VarData>& vsetdatas, std::vector<int>& tstep_nums);
+
 protected:
+  // Write non-set variables (implemented in child classes)
+  virtual ErrorCode write_nonset_variables(std::vector<WriteNC::VarData>& vdatas, std::vector<int>& tstep_nums) = 0;
+
   template <typename T> void jik_to_kji(size_t ni, size_t nj, size_t nk, T* dest, T* source)
   {
     size_t nik = ni * nk, nij = ni * nj;
   //! Collect data for specified variables
   virtual ErrorCode collect_variable_data(std::vector<std::string>& var_names, std::vector<int>& tstep_nums);
 
-  //! Implementation of NCWriteHelper::write_values()
-  virtual ErrorCode write_values(std::vector<std::string>& var_names, std::vector<int>& tstep_nums);
+  //! Implementation of NCWriteHelper::write_nonset_variables()
+  virtual ErrorCode write_nonset_variables(std::vector<WriteNC::VarData>& vdatas, std::vector<int>& tstep_nums);
 
 protected:
   //! Dimensions of my local part of grid

src/io/NCWriteMPAS.cpp

   return MB_SUCCESS;
 }
 
-ErrorCode NCWriteMPAS::write_values(std::vector<std::string>& var_names, std::vector<int>& tstep_nums)
+ErrorCode NCWriteMPAS::write_nonset_variables(std::vector<WriteNC::VarData>& vdatas, std::vector<int>& tstep_nums)
 {
   Interface*& mbImpl = _writeNC->mbImpl;
-  std::set<std::string>& usedCoordinates = _writeNC->usedCoordinates;
-  std::set<std::string>& dummyVarNames = _writeNC->dummyVarNames;
-  std::map<std::string, WriteNC::VarData>& varInfo = _writeNC->varInfo;
 
   int success;
   Range* pLocalEntsOwned = NULL;
   // Now look at requested var_names; if they have time, we will have a list, and write one at a time
   // For each variable tag in the indexed lists, write a time step data
   // Assume the first dimension is time (need to check); if not, just write regularly
-  for (size_t i = 0; i < var_names.size(); i++) {
-    std::map<std::string, WriteNC::VarData>::iterator vit = varInfo.find(var_names[i]);
-    if (vit == varInfo.end())
-      ERRORR(MB_FAILURE, "Can't find variable requested.");
-
-    WriteNC::VarData& variableData = vit->second;
+  for (unsigned int i = 0; i < vdatas.size(); i++) {
+    WriteNC::VarData& variableData = vdatas[i];
 
     // Skip edge variables, if there are no edges
     if (localEdgesOwned.empty() && variableData.entLoc == WriteNC::ENTLOCEDGE)
       continue;
 
-    if (variableData.has_tsteps) {
-      // Time should be the first dimension
-      assert(tDim == variableData.varDims[0]);
-
-      // Get local owned entities of this variable
-      switch (variableData.entLoc) {
-        case WriteNC::ENTLOCVERT:
-          // Vertices
-          pLocalEntsOwned = &localVertsOwned;
-          pLocalGidEntsOwned = &localGidVertsOwned;
-          break;
-        case WriteNC::ENTLOCEDGE:
-          // Edges
-          pLocalEntsOwned = &localEdgesOwned;
-          pLocalGidEntsOwned = &localGidEdgesOwned;
-          break;
-        case WriteNC::ENTLOCFACE:
-          // Cells
-          pLocalEntsOwned = &localCellsOwned;
-          pLocalGidEntsOwned = &localGidCellsOwned;
-          break;
-        default:
-          ERRORR(MB_FAILURE, "Unexpected entity location type for MPAS non-set variable.");
-      }
+    // Time should be the first dimension
+    assert(tDim == variableData.varDims[0]);
+
+    // Get local owned entities of this variable
+    switch (variableData.entLoc) {
+      case WriteNC::ENTLOCVERT:
+        // Vertices
+        pLocalEntsOwned = &localVertsOwned;
+        pLocalGidEntsOwned = &localGidVertsOwned;
+        break;
+      case WriteNC::ENTLOCEDGE:
+        // Edges
+        pLocalEntsOwned = &localEdgesOwned;
+        pLocalGidEntsOwned = &localGidEdgesOwned;
+        break;
+      case WriteNC::ENTLOCFACE:
+        // Cells
+        pLocalEntsOwned = &localCellsOwned;
+        pLocalGidEntsOwned = &localGidCellsOwned;
+        break;
+      default:
+        ERRORR(MB_FAILURE, "Unexpected entity location type for MPAS non-set variable.");
+    }
 
-      // A typical variable has 3 dimensions as (Time, nCells, nVertLevels)
-      for (unsigned int t = 0; t < tstep_nums.size(); t++) {
-        // We will write one time step, and count will be one; start will be different
-        // Use tag_get_data instead of tag_iterate to get values, as localEntsOwned
-        // might not be contiguous.
-        variableData.writeStarts[0] = t; // This is time, again
-        std::vector<double> tag_data(pLocalEntsOwned->size() * variableData.numLev);
-        ErrorCode rval = mbImpl->tag_get_data(variableData.varTags[t], *pLocalEntsOwned, &tag_data[0]);
-        ERRORR(rval, "Trouble getting tag data on owned vertices.");
+    // A typical variable has 3 dimensions as (Time, nCells, nVertLevels)
+    for (unsigned int t = 0; t < tstep_nums.size(); t++) {
+      // We will write one time step, and count will be one; start will be different
+      // Use tag_get_data instead of tag_iterate to get values, as localEntsOwned
+      // might not be contiguous.
+      variableData.writeStarts[0] = t; // This is time, again
+      std::vector<double> tag_data(pLocalEntsOwned->size() * variableData.numLev);
+      ErrorCode rval = mbImpl->tag_get_data(variableData.varTags[t], *pLocalEntsOwned, &tag_data[0]);
+      ERRORR(rval, "Trouble getting tag data on owned vertices.");
 
 #ifdef PNETCDF_FILE
-        size_t nb_writes = pLocalGidEntsOwned->psize();
-        std::vector<int> requests(nb_writes), statuss(nb_writes);
-        size_t idxReq = 0;
+      size_t nb_writes = pLocalGidEntsOwned->psize();
+      std::vector<int> requests(nb_writes), statuss(nb_writes);
+      size_t idxReq = 0;
 #endif
 
-        // Now write from memory directly
-        switch (variableData.varDataType) {
-          case NC_DOUBLE: {
-            size_t indexInDoubleArray = 0;
-            size_t ic = 0;
-            for (Range::pair_iterator pair_iter = pLocalGidEntsOwned->pair_begin();
-                pair_iter != pLocalGidEntsOwned->pair_end(); ++pair_iter, ic++) {
-              EntityHandle starth = pair_iter->first;
-              EntityHandle endh = pair_iter->second;
-              variableData.writeStarts[1] = (NCDF_SIZE)(starth - 1);
-              variableData.writeCounts[1] = (NCDF_SIZE)(endh - starth + 1);
-
-              // Do a partial write, in each subrange
+      // Now write from memory directly
+      switch (variableData.varDataType) {
+        case NC_DOUBLE: {
+          size_t indexInDoubleArray = 0;
+          size_t ic = 0;
+          for (Range::pair_iterator pair_iter = pLocalGidEntsOwned->pair_begin();
+              pair_iter != pLocalGidEntsOwned->pair_end(); ++pair_iter, ic++) {
+            EntityHandle starth = pair_iter->first;
+            EntityHandle endh = pair_iter->second;
+            variableData.writeStarts[1] = (NCDF_SIZE)(starth - 1);
+            variableData.writeCounts[1] = (NCDF_SIZE)(endh - starth + 1);
+
+            // Do a partial write, in each subrange
 #ifdef PNETCDF_FILE
-              // Wait outside this loop
-              success = NCFUNCREQP(_vara_double)(_fileId, variableData.varId,
-                  &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
-                             &(tag_data[indexInDoubleArray]), &requests[idxReq++]);
+            // Wait outside this loop
+            success = NCFUNCREQP(_vara_double)(_fileId, variableData.varId,
+                &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
+                           &(tag_data[indexInDoubleArray]), &requests[idxReq++]);
 #else
-              success = NCFUNCAP(_vara_double)(_fileId, variableData.varId,
-                  &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
-                             &(tag_data[indexInDoubleArray]));
-#endif
-              ERRORS(success, "Failed to read double data in loop");
-              // We need to increment the index in double array for the
-              // next subrange
-              indexInDoubleArray += (endh - starth + 1) * variableData.numLev;
-            }
-            assert(ic == pLocalGidEntsOwned->psize());
-#ifdef PNETCDF_FILE
-            success = ncmpi_wait_all(_fileId, requests.size(), &requests[0], &statuss[0]);
-            ERRORS(success, "Failed on wait_all.");
+            success = NCFUNCAP(_vara_double)(_fileId, variableData.varId,
+                &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
+                           &(tag_data[indexInDoubleArray]));
 #endif
-            break;
+            ERRORS(success, "Failed to read double data in loop");
+            // We need to increment the index in double array for the
+            // next subrange
+            indexInDoubleArray += (endh - starth + 1) * variableData.numLev;
           }
-          default:
-            ERRORR(MB_FAILURE, "Not implemented yet.");
-        }
-      }
-    } // if (variableData.has_tsteps)
-    else {
-      switch (variableData.varDataType) {
-        case NC_DOUBLE:
-          success = NCFUNCAP(_vara_double)(_fileId, variableData.varId, &variableData.writeStarts[0],
-                    &variableData.writeCounts[0], (double*)(variableData.memoryHogs[0]));
-          ERRORS(success, "Failed to write double data.");
-          break;
-        default:
-          ERRORR(MB_FAILURE, "Not implemented yet.");
-      }
-    }
-  }
-
-  // Write coordinates used by requested var_names
-  // Use independent I/O mode put, since this write is only for the root processor
-  // CAUTION: if the NetCDF ID is from a previous call to ncmpi_create rather than ncmpi_open,
-  // all processors need to call ncmpi_begin_indep_data(). If only the root processor does so,
-  // ncmpi_begin_indep_data() call will be blocked forever :(
+          assert(ic == pLocalGidEntsOwned->psize());
 #ifdef PNETCDF_FILE
-  // Enter independent I/O mode
-  success = NCFUNC(begin_indep_data)(_fileId);
-  ERRORS(success, "Failed to begin independent I/O mode.");
+          success = ncmpi_wait_all(_fileId, requests.size(), &requests[0], &statuss[0]);
+          ERRORS(success, "Failed on wait_all.");
 #endif
-
-  int rank = 0;
-#ifdef USE_MPI
-  bool& isParallel = _writeNC->isParallel;
-  if (isParallel) {
-    ParallelComm*& myPcomm = _writeNC->myPcomm;
-    rank = myPcomm->proc_config().proc_rank();
-  }
-#endif
-  if (0 == rank) {
-    for (std::set<std::string>::iterator setIt = usedCoordinates.begin();
-        setIt != usedCoordinates.end(); ++setIt) {
-      const std::string& coordName = *setIt;
-
-      // Skip dummy coordinate variables (e.g. ncol)
-      if (dummyVarNames.find(coordName) != dummyVarNames.end())
-        continue;
-
-      std::map<std::string, WriteNC::VarData>::iterator vit = varInfo.find(coordName);
-      if (vit == varInfo.end())
-        ERRORR(MB_FAILURE, "Can't find one coordinate variable.");
-
-      WriteNC::VarData& varCoordData = vit->second;
-
-      switch (varCoordData.varDataType) {
-        case NC_DOUBLE:
-          // Independent I/O mode put
-          success = NCFUNCP(_vara_double)(_fileId, varCoordData.varId, &varCoordData.writeStarts[0],
-                    &varCoordData.writeCounts[0], (double*)(varCoordData.memoryHogs[0]));
-          ERRORS(success, "Failed to write double data.");
-          break;
-        case NC_INT:
-          // Independent I/O mode put
-          success = NCFUNCP(_vara_int)(_fileId, varCoordData.varId, &varCoordData.writeStarts[0],
-                    &varCoordData.writeCounts[0], (int*)(varCoordData.memoryHogs[0]));
-          ERRORS(success, "Failed to write int data.");
           break;
+        }
         default:
-          ERRORR(MB_FAILURE, "Not implemented yet.");
+          ERRORR(MB_NOT_IMPLEMENTED, "Writing with current data type not implemented yet.");
       }
     }
   }
 
-#ifdef PNETCDF_FILE
-  // End independent I/O mode
-  success = NCFUNC(end_indep_data)(_fileId);
-  ERRORS(success, "Failed to end independent I/O mode.");
-#endif
-
   return MB_SUCCESS;
 }
 

src/io/NCWriteMPAS.hpp

   //! Collect data for specified variables
   virtual ErrorCode collect_variable_data(std::vector<std::string>& var_names, std::vector<int>& tstep_nums);
 
-  //! Implementation of NCWriteHelper::write_values()
-  virtual ErrorCode write_values(std::vector<std::string>& var_names, std::vector<int>& tstep_nums);
+  //! Implementation of NCWriteHelper::write_nonset_variables()
+  virtual ErrorCode write_nonset_variables(std::vector<WriteNC::VarData>& vdatas, std::vector<int>& tstep_nums);
 };
 
 } // namespace moab