Sam  Leitner avatar Sam Leitner committed e7086b9

really messy commit, but uses callback from artio to pull positions into the oct_handler; a lot to resolve and clean up here...

Comments (0)

Files changed (17)

 detailed-errors=1
 where=yt
 exclude=answer_testing
-with-xunit=1
-#with-answer-testing=1
-#answer-compare=gold001
+with-xunit=1

yt/frontends/artio/_artio_caller.pyx

 
 from libc.stdint cimport int32_t, int64_t
 from libc.stdlib cimport malloc, free
+#from  data_structures  import cell_pos_callback
+import  data_structures  
 
 cdef struct artio_context_struct: 
     int comm
 cdef artio_context artio_context_global
 
 
+cdef extern from "sfc.h":
+    ctypedef void sfc_coords( int index, int coords[3], int )
+
 cdef extern from "artio.h":
     ctypedef struct artio_file_struct "artio_file_struct" :
         pass
     ctypedef struct artio_context_struct "artio_context_struct" :
         pass
     ctypedef artio_context_struct *artio_context "artio_context"
+
+
     cdef artio_context artio_context_global
+    
 
     # open modes
     cdef int ARTIO_OPEN_HEADER "ARTIO_OPEN_HEADER"
 
     cdef int ARTIO_PARAMETER_EXHAUSTED "ARTIO_PARAMETER_EXHAUSTED"
 
+    # grid read options
+    cdef int ARTIO_READ_LEAFS "ARTIO_READ_LEAFS"
+    cdef int ARTIO_READ_REFINED "ARTIO_READ_REFINED"
+    cdef int ARTIO_READ_ALL "ARTIO_READ_ALL"
+    
+
     # errors
     cdef int ARTIO_SUCCESS "ARTIO_SUCCESS"
+    cdef int ARTIO_ERR_MEMORY_ALLOCATION "ARTIO_ERR_MEMORY_ALLOCATION"
 
     artio_file artio_fileset_open(char *file_prefix, int type, artio_context context )
     int artio_fileset_close( artio_file handle )
-    int artio_fileset_open_grid( artio_file )
-    int artio_fileset_open_particle( artio_file )
+    int artio_fileset_open_particle( artio_file handle )
+    int artio_fileset_open_grid(artio_file handle) 
+    int artio_fileset_close_grid(artio_file handle) 
 
     # parameter functions
     int artio_parameter_iterate( artio_file handle, char *key, int *type, int *length )
     int artio_parameter_get_long_array(artio_file handle, char * key, int length, int64_t *values)
     int artio_parameter_get_double_array(artio_file handle, char * key, int length, double *values)
     int artio_parameter_get_string_array(artio_file handle, char * key, int length, char **values, int max_length)
+    int artio_grid_read_root_cell_end(artio_file handle)
+    int artio_grid_read_root_nocts(artio_file handle, int64_t sfc,\
+                                  float *variables, int32_t *num_oct_levels, int32_t *num_octs_per_level)
 
+    ctypedef void (* GridCallBackPos)(float * variables, int level, int refined, int64_t sfc_index, double pos[3])
+    int artio_grid_read_sfc_range_pos(artio_file handle,\
+                int64_t sfc1, int64_t sfc2,\
+                int min_level_to_read, int max_level_to_read,\
+                int options,\
+                GridCallBackPos callback)
+    ctypedef void (* GridCallBack)(float * variables, int level, int refined,int64_t sfc_index)
+    int artio_grid_read_sfc_range(artio_file handle,\
+                int64_t sfc1, int64_t sfc2,\
+                int min_level_to_read, int max_level_to_read,\
+                int options,\
+                GridCallBack callback)
+    
 ########## wrappers calling c
 cdef extern from "artio.c":
     artio_file artio_fileset_open(char * file_prefix, int type, artio_context context) 
 
-cpdef read_header(char * file_prefix, int type, rheader) :
-    cdef artio_file junk
-    cdef artio_context context=NULL
-    print "file_prefix=",file_prefix,"type=",type
-    junk = artio_fileset_open(file_prefix, type, context)
 
+cdef class read_parameters_artio : 
+    cdef public object parameters 
+    cdef artio_file handle
 
-########## wrappers called by python
-cdef class artio_fileset :
-    cdef artio_file handle
-    cdef public object parameters
- 
-    def __init__(self, char *file_prefix) :
-        self.handle = artio_fileset_open( file_prefix, ARTIO_OPEN_HEADER, artio_context_global )
+    def __init__(self, char *file_prefix, int artio_type) :
+        self.handle = artio_fileset_open( file_prefix, artio_type, artio_context_global ) 
         self.read_parameters()
+        artio_fileset_close(self.handle)  #snl why didn't Doug close?
 
     def read_parameters(self) :
         cdef char key[64]
 
             self.parameters[key] = parameter
 
+#        print self.parameters
+        
+#    def read_parameters(self) : 
+#        cdef char key[64]
+#        cdef int type
+#        cdef int length
+#        while artio_parameter_iterate( self.handle, key, &type, &length ) == ARTIO_SUCCESS :
+#            print 'hi!!'
+
+cdef class artio_fileset :
+    cdef public object parameters 
+    cdef artio_file handle
+    def __init__(self, char *file_prefix) :
+        cdef int artio_type = ARTIO_OPEN_HEADER
+        self.handle = artio_fileset_open( file_prefix, artio_type, artio_context_global ) 
+        d = read_parameters_artio(file_prefix, artio_type)
+        self.parameters = {}
+        self.parameters = d.parameters
         print self.parameters
-            
+        print 'done reading header parameters'
 
+cdef class artio_fileset_grid :
+    cdef public object parameters #is self.parameters public or is parameters public?
+    cdef artio_file handle
+
+    def __init__(self, char *file_prefix) :
+        cdef int artio_type = ARTIO_OPEN_GRID
+        self.handle = artio_fileset_open( file_prefix, artio_type, artio_context_global ) 
+        artio_fileset_open_grid( self.handle ) 
+        d = read_parameters_artio(file_prefix, artio_type)
+        self.parameters = {}
+        self.parameters = d.parameters
+#        print self.parameters
+        print 'done reading grid parameters'
+
+###### callback for positions #############
+def count_octs(char *file_prefix,\
+                   int64_t sfc1, int64_t sfc2,\
+                   int min_level_to_read, int max_level_to_read,\
+                   int num_grid_variables
+               ) :
+    #max_level_to_read is currently unused, but could be useful
+    cdef artio_file handle
+    handle = artio_fileset_open( file_prefix, ARTIO_OPEN_GRID, artio_context_global ) 
+    artio_fileset_open_grid( handle ) 
+    num_total_octs = 0 
+    cdef float * variables  
+    cdef int32_t * num_oct_levels 
+    cdef int32_t * num_octs_per_level 
+    length = num_grid_variables * 8
+    variables = <float *>malloc(length*sizeof(float))
+    num_oct_levels = <int32_t *>malloc(1*sizeof(int32_t))
+    length = max_level_to_read
+    num_octs_per_level = <int32_t *>malloc(length*sizeof(int32_t))
+    
+    for sfc in xrange(sfc1,sfc2):
+        artio_grid_read_root_nocts(handle,\
+                                  sfc,\
+                                  variables, num_oct_levels,\
+                                  num_octs_per_level)
+        count_level_octs = {}          
+        count_level_octs = [ num_octs_per_level[i] for i in xrange(min(max_level_to_read,1)) ]
+        num_sfc_octs = sum(count_level_octs)
+        num_total_octs += num_sfc_octs
+
+    artio_grid_read_root_cell_end(handle)
+    return num_total_octs
+
+###### callback for positions #############
+def grid_pos_fill(char *file_prefix,\
+                int64_t sfc1, int64_t sfc2,\
+                int min_level_to_read, int max_level_to_read) :
+    cdef artio_file handle
+    handle = artio_fileset_open( file_prefix, ARTIO_OPEN_GRID, artio_context_global ) 
+    artio_fileset_open_grid( handle ) 
+    status = artio_grid_read_sfc_range_pos(handle,\
+                sfc1, sfc2,\
+                min_level_to_read, max_level_to_read,\
+                ARTIO_READ_LEAFS,\
+                wrap_cell_pos_callback)
+    if status != ARTIO_SUCCESS :
+        print "exiting sfc range read with error status", status
+        if status == ARTIO_ERR_MEMORY_ALLOCATION :
+            print "cannot allocate enough memory in one sfc range,",\
+                "try loading smaller pieces"
+
+cdef void wrap_cell_pos_callback(float *variables, int level, int refined, int64_t sfc_index, double *pos):
+    position = {}
+    position = [ pos[i] for i in range(3) ]
+    data_structures.cell_pos_callback(level, refined, sfc_index, position)
+
+###### callback for variables #############
+def grid_var_fill(char *file_prefix,\
+                int64_t sfc1, int64_t sfc2,\
+                int min_level_to_read, int max_level_to_read) :
+    cdef artio_file handle
+    handle = artio_fileset_open( file_prefix, ARTIO_OPEN_GRID, artio_context_global ) 
+    artio_fileset_open_grid( handle ) 
+    artio_grid_read_sfc_range(handle,\
+                sfc1, sfc2,\
+                min_level_to_read, max_level_to_read,\
+                ARTIO_READ_LEAFS,\
+                wrap_cell_var_callback)
+def cell_var_callback(level, refined, sfc_index, cell_var):
+    print "variable callback success! ",level, refined, sfc_index 
+cdef void wrap_cell_var_callback(float *variables, int level, int refined, int64_t sfc_index):
+    cell_var={}
+#     cdef int num_grid_variables=1 # really from read_parameters_artio(file_prefix, artio_type)
+#        for label in parameters.grid_variable_labels
+#            cell_var = [ variables[i] for i in range(num_grid_variables) ]
+#            if(label == 'density'): 
+#                cell_var_callback(cell_var, level, refined, sfc_index)
+    cell_var_callback(level, refined, sfc_index, cell_var)
+ 
 def artio_is_valid( char *file_prefix ) :
     cdef artio_file handle = artio_fileset_open( file_prefix, 
             ARTIO_OPEN_HEADER, artio_context_global )

yt/frontends/artio/artio_headers/artio.c

-/Users/sleitner/repos/svntrunk_temp/src/tools/artio/artio.c
+/*
+ * artio.c
+ *
+ *  Created on: Feb 21, 2010
+ *  Author: Yongen Yu
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <math.h>
+
+#include "artio.h"
+#include "artio_internal.h"
+
+artio_file artio_fileset_allocate( char *file_prefix, int mode,
+        artio_context context );
+void artio_fileset_destroy( artio_file handle );
+
+artio_file artio_fileset_open(char * file_prefix, int type, artio_context context) {
+	artio_fh head_fh;
+	char filename[256];
+	int ret;
+
+	artio_file handle = 
+		artio_fileset_allocate( file_prefix, ARTIO_FILESET_READ, context );
+	if ( handle == NULL ) {
+		return NULL;
+	}
+
+	/* open header file */
+	sprintf(filename, "%s.art", handle->file_prefix);
+	head_fh = artio_file_fopen(filename, 
+			ARTIO_MODE_READ | ARTIO_MODE_ACCESS | ARTIO_MODE_DIRECT, context);
+
+	if ( head_fh == NULL ) {
+		artio_fileset_destroy(handle);
+		return NULL;
+	}
+
+	ret = artio_parameter_read(head_fh, &handle->param_list);
+	if ( ret != ARTIO_SUCCESS ) {
+		artio_fileset_destroy(handle);
+		return NULL;
+	}
+
+	artio_file_fclose(head_fh);
+
+	artio_parameter_get_long(handle, "num_root_cells", &handle->num_root_cells);
+
+	/* default to accessing all sfc indices */
+	handle->proc_sfc_begin = 0;
+	handle->proc_sfc_end = handle->num_root_cells-1;
+
+	/* open data files */
+	if (type & ARTIO_OPEN_PARTICLES) {
+		ret = artio_fileset_open_particles(handle);
+		if ( ret != ARTIO_SUCCESS ) {
+			artio_fileset_destroy(handle);
+			return NULL;
+		}
+	}
+
+	if (type & ARTIO_OPEN_GRID) {
+		ret = artio_fileset_open_grid(handle);
+		if ( ret != ARTIO_SUCCESS ) {
+			artio_fileset_destroy(handle);
+			return NULL;
+		}
+	}
+
+	return handle;
+}
+
+artio_file artio_fileset_create(char * file_prefix, int64_t root_cells, 
+		int64_t proc_sfc_begin, int64_t proc_sfc_end, artio_context context) {
+    artio_file handle = 
+		artio_fileset_allocate( file_prefix, ARTIO_FILESET_WRITE, context );
+    if ( handle == NULL ) {
+        return NULL;
+    }
+
+	handle->proc_sfc_index = 
+		(int64_t*)malloc((handle->num_procs+1)*sizeof(int64_t));
+	if ( handle->proc_sfc_index == NULL ) {
+		artio_fileset_destroy(handle);
+		return NULL;
+	}
+
+#ifdef ARTIO_MPI
+	MPI_Allgather( &proc_sfc_begin, 1, MPI_LONG_LONG, 
+			handle->proc_sfc_index, 1, MPI_LONG_LONG, context->comm );
+#else
+	handle->proc_sfc_index[0] = 0;
+#endif /* ARTIO_MPI */
+	handle->proc_sfc_index[handle->num_procs] = root_cells;
+
+	handle->proc_sfc_begin = proc_sfc_begin;
+	handle->proc_sfc_end = proc_sfc_end;
+	handle->num_root_cells = root_cells;
+
+	artio_parameter_set_long(handle, "num_root_cells", root_cells);
+
+	return handle;
+}
+
+int artio_fileset_close(artio_file handle) {
+	char header_filename[256];
+	artio_fh head_fh;
+	
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode == ARTIO_FILESET_WRITE) {
+		/* ensure we've flushed open particle and 
+		 * grid files before writing header */
+		if ( handle->grid != NULL ) {
+			artio_fileset_close_grid(handle);
+		}
+
+		if ( handle->particle != NULL ) {
+			artio_fileset_close_particles(handle);
+		}
+
+		sprintf(header_filename, "%s.art", handle->file_prefix);
+		head_fh = artio_file_fopen(header_filename, 
+				ARTIO_MODE_WRITE | ARTIO_MODE_DIRECT |
+					   ((handle->rank == 0) ? ARTIO_MODE_ACCESS : 0), 
+				handle->context);
+
+		if (head_fh == NULL) {
+			return ARTIO_ERR_FILE_CREATE;
+		}
+
+		if (0 == handle->rank) {
+			artio_parameter_write(head_fh, &handle->param_list);
+		}
+
+		artio_file_fclose(head_fh);
+	}
+
+	artio_fileset_destroy(handle);
+
+	return ARTIO_SUCCESS;
+}
+
+artio_file artio_fileset_allocate( char *file_prefix, int mode, 
+		artio_context context ) {
+	int my_rank;
+	int num_procs;
+
+    artio_file handle = (artio_file)malloc(sizeof(struct artio_file_struct));
+	if ( handle != NULL ) {
+		artio_parameter_list_init(&handle->param_list);
+
+#ifdef ARTIO_MPI
+		MPI_Comm_size(context->comm, &num_procs);
+		MPI_Comm_rank(context->comm, &my_rank);
+#else
+		num_procs = 1;
+		my_rank = 0;
+#endif /* MPI */
+
+		strncpy(handle->file_prefix, file_prefix, 250);
+
+		handle->open_mode = mode;
+		handle->open_type = ARTIO_OPEN_HEADER;
+
+		handle->rank = my_rank;
+		handle->num_procs = num_procs;
+		handle->context = context;
+		handle->endian_swap = 0;
+
+		handle->proc_sfc_index = NULL;
+		handle->proc_sfc_begin = -1;
+		handle->proc_sfc_end = -1;
+		handle->num_root_cells = -1;
+		
+		handle->grid = NULL;
+		handle->particle = NULL;
+	}	
+	return handle;
+}
+
+void artio_fileset_destroy( artio_file handle ) {
+	if ( handle == NULL ) return;
+
+	if ( handle->proc_sfc_index != NULL ) free( handle->proc_sfc_index );
+	
+	if ( handle->grid != NULL ) {
+        artio_fileset_close_grid(handle);
+    }
+
+    if ( handle->particle != NULL ) {
+        artio_fileset_close_particles(handle);
+    }
+
+	artio_parameter_free_list(&handle->param_list);
+
+	free(handle);
+}

yt/frontends/artio/artio_headers/artio.h

-/Users/sleitner/repos/svntrunk_temp/src/tools/artio/artio.h
+/*
+ * artio.h
+ *
+ *  Created on: Feb 21, 2010
+ *      Author: Yongen Yu
+ *  Modified: Jun 6, 2010 - Doug Rudd
+ *            Nov 18, 2010 - Doug Rudd
+ *            Nov 14, 2012 - Doug Rudd
+ */
+
+#ifndef __ARTIO_H__
+#define __ARTIO_H__
+
+#include <stdint.h>
+#ifndef int64_t
+#ifdef _WIN32
+typedef __int64 int64_t;
+#endif
+#else
+#error "Undefined int64_t!"
+#endif
+
+#define ARTIO_OPEN_HEADER					0
+#define ARTIO_OPEN_PARTICLES                1
+#define ARTIO_OPEN_GRID                     2
+
+#define ARTIO_READ_LEAFS                    1
+#define ARTIO_READ_REFINED                  2
+#define	ARTIO_READ_ALL                      3
+
+/* allocation strategy */
+#define ARTIO_ALLOC_EQUAL_SFC               0
+#define ARTIO_ALLOC_EQUAL_PROC              1
+#define ARTIO_ALLOC_MAX_FILE_SIZE  	        2
+
+#define ARTIO_TYPE_STRING                   0
+#define ARTIO_TYPE_CHAR                     1
+#define ARTIO_TYPE_INT                      2
+#define ARTIO_TYPE_FLOAT                    3
+#define ARTIO_TYPE_DOUBLE                   4
+#define ARTIO_TYPE_LONG                     5
+
+/* error codes */
+#define ARTIO_SUCCESS                       0
+
+#define ARTIO_ERR_PARAM_NOT_FOUND           1
+#define ARTIO_ERR_PARAM_INVALID_LENGTH      2
+#define ARTIO_ERR_PARAM_TYPE_MISMATCH       3
+#define ARTIO_ERR_PARAM_LENGTH_MISMATCH     4
+#define ARTIO_ERR_PARAM_LENGTH_INVALID      5
+#define ARTIO_ERR_PARAM_DUPLICATE           6
+
+#define ARTIO_ERR_INVALID_FILESET_MODE      100
+#define	ARTIO_ERR_INVALID_FILE_NUMBER       101
+#define ARTIO_ERR_INVALID_FILE_MODE         102
+#define ARTIO_ERR_INVALID_SFC_RANGE         103
+#define ARTIO_ERR_INVALID_SFC               104
+#define ARTIO_ERR_INVALID_STATE             105
+#define ARTIO_ERR_INVALID_SEEK              106
+#define ARTIO_ERR_INVALID_OCT_LEVELS        107
+#define ARTIO_ERR_INVALID_SPECIES           108
+#define ARTIO_ERR_INVALID_ALLOC_STRATEGY    109
+#define ARTIO_ERR_INVALID_LEVEL             110
+#define ARTIO_ERR_INVALID_PARAM_LIST        111
+#define ARTIO_ERR_INVALID_DATATYPE          112
+#define ARTIO_ERR_INVALID_OCT_REFINED       113
+#define ARTIO_ERR_INVALID_HANDLE            114
+
+#define ARTIO_ERR_DATA_EXISTS               200
+#define ARTIO_ERR_INSUFFICIENT_DATA         201
+#define ARTIO_ERR_FILE_CREATE               202
+#define ARTIO_ERR_PARTICLE_FILE_NOT_FOUND   203
+#define ARTIO_ERR_GRID_FILE_NOT_FOUND       204
+
+#define ARTIO_ERR_PARAM_CORRUPTED           207
+#define ARTIO_ERR_PARAM_CORRUPTED_MAGIC     208
+
+#define ARTIO_ERR_64_TO_32_BIT_TRUNCATION   209
+#define ARTIO_ERR_MEMORY_ALLOCATION         210
+
+#define ARTIO_PARAMETER_EXHAUSTED           300
+
+#ifdef ARTIO_MPI
+#include <mpi.h>
+
+struct artio_context_struct {
+    MPI_Comm comm;
+};
+#else
+struct artio_context_struct {
+    int comm;
+};
+#endif
+
+typedef struct artio_file_struct * artio_file;
+typedef struct artio_param_list * artio_parameters;
+typedef struct artio_context_struct * artio_context;
+
+extern artio_context artio_context_global;
+
+/*
+ * Description: Open the file
+ *
+ *  filename			The file prefix
+ *  type			combination of ARTIO_OPEN_PARTICLES and ARTIO_OPEN_GRID flags
+ */
+artio_file artio_fileset_open(char * file_name, int type, artio_context context);
+
+/**
+ * Description: Create fileset and begin populating header information
+ *
+ *  file_name			file name of refined cells
+ *  root_cells			the number of root level cells
+ *  proc_sfc_begin-end		the range of local space-filling-curve indices
+ *  handle			the artio file handle
+ *
+ */
+artio_file artio_fileset_create(char * file_prefix, 
+        int64_t root_cells, int64_t proc_sfc_begin, int64_t proc_sfc_end, artio_context context);
+
+/*
+ * Description	Close the file
+ */
+int artio_fileset_close(artio_file handle);
+
+/* public parameter interface */
+int artio_parameter_iterate( artio_file handle, char *key, int *type, int *length );
+int artio_parameter_get_array_length(artio_file handle, char * key, int *length);
+
+void artio_parameter_set_int(artio_file handle, char * key, int32_t value);
+int artio_parameter_get_int(artio_file handle, char * key, int32_t * value);
+
+void artio_parameter_set_int_array(artio_file handle, char * key, int length,
+		int32_t *values);
+int artio_parameter_get_int_array(artio_file handle, char * key, int length,
+		int32_t *values);
+
+void artio_parameter_set_string(artio_file handle, char * key, char * value);
+int artio_parameter_get_string(artio_file handle, char * key, char * value, int max_length);
+
+void artio_parameter_set_string_array(artio_file handle, char * key,
+		int length, char ** values);
+int artio_parameter_get_string_array(artio_file handle, char * key,
+		int length, char ** values, int max_length);
+
+void artio_parameter_set_float(artio_file handle, char * key, float value);
+int artio_parameter_get_float(artio_file handle, char * key, float * value);
+
+void artio_parameter_set_float_array(artio_file handle, char * key,
+		int length, float *values);
+int artio_parameter_get_float_array(artio_file handle, char * key,
+		int length, float * values);
+
+void artio_parameter_set_double(artio_file handle, char * key, double value);
+int  artio_parameter_get_double(artio_file handle, char * key, double * value);
+
+void artio_parameter_set_double_array(artio_file handle, char * key,
+		int length, double * values);
+int artio_parameter_get_double_array(artio_file handle, char * key,
+        int length, double *values);
+
+void artio_parameter_set_long(artio_file handle, char * key, int64_t value);
+int artio_parameter_get_long(artio_file handle, char * key, int64_t *value);
+
+void artio_parameter_set_long_array(artio_file handle, char * key,
+        int length, int64_t *values);
+int artio_parameter_get_long_array(artio_file handle, char * key,
+        int length, int64_t *values);
+
+/* public grid interface */
+typedef void (* GridCallBack)(float * variables, int level, int refined,
+		int64_t sfc_index);
+typedef void (* GridCallBackPos)(double * variables, int level, int refined,
+                int64_t sfc_index, double pos[3]);
+
+/*
+ * Description:	Add a grid component to a fileset open for writing
+ *
+ *  handle			The fileset handle
+ *  num_grid_files		The number of grid files to create
+ *  allocation_strategy		How to apportion sfc indices to each grid file
+ *  num_grid_variables		The number of variables per cell
+ *  grid_variable_labels	Identifying labels for each variable
+ *  num_levels_per_root_tree	Maximum tree depth for each oct tree
+ *  num_octs_per_root_tree	Total octs in each oct tree
+ */
+int artio_fileset_add_grid(artio_file handle,
+        int num_grid_files, int allocation_strategy,
+        int num_grid_variables,
+        char ** grid_variable_labels,
+        int * num_levels_per_root_tree,
+        int * num_octs_per_root_tree );
+
+int artio_fileset_open_grid(artio_file handle);
+int artio_fileset_close_grid(artio_file handle);
+int artio_fileset_open_particle(artio_file handle);
+int artio_fileset_close_particle(artio_file handle);
+
+/*
+ * Description:	Output the variables of the root level cell and the hierarchy of the Oct tree correlated with this root level cell
+ *
+ *  handle			The File handle
+ *  sfc				The sfc index of root cell
+ *  variables			The variables of the root level cell
+ *  level			The depth of the Oct tree correlated to the root level cell
+ *  num_level_octs		The array store the number of Oct nodes each level
+ */
+int artio_grid_write_root_cell_begin(artio_file handle, int64_t sfc, 
+		float * variables, int level, int * num_octs_per_level);
+
+/*
+ * Description:	Do something at the end of writing the root level cell
+ */
+int artio_grid_write_root_cell_end(artio_file handle);
+
+/*
+ * Description:	Do something at the beginning of each level
+ */
+int artio_grid_write_level_begin(artio_file handle, int level );
+
+/*
+ * Description:	Do something at the end of each level
+ */
+int artio_grid_write_level_end(artio_file handle);
+
+/*
+ * Description:	Output the data of a special oct tree node to the file
+ *
+ *  handle			The handle of the file
+ *  variables 			The array recording the variables of the eight cells belonging to this Octree node.
+ */
+int artio_grid_write_oct(artio_file handle, float *variables, int *refined);
+
+/*
+ * Description:	Read the variables of the root level cell and the hierarchy of the Octtree
+ *              correlated with this root level cell
+ *
+ *  handle			The File handle
+ *  variables			The variables of the root level cell
+ *  level 			The depth of the OCT tree
+ *  num_octs_per_level		The number of node of each oct level
+ *
+ */
+int artio_grid_read_root_nocts(artio_file handle, int64_t sfc, float *variables,
+		int *num_tree_levels, int *num_octs_per_level);
+int artio_grid_read_root_cell_begin(artio_file handle, int64_t sfc, float *variables,
+		int *num_tree_levels, int *num_octs_per_level);
+
+/*
+ * Description:	Do something at the end of reading the root level cell
+ */
+int artio_grid_read_root_cell_end(artio_file handle);
+
+/*
+ * Description:	Do something at the beginning of each level
+ */
+int artio_grid_read_level_begin(artio_file handle, int level );
+
+/*
+ * Description:	Do something at the end of each level
+ */
+int artio_grid_read_level_end(artio_file handle);
+
+/*
+ * Description:	Read the data of a special oct tree node from the file
+ */
+int artio_grid_read_oct(artio_file handle, float *variables, int *refined);
+
+int artio_grid_cache_sfc_range(artio_file handle, int64_t sfc_start, int64_t sfc_end);
+
+/*
+ * Description:	Read a segment of oct nodes
+ *
+ *  handle			file pointer
+ *  sfc1			the start sfc index
+ *  sfc2			the end sfc index
+ *  max_level_to_read		max level to read for each oct tree
+ *  option			1. refined nodes; 2 leaf nodes; 3 all nodes
+ *  callback			callback function
+ */
+int artio_grid_read_sfc_range(artio_file handle, int64_t sfc1, int64_t sfc2, int min_level_to_read,int max_level_to_read, int options, GridCallBack callback);
+int artio_grid_read_sfc_range_pos(artio_file handle, int64_t sfc1, int64_t sfc2, int min_level_to_read,int max_level_to_read, int options, GridCallBackPos callback);
+		
+
+typedef void (* ParticleCallBack)(int64_t pid, 
+		double *primary_variables, float *secondary_variables, 
+		int species, int subspecies, int64_t sfc_index);
+
+/**
+ *  header			head file name
+ *  num_particle_files		the number of files to record refined cells
+ *  allocation_strategy
+ *  num_species			number of particle species
+ *  species_labels		string identifier for each species
+ *  handle			the artio file handle
+ *
+ */
+int artio_fileset_add_particles(artio_file handle, 
+        int num_particle_files, int allocation_strategy,
+        int num_species, char **species_labels,
+        int *num_primary_variables,
+        int *num_secondary_variables,
+        char ***primary_variable_labels_per_species,
+        char ***secondary_variable_labels_per_species,
+        int *num_particles_per_species_per_root_tree );
+
+int artio_fileset_open_particles(artio_file handle);
+int artio_fileset_close_particles(artio_file handle);
+
+/*
+ * Description:	Output the variables of the root level cell and the hierarchy of 
+ *                  the oct-tree correlated with this root level cell
+ *
+ *  handle			The File handle
+ *  sfc				The sfc index of root cell
+ *  variables			The variables of the root level cell
+ *  level			The depth of the Oct tree correlated to the root level cell
+ *  num_level_octs		The array store the number of Oct nodes each level
+ */
+int artio_particle_write_root_cell_begin(artio_file handle, int64_t sfc,
+		int *num_particles_per_species);
+
+/*
+ * Description:	Do something at the end of writing the root level cell
+ */
+int artio_particle_write_root_cell_end(artio_file handle);
+
+/*
+ * Description:	Do something at the beginning of each level
+ */
+int artio_particle_write_species_begin(artio_file handle, int species );
+
+/*
+ * Description:	Do something at the end of each level
+ */
+int artio_particle_write_species_end(artio_file handle);
+
+/*
+ * Description: Output the data of a special oct tree node to the file
+ *
+ *  handle			The handle of the file
+ *  variables 			The array recording the variables of the eight cells belonging to this Octree node.
+ */
+int artio_particle_write_particle(artio_file handle, int64_t pid, int subspecies, 
+			double* primary_variables, float *secondary_variables);
+
+/*
+ * Description:	Read the variables of the root level cell and the hierarchy of the Octtree
+ *              correlated with this root level cell
+ *
+ *  handle			The File handle
+ *  variables			The variables of the root level cell
+ *  level 			The depth of the OCT tree
+ *  num_octs_per_level		The number of node of each oct level
+ *
+ */
+int artio_particle_read_root_cell_begin(artio_file handle, int64_t sfc, 
+			int * num_particle_per_species);
+
+/*
+ * Description:	Do something at the end of reading the root level cell
+ */
+int artio_particle_read_root_cell_end(artio_file handle);
+
+/*
+ * Description:	Do something at the beginning of each level
+ */
+int artio_particle_read_species_begin(artio_file handle, int species );
+
+/*
+ * Description:  Do something at the end of each level
+ */
+int artio_particle_read_species_end(artio_file handle);
+
+/*
+ * Description:	Read the data of a single particle from the file
+ */
+int artio_particle_read_particle(artio_file handle, int64_t *pid, int *subspecies,
+			double *primary_variables, float *secondary_variables);
+
+int artio_particle_cache_sfc_range(artio_file handle, int64_t sfc_start, int64_t sfc_end);
+
+/*
+ * Description: Read a segment of particles
+ *
+ *  handle			file pointer
+ *  sfc1			the start sfc index
+ *  sfc2			the end sfc index
+ *  start_species		the first particle species to read
+ *  end_species			the last particle species to read
+ *  callback			callback function
+ */
+int artio_particle_read_sfc_range(artio_file handle, 
+		int64_t sfc1, int64_t sfc2, 
+		int start_species, int end_species,
+		ParticleCallBack callback);
+
+#endif /* __ARTIO_H__ */

yt/frontends/artio/artio_headers/artio_endian.c

-/Users/sleitner/repos/svntrunk_temp/src/tools/artio/artio_endian.c
+#include "artio_endian.h"
+
+#include <stdint.h>
+
+void artio_int_swap(int32_t *src, int count) {
+	int i;
+	union {
+		int32_t f;
+		unsigned char c[4];
+	} d1, d2;
+
+	for ( i = 0; i < count; i++ ) {
+		d1.f = src[i];
+		d2.c[0] = d1.c[3];
+		d2.c[1] = d1.c[2];
+		d2.c[2] = d1.c[1];
+		d2.c[3] = d1.c[0];
+		src[i] = d2.f;
+	}
+}
+
+void artio_float_swap(float *src, int count) {
+	int i;
+	union {
+		float f;
+		unsigned char c[4];
+	} d1, d2;
+
+	for ( i = 0; i < count; i++ ) {
+		d1.f = src[i];
+		d2.c[0] = d1.c[3];
+		d2.c[1] = d1.c[2];
+		d2.c[2] = d1.c[1];
+		d2.c[3] = d1.c[0];
+		src[i] = d2.f;
+	}
+}
+
+void artio_double_swap(double *src, int count) {	
+	int i;
+	union
+	{
+		double d;
+		unsigned char c[8];
+	} d1, d2;
+
+	for ( i = 0; i < count; i++ ) {
+		d1.d = src[i];
+		d2.c[0] = d1.c[7];
+		d2.c[1] = d1.c[6];
+		d2.c[2] = d1.c[5];
+		d2.c[3] = d1.c[4];
+		d2.c[4] = d1.c[3];
+		d2.c[5] = d1.c[2];
+		d2.c[6] = d1.c[1];
+		d2.c[7] = d1.c[0];
+		src[i] = d2.d;
+	}
+}
+
+void artio_long_swap(int64_t *src, int count) {
+	int i;
+	union
+	{
+		int64_t d;
+		unsigned char c[8];
+	} d1, d2;
+
+	for ( i = 0; i < count; i++ ) {
+		d1.d = src[i];
+		d2.c[0] = d1.c[7];
+		d2.c[1] = d1.c[6];
+		d2.c[2] = d1.c[5];
+		d2.c[3] = d1.c[4];
+		d2.c[4] = d1.c[3];
+		d2.c[5] = d1.c[2];
+		d2.c[6] = d1.c[1];
+		d2.c[7] = d1.c[0];
+		src[i] = d2.d;
+	}
+}

yt/frontends/artio/artio_headers/artio_endian.h

-/Users/sleitner/repos/svntrunk_temp/src/tools/artio/artio_endian.h
+#ifndef __ARTIO_EDIAN_H__
+#define __ARTIO_EDIAN_H__
+
+#include <stdint.h>
+
+void artio_int_swap(int32_t *src, int count);
+void artio_float_swap(float *src, int count);
+void artio_double_swap(double *src, int count);
+void artio_long_swap(int64_t *src, int count);
+
+#endif /* __ARTIO_ENDIAN_H__ */

yt/frontends/artio/artio_headers/artio_grid.c

-/Users/sleitner/repos/svntrunk_temp/src/tools/artio/artio_grid.c
+/*
+ * artio_grid.c
+ *
+ *  Created on: May 10, 2011
+ *      Author: Yongen Yu
+ */
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+#include "sfc.h"
+#include "artio.h"
+#include "artio_internal.h"
+
+int artio_grid_find_file(artio_grid_file ghandle, int start, int end, int64_t sfc);
+artio_grid_file artio_grid_file_allocate(void);
+void artio_grid_file_destroy(artio_grid_file ghandle);
+
+/*
+ * Open grid component of the fileset
+ */
+int artio_fileset_open_grid(artio_file handle) {
+	int i;
+	char filename[256];
+	int first_file, last_file;
+	int mode;
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	/* check that the fileset doesn't already contain a grid component */
+	if ( handle->open_type & ARTIO_OPEN_GRID ||
+			handle->open_mode != ARTIO_FILESET_READ ||
+			handle->grid != NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+	handle->open_type |= ARTIO_OPEN_GRID;
+
+	ghandle = artio_grid_file_allocate();
+	if ( ghandle == NULL ) {
+		return ARTIO_ERR_MEMORY_ALLOCATION;
+	}
+
+	/* load grid parameters from header file (should be doing error handling...) */
+	artio_parameter_get_int(handle, "num_grid_variables", &ghandle->num_grid_variables);
+	artio_parameter_get_int(handle, "num_grid_files", &ghandle->num_grid_files);
+
+	ghandle->file_sfc_index = (int64_t *)malloc(sizeof(int64_t) * (ghandle->num_grid_files + 1));
+	if ( ghandle->file_sfc_index == NULL ) {
+		artio_grid_file_destroy(ghandle);
+		return ARTIO_ERR_MEMORY_ALLOCATION;
+	}
+
+	artio_parameter_get_long_array(handle, "grid_file_sfc_index",
+				       ghandle->num_grid_files + 1, ghandle->file_sfc_index);
+	artio_parameter_get_int(handle, "grid_max_level",
+				&ghandle->file_max_level);
+
+	ghandle->octs_per_level = (int *)malloc(ghandle->file_max_level * sizeof(int));
+	if ( ghandle->octs_per_level == NULL ) {
+		artio_grid_file_destroy(ghandle);
+		return ARTIO_ERR_MEMORY_ALLOCATION;
+	}
+
+	ghandle->ffh = (artio_fh *)malloc(ghandle->num_grid_files * sizeof(artio_fh));
+	if ( ghandle->ffh == NULL ) {
+		artio_grid_file_destroy(ghandle);
+		return ARTIO_ERR_MEMORY_ALLOCATION;
+	}
+ 
+	for ( i = 0; i < ghandle->num_grid_files; i++ ) {
+		ghandle->ffh[i] = NULL;
+	}
+
+	first_file = artio_grid_find_file(ghandle, 0, 
+			ghandle->num_grid_files, handle->proc_sfc_begin);
+	last_file = artio_grid_find_file(ghandle, first_file,
+			ghandle->num_grid_files, handle->proc_sfc_end);
+
+	/* open files on all processes */
+	for (i = 0; i < ghandle->num_grid_files; i++) {
+		sprintf(filename, "%s.g%03d", handle->file_prefix, i);
+
+		mode = ARTIO_MODE_READ;
+		if (i >= first_file && i <= last_file) {
+			mode |= ARTIO_MODE_ACCESS;
+		}
+
+		if (handle->endian_swap) {
+			mode |= ARTIO_MODE_ENDIAN_SWAP;
+		}
+
+		ghandle->ffh[i] = artio_file_fopen(filename, mode, handle->context);
+		if ( ghandle->ffh[i] == NULL ) {
+			artio_grid_file_destroy(ghandle);
+			return ARTIO_ERR_GRID_FILE_NOT_FOUND;
+		}
+	}
+
+	handle->grid = ghandle;
+	return ARTIO_SUCCESS;
+}
+
+int artio_fileset_add_grid(artio_file handle, 
+		int num_grid_files, int allocation_strategy, 
+		int num_grid_variables, 
+		char ** grid_variable_labels,
+		int * num_levels_per_root_tree,
+		int * num_octs_per_root_tree ) {
+
+	int i;
+	int file_max_level, local_max_level;
+	int64_t cur, sfc, l;
+	int64_t first_file_sfc, last_file_sfc;
+	int first_file, last_file;
+	char filename[256];
+	int mode;
+	int ret;
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if ( handle->open_mode != ARTIO_FILESET_WRITE ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	if ( handle->open_type & ARTIO_OPEN_GRID) {
+		return ARTIO_ERR_DATA_EXISTS;
+	}
+	handle->open_type |= ARTIO_OPEN_GRID;
+
+	artio_parameter_set_int(handle, "num_grid_files", num_grid_files);
+	artio_parameter_set_int(handle, "num_grid_variables", num_grid_variables);
+	artio_parameter_set_string_array(handle, "grid_variable_labels",
+			num_grid_variables, grid_variable_labels);
+
+	ghandle = artio_grid_file_allocate();
+	if ( ghandle == NULL ) {
+		return ARTIO_ERR_MEMORY_ALLOCATION;
+	}
+
+	ghandle->file_sfc_index = (int64_t *)malloc(sizeof(int64_t) * (num_grid_files + 1));
+	if ( ghandle->file_sfc_index == NULL ) {
+		artio_grid_file_destroy(ghandle);
+		return ARTIO_ERR_MEMORY_ALLOCATION;
+	}
+
+	/* compute global maximum level */
+	local_max_level = 0;
+	for (sfc = 0; sfc < handle->proc_sfc_end - handle->proc_sfc_begin + 1; sfc++) {
+		if (num_levels_per_root_tree[sfc] > local_max_level) {
+			local_max_level = num_levels_per_root_tree[sfc];
+		}
+	}
+
+#ifdef ARTIO_MPI
+	MPI_Allreduce( &local_max_level, &file_max_level, 
+			1, MPI_INT, MPI_MAX, handle->context->comm );
+#else
+	file_max_level = local_max_level;
+#endif /* ARTIO_MPI */
+
+	switch (allocation_strategy) {
+		case ARTIO_ALLOC_EQUAL_PROC:
+			if (num_grid_files > handle->num_procs) {
+				return ARTIO_ERR_INVALID_FILE_NUMBER;
+			}
+
+			for (i = 0; i < num_grid_files; i++) {
+				ghandle->file_sfc_index[i] = 
+					handle->proc_sfc_index[(handle->num_procs*i+num_grid_files-1) / num_grid_files];
+			}
+			ghandle->file_sfc_index[num_grid_files] = 
+				handle->proc_sfc_index[handle->num_procs];
+			break;
+		case ARTIO_ALLOC_EQUAL_SFC:
+			if ( num_grid_files > handle->num_root_cells ) {
+				return ARTIO_ERR_INVALID_FILE_NUMBER;
+			}
+
+			for (i = 0; i < num_grid_files; i++) {
+				ghandle->file_sfc_index[i] = 
+					(handle->num_root_cells*i+num_grid_files-1) / num_grid_files;
+			}
+			ghandle->file_sfc_index[num_grid_files] = handle->num_root_cells;
+			break;
+		default:
+			artio_grid_file_destroy(ghandle);
+			return ARTIO_ERR_INVALID_ALLOC_STRATEGY;
+	}
+
+	ghandle->num_grid_files = num_grid_files;
+	ghandle->num_grid_variables = num_grid_variables;
+	ghandle->file_max_level = file_max_level;
+
+	/* allocate space for sfc offset cache */
+	ghandle->cache_sfc_begin = handle->proc_sfc_begin;
+	ghandle->cache_sfc_end = handle->proc_sfc_end;
+	ghandle->sfc_offset_table = 
+		(int64_t *)malloc((size_t)(ghandle->cache_sfc_end - 
+					ghandle->cache_sfc_begin + 1) * sizeof(int64_t));
+	if ( ghandle->sfc_offset_table == NULL ) {
+		artio_grid_file_destroy(ghandle);
+		return ARTIO_ERR_MEMORY_ALLOCATION;
+	}
+
+	ghandle->octs_per_level = (int *)malloc(ghandle->file_max_level * sizeof(int));
+	if ( ghandle->octs_per_level == NULL ) {
+		artio_grid_file_destroy(ghandle);
+		return ARTIO_ERR_MEMORY_ALLOCATION;
+	}
+
+	/* allocate file handles */
+	ghandle->ffh = (artio_fh *)malloc(num_grid_files * sizeof(artio_fh));
+	if ( ghandle->ffh == NULL ) {
+		artio_grid_file_destroy(ghandle);
+		return ARTIO_ERR_MEMORY_ALLOCATION;
+	}
+	for ( i = 0; i < num_grid_files; i++ ) {
+		ghandle->ffh[i] = NULL;
+	}
+
+	/* open file handles */
+	first_file = artio_grid_find_file(ghandle, 0, num_grid_files, 
+					handle->proc_sfc_begin);
+	last_file = artio_grid_find_file(ghandle, first_file, num_grid_files, 
+					handle->proc_sfc_end);
+
+	if ( first_file < 0 || first_file >= num_grid_files ||
+			last_file < first_file || last_file >= num_grid_files ) {
+		return ARTIO_ERR_INVALID_FILE_NUMBER;
+	}
+
+	for (i = 0; i < num_grid_files; i++) {
+		sprintf(filename, "%s.g%03d", handle->file_prefix, i);
+
+		mode = ARTIO_MODE_WRITE;
+		if (i >= first_file && i <= last_file) {
+			mode |= ARTIO_MODE_ACCESS;
+		}
+
+		ghandle->ffh[i] = artio_file_fopen(filename, mode, handle->context);
+		if ( ghandle->ffh[i] == NULL ) {
+			artio_grid_file_destroy(ghandle);
+			return ARTIO_ERR_FILE_CREATE;
+		}
+
+		/* write sfc offset header if we contribute to this file */
+		if (i >= first_file && i <= last_file) {
+#ifdef ARTIO_MPI
+			if (ghandle->file_sfc_index[i] >= handle->proc_sfc_index[ handle->rank ] &&
+					ghandle->file_sfc_index[i] < handle->proc_sfc_index[ handle->rank + 1] ) {
+				cur = (ghandle->file_sfc_index[i + 1] - ghandle->file_sfc_index[i]) * sizeof(int64_t);
+			} else {
+				/* obtain offset from previous process */
+				MPI_Recv( &cur, 1, MPI_LONG_LONG_INT, handle->rank - 1, i,
+						handle->context->comm, MPI_STATUS_IGNORE );
+			}
+#else
+			cur = (ghandle->file_sfc_index[i + 1] - ghandle->file_sfc_index[i]) * sizeof(int64_t);
+#endif /* ARTIO_MPI */
+
+			first_file_sfc = MAX( handle->proc_sfc_begin, ghandle->file_sfc_index[i] );
+			last_file_sfc = MIN( handle->proc_sfc_end, ghandle->file_sfc_index[i+1]-1 );
+
+			for (l = first_file_sfc - ghandle->cache_sfc_begin; 
+					l < last_file_sfc - ghandle->cache_sfc_begin + 1; l++) {
+				ghandle->sfc_offset_table[l] = cur;
+				cur += sizeof(float) * ghandle->num_grid_variables + sizeof(int) * (1
+						+ num_levels_per_root_tree[l])
+						+ num_octs_per_root_tree[l] * 8 * (sizeof(float)
+								* ghandle->num_grid_variables + sizeof(int));
+			}
+
+#ifdef ARTIO_MPI
+			if ( ghandle->file_sfc_index[i+1] > handle->proc_sfc_end+1 ) {
+				MPI_Send( &cur, 1, MPI_LONG_LONG_INT, handle->rank + 1, i, handle->context->comm );
+			}
+#endif /* ARTIO_MPI */
+
+			/* seek and write our portion of sfc table */
+			ret = artio_file_fseek(ghandle->ffh[i], 
+					(first_file_sfc - ghandle->file_sfc_index[i]) * sizeof(int64_t), 
+					ARTIO_SEEK_SET);
+			if ( ret != ARTIO_SUCCESS ) {
+				artio_grid_file_destroy(ghandle);
+				return ret;
+			}
+
+			ret = artio_file_fwrite(ghandle->ffh[i],
+					&ghandle->sfc_offset_table[first_file_sfc - ghandle->cache_sfc_begin],
+					last_file_sfc - first_file_sfc + 1, ARTIO_TYPE_LONG);
+			if ( ret != ARTIO_SUCCESS ) {
+				artio_grid_file_destroy(ghandle);
+				return ret;
+			}
+		}
+	}
+
+	handle->grid = ghandle;
+
+	artio_parameter_set_long_array(handle, "grid_file_sfc_index",
+			ghandle->num_grid_files + 1, ghandle->file_sfc_index);
+	artio_parameter_set_int(handle, "grid_max_level", ghandle->file_max_level);
+	
+	return ARTIO_SUCCESS;
+}
+
+artio_grid_file artio_grid_file_allocate(void) {
+	artio_grid_file ghandle = 
+			(artio_grid_file)malloc(sizeof(struct artio_grid_file_struct));                                          
+	if ( ghandle != NULL ) {
+		ghandle->ffh = NULL;
+		ghandle->num_grid_variables = -1;
+		ghandle->num_grid_files = -1;
+		ghandle->file_sfc_index = NULL;
+		ghandle->cache_sfc_begin = -1;
+		ghandle->cache_sfc_end = -1;
+		ghandle->sfc_offset_table = NULL;
+		ghandle->file_max_level = -1;
+		ghandle->cur_file = -1;
+		ghandle->cur_num_levels = -1;
+		ghandle->cur_level = -1;
+		ghandle->cur_octs = -1;
+		ghandle->cur_sfc = -1;
+		ghandle->octs_per_level = NULL;
+    }
+	return ghandle;
+}
+
+void artio_grid_file_destroy(artio_grid_file ghandle) {
+	int i;
+	if ( ghandle == NULL ) return;	
+
+	if ( ghandle->ffh != NULL ) {
+		for (i = 0; i < ghandle->num_grid_files; i++) {
+			if ( ghandle->ffh[i] != NULL ) {
+				artio_file_fclose(ghandle->ffh[i]);
+			}
+		}
+		free(ghandle->ffh);
+	}
+
+	if ( ghandle->sfc_offset_table != NULL ) free(ghandle->sfc_offset_table);
+	if ( ghandle->octs_per_level != NULL ) free(ghandle->octs_per_level);
+	if ( ghandle->file_sfc_index != NULL ) free(ghandle->file_sfc_index);
+	free(ghandle);
+}
+
+int artio_fileset_close_grid(artio_file handle) {
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if ( !(handle->open_type & ARTIO_OPEN_GRID) || 
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	artio_grid_file_destroy(handle->grid);
+	handle->grid = NULL;
+	return ARTIO_SUCCESS;
+}
+
+int artio_grid_cache_sfc_range(artio_file handle, int64_t start, int64_t end) {
+	int i;
+	int ret;
+	int first_file, last_file;
+	int64_t first, count, cur;
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_READ || 
+			!(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+
+	if ( start > end || start < handle->proc_sfc_begin ||
+			end > handle->proc_sfc_end ) {
+		return ARTIO_ERR_INVALID_SFC_RANGE;
+	}
+
+
+	if (ghandle->sfc_offset_table != NULL) {
+		free(ghandle->sfc_offset_table);
+	}
+
+	first_file = artio_grid_find_file(ghandle, 0, ghandle->num_grid_files, start);
+	last_file = artio_grid_find_file(ghandle, first_file, ghandle->num_grid_files, end);
+
+	ghandle->cache_sfc_begin = start;
+	ghandle->cache_sfc_end = end;
+	ghandle->sfc_offset_table = (int64_t *)malloc(sizeof(int64_t) * (size_t)(end - start + 1));
+	if ( ghandle->sfc_offset_table == NULL ) {
+		return ARTIO_ERR_MEMORY_ALLOCATION;
+	}
+
+	cur = 0;
+	for (i = first_file; i <= last_file; i++) {
+		first = MAX( 0, start - ghandle->file_sfc_index[i] );
+		count = MIN( ghandle->file_sfc_index[i+1], end+1 )
+				- MAX( start, ghandle->file_sfc_index[i]);
+		ret = artio_file_fseek(ghandle->ffh[i], 
+				sizeof(int64_t) * first, ARTIO_SEEK_SET);
+		if ( ret != ARTIO_SUCCESS ) return ret;
+
+		ret = artio_file_fread(ghandle->ffh[i], 
+				&ghandle->sfc_offset_table[cur],
+				count, ARTIO_TYPE_LONG);
+		if ( ret != ARTIO_SUCCESS ) return ret;
+
+		cur += count;
+	}
+
+	return ARTIO_SUCCESS;
+}
+
+int artio_grid_find_file(artio_grid_file ghandle, int start, int end, int64_t sfc) {
+	int j;
+
+	if ( ghandle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if ( start < 0 || start > ghandle->num_grid_files ||
+			end < 0 || end > ghandle->num_grid_files || 
+			sfc < ghandle->file_sfc_index[start] ||
+			sfc >= ghandle->file_sfc_index[end] ) {
+		return -1;
+	}
+
+	if (start == end || sfc == ghandle->file_sfc_index[start]) {
+		return start;
+	}
+
+	if (1 == end - start) {
+		if (sfc < ghandle->file_sfc_index[end]) {
+			return start;
+		} else {
+			return end;
+		}
+	}
+
+	j = start + (end - start) / 2;
+	if (sfc > ghandle->file_sfc_index[j]) {
+		return artio_grid_find_file(ghandle, j, end, sfc);
+	} else if (sfc < ghandle->file_sfc_index[j]) {
+		return artio_grid_find_file(ghandle, start, j, sfc);
+	} else {
+		return j;
+	}
+}
+
+int artio_grid_seek_to_sfc(artio_file handle, int64_t sfc) {
+	int64_t offset;
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if ( !(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+
+	if (ghandle->cache_sfc_begin == -1 || 
+			sfc < ghandle->cache_sfc_begin || 
+			sfc > ghandle->cache_sfc_end) {
+		return ARTIO_ERR_INVALID_SFC;
+	}
+
+	ghandle->cur_file = artio_grid_find_file(ghandle, 0, ghandle->num_grid_files, sfc);
+	offset = ghandle->sfc_offset_table[sfc - ghandle->cache_sfc_begin];
+	return artio_file_fseek(ghandle->ffh[ghandle->cur_file], 
+			offset, ARTIO_SEEK_SET);
+}
+
+int artio_grid_write_root_cell_begin(artio_file handle, int64_t sfc,
+		float *variables, int num_oct_levels, int *num_octs_per_level) {
+	int i;
+	int ret;
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_WRITE || 
+			!(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+
+	if (num_oct_levels < 0 || num_oct_levels > ghandle->file_max_level) {
+		return ARTIO_ERR_INVALID_OCT_LEVELS;
+	}	
+
+	ret = artio_grid_seek_to_sfc(handle, sfc);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	ret = artio_file_fwrite(ghandle->ffh[ghandle->cur_file], variables, 
+			ghandle->num_grid_variables, ARTIO_TYPE_FLOAT);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	ret = artio_file_fwrite(ghandle->ffh[ghandle->cur_file], 
+			&num_oct_levels, 1, ARTIO_TYPE_INT);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	ret = artio_file_fwrite(ghandle->ffh[ghandle->cur_file], 
+			num_octs_per_level, num_oct_levels, ARTIO_TYPE_INT);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	for (i = 0; i < num_oct_levels; i++) {
+		ghandle->octs_per_level[i] = num_octs_per_level[i];
+	}
+
+	ghandle->cur_sfc = sfc;
+	ghandle->cur_num_levels = num_oct_levels;
+	ghandle->cur_level = -1;
+	ghandle->cur_octs = 0;
+
+	return ARTIO_SUCCESS;
+}
+
+int artio_grid_write_root_cell_end(artio_file handle) {
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_WRITE || 
+			!(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	handle->grid->cur_sfc = -1;
+	return ARTIO_SUCCESS;
+}
+
+int artio_grid_write_level_begin(artio_file handle, int level) {
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_WRITE || 
+			!(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+
+	if (ghandle->cur_sfc == -1 ||
+			level <= 0 || level > ghandle->cur_num_levels) {
+		return ARTIO_ERR_INVALID_STATE;
+	}
+
+	ghandle->cur_level = level;
+	return ARTIO_SUCCESS;
+}
+
+int artio_grid_write_level_end(artio_file handle) {
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_WRITE || 
+			!(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+
+	if (ghandle->cur_level == -1 || 
+			ghandle->cur_octs != ghandle->octs_per_level[ghandle->cur_level - 1] ) {
+		return ARTIO_ERR_INVALID_STATE;
+	}
+
+	ghandle->cur_level = -1;
+	ghandle->cur_octs = 0;
+
+	return ARTIO_SUCCESS;
+}
+
+int artio_grid_write_oct(artio_file handle, float *variables,
+		int *cellrefined) {
+	int i;
+	int ret;
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_WRITE || 
+			!(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+
+	if (ghandle->cur_level == -1 || 
+			ghandle->cur_octs >= ghandle->octs_per_level[ghandle->cur_level - 1]) {
+		return ARTIO_ERR_INVALID_STATE;
+	}
+
+	/* check that no last-level octs have refined cells */
+	if ( ghandle->cur_level == ghandle->cur_num_levels ) {
+		for ( i = 0; i < 8; i++ ) {
+			if ( cellrefined[i] ) {
+				return ARTIO_ERR_INVALID_OCT_REFINED;
+			}
+		}
+	}
+
+	ret = artio_file_fwrite(ghandle->ffh[ghandle->cur_file], 
+			variables, 8 * ghandle->num_grid_variables,
+			ARTIO_TYPE_FLOAT);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	ret = artio_file_fwrite(ghandle->ffh[ghandle->cur_file], 
+			cellrefined, 8, ARTIO_TYPE_INT);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	ghandle->cur_octs++;
+	return ARTIO_SUCCESS;
+}
+
+/*
+ *
+ */
+int artio_grid_read_root_nocts(artio_file handle, int64_t sfc,
+		float *variables, int *num_oct_levels, int *num_octs_per_level) {
+	int i;
+	int ret;
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_READ || 
+			!(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+/*         variables = (float *)malloc(8*ghandle->num_grid_variables * sizeof(float)); */
+/* 	num_octs_per_level = (int *)malloc(ghandle->file_max_level * sizeof(int)); */
+/* 	num_oct_levels = (int *)malloc(sizeof(int)); */
+/*         printf("%d %d %d\n snl in artio_grid.c", ghandle->num_grid_variables ,ghandle->file_max_level, 1); */
+
+	ret = artio_grid_seek_to_sfc(handle, sfc);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	ret = artio_file_fread(ghandle->ffh[ghandle->cur_file], 
+			variables, ghandle->num_grid_variables,
+			ARTIO_TYPE_FLOAT);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	ret = artio_file_fread(ghandle->ffh[ghandle->cur_file], 
+			num_oct_levels, 1, ARTIO_TYPE_INT);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	if ( *num_oct_levels > ghandle->file_max_level ) {
+		return ARTIO_ERR_INVALID_OCT_LEVELS;
+	}
+
+	if (*num_oct_levels > 0) {
+		ret = artio_file_fread(ghandle->ffh[ghandle->cur_file], 
+				num_octs_per_level, *num_oct_levels,
+				ARTIO_TYPE_INT);
+		if ( ret != ARTIO_SUCCESS ) return ret;
+
+		for (i = 0; i < *num_oct_levels; i++) {
+			ghandle->octs_per_level[i] = num_octs_per_level[i];
+		}
+	}
+
+	ghandle->cur_sfc = sfc;
+	ghandle->cur_num_levels = *num_oct_levels;
+	ghandle->cur_level = -1;
+
+	return ARTIO_SUCCESS;
+}
+int artio_grid_read_root_cell_begin(artio_file handle, int64_t sfc,
+		float *variables, int *num_oct_levels, int *num_octs_per_level) {
+	int i;
+	int ret;
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_READ || 
+			!(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+
+	ret = artio_grid_seek_to_sfc(handle, sfc);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	ret = artio_file_fread(ghandle->ffh[ghandle->cur_file], 
+			variables, ghandle->num_grid_variables,
+			ARTIO_TYPE_FLOAT);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	ret = artio_file_fread(ghandle->ffh[ghandle->cur_file], 
+			num_oct_levels, 1, ARTIO_TYPE_INT);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	if ( *num_oct_levels > ghandle->file_max_level ) {
+		return ARTIO_ERR_INVALID_OCT_LEVELS;
+	}
+
+	if (*num_oct_levels > 0) {
+		ret = artio_file_fread(ghandle->ffh[ghandle->cur_file], 
+				num_octs_per_level, *num_oct_levels,
+				ARTIO_TYPE_INT);
+		if ( ret != ARTIO_SUCCESS ) return ret;
+
+		for (i = 0; i < *num_oct_levels; i++) {
+			ghandle->octs_per_level[i] = num_octs_per_level[i];
+		}
+	}
+
+	ghandle->cur_sfc = sfc;
+	ghandle->cur_num_levels = *num_oct_levels;
+	ghandle->cur_level = -1;
+
+	return ARTIO_SUCCESS;
+}
+
+/* Description  */
+int artio_grid_read_oct(artio_file handle, float *variables,
+		int *refined) {
+	int ret;
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_READ || 
+			!(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+
+	if (ghandle->cur_level == -1 || 
+			ghandle->cur_octs > ghandle->octs_per_level[ghandle->cur_level - 1]) {
+		return ARTIO_ERR_INVALID_STATE;
+	}
+
+	ret = artio_file_fread(ghandle->ffh[ghandle->cur_file], 
+			variables, 8 * ghandle->num_grid_variables,
+			ARTIO_TYPE_FLOAT);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	ret = artio_file_fread(ghandle->ffh[ghandle->cur_file], 
+			refined, 8, ARTIO_TYPE_INT);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	ghandle->cur_octs++;
+
+	return ARTIO_SUCCESS;
+}
+
+/*
+ * Description        Obtain data from an appointed level tree node
+ */
+int artio_grid_read_level_begin(artio_file handle, int level) {
+	int i;
+	int ret;
+	int64_t offset = 0;
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_READ ||
+			!(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+
+	if ( ghandle->cur_sfc == -1 || level <= 0 || 
+			level > ghandle->cur_num_levels ) {
+		return ARTIO_ERR_INVALID_STATE;
+	}
+
+	offset = ghandle->sfc_offset_table[ghandle->cur_sfc - ghandle->cache_sfc_begin];
+	offset += sizeof(float) * ghandle->num_grid_variables + sizeof(int)
+			* (ghandle->cur_num_levels + 1);
+	for (i = 0; i < level - 1; i++) {
+		offset += 8 * (sizeof(float) * ghandle->num_grid_variables + sizeof(int))
+				* ghandle->octs_per_level[i];
+	}
+
+	ret = artio_file_fseek(ghandle->ffh[ghandle->cur_file], 
+			offset, ARTIO_SEEK_SET);
+	if ( ret != ARTIO_SUCCESS ) return ret;
+
+	ghandle->cur_level = level;
+	ghandle->cur_octs = 0;
+
+	return ARTIO_SUCCESS;
+}
+
+/*
+ * Description        Do something at the end of each kind of read operation
+ */
+int artio_grid_read_level_end(artio_file handle) {
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_READ ||
+			!(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+
+	if (ghandle->cur_level == -1) {
+		return ARTIO_ERR_INVALID_STATE;
+	}
+
+	ghandle->cur_level = -1;
+	ghandle->cur_octs = -1;
+
+	return ARTIO_SUCCESS;
+}
+
+int artio_grid_read_root_cell_end(artio_file handle) {
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_READ ||
+			!(handle->open_type & ARTIO_OPEN_GRID) ||
+			handle->grid == NULL ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+	handle->grid->cur_sfc = -1;
+	return ARTIO_SUCCESS;
+}
+
+int artio_grid_read_sfc_range(artio_file handle, 
+		int64_t sfc1, int64_t sfc2,
+		int min_level_to_read, int max_level_to_read, 
+		int options,
+		GridCallBack callback) {
+	int64_t sfc;
+	int oct, level, j;
+	int ret;
+	int *octs_per_level = NULL;
+	int refined;
+	int oct_refined[8];
+	int root_tree_levels;
+	float * variables = NULL;
+
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_READ ||
+			!(handle->open_type & ARTIO_OPEN_GRID) ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+
+	if ((min_level_to_read < 0) || (min_level_to_read > max_level_to_read)) {
+		return ARTIO_ERR_INVALID_LEVEL;
+	}
+
+	octs_per_level = (int *)malloc(ghandle->file_max_level * sizeof(int));
+	variables = (float *)malloc(8*ghandle->num_grid_variables * sizeof(float));
+
+	if ( octs_per_level == NULL || variables == NULL ) {
+		if ( octs_per_level != NULL ) free(octs_per_level);
+		if ( variables != NULL ) free(variables);
+		return ARTIO_ERR_MEMORY_ALLOCATION;
+	}
+
+	ret = artio_grid_cache_sfc_range(handle, sfc1, sfc2);
+	if ( ret != ARTIO_SUCCESS ) {
+		free(octs_per_level);
+		free(variables);
+		return ret;
+	}
+		
+	for (sfc = sfc1; sfc <= sfc2; sfc++) {
+		ret = artio_grid_read_root_cell_begin(handle, sfc, variables,
+				&root_tree_levels, octs_per_level);
+		if ( ret != ARTIO_SUCCESS ) {
+			free(octs_per_level);
+			free(variables);
+			return ret;
+		}
+
+		if (min_level_to_read == 0 && (options == ARTIO_READ_ALL || 
+				(options == ARTIO_READ_REFINED && root_tree_levels > 0) || 
+				(options == ARTIO_READ_LEAFS && root_tree_levels == 0)) ) {
+			refined = (root_tree_levels > 0) ? 1 : 0;
+			callback(variables, 0, refined, sfc);
+		}
+
+		for (level = MAX(min_level_to_read,1); 
+				level <= MIN(root_tree_levels,max_level_to_read); level++) {
+			ret = artio_grid_read_level_begin(handle, level);
+			if ( ret != ARTIO_SUCCESS ) {
+				free(octs_per_level);
+				free(variables);
+				return ret;
+			}
+
+			for (oct = 0; oct < octs_per_level[level - 1]; oct++) {
+				ret = artio_grid_read_oct(handle, variables, oct_refined);
+				if ( ret != ARTIO_SUCCESS ) {
+					free(octs_per_level);
+					free(variables);
+					return ret;
+				}
+
+				for (j = 0; j < 8; j++) {
+					if (options == ARTIO_READ_ALL || 
+							(options == ARTIO_READ_REFINED && oct_refined[j]) ||
+							(options == ARTIO_READ_LEAFS && !oct_refined[j]) ) {
+						callback(&variables[j * ghandle->num_grid_variables],
+								level, oct_refined[j], sfc);
+					}
+				}
+			}
+			artio_grid_read_level_end(handle);
+		}
+		artio_grid_read_root_cell_end(handle);
+	}
+
+	free(variables);
+	free(octs_per_level);
+
+	return ARTIO_SUCCESS;
+}
+
+
+/* array which describes how child cells are offset from 
+ * the corner of their parent oct */
+#define num_children 8
+#define min_level 0
+const double cell_delta_corner[num_children][nDim] = {
+#if nDim == 1
+    { 0.0 }, { 1.0 }
+#elif nDim == 2
+    { 0.0 , 0.0  }, { 1.0, 0.0  }, { 0.0 , 1.0 }, { 1.0, 1.0 }
+#elif nDim == 3
+    { 0.0 , 0.0 , 0.0  }, {  1.0, 0.0 , 0.0  }, { 0.0 ,  1.0, 0.0  },
+    {  1.0,  1.0, 0.0  }, { 0.0 , 0.0 ,  1.0 }, {  1.0, 0.0 ,  1.0 },
+    { 0.0 ,  1.0,  1.0 }, {  1.0,  1.0,  1.0 }
+#else
+#error "No valid cell_delta for that number of dimensions!"
+#endif
+};
+int artio_grid_read_sfc_range_pos(artio_file handle, 
+		int64_t sfc1, int64_t sfc2,
+		int min_level_to_read, int max_level_to_read, 
+		int options,
+		GridCallBackPos callback) {
+	int64_t sfc;
+	int oct, level, j, i;
+	int ret;
+	int *num_octs_per_level = NULL;
+        int num_level_octs, num_next_level_octs;
+        int num_root_levels;
+
+        int coords[nDim];
+        double pos[nDim];
+        double cell_size;
+        double *level_octs_pos_x, *next_level_octs_pos_x;
+        double *level_octs_pos_y, *next_level_octs_pos_y;
+        double *level_octs_pos_z, *next_level_octs_pos_z;
+	int refined;
+	int oct_refined[num_children];
+	int root_tree_levels;
+	float * variables = NULL;
+
+	artio_grid_file ghandle;
+
+	if ( handle == NULL ) {
+		return ARTIO_ERR_INVALID_HANDLE;
+	}
+
+	if (handle->open_mode != ARTIO_FILESET_READ ||
+			!(handle->open_type & ARTIO_OPEN_GRID) ) {
+		return ARTIO_ERR_INVALID_FILESET_MODE;
+	}
+
+	ghandle = handle->grid;
+
+	if ((min_level_to_read < 0) || (min_level_to_read > max_level_to_read)) {
+		return ARTIO_ERR_INVALID_LEVEL;
+	}
+
+        num_root_levels = log(handle->num_root_cells)/(3*log(2));
+	num_octs_per_level = (int *)malloc(ghandle->file_max_level * sizeof(int));
+	variables = (float *)malloc(num_children*ghandle->num_grid_variables * sizeof(float));
+
+	if ( num_octs_per_level == NULL || variables == NULL ) {
+		if ( num_octs_per_level != NULL ) free(num_octs_per_level);
+		if ( variables != NULL ) free(variables);
+		return ARTIO_ERR_MEMORY_ALLOCATION;
+	}
+
+	ret = artio_grid_cache_sfc_range(handle, sfc1, sfc2);
+	if ( ret != ARTIO_SUCCESS ) {
+		free(num_octs_per_level);
+		free(variables);
+		return ret;
+	}
+		
+	for (sfc = sfc1; sfc <= sfc2; sfc++) {
+		ret = artio_grid_read_root_cell_begin(handle, sfc, variables,
+				&root_tree_levels, num_octs_per_level);
+
+		if ( ret != ARTIO_SUCCESS ) {
+			free(num_octs_per_level);
+			free(variables);
+			return ret;
+		}
+                //////////////////////////////////////
+                sfc_coords(sfc, coords, num_root_levels); 
+                for(i=0; i<nDim; i++){ pos[i] = (double)coords[i]; }
+                refined = (root_tree_levels > 0) ? 1 : 0;
+                if( refined ){
+                    
+                    num_next_level_octs = 1;
+                    next_level_octs_pos_x = malloc( sizeof(double)*num_next_level_octs );
+                    next_level_octs_pos_y = malloc( sizeof(double)*num_next_level_octs );
+                    next_level_octs_pos_z = malloc( sizeof(double)*num_next_level_octs );
+                    if ( next_level_octs_pos_x == NULL ||
+                         next_level_octs_pos_y == NULL ||
+                         next_level_octs_pos_z == NULL ){
+                        return ARTIO_ERR_MEMORY_ALLOCATION;
+                    }
+                    next_level_octs_pos_x[0] = pos[0] ;  //need a stupid 2D array
+                    next_level_octs_pos_y[0] = pos[1] ;  //need a stupid 2D array
+                    next_level_octs_pos_z[0] = pos[2] ;  //need a stupid 2D array
+                }else{
+                    num_next_level_octs = 0;
+                }
+                //////////////////////////////////////
+                        
+
+		if (min_level_to_read == 0 && (options == ARTIO_READ_ALL || 
+				(options == ARTIO_READ_REFINED && root_tree_levels > 0) || 
+				(options == ARTIO_READ_LEAFS && root_tree_levels == 0)) ) {
+			refined = (root_tree_levels > 0) ? 1 : 0;
+			callback(variables, min_level, refined, sfc, pos);
+		}
+// level is the cell_level; current octs live at level-1.
+		for (level = min_level+1; level <= MIN(root_tree_levels,max_level_to_read); level++) { 
+			ret = artio_grid_read_level_begin(handle, level);
+			if ( ret != ARTIO_SUCCESS ) {
+				free(num_octs_per_level);
+				free(variables);
+				return ret;
+			}
+                        
+                        //////////////////////////////////////
+                        if( num_next_level_octs > 0 ){
+                            num_level_octs = num_next_level_octs;
+                            level_octs_pos_x = next_level_octs_pos_x;
+                            level_octs_pos_y = next_level_octs_pos_y;
+                            level_octs_pos_z = next_level_octs_pos_z;
+                            
+                            // at max_level-1 the next level can only have cells -- no more octs
+                            if(num_level_octs != num_octs_per_level[level-1]){
+                                printf("bad oct counting next expected:%d octs per level array %d\n",
+                                       num_level_octs, num_octs_per_level[level-1]);
+                                       printf("sfc %d \n",sfc);
+                            }
+                            if ( level < MIN(root_tree_levels,max_level_to_read) && num_octs_per_level[level] > 0 ) {
+                                next_level_octs_pos_x = malloc( sizeof(double)*num_children*num_level_octs );
+                                next_level_octs_pos_y = malloc( sizeof(double)*num_children*num_level_octs );
+                                next_level_octs_pos_z = malloc( sizeof(double)*num_children*num_level_octs );
+                                if ( next_level_octs_pos_x == NULL ||
+                                     next_level_octs_pos_y == NULL ||
+                                     next_level_octs_pos_z == NULL ){
+                                    return ARTIO_ERR_MEMORY_ALLOCATION;
+                                }
+                            }
+                        }
+                        num_next_level_octs = 0;
+                        cell_size = pow(2,-level);
+                        //////////////////////////////////////
+