Commits

Tom Roche committed 62f9d9f

Gets helpers from project=regrid_utils, adapts to R package upgrades on terrae

* bash_utilities.sh, plotLayersForTimestep.r (or later versions of these) moved to
https://bitbucket.org/tlroche/regrid_utils

* GEIA_driver.sh
** clones regrid_utils if not already present
** pulls helpers from folder=regrid_utils

* uber_driver::file copies folder=regrid_utils

* regrid_global_to_AQMEII.r adapts to upgraded R package=rgdal, plus other tweaks

Tested on terrae/EMVL from 'uber_driver.sh' in fresh terminal.

TODO:
* all regrids: how to nudge off/onshore as required? e.g., soil or burning emissions should never be offshore, marine emissions should never be onshore.
* all regrid maps: add Caribbean islands (esp Bahamas! for offshore burning), Canadian provinces, Mexican states
* test on
** tlrPanP5 (which now has R package=ncdf4, but readAsciiTable of input .txt's is very slow compared to terrae)
** HPCC (once problem with ncdf4 on amad1 is debugged: in process with JOB and KMF)

Comments (0)

Files changed (5)

 # See https://bitbucket.org/tlroche/aqmeii_ag_soil
 
 # Expects to run on linux. Utility dependencies include:
-# * bash
-# * basename, dirname
-# * unzip
-# * curl or wget
-# * ncdump
-# * NCL
-# * R
+
+# * definitely
+
+# ** bash: required to run this script. Known to work with bash --version==3.2.25
+
+# * potentially initially. If you have not already retrieved regrid_utils (see `get_regrid_utils`), you will need
+
+# ** git: to clone their repo
+# ** web access (HTTP currently)
+
+#   These must be available to the script when initially run.
+
+# * ultimately
+
+# ** basename, dirname
+# ** unzip
+# ** curl or wget (latter {preferred, coded} for ability to deal with redirects)
+# ** ncdump
+# ** NCL
+# ** R
+
+#   Paths to the above can be setup in `bash_utilities::setup_paths`
 
 # TODO: failing functions should fail this (entire) driver!
 
 export CALL_CONSERV_FN='check_conservation.ncl'
 export CALL_CONSERV_FP="${WORK_DIR}/${CALL_CONSERV_FN}"
 
-# PLOT_FUNCS_URI='https://bitbucket.org/tlroche/geia_regrid/raw/cbb6368fb7d9924280095f5f9add6a3c62684be0/plotLayersForTimestep.r'
-# PLOT_FUNCS_FN="$(basename ${PLOT_FUNCS_URI%%\?*})"
-# in this repo
-PLOT_FUNCS_FN='plotLayersForTimestep.r'
-export PLOT_FUNCS_FP="${WORK_DIR}/${PLOT_FUNCS_FN}"
+### helpers retrieved from elsewhere. TODO: R-package my code
+
+# path to a project, not a .git
+REGRID_UTILS_URI='https://bitbucket.org/tlroche/regrid_utils'
+REGRID_UTILS_PN="$(basename ${REGRID_UTILS_URI})" # project name
+# can also use   'git@bitbucket.org:tlroche/regrid_utils' if supported
+# path to a folder, not a file: needed by NCL to get to initial helpers
+export REGRID_UTILS_FP="${WORK_DIR}/${REGRID_UTILS_PN}" # folder, not file
 
-# BASH_UTILS_URI='https://bitbucket.org/tlroche/gfed-3.1_global_to_aqmeii-na/raw/95484c5d63502ab146402cedc3612dcdaf629bd7/bash_utilities.sh'
-# export BASH_UTILS_FN="$(basename ${BASH_UTILS_URI})"
-# in this repo
 export BASH_UTILS_FN='bash_utilities.sh'
-BASH_UTILS_FP="${WORK_DIR}/${BASH_UTILS_FN}"
+export BASH_UTILS_FP="${REGRID_UTILS_FP}/${BASH_UTILS_FN}"
 
-## helpers retrieved from elsewhere. TODO: R-package my code
+export PLOT_FUNCS_FN='plotLayersForTimestep.r'
+export PLOT_FUNCS_FP="${REGRID_UTILS_FP}/${PLOT_FUNCS_FN}"
 
-STATS_FUNCS_URI='https://bitbucket.org/tlroche/aqmeii_ag_soil/raw/0fa7d1180b67265540cdf0dd1d82e06d9b4fa789/netCDF.stats.to.stdout.r'
-# get base filename, ignoring query string (if any)
-STATS_FUNCS_FN="$(basename ${STATS_FUNCS_URI%%\?*})"
-export STATS_FUNCS_FP="${WORK_DIR}/${STATS_FUNCS_FN}"
+export STATS_FUNCS_FN='netCDF.stats.to.stdout.r'
+export STATS_FUNCS_FP="${REGRID_UTILS_FP}/${STATS_FUNCS_FN}"
 
-SUMMARIZE_FUNCS_URI='https://bitbucket.org/tlroche/edgar-4.2_minus_soil_and_biomass_to_aqmeii-na/raw/618f09df886a86c9bd15880f37bbf1bf9295b638/summarize.ncl'
-SUMMARIZE_FUNCS_FN="$(basename ${SUMMARIZE_FUNCS_URI})"
-export SUMMARIZE_FUNCS_FP="${WORK_DIR}/${SUMMARIZE_FUNCS_FN}"
+export SUMMARIZE_FUNCS_FN='summarize.ncl'
+export SUMMARIZE_FUNCS_FP="${REGRID_UTILS_FP}/${SUMMARIZE_FUNCS_FN}"
 
-TIME_FUNCS_URI='https://bitbucket.org/tlroche/edgar-4.2_minus_soil_and_biomass_to_aqmeii-na/raw/618f09df886a86c9bd15880f37bbf1bf9295b638/time.ncl'
-TIME_FUNCS_FN="$(basename ${TIME_FUNCS_URI})"
-export TIME_FUNCS_FP="${WORK_DIR}/${TIME_FUNCS_FN}"
+export TIME_FUNCS_FN='time.ncl'
+export TIME_FUNCS_FP="${REGRID_UTILS_FP}/${TIME_FUNCS_FN}"
 
-VIS_FUNCS_URI='https://bitbucket.org/tlroche/edgar-4.2_minus_soil_and_biomass_to_aqmeii-na/raw/618f09df886a86c9bd15880f37bbf1bf9295b638/visualization.r'
-VIS_FUNCS_FN="$(basename ${VIZ_SCRIPT_URI})"
-export VIS_FUNCS_FP="${WORK_DIR}/${VIS_FUNCS_FN}"
+export VIS_FUNCS_FN='visualization.r'
+export VIS_FUNCS_FP="${REGRID_UTILS_FP}/${VIS_FUNCS_FN}"
 
 ## Raw input data
 
 #  echo -e "\n$ ${THIS_FN}: PDF_VIEWER='${PDF_VIEWER}'" # debugging
 } # end function setup
 
+# Must `get_regrid_utils` first.
 function get_helpers {
   for CMD in \
+    'get_regrid_utils' \
     'get_bash_utils' \
     'get_plot_funcs' \
     'get_stats_funcs' \
   done
 } # end function get_helpers
 
+function get_regrid_utils {
+  if [[ -z "${REGRID_UTILS_FP}" ]] ; then
+    echo -e "${THIS_FN}: ERROR: REGRID_UTILS_FP not defined"
+    exit 4
+  fi
+  if [[ ! -r "${REGRID_UTILS_FP}" ]] ; then
+#      "rm -fr ${REGRID_UTILS_FP}/.git" \
+    for CMD in \
+      "git clone ${REGRID_UTILS_URI}.git" \
+    ; do
+      echo -e "$ ${CMD}"
+      eval "${CMD}"
+    done
+  fi
+  if [[ ! -r "${REGRID_UTILS_FP}" ]] ; then
+    echo -e "${THIS_FN}: ERROR: cannot download REGRID_UTILS_FP=='${REGRID_UTILS_FP}'"
+    exit 5
+  fi  
+} # end function get_regrid_utils
+
 function setup_resources {
   for CMD in \
     'get_map_table' \
 #   fi
 } # end function get_geia_regrid
 
+# isa regrid_utils
 function get_bash_utils {
   if [[ -z "${BASH_UTILS_FP}" ]] ; then
     echo -e "${THIS_FN}: ERROR: BASH_UTILS_FP not defined"
     exit 18
   fi
-  # in this repository
-#   if [[ ! -r "${BASH_UTILS_FP}" ]] ; then
-#     for CMD in \
-#       "${WGET_TO_FILE} ${BASH_UTILS_FP} ${BASH_UTILS_URI}" \
-#     ; do
-#       echo -e "$ ${CMD}"
-#       eval "${CMD}"
-#     done
-#   fi
   if [[ ! -r "${BASH_UTILS_FP}" ]] ; then
     echo -e "${THIS_FN}::${FUNCNAME[0]}: ERROR: BASH_UTILS_FP=='${BASH_UTILS_FP}' not readable"
     exit 19
   fi
   # This is bash, so gotta ...
   source "${BASH_UTILS_FP}"
+  # ... for its functions to be available later in this script
 } # end function get_bash_utils
 
+# isa regrid_utils
 function get_plot_funcs {
   if [[ -z "${PLOT_FUNCS_FP}" ]] ; then
     echo -e "${THIS_FN}: ERROR: PLOT_FUNCS_FP not defined"
     exit 20
   fi
-  # is in this repo
-#  if [[ ! -r "${PLOT_FUNCS_FP}" ]] ; then
-#    for CMD in \
-#      "${WGET_TO_FILE} ${PLOT_FUNCS_FP} ${PLOT_FUNCS_URI}" \
-#    ; do
-#      echo -e "$ ${CMD}"
-#      eval "${CMD}"
-#    done
-#  fi
   if [[ ! -r "${PLOT_FUNCS_FP}" ]] ; then
     echo -e "${THIS_FN}::${FUNCNAME[0]}: ERROR: PLOT_FUNCS_FP=='${PLOT_FUNCS_FP}' not readable"
     exit 21
   fi
 } # end function get_plot_funcs
 
+# isa regrid_utils
 function get_stats_funcs {
   if [[ -z "${STATS_FUNCS_FP}" ]] ; then
     echo -e "${THIS_FN}: ERROR: STATS_FUNCS_FP not defined"
     exit 22
   fi
   if [[ ! -r "${STATS_FUNCS_FP}" ]] ; then
-    for CMD in \
-      "${WGET_TO_FILE} ${STATS_FUNCS_FP} ${STATS_FUNCS_URI}" \
-    ; do
-      echo -e "$ ${CMD}"
-      eval "${CMD}"
-    done
-  fi
-  if [[ ! -r "${STATS_FUNCS_FP}" ]] ; then
     echo -e "${THIS_FN}::${FUNCNAME[0]}: ERROR: STATS_FUNCS_FP=='${STATS_FUNCS_FP}' not readable"
     exit 23
   fi
 } # end function get_stats_funcs
 
+# isa regrid_utils
 function get_summarize_funcs {
   if [[ -z "${SUMMARIZE_FUNCS_FP}" ]] ; then
     echo -e "${THIS_FN}: ERROR: SUMMARIZE_FUNCS_FP not defined"
     exit 24
   fi
   if [[ ! -r "${SUMMARIZE_FUNCS_FP}" ]] ; then
-    for CMD in \
-      "${WGET_TO_FILE} ${SUMMARIZE_FUNCS_FP} ${SUMMARIZE_FUNCS_URI}" \
-    ; do
-      echo -e "$ ${CMD}"
-      eval "${CMD}"
-    done
-  fi
-  if [[ ! -r "${SUMMARIZE_FUNCS_FP}" ]] ; then
     echo -e "${THIS_FN}::${FUNCNAME[0]}: ERROR: SUMMARIZE_FUNCS_FP=='${SUMMARIZE_FUNCS_FP}' not readable"
     exit 25
   fi
 } # end function get_summarize_funcs
 
+# isa regrid_utils
 function get_time_funcs {
   if [[ -z "${TIME_FUNCS_FP}" ]] ; then
     echo -e "${THIS_FN}: ERROR: TIME_FUNCS_FP not defined"
     exit 26
   fi
   if [[ ! -r "${TIME_FUNCS_FP}" ]] ; then
-    for CMD in \
-      "${WGET_TO_FILE} ${TIME_FUNCS_FP} ${TIME_FUNCS_URI}" \
-    ; do
-      echo -e "$ ${CMD}"
-      eval "${CMD}"
-    done
-  fi
-  if [[ ! -r "${TIME_FUNCS_FP}" ]] ; then
     echo -e "${THIS_FN}::${FUNCNAME[0]}: ERROR: TIME_FUNCS_FP=='${TIME_FUNCS_FP}' not readable"
     exit 27
   fi
 } # end function get_time_funcs
 
+# isa regrid_utils
 function get_vis_funcs {
   if [[ -z "${VIS_FUNCS_FP}" ]] ; then
     echo -e "${THIS_FN}: ERROR: VIS_FUNCS_FP not defined"
     exit 28
   fi
   if [[ ! -r "${VIS_FUNCS_FP}" ]] ; then
-    for CMD in \
-      "${WGET_TO_FILE} ${VIS_FUNCS_FP} ${VIS_FUNCS_URI}" \
-    ; do
-      echo -e "$ ${CMD}"
-      eval "${CMD}"
-    done
-  fi
-  if [[ ! -r "${VIS_FUNCS_FP}" ]] ; then
     echo -e "${THIS_FN}::${FUNCNAME[0]}: ERROR: VIS_FUNCS_FP=='${VIS_FUNCS_FP}' not readable"
     exit 29
   fi
 # should always
 # * begin with `setup` to setup paths, apps, helpers, resources
 # * end with `teardown` for tidy and testing (e.g., plot display)
+  # 'setup' \
+  # 'reformat' \
+  # 'regrid' \
+  # 'retemp' \
+  # 'check_conserv' \
+  # 'teardown' \
 for CMD in \
   'setup' \
   'reformat' \

bash_utilities.sh

-# description---------------------------------------------------------
-
-# Source me for bash convenience functions: notably,
-# ensuring system/environment dependencies are available.
-# Note this assumes linux platform: dependencies include
-# * `hostname`
-
-# code----------------------------------------------------------------
-
-# constants-----------------------------------------------------------
-
-### TODO: take switches for help, debugging, no/eval, target drive
-
-# Following does not work with `source`
-# THIS="$0"
-# THIS_FN="$(basename ${THIS})"
-# THIS_DIR="$(dirname ${THIS})"
-
-### constants by model/run
-# TODO: read IOAPI info from CCTM Makefile
-IOAPI_VERSION="3.1" # desired
-
-### constants by host/cluster
-
-## tlrPanP5
-# HDF5 version per http://www.hdfgroup.org/hdf5-quest.html#vers1 :
-# `find /usr/lib/ | fgrep -ie 'libhdf5.a' | xargs strings | fgrep -ne 'HDF5 library version:'`
-TLRP_HDF5_VERSION='1.8.8'
-TLRP_NCL_VERSION="ncl_ncarg-6.1.1.Linux_Debian6.0_x86_64_nodap_gcc445"
-TLRP_NETCDF_VERSION='4.1.3' # AND I NOW HAVE R PACKAGE=ncdf4 !!!
-TLRP_R_VERSION='2.15.1'
-
-# NCARG_ROOT required by NCL
-TLRP_NCARG_ROOT="${HOME}/bin/${TLRP_NCL_VERSION}"
-TLRP_NCL_PATH="${TLRP_NCARG_ROOT}/bin"
-TLRP_R_PATH='/usr/bin'
-
-## HPCC
-HPCC_NCO_VERSION='4.0.8' # on infinity, anyway
-# a much less regular environment than EMVL :-(
-AMAD1_NCL_VERSION='ncl_ncarg-6.1.2.Linux_RHEL5.6_x86_64_nodap_gcc412'
-
-HPCC_R_PATH='/share/linux86_64/bin'
-# `ncdump` now on hpcc in /usr/bin
-#HPCC_NCDUMP_PATH='/share/linux86_64/grads/supplibs-2.2.0/x86_64-unknown-linux-gnu/bin'
-HPCC_IOAPI_LIB_PATH="/project/air5/roche/CMAQ-5-eval/lib/ioapi_${IOAPI_VERSION}"
-HPCC_IOAPI_BIN_PATH="${HPCC_IOAPI_LIB_PATH}"
-HPCC_NCO_PATH="/share/linux86_64/nco/nco-${NCO_VERSION}/bin"
-
-AMAD1_NCARG_ROOT="${HOME}/bin/${AMAD1_NCL_VERSION}"
-AMAD1_NCL_PATH="${AMAD1_NCARG_ROOT}/bin"
-
-## terrae/EMVL
-TERRAE_MODULE_COMPILER='gcc-4.1.2'
-
-#TERRAE_HDF5_VERSION='1.8.7'
-TERRAE_HDF5_VERSION='1.8.10/intel-13.0'
-TERRAE_INTEL_VERSION='13.1'
-#TERRAE_NCO_VERSION='4.0.5'
-#TERRAE_NCO_VERSION='4.2.3/${TERRAE_MODULE_COMPILER}'
-TERRAE_NCO_VERSION='4.2.3'
-#TERRAE_NETCDF_VERSION='4.1.2'
-TERRAE_NETCDF_VERSION='4.1.2/intel-13.0'
-#TERRAE_R_VERSION='2.15.2'
-TERRAE_R_VERSION='3.0.0'
-
-TERRAE_HDF5_MODULE="hdf5-${TERRAE_HDF5_VERSION}"
-
-TERRAE_INTEL_MODULE="intel-${TERRAE_INTEL_VERSION}"
-
-#TERRAE_IOAPI_MODULE="ioapi-${IOAPI_VERSION}/${TERRAE_MODULE_COMPILER}"
-TERRAE_IOAPI_MODULE="ioapi-${IOAPI_VERSION}"
-
-TERRAE_NCO_MODULE="nco-${TERRAE_NCO_VERSION}"
-
-TERRAE_NETCDF_MODULE="netcdf-${TERRAE_NETCDF_VERSION}"
-
-#TERRAE_NCL_MODULE_VERSION='6.0.0' # outdated
-TERRAE_NCL_VERSION='ncl_ncarg-6.1.2.Linux_RHEL5.6_x86_64_gcc412'
-# NCARG_ROOT required by NCL
-# use Herwehe's while EMVL upgrades RHEL
-TERRAE_NCARG_ROOT="/home/hhg/ncl_ncarg/${TERRAE_NCL_VERSION}"
-
-#TERRAE_R_MODULE='R'
-# note different separator
-TERRAE_R_MODULE="R/${TERRAE_R_VERSION}"
-
-# ----------------------------------------------------------------------
-# functions
-# ----------------------------------------------------------------------
-
-# Ensure various needed artifacts are on various *xy paths,
-# but assumes `hostname` is already available.
-# TODO: ensure your hostname matches here!
-# TODO: setup packages={ncdf4} on infinity, not just amad
-function setup_paths {
-  H="$(hostname)"
-  case "${H}" in
-    amad*)
-      echo -e "${FUNCNAME[0]}: ${H} is on hpcc"
-# as of 22 May 12, on the hpcc R servers NCO is installed normally, in /usr/bin
-#      addPath "${HPCC_NCO_PATH}"
-      addPath "${HPCC_IOAPI_BIN_PATH}"
-      addPath "${AMAD1_NCL_PATH}"
-      addPath "${HPCC_R_PATH}"
-      addLdLibraryPath "${HPCC_IOAPI_LIB_PATH}"
-      ;;
-    global*)
-      echo -e "${FUNCNAME[0]}: ${H} is on hpcc"
-#      addPath "${HPCC_NCO_PATH}"
-      addPath "${HPCC_IOAPI_BIN_PATH}"
-      addPath "${HPCC_R_PATH}"
-      addLdLibraryPath "${HPCC_IOAPI_LIB_PATH}"
-      ;;
-    imaster*) # == infinity
-      echo -e "${FUNCNAME[0]}: ${H} is on hpcc"
-      echo -e "For R packages such as ncdf4, must run on amad"
-      addPath "${HPCC_NCO_PATH}"
-      addPath "${HPCC_IOAPI_BIN_PATH}"
-      addPath "${HPCC_R_PATH}"
-      addLdLibraryPath "${HPCC_IOAPI_LIB_PATH}"
-      ;;
-    inode*) # == node39
-      echo -e "${FUNCNAME[0]}: ${H} is on hpcc"
-      echo -e "For R packages such as ncdf4, must run on amad"
-      addPath "${HPCC_NCO_PATH}"
-      addPath "${HPCC_IOAPI_BIN_PATH}"
-      addPath "${HPCC_R_PATH}"
-      addLdLibraryPath "${HPCC_IOAPI_LIB_PATH}"
-      ;;
-    terra*)
-      echo -e "${FUNCNAME[0]}: ${H} is on terrae"
-      removeModule 'intel' # required on terrae/EMVL as of 23 Apr 2013
-      addModule "${TERRAE_INTEL_MODULE}"
-      addModule "${TERRAE_HDF5_MODULE}"
-      addModule "${TERRAE_NETCDF_MODULE}"
-      addModule "${TERRAE_R_MODULE}"
-      echo -e "${FUNCNAME[0]}: 'module list -t'=="
-      modulecmd bash list -t > ${TEMPFILE}
-      source ${TEMPFILE}
-      # until NCL is module'd on terrae
-      export NCARG_ROOT="${TERRAE_NCARG_ROOT}"
-      addPath "${TERRAE_NCARG_ROOT}/bin"
-      ;;
-    tlr*)
-      echo -e "${FUNCNAME[0]}: ${H} is a Tom box"
-      # NCL
-      export NCARG_ROOT="${TLRP_NCARG_ROOT}"
-      addPath "${TLRP_NCL_PATH}"
-      addPath "${TLRP_R_PATH}"
-      ;;
-    *)
-      echo -e "${FUNCNAME[0]}: unknown ${H}"
-#      exit 1
-      ;;
-  esac
-} # end function setup_paths
-
-# Choose which app to use based on host. Assumes
-# * paths (et al) are properly configured by setup_paths (above)
-# * `hostname` is available
-# TODO: ensure your hostname matches here!
-function setup_apps {
-  H="$(hostname)"
-  case "${H}" in
-    amad*)
-      echo -e "${FUNCNAME[0]}: ${H} is on hpcc"
-      export PDF_VIEWER='xpdf'
-      ;;
-    global*)
-      echo -e "${FUNCNAME[0]}: ${H} is on hpcc"
-      export PDF_VIEWER='xpdf'
-      ;;
-    imaster*) # == infinity
-      echo -e "${FUNCNAME[0]}: ${H} is on hpcc"
-      export PDF_VIEWER='xpdf'
-      ;;
-    inode*) # == node39
-      echo -e "${FUNCNAME[0]}: ${H} is on hpcc"
-      export PDF_VIEWER='xpdf'
-      ;;
-    terra*)
-      echo -e "${FUNCNAME[0]}: ${H} is on terrae"
-      export PDF_VIEWER='xpdf'
-      ;;
-    tlr*)
-      echo -e "${FUNCNAME[0]}: ${H} is a Tom box"
-      export PDF_VIEWER='evince'
-      ;;
-    *)
-      echo -e "${FUNCNAME[0]}: unknown ${H}"
-#      exit 1
-      ;;
-  esac
-} # end function setup_apps
-
-# add $1 to PATH if not already there
-function addPath {
-    DIR="$1"
-    if [[ -n "${DIR}" ]] ; then
-      if [ -d "${DIR}" ] ; then
-        if [[ ":${PATH}:" != *":${DIR}:"* ]] ; then
-          PATH="${DIR}:${PATH}"
-        else
-          echo -e "${FUNCNAME[0]}: PATH contains '${DIR}'"
-        fi
-      else
-        echo -e "${THIS_FN}::${FUNCNAME[0]}: ERROR: '${DIR}' is not a directory" 1>&2
-      fi
-    else
-      echo -e "${THIS_FN}::${FUNCNAME[0]}: ERROR: DIR not defined" 1>&2
-    fi
-}
-
-# add $1 to LD_LIBRARY_PATH if not already there
-function addLdLibraryPath {
-    DIR="$1"
-    if [[ -n "${DIR}" ]] ; then
-      if [ -d "${DIR}" ] ; then
-        if [[ ":${LD_LIBRARY_PATH}:" != *":${DIR}:"* ]] ; then
-          LD_LIBRARY_PATH="${DIR}:${LD_LIBRARY_PATH}"
-        else
-          echo -e "LD_LIBRARY_PATH contains '${DIR}'"
-        fi
-      else
-        echo -e "${THIS_FN}::${FUNCNAME[0]}: ERROR: '${DIR}' is not a directory" 1>&2
-      fi
-    else
-      echo -e "${THIS_FN}::${FUNCNAME[0]}: ERROR: DIR not defined" 1>&2
-    fi
-}
-
-# add module=$1 to your  Environment Modules (
-# http://modules.sourceforge.net/
-# TODO: check for existence of module
-function addModule {
-  MODULE="$1"
-  TEMPFILE="$(mktemp)"
-  modulecmd bash add ${MODULE} > ${TEMPFILE}
-  source ${TEMPFILE}
-}
-
-# remove module=$1 from your  Environment Modules (
-# http://modules.sourceforge.net/
-# TODO: check for existence of module
-function removeModule {
-  MODULE="$1"
-  TEMPFILE="$(mktemp)"
-  modulecmd bash rm ${MODULE} > ${TEMPFILE}
-  source ${TEMPFILE}
-}
-
-# # If your computing platform uses Environment Modules (
-# # http://modules.sourceforge.net/
-# # ), load modules for current NCO and IOAPI, noting
-# # how this syntax differs from the commandline.
-# # (Thanks, Barron Henderson for noting this.)
-# # TODO: test for non/existence of paths above!
-# function setupModules {
-#   # for CMD in \
-#   #   "modulecmd bash add ${TERRAE_NCO_MODULE} ${TERRAE_IOAPI_MODULE}" \
-#   # ; do
-#   #   echo -e "$ ${CMD}"
-#   #   eval "${CMD}"
-#   # done
-#   TEMPFILE="$(mktemp)"
-#   modulecmd bash add ${TERRAE_NCO_MODULE} ${TERRAE_IOAPI_MODULE} > ${TEMPFILE}
-#   source ${TEMPFILE}
-# }
-
-# "Comments" lines from running iff _DEBUG='on' (which can be export'ed by caller),
-# and runs with `set xtrace`
-# For `echo`, use DEBUG()
-function DEBUGx {
-  if [[ "${_DEBUG}" == 'on' ]] ; then
-    set -x
-    "$@" 1>&2
-    set +x
-  fi
-} # end function DEBUG
-
-# "Comments" lines from running iff _DEBUG='on'
-# (which can be export'ed by caller)
-function DEBUG {
-  if [[ "${_DEBUG}" == 'on' ]] ; then
-    "$@" 1>&2
-  fi
-} # end function DEBUG

plotLayersForTimestep.r

-# modified from plotLayersForTimestep.r.1
-# * move device control outside of these methods
-# * refactor code for plotting single dataset, for reuse
-
-library(fields)
-library(raster)
-
-# double-sprintf-ing to set precision by constant: cool or brittle?
-stats.precision <- 3 # sigdigs to use for min, median, max of obs
-stat.str <- sprintf('%%.%ig', stats.precision)
-# use these in function=subtitle.stats as sprintf inputs
-max.str <- sprintf('max=%s', stat.str)
-med.str <- sprintf('med=%s', stat.str)
-min.str <- sprintf('min=%s', stat.str)
-
-plot.layers.for.timestep <- function(
-  datavar,          # data variable
-  datavar.name,     # string naming the datavar # TODO: get from datavar
-  datavar.parent,   # file object containing the datavar
-  i.timestep=1,     # index of timestep to plot
-  n.layers=0,       # max number of layers (in timestep) to plot
-  attrs.list,       # list of global attributes
-                    # TODO: handle when null!
-  q.vec=NULL,       # quantile bounds
-  l2d.fp=NULL,      # maps layer# to crop description
-  colors,
-  map
-) {
-  for (i.layer in 1:n.layers) {
-# debugging
-# i.layer <- 1
-
-    # get title string:
-    # minimally:
-    title <- sprintf('%s: layer#=%2i', datavar.name, i.layer)
-    if (!is.null(l2d.fp)) {      # TODO: test file readability
-      l2d.env <- readRDS(l2d.fp) # TODO: test me!
-      title <- sprintf('%s (%s)',
-        title, l2d.env[[as.character(i.layer)]])
-    } else {
-      cat(sprintf(
-        'ERROR: plot.layers.for.timestep: no file mapping layer#s to descriptions\n'))
-    }
-    attr.list <-
-      ncatt_get(datavar.parent, varid=datavar.name, attname="units")
-    if (attr.list$hasatt) {
-      title <- sprintf('%s, units=%s', title, attr.list$value)
-    } else {
-      cat(sprintf(
-        'ERROR: plot.layers.for.timestep: no units for var=%s\n',
-        datavar.name))
-    }
-
-    data <- datavar[,,i.layer]
-# start debugging for Doug Nychka Mon, 13 Feb 2012 21:33:36 -0700
-#    print(paste('class(data)==', class(data), sep=""))
-#   end debugging for Doug Nychka Mon, 13 Feb 2012 21:33:36 -0700
-
-    # get stats for subtitle
-    # to put under title, just create second line (thanks, Doug Nychka)
-    title <- sprintf('%s\n%s', title, subtitle.stats(data))
-
-    plot.layer(data,
-    title=title,
-    attrs.list=attrs.list,
-    q.vec=q.vec,
-    colors=colors,
-    map=map)
-  } # end interating layers
-} # end function plot.layers.for.timestep
-
-subtitle.stats <- function(vec) {
-  return.str <- ""
-  # is it numeric, and not empty?
-  if (is.numeric(vec) && sum(!is.na(vec))) {
-#    unsparse.vec <- subset(vec, !is.na(vec)) # fail: intended for interactive use
-#    unsparse.vec <- na.omit(vec) # fail: omits all *rows* containing an NA!
-    grids <- length(vec)
-    grids.str <- sprintf('(of cells=%i)', grids)
-    unsparse.vec <- vec[!is.na(vec)]
-    obs <- length(unsparse.vec)
-    obs.str <- sprintf('obs=%i', obs)
-    # use constants defined above. TODO: compute these once!
-    max.str <- sprintf(max.str, max(unsparse.vec))
-    med.str <- sprintf(med.str, median(unsparse.vec))
-    min.str <- sprintf(min.str, min(unsparse.vec))
-    return.str <-
-      sprintf('%s %s: %s, %s, %s',
-              obs.str, grids.str, min.str, med.str, max.str)
-  } else {
-    return.str <-"no data"
-  }
-  return.str
-} # end function subtitle.stats
-
-plot.before.and.after.layers.for.timestep <- function(
-  source.datavar,   # source/unmodified data variable
-  target.datavar,   # target/modified data variable
-  datavar.name,     # string naming the datavar
-  i.timestep=1,     # index of timestep to plot
-  datavar.n.layers, # max number of layers (in timestep) to plot
-  attrs.list,       # list of global attributes
-  q.vec=NULL,       # for quantiles # TODO: handle when null!
-  colors,
-  map
-) {
-  for (i.layer in 1:datavar.n.layers) {
-# debugging
-# i.layer <- 1
-
-#    source.data <- source.datavar[,,i.layer,i.timestep]
-    source.data <- source.datavar[,,i.layer]
-#    target.data <- target.datavar[,,i.layer,i.timestep]
-    target.data <- target.datavar[,,i.layer]
-# start debugging for Doug Nychka Mon, 13 Feb 2012 21:33:36 -0700
-#    print(paste('class(source.data)==', class(source.data), sep=""))
-#    print(paste('class(target.data)==', class(target.data), sep=""))
-#   end debugging for Doug Nychka Mon, 13 Feb 2012 21:33:36 -0700
-
-    source.title <-
-      sprintf('%s original: layer#=%2i', datavar.name, i.layer)
-#    source.title <-
-#      sprintf('%s original: layer#=%2i (%s) # when we have the crop name
-    target.title <-
-      sprintf('%s modified: layer#=%2i', datavar.name, i.layer)
-#    target.title <-
-#      sprintf('%s modified: layer#=%2i (%s) # when we have the crop name
-    attr.list <-
-      ncatt_get(datavar.parent, varid=datavar.name, attname="units")
-    if (attr.list$hasatt) {
-      source.title <-
-        sprintf('%s, units=%s', source.title, attr.list$value)
-      target.title <-
-        sprintf('%s, units=%s', target.title, attr.list$value)
-    } else {
-      cat(sprintf(
-        'plot.layers.for.timestep: ERROR: no units for var=%s\n',
-        datavar.name))
-    }
-    plot.layer(source.data,
-      title=source.title,
-#      sub="subtitle",
-      sub=subtitle.stats(data),
-      attrs.list=attrs.list,
-      q.vec=q.vec,
-      colors=colors,
-      map=map)
-    plot.layer(target.data,
-      title=target.title,
-#      sub="subtitle",
-      sub=subtitle.stats(data),
-      attrs.list=attrs.list,
-      q.vec=q.vec,
-      colors=colors,
-      map=map)
-  } # end interating layers
-} # end function plot.before.and.after.layers.for.timestep
-
-plot.layer <- function(
-  data,             # data to plot (required)
-  title,            # string for plot title (required?)
-                    # TODO: handle when null!
-  subtitle=NULL,    # string for plot subtitle
-  attrs.list=NULL,  # list of global attributes (used for plotting)
-  q.vec=NULL,       # for quantiles
-  colors,
-  map
-) {
-  x.centers <- attrs.list$x.cell.centers.km
-  y.centers <- attrs.list$y.cell.centers.km
-  plot.data(data, title, subtitle, x.centers, y.centers, q.vec, colors, map)
-} # end function plot.layer
-
-# return a vector of the centers of the columns of a Raster
-raster.centers.x <- function(raster) {
-  # indices of the cells in the first row==c(1:ncol(r))
-  # xyFromCell(r, c(1:ncol(r))) returns a matrix of the centers of those cells
-  # we want all, and only, the x values (y values are identical),
-  # i.e., column=1, 
-  # and I believe we want them in ascending order (default for `sort`)
-  return(sort(xyFromCell(raster, c(1:ncol(raster)) )[,1]))
-}
-
-# return a vector of the centers of the rows of a Raster
-raster.centers.y <- function(raster) {
-  # indices of the cells in the first col==seq(1, ncell(r), by=ncol(r))
-  # xyFromCell(r, seq(1, ncell(r), by=ncol(r))) returns a matrix of the centers of those cells
-  # we want all, and only, the y values (x values are identical),
-  # i.e., column=2, 
-  # and I believe we want them in ascending order (default for `sort`)
-  return(sort(xyFromCell(raster, seq(1, ncell(raster), by=ncol(raster)))[,2]))
-}
-
-plot.raster <- function(
-  raster,           # data to plot (required)
-  title,            # string for plot title (required?)
-                    # TODO: handle when null!
-  subtitle=NULL,    # string for plot subtitle
-  q.vec=NULL,       # for quantiles
-  colors,
-  map
-) {
-  x.centers <- raster.centers.x(raster)
-  y.centers <- raster.centers.y(raster)
-  # package=fields needs data as matrix, not vector
-  data.mx <- getValues(raster)
-  dim(data.mx) <- c(length(x.centers), length(y.centers)) # cols, rows
-#  plot.data(data.mx, title, subtitle, x.centers, y.centers, q.vec, colors, map)
-  # Above fails: data is reversed relative to map ?!?
-#  plot.mx <- data.mx[nrow(data.mx):1,] # just change the row indices
-  # And reversing matrix rows vertically fails--
-  # it mirrors the plot horizontally/east-west!
-  # What works: reverse the _cols_ horizontally (not transpose)
-  plot.mx <- data.mx[,ncol(data.mx):1] # just change the column indices
-  plot.data(plot.mx, title, subtitle, x.centers, y.centers, q.vec, colors, map)
-} # end function plot.raster
-
-plot.data <- function(
-  data,             # data to plot (required)
-  title,            # string for plot title (required?)
-                    # TODO: handle when null!
-  subtitle=NULL,    # string for plot subtitle
-  x.centers,        # vector of centers of columns
-  y.centers,        # vector of centers of rows
-  q.vec=NULL,       # for quantiles
-  colors,
-  map
-) {
-  if (sum(!is.na(data)) && (!is.null(q.vec))) {
-    plot.list <- list(x=x.centers, y=y.centers, z=data)
-    quantiles <- quantile(c(data), q.vec, na.rm=TRUE)
-    quantiles.formatted <- format(as.numeric(quantiles), digits=3)
-# start debugging
-#      print(paste('Non-null image.plot for source layer==', i.layer, ', quantile bounds=='))
-#      print(quantiles)
-#   end debugging
-    if (is.null(subtitle)) {
-      image.plot(plot.list, xlab="", ylab="", axes=F, col=colors(100),
-        axis.args=list(at=quantiles, labels=quantiles.formatted),
-        main=title)
-    } else {
-      image.plot(plot.list, xlab="", ylab="", axes=F, col=colors(100),
-        axis.args=list(at=quantiles, labels=quantiles.formatted),
-        main=title, sub=subtitle)
-    }
-    lines(map)
-  } else {
-# debugging
-#      print(paste('Null image.plot for source layer=', i.layer))
-    if (is.null(subtitle)) {
-      plot(0, type="n", axes=F, xlab="", ylab="",
-        xlim=range(x.centers), ylim=range(y.centers),
-        main=title)
-    } else {
-      plot(0, type="n", axes=F, xlab="", ylab="",
-        xlim=range(x.centers), ylim=range(y.centers),
-        main=title, sub=subtitle)
-    }
-    lines(map)
-  } # end testing data
-} # end function plot.data

regrid_global_to_AQMEII.r

 #   mean=71.9
 #   q3=85.5
 #   max=523
+#   sum=3.34e+06
 
 # plot to PDF
 
 data(wrld_simpl) # from maptools
 # map.us.unproj <- wrld_simpl[wrld_simpl$ISO3 == 'USA', ]  # unprojected
 library(rgdal)
-# map.us.proj <- rgdal::spTransform(map.us.unproj, sp::CRS(out.proj4))  # projected
-# do North America instead
-map.us.unproj <- wrld_simpl[wrld_simpl$ISO3 %in% c('CAN', 'MEX', 'USA'),]
-map.us.proj <-
-  rgdal::spTransform(map.us.unproj, sp::CRS(out.proj4)) # projected
+map.NorAm.unproj <- wrld_simpl[wrld_simpl$ISO3 %in% c('CAN', 'MEX', 'USA'),]
+map.NorAm.proj <-
+# broken by R package upgrades on terrae 25 Apr 13
+#  rgdal::spTransform(map.NorAm.unproj, sp::CRS(out.proj4)) # projected
+  spTransform(map.NorAm.unproj, sp::CRS(out.proj4)) # projected
 
 title <- sprintf('%s\n%s',
   Sys.getenv('GEIA_REGRID_PDF_TITLE'), Sys.getenv('GEIA_REGRID_DATAVAR_UNIT'))
   xlab='', ylab='', axes=F, col=colors(100),
   axis.args=list(at=quantiles, labels=quantiles.formatted))
 # add a projected CONUS map
-plot(map.us.proj, add=TRUE)
+plot(map.NorAm.proj, add=TRUE)
 
 # plot page 2: fields::image.plot-------------------------------------
 
 
 REPO_ROOT='/tmp' # or location of your choice: repo/workspace created in subdir
 PROJECT_NAME='geia_regrid'
+REPO_DIR="${REPO_ROOT}/${PROJECT_NAME}"
 # what actually does the work
 DRIVER_FN='GEIA_driver.sh'
 
 # if [[ "${ACCESS}" == 'file' ]], you'll need
 FILE_LIST="./${DRIVER_FN} \
-    ./bash_utilities.sh \
+    ./regrid_utils/* \
     ./check_conservation.ncl \
-    ./plotLayersForTimestep.r \
     ./reformat_GEIA_to_netCDF.r \
     ./regrid_global_to_AQMEII.r \
     ./retemp_reunit.ncl"
 
-REPO_DIR="${REPO_ROOT}/${PROJECT_NAME}"
 if [[ -d "${REPO_DIR}" ]] ; then
   echo -e "ERROR? repo dir='${REPO_DIR}' exists: move or delete it before running this script"
   exit 1