Commits

BarryFSmith committed 28176ea

removed all HMPI code
- didn't work any more because of randomized memory messing up function pointers
- complicated code to maintain that no one used and they would prefer threads anyways

  • Participants
  • Parent commits 50339e3

Comments (1)

Files changed (17)

bin/maint/getinterfaces.py

   classes['PetscBinary'] = {}
   classes['PetscOptions'] = {}
   classes['PetscMalloc'] = {}
-  classes['PetscHMPI'] = {}
   classes['PetscToken'] = {}
   for i in args:
     getclasses(i)

include/finclude/petscpcdef.h

 #define PCML 'ml'
 #define PCGALERKIN 'galerkin'
 #define PCEXOTIC 'exotic'
-#define PCHMPI 'hmpi'
 #define PCSUPPORTGRAPH 'supportgraph'
 #define PCASA 'asa'
 #define PCCP 'cp'

include/petsc-private/pcimpl.h

   Mat            mat,pmat;
   Vec            diagonalscaleright,diagonalscaleleft; /* used for time integration scaling */
   PetscBool      diagonalscale;
-  PetscBool      nonzero_guess; /* used by PCKSP, PCREDUNDANT and PCHMPI */
+  PetscBool      nonzero_guess; /* used by PCKSP, PCREDUNDANT */
   PetscBool      useAmat; /* used by several PC that including applying the operator inside the preconditioner */
   PetscErrorCode (*modifysubmatrices)(PC,PetscInt,const IS[],const IS[],Mat[],void*); /* user provided routine */
   void           *modifysubmatricesP; /* context for user routine */

include/petscpc.h

 #define PCML              "ml"
 #define PCGALERKIN        "galerkin"
 #define PCEXOTIC          "exotic"
-#define PCHMPI            "hmpi"
 #define PCASA             "asa"
 #define PCCP              "cp"
 #define PCBFBT            "bfbt"

include/petscsys.h

 PETSC_EXTERN PetscErrorCode PetscEnd(void);
 PETSC_EXTERN PetscErrorCode PetscSysInitializePackage(void);
 
-PETSC_EXTERN MPI_Comm PETSC_COMM_LOCAL_WORLD;
-PETSC_EXTERN PetscErrorCode PetscHMPIMerge(PetscMPIInt,PetscErrorCode (*)(void*),void*);
-PETSC_EXTERN PetscErrorCode PetscHMPISpawn(PetscMPIInt);
-PETSC_EXTERN PetscErrorCode PetscHMPIFinalize(void);
-PETSC_EXTERN PetscErrorCode PetscHMPIRun(MPI_Comm,PetscErrorCode (*)(MPI_Comm,void *),void*);
-PETSC_EXTERN PetscErrorCode PetscHMPIRunCtx(MPI_Comm,PetscErrorCode (*)(MPI_Comm,void*,void *),void*);
-PETSC_EXTERN PetscErrorCode PetscHMPIFree(MPI_Comm,void*);
-PETSC_EXTERN PetscErrorCode PetscHMPIMalloc(MPI_Comm,size_t,void**);
-
 PETSC_EXTERN PetscErrorCode PetscPythonInitialize(const char[],const char[]);
 PETSC_EXTERN PetscErrorCode PetscPythonFinalize(void);
 PETSC_EXTERN PetscErrorCode PetscPythonPrintError(void);

src/ksp/ksp/examples/tests/ex34.c

-
-
-static char help[] = "Demonstrates PetscHMPIMerge() usage\n\n";
-
-#include <petscmat.h>
-#include <petscksp.h>
-
-typedef struct {
-  MPI_Comm   comm;
-  Mat        A;
-  Vec        x,y;      /* contains the vector values spread across all the processes */
-  Vec        xr,yr;    /* contains the vector values on the master processes, all other processes have zero length */
-  VecScatter sct;
-} MyMultCtx;
-
-#undef __FUNCT__
-#define __FUNCT__ "MyMult"
-/*
-    This is called on ALL processess, master and worker
-*/
-PetscErrorCode MyMult(MPI_Comm comm,MyMultCtx *ctx,void *dummy)
-{
-  PetscErrorCode ierr;
-
-  PetscFunctionBeginUser;
-  ierr = PetscSynchronizedPrintf(ctx->comm,"Doing multiply\n");
-  ierr = PetscSynchronizedFlush(ctx->comm,PETSC_STDOUT);CHKERRQ(ierr);
-  /* moves data that lives only on master processes to all processes */
-  ierr = VecScatterBegin(ctx->sct,ctx->xr,ctx->x,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
-  ierr = VecScatterEnd(ctx->sct,ctx->xr,ctx->x,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
-  ierr = MatMult(ctx->A,ctx->x,ctx->y);CHKERRQ(ierr);
-  /* moves data that lives on all processes to master processes */
-  ierr = VecScatterBegin(ctx->sct,ctx->y,ctx->yr,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
-  ierr = VecScatterEnd(ctx->sct,ctx->y,ctx->yr,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "MySubsolver"
-/*
-    This is called only on the master processes
-*/
-PetscErrorCode MySubsolver(MyMultCtx *ctx)
-{
-  PetscErrorCode ierr;
-  void           *subctx;
-
-  PetscFunctionBeginUser;
-  ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"MySubsolver\n");
-  ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
-  /* allocates memory on each process, both masters and workers */
-  ierr = PetscHMPIMalloc(PETSC_COMM_LOCAL_WORLD,sizeof(int),&subctx);CHKERRQ(ierr);
-  /* runs MyMult() function on each process, both masters and workers */
-  ierr = PetscHMPIRunCtx(PETSC_COMM_LOCAL_WORLD,(PetscErrorCode (*)(MPI_Comm,void*,void*))MyMult,subctx);CHKERRQ(ierr);
-  ierr = PetscHMPIFree(PETSC_COMM_LOCAL_WORLD,subctx);CHKERRQ(ierr);
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "main"
-int main(int argc,char **args)
-{
-  PetscErrorCode ierr;
-  char           file[PETSC_MAX_PATH_LEN];
-  PetscViewer    fd;
-  PetscMPIInt    rank,size,nodesize = 1;
-  MyMultCtx      ctx;
-  const PetscInt *ns; /* length of vector ctx.x on all process */
-  PetscInt       i,rstart,n = 0;   /* length of vector ctx.xr on this process */
-  IS             is;
-
-  PetscInitialize(&argc,&args,(char*)0,help);
-  ctx.comm = PETSC_COMM_WORLD;
-  ierr     = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
-  ierr     = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
-  ierr     = PetscOptionsGetInt(NULL,"-nodesize",&nodesize,NULL);CHKERRQ(ierr);
-  if (size % nodesize) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"MPI_COMM_WORLD size must be divisible by nodesize");
-
-  /* Read matrix */
-  ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,NULL);CHKERRQ(ierr);
-  ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr);
-  ierr = MatCreate(PETSC_COMM_WORLD,&ctx.A);CHKERRQ(ierr);
-  ierr = MatSetType(ctx.A,MATMPIAIJ);CHKERRQ(ierr);
-  ierr = MatLoad(ctx.A,fd);CHKERRQ(ierr);
-  ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
-
-  /* Create work vectors for matrix-vector product */
-  ierr = MatGetVecs(ctx.A,&ctx.x,&ctx.y);CHKERRQ(ierr);
-  ierr = VecGetOwnershipRanges(ctx.x,&ns);CHKERRQ(ierr);
-  if (!(rank % nodesize)) { /* I am master process; I will own all vector elements on all my worker processes*/
-    for (i=0; i<nodesize; i++) n += ns[rank+i+1] - ns[rank+i];
-  }
-  ierr = VecCreateMPI(MPI_COMM_WORLD,n,PETSC_DETERMINE,&ctx.xr);CHKERRQ(ierr);
-  ierr = VecDuplicate(ctx.xr,&ctx.yr);CHKERRQ(ierr);
-  /* create scatter from ctx.xr to ctx.x vector */
-  ierr = VecGetOwnershipRange(ctx.x,&rstart,NULL);CHKERRQ(ierr);
-  ierr = ISCreateStride(PETSC_COMM_WORLD,ns[rank],rstart,1,&is);CHKERRQ(ierr);
-  ierr = VecScatterCreate(ctx.xr,is,ctx.x,is,&ctx.sct);CHKERRQ(ierr);
-  ierr = ISDestroy(&is);CHKERRQ(ierr);
-
-  /*
-     The master nodes call the function MySubsolver() while the worker nodes wait for requests to call functions
-     These requests are triggered by the calls from the masters on PetscHMPIRunCtx()
-  */
-  ierr = PetscHMPIMerge(nodesize,(PetscErrorCode (*)(void*))MySubsolver,&ctx);CHKERRQ(ierr);
-
-  ierr = PetscHMPIFinalize();CHKERRQ(ierr);
-  ierr = MatDestroy(&ctx.A);CHKERRQ(ierr);
-  ierr = VecDestroy(&ctx.x);CHKERRQ(ierr);
-  ierr = VecDestroy(&ctx.y);CHKERRQ(ierr);
-  ierr = VecDestroy(&ctx.xr);CHKERRQ(ierr);
-  ierr = VecDestroy(&ctx.yr);CHKERRQ(ierr);
-  ierr = VecScatterDestroy(&ctx.sct);CHKERRQ(ierr);
-
-  ierr = PetscFinalize();
-  return 0;
-}
-

src/ksp/ksp/examples/tests/makefile

 EXAMPLESC       = ex1.c ex3.c ex4.c ex6.c ex7.c ex10.c ex11.c ex14.c \
                 ex15.c ex17.c ex18.c ex19.c ex20.c ex21.c ex22.c ex24.c \
                 ex25.c ex26.c ex27.c ex28.c ex29.c ex30.c ex31.c ex32.c \
-                ex33.c ex34.c ex35.c ex36.c ex37.c ex38.c ex39.c ex40.c ex41.c ex42.c \
+                ex33.c ex35.c ex36.c ex37.c ex38.c ex39.c ex40.c ex41.c ex42.c \
                 ex43.c ex44.c
 EXAMPLESCH      =
 EXAMPLESF       = ex5f.F ex12f.F ex16f.F
 	   else echo ${PWD} ; echo "Possible problem with with ex33_mumps_2, diffs above \n========================================="; fi; \
 	   ${RM} -f ex33.tmp
 # See http://www.mcs.anl.gov/petsc/documentation/faq.html#datafiles for how to obtain the datafiles used below
-runex34:
-	-@${MPIEXEC} -n 2 ./ex34 -f  ${DATAFILESPATH}/matrices/small -nodesize 2 > ex34.tmp 2>&1; \
-	if (${DIFF} output/ex34_1.out ex34.tmp) then true; \
-	   else echo ${PWD} ; echo "Possible problem with with ex34, diffs above \n========================================="; fi; \
-	   ${RM} -f ex34.tmp
-
 runex35_1:
 	-@${MPIEXEC} -n 1 ./ex35 -dof 1 -mat_view -check_final_residual> ex35.tmp 2>&1;\
 	if (${DIFF} output/ex35_1.out ex35.tmp) then true; \

src/ksp/ksp/examples/tutorials/makefile

 	${RM} -f ex10_19aij.tmp ex10_19sbaij.tmp
 # See http://www.mcs.anl.gov/petsc/documentation/faq.html#datafiles for how to obtain the datafiles used below
 
-# Start a sequential user code [on 1 node, assembling seqaij] and then
-# spwan a parallel solver on 2 procs.
-# See http://www.mcs.anl.gov/petsc/documentation/faq.html#datafiles for how to obtain the datafiles used below
-runex10_22:
-	-@${MPIEXEC} -n 2 ./ex10 -hmpi_merge_size 2 -pc_type hmpi  -ksp_type preonly -hmpi_pc_type ksp -f0 ${DATAFILESPATH}/matrices/medium > ex10_22.tmp 2>&1;\
-	   if (${DIFF} output/ex10_22.out ex10_22.tmp) then true; \
-	   else echo ${PWD} ; echo "Possible problem with with ex10_22, diffs above \n========================================="; fi; \
-	   ${RM} -f ex10_22.tmp
-runex10_23:
-	-@${MPIEXEC} -n 2 ./ex10 -hmpi_merge_size 2 -pc_type hmpi  -ksp_type preonly -hmpi_pc_type ksp -hmpi_ksp_pc_type bjacobi -hmpi_ksp_ksp_type gmres -f0 ${DATAFILESPATH}/matrices/medium > ex10_23.tmp 2>&1;\
-	   if (${DIFF} output/ex10_23.out ex10_23.tmp) then true; \
-	   else echo ${PWD} ; echo "Possible problem with with ex10_23, diffs above \n========================================="; fi; \
-	   ${RM} -f ex10_23.tmp
-
-runex10_24:
-	-@${MPIEXEC} -n 2 ./ex10 -hmpi_merge_size 2 -pc_type hmpi -hmpi_pc_type sor -f0 ${DATAFILESPATH}/matrices/medium -hmpi_ksp_monitor_short -initialguess -ksp_type gmres -ksp_monitor_short -ksp_view > ex10_24.tmp 2>&1;\
-	   if (${DIFF} output/ex10_24.out ex10_24.tmp) then true; \
-	   else echo ${PWD} ; echo "Possible problem with with ex10_24, diffs above \n========================================="; fi; \
-	   ${RM} -f ex10_24.tmp
-
-# Start a parallel user code [on 4 nodes, assembling MPIAIJ with
-# np=4] and then spwan a parallel sub-domain-solver on each node
-# [with np=2]. This emulates mixed MPI-shared memory model [MPI between
-# nodes, MPI within the nodes]
-# See http://www.mcs.anl.gov/petsc/documentation/faq.html#datafiles for how to obtain the datafiles used below
-runex10_25:
-	-@${MPIEXEC} -n 8 ./ex10 -hmpi_merge_size 2 -sub_pc_type hmpi -f0 ${DATAFILESPATH}/matrices/medium -ksp_monitor_short> ex10_25.tmp 2>&1;\
-	   if (${DIFF} output/ex10_25.out ex10_25.tmp) then true; \
-	   else echo ${PWD} ; echo "Possible problem with with ex10_25, diffs above \n========================================="; fi; \
-	   ${RM} -f ex10_25.tmp
-
 runex10_superlu_lu_1:
 	-@${MPIEXEC} -n 1 ./ex10 -f0 ${DATAFILESPATH}/matrices/small -ksp_type preonly -pc_type lu -pc_factor_mat_solver_package superlu -num_numfac 2 -num_rhs 2 > ex10_superlu_lu_1.tmp 2>&1; \
 	   if (${DIFF} output/ex10_mumps.out ex10_superlu_lu_1.tmp) then true; \
                                  ex23.PETSc runex23 ex23.rm
 TESTEXAMPLES_C_COMPLEX	       = ex10.PETSc ex10.rm ex11.PETSc runex11 ex11.rm ex39.PETSc runex39 ex39.rm ex40.PETSc runex40 ex40.rm
 TESTEXAMPLES_DATAFILESPATH     = ex10.PETSc runex10_2 runex10_3 runex10_4 runex10_5 runex10_6 runex10_7 runex10_8 \
-                                 runex10_9 runex10_10 runex10_19 runex10_22 runex10_23 runex10_24 runex10_25  runex10_ILU runex10_ILUBAIJ runex10_cg \
+                                 runex10_9 runex10_10 runex10_19  runex10_ILU runex10_ILUBAIJ runex10_cg \
                                  runex10_cg_singlereduction runex10_seqaijcrl runex10_mpiaijcrl runex10_seqaijperm runex10_mpiaijperm ex10.rm
 # even though ex10.c is -pc_mg_smoothdown na C example to run with -mat_type lusol requires a Fortran compiler, hence
 # we list it with the fortran examples

src/ksp/pc/impls/makefile

 
 LIBBASE  = libpetscksp
 DIRS     = jacobi none sor shell bjacobi mg eisens asm ksp composite redundant spai is pbjacobi ml\
-           mat hypre tfs fieldsplit factor galerkin openmp asa cp wb python ainvcusp sacusp bicgstabcusp\
+           mat hypre tfs fieldsplit factor galerkin asa cp wb python ainvcusp sacusp bicgstabcusp\
            lsc redistribute gasm svd gamg parms bddc
 LOCDIR   = src/ksp/pc/impls/
 

src/ksp/pc/impls/openmp/hpc.c

-
-#include <petsc-private/pcimpl.h>     /*I "petscpc.h" I*/
-#include <petscksp.h>
-
-typedef struct {
-  MatStructure flag;               /* pc->flag */
-  PetscInt     setupcalled;        /* pc->setupcalled */
-  PetscInt     n;
-  MPI_Comm     comm;                 /* local world used by this preconditioner */
-  KSP          ksp;                  /* actual solver used across local world */
-  Mat          mat;                  /* matrix in local world */
-  Mat          gmat;                 /* matrix known only to process 0 in the local world */
-  Vec          x,y,xdummy,ydummy;
-  VecScatter   scatter;
-  PetscBool    nonzero_guess;
-} PC_HMPI;
-
-
-#undef __FUNCT__
-#define __FUNCT__ "PCView_HMPI_MP"
-/*
-    Would like to have this simply call PCView() on the inner PC. The problem is
-  that the outer comm does not live on the inside so cannot do this. Instead
-  handle the special case when the viewer is stdout, construct a new one just
-  for this call.
-*/
-
-static PetscErrorCode PCView_HMPI_MP(MPI_Comm comm,void *ctx)
-{
-  PC_HMPI        *red = (PC_HMPI*)ctx;
-  PetscErrorCode ierr;
-  PetscViewer    viewer;
-
-  PetscFunctionBegin;
-  ierr = PetscViewerASCIIGetStdout(comm,&viewer);CHKERRQ(ierr);
-  ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);         /* this is bogus in general */
-  ierr = KSPView(red->ksp,viewer);CHKERRQ(ierr);
-  ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PCView_HMPI"
-static PetscErrorCode PCView_HMPI(PC pc,PetscViewer viewer)
-{
-  PC_HMPI        *red = (PC_HMPI*)pc->data;
-  PetscMPIInt    size;
-  PetscErrorCode ierr;
-  PetscBool      iascii;
-
-  PetscFunctionBegin;
-  ierr = MPI_Comm_size(red->comm,&size);CHKERRQ(ierr);
-  ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);CHKERRQ(ierr);
-  if (iascii) {
-    ierr = PetscViewerASCIIPrintf(viewer,"  Size of solver nodes %d\n",size);CHKERRQ(ierr);
-    ierr = PetscViewerASCIIPrintf(viewer,"  Parallel sub-solver given next\n",size);CHKERRQ(ierr);
-    /* should only make the next call if the viewer is associated with stdout */
-    ierr = PetscHMPIRun(red->comm,PCView_HMPI_MP,red);CHKERRQ(ierr);
-  }
-  PetscFunctionReturn(0);
-}
-
-PETSC_EXTERN PetscErrorCode MatDistribute_MPIAIJ(MPI_Comm,Mat,PetscInt,MatReuse,Mat*);
-
-#undef __FUNCT__
-#define __FUNCT__ "PCApply_HMPI_1"
-static PetscErrorCode PCApply_HMPI_1(PC pc,Vec x,Vec y)
-{
-  PC_HMPI        *red = (PC_HMPI*)pc->data;
-  PetscErrorCode ierr;
-
-  PetscFunctionBegin;
-  ierr = KSPSetInitialGuessNonzero(red->ksp,pc->nonzero_guess);CHKERRQ(ierr);
-  ierr = KSPSolve(red->ksp,x,y);CHKERRQ(ierr);
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PCSetUp_HMPI_MP"
-static PetscErrorCode PCSetUp_HMPI_MP(MPI_Comm comm,void *ctx)
-{
-  PC_HMPI        *red = (PC_HMPI*)ctx;
-  PetscErrorCode ierr;
-  PetscInt       m;
-  MatReuse       scal;
-  PetscMPIInt    rank;
-
-  PetscFunctionBegin;
-  red->comm = comm;
-
-  ierr = MPI_Bcast(&red->setupcalled,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-  ierr = MPI_Bcast((PetscEnum*)&red->flag,1,MPIU_ENUM,0,comm);CHKERRQ(ierr);
-  if (!red->setupcalled) {
-    /* setup vector communication */
-    ierr = MPI_Bcast(&red->n,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-    ierr = VecCreateMPI(comm,PETSC_DECIDE,red->n,&red->x);CHKERRQ(ierr);
-    ierr = VecCreateMPI(comm,PETSC_DECIDE,red->n,&red->y);CHKERRQ(ierr);
-    ierr = VecScatterCreateToZero(red->x,&red->scatter,&red->xdummy);CHKERRQ(ierr);
-    ierr = VecDuplicate(red->xdummy,&red->ydummy);CHKERRQ(ierr);
-    ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
-    if (!rank) {
-      ierr = VecDestroy(&red->xdummy);CHKERRQ(ierr);
-      ierr = VecDestroy(&red->ydummy);CHKERRQ(ierr);
-    }
-    scal = MAT_INITIAL_MATRIX;
-  } else {
-    if (red->flag == DIFFERENT_NONZERO_PATTERN) {
-      ierr = MatDestroy(&red->mat);CHKERRQ(ierr);
-      scal = MAT_INITIAL_MATRIX;
-    } else {
-      scal = MAT_REUSE_MATRIX;
-    }
-  }
-
-  /* copy matrix out onto processes */
-  ierr = VecGetLocalSize(red->x,&m);CHKERRQ(ierr);
-  ierr = MatDistribute_MPIAIJ(comm,red->gmat,m,scal,&red->mat);CHKERRQ(ierr);
-  if (!red->setupcalled) {
-    /* create the solver */
-    ierr = KSPCreate(comm,&red->ksp);CHKERRQ(ierr);
-    /* would like to set proper tablevel for KSP, but do not have direct access to parent pc */
-    ierr = KSPSetOptionsPrefix(red->ksp,"hmpi_");CHKERRQ(ierr); /* should actually append with global pc prefix */
-    ierr = KSPSetOperators(red->ksp,red->mat,red->mat,red->flag);CHKERRQ(ierr);
-    ierr = KSPSetFromOptions(red->ksp);CHKERRQ(ierr);
-  } else {
-    ierr = KSPSetOperators(red->ksp,red->mat,red->mat,red->flag);CHKERRQ(ierr);
-  }
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PCSetUp_HMPI"
-static PetscErrorCode PCSetUp_HMPI(PC pc)
-{
-  PC_HMPI        *red = (PC_HMPI*)pc->data;
-  PetscErrorCode ierr;
-  PetscMPIInt    size;
-
-  PetscFunctionBegin;
-  red->gmat        = pc->mat;
-  red->flag        = pc->flag;
-  red->setupcalled = pc->setupcalled;
-
-  ierr = MPI_Comm_size(red->comm,&size);CHKERRQ(ierr);
-  if (size == 1) {  /* special case where copy of matrix is not needed */
-    if (!red->setupcalled) {
-      /* create the solver */
-      ierr = KSPCreate(PetscObjectComm((PetscObject)pc),&red->ksp);CHKERRQ(ierr);
-      ierr = PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);CHKERRQ(ierr);
-      ierr = KSPSetOptionsPrefix(red->ksp,"hmpi_");CHKERRQ(ierr); /* should actually append with global pc prefix */
-      ierr = KSPSetOperators(red->ksp,red->gmat,red->gmat,red->flag);CHKERRQ(ierr);
-      ierr = KSPSetFromOptions(red->ksp);CHKERRQ(ierr);
-    } else {
-      ierr = KSPSetOperators(red->ksp,red->gmat,red->gmat,red->flag);CHKERRQ(ierr);
-    }
-    pc->ops->apply = PCApply_HMPI_1;
-    PetscFunctionReturn(0);
-  } else {
-    ierr = MatGetSize(pc->mat,&red->n,PETSC_IGNORE);CHKERRQ(ierr);
-    ierr = PetscHMPIRun(red->comm,PCSetUp_HMPI_MP,red);CHKERRQ(ierr);
-  }
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PCApply_HMPI_MP"
-static PetscErrorCode PCApply_HMPI_MP(MPI_Comm comm,void *ctx)
-{
-  PC_HMPI        *red = (PC_HMPI*)ctx;
-  PetscErrorCode ierr;
-
-  PetscFunctionBegin;
-  ierr = VecScatterBegin(red->scatter,red->xdummy,red->x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
-  ierr = VecScatterEnd(red->scatter,red->xdummy,red->x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
-  ierr = MPI_Bcast(&red->nonzero_guess,1,MPIU_BOOL,0,red->comm);CHKERRQ(ierr);
-  if (red->nonzero_guess) {
-    ierr = VecScatterBegin(red->scatter,red->ydummy,red->y,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
-    ierr = VecScatterEnd(red->scatter,red->ydummy,red->y,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
-  }
-  ierr = KSPSetInitialGuessNonzero(red->ksp,red->nonzero_guess);CHKERRQ(ierr);
-
-  ierr = KSPSolve(red->ksp,red->x,red->y);CHKERRQ(ierr);
-
-  ierr = VecScatterBegin(red->scatter,red->y,red->ydummy,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
-  ierr = VecScatterEnd(red->scatter,red->y,red->ydummy,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PCApply_HMPI"
-static PetscErrorCode PCApply_HMPI(PC pc,Vec x,Vec y)
-{
-  PC_HMPI        *red = (PC_HMPI*)pc->data;
-  PetscErrorCode ierr;
-
-  PetscFunctionBegin;
-  red->xdummy        = x;
-  red->ydummy        = y;
-  red->nonzero_guess = pc->nonzero_guess;
-
-  ierr = PetscHMPIRun(red->comm,PCApply_HMPI_MP,red);CHKERRQ(ierr);
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PCDestroy_HMPI_MP"
-static PetscErrorCode PCDestroy_HMPI_MP(MPI_Comm comm,void *ctx)
-{
-  PC_HMPI        *red = (PC_HMPI*)ctx;
-  PetscMPIInt    rank;
-  PetscErrorCode ierr;
-
-  PetscFunctionBegin;
-  ierr = VecScatterDestroy(&red->scatter);CHKERRQ(ierr);
-  ierr = VecDestroy(&red->x);CHKERRQ(ierr);
-  ierr = VecDestroy(&red->y);CHKERRQ(ierr);
-  ierr = KSPDestroy(&red->ksp);CHKERRQ(ierr);
-  ierr = MatDestroy(&red->mat);CHKERRQ(ierr);
-  ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
-  if (rank) {
-    ierr = VecDestroy(&red->xdummy);CHKERRQ(ierr);
-    ierr = VecDestroy(&red->ydummy);CHKERRQ(ierr);
-  }
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PCDestroy_HMPI"
-static PetscErrorCode PCDestroy_HMPI(PC pc)
-{
-  PC_HMPI        *red = (PC_HMPI*)pc->data;
-  PetscErrorCode ierr;
-
-  PetscFunctionBegin;
-  ierr = PetscHMPIRun(red->comm,PCDestroy_HMPI_MP,red);CHKERRQ(ierr);
-  ierr = PetscHMPIFree(red->comm,red);CHKERRQ(ierr);
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PCSetFromOptions_HMPI"
-static PetscErrorCode PCSetFromOptions_HMPI(PC pc)
-{
-  PetscFunctionBegin;
-  PetscFunctionReturn(0);
-}
-
-
-/* -------------------------------------------------------------------------------------*/
-/*MC
-     PCHMPI - Runs a preconditioner for a single process matrix across several MPI processes
-
-$     This will usually be run with -pc_type hmpi -ksp_type preonly
-$     solver options are set with -hmpi_ksp_... and -hmpi_pc_... for example
-$     -hmpi_ksp_type cg would use cg as the Krylov method or -hmpi_ksp_monitor or
-$     -hmpi_pc_type hypre -hmpi_pc_hypre_type boomeramg
-
-       Always run with -ksp_view (or -snes_view) to see what solver is actually being used.
-
-       Currently the solver options INSIDE the HMPI preconditioner can ONLY be set via the
-      options database.
-
-   Level: intermediate
-
-   See PetscHMPIMerge() and PetscHMPISpawn() for two ways to start up MPI for use with this preconditioner
-
-.seealso:  PCCreate(), PCSetType(), PCType (for list of available types)
-
-M*/
-
-#undef __FUNCT__
-#define __FUNCT__ "PCCreate_HMPI"
-PETSC_EXTERN PetscErrorCode PCCreate_HMPI(PC pc)
-{
-  PetscErrorCode ierr;
-  PC_HMPI        *red;
-  PetscMPIInt    size;
-
-  PetscFunctionBegin;
-  ierr = MPI_Comm_size(PetscObjectComm((PetscObject)pc),&size);CHKERRQ(ierr);
-  if (size > 1) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_SIZ,"HMPI preconditioner only works for sequential solves");
-  if (!PETSC_COMM_LOCAL_WORLD) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"PETSc not initialized for PCMPI see the manual pages for PetscHMPISpawn() and PetscHMPIMerge()");
-  /* caste the struct length to a PetscInt for easier MPI calls */
-
-  ierr      = PetscHMPIMalloc(PETSC_COMM_LOCAL_WORLD,(PetscInt)sizeof(PC_HMPI),(void**)&red);CHKERRQ(ierr);
-  red->comm = PETSC_COMM_LOCAL_WORLD;
-  pc->data  = (void*) red;
-
-  pc->ops->apply          = PCApply_HMPI;
-  pc->ops->destroy        = PCDestroy_HMPI;
-  pc->ops->setfromoptions = PCSetFromOptions_HMPI;
-  pc->ops->setup          = PCSetUp_HMPI;
-  pc->ops->view           = PCView_HMPI;
-  PetscFunctionReturn(0);
-}

src/ksp/pc/impls/openmp/makefile

-
-ALL: lib
-
-CFLAGS   =
-FFLAGS   =
-SOURCEC  = hpc.c
-SOURCEF  =
-SOURCEH  =
-LIBBASE  = libpetscksp
-DIRS     =
-MANSEC   = PC
-LOCDIR   = src/ksp/pc/impls/openmp/
-
-include ${PETSC_DIR}/conf/variables
-include ${PETSC_DIR}/conf/rules
-include ${PETSC_DIR}/conf/test

src/ksp/pc/interface/pcregis.c

 PETSC_EXTERN PetscErrorCode PCCreate_Cholesky(PC);
 PETSC_EXTERN PetscErrorCode PCCreate_FieldSplit(PC);
 PETSC_EXTERN PetscErrorCode PCCreate_Galerkin(PC);
-PETSC_EXTERN PetscErrorCode PCCreate_HMPI(PC);
 PETSC_EXTERN PetscErrorCode PCCreate_Exotic(PC);
 PETSC_EXTERN PetscErrorCode PCCreate_ASA(PC);
 PETSC_EXTERN PetscErrorCode PCCreate_CP(PC);
   ierr = PCRegister(PCFIELDSPLIT   ,PCCreate_FieldSplit);CHKERRQ(ierr);
   ierr = PCRegister(PCGALERKIN     ,PCCreate_Galerkin);CHKERRQ(ierr);
   ierr = PCRegister(PCEXOTIC       ,PCCreate_Exotic);CHKERRQ(ierr);
-  ierr = PCRegister(PCHMPI         ,PCCreate_HMPI);CHKERRQ(ierr);
   ierr = PCRegister(PCASA          ,PCCreate_ASA);CHKERRQ(ierr);
   ierr = PCRegister(PCCP           ,PCCreate_CP);CHKERRQ(ierr);
   ierr = PCRegister(PCLSC          ,PCCreate_LSC);CHKERRQ(ierr);

src/ksp/pc/interface/precon.c

    Notes:
     This is a weird function. Since PC's are linear operators on the right hand side they
     CANNOT use an initial guess. This function is for the "pass-through" preconditioners
-    PCKSP, PCREDUNDANT and PCHMPI and causes the inner KSP object to use the nonzero
+    PCKSP and PCREDUNDANT  and causes the inner KSP object to use the nonzero
     initial guess. Not currently working for PCREDUNDANT, that has to be rewritten to use KSP.
 
 

src/sys/objects/ftn-custom/zstart.c

 #endif
 #include <petscthreadcomm.h>
 
-extern PetscBool PetscHMPIWorker;
-
 #if defined(PETSC_HAVE_FORTRAN_CAPS)
 #define petscinitialize_              PETSCINITIALIZE
 #define petscfinalize_                PETSCFINALIZE
   PetscMPIInt size;
   char        *t1,name[256],hostname[64];
   PetscMPIInt f_petsc_comm_world;
-  PetscInt    nodesize;
-  PetscBool   flg;
 
   *ierr = PetscMemzero(name,256); if (*ierr) return;
   if (PetscInitializeCalled) {*ierr = 0; return;}
   if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:PetscStackCreate()\n");return;}
 #endif
 
-  *ierr = PetscOptionsGetInt(NULL,"-hmpi_spawn_size",&nodesize,&flg);
-  if (flg) {
-#if defined(PETSC_HAVE_MPI_COMM_SPAWN)
-    *ierr = PetscHMPISpawn((PetscMPIInt) nodesize); /* worker nodes never return from here; they go directly to PetscEnd() */
-    if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:PetscHMPIS-pawn()\n");return;}
-#else
-    *ierr = PETSC_ERR_SUP;
-    (*PetscErrorPrintf)("PetscInitialize: PETSc built without MPI 2 (MPI_Comm_spawn) support, use -hmpi_merge_size instead");
-    return;
-#endif
-  } else {
-    *ierr = PetscOptionsGetInt(NULL,"-hmpi_merge_size",&nodesize,&flg);
-    if (flg) {
-      *ierr = PetscHMPIMerge((PetscMPIInt) nodesize,NULL,NULL);
-      if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:PetscHMPIMerge()\n");return;}
-      if (PetscHMPIWorker) { /* if worker then never enter user code */
-        PetscInitializeCalled = PETSC_TRUE;
-        *ierr = PetscEnd();
-      }
-    }
-  }
-
 #if defined(PETSC_HAVE_CUDA)
   flg  = PETSC_TRUE;
   *ierr = PetscOptionsGetBool(NULL,"-cublas",&flg,NULL);

src/sys/objects/makefile

 CPPFLAGS  =
 SOURCEC	  = version.c gcomm.c   gtype.c   olist.c    pname.c  tagm.c \
             destroy.c gcookie.c inherit.c options.c pgname.c prefix.c init.c \
-	    pinit.c ptype.c state.c aoptions.c mpinit.c subcomm.c fcallback.c
+	    pinit.c ptype.c state.c aoptions.c subcomm.c fcallback.c
 SOURCEF	  =
 SOURCEH	  = ../../../include/petscoptions.h
 MANSEC	  = Sys

src/sys/objects/mpinit.c

-
-
-#include <petscsys.h>        /*I  "petscsys.h"   I*/
-
-static MPI_Comm saved_PETSC_COMM_WORLD = 0;
-MPI_Comm        PETSC_COMM_LOCAL_WORLD = 0;           /* comm for a single node (local set of processes) */
-PetscBool       PetscHMPIWorker        = PETSC_FALSE; /* this is a regular process, nonworker process */
-void            * PetscHMPICtx         = 0;
-
-extern PetscErrorCode  PetscHMPIHandle(MPI_Comm);
-
-#if defined(PETSC_HAVE_MPI_COMM_SPAWN)
-#undef __FUNCT__
-#define __FUNCT__ "PetscHMPISpawn"
-/*@C
-   PetscHMPISpawn - Initialize additional processes to be used as "worker" processes. This is not generally
-     called by users. One should use -hmpi_spawn_size <n> to indicate that you wish to have n-1 new MPI
-     processes spawned for each current process.
-
-   Not Collective (could make collective on MPI_COMM_WORLD, generate one huge comm and then split it up)
-
-   Input Parameter:
-.  nodesize - size of each compute node that will share processors
-
-   Options Database:
-.   -hmpi_spawn_size nodesize
-
-   Notes: This is only supported on systems with an MPI 2 implementation that includes the MPI_Comm_Spawn() routine.
-
-$    Comparison of two approaches for HMPI usage (MPI started with N processes)
-$
-$    -hmpi_spawn_size <n> requires MPI 2, results in n*N total processes with N directly used by application code
-$                                           and n-1 worker processes (used by PETSc) for each application node.
-$                           You MUST launch MPI so that only ONE MPI process is created for each hardware node.
-$
-$    -hmpi_merge_size <n> results in N total processes, N/n used by the application code and the rest worker processes
-$                            (used by PETSc)
-$                           You MUST launch MPI so that n MPI processes are created for each hardware node.
-$
-$    petscmpiexec -n 2 ./ex1 -hmpi_spawn_size 3 gives 2 application nodes (and 4 PETSc worker nodes)
-$    petscmpiexec -n 6 ./ex1 -hmpi_merge_size 3 gives the SAME 2 application nodes and 4 PETSc worker nodes
-$       This is what would use if each of the computers hardware nodes had 3 CPUs.
-$
-$      These are intended to be used in conjunction with USER HMPI code. The user will have 1 process per
-$   computer (hardware) node (where the computer node has p cpus), the user's code will use threads to fully
-$   utilize all the CPUs on the node. The PETSc code will have p processes to fully use the compute node for
-$   PETSc calculations. The user THREADS and PETSc PROCESSES will NEVER run at the same time so the p CPUs
-$   are always working on p task, never more than p.
-$
-$    See PCHMPI for a PETSc preconditioner that can use this functionality
-$
-
-   For both PetscHMPISpawn() and PetscHMPIMerge() PETSC_COMM_WORLD consists of one process per "node", PETSC_COMM_LOCAL_WORLD
-   consists of all the processes in a "node."
-
-   In both cases the user's code is running ONLY on PETSC_COMM_WORLD (that was newly generated by running this command).
-
-   Level: developer
-
-   Concepts: HMPI
-
-.seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscHMPIFinalize(), PetscInitialize(), PetscHMPIMerge(), PetscHMPIRun()
-
-@*/
-PetscErrorCode  PetscHMPISpawn(PetscMPIInt nodesize)
-{
-
-  SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"HMPI functionality is currently broken");
-#if defined(broken_functionality_commented_out)
-  PetscErrorCode ierr;
-  PetscMPIInt    size;
-  MPI_Comm       parent,children;
-
-  PetscFunctionBegin;
-  ierr = MPI_Comm_get_parent(&parent);CHKERRQ(ierr);
-  if (parent == MPI_COMM_NULL) {  /* the original processes started by user */
-    char programname[PETSC_MAX_PATH_LEN];
-    char **argv;
-
-    ierr = PetscGetProgramName(programname,PETSC_MAX_PATH_LEN);CHKERRQ(ierr);
-    ierr = PetscGetArguments(&argv);CHKERRQ(ierr);
-    ierr = MPI_Comm_spawn(programname,argv,nodesize-1,MPI_INFO_NULL,0,PETSC_COMM_SELF,&children,MPI_ERRCODES_IGNORE);CHKERRQ(ierr);
-    ierr = PetscFreeArguments(argv);CHKERRQ(ierr);
-    ierr = MPI_Intercomm_merge(children,0,&PETSC_COMM_LOCAL_WORLD);CHKERRQ(ierr);
-
-    ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
-    ierr = PetscInfo2(0,"PETSc HMPI successfully spawned: number of nodes = %d node size = %d\n",size,nodesize);CHKERRQ(ierr);
-
-    saved_PETSC_COMM_WORLD = PETSC_COMM_WORLD;
-  } else { /* worker nodes that get spawned */
-    ierr            = MPI_Intercomm_merge(parent,1,&PETSC_COMM_LOCAL_WORLD);CHKERRQ(ierr);
-    ierr            = PetscHMPIHandle(PETSC_COMM_LOCAL_WORLD);CHKERRQ(ierr);
-    PetscHMPIWorker = PETSC_TRUE; /* so that PetscHMPIFinalize() will not attempt a broadcast from this process */
-    PetscEnd();  /* cannot continue into user code */
-  }
-  PetscFunctionReturn(0);
-#endif
-}
-#endif
-
-#undef __FUNCT__
-#define __FUNCT__ "PetscHMPIMerge"
-/*@C
-   PetscHMPIMerge - Initializes the PETSc and MPI to work with HMPI. This is not usually called
-      by the user. One should use -hmpi_merge_size <n> to indicate the node size of merged communicator
-      to be.
-
-   Collective on MPI_COMM_WORLD or PETSC_COMM_WORLD if it has been set
-
-   Input Parameter:
-+  nodesize - size of each compute node that will share processors
-.  func - optional function to call on the master nodes
--  ctx - context passed to function on master nodes
-
-   Options Database:
-.   -hmpi_merge_size <n>
-
-   Level: developer
-
-$    Comparison of two approaches for HMPI usage (MPI started with N processes)
-$
-$    -hmpi_spawn_size <n> requires MPI 2, results in n*N total processes with N directly used by application code
-$                                           and n-1 worker processes (used by PETSc) for each application node.
-$                           You MUST launch MPI so that only ONE MPI process is created for each hardware node.
-$
-$    -hmpi_merge_size <n> results in N total processes, N/n used by the application code and the rest worker processes
-$                            (used by PETSc)
-$                           You MUST launch MPI so that n MPI processes are created for each hardware node.
-$
-$    petscmpiexec -n 2 ./ex1 -hmpi_spawn_size 3 gives 2 application nodes (and 4 PETSc worker nodes)
-$    petscmpiexec -n 6 ./ex1 -hmpi_merge_size 3 gives the SAME 2 application nodes and 4 PETSc worker nodes
-$       This is what would use if each of the computers hardware nodes had 3 CPUs.
-$
-$      These are intended to be used in conjunction with USER HMPI code. The user will have 1 process per
-$   computer (hardware) node (where the computer node has p cpus), the user's code will use threads to fully
-$   utilize all the CPUs on the node. The PETSc code will have p processes to fully use the compute node for
-$   PETSc calculations. The user THREADS and PETSc PROCESSES will NEVER run at the same time so the p CPUs
-$   are always working on p task, never more than p.
-$
-$    See PCHMPI for a PETSc preconditioner that can use this functionality
-$
-
-   For both PetscHMPISpawn() and PetscHMPIMerge() PETSC_COMM_WORLD consists of one process per "node", PETSC_COMM_LOCAL_WORLD
-   consists of all the processes in a "node."
-
-   In both cases the user's code is running ONLY on PETSC_COMM_WORLD (that was newly generated by running this command).
-
-   Concepts: HMPI
-
-.seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscHMPIFinalize(), PetscInitialize(), PetscHMPISpawn(), PetscHMPIRun()
-
-@*/
-PetscErrorCode  PetscHMPIMerge(PetscMPIInt nodesize,PetscErrorCode (*func)(void*),void *ctx)
-{
-
-  SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"HMPI functionality is currently broken");
-#if defined(broken_functionality_commented_out)
-  PetscErrorCode ierr;
-  PetscMPIInt    size,rank,*ranks,i;
-  MPI_Group      group,newgroup;
-
-  PetscFunctionBegin;
-  saved_PETSC_COMM_WORLD = PETSC_COMM_WORLD;
-
-  ierr = MPI_Comm_size(saved_PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
-  if (size % nodesize) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Total number of process nodes %d is not divisible by number of processes per node %d",size,nodesize);
-  ierr = MPI_Comm_rank(saved_PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
-
-
-  /* create two communicators
-      *) one that contains the first process from each node: 0,nodesize,2*nodesize,...
-      *) one that contains all processes in a node:  (0,1,2...,nodesize-1), (nodesize,nodesize+1,...2*nodesize-), ...
-  */
-  ierr = MPI_Comm_group(saved_PETSC_COMM_WORLD,&group);CHKERRQ(ierr);
-  ierr = PetscMalloc((size/nodesize)*sizeof(PetscMPIInt),&ranks);CHKERRQ(ierr);
-  for (i=0; i<(size/nodesize); i++) ranks[i] = i*nodesize;
-  ierr = MPI_Group_incl(group,size/nodesize,ranks,&newgroup);CHKERRQ(ierr);
-  ierr = PetscFree(ranks);CHKERRQ(ierr);
-  ierr = MPI_Comm_create(saved_PETSC_COMM_WORLD,newgroup,&PETSC_COMM_WORLD);CHKERRQ(ierr);
-  if (rank % nodesize) PETSC_COMM_WORLD = 0; /* mark invalid processes for easy debugging */
-  ierr = MPI_Group_free(&group);CHKERRQ(ierr);
-  ierr = MPI_Group_free(&newgroup);CHKERRQ(ierr);
-
-  ierr = MPI_Comm_split(saved_PETSC_COMM_WORLD,rank/nodesize,rank % nodesize,&PETSC_COMM_LOCAL_WORLD);CHKERRQ(ierr);
-
-  ierr = PetscInfo2(0,"PETSc HMPI successfully started: number of nodes = %d node size = %d\n",size/nodesize,nodesize);CHKERRQ(ierr);
-  ierr = PetscInfo1(0,"PETSc HMPI process %sactive\n",(rank % nodesize) ? "in" : "");CHKERRQ(ierr);
-
-  PetscHMPICtx = ctx;
-  /*
-     All process not involved in user application code wait here
-  */
-  if (!PETSC_COMM_WORLD) {
-    ierr             = PetscHMPIHandle(PETSC_COMM_LOCAL_WORLD);CHKERRQ(ierr);
-    PETSC_COMM_WORLD = saved_PETSC_COMM_WORLD;
-    PetscHMPIWorker  = PETSC_TRUE; /* so that PetscHMPIFinalize() will not attempt a broadcast from this process */
-    ierr             = PetscInfo(0,"PETSc HMPI inactive process becoming active");CHKERRQ(ierr);
-  } else if (func) {
-    ierr = (*func)(ctx);CHKERRQ(ierr);
-  }
-  PetscFunctionReturn(0);
-#endif
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PetscHMPIFinalize"
-/*@C
-   PetscHMPIFinalize - Finalizes the PETSc and MPI to work with HMPI. Called by PetscFinalize() cannot
-       be called by user.
-
-   Collective on the entire system
-
-   Level: developer
-
-.seealso: PetscFinalize(), PetscGetArgs(), PetscHMPIMerge(), PCHMPIRun()
-
-@*/
-PetscErrorCode  PetscHMPIFinalize(void)
-{
-  PetscErrorCode ierr    = 0;
-  PetscInt       command = 3;
-
-  PetscFunctionBegin;
-  if (!PetscHMPIWorker && PETSC_COMM_LOCAL_WORLD) {
-    ierr = MPI_Bcast(&command,1,MPIU_INT,0,PETSC_COMM_LOCAL_WORLD);CHKERRQ(ierr); /* broadcast to my worker group to end program */
-
-    PETSC_COMM_WORLD = saved_PETSC_COMM_WORLD;
-
-    ierr = PetscInfo(0,"PETSc HMPI active process ending PetscHMPIMerge()");CHKERRQ(ierr);
-  }
-  PetscFunctionReturn(ierr);
-}
-
-static PetscInt numberobjects = 0;
-static void     *objects[100];
-
-#undef __FUNCT__
-#define __FUNCT__ "PetscHMPIHandle"
-/*@C
-   PetscHMPIHandle - Receives commands from the master node and processes them
-
-   Collective on MPI_Comm
-
-   Input Parameter:
-.   comm - Must be PETSC_COMM_LOCAL_WORLD
-
-   Level: developer
-
-   Notes: this is usually handled automatically, likely you do not need to use this directly
-
-   Developer Notes: Since comm must be PETSC_COMM_LOCAL_WORLD, why have this argument?
-
-.seealso: PetscHMPIMerge(), PCHMPIRun(), PCHMPINew()
-
-@*/
-PetscErrorCode  PetscHMPIHandle(MPI_Comm comm)
-{
-  PetscErrorCode ierr;
-  PetscInt       command       = 0; /* dummy value so MPI-Uni doesn't think it is not set*/
-  PetscBool      exitwhileloop = PETSC_FALSE;
-
-  PetscFunctionBegin;
-  while (!exitwhileloop) {
-    ierr = MPI_Bcast(&command,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-    switch (command) {
-    case 0: { /* allocate some memory on this worker process */
-      size_t n = 0;   /* dummy value so MPI-Uni doesn't think it is not set*/
-      void   *ptr;
-      ierr = MPI_Bcast(&n,1,MPIU_SIZE_T,0,comm);CHKERRQ(ierr);
-      /* cannot use PetscNew() cause it requires struct argument */
-      ierr = PetscMalloc(n,&ptr);CHKERRQ(ierr);
-      ierr = PetscMemzero(ptr,n);CHKERRQ(ierr);
-
-      objects[numberobjects++] = ptr;
-      break;
-    }
-    case 1: {  /* free some memory on this worker process */
-      PetscInt i;
-      ierr = MPI_Bcast(&i,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-      ierr = PetscFree(objects[i]);CHKERRQ(ierr);
-      break;
-    }
-    case 2: {  /* run a function on this worker process */
-      PetscInt       i;
-      PetscErrorCode (*f)(MPI_Comm,void*);
-      ierr = MPI_Bcast(&i,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-      ierr = MPI_Bcast((PETSC_UINTPTR_T*)&f,1,MPIU_SIZE_T,0,comm);CHKERRQ(ierr);
-      ierr = (*f)(comm,objects[i]);CHKERRQ(ierr);
-      break;
-    }
-    case 4: {  /* run a function on this worker process with provided context */
-      PetscInt       i;
-      PetscErrorCode (*f)(MPI_Comm,void*,void*);
-      ierr = MPI_Bcast(&i,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-      ierr = MPI_Bcast((PETSC_UINTPTR_T*)&f,1,MPIU_SIZE_T,0,comm);CHKERRQ(ierr);
-      ierr = (*f)(comm,PetscHMPICtx,objects[i]);CHKERRQ(ierr);
-      break;
-    }
-    case 3: {
-      exitwhileloop = PETSC_TRUE;
-      break;
-    }
-    default:
-      SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unknown HMPI command %D",command);
-    }
-  }
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PetscHMPIMalloc"
-/*@C
-   PetscHMPIMalloc - Creates a "c struct" on all nodes of an HMPI communicator
-
-   Collective on MPI_Comm
-
-   Input Parameters:
-+   comm - Must be PETSC_COMM_LOCAL_WORLD
--   n  - amount of memory requested
-
-   Level: developer
-
-   Developer Notes: Since comm must be PETSC_COMM_LOCAL_WORLD, why have this argument?
-
-.seealso: PetscHMPIMerge(), PCHMPIRun(), PCHMPIFree()
-
-@*/
-PetscErrorCode  PetscHMPIMalloc(MPI_Comm comm,size_t n,void **ptr)
-{
-  PetscErrorCode ierr;
-  PetscInt       command = 0;
-
-  PetscFunctionBegin;
-  if (PetscHMPIWorker) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Not using HMPI feature of PETSc");
-
-  ierr = MPI_Bcast(&command,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-  ierr = MPI_Bcast(&n,1,MPIU_SIZE_T,0,comm);CHKERRQ(ierr);
-
-  /* cannot use PetscNew() cause it requires struct argument */
-  ierr = PetscMalloc(n,ptr);CHKERRQ(ierr);
-  ierr = PetscMemzero(*ptr,n);CHKERRQ(ierr);
-
-  objects[numberobjects++] = *ptr;
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PetscHMPIFree"
-/*@C
-   PetscHMPIFree - Frees a "c struct" on all nodes of an HMPI communicator
-
-   Collective on MPI_Comm
-
-   Input Parameters:
-+   comm - Must be PETSC_COMM_LOCAL_WORLD
--   ptr - pointer to data to be freed, must have been obtained with PetscHMPIMalloc()
-
-   Level: developer
-
-  Developer Notes: Since comm must be PETSC_COMM_LOCAL_WORLD, why have this argument?
-
-.seealso: PetscHMPIMerge(), PetscHMPIMalloc()
-
-@*/
-PetscErrorCode  PetscHMPIFree(MPI_Comm comm,void *ptr)
-{
-  PetscErrorCode ierr;
-  PetscInt       command = 1,i;
-
-  PetscFunctionBegin;
-  if (PetscHMPIWorker) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Not using HMPI feature of PETSc");
-
-  ierr = MPI_Bcast(&command,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-  for (i=0; i<numberobjects; i++) {
-    if (objects[i] == ptr) {
-      ierr = MPI_Bcast(&i,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-      ierr = PetscFree(objects[i]);CHKERRQ(ierr);
-      PetscFunctionReturn(0);
-    }
-  }
-  SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Pointer does not appear to have been created with PetscHMPIMalloc()");
-  PetscFunctionReturn(ierr);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PetscHMPIRun"
-/*@C
-   PetscHMPIRun - runs a function on all the processes of a node
-
-   Collective on MPI_Comm
-
-   Input Parameters:
-+   comm - communicator to run function on, must be PETSC_COMM_LOCAL_WORLD
-.   f - function to run
--   ptr - pointer to data to pass to function; must be obtained with PetscHMPIMalloc()
-
-   Level: developer
-
-   Developer Notes: Since comm must be PETSC_COMM_LOCAL_WORLD, why have this argument?
-
-.seealso: PetscHMPIMerge(), PetscHMPIMalloc(), PetscHMPIFree(), PetscHMPIRunCtx()
-
-@*/
-PetscErrorCode  PetscHMPIRun(MPI_Comm comm,PetscErrorCode (*f)(MPI_Comm,void*),void *ptr)
-{
-  PetscErrorCode ierr;
-  PetscInt       command = 2,i;
-
-  PetscFunctionBegin;
-  if (PetscHMPIWorker) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Not using HMPI feature of PETSc");
-
-  ierr = MPI_Bcast(&command,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-  for (i=0; i<numberobjects; i++) {
-    if (objects[i] == ptr) {
-      ierr = MPI_Bcast(&i,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-      ierr = MPI_Bcast((PETSC_UINTPTR_T*)&f,1,MPIU_SIZE_T,0,comm);CHKERRQ(ierr);
-      ierr = (*f)(comm,ptr);CHKERRQ(ierr);
-      PetscFunctionReturn(0);
-    }
-  }
-  SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Pointer does not appear to have been created with PetscHMPIMalloc()");
-  PetscFunctionReturn(ierr);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PetscHMPIRunCtx"
-/*@C
-   PetscHMPIRunCtx - runs a function on all the processes of a node
-
-   Collective on MPI_Comm
-
-   Input Parameters:
-+   comm - communicator to run function on, must be PETSC_COMM_LOCAL_WORLD
-.   f - function to run
--   ptr - pointer to data to pass to function; must be obtained with PetscHMPIMalloc()
-
-   Notes: This is like PetscHMPIRun() except it also passes the context passed in PetscHMPIMerge()
-   Level: developer
-
-   Developer Notes: Since comm must be PETSC_COMM_LOCAL_WORLD, why have this argument?
-
-.seealso: PetscHMPIMerge(), PetscHMPIMalloc(), PetscHMPIFree(), PetscHMPIRun()
-
-@*/
-PetscErrorCode  PetscHMPIRunCtx(MPI_Comm comm,PetscErrorCode (*f)(MPI_Comm,void*,void*),void *ptr)
-{
-  PetscErrorCode ierr;
-  PetscInt       command = 4,i;
-
-  PetscFunctionBegin;
-  if (PetscHMPIWorker) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Not using HMPI feature of PETSc");
-
-  ierr = MPI_Bcast(&command,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-  for (i=0; i<numberobjects; i++) {
-    if (objects[i] == ptr) {
-      ierr = MPI_Bcast(&i,1,MPIU_INT,0,comm);CHKERRQ(ierr);
-      ierr = MPI_Bcast((PETSC_UINTPTR_T*)&f,1,MPIU_SIZE_T,0,comm);CHKERRQ(ierr);
-      ierr = (*f)(comm,PetscHMPICtx,ptr);CHKERRQ(ierr);
-      PetscFunctionReturn(0);
-    }
-  }
-  SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Pointer does not appear to have been created with PetscHMPIMalloc()");
-  PetscFunctionReturn(ierr);
-}

src/sys/objects/pinit.c

 #if defined(PETSC_USE_LOG)
 extern PetscErrorCode PetscLogBegin_Private(void);
 #endif
-extern PetscBool PetscHMPIWorker;
 
 #if defined(PETSC_SERIALIZE_FUNCTIONS)
 PetscFPT PetscFPTData = 0;
 {
   PetscErrorCode ierr;
   PetscMPIInt    flag, size;
-  PetscInt       nodesize;
   PetscBool      flg;
   char           hostname[256];
 
 #endif
 #endif
 
-  ierr = PetscOptionsGetInt(NULL,"-hmpi_spawn_size",&nodesize,&flg);CHKERRQ(ierr);
-  if (flg) {
-#if defined(PETSC_HAVE_MPI_COMM_SPAWN)
-    ierr = PetscHMPISpawn((PetscMPIInt) nodesize);CHKERRQ(ierr); /* worker nodes never return from here; they go directly to PetscEnd() */
-#else
-    SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"PETSc built without MPI 2 (MPI_Comm_spawn) support, use -hmpi_merge_size instead");
-#endif
-  } else {
-    ierr = PetscOptionsGetInt(NULL,"-hmpi_merge_size",&nodesize,&flg);CHKERRQ(ierr);
-    if (flg) {
-      ierr = PetscHMPIMerge((PetscMPIInt) nodesize,NULL,NULL);CHKERRQ(ierr);
-      if (PetscHMPIWorker) { /* if worker then never enter user code */
-        PetscInitializeCalled = PETSC_TRUE;
-        PetscEnd();
-      }
-    }
-  }
-
 #if defined(PETSC_HAVE_CUDA)
   flg  = PETSC_TRUE;
   ierr = PetscOptionsGetBool(NULL,"-cublas",&flg,NULL);CHKERRQ(ierr);
   }
 #endif
 
-  ierr = PetscHMPIFinalize();CHKERRQ(ierr);
-
   ierr = PetscOptionsGetBool(NULL,"-malloc_info",&flg2,NULL);CHKERRQ(ierr);
   if (!flg2) {
     flg2 = PETSC_FALSE;
   ierr = PetscOptionsHasName(NULL,"-nox",&flg1);CHKERRQ(ierr);
   ierr = PetscOptionsHasName(NULL,"-nox_warning",&flg1);CHKERRQ(ierr);
 
-  if (!PetscHMPIWorker) { /* worker processes skip this because they do not usually process options */
-    flg3 = PETSC_FALSE; /* default value is required */
-    ierr = PetscOptionsGetBool(NULL,"-options_left",&flg3,&flg1);CHKERRQ(ierr);
-    ierr = PetscOptionsAllUsed(&nopt);CHKERRQ(ierr);
-    if (flg3) {
-      if (!flg2) { /* have not yet printed the options */
-        PetscViewer viewer;
-        ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr);
-        ierr = PetscOptionsView(viewer);CHKERRQ(ierr);
-        ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
-      }
-      if (!nopt) {
-        ierr = PetscPrintf(PETSC_COMM_WORLD,"There are no unused options.\n");CHKERRQ(ierr);
-      } else if (nopt == 1) {
-        ierr = PetscPrintf(PETSC_COMM_WORLD,"There is one unused database option. It is:\n");CHKERRQ(ierr);
-      } else {
-        ierr = PetscPrintf(PETSC_COMM_WORLD,"There are %D unused database options. They are:\n",nopt);CHKERRQ(ierr);
-      }
+  flg3 = PETSC_FALSE; /* default value is required */
+  ierr = PetscOptionsGetBool(NULL,"-options_left",&flg3,&flg1);CHKERRQ(ierr);
+  ierr = PetscOptionsAllUsed(&nopt);CHKERRQ(ierr);
+  if (flg3) {
+    if (!flg2) { /* have not yet printed the options */
+      PetscViewer viewer;
+      ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr);
+      ierr = PetscOptionsView(viewer);CHKERRQ(ierr);
+      ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
+    }
+    if (!nopt) {
+      ierr = PetscPrintf(PETSC_COMM_WORLD,"There are no unused options.\n");CHKERRQ(ierr);
+    } else if (nopt == 1) {
+      ierr = PetscPrintf(PETSC_COMM_WORLD,"There is one unused database option. It is:\n");CHKERRQ(ierr);
+    } else {
+      ierr = PetscPrintf(PETSC_COMM_WORLD,"There are %D unused database options. They are:\n",nopt);CHKERRQ(ierr);
     }
 #if defined(PETSC_USE_DEBUG)
     if (nopt && !flg3 && !flg1) {