Commits

Jed Brown committed fc4a80a

Rename some threadcomm impl files to avoid multiple files with same name

Hg-commit: 811a350d363e866edc3e3ebd77231296a2d23565

  • Participants
  • Parent commits 6eb941e

Comments (0)

Files changed (7)

src/sys/threadcomm/impls/pthread/lockfree.c

-#include <../src/sys/threadcomm/impls/pthread/pthreadimpl.h>
-
-#define THREAD_TERMINATE      -1
-#define THREAD_WAITING_FOR_JOB 0
-#define THREAD_RECIEVED_JOB    1
-#define THREAD_INITIALIZE      2
-
-/* lock-free data structure */
-typedef struct {
-  PetscThreadCommJobCtx *data;
-  PetscInt           *my_job_status;
-} sjob_lockfree;
-
-static sjob_lockfree job_lockfree = {NULL,NULL};
-
-
-void SparkThreads_LockFree(PetscInt myrank,PetscThreadComm tcomm,PetscThreadCommJobCtx job)
-{
-  PetscInt i,thread_num;
-  PetscThreadComm_PThread  ptcomm;
-  PetscInt                 next;
-
-  ptcomm = (PetscThreadComm_PThread)tcomm->data;
-  
-  switch(ptcomm->spark) {
-  case PTHREADPOOLSPARK_LEADER:
-    if(PetscReadOnce(int,tcomm->leader) == myrank) {
-      /* Only leader sparks all the other threads */
-      for(i=ptcomm->thread_num_start+1; i < tcomm->nworkThreads;i++) {
-	thread_num = ptcomm->granks[i];
-	while(PetscReadOnce(int,job_lockfree.my_job_status[thread_num]) != THREAD_WAITING_FOR_JOB)
-	  ;
-	job_lockfree.data[thread_num] = job;
-	job_lockfree.my_job_status[thread_num] = THREAD_RECIEVED_JOB;
-      }
-    }
-    break;
-  case PTHREADPOOLSPARK_CHAIN:
-    /* Spark the next thread */
-    next = ptcomm->ngranks[myrank];
-    if(next != -1) {
-      thread_num = next;
-      while(PetscReadOnce(int,job_lockfree.my_job_status[thread_num]) != THREAD_WAITING_FOR_JOB)
-	;
-      job_lockfree.data[thread_num] = job;
-      job_lockfree.my_job_status[thread_num] = THREAD_RECIEVED_JOB;
-    }
-    break;
-  }
-}
-
-void* PetscPThreadCommFunc_LockFree(void* arg)
-{
-
-#if defined(PETSC_PTHREAD_LOCAL)
-  PetscPThreadRank = *(PetscInt*)arg;
-#else
-  PetscInt PetscPThreadRank=*(PetscInt*)arg;
-  pthread_setspecific(PetscPThreadRankkey,&PetscPThreadRank);
-#endif
-
-#if defined(PETSC_HAVE_SCHED_CPU_SET_T)
-  PetscPThreadCommDoCoreAffinity();
-#endif
-  job_lockfree.my_job_status[PetscPThreadRank] = THREAD_WAITING_FOR_JOB;
-
-  /* Spin loop */
-  while(PetscReadOnce(int,job_lockfree.my_job_status[PetscPThreadRank]) != THREAD_TERMINATE) {
-    if(job_lockfree.my_job_status[PetscPThreadRank] == THREAD_RECIEVED_JOB) {
-      /* Spark the thread pool */
-      SparkThreads_LockFree(PetscPThreadRank,job_lockfree.data[PetscPThreadRank]->tcomm,job_lockfree.data[PetscPThreadRank]);
-      /* Do own job */
-      PetscRunKernel(PetscPThreadRank,job_lockfree.data[PetscPThreadRank]->nargs,job_lockfree.data[PetscPThreadRank]);
-      /* Reset own status */ 
-      job_lockfree.my_job_status[PetscPThreadRank] = THREAD_WAITING_FOR_JOB;
-    }
-  }
-
-  return NULL;
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PetscThreadCommBarrier_PThread_LockFree"
-PetscErrorCode PetscThreadCommBarrier_PThread_LockFree(PetscThreadComm tcomm)
-{
-  PetscInt active_threads=0,i;
-  PetscBool wait=PETSC_TRUE;
-  PetscThreadComm_PThread ptcomm=(PetscThreadComm_PThread)tcomm->data;
-
-  PetscFunctionBegin;
-  /* Loop till all threads signal that they have done their job */
-  while(wait) {
-    for(i=0;i<tcomm->nworkThreads;i++) active_threads += job_lockfree.my_job_status[ptcomm->granks[i]];
-    if(active_threads) active_threads = 0;
-    else wait=PETSC_FALSE;
-  }
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PetscPThreadCommInitialize_LockFree"
-PetscErrorCode PetscPThreadCommInitialize_LockFree(PetscThreadComm tcomm)
-{
-  PetscErrorCode ierr;
-  PetscInt       i;
-  PetscThreadComm_PThread ptcomm=(PetscThreadComm_PThread)tcomm->data;
-
-  PetscFunctionBegin;
-
-  ierr = PetscMalloc(tcomm->nworkThreads*sizeof(PetscThreadCommJobCtx),&job_lockfree.data);CHKERRQ(ierr);
-  ierr = PetscMalloc(tcomm->nworkThreads*sizeof(PetscInt),&job_lockfree.my_job_status);CHKERRQ(ierr);
-
-  /* Create threads */
-  for(i=ptcomm->thread_num_start; i < tcomm->nworkThreads;i++) {
-    job_lockfree.my_job_status[i] = THREAD_INITIALIZE;
-    ierr = pthread_create(&ptcomm->tid[i],NULL,&PetscPThreadCommFunc_LockFree,&ptcomm->granks[i]);CHKERRQ(ierr);
-  }
-  if(ptcomm->ismainworker) {
-    job_lockfree.my_job_status[0] = THREAD_WAITING_FOR_JOB;
-  }
-
-  /* Put a barrier so that all threads get pinned properly */
-  ierr = PetscThreadCommBarrier_PThread_LockFree(tcomm);CHKERRQ(ierr);
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PetscPThreadCommFinalize_LockFree"
-PetscErrorCode PetscPThreadCommFinalize_LockFree(PetscThreadComm tcomm)
-{
-  PetscErrorCode           ierr;
-  void*                    jstatus;
-  PetscThreadComm_PThread  ptcomm=(PetscThreadComm_PThread)tcomm->data;
-  PetscInt                 i;
-  PetscFunctionBegin;
-  ierr = PetscThreadCommBarrier_PThread_LockFree(tcomm);CHKERRQ(ierr);
-  for(i=ptcomm->thread_num_start; i < tcomm->nworkThreads;i++) {
-    job_lockfree.my_job_status[i] = THREAD_TERMINATE;
-    ierr = pthread_join(ptcomm->tid[i],&jstatus);CHKERRQ(ierr);
-  }
-  ierr = PetscFree(job_lockfree.my_job_status);CHKERRQ(ierr);
-  ierr = PetscFree(job_lockfree.data);CHKERRQ(ierr);
-
-  PetscFunctionReturn(0);
-}
-
-#undef __FUNCT__
-#define __FUNCT__ "PetscThreadCommRunKernel_PThread_LockFree"
-PetscErrorCode PetscThreadCommRunKernel_PThread_LockFree(MPI_Comm comm,PetscThreadCommJobCtx job)
-{
-  PetscErrorCode           ierr;
-  PetscThreadComm          tcomm=0;
-  PetscThreadComm_PThread  ptcomm;    
-  
-  PetscFunctionBegin;
-  ierr = PetscCommGetThreadComm(comm,&tcomm);CHKERRQ(ierr);
-  ptcomm = (PetscThreadComm_PThread)tcomm->data;
-  if(ptcomm->nthreads) {
-    PetscInt thread_num;
-    /* Spark the leader thread */
-    thread_num = tcomm->leader;
-    /* Wait till the leader thread has finished its previous job */
-    while(PetscReadOnce(int,job_lockfree.my_job_status[thread_num]) != THREAD_WAITING_FOR_JOB)
-      ;
-    job_lockfree.data[thread_num] = job;
-    job_lockfree.my_job_status[thread_num] = THREAD_RECIEVED_JOB;
-  }
-  if(ptcomm->ismainworker) {
-    job_lockfree.my_job_status[0] = THREAD_RECIEVED_JOB;
-    job_lockfree.data[0] = job;
-    PetscRunKernel(0,job->nargs, job_lockfree.data[0]);
-    job_lockfree.my_job_status[0] = THREAD_WAITING_FOR_JOB;
-  }
-  if(ptcomm->synchronizeafter) {
-    ierr = PetscThreadCommBarrier(comm);CHKERRQ(ierr);
-  }
-  PetscFunctionReturn(0);
-}

src/sys/threadcomm/impls/pthread/makefile

 
 CFLAGS   =
 FFLAGS   =
-SOURCEC  = pthread.c lockfree.c
+SOURCEC  = tcpthread.c tclockfree.c
 SOURCEF  =
-SOURCEH  = pthreadimpl.h
-OBJSC    = pthread.o lockfree.o
+SOURCEH  = tcpthreadimpl.h
+OBJSC    = $(SOURCEC:.c=.o)
 OBJSF    =
 LIBBASE  = libpetscsys
 MANSEC   = PetscThreadComm

src/sys/threadcomm/impls/pthread/pthread.c

-/* Define feature test macros to make sure CPU_SET and other functions are available
- */
-#define PETSC_DESIRE_FEATURE_TEST_MACROS
-
-#include <../src/sys/threadcomm/impls/pthread/pthreadimpl.h>
-
-#if defined(PETSC_PTHREAD_LOCAL)
-PETSC_PTHREAD_LOCAL PetscInt PetscPThreadRank;
-#else
-pthread_key_t PetscPThreadRankkey;
-#endif
-
-static PetscBool PetscPThreadCommInitializeCalled = PETSC_FALSE;
-
-const char *const PetscPThreadCommSynchronizationTypes[] = {"LOCKFREE","PetscPThreadCommSynchronizationType","PTHREADSYNC_",0};
-const char *const PetscPThreadCommAffinityPolicyTypes[] = {"ALL","ONECORE","NONE","PetscPThreadCommAffinityPolicyType","PTHREADAFFPOLICY_",0};
-const char *const PetscPThreadCommPoolSparkTypes[] = {"LEADER","CHAIN","PetscPThreadCommPoolSparkType","PTHREADPOOLSPARK_",0};
-
-static PetscInt ptcommcrtct = 0; /* PThread communicator creation count. Incremented whenever a pthread
-                                    communicator is created and decremented when it is destroyed. On the
-                                    last pthread communicator destruction, the thread pool is also terminated
-                                  */
-
-PetscInt PetscThreadCommGetRank_PThread()
-{
-#if defined(PETSC_PTHREAD_LOCAL)
-  return PetscPThreadRank;
-#else
-  return *((PetscInt*)pthread_getspecific(PetscPThreadRankkey));
-#endif
-}
-
-
-#if defined(PETSC_HAVE_SCHED_CPU_SET_T)
-void PetscPThreadCommDoCoreAffinity(void)
-{
-  PetscInt                 i,icorr=0; 
-  cpu_set_t                mset;
-  PetscInt                 ncores,myrank=PetscThreadCommGetRank_PThread();
-  PetscThreadComm          tcomm;
-  PetscThreadComm_PThread  gptcomm;
-  
-  PetscCommGetThreadComm(PETSC_COMM_WORLD,&tcomm);
-  PetscGetNCores(&ncores);
-  gptcomm=(PetscThreadComm_PThread)tcomm->data;
-  switch(gptcomm->aff) {
-  case PTHREADAFFPOLICY_ONECORE:
-    icorr = tcomm->affinities[myrank];
-    CPU_ZERO(&mset);
-    CPU_SET(icorr%ncores,&mset);
-    pthread_setaffinity_np(pthread_self(),sizeof(cpu_set_t),&mset);
-    break;
-  case PTHREADAFFPOLICY_ALL:
-    CPU_ZERO(&mset);
-    for(i=0;i<ncores;i++) CPU_SET(i,&mset);
-    pthread_setaffinity_np(pthread_self(),sizeof(cpu_set_t),&mset);
-    break;
-  case PTHREADAFFPOLICY_NONE:
-    break;
-  }
-}
-#endif
-
-#undef __FUNCT__
-#define __FUNCT__ "PetscThreadCommDestroy_PThread"
-PetscErrorCode PetscThreadCommDestroy_PThread(PetscThreadComm tcomm)
-{
-  PetscThreadComm_PThread ptcomm=(PetscThreadComm_PThread)tcomm->data;
-  PetscErrorCode          ierr;
-
-  PetscFunctionBegin;
-  if(!ptcomm) PetscFunctionReturn(0);
-  ptcommcrtct--;
-  if(!ptcommcrtct) {
-    /* Terminate the thread pool */
-    ierr = (*ptcomm->finalize)(tcomm);CHKERRQ(ierr);
-    ierr = PetscFree(ptcomm->tid);CHKERRQ(ierr);
-  }
-  ierr = PetscFree(ptcomm->granks);CHKERRQ(ierr);
-  if(ptcomm->spark == PTHREADPOOLSPARK_CHAIN) {
-    ierr = PetscFree(ptcomm->ngranks);CHKERRQ(ierr);
-  }
-  ierr = PetscFree(ptcomm);CHKERRQ(ierr);
-  PetscFunctionReturn(0);
-}
-
-EXTERN_C_BEGIN
-#undef __FUNCT__
-#define __FUNCT__ "PetscThreadCommCreate_PThread"
-PetscErrorCode PetscThreadCommCreate_PThread(PetscThreadComm tcomm)
-{
-  PetscThreadComm_PThread ptcomm;
-  PetscErrorCode          ierr;
-  PetscInt                i;
-
-  PetscFunctionBegin;
-  ptcommcrtct++;
-  ierr = PetscStrcpy(tcomm->type,PTHREAD);CHKERRQ(ierr);
-  ierr = PetscNew(struct _p_PetscThreadComm_PThread,&ptcomm);CHKERRQ(ierr);
-  tcomm->data = (void*)ptcomm;
-  ptcomm->nthreads = 0;
-  ptcomm->sync = PTHREADSYNC_LOCKFREE;
-  ptcomm->aff = PTHREADAFFPOLICY_ONECORE;
-  ptcomm->spark = PTHREADPOOLSPARK_LEADER;
-  ptcomm->ismainworker = PETSC_TRUE;
-  ptcomm->synchronizeafter = PETSC_TRUE;
-  tcomm->ops->destroy = PetscThreadCommDestroy_PThread;
-  tcomm->ops->runkernel = PetscThreadCommRunKernel_PThread_LockFree;
-  tcomm->ops->barrier   = PetscThreadCommBarrier_PThread_LockFree;
-  tcomm->ops->getrank   = PetscThreadCommGetRank_PThread;
-
-  ierr = PetscMalloc(tcomm->nworkThreads*sizeof(PetscInt),&ptcomm->granks);CHKERRQ(ierr);
-
-  if(!PetscPThreadCommInitializeCalled) { /* Only done for PETSC_THREAD_COMM_WORLD */
-    PetscPThreadCommInitializeCalled = PETSC_TRUE;
-    PetscBool               flg1,flg2,flg3,flg4;
-
-    ierr = PetscOptionsBegin(PETSC_COMM_WORLD,PETSC_NULL,"PThread communicator options",PETSC_NULL);CHKERRQ(ierr);
-    ierr = PetscOptionsBool("-threadcomm_pthread_main_is_worker","Main thread is also a worker thread",PETSC_NULL,PETSC_TRUE,&ptcomm->ismainworker,&flg1);CHKERRQ(ierr);
-    ierr = PetscOptionsEnum("-threadcomm_pthread_affpolicy","Thread affinity policy"," ",PetscPThreadCommAffinityPolicyTypes,(PetscEnum)ptcomm->aff,(PetscEnum*)&ptcomm->aff,&flg2);CHKERRQ(ierr);
-    ierr = PetscOptionsEnum("-threadcomm_pthread_type","Thread pool type"," ",PetscPThreadCommSynchronizationTypes,(PetscEnum)ptcomm->sync,(PetscEnum*)&ptcomm->sync,&flg3);CHKERRQ(ierr);
-    ierr = PetscOptionsEnum("-threadcomm_pthread_spark","Thread pool spark type"," ",PetscPThreadCommPoolSparkTypes,(PetscEnum)ptcomm->spark,(PetscEnum*)&ptcomm->spark,&flg4);CHKERRQ(ierr);
-    ierr = PetscOptionsBool("-threadcomm_pthread_synchronizeafter","Puts a barrier after every kernel call",PETSC_NULL,PETSC_TRUE,&ptcomm->synchronizeafter,&flg1);CHKERRQ(ierr);
-    ierr = PetscOptionsEnd();CHKERRQ(ierr);
-
-    if(ptcomm->ismainworker) {
-      ptcomm->nthreads = tcomm->nworkThreads-1;
-      ptcomm->thread_num_start = 1;
-    } else {
-      ptcomm->nthreads = tcomm->nworkThreads;
-      ptcomm->thread_num_start = 0;
-    }
-
-    switch(ptcomm->sync) {
-    case PTHREADSYNC_LOCKFREE:
-      ptcomm->initialize      = PetscPThreadCommInitialize_LockFree;
-      ptcomm->finalize        = PetscPThreadCommFinalize_LockFree;
-      tcomm->ops->runkernel   = PetscThreadCommRunKernel_PThread_LockFree;
-      tcomm->ops->barrier     = PetscThreadCommBarrier_PThread_LockFree;
-      break;
-    default:
-      SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Only Lock-free synchronization scheme supported currently");
-    }
-    /* Set up thread ranks */
-    for(i=0;i< tcomm->nworkThreads;i++) ptcomm->granks[i] = i;
-
-    if(ptcomm->ismainworker) {
-#if defined(PETSC_PTHREAD_LOCAL)
-      PetscPThreadRank=0; /* Main thread rank */
-#else
-      ierr = pthread_key_create(&PetscPThreadRankkey,NULL);CHKERRQ(ierr);
-      ierr = pthread_setspecific(PetscPThreadRankkey,&ptcomm->granks[0]);CHKERRQ(ierr);
-#endif
-    }
-    /* Set the leader thread rank */
-    if(ptcomm->nthreads) {
-      if(ptcomm->ismainworker) tcomm->leader = ptcomm->granks[1];
-      else tcomm->leader = ptcomm->granks[0];
-    }
-  
-    if(ptcomm->spark == PTHREADPOOLSPARK_CHAIN) {
-      ierr = PetscMalloc(tcomm->nworkThreads*sizeof(PetscInt),&ptcomm->ngranks);CHKERRQ(ierr);
-      for(i=ptcomm->thread_num_start;i < tcomm->nworkThreads-1;i++) ptcomm->ngranks[i] = ptcomm->granks[i+1];
-      ptcomm->ngranks[tcomm->nworkThreads-1] = -1;
-    }
-
-    /* Create array holding pthread ids */
-    ierr = PetscMalloc(tcomm->nworkThreads*sizeof(pthread_t),&ptcomm->tid);CHKERRQ(ierr);
-
-    /* Set affinity of the main thread */
-    
-#if defined(PETSC_HAVE_SCHED_CPU_SET_T)
-    cpu_set_t mset;
-    PetscInt  ncores,icorr;
-    
-
-    ierr = PetscGetNCores(&ncores);CHKERRQ(ierr);
-    CPU_ZERO(&mset);
-    icorr = tcomm->affinities[0]%ncores;
-    CPU_SET(icorr,&mset);
-    sched_setaffinity(0,sizeof(cpu_set_t),&mset);
-#endif
-
-    /* Initialize thread pool */
-    ierr = (*ptcomm->initialize)(tcomm);CHKERRQ(ierr);
-
-  } else {
-    PetscThreadComm          gtcomm;
-    PetscThreadComm_PThread  gptcomm;
-    PetscInt                 *granks,j,*gaffinities;
-
-    ierr = PetscCommGetThreadComm(PETSC_COMM_WORLD,&gtcomm);CHKERRQ(ierr);
-    gaffinities = gtcomm->affinities;
-    gptcomm = (PetscThreadComm_PThread)tcomm->data;
-    granks = gptcomm->granks;
-    /* Copy over the data from the global thread communicator structure */
-    ptcomm->ismainworker     = gptcomm->ismainworker;
-    ptcomm->thread_num_start = gptcomm->thread_num_start;
-    ptcomm->sync             = gptcomm->sync;
-    ptcomm->aff              = gptcomm->aff;
-    tcomm->ops->runkernel    = gtcomm->ops->runkernel;
-    tcomm->ops->barrier      = gtcomm->ops->barrier;
-    
-    for(i=0; i < tcomm->nworkThreads;i++) {
-      for(j=0;j < gtcomm->nworkThreads; j++) {
-	if(tcomm->affinities[i] == gaffinities[j]) {
-	  ptcomm->granks[i] = granks[j];
-	}
-      }
-    }
-  }  
-
-  PetscFunctionReturn(0);
-}
-EXTERN_C_END
-    

src/sys/threadcomm/impls/pthread/pthreadimpl.h

-
-#ifndef __PTHREADIMPLH
-#define __PTHREADIMPLH
-
-#include <petsc-private/threadcommimpl.h>
-
-#if defined(PETSC_HAVE_PTHREAD_H)
-#include <pthread.h>
-#elif defined(PETSC_HAVE_WINPTHREADS_H)
-#include "winpthreads.h"       /* http://locklessinc.com/downloads/winpthreads.h */
-#endif
-
-/*
-  PetscPThreadCommSynchronizationType - Type of thread synchronization for pthreads communicator.
-
-$ PTHREADSYNC_LOCKFREE -  A lock-free variant.
-
-*/
-typedef enum {PTHREADSYNC_LOCKFREE} PetscPThreadCommSynchronizationType;
-extern const char *const PetscPThreadCommSynchronizationTypes[];
-
-/*
-  PetscPThreadCommAffinityPolicy - Core affinity policy for pthreads
-
-$ PTHREADAFFPOLICY_ALL     - threads can run on any core. OS decides thread scheduling
-$ PTHREADAFFPOLICY_ONECORE - threads can run on only one core.
-$ PTHREADAFFPOLICY_NONE    - No set affinity policy. OS decides thread scheduling
-*/
-typedef enum {PTHREADAFFPOLICY_ALL,PTHREADAFFPOLICY_ONECORE,PTHREADAFFPOLICY_NONE} PetscPThreadCommAffinityPolicyType;
-extern const char *const PetscPTheadCommAffinityPolicyTypes[];
-
-typedef enum {PTHREADPOOLSPARK_LEADER,PTHREADPOOLSPARK_CHAIN} PetscPThreadCommPoolSparkType;
-extern const char *const PetscPThreadCommPoolSparkTypes[];
-
-/*
-   PetscThreadComm_PThread - The main data structure to manage the thread
-   communicator using pthreads. This data structure is shared by NONTHREADED
-   and PTHREAD threading models. For NONTHREADED threading model, no extra
-   pthreads are created
-*/
-struct _p_PetscThreadComm_PThread{
-  PetscInt    nthreads;                      /* Number of threads created */
-  pthread_t  *tid;                           /* thread ids */
-  PetscBool  ismainworker;                   /* Is the main thread also a work thread?*/
-  PetscInt   *granks;                        /* Thread ranks - if main thread is a worker then main thread 
-				                rank is 0 and ranks for other threads start from 1, 
-				                otherwise the thread ranks start from 0.
-                                                These ranks are with respect to the first initialized thread pool */
-  PetscInt    thread_num_start;              /* index for the first created thread (= 1 if the main thread is a worker
-                                                else 0) */
-  PetscInt    *ngranks;                      /* Stores the rank of the next thread to be sparked by this thread 
-						Only used for CHAIN PoolSpark type */
-  PetscPThreadCommSynchronizationType sync;   /* Synchronization type */
-  PetscPThreadCommAffinityPolicyType  aff;    /* affinity policy */
-  PetscPThreadCommPoolSparkType       spark;  /* Type for sparking threads */
-  PetscBool                           synchronizeafter; /* Whether the main thread should be blocked till all threads complete the given kernel */     
-  PetscErrorCode (*initialize)(PetscThreadComm);
-  PetscErrorCode (*finalize)(PetscThreadComm);
-};
-
-typedef struct _p_PetscThreadComm_PThread *PetscThreadComm_PThread;
-
-#if defined(PETSC_PTHREAD_LOCAL)
-extern PETSC_PTHREAD_LOCAL PetscInt PetscPThreadRank; /* Rank of the calling thread ... thread local variable */
-#else
-extern pthread_key_t  PetscPThreadRankkey;
-#endif
-
-
-EXTERN_C_BEGIN
-extern PetscErrorCode PetscThreadCommCreate_PThread(PetscThreadComm);
-EXTERN_C_END
-
-extern PetscErrorCode PetscPThreadCommInitialize_LockFree(PetscThreadComm);
-extern PetscErrorCode PetscPThreadCommFinalize_LockFree(PetscThreadComm);
-extern PetscErrorCode PetscThreadCommRunKernel_PThread_LockFree(MPI_Comm,PetscThreadCommJobCtx);
-extern PetscErrorCode PetscThreadCommBarrier_PThread_LockFree(PetscThreadComm);
-
-#if defined(PETSC_HAVE_SCHED_CPU_SET_T)
-extern void PetscPThreadCommDoCoreAffinity();
-#endif
-
-
-#endif

src/sys/threadcomm/impls/pthread/tclockfree.c

+#include <../src/sys/threadcomm/impls/pthread/tcpthreadimpl.h>
+
+#define THREAD_TERMINATE      -1
+#define THREAD_WAITING_FOR_JOB 0
+#define THREAD_RECIEVED_JOB    1
+#define THREAD_INITIALIZE      2
+
+/* lock-free data structure */
+typedef struct {
+  PetscThreadCommJobCtx *data;
+  PetscInt           *my_job_status;
+} sjob_lockfree;
+
+static sjob_lockfree job_lockfree = {NULL,NULL};
+
+
+void SparkThreads_LockFree(PetscInt myrank,PetscThreadComm tcomm,PetscThreadCommJobCtx job)
+{
+  PetscInt i,thread_num;
+  PetscThreadComm_PThread  ptcomm;
+  PetscInt                 next;
+
+  ptcomm = (PetscThreadComm_PThread)tcomm->data;
+  
+  switch(ptcomm->spark) {
+  case PTHREADPOOLSPARK_LEADER:
+    if(PetscReadOnce(int,tcomm->leader) == myrank) {
+      /* Only leader sparks all the other threads */
+      for(i=ptcomm->thread_num_start+1; i < tcomm->nworkThreads;i++) {
+	thread_num = ptcomm->granks[i];
+	while(PetscReadOnce(int,job_lockfree.my_job_status[thread_num]) != THREAD_WAITING_FOR_JOB)
+	  ;
+	job_lockfree.data[thread_num] = job;
+	job_lockfree.my_job_status[thread_num] = THREAD_RECIEVED_JOB;
+      }
+    }
+    break;
+  case PTHREADPOOLSPARK_CHAIN:
+    /* Spark the next thread */
+    next = ptcomm->ngranks[myrank];
+    if(next != -1) {
+      thread_num = next;
+      while(PetscReadOnce(int,job_lockfree.my_job_status[thread_num]) != THREAD_WAITING_FOR_JOB)
+	;
+      job_lockfree.data[thread_num] = job;
+      job_lockfree.my_job_status[thread_num] = THREAD_RECIEVED_JOB;
+    }
+    break;
+  }
+}
+
+void* PetscPThreadCommFunc_LockFree(void* arg)
+{
+
+#if defined(PETSC_PTHREAD_LOCAL)
+  PetscPThreadRank = *(PetscInt*)arg;
+#else
+  PetscInt PetscPThreadRank=*(PetscInt*)arg;
+  pthread_setspecific(PetscPThreadRankkey,&PetscPThreadRank);
+#endif
+
+#if defined(PETSC_HAVE_SCHED_CPU_SET_T)
+  PetscPThreadCommDoCoreAffinity();
+#endif
+  job_lockfree.my_job_status[PetscPThreadRank] = THREAD_WAITING_FOR_JOB;
+
+  /* Spin loop */
+  while(PetscReadOnce(int,job_lockfree.my_job_status[PetscPThreadRank]) != THREAD_TERMINATE) {
+    if(job_lockfree.my_job_status[PetscPThreadRank] == THREAD_RECIEVED_JOB) {
+      /* Spark the thread pool */
+      SparkThreads_LockFree(PetscPThreadRank,job_lockfree.data[PetscPThreadRank]->tcomm,job_lockfree.data[PetscPThreadRank]);
+      /* Do own job */
+      PetscRunKernel(PetscPThreadRank,job_lockfree.data[PetscPThreadRank]->nargs,job_lockfree.data[PetscPThreadRank]);
+      /* Reset own status */ 
+      job_lockfree.my_job_status[PetscPThreadRank] = THREAD_WAITING_FOR_JOB;
+    }
+  }
+
+  return NULL;
+}
+
+#undef __FUNCT__
+#define __FUNCT__ "PetscThreadCommBarrier_PThread_LockFree"
+PetscErrorCode PetscThreadCommBarrier_PThread_LockFree(PetscThreadComm tcomm)
+{
+  PetscInt active_threads=0,i;
+  PetscBool wait=PETSC_TRUE;
+  PetscThreadComm_PThread ptcomm=(PetscThreadComm_PThread)tcomm->data;
+
+  PetscFunctionBegin;
+  /* Loop till all threads signal that they have done their job */
+  while(wait) {
+    for(i=0;i<tcomm->nworkThreads;i++) active_threads += job_lockfree.my_job_status[ptcomm->granks[i]];
+    if(active_threads) active_threads = 0;
+    else wait=PETSC_FALSE;
+  }
+  PetscFunctionReturn(0);
+}
+
+#undef __FUNCT__
+#define __FUNCT__ "PetscPThreadCommInitialize_LockFree"
+PetscErrorCode PetscPThreadCommInitialize_LockFree(PetscThreadComm tcomm)
+{
+  PetscErrorCode ierr;
+  PetscInt       i;
+  PetscThreadComm_PThread ptcomm=(PetscThreadComm_PThread)tcomm->data;
+
+  PetscFunctionBegin;
+
+  ierr = PetscMalloc(tcomm->nworkThreads*sizeof(PetscThreadCommJobCtx),&job_lockfree.data);CHKERRQ(ierr);
+  ierr = PetscMalloc(tcomm->nworkThreads*sizeof(PetscInt),&job_lockfree.my_job_status);CHKERRQ(ierr);
+
+  /* Create threads */
+  for(i=ptcomm->thread_num_start; i < tcomm->nworkThreads;i++) {
+    job_lockfree.my_job_status[i] = THREAD_INITIALIZE;
+    ierr = pthread_create(&ptcomm->tid[i],NULL,&PetscPThreadCommFunc_LockFree,&ptcomm->granks[i]);CHKERRQ(ierr);
+  }
+  if(ptcomm->ismainworker) {
+    job_lockfree.my_job_status[0] = THREAD_WAITING_FOR_JOB;
+  }
+
+  /* Put a barrier so that all threads get pinned properly */
+  ierr = PetscThreadCommBarrier_PThread_LockFree(tcomm);CHKERRQ(ierr);
+  PetscFunctionReturn(0);
+}
+
+#undef __FUNCT__
+#define __FUNCT__ "PetscPThreadCommFinalize_LockFree"
+PetscErrorCode PetscPThreadCommFinalize_LockFree(PetscThreadComm tcomm)
+{
+  PetscErrorCode           ierr;
+  void*                    jstatus;
+  PetscThreadComm_PThread  ptcomm=(PetscThreadComm_PThread)tcomm->data;
+  PetscInt                 i;
+  PetscFunctionBegin;
+  ierr = PetscThreadCommBarrier_PThread_LockFree(tcomm);CHKERRQ(ierr);
+  for(i=ptcomm->thread_num_start; i < tcomm->nworkThreads;i++) {
+    job_lockfree.my_job_status[i] = THREAD_TERMINATE;
+    ierr = pthread_join(ptcomm->tid[i],&jstatus);CHKERRQ(ierr);
+  }
+  ierr = PetscFree(job_lockfree.my_job_status);CHKERRQ(ierr);
+  ierr = PetscFree(job_lockfree.data);CHKERRQ(ierr);
+
+  PetscFunctionReturn(0);
+}
+
+#undef __FUNCT__
+#define __FUNCT__ "PetscThreadCommRunKernel_PThread_LockFree"
+PetscErrorCode PetscThreadCommRunKernel_PThread_LockFree(MPI_Comm comm,PetscThreadCommJobCtx job)
+{
+  PetscErrorCode           ierr;
+  PetscThreadComm          tcomm=0;
+  PetscThreadComm_PThread  ptcomm;    
+  
+  PetscFunctionBegin;
+  ierr = PetscCommGetThreadComm(comm,&tcomm);CHKERRQ(ierr);
+  ptcomm = (PetscThreadComm_PThread)tcomm->data;
+  if(ptcomm->nthreads) {
+    PetscInt thread_num;
+    /* Spark the leader thread */
+    thread_num = tcomm->leader;
+    /* Wait till the leader thread has finished its previous job */
+    while(PetscReadOnce(int,job_lockfree.my_job_status[thread_num]) != THREAD_WAITING_FOR_JOB)
+      ;
+    job_lockfree.data[thread_num] = job;
+    job_lockfree.my_job_status[thread_num] = THREAD_RECIEVED_JOB;
+  }
+  if(ptcomm->ismainworker) {
+    job_lockfree.my_job_status[0] = THREAD_RECIEVED_JOB;
+    job_lockfree.data[0] = job;
+    PetscRunKernel(0,job->nargs, job_lockfree.data[0]);
+    job_lockfree.my_job_status[0] = THREAD_WAITING_FOR_JOB;
+  }
+  if(ptcomm->synchronizeafter) {
+    ierr = PetscThreadCommBarrier(comm);CHKERRQ(ierr);
+  }
+  PetscFunctionReturn(0);
+}

src/sys/threadcomm/impls/pthread/tcpthread.c

+/* Define feature test macros to make sure CPU_SET and other functions are available
+ */
+#define PETSC_DESIRE_FEATURE_TEST_MACROS
+
+#include <../src/sys/threadcomm/impls/pthread/tcpthreadimpl.h>
+
+#if defined(PETSC_PTHREAD_LOCAL)
+PETSC_PTHREAD_LOCAL PetscInt PetscPThreadRank;
+#else
+pthread_key_t PetscPThreadRankkey;
+#endif
+
+static PetscBool PetscPThreadCommInitializeCalled = PETSC_FALSE;
+
+const char *const PetscPThreadCommSynchronizationTypes[] = {"LOCKFREE","PetscPThreadCommSynchronizationType","PTHREADSYNC_",0};
+const char *const PetscPThreadCommAffinityPolicyTypes[] = {"ALL","ONECORE","NONE","PetscPThreadCommAffinityPolicyType","PTHREADAFFPOLICY_",0};
+const char *const PetscPThreadCommPoolSparkTypes[] = {"LEADER","CHAIN","PetscPThreadCommPoolSparkType","PTHREADPOOLSPARK_",0};
+
+static PetscInt ptcommcrtct = 0; /* PThread communicator creation count. Incremented whenever a pthread
+                                    communicator is created and decremented when it is destroyed. On the
+                                    last pthread communicator destruction, the thread pool is also terminated
+                                  */
+
+PetscInt PetscThreadCommGetRank_PThread()
+{
+#if defined(PETSC_PTHREAD_LOCAL)
+  return PetscPThreadRank;
+#else
+  return *((PetscInt*)pthread_getspecific(PetscPThreadRankkey));
+#endif
+}
+
+
+#if defined(PETSC_HAVE_SCHED_CPU_SET_T)
+void PetscPThreadCommDoCoreAffinity(void)
+{
+  PetscInt                 i,icorr=0; 
+  cpu_set_t                mset;
+  PetscInt                 ncores,myrank=PetscThreadCommGetRank_PThread();
+  PetscThreadComm          tcomm;
+  PetscThreadComm_PThread  gptcomm;
+  
+  PetscCommGetThreadComm(PETSC_COMM_WORLD,&tcomm);
+  PetscGetNCores(&ncores);
+  gptcomm=(PetscThreadComm_PThread)tcomm->data;
+  switch(gptcomm->aff) {
+  case PTHREADAFFPOLICY_ONECORE:
+    icorr = tcomm->affinities[myrank];
+    CPU_ZERO(&mset);
+    CPU_SET(icorr%ncores,&mset);
+    pthread_setaffinity_np(pthread_self(),sizeof(cpu_set_t),&mset);
+    break;
+  case PTHREADAFFPOLICY_ALL:
+    CPU_ZERO(&mset);
+    for(i=0;i<ncores;i++) CPU_SET(i,&mset);
+    pthread_setaffinity_np(pthread_self(),sizeof(cpu_set_t),&mset);
+    break;
+  case PTHREADAFFPOLICY_NONE:
+    break;
+  }
+}
+#endif
+
+#undef __FUNCT__
+#define __FUNCT__ "PetscThreadCommDestroy_PThread"
+PetscErrorCode PetscThreadCommDestroy_PThread(PetscThreadComm tcomm)
+{
+  PetscThreadComm_PThread ptcomm=(PetscThreadComm_PThread)tcomm->data;
+  PetscErrorCode          ierr;
+
+  PetscFunctionBegin;
+  if(!ptcomm) PetscFunctionReturn(0);
+  ptcommcrtct--;
+  if(!ptcommcrtct) {
+    /* Terminate the thread pool */
+    ierr = (*ptcomm->finalize)(tcomm);CHKERRQ(ierr);
+    ierr = PetscFree(ptcomm->tid);CHKERRQ(ierr);
+  }
+  ierr = PetscFree(ptcomm->granks);CHKERRQ(ierr);
+  if(ptcomm->spark == PTHREADPOOLSPARK_CHAIN) {
+    ierr = PetscFree(ptcomm->ngranks);CHKERRQ(ierr);
+  }
+  ierr = PetscFree(ptcomm);CHKERRQ(ierr);
+  PetscFunctionReturn(0);
+}
+
+EXTERN_C_BEGIN
+#undef __FUNCT__
+#define __FUNCT__ "PetscThreadCommCreate_PThread"
+PetscErrorCode PetscThreadCommCreate_PThread(PetscThreadComm tcomm)
+{
+  PetscThreadComm_PThread ptcomm;
+  PetscErrorCode          ierr;
+  PetscInt                i;
+
+  PetscFunctionBegin;
+  ptcommcrtct++;
+  ierr = PetscStrcpy(tcomm->type,PTHREAD);CHKERRQ(ierr);
+  ierr = PetscNew(struct _p_PetscThreadComm_PThread,&ptcomm);CHKERRQ(ierr);
+  tcomm->data = (void*)ptcomm;
+  ptcomm->nthreads = 0;
+  ptcomm->sync = PTHREADSYNC_LOCKFREE;
+  ptcomm->aff = PTHREADAFFPOLICY_ONECORE;
+  ptcomm->spark = PTHREADPOOLSPARK_LEADER;
+  ptcomm->ismainworker = PETSC_TRUE;
+  ptcomm->synchronizeafter = PETSC_TRUE;
+  tcomm->ops->destroy = PetscThreadCommDestroy_PThread;
+  tcomm->ops->runkernel = PetscThreadCommRunKernel_PThread_LockFree;
+  tcomm->ops->barrier   = PetscThreadCommBarrier_PThread_LockFree;
+  tcomm->ops->getrank   = PetscThreadCommGetRank_PThread;
+
+  ierr = PetscMalloc(tcomm->nworkThreads*sizeof(PetscInt),&ptcomm->granks);CHKERRQ(ierr);
+
+  if(!PetscPThreadCommInitializeCalled) { /* Only done for PETSC_THREAD_COMM_WORLD */
+    PetscPThreadCommInitializeCalled = PETSC_TRUE;
+    PetscBool               flg1,flg2,flg3,flg4;
+
+    ierr = PetscOptionsBegin(PETSC_COMM_WORLD,PETSC_NULL,"PThread communicator options",PETSC_NULL);CHKERRQ(ierr);
+    ierr = PetscOptionsBool("-threadcomm_pthread_main_is_worker","Main thread is also a worker thread",PETSC_NULL,PETSC_TRUE,&ptcomm->ismainworker,&flg1);CHKERRQ(ierr);
+    ierr = PetscOptionsEnum("-threadcomm_pthread_affpolicy","Thread affinity policy"," ",PetscPThreadCommAffinityPolicyTypes,(PetscEnum)ptcomm->aff,(PetscEnum*)&ptcomm->aff,&flg2);CHKERRQ(ierr);
+    ierr = PetscOptionsEnum("-threadcomm_pthread_type","Thread pool type"," ",PetscPThreadCommSynchronizationTypes,(PetscEnum)ptcomm->sync,(PetscEnum*)&ptcomm->sync,&flg3);CHKERRQ(ierr);
+    ierr = PetscOptionsEnum("-threadcomm_pthread_spark","Thread pool spark type"," ",PetscPThreadCommPoolSparkTypes,(PetscEnum)ptcomm->spark,(PetscEnum*)&ptcomm->spark,&flg4);CHKERRQ(ierr);
+    ierr = PetscOptionsBool("-threadcomm_pthread_synchronizeafter","Puts a barrier after every kernel call",PETSC_NULL,PETSC_TRUE,&ptcomm->synchronizeafter,&flg1);CHKERRQ(ierr);
+    ierr = PetscOptionsEnd();CHKERRQ(ierr);
+
+    if(ptcomm->ismainworker) {
+      ptcomm->nthreads = tcomm->nworkThreads-1;
+      ptcomm->thread_num_start = 1;
+    } else {
+      ptcomm->nthreads = tcomm->nworkThreads;
+      ptcomm->thread_num_start = 0;
+    }
+
+    switch(ptcomm->sync) {
+    case PTHREADSYNC_LOCKFREE:
+      ptcomm->initialize      = PetscPThreadCommInitialize_LockFree;
+      ptcomm->finalize        = PetscPThreadCommFinalize_LockFree;
+      tcomm->ops->runkernel   = PetscThreadCommRunKernel_PThread_LockFree;
+      tcomm->ops->barrier     = PetscThreadCommBarrier_PThread_LockFree;
+      break;
+    default:
+      SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Only Lock-free synchronization scheme supported currently");
+    }
+    /* Set up thread ranks */
+    for(i=0;i< tcomm->nworkThreads;i++) ptcomm->granks[i] = i;
+
+    if(ptcomm->ismainworker) {
+#if defined(PETSC_PTHREAD_LOCAL)
+      PetscPThreadRank=0; /* Main thread rank */
+#else
+      ierr = pthread_key_create(&PetscPThreadRankkey,NULL);CHKERRQ(ierr);
+      ierr = pthread_setspecific(PetscPThreadRankkey,&ptcomm->granks[0]);CHKERRQ(ierr);
+#endif
+    }
+    /* Set the leader thread rank */
+    if(ptcomm->nthreads) {
+      if(ptcomm->ismainworker) tcomm->leader = ptcomm->granks[1];
+      else tcomm->leader = ptcomm->granks[0];
+    }
+  
+    if(ptcomm->spark == PTHREADPOOLSPARK_CHAIN) {
+      ierr = PetscMalloc(tcomm->nworkThreads*sizeof(PetscInt),&ptcomm->ngranks);CHKERRQ(ierr);
+      for(i=ptcomm->thread_num_start;i < tcomm->nworkThreads-1;i++) ptcomm->ngranks[i] = ptcomm->granks[i+1];
+      ptcomm->ngranks[tcomm->nworkThreads-1] = -1;
+    }
+
+    /* Create array holding pthread ids */
+    ierr = PetscMalloc(tcomm->nworkThreads*sizeof(pthread_t),&ptcomm->tid);CHKERRQ(ierr);
+
+    /* Set affinity of the main thread */
+    
+#if defined(PETSC_HAVE_SCHED_CPU_SET_T)
+    cpu_set_t mset;
+    PetscInt  ncores,icorr;
+    
+
+    ierr = PetscGetNCores(&ncores);CHKERRQ(ierr);
+    CPU_ZERO(&mset);
+    icorr = tcomm->affinities[0]%ncores;
+    CPU_SET(icorr,&mset);
+    sched_setaffinity(0,sizeof(cpu_set_t),&mset);
+#endif
+
+    /* Initialize thread pool */
+    ierr = (*ptcomm->initialize)(tcomm);CHKERRQ(ierr);
+
+  } else {
+    PetscThreadComm          gtcomm;
+    PetscThreadComm_PThread  gptcomm;
+    PetscInt                 *granks,j,*gaffinities;
+
+    ierr = PetscCommGetThreadComm(PETSC_COMM_WORLD,&gtcomm);CHKERRQ(ierr);
+    gaffinities = gtcomm->affinities;
+    gptcomm = (PetscThreadComm_PThread)tcomm->data;
+    granks = gptcomm->granks;
+    /* Copy over the data from the global thread communicator structure */
+    ptcomm->ismainworker     = gptcomm->ismainworker;
+    ptcomm->thread_num_start = gptcomm->thread_num_start;
+    ptcomm->sync             = gptcomm->sync;
+    ptcomm->aff              = gptcomm->aff;
+    tcomm->ops->runkernel    = gtcomm->ops->runkernel;
+    tcomm->ops->barrier      = gtcomm->ops->barrier;
+    
+    for(i=0; i < tcomm->nworkThreads;i++) {
+      for(j=0;j < gtcomm->nworkThreads; j++) {
+	if(tcomm->affinities[i] == gaffinities[j]) {
+	  ptcomm->granks[i] = granks[j];
+	}
+      }
+    }
+  }  
+
+  PetscFunctionReturn(0);
+}
+EXTERN_C_END
+    

src/sys/threadcomm/impls/pthread/tcpthreadimpl.h

+
+#ifndef __PTHREADIMPLH
+#define __PTHREADIMPLH
+
+#include <petsc-private/threadcommimpl.h>
+
+#if defined(PETSC_HAVE_PTHREAD_H)
+#include <pthread.h>
+#elif defined(PETSC_HAVE_WINPTHREADS_H)
+#include "winpthreads.h"       /* http://locklessinc.com/downloads/winpthreads.h */
+#endif
+
+/*
+  PetscPThreadCommSynchronizationType - Type of thread synchronization for pthreads communicator.
+
+$ PTHREADSYNC_LOCKFREE -  A lock-free variant.
+
+*/
+typedef enum {PTHREADSYNC_LOCKFREE} PetscPThreadCommSynchronizationType;
+extern const char *const PetscPThreadCommSynchronizationTypes[];
+
+/*
+  PetscPThreadCommAffinityPolicy - Core affinity policy for pthreads
+
+$ PTHREADAFFPOLICY_ALL     - threads can run on any core. OS decides thread scheduling
+$ PTHREADAFFPOLICY_ONECORE - threads can run on only one core.
+$ PTHREADAFFPOLICY_NONE    - No set affinity policy. OS decides thread scheduling
+*/
+typedef enum {PTHREADAFFPOLICY_ALL,PTHREADAFFPOLICY_ONECORE,PTHREADAFFPOLICY_NONE} PetscPThreadCommAffinityPolicyType;
+extern const char *const PetscPTheadCommAffinityPolicyTypes[];
+
+typedef enum {PTHREADPOOLSPARK_LEADER,PTHREADPOOLSPARK_CHAIN} PetscPThreadCommPoolSparkType;
+extern const char *const PetscPThreadCommPoolSparkTypes[];
+
+/*
+   PetscThreadComm_PThread - The main data structure to manage the thread
+   communicator using pthreads. This data structure is shared by NONTHREADED
+   and PTHREAD threading models. For NONTHREADED threading model, no extra
+   pthreads are created
+*/
+struct _p_PetscThreadComm_PThread{
+  PetscInt    nthreads;                      /* Number of threads created */
+  pthread_t  *tid;                           /* thread ids */
+  PetscBool  ismainworker;                   /* Is the main thread also a work thread?*/
+  PetscInt   *granks;                        /* Thread ranks - if main thread is a worker then main thread 
+				                rank is 0 and ranks for other threads start from 1, 
+				                otherwise the thread ranks start from 0.
+                                                These ranks are with respect to the first initialized thread pool */
+  PetscInt    thread_num_start;              /* index for the first created thread (= 1 if the main thread is a worker
+                                                else 0) */
+  PetscInt    *ngranks;                      /* Stores the rank of the next thread to be sparked by this thread 
+						Only used for CHAIN PoolSpark type */
+  PetscPThreadCommSynchronizationType sync;   /* Synchronization type */
+  PetscPThreadCommAffinityPolicyType  aff;    /* affinity policy */
+  PetscPThreadCommPoolSparkType       spark;  /* Type for sparking threads */
+  PetscBool                           synchronizeafter; /* Whether the main thread should be blocked till all threads complete the given kernel */     
+  PetscErrorCode (*initialize)(PetscThreadComm);
+  PetscErrorCode (*finalize)(PetscThreadComm);
+};
+
+typedef struct _p_PetscThreadComm_PThread *PetscThreadComm_PThread;
+
+#if defined(PETSC_PTHREAD_LOCAL)
+extern PETSC_PTHREAD_LOCAL PetscInt PetscPThreadRank; /* Rank of the calling thread ... thread local variable */
+#else
+extern pthread_key_t  PetscPThreadRankkey;
+#endif
+
+
+EXTERN_C_BEGIN
+extern PetscErrorCode PetscThreadCommCreate_PThread(PetscThreadComm);
+EXTERN_C_END
+
+extern PetscErrorCode PetscPThreadCommInitialize_LockFree(PetscThreadComm);
+extern PetscErrorCode PetscPThreadCommFinalize_LockFree(PetscThreadComm);
+extern PetscErrorCode PetscThreadCommRunKernel_PThread_LockFree(MPI_Comm,PetscThreadCommJobCtx);
+extern PetscErrorCode PetscThreadCommBarrier_PThread_LockFree(PetscThreadComm);
+
+#if defined(PETSC_HAVE_SCHED_CPU_SET_T)
+extern void PetscPThreadCommDoCoreAffinity();
+#endif
+
+
+#endif