The flesh is not thread safe

The flesh is not thread safe, and therefore CarpetX’s calling of functions through the flesh via openmp is broken. In particular, the function below The variable :current_scheduled_function is a global variable.

const cFunctionData *CCTK_ScheduleQueryCurrentFunction(const cGH * CCTK_ATTRIBUTE_UNUSED GH)
{
  return current_scheduled_function;
}

If one makes the following hackish edit (which I do not recommend) the problem goes away.

db1 flesh$ git diff src/main/ScheduleInterface.c|cat -
diff --git a/src/main/ScheduleInterface.c b/src/main/ScheduleInterface.c
index 93bdc20..c922904 100644
--- a/src/main/ScheduleInterface.c
+++ b/src/main/ScheduleInterface.c
@@ -12,6 +12,7 @@
 #include <stdlib.h>
 #include <stdarg.h>
 #include <string.h>
+#include <assert.h>

 #include "cctk_Flesh.h"
 #include "cctk_WarnLevel.h"
@@ -35,6 +36,8 @@

 #include "util_Table.h"

+#include "omp.h"
+
 static const char *rcsid = "$Header$";

 CCTK_FILEVERSION(main_ScheduleInterface_c);
@@ -238,7 +241,7 @@ static int *scheduled_storage_groups_timelevels = NULL;
 static cTimerData *timerinfo = NULL;
 static int total_timer = -1;

-static const cFunctionData *current_scheduled_function = NULL;
+static const cFunctionData *current_scheduled_function[100];// = NULL;


 /********************************************************************
@@ -286,21 +289,22 @@ int CCTK_CallFunction(void *function,

   int (*oneargfunc)(void *);

+  int tn = omp_get_thread_num();
 #if ALLOW_RECURSIVE_SCHEDULE_CALLS
-  const cFunctionData *previous_scheduled_function = current_scheduled_function;
+  const cFunctionData *previous_scheduled_function = current_scheduled_function[tn];
 #else
-  if(current_scheduled_function != NULL)
+  if(current_scheduled_function[tn] != NULL)
   {
     CCTK_VWarn(CCTK_WARN_PICKY, __LINE__, __FILE__, "Cactus",
                "CCTK_CallFunction: recursive call, calling "
                "'%s: %s::%s' while within '%s: %s::%s'",
                fdata->where, fdata->thorn, fdata->routine,
-               current_scheduled_function->where,
-               current_scheduled_function->thorn,
-               current_scheduled_function->routine);
+               current_scheduled_function[tn]->where,
+               current_scheduled_function[tn]->thorn,
+               current_scheduled_function[tn]->routine);
   }
 #endif
-  current_scheduled_function = fdata;
+  current_scheduled_function[tn] = fdata;

   switch(fdata->type)
   {
@@ -317,6 +321,7 @@ int CCTK_CallFunction(void *function,
       {
         case LangC:
           standardfunc = (void (*)(void *))function;
+          //assert(data != 0);
           standardfunc(data);
           break;
         case LangFortran:
@@ -336,9 +341,9 @@ int CCTK_CallFunction(void *function,
   }

 #if ALLOW_RECURSIVE_SCHEDULE_CALLS
-  current_scheduled_function = previous_scheduled_function;
+  current_scheduled_function[tn] = previous_scheduled_function;
 #else
-  current_scheduled_function = NULL;
+  current_scheduled_function[tn] = NULL;
 #endif

   /* Return 0, meaning didn't synchronise */
@@ -369,7 +374,8 @@ int CCTK_CallFunction(void *function,
 @@*/
 const cFunctionData *CCTK_ScheduleQueryCurrentFunction(const cGH * CCTK_ATTRIBUTE_UNUSED GH)
 {
-  return current_scheduled_function;
+  int tn = omp_get_thread_num();
+  return current_scheduled_function[tn];
 }

 /*@@

I think what needs to be done is cctkGH needs to hold the current_scheduled_function and CarpetX needs to set it appropriately in its various copies.

Comments (8)