openmp/runtime/test/affinity/root-threads-affinity.c

   1 // RUN: %libomp-compile && env LIBOMP_NUM_HIDDEN_HELPER_THREADS=0 OMP_PROC_BIND=close OMP_PLACES=cores KMP_AFFINITY=verbose %libomp-run 8 1 4
   2 // REQUIRES: linux
   3 //
   4 // This test pthread_creates 8 root threads before any OpenMP
   5 // runtime entry is ever called. We have all the root threads
   6 // register with the runtime by calling omp_set_num_threads(),
   7 // but this does not initialize their affinity. The fourth root thread
   8 // then calls a parallel region and we make sure its affinity
   9 // is correct. We also make sure all the other root threads are
  10 // free-floating since they have not called into a parallel region.
  11
  12 #define _GNU_SOURCE
  13 #include <stdio.h>
  14 #include <stdlib.h>
  15 #include <omp.h>
  16 #include <pthread.h>
  17 #include <unistd.h>
  18 #include <assert.h>
  19 #include <sys/types.h>
  20 #include <sys/syscall.h>
  21 #include "libomp_test_affinity.h"
  22
  23 volatile int entry_flag = 0;
  24 volatile int flag = 0;
  25 volatile int num_roots_arrived = 0;
  26 int num_roots;
  27 int spawner = 0;
  28 pthread_mutex_t lock;
  29 int register_workers = 0; // boolean
  30 affinity_mask_t *full_mask;
  31
  32 int __kmpc_global_thread_num(void*);
  33
  34 int get_os_thread_id() {
  35   return (int)syscall(SYS_gettid);
  36 }
  37
  38 int place_and_affinity_match() {
  39   int i, max_cpu;
  40   char buf[512];
  41   affinity_mask_t *mask = affinity_mask_alloc();
  42   int place = omp_get_place_num();
  43   int num_procs = omp_get_place_num_procs(place);
  44   int *ids = (int*)malloc(sizeof(int) * num_procs);
  45   omp_get_place_proc_ids(place, ids);
  46   get_thread_affinity(mask);
  47   affinity_mask_snprintf(buf, sizeof(buf), mask);
  48   printf("Primary Thread Place: %d\n", place);
  49   printf("Primary Thread mask: %s\n", buf);
  50
  51   for (i = 0; i < num_procs; ++i) {
  52     int cpu = ids[i];
  53     if (!affinity_mask_isset(mask, cpu))
  54       return 0;
  55   }
  56
  57   max_cpu = AFFINITY_MAX_CPUS;
  58   for (i = 0; i < max_cpu; ++i) {
  59     int cpu = i;
  60     if (affinity_mask_isset(mask, cpu)) {
  61       int j, found = 0;
  62       for (j = 0; j < num_procs; ++j) {
  63         if (ids[j] == cpu) {
  64           found = 1;
  65           break;
  66         }
  67       }
  68       if (!found)
  69         return 0;
  70     }
  71   }
  72
  73   affinity_mask_free(mask);
  74   free(ids);
  75   return 1;
  76 }
  77
  78 void* thread_func(void *arg) {
  79   int place, nplaces;
  80   int root_id = *((int*)arg);
  81   int pid = getpid();
  82   int tid = get_os_thread_id();
  83
  84   // Order how the root threads are assigned a gtid in the runtime
  85   // i.e., root_id = gtid
  86   while (1) {
  87     int v = entry_flag;
  88     if (v == root_id)
  89       break;
  90   }
  91
  92   // If main root thread
  93   if (root_id == spawner) {
  94     printf("Initial application thread (pid=%d, tid=%d, spawner=%d) reached thread_func (will call OpenMP)\n", pid, tid, spawner);
  95     omp_set_num_threads(4);
  96     #pragma omp atomic
  97     entry_flag++;
  98     // Wait for the workers to signal their arrival before #pragma omp parallel
  99     while (num_roots_arrived < num_roots - 1) {}
 100     // This will trigger the output for KMP_AFFINITY in this case
 101     #pragma omp parallel
 102     {
 103       int gtid = __kmpc_global_thread_num(NULL);
 104       #pragma omp single
 105       {
 106         printf("Exactly %d threads in the #pragma omp parallel\n",
 107                omp_get_num_threads());
 108       }
 109       #pragma omp critical
 110       {
 111         printf("OpenMP thread %d: gtid=%d\n", omp_get_thread_num(), gtid);
 112       }
 113     }
 114     flag = 1;
 115     if (!place_and_affinity_match()) {
 116       fprintf(stderr, "error: place and affinity mask do not match for primary thread\n");
 117       exit (EXIT_FAILURE);
 118     }
 119
 120   } else { // If worker root thread
 121     // Worker root threads, register with OpenMP through omp_set_num_threads()
 122     // if designated to, signal their arrival and then wait for the main root
 123     // thread to signal them to exit.
 124     printf("New root pthread (pid=%d, tid=%d) reached thread_func\n", pid, tid);
 125     if (register_workers)
 126       omp_set_num_threads(4);
 127     #pragma omp atomic
 128     entry_flag++;
 129
 130     pthread_mutex_lock(&lock);
 131     num_roots_arrived++;
 132     pthread_mutex_unlock(&lock);
 133     while (flag == 0) {}
 134
 135     // Main check whether root threads' mask is equal to the
 136     // initial affinity mask
 137     affinity_mask_t *mask = affinity_mask_alloc();
 138     get_thread_affinity(mask);
 139     if (!affinity_mask_equal(mask, full_mask)) {
 140       char buf[1024];
 141       printf("root thread %d mask: ", root_id);
 142       affinity_mask_snprintf(buf, sizeof(buf), mask);
 143       printf("initial affinity mask: %s\n", buf);
 144       fprintf(stderr, "error: root thread %d affinity mask not equal"
 145                       " to initial full mask\n", root_id);
 146       affinity_mask_free(mask);
 147       exit(EXIT_FAILURE);
 148     }
 149     affinity_mask_free(mask);
 150   }
 151   return NULL;
 152 }
 153
 154 int main(int argc, char** argv) {
 155   int i;
 156   if (argc != 3 && argc != 4) {
 157     fprintf(stderr, "usage: %s <num_roots> <register_workers_bool> [<spawn_root_number>]\n", argv[0]);
 158     exit(EXIT_FAILURE);
 159   }
 160
 161   // Initialize pthread mutex
 162   pthread_mutex_init(&lock, NULL);
 163
 164   // Get initial full mask
 165   full_mask = affinity_mask_alloc();
 166   get_thread_affinity(full_mask);
 167
 168   // Get the number of root pthreads to create and allocate resources for them
 169   num_roots = atoi(argv[1]);
 170   pthread_t *roots = (pthread_t*)malloc(sizeof(pthread_t) * num_roots);
 171   int *root_ids = (int*)malloc(sizeof(int) * num_roots);
 172
 173   // Get the flag indicating whether to have root pthreads call omp_set_num_threads() or not
 174   register_workers = atoi(argv[2]);
 175
 176   if (argc == 4)
 177     spawner = atoi(argv[3]);
 178
 179   // Spawn worker root threads
 180   for (i = 1; i < num_roots; ++i) {
 181     *(root_ids + i) = i;
 182     pthread_create(roots + i, NULL, thread_func, root_ids + i);
 183   }
 184   // Have main root thread (root 0) go into thread_func
 185   *root_ids = 0;
 186   thread_func(root_ids);
 187
 188   // Cleanup all resources
 189   for (i = 1; i < num_roots; ++i) {
 190     void *status;
 191     pthread_join(roots[i], &status);
 192   }
 193   free(roots);
 194   free(root_ids);
 195   pthread_mutex_destroy(&lock);
 196   return EXIT_SUCCESS;
 197 }