libgomp/ordered.c

   1 /* Copyright (C) 2005-2024 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file handles the ORDERED construct.  */
  27
  28 #include "libgomp.h"
  29 #include <stdarg.h>
  30 #include <string.h>
  31 #include "doacross.h"
  32
  33
  34 /* This function is called when first allocating an iteration block.  That
  35    is, the thread is not currently on the queue.  The work-share lock must
  36    be held on entry.  */
  37
  38 void
  39 gomp_ordered_first (void)
  40 {
  41   struct gomp_thread *thr = gomp_thread ();
  42   struct gomp_team *team = thr->ts.team;
  43   struct gomp_work_share *ws = thr->ts.work_share;
  44   unsigned index;
  45
  46   /* Work share constructs can be orphaned.  */
  47   if (team == NULL || team->nthreads == 1)
  48     return;
  49
  50   index = ws->ordered_cur + ws->ordered_num_used;
  51   if (index >= team->nthreads)
  52     index -= team->nthreads;
  53   ws->ordered_team_ids[index] = thr->ts.team_id;
  54
  55   /* If this is the first and only thread in the queue, then there is
  56      no one to release us when we get to our ordered section.  Post to
  57      our own release queue now so that we won't block later.  */
  58   if (ws->ordered_num_used++ == 0)
  59     gomp_sem_post (team->ordered_release[thr->ts.team_id]);
  60 }
  61
  62 /* This function is called when completing the last iteration block.  That
  63    is, there are no more iterations to perform and so the thread should be
  64    removed from the queue entirely.  Because of the way ORDERED blocks are
  65    managed, it follows that we currently own access to the ORDERED block,
  66    and should now pass it on to the next thread.  The work-share lock must
  67    be held on entry.  */
  68
  69 void
  70 gomp_ordered_last (void)
  71 {
  72   struct gomp_thread *thr = gomp_thread ();
  73   struct gomp_team *team = thr->ts.team;
  74   struct gomp_work_share *ws = thr->ts.work_share;
  75   unsigned next_id;
  76
  77   /* Work share constructs can be orphaned.  */
  78   if (team == NULL || team->nthreads == 1)
  79     return;
  80
  81   /* We're no longer the owner.  */
  82   ws->ordered_owner = -1;
  83
  84   /* If we're not the last thread in the queue, then wake the next.  */
  85   if (--ws->ordered_num_used > 0)
  86     {
  87       unsigned next = ws->ordered_cur + 1;
  88       if (next == team->nthreads)
  89         next = 0;
  90       ws->ordered_cur = next;
  91
  92       next_id = ws->ordered_team_ids[next];
  93       gomp_sem_post (team->ordered_release[next_id]);
  94     }
  95 }
  96
  97
  98 /* This function is called when allocating a subsequent allocation block.
  99    That is, we're done with the current iteration block and we're allocating
 100    another.  This is the logical combination of a call to gomp_ordered_last
 101    followed by a call to gomp_ordered_first.  The work-share lock must be
 102    held on entry. */
 103
 104 void
 105 gomp_ordered_next (void)
 106 {
 107   struct gomp_thread *thr = gomp_thread ();
 108   struct gomp_team *team = thr->ts.team;
 109   struct gomp_work_share *ws = thr->ts.work_share;
 110   unsigned index, next_id;
 111
 112   /* Work share constructs can be orphaned.  */
 113   if (team == NULL || team->nthreads == 1)
 114     return;
 115
 116   /* We're no longer the owner.  */
 117   ws->ordered_owner = -1;
 118
 119   /* If there's only one thread in the queue, that must be us.  */
 120   if (ws->ordered_num_used == 1)
 121     {
 122       /* We have a similar situation as in gomp_ordered_first
 123          where we need to post to our own release semaphore.  */
 124       gomp_sem_post (team->ordered_release[thr->ts.team_id]);
 125       return;
 126     }
 127
 128   /* If the queue is entirely full, then we move ourself to the end of
 129      the queue merely by incrementing ordered_cur.  Only if it's not
 130      full do we have to write our id.  */
 131   if (ws->ordered_num_used < team->nthreads)
 132     {
 133       index = ws->ordered_cur + ws->ordered_num_used;
 134       if (index >= team->nthreads)
 135         index -= team->nthreads;
 136       ws->ordered_team_ids[index] = thr->ts.team_id;
 137     }
 138
 139   index = ws->ordered_cur + 1;
 140   if (index == team->nthreads)
 141     index = 0;
 142   ws->ordered_cur = index;
 143
 144   next_id = ws->ordered_team_ids[index];
 145   gomp_sem_post (team->ordered_release[next_id]);
 146 }
 147
 148
 149 /* This function is called when a statically scheduled loop is first
 150    being created.  */
 151
 152 void
 153 gomp_ordered_static_init (void)
 154 {
 155   struct gomp_thread *thr = gomp_thread ();
 156   struct gomp_team *team = thr->ts.team;
 157
 158   if (team == NULL || team->nthreads == 1)
 159     return;
 160
 161   gomp_sem_post (team->ordered_release[0]);
 162 }
 163
 164 /* This function is called when a statically scheduled loop is moving to
 165    the next allocation block.  Static schedules are not first come first
 166    served like the others, so we're to move to the numerically next thread,
 167    not the next thread on a list.  The work-share lock should *not* be held
 168    on entry.  */
 169
 170 void
 171 gomp_ordered_static_next (void)
 172 {
 173   struct gomp_thread *thr = gomp_thread ();
 174   struct gomp_team *team = thr->ts.team;
 175   struct gomp_work_share *ws = thr->ts.work_share;
 176   unsigned id = thr->ts.team_id;
 177
 178   if (team == NULL || team->nthreads == 1)
 179     return;
 180
 181   ws->ordered_owner = -1;
 182
 183   /* This thread currently owns the lock.  Increment the owner.  */
 184   if (++id == team->nthreads)
 185     id = 0;
 186   ws->ordered_team_ids[0] = id;
 187   gomp_sem_post (team->ordered_release[id]);
 188 }
 189
 190 /* This function is called when we need to assert that the thread owns the
 191    ordered section.  Due to the problem of posted-but-not-waited semaphores,
 192    this needs to happen before completing a loop iteration.  */
 193
 194 void
 195 gomp_ordered_sync (void)
 196 {
 197   struct gomp_thread *thr = gomp_thread ();
 198   struct gomp_team *team = thr->ts.team;
 199   struct gomp_work_share *ws = thr->ts.work_share;
 200
 201   /* Work share constructs can be orphaned.  But this clearly means that
 202      we are the only thread, and so we automatically own the section.  */
 203   if (team == NULL || team->nthreads == 1)
 204     return;
 205
 206   /* ??? I believe it to be safe to access this data without taking the
 207      ws->lock.  The only presumed race condition is with the previous
 208      thread on the queue incrementing ordered_cur such that it points
 209      to us, concurrently with our check below.  But our team_id is
 210      already present in the queue, and the other thread will always
 211      post to our release semaphore.  So the two cases are that we will
 212      either win the race an momentarily block on the semaphore, or lose
 213      the race and find the semaphore already unlocked and so not block.
 214      Either way we get correct results.
 215      However, there is an implicit flush on entry to an ordered region,
 216      so we do need to have a barrier here.  If we were taking a lock
 217      this could be MEMMODEL_RELEASE since the acquire would be covered
 218      by the lock.  */
 219
 220   __atomic_thread_fence (MEMMODEL_ACQ_REL);
 221   if (ws->ordered_owner != thr->ts.team_id)
 222     {
 223       gomp_sem_wait (team->ordered_release[thr->ts.team_id]);
 224       ws->ordered_owner = thr->ts.team_id;
 225     }
 226 }
 227
 228 /* This function is called by user code when encountering the start of an
 229    ORDERED block.  We must check to see if the current thread is at the
 230    head of the queue, and if not, block.  */
 231
 232 #ifdef HAVE_ATTRIBUTE_ALIAS
 233 extern void GOMP_ordered_start (void)
 234         __attribute__((alias ("gomp_ordered_sync")));
 235 #else
 236 void
 237 GOMP_ordered_start (void)
 238 {
 239   gomp_ordered_sync ();
 240 }
 241 #endif
 242
 243 /* This function is called by user code when encountering the end of an
 244    ORDERED block.  With the current ORDERED implementation there's nothing
 245    for us to do.
 246
 247    However, the current implementation has a flaw in that it does not allow
 248    the next thread into the ORDERED section immediately after the current
 249    thread exits the ORDERED section in its last iteration.  The existence
 250    of this function allows the implementation to change.  */
 251
 252 void
 253 GOMP_ordered_end (void)
 254 {
 255 }
 256
 257 /* DOACROSS initialization.  */
 258
 259 #define MAX_COLLAPSED_BITS (__SIZEOF_LONG__ * __CHAR_BIT__)
 260
 261 void
 262 gomp_doacross_init (unsigned ncounts, long *counts, long chunk_size,
 263                     size_t extra)
 264 {
 265   struct gomp_thread *thr = gomp_thread ();
 266   struct gomp_team *team = thr->ts.team;
 267   struct gomp_work_share *ws = thr->ts.work_share;
 268   unsigned int i, bits[MAX_COLLAPSED_BITS], num_bits = 0;
 269   unsigned long ent, num_ents, elt_sz, shift_sz;
 270   struct gomp_doacross_work_share *doacross;
 271
 272   if (team == NULL || team->nthreads == 1)
 273     {
 274     empty:
 275       if (!extra)
 276         ws->doacross = NULL;
 277       else
 278         {
 279           doacross = gomp_malloc_cleared (sizeof (*doacross) + extra);
 280           doacross->extra = (void *) (doacross + 1);
 281           ws->doacross = doacross;
 282         }
 283       return;
 284     }
 285
 286   for (i = 0; i < ncounts; i++)
 287     {
 288       /* If any count is 0, GOMP_doacross_{post,wait} can't be called.  */
 289       if (counts[i] == 0)
 290         goto empty;
 291
 292       if (num_bits <= MAX_COLLAPSED_BITS)
 293         {
 294           unsigned int this_bits;
 295           if (counts[i] == 1)
 296             this_bits = 1;
 297           else
 298             this_bits = __SIZEOF_LONG__ * __CHAR_BIT__
 299                         - __builtin_clzl (counts[i] - 1);
 300           if (num_bits + this_bits <= MAX_COLLAPSED_BITS)
 301             {
 302               bits[i] = this_bits;
 303               num_bits += this_bits;
 304             }
 305           else
 306             num_bits = MAX_COLLAPSED_BITS + 1;
 307         }
 308     }
 309
 310   if (ws->sched == GFS_STATIC)
 311     num_ents = team->nthreads;
 312   else if (ws->sched == GFS_GUIDED)
 313     num_ents = counts[0];
 314   else
 315     num_ents = (counts[0] - 1) / chunk_size + 1;
 316   if (num_bits <= MAX_COLLAPSED_BITS)
 317     {
 318       elt_sz = sizeof (unsigned long);
 319       shift_sz = ncounts * sizeof (unsigned int);
 320     }
 321   else
 322     {
 323       elt_sz = sizeof (unsigned long) * ncounts;
 324       shift_sz = 0;
 325     }
 326   elt_sz = (elt_sz + 63) & ~63UL;
 327
 328   doacross = gomp_malloc (sizeof (*doacross) + 63 + num_ents * elt_sz
 329                           + shift_sz + extra);
 330   doacross->chunk_size = chunk_size;
 331   doacross->elt_sz = elt_sz;
 332   doacross->ncounts = ncounts;
 333   doacross->flattened = false;
 334   doacross->array = (unsigned char *)
 335                     ((((uintptr_t) (doacross + 1)) + 63 + shift_sz)
 336                      & ~(uintptr_t) 63);
 337   if (extra)
 338     {
 339       doacross->extra = doacross->array + num_ents * elt_sz;
 340       memset (doacross->extra, '\0', extra);
 341     }
 342   else
 343     doacross->extra = NULL;
 344   if (num_bits <= MAX_COLLAPSED_BITS)
 345     {
 346       unsigned int shift_count = 0;
 347       doacross->flattened = true;
 348       for (i = ncounts; i > 0; i--)
 349         {
 350           doacross->shift_counts[i - 1] = shift_count;
 351           shift_count += bits[i - 1];
 352         }
 353       for (ent = 0; ent < num_ents; ent++)
 354         *(unsigned long *) (doacross->array + ent * elt_sz) = 0;
 355     }
 356   else
 357     for (ent = 0; ent < num_ents; ent++)
 358       memset (doacross->array + ent * elt_sz, '\0',
 359               sizeof (unsigned long) * ncounts);
 360   if (ws->sched == GFS_STATIC && chunk_size == 0)
 361     {
 362       unsigned long q = counts[0] / num_ents;
 363       unsigned long t = counts[0] % num_ents;
 364       doacross->boundary = t * (q + 1);
 365       doacross->q = q;
 366       doacross->t = t;
 367     }
 368   ws->doacross = doacross;
 369 }
 370
 371 /* DOACROSS POST operation.  */
 372
 373 void
 374 GOMP_doacross_post (long *counts)
 375 {
 376   struct gomp_thread *thr = gomp_thread ();
 377   struct gomp_work_share *ws = thr->ts.work_share;
 378   struct gomp_doacross_work_share *doacross = ws->doacross;
 379   unsigned long ent;
 380   unsigned int i;
 381
 382   if (__builtin_expect (doacross == NULL, 0)
 383       || __builtin_expect (doacross->array == NULL, 0))
 384     {
 385       __sync_synchronize ();
 386       return;
 387     }
 388
 389   if (__builtin_expect (ws->sched == GFS_STATIC, 1))
 390     ent = thr->ts.team_id;
 391   else if (ws->sched == GFS_GUIDED)
 392     ent = counts[0];
 393   else
 394     ent = counts[0] / doacross->chunk_size;
 395   unsigned long *array = (unsigned long *) (doacross->array
 396                                             + ent * doacross->elt_sz);
 397
 398   if (__builtin_expect (doacross->flattened, 1))
 399     {
 400       unsigned long flattened
 401         = (unsigned long) counts[0] << doacross->shift_counts[0];
 402
 403       for (i = 1; i < doacross->ncounts; i++)
 404         flattened |= (unsigned long) counts[i]
 405                      << doacross->shift_counts[i];
 406       flattened++;
 407       if (flattened == __atomic_load_n (array, MEMMODEL_ACQUIRE))
 408         __atomic_thread_fence (MEMMODEL_RELEASE);
 409       else
 410         __atomic_store_n (array, flattened, MEMMODEL_RELEASE);
 411       return;
 412     }
 413
 414   __atomic_thread_fence (MEMMODEL_ACQUIRE);
 415   for (i = doacross->ncounts; i-- > 0; )
 416     {
 417       if (counts[i] + 1UL != __atomic_load_n (&array[i], MEMMODEL_RELAXED))
 418         __atomic_store_n (&array[i], counts[i] + 1UL, MEMMODEL_RELEASE);
 419     }
 420 }
 421
 422 /* DOACROSS WAIT operation.  */
 423
 424 void
 425 GOMP_doacross_wait (long first, ...)
 426 {
 427   struct gomp_thread *thr = gomp_thread ();
 428   struct gomp_work_share *ws = thr->ts.work_share;
 429   struct gomp_doacross_work_share *doacross = ws->doacross;
 430   va_list ap;
 431   unsigned long ent;
 432   unsigned int i;
 433
 434   if (__builtin_expect (doacross == NULL, 0)
 435       || __builtin_expect (doacross->array == NULL, 0))
 436     {
 437       __sync_synchronize ();
 438       return;
 439     }
 440
 441   if (__builtin_expect (ws->sched == GFS_STATIC, 1))
 442     {
 443       if (ws->chunk_size == 0)
 444         {
 445           if (first < doacross->boundary)
 446             ent = first / (doacross->q + 1);
 447           else
 448             ent = (first - doacross->boundary) / doacross->q
 449                   + doacross->t;
 450         }
 451       else
 452         ent = first / ws->chunk_size % thr->ts.team->nthreads;
 453     }
 454   else if (ws->sched == GFS_GUIDED)
 455     ent = first;
 456   else
 457     ent = first / doacross->chunk_size;
 458   unsigned long *array = (unsigned long *) (doacross->array
 459                                             + ent * doacross->elt_sz);
 460
 461   if (__builtin_expect (doacross->flattened, 1))
 462     {
 463       unsigned long flattened
 464         = (unsigned long) first << doacross->shift_counts[0];
 465       unsigned long cur;
 466
 467       va_start (ap, first);
 468       for (i = 1; i < doacross->ncounts; i++)
 469         flattened |= (unsigned long) va_arg (ap, long)
 470                      << doacross->shift_counts[i];
 471       cur = __atomic_load_n (array, MEMMODEL_ACQUIRE);
 472       if (flattened < cur)
 473         {
 474           __atomic_thread_fence (MEMMODEL_RELEASE);
 475           va_end (ap);
 476           return;
 477         }
 478       doacross_spin (array, flattened, cur);
 479       __atomic_thread_fence (MEMMODEL_RELEASE);
 480       va_end (ap);
 481       return;
 482     }
 483
 484   do
 485     {
 486       va_start (ap, first);
 487       for (i = 0; i < doacross->ncounts; i++)
 488         {
 489           unsigned long thisv
 490             = (unsigned long) (i ? va_arg (ap, long) : first) + 1;
 491           unsigned long cur = __atomic_load_n (&array[i], MEMMODEL_RELAXED);
 492           if (thisv < cur)
 493             {
 494               i = doacross->ncounts;
 495               break;
 496             }
 497           if (thisv > cur)
 498             break;
 499         }
 500       va_end (ap);
 501       if (i == doacross->ncounts)
 502         break;
 503       cpu_relax ();
 504     }
 505   while (1);
 506   __sync_synchronize ();
 507 }
 508
 509 typedef unsigned long long gomp_ull;
 510
 511 void
 512 gomp_doacross_ull_init (unsigned ncounts, gomp_ull *counts,
 513                         gomp_ull chunk_size, size_t extra)
 514 {
 515   struct gomp_thread *thr = gomp_thread ();
 516   struct gomp_team *team = thr->ts.team;
 517   struct gomp_work_share *ws = thr->ts.work_share;
 518   unsigned int i, bits[MAX_COLLAPSED_BITS], num_bits = 0;
 519   unsigned long ent, num_ents, elt_sz, shift_sz;
 520   struct gomp_doacross_work_share *doacross;
 521
 522   if (team == NULL || team->nthreads == 1)
 523     {
 524     empty:
 525       if (!extra)
 526         ws->doacross = NULL;
 527       else
 528         {
 529           doacross = gomp_malloc_cleared (sizeof (*doacross) + extra);
 530           doacross->extra = (void *) (doacross + 1);
 531           ws->doacross = doacross;
 532         }
 533       return;
 534     }
 535
 536   for (i = 0; i < ncounts; i++)
 537     {
 538       /* If any count is 0, GOMP_doacross_{post,wait} can't be called.  */
 539       if (counts[i] == 0)
 540         goto empty;
 541
 542       if (num_bits <= MAX_COLLAPSED_BITS)
 543         {
 544           unsigned int this_bits;
 545           if (counts[i] == 1)
 546             this_bits = 1;
 547           else
 548             this_bits = __SIZEOF_LONG_LONG__ * __CHAR_BIT__
 549                         - __builtin_clzll (counts[i] - 1);
 550           if (num_bits + this_bits <= MAX_COLLAPSED_BITS)
 551             {
 552               bits[i] = this_bits;
 553               num_bits += this_bits;
 554             }
 555           else
 556             num_bits = MAX_COLLAPSED_BITS + 1;
 557         }
 558     }
 559
 560   if (ws->sched == GFS_STATIC)
 561     num_ents = team->nthreads;
 562   else if (ws->sched == GFS_GUIDED)
 563     num_ents = counts[0];
 564   else
 565     num_ents = (counts[0] - 1) / chunk_size + 1;
 566   if (num_bits <= MAX_COLLAPSED_BITS)
 567     {
 568       elt_sz = sizeof (unsigned long);
 569       shift_sz = ncounts * sizeof (unsigned int);
 570     }
 571   else
 572     {
 573       if (sizeof (gomp_ull) == sizeof (unsigned long))
 574         elt_sz = sizeof (gomp_ull) * ncounts;
 575       else if (sizeof (gomp_ull) == 2 * sizeof (unsigned long))
 576         elt_sz = sizeof (unsigned long) * 2 * ncounts;
 577       else
 578         abort ();
 579       shift_sz = 0;
 580     }
 581   elt_sz = (elt_sz + 63) & ~63UL;
 582
 583   doacross = gomp_malloc (sizeof (*doacross) + 63 + num_ents * elt_sz
 584                           + shift_sz);
 585   doacross->chunk_size_ull = chunk_size;
 586   doacross->elt_sz = elt_sz;
 587   doacross->ncounts = ncounts;
 588   doacross->flattened = false;
 589   doacross->boundary = 0;
 590   doacross->array = (unsigned char *)
 591                     ((((uintptr_t) (doacross + 1)) + 63 + shift_sz)
 592                      & ~(uintptr_t) 63);
 593   if (extra)
 594     {
 595       doacross->extra = doacross->array + num_ents * elt_sz;
 596       memset (doacross->extra, '\0', extra);
 597     }
 598   else
 599     doacross->extra = NULL;
 600   if (num_bits <= MAX_COLLAPSED_BITS)
 601     {
 602       unsigned int shift_count = 0;
 603       doacross->flattened = true;
 604       for (i = ncounts; i > 0; i--)
 605         {
 606           doacross->shift_counts[i - 1] = shift_count;
 607           shift_count += bits[i - 1];
 608         }
 609       for (ent = 0; ent < num_ents; ent++)
 610         *(unsigned long *) (doacross->array + ent * elt_sz) = 0;
 611     }
 612   else
 613     for (ent = 0; ent < num_ents; ent++)
 614       memset (doacross->array + ent * elt_sz, '\0',
 615               sizeof (unsigned long) * ncounts);
 616   if (ws->sched == GFS_STATIC && chunk_size == 0)
 617     {
 618       gomp_ull q = counts[0] / num_ents;
 619       gomp_ull t = counts[0] % num_ents;
 620       doacross->boundary_ull = t * (q + 1);
 621       doacross->q_ull = q;
 622       doacross->t = t;
 623     }
 624   ws->doacross = doacross;
 625 }
 626
 627 /* DOACROSS POST operation.  */
 628
 629 void
 630 GOMP_doacross_ull_post (gomp_ull *counts)
 631 {
 632   struct gomp_thread *thr = gomp_thread ();
 633   struct gomp_work_share *ws = thr->ts.work_share;
 634   struct gomp_doacross_work_share *doacross = ws->doacross;
 635   unsigned long ent;
 636   unsigned int i;
 637
 638   if (__builtin_expect (doacross == NULL, 0)
 639       || __builtin_expect (doacross->array == NULL, 0))
 640     {
 641       __sync_synchronize ();
 642       return;
 643     }
 644
 645   if (__builtin_expect (ws->sched == GFS_STATIC, 1))
 646     ent = thr->ts.team_id;
 647   else if (ws->sched == GFS_GUIDED)
 648     ent = counts[0];
 649   else
 650     ent = counts[0] / doacross->chunk_size_ull;
 651
 652   if (__builtin_expect (doacross->flattened, 1))
 653     {
 654       unsigned long *array = (unsigned long *) (doacross->array
 655                               + ent * doacross->elt_sz);
 656       gomp_ull flattened
 657         = counts[0] << doacross->shift_counts[0];
 658
 659       for (i = 1; i < doacross->ncounts; i++)
 660         flattened |= counts[i] << doacross->shift_counts[i];
 661       flattened++;
 662       if (flattened == __atomic_load_n (array, MEMMODEL_ACQUIRE))
 663         __atomic_thread_fence (MEMMODEL_RELEASE);
 664       else
 665         __atomic_store_n (array, flattened, MEMMODEL_RELEASE);
 666       return;
 667     }
 668
 669   __atomic_thread_fence (MEMMODEL_ACQUIRE);
 670   if (sizeof (gomp_ull) == sizeof (unsigned long))
 671     {
 672       gomp_ull *array = (gomp_ull *) (doacross->array
 673                                       + ent * doacross->elt_sz);
 674
 675       for (i = doacross->ncounts; i-- > 0; )
 676         {
 677           if (counts[i] + 1UL != __atomic_load_n (&array[i], MEMMODEL_RELAXED))
 678             __atomic_store_n (&array[i], counts[i] + 1UL, MEMMODEL_RELEASE);
 679         }
 680     }
 681   else
 682     {
 683       unsigned long *array = (unsigned long *) (doacross->array
 684                                                 + ent * doacross->elt_sz);
 685
 686       for (i = doacross->ncounts; i-- > 0; )
 687         {
 688           gomp_ull cull = counts[i] + 1UL;
 689           unsigned long c = (unsigned long) cull;
 690           if (c != __atomic_load_n (&array[2 * i + 1], MEMMODEL_RELAXED))
 691             __atomic_store_n (&array[2 * i + 1], c, MEMMODEL_RELEASE);
 692           c = cull >> (__SIZEOF_LONG_LONG__ * __CHAR_BIT__ / 2);
 693           if (c != __atomic_load_n (&array[2 * i], MEMMODEL_RELAXED))
 694             __atomic_store_n (&array[2 * i], c, MEMMODEL_RELEASE);
 695         }
 696     }
 697 }
 698
 699 /* DOACROSS WAIT operation.  */
 700
 701 void
 702 GOMP_doacross_ull_wait (gomp_ull first, ...)
 703 {
 704   struct gomp_thread *thr = gomp_thread ();
 705   struct gomp_work_share *ws = thr->ts.work_share;
 706   struct gomp_doacross_work_share *doacross = ws->doacross;
 707   va_list ap;
 708   unsigned long ent;
 709   unsigned int i;
 710
 711   if (__builtin_expect (doacross == NULL, 0)
 712       || __builtin_expect (doacross->array == NULL, 0))
 713     {
 714       __sync_synchronize ();
 715       return;
 716     }
 717
 718   if (__builtin_expect (ws->sched == GFS_STATIC, 1))
 719     {
 720       if (ws->chunk_size_ull == 0)
 721         {
 722           if (first < doacross->boundary_ull)
 723             ent = first / (doacross->q_ull + 1);
 724           else
 725             ent = (first - doacross->boundary_ull) / doacross->q_ull
 726                   + doacross->t;
 727         }
 728       else
 729         ent = first / ws->chunk_size_ull % thr->ts.team->nthreads;
 730     }
 731   else if (ws->sched == GFS_GUIDED)
 732     ent = first;
 733   else
 734     ent = first / doacross->chunk_size_ull;
 735
 736   if (__builtin_expect (doacross->flattened, 1))
 737     {
 738       unsigned long *array = (unsigned long *) (doacross->array
 739                                                 + ent * doacross->elt_sz);
 740       gomp_ull flattened = first << doacross->shift_counts[0];
 741       unsigned long cur;
 742
 743       va_start (ap, first);
 744       for (i = 1; i < doacross->ncounts; i++)
 745         flattened |= va_arg (ap, gomp_ull)
 746                      << doacross->shift_counts[i];
 747       cur = __atomic_load_n (array, MEMMODEL_ACQUIRE);
 748       if (flattened < cur)
 749         {
 750           __atomic_thread_fence (MEMMODEL_RELEASE);
 751           va_end (ap);
 752           return;
 753         }
 754       doacross_spin (array, flattened, cur);
 755       __atomic_thread_fence (MEMMODEL_RELEASE);
 756       va_end (ap);
 757       return;
 758     }
 759
 760   if (sizeof (gomp_ull) == sizeof (unsigned long))
 761     {
 762       gomp_ull *array = (gomp_ull *) (doacross->array
 763                                       + ent * doacross->elt_sz);
 764       do
 765         {
 766           va_start (ap, first);
 767           for (i = 0; i < doacross->ncounts; i++)
 768             {
 769               gomp_ull thisv
 770                 = (i ? va_arg (ap, gomp_ull) : first) + 1;
 771               gomp_ull cur = __atomic_load_n (&array[i], MEMMODEL_RELAXED);
 772               if (thisv < cur)
 773                 {
 774                   i = doacross->ncounts;
 775                   break;
 776                 }
 777               if (thisv > cur)
 778                 break;
 779             }
 780           va_end (ap);
 781           if (i == doacross->ncounts)
 782             break;
 783           cpu_relax ();
 784         }
 785       while (1);
 786     }
 787   else
 788     {
 789       unsigned long *array = (unsigned long *) (doacross->array
 790                                                 + ent * doacross->elt_sz);
 791       do
 792         {
 793           va_start (ap, first);
 794           for (i = 0; i < doacross->ncounts; i++)
 795             {
 796               gomp_ull thisv
 797                 = (i ? va_arg (ap, gomp_ull) : first) + 1;
 798               unsigned long t
 799                 = thisv >> (__SIZEOF_LONG_LONG__ * __CHAR_BIT__ / 2);
 800               unsigned long cur
 801                 = __atomic_load_n (&array[2 * i], MEMMODEL_RELAXED);
 802               if (t < cur)
 803                 {
 804                   i = doacross->ncounts;
 805                   break;
 806                 }
 807               if (t > cur)
 808                 break;
 809               t = thisv;
 810               cur = __atomic_load_n (&array[2 * i + 1], MEMMODEL_RELAXED);
 811               if (t < cur)
 812                 {
 813                   i = doacross->ncounts;
 814                   break;
 815                 }
 816               if (t > cur)
 817                 break;
 818             }
 819           va_end (ap);
 820           if (i == doacross->ncounts)
 821             break;
 822           cpu_relax ();
 823         }
 824       while (1);
 825     }
 826   __sync_synchronize ();
 827 }