nptl/allocatestack.c

   1 /* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #include <assert.h>
  21 #include <errno.h>
  22 #include <signal.h>
  23 #include <stdint.h>
  24 #include <string.h>
  25 #include <unistd.h>
  26 #include <sys/mman.h>
  27 #include <sys/param.h>
  28 #include <dl-sysdep.h>
  29 #include <tls.h>
  30 #include <lowlevellock.h>
  31
  32
  33 #ifndef NEED_SEPARATE_REGISTER_STACK
  34
  35 /* Most architectures have exactly one stack pointer.  Some have more.  */
  36 # define STACK_VARIABLES void *stackaddr
  37
  38 /* How to pass the values to the 'create_thread' function.  */
  39 # define STACK_VARIABLES_ARGS stackaddr
  40
  41 /* How to declare function which gets there parameters.  */
  42 # define STACK_VARIABLES_PARMS void *stackaddr
  43
  44 /* How to declare allocate_stack.  */
  45 # define ALLOCATE_STACK_PARMS void **stack
  46
  47 /* This is how the function is called.  We do it this way to allow
  48    other variants of the function to have more parameters.  */
  49 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
  50
  51 #else
  52
  53 /* We need two stacks.  The kernel will place them but we have to tell
  54    the kernel about the size of the reserved address space.  */
  55 # define STACK_VARIABLES void *stackaddr; size_t stacksize
  56
  57 /* How to pass the values to the 'create_thread' function.  */
  58 # define STACK_VARIABLES_ARGS stackaddr, stacksize
  59
  60 /* How to declare function which gets there parameters.  */
  61 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
  62
  63 /* How to declare allocate_stack.  */
  64 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
  65
  66 /* This is how the function is called.  We do it this way to allow
  67    other variants of the function to have more parameters.  */
  68 # define ALLOCATE_STACK(attr, pd) \
  69   allocate_stack (attr, pd, &stackaddr, &stacksize)
  70
  71 #endif
  72
  73
  74 /* Default alignment of stack.  */
  75 #ifndef STACK_ALIGN
  76 # define STACK_ALIGN __alignof__ (long double)
  77 #endif
  78
  79 /* Default value for minimal stack size after allocating thread
  80    descriptor and guard.  */
  81 #ifndef MINIMAL_REST_STACK
  82 # define MINIMAL_REST_STACK     4096
  83 #endif
  84
  85
  86 /* Let the architecture add some flags to the mmap() call used to
  87    allocate stacks.  */
  88 #ifndef ARCH_MAP_FLAGS
  89 # define ARCH_MAP_FLAGS 0
  90 #endif
  91
  92 /* This yields the pointer that TLS support code calls the thread pointer.  */
  93 #if TLS_TCB_AT_TP
  94 # define TLS_TPADJ(pd) (pd)
  95 #elif TLS_DTV_AT_TP
  96 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
  97 #endif
  98
  99 /* Cache handling for not-yet free stacks.  */
 100
 101 /* Maximum size in kB of cache.  */
 102 static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default.  */
 103 static size_t stack_cache_actsize;
 104
 105 /* Mutex protecting this variable.  */
 106 static lll_lock_t stack_cache_lock = LLL_LOCK_INITIALIZER;
 107
 108 /* List of queued stack frames.  */
 109 static LIST_HEAD (stack_cache);
 110
 111 /* List of the stacks in use.  */
 112 static LIST_HEAD (stack_used);
 113
 114 /* List of the threads with user provided stacks in use.  No need to
 115    initialize this, since it's done in __pthread_initialize_minimal.  */
 116 list_t __stack_user __attribute__ ((nocommon));
 117 hidden_data_def (__stack_user)
 118
 119 #if COLORING_INCREMENT != 0
 120 /* Number of threads created.  */
 121 static unsigned int nptl_ncreated;
 122 #endif
 123
 124
 125 /* Check whether the stack is still used or not.  */
 126 #define FREE_P(descr) ((descr)->tid <= 0)
 127
 128
 129 /* We create a double linked list of all cache entries.  Double linked
 130    because this allows removing entries from the end.  */
 131
 132
 133 /* Get a stack frame from the cache.  We have to match by size since
 134    some blocks might be too small or far too large.  */
 135 static struct pthread *
 136 get_cached_stack (size_t *sizep, void **memp)
 137 {
 138   size_t size = *sizep;
 139   struct pthread *result = NULL;
 140   list_t *entry;
 141
 142   lll_lock (stack_cache_lock);
 143
 144   /* Search the cache for a matching entry.  We search for the
 145      smallest stack which has at least the required size.  Note that
 146      in normal situations the size of all allocated stacks is the
 147      same.  As the very least there are only a few different sizes.
 148      Therefore this loop will exit early most of the time with an
 149      exact match.  */
 150   list_for_each (entry, &stack_cache)
 151     {
 152       struct pthread *curr;
 153
 154       curr = list_entry (entry, struct pthread, list);
 155       if (FREE_P (curr) && curr->stackblock_size >= size)
 156         {
 157           if (curr->stackblock_size == size)
 158             {
 159               result = curr;
 160               break;
 161             }
 162
 163           if (result == NULL
 164               || result->stackblock_size > curr->stackblock_size)
 165             result = curr;
 166         }
 167     }
 168
 169   if (__builtin_expect (result == NULL, 0)
 170       /* Make sure the size difference is not too excessive.  In that
 171          case we do not use the block.  */
 172       || __builtin_expect (result->stackblock_size > 4 * size, 0))
 173     {
 174       /* Release the lock.  */
 175       lll_unlock (stack_cache_lock);
 176
 177       return NULL;
 178     }
 179
 180   /* Dequeue the entry.  */
 181   list_del (&result->list);
 182
 183   /* And add to the list of stacks in use.  */
 184   list_add (&result->list, &stack_used);
 185
 186   /* And decrease the cache size.  */
 187   stack_cache_actsize -= result->stackblock_size;
 188
 189   /* Release the lock early.  */
 190   lll_unlock (stack_cache_lock);
 191
 192   /* Report size and location of the stack to the caller.  */
 193   *sizep = result->stackblock_size;
 194   *memp = result->stackblock;
 195
 196   /* Cancellation handling is back to the default.  */
 197   result->cancelhandling = 0;
 198   result->cleanup = NULL;
 199
 200   /* No pending event.  */
 201   result->nextevent = NULL;
 202
 203   /* Clear the DTV.  */
 204   dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
 205   memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
 206
 207   /* Re-initialize the TLS.  */
 208   _dl_allocate_tls_init (TLS_TPADJ (result));
 209
 210   return result;
 211 }
 212
 213
 214 /* Add a stack frame which is not used anymore to the stack.  Must be
 215    called with the cache lock held.  */
 216 static inline void
 217 __attribute ((always_inline))
 218 queue_stack (struct pthread *stack)
 219 {
 220   /* We unconditionally add the stack to the list.  The memory may
 221      still be in use but it will not be reused until the kernel marks
 222      the stack as not used anymore.  */
 223   list_add (&stack->list, &stack_cache);
 224
 225   stack_cache_actsize += stack->stackblock_size;
 226   if (__builtin_expect (stack_cache_actsize > stack_cache_maxsize, 0))
 227     {
 228       /* We reduce the size of the cache.  Remove the last entries
 229          until the size is below the limit.  */
 230       list_t *entry;
 231       list_t *prev;
 232
 233       /* Search from the end of the list.  */
 234       list_for_each_prev_safe (entry, prev, &stack_cache)
 235         {
 236           struct pthread *curr;
 237
 238           curr = list_entry (entry, struct pthread, list);
 239           if (FREE_P (curr))
 240             {
 241               /* Unlink the block.  */
 242               list_del (entry);
 243
 244               /* Account for the freed memory.  */
 245               stack_cache_actsize -= curr->stackblock_size;
 246
 247               /* Free the memory associated with the ELF TLS.  */
 248               _dl_deallocate_tls (TLS_TPADJ (curr), false);
 249
 250               /* Remove this block.  This should never fail.  If it
 251                  does something is really wrong.  */
 252               if (munmap (curr->stackblock, curr->stackblock_size) != 0)
 253                 abort ();
 254
 255               /* Maybe we have freed enough.  */
 256               if (stack_cache_actsize <= stack_cache_maxsize)
 257                 break;
 258             }
 259         }
 260     }
 261 }
 262
 263
 264 static int
 265 internal_function
 266 change_stack_perm (struct pthread *pd
 267 #ifdef NEED_SEPARATE_REGISTER_STACK
 268                    , size_t pagemask
 269 #endif
 270                    )
 271 {
 272 #ifdef NEED_SEPARATE_REGISTER_STACK
 273   void *stack = (pd->stackblock
 274                  + (((((pd->stackblock_size - pd->guardsize) / 2)
 275                       & pagemask) + pd->guardsize) & pagemask));
 276   size_t len = pd->stackblock + pd->stackblock_size - stack;
 277 #else
 278   void *stack = pd->stackblock + pd->guardsize;
 279   size_t len = pd->stackblock_size - pd->guardsize;
 280 #endif
 281   if (mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
 282     return errno;
 283
 284   return 0;
 285 }
 286
 287
 288 static int
 289 allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
 290                 ALLOCATE_STACK_PARMS)
 291 {
 292   struct pthread *pd;
 293   size_t size;
 294   size_t pagesize_m1 = __getpagesize () - 1;
 295   void *stacktop;
 296
 297   assert (attr != NULL);
 298   assert (powerof2 (pagesize_m1 + 1));
 299   assert (TCB_ALIGNMENT >= STACK_ALIGN);
 300
 301   /* Get the stack size from the attribute if it is set.  Otherwise we
 302      use the default we determined at start time.  */
 303   size = attr->stacksize ?: __default_stacksize;
 304
 305   /* Get memory for the stack.  */
 306   if (__builtin_expect (attr->flags & ATTR_FLAG_STACKADDR, 0))
 307     {
 308       uintptr_t adj;
 309
 310       /* If the user also specified the size of the stack make sure it
 311          is large enough.  */
 312       if (attr->stacksize != 0
 313           && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
 314         return EINVAL;
 315
 316       /* Adjust stack size for alignment of the TLS block.  */
 317 #if TLS_TCB_AT_TP
 318       adj = ((uintptr_t) attr->stackaddr - TLS_TCB_SIZE)
 319             & __static_tls_align_m1;
 320       assert (size > adj + TLS_TCB_SIZE);
 321 #elif TLS_DTV_AT_TP
 322       adj = ((uintptr_t) attr->stackaddr - __static_tls_size)
 323             & __static_tls_align_m1;
 324       assert (size > adj);
 325 #endif
 326
 327       /* The user provided some memory.  Let's hope it matches the
 328          size...  We do not allocate guard pages if the user provided
 329          the stack.  It is the user's responsibility to do this if it
 330          is wanted.  */
 331 #if TLS_TCB_AT_TP
 332       pd = (struct pthread *) ((uintptr_t) attr->stackaddr
 333                                - TLS_TCB_SIZE - adj);
 334 #elif TLS_DTV_AT_TP
 335       pd = (struct pthread *) (((uintptr_t) attr->stackaddr
 336                                 - __static_tls_size - adj)
 337                                - TLS_PRE_TCB_SIZE);
 338 #endif
 339
 340       /* The user provided stack memory needs to be cleared.  */
 341       memset (pd, '\0', sizeof (struct pthread));
 342
 343       /* The first TSD block is included in the TCB.  */
 344       pd->specific[0] = pd->specific_1stblock;
 345
 346       /* Remember the stack-related values.  */
 347       pd->stackblock = (char *) attr->stackaddr - size;
 348       pd->stackblock_size = size;
 349
 350       /* This is a user-provided stack.  It will not be queued in the
 351          stack cache nor will the memory (except the TLS memory) be freed.  */
 352       pd->user_stack = true;
 353
 354       /* This is at least the second thread.  */
 355       pd->header.multiple_threads = 1;
 356 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 357       __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 358 #endif
 359
 360 #ifdef NEED_DL_SYSINFO
 361       /* Copy the sysinfo value from the parent.  */
 362       THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 363 #endif
 364
 365       /* The process ID is also the same as that of the caller.  */
 366       pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 367
 368       /* Allocate the DTV for this thread.  */
 369       if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 370         {
 371           /* Something went wrong.  */
 372           assert (errno == ENOMEM);
 373           return EAGAIN;
 374         }
 375
 376
 377       /* Prepare to modify global data.  */
 378       lll_lock (stack_cache_lock);
 379
 380       /* And add to the list of stacks in use.  */
 381       list_add (&pd->list, &__stack_user);
 382
 383       lll_unlock (stack_cache_lock);
 384     }
 385   else
 386     {
 387       /* Allocate some anonymous memory.  If possible use the cache.  */
 388       size_t guardsize;
 389       size_t reqsize;
 390       void *mem;
 391       const int prot = (PROT_READ | PROT_WRITE
 392                         | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
 393
 394 #if COLORING_INCREMENT != 0
 395       /* Add one more page for stack coloring.  Don't do it for stacks
 396          with 16 times pagesize or larger.  This might just cause
 397          unnecessary misalignment.  */
 398       if (size <= 16 * pagesize_m1)
 399         size += pagesize_m1 + 1;
 400 #endif
 401
 402       /* Adjust the stack size for alignment.  */
 403       size &= ~__static_tls_align_m1;
 404       assert (size != 0);
 405
 406       /* Make sure the size of the stack is enough for the guard and
 407          eventually the thread descriptor.  */
 408       guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
 409       if (__builtin_expect (size < (guardsize + __static_tls_size
 410                                     + MINIMAL_REST_STACK + pagesize_m1 + 1),
 411                             0))
 412         /* The stack is too small (or the guard too large).  */
 413         return EINVAL;
 414
 415       /* Try to get a stack from the cache.  */
 416       reqsize = size;
 417       pd = get_cached_stack (&size, &mem);
 418       if (pd == NULL)
 419         {
 420           /* To avoid aliasing effects on a larger scale than pages we
 421              adjust the allocated stack size if necessary.  This way
 422              allocations directly following each other will not have
 423              aliasing problems.  */
 424 #if MULTI_PAGE_ALIASING != 0
 425           if ((size % MULTI_PAGE_ALIASING) == 0)
 426             size += pagesize_m1 + 1;
 427 #endif
 428
 429           mem = mmap (NULL, size, prot,
 430                       MAP_PRIVATE | MAP_ANONYMOUS | ARCH_MAP_FLAGS, -1, 0);
 431
 432           if (__builtin_expect (mem == MAP_FAILED, 0))
 433             {
 434 #ifdef ARCH_RETRY_MMAP
 435               mem = ARCH_RETRY_MMAP (size);
 436               if (__builtin_expect (mem == MAP_FAILED, 0))
 437 #endif
 438                 return errno;
 439             }
 440
 441           /* SIZE is guaranteed to be greater than zero.
 442              So we can never get a null pointer back from mmap.  */
 443           assert (mem != NULL);
 444
 445 #if COLORING_INCREMENT != 0
 446           /* Atomically increment NCREATED.  */
 447           unsigned int ncreated = atomic_increment_val (&nptl_ncreated);
 448
 449           /* We chose the offset for coloring by incrementing it for
 450              every new thread by a fixed amount.  The offset used
 451              module the page size.  Even if coloring would be better
 452              relative to higher alignment values it makes no sense to
 453              do it since the mmap() interface does not allow us to
 454              specify any alignment for the returned memory block.  */
 455           size_t coloring = (ncreated * COLORING_INCREMENT) & pagesize_m1;
 456
 457           /* Make sure the coloring offsets does not disturb the alignment
 458              of the TCB and static TLS block.  */
 459           if (__builtin_expect ((coloring & __static_tls_align_m1) != 0, 0))
 460             coloring = (((coloring + __static_tls_align_m1)
 461                          & ~(__static_tls_align_m1))
 462                         & ~pagesize_m1);
 463 #else
 464           /* Unless specified we do not make any adjustments.  */
 465 # define coloring 0
 466 #endif
 467
 468           /* Place the thread descriptor at the end of the stack.  */
 469 #if TLS_TCB_AT_TP
 470           pd = (struct pthread *) ((char *) mem + size - coloring) - 1;
 471 #elif TLS_DTV_AT_TP
 472           pd = (struct pthread *) ((((uintptr_t) mem + size - coloring
 473                                     - __static_tls_size)
 474                                     & ~__static_tls_align_m1)
 475                                    - TLS_PRE_TCB_SIZE);
 476 #endif
 477
 478           /* Remember the stack-related values.  */
 479           pd->stackblock = mem;
 480           pd->stackblock_size = size;
 481
 482           /* We allocated the first block thread-specific data array.
 483              This address will not change for the lifetime of this
 484              descriptor.  */
 485           pd->specific[0] = pd->specific_1stblock;
 486
 487           /* This is at least the second thread.  */
 488           pd->header.multiple_threads = 1;
 489 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 490           __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 491 #endif
 492
 493 #ifdef NEED_DL_SYSINFO
 494           /* Copy the sysinfo value from the parent.  */
 495           THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 496 #endif
 497
 498           /* The process ID is also the same as that of the caller.  */
 499           pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 500
 501           /* Allocate the DTV for this thread.  */
 502           if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 503             {
 504               /* Something went wrong.  */
 505               assert (errno == ENOMEM);
 506
 507               /* Free the stack memory we just allocated.  */
 508               (void) munmap (mem, size);
 509
 510               return EAGAIN;
 511             }
 512
 513
 514           /* Prepare to modify global data.  */
 515           lll_lock (stack_cache_lock);
 516
 517           /* And add to the list of stacks in use.  */
 518           list_add (&pd->list, &stack_used);
 519
 520           lll_unlock (stack_cache_lock);
 521
 522
 523           /* There might have been a race.  Another thread might have
 524              caused the stacks to get exec permission while this new
 525              stack was prepared.  Detect if this was possible and
 526              change the permission if necessary.  */
 527           if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
 528                                 && (prot & PROT_EXEC) == 0, 0))
 529             {
 530               int err = change_stack_perm (pd
 531 #ifdef NEED_SEPARATE_REGISTER_STACK
 532                                            , ~pagesize_m1
 533 #endif
 534                                            );
 535               if (err != 0)
 536                 {
 537                   /* Free the stack memory we just allocated.  */
 538                   (void) munmap (mem, size);
 539
 540                   return err;
 541                 }
 542             }
 543
 544
 545           /* Note that all of the stack and the thread descriptor is
 546              zeroed.  This means we do not have to initialize fields
 547              with initial value zero.  This is specifically true for
 548              the 'tid' field which is always set back to zero once the
 549              stack is not used anymore and for the 'guardsize' field
 550              which will be read next.  */
 551         }
 552
 553       /* Create or resize the guard area if necessary.  */
 554       if (__builtin_expect (guardsize > pd->guardsize, 0))
 555         {
 556 #ifdef NEED_SEPARATE_REGISTER_STACK
 557           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 558 #else
 559           char *guard = mem;
 560 #endif
 561           if (mprotect (guard, guardsize, PROT_NONE) != 0)
 562             {
 563               int err;
 564             mprot_error:
 565               err = errno;
 566
 567               lll_lock (stack_cache_lock);
 568
 569               /* Remove the thread from the list.  */
 570               list_del (&pd->list);
 571
 572               lll_unlock (stack_cache_lock);
 573
 574               /* Get rid of the TLS block we allocated.  */
 575               _dl_deallocate_tls (TLS_TPADJ (pd), false);
 576
 577               /* Free the stack memory regardless of whether the size
 578                  of the cache is over the limit or not.  If this piece
 579                  of memory caused problems we better do not use it
 580                  anymore.  Uh, and we ignore possible errors.  There
 581                  is nothing we could do.  */
 582               (void) munmap (mem, size);
 583
 584               return err;
 585             }
 586
 587           pd->guardsize = guardsize;
 588         }
 589       else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
 590                                  0))
 591         {
 592           /* The old guard area is too large.  */
 593
 594 #ifdef NEED_SEPARATE_REGISTER_STACK
 595           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 596           char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
 597
 598           if (oldguard < guard
 599               && mprotect (oldguard, guard - oldguard, prot) != 0)
 600             goto mprot_error;
 601
 602           if (mprotect (guard + guardsize,
 603                         oldguard + pd->guardsize - guard - guardsize,
 604                         prot) != 0)
 605             goto mprot_error;
 606 #else
 607           if (mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
 608                         prot) != 0)
 609             goto mprot_error;
 610 #endif
 611
 612           pd->guardsize = guardsize;
 613         }
 614       /* The pthread_getattr_np() calls need to get passed the size
 615          requested in the attribute, regardless of how large the
 616          actually used guardsize is.  */
 617       pd->reported_guardsize = guardsize;
 618     }
 619
 620   /* Initialize the lock.  We have to do this unconditionally since the
 621      stillborn thread could be canceled while the lock is taken.  */
 622   pd->lock = LLL_LOCK_INITIALIZER;
 623
 624   /* We place the thread descriptor at the end of the stack.  */
 625   *pdp = pd;
 626
 627 #if TLS_TCB_AT_TP
 628   /* The stack begins before the TCB and the static TLS block.  */
 629   stacktop = ((char *) (pd + 1) - __static_tls_size);
 630 #elif TLS_DTV_AT_TP
 631   stacktop = (char *) (pd - 1);
 632 #endif
 633
 634 #ifdef NEED_SEPARATE_REGISTER_STACK
 635   *stack = pd->stackblock;
 636   *stacksize = stacktop - *stack;
 637 #else
 638   *stack = stacktop;
 639 #endif
 640
 641   return 0;
 642 }
 643
 644
 645 void
 646 internal_function
 647 __deallocate_stack (struct pthread *pd)
 648 {
 649   lll_lock (stack_cache_lock);
 650
 651   /* Remove the thread from the list of threads with user defined
 652      stacks.  */
 653   list_del (&pd->list);
 654
 655   /* Not much to do.  Just free the mmap()ed memory.  Note that we do
 656      not reset the 'used' flag in the 'tid' field.  This is done by
 657      the kernel.  If no thread has been created yet this field is
 658      still zero.  */
 659   if (__builtin_expect (! pd->user_stack, 1))
 660     (void) queue_stack (pd);
 661   else
 662     /* Free the memory associated with the ELF TLS.  */
 663     _dl_deallocate_tls (TLS_TPADJ (pd), false);
 664
 665   lll_unlock (stack_cache_lock);
 666 }
 667
 668
 669 int
 670 internal_function
 671 __make_stacks_executable (void **stack_endp)
 672 {
 673   /* First the main thread's stack.  */
 674   int err = _dl_make_stack_executable (stack_endp);
 675   if (err != 0)
 676     return err;
 677
 678 #ifdef NEED_SEPARATE_REGISTER_STACK
 679   const size_t pagemask = ~(__getpagesize () - 1);
 680 #endif
 681
 682   lll_lock (stack_cache_lock);
 683
 684   list_t *runp;
 685   list_for_each (runp, &stack_used)
 686     {
 687       err = change_stack_perm (list_entry (runp, struct pthread, list)
 688 #ifdef NEED_SEPARATE_REGISTER_STACK
 689                                , pagemask
 690 #endif
 691                                );
 692       if (err != 0)
 693         break;
 694     }
 695
 696   /* Also change the permission for the currently unused stacks.  This
 697      might be wasted time but better spend it here than adding a check
 698      in the fast path.  */
 699   if (err == 0)
 700     list_for_each (runp, &stack_cache)
 701       {
 702         err = change_stack_perm (list_entry (runp, struct pthread, list)
 703 #ifdef NEED_SEPARATE_REGISTER_STACK
 704                                  , pagemask
 705 #endif
 706                                  );
 707         if (err != 0)
 708           break;
 709       }
 710
 711   lll_unlock (stack_cache_lock);
 712
 713   return err;
 714 }
 715
 716
 717 /* In case of a fork() call the memory allocation in the child will be
 718    the same but only one thread is running.  All stacks except that of
 719    the one running thread are not used anymore.  We have to recycle
 720    them.  */
 721 void
 722 __reclaim_stacks (void)
 723 {
 724   struct pthread *self = (struct pthread *) THREAD_SELF;
 725
 726   /* No locking necessary.  The caller is the only stack in use.  */
 727
 728   /* Mark all stacks except the still running one as free.  */
 729   list_t *runp;
 730   list_for_each (runp, &stack_used)
 731     {
 732       struct pthread *curp;
 733
 734       curp = list_entry (runp, struct pthread, list);
 735       if (curp != self)
 736         {
 737           /* This marks the stack as free.  */
 738           curp->tid = 0;
 739
 740           /* The PID field must be initialized for the new process.  */
 741           curp->pid = self->pid;
 742
 743           /* Account for the size of the stack.  */
 744           stack_cache_actsize += curp->stackblock_size;
 745         }
 746     }
 747
 748   /* Add the stack of all running threads to the cache.  */
 749   list_splice (&stack_used, &stack_cache);
 750
 751   /* Remove the entry for the current thread to from the cache list
 752      and add it to the list of running threads.  Which of the two
 753      lists is decided by the user_stack flag.  */
 754   list_del (&self->list);
 755
 756   /* Re-initialize the lists for all the threads.  */
 757   INIT_LIST_HEAD (&stack_used);
 758   INIT_LIST_HEAD (&__stack_user);
 759
 760   if (__builtin_expect (THREAD_GETMEM (self, user_stack), 0))
 761     list_add (&self->list, &__stack_user);
 762   else
 763     list_add (&self->list, &stack_used);
 764
 765   /* There is one thread running.  */
 766   __nptl_nthreads = 1;
 767
 768   /* Initialize the lock.  */
 769   stack_cache_lock = LLL_LOCK_INITIALIZER;
 770 }
 771
 772
 773 #if HP_TIMING_AVAIL
 774 # undef __find_thread_by_id
 775 /* Find a thread given the thread ID.  */
 776 attribute_hidden
 777 struct pthread *
 778 __find_thread_by_id (pid_t tid)
 779 {
 780   struct pthread *result = NULL;
 781
 782   lll_lock (stack_cache_lock);
 783
 784   /* Iterate over the list with system-allocated threads first.  */
 785   list_t *runp;
 786   list_for_each (runp, &stack_used)
 787     {
 788       struct pthread *curp;
 789
 790       curp = list_entry (runp, struct pthread, list);
 791
 792       if (curp->tid == tid)
 793         {
 794           result = curp;
 795           goto out;
 796         }
 797     }
 798
 799   /* Now the list with threads using user-allocated stacks.  */
 800   list_for_each (runp, &__stack_user)
 801     {
 802       struct pthread *curp;
 803
 804       curp = list_entry (runp, struct pthread, list);
 805
 806       if (curp->tid == tid)
 807         {
 808           result = curp;
 809           goto out;
 810         }
 811     }
 812
 813  out:
 814   lll_unlock (stack_cache_lock);
 815
 816   return result;
 817 }
 818 #endif
 819
 820 int
 821 attribute_hidden
 822 __nptl_setxid (struct xid_command *cmdp)
 823 {
 824   int result;
 825   lll_lock (stack_cache_lock);
 826
 827   __xidcmd = cmdp;
 828   cmdp->cntr = 0;
 829
 830   INTERNAL_SYSCALL_DECL (err);
 831
 832   struct pthread *self = THREAD_SELF;
 833
 834   /* Iterate over the list with system-allocated threads first.  */
 835   list_t *runp;
 836   list_for_each (runp, &stack_used)
 837     {
 838       struct pthread *t = list_entry (runp, struct pthread, list);
 839       if (t != self)
 840         {
 841           int val;
 842 #if __ASSUME_TGKILL
 843           val = INTERNAL_SYSCALL (tgkill, err, 3,
 844                                   THREAD_GETMEM (THREAD_SELF, pid),
 845                                   t->tid, SIGSETXID);
 846 #else
 847 # ifdef __NR_tgkill
 848           val = INTERNAL_SYSCALL (tgkill, err, 3,
 849                                   THREAD_GETMEM (THREAD_SELF, pid),
 850                                   t->tid, SIGSETXID);
 851           if (INTERNAL_SYSCALL_ERROR_P (val, err)
 852               && INTERNAL_SYSCALL_ERRNO (val, err) == ENOSYS)
 853 # endif
 854             val = INTERNAL_SYSCALL (tkill, err, 2, t->tid, SIGSETXID);
 855 #endif
 856
 857           if (!INTERNAL_SYSCALL_ERROR_P (val, err))
 858             atomic_increment (&cmdp->cntr);
 859         }
 860     }
 861
 862   /* Now the list with threads using user-allocated stacks.  */
 863   list_for_each (runp, &__stack_user)
 864     {
 865       struct pthread *t = list_entry (runp, struct pthread, list);
 866       if (t != self)
 867         {
 868           int val;
 869 #if __ASSUME_TGKILL
 870           val = INTERNAL_SYSCALL (tgkill, err, 3,
 871                                   THREAD_GETMEM (THREAD_SELF, pid),
 872                                   t->tid, SIGSETXID);
 873 #else
 874 # ifdef __NR_tgkill
 875           val = INTERNAL_SYSCALL (tgkill, err, 3,
 876                                   THREAD_GETMEM (THREAD_SELF, pid),
 877                                   t->tid, SIGSETXID);
 878           if (INTERNAL_SYSCALL_ERROR_P (val, err)
 879               && INTERNAL_SYSCALL_ERRNO (val, err) == ENOSYS)
 880 # endif
 881             val = INTERNAL_SYSCALL (tkill, err, 2, t->tid, SIGSETXID);
 882 #endif
 883
 884           if (!INTERNAL_SYSCALL_ERROR_P (val, err))
 885             atomic_increment (&cmdp->cntr);
 886         }
 887     }
 888
 889   int cur = cmdp->cntr;
 890   while (cur != 0)
 891     {
 892       lll_futex_wait (&cmdp->cntr, cur);
 893       cur = cmdp->cntr;
 894     }
 895
 896   /* This must be last, otherwise the current thread might not have
 897      permissions to send SIGSETXID syscall to the other threads.  */
 898   result = INTERNAL_SYSCALL_NCS (cmdp->syscall_no, err, 3,
 899                                  cmdp->id[0], cmdp->id[1], cmdp->id[2]);
 900   if (INTERNAL_SYSCALL_ERROR_P (result, err))
 901     {
 902       __set_errno (INTERNAL_SYSCALL_ERRNO (result, err));
 903       result = -1;
 904     }
 905
 906   lll_unlock (stack_cache_lock);
 907   return result;
 908 }
 909
 910 static inline void __attribute__((always_inline))
 911 init_one_static_tls (struct pthread *curp, struct link_map *map)
 912 {
 913   dtv_t *dtv = GET_DTV (TLS_TPADJ (curp));
 914 # if TLS_TCB_AT_TP
 915   void *dest = (char *) curp - map->l_tls_offset;
 916 # elif TLS_DTV_AT_TP
 917   void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
 918 # else
 919 #  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
 920 # endif
 921
 922   /* Fill in the DTV slot so that a later LD/GD access will find it.  */
 923   dtv[map->l_tls_modid].pointer = dest;
 924
 925   /* Initialize the memory.  */
 926   memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
 927           '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
 928 }
 929
 930 void
 931 attribute_hidden
 932 __pthread_init_static_tls (struct link_map *map)
 933 {
 934   lll_lock (stack_cache_lock);
 935
 936   /* Iterate over the list with system-allocated threads first.  */
 937   list_t *runp;
 938   list_for_each (runp, &stack_used)
 939     init_one_static_tls (list_entry (runp, struct pthread, list), map);
 940
 941   /* Now the list with threads using user-allocated stacks.  */
 942   list_for_each (runp, &__stack_user)
 943     init_one_static_tls (list_entry (runp, struct pthread, list), map);
 944
 945   lll_unlock (stack_cache_lock);
 946 }