src/kmk/incdep.c

   1 #ifdef CONFIG_WITH_INCLUDEDEP
   2 /* $Id$ */
   3 /** @file
   4  * incdep - Simple dependency files.
   5  */
   6
   7 /*
   8  * Copyright (c) 2007-2008 knut st. osmundsen <bird-src-spam@anduin.net>
   9  *
  10  * This file is part of kBuild.
  11  *
  12  * kBuild is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License as published by
  14  * the Free Software Foundation; either version 3 of the License, or
  15  * (at your option) any later version.
  16  *
  17  * kBuild is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20  * GNU General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU General Public License
  23  * along with kBuild.  If not, see <http://www.gnu.org/licenses/>
  24  *
  25  */
  26
  27 /*******************************************************************************
  28 *   Header Files                                                               *
  29 *******************************************************************************/
  30 #ifdef __OS2__
  31 # define INCL_BASE
  32 # define INCL_ERRORS
  33 #endif
  34
  35 #include "make.h"
  36
  37 #if !defined(WINDOWS32) && !defined(__OS2__)
  38 # define HAVE_PTHREAD
  39 #endif
  40
  41 #include <assert.h>
  42
  43 #include <glob.h>
  44
  45 #include "dep.h"
  46 #include "filedef.h"
  47 #include "job.h"
  48 #include "commands.h"
  49 #include "variable.h"
  50 #include "rule.h"
  51 #include "debug.h"
  52 #include "strcache2.h"
  53
  54 #ifdef HAVE_FCNTL_H
  55 # include <fcntl.h>
  56 #else
  57 # include <sys/file.h>
  58 #endif
  59
  60 #ifdef WINDOWS32
  61 # include <io.h>
  62 # include <process.h>
  63 # include <Windows.h>
  64 # define PARSE_IN_WORKER
  65 #endif
  66
  67 #ifdef __OS2__
  68 # include <os2.h>
  69 # include <sys/fmutex.h>
  70 #endif
  71
  72 #ifdef HAVE_PTHREAD
  73 # include <pthread.h>
  74 #endif
  75
  76 #ifdef __APPLE__
  77 # include <malloc/malloc.h>
  78 # define PARSE_IN_WORKER
  79 #endif
  80
  81 #if defined(__gnu_linux__) || defined(__linux__)
  82 # define PARSE_IN_WORKER
  83 #endif
  84
  85
  86 /*******************************************************************************
  87 *   Structures and Typedefs                                                    *
  88 *******************************************************************************/
  89 struct incdep_variable_in_set
  90 {
  91     struct incdep_variable_in_set *next;
  92     /* the parameters */
  93     struct strcache2_entry *name_entry;     /* dep strcache - WRONG */
  94     const char *value;                      /* xmalloc'ed */
  95     unsigned int value_length;
  96     int duplicate_value;                    /* 0 */
  97     enum variable_origin origin;
  98     int recursive;
  99     struct variable_set *set;
 100     const struct floc *flocp;               /* NILF */
 101 };
 102
 103 struct incdep_variable_def
 104 {
 105     struct incdep_variable_def *next;
 106     /* the parameters */
 107     const struct floc *flocp;               /* NILF */
 108     struct strcache2_entry *name_entry;     /* dep strcache - WRONG */
 109     char *value;                            /* xmalloc'ed, free it */
 110     unsigned int value_length;
 111     enum variable_origin origin;
 112     enum variable_flavor flavor;
 113     int target_var;
 114 };
 115
 116 struct incdep_recorded_files
 117 {
 118     struct incdep_recorded_files *next;
 119
 120     /* the parameters */
 121     struct strcache2_entry *filename_entry; /* dep strcache; converted to a nameseq record. */
 122     const char *pattern;                    /* NULL */
 123     const char *pattern_percent;            /* NULL */
 124     struct dep *deps;                       /* All the names are dep strcache entries. */
 125     unsigned int cmds_started;              /* 0 */
 126     char *commands;                         /* NULL */
 127     unsigned int commands_idx;              /* 0 */
 128     int two_colon;                          /* 0 */
 129     const struct floc *flocp;               /* NILF */
 130 };
 131
 132
 133 /* per dep file structure. */
 134 struct incdep
 135 {
 136   struct incdep *next;
 137   char *file_base;
 138   char *file_end;
 139
 140   int worker_tid;
 141 #ifdef PARSE_IN_WORKER
 142   unsigned int err_line_no;
 143   const char *err_msg;
 144
 145   struct incdep_variable_in_set *recorded_variables_in_set_head;
 146   struct incdep_variable_in_set *recorded_variables_in_set_tail;
 147
 148   struct incdep_variable_def *recorded_variable_defs_head;
 149   struct incdep_variable_def *recorded_variable_defs_tail;
 150
 151   struct incdep_recorded_files *recorded_files_head;
 152   struct incdep_recorded_files *recorded_files_tail;
 153 #endif
 154
 155   char name[1];
 156 };
 157
 158
 159 /*******************************************************************************
 160 *   Global Variables                                                           *
 161 *******************************************************************************/
 162
 163 /* mutex protecting the globals and an associated condition/event. */
 164 #ifdef HAVE_PTHREAD
 165 static pthread_mutex_t incdep_mtx;
 166 static pthread_cond_t  incdep_cond_todo;
 167 static pthread_cond_t  incdep_cond_done;
 168
 169 #elif defined (WINDOWS32)
 170 static CRITICAL_SECTION incdep_mtx;
 171 static HANDLE incdep_hev_todo;
 172 static HANDLE incdep_hev_done;
 173 static int volatile incdep_hev_todo_waiters;
 174 static int volatile incdep_hev_done_waiters;
 175
 176 #elif defined (__OS2__)
 177 static _fmutex incdep_mtx;
 178 static HEV incdep_hev_todo;
 179 static HEV incdep_hev_done;
 180 static int volatile incdep_hev_todo_waiters;
 181 static int volatile incdep_hev_done_waiters;
 182 #endif
 183
 184 /* flag indicating whether the threads, lock and event/condvars has
 185    been initialized or not. */
 186 static int incdep_initialized;
 187
 188 /* the list of files that needs reading. */
 189 static struct incdep * volatile incdep_head_todo;
 190 static struct incdep * volatile incdep_tail_todo;
 191
 192 /* the number of files that are currently being read. */
 193 static int volatile incdep_num_reading;
 194
 195 /* the list of files that have been read. */
 196 static struct incdep * volatile incdep_head_done;
 197 static struct incdep * volatile incdep_tail_done;
 198
 199
 200 /* The handles to the worker threads. */
 201 #ifdef HAVE_PTHREAD
 202 # define INCDEP_MAX_THREADS 1
 203 static pthread_t incdep_threads[INCDEP_MAX_THREADS];
 204
 205 #elif defined (WINDOWS32)
 206 # define INCDEP_MAX_THREADS 2
 207 static HANDLE incdep_threads[INCDEP_MAX_THREADS];
 208
 209 #elif defined (__OS2__)
 210 # define INCDEP_MAX_THREADS 2
 211 static TID incdep_threads[INCDEP_MAX_THREADS];
 212 #endif
 213
 214 static struct alloccache incdep_rec_caches[INCDEP_MAX_THREADS];
 215 static struct alloccache incdep_dep_caches[INCDEP_MAX_THREADS];
 216 static struct strcache2 incdep_dep_strcaches[INCDEP_MAX_THREADS];
 217 static struct strcache2 incdep_var_strcaches[INCDEP_MAX_THREADS];
 218 static unsigned incdep_num_threads;
 219
 220 /* flag indicating whether the worker threads should terminate or not. */
 221 static int volatile incdep_terminate;
 222
 223 #ifdef __APPLE__
 224 /* malloc zone for the incdep threads. */
 225 static malloc_zone_t *incdep_zone;
 226 #endif
 227
 228
 229 /*******************************************************************************
 230 *   Internal Functions                                                         *
 231 *******************************************************************************/
 232 static void incdep_flush_it (struct floc *);
 233 static void eval_include_dep_file (struct incdep *, struct floc *);
 234
 235
 236 /* xmalloc wrapper.
 237    For working around multithreaded performance problems found on Darwin,
 238    Linux (glibc), and possibly other systems. */
 239 static void *
 240 incdep_xmalloc (struct incdep *cur, size_t size)
 241 {
 242   void *ptr;
 243
 244 #ifdef __APPLE__
 245   if (cur && cur->worker_tid != -1)
 246     {
 247       ptr = malloc_zone_malloc (incdep_zone, size);
 248       if (!ptr)
 249         fatal (NILF, _("virtual memory exhausted"));
 250     }
 251   else
 252     ptr = xmalloc (size);
 253 #else
 254   ptr = xmalloc (size);
 255 #endif
 256
 257   (void)cur;
 258   return ptr;
 259 }
 260
 261 #if 0
 262 /* memset(malloc(sz),'\0',sz) wrapper. */
 263 static void *
 264 incdep_xcalloc (struct incdep *cur, size_t size)
 265 {
 266   void *ptr;
 267
 268 #ifdef __APPLE__
 269   if (cur && cur->worker_tid != -1)
 270     ptr = malloc_zone_calloc (incdep_zone, size, 1);
 271   else
 272     ptr = calloc (size, 1);
 273 #else
 274   ptr = calloc (size, 1);
 275 #endif
 276   if (!ptr)
 277     fatal (NILF, _("virtual memory exhausted"));
 278
 279   (void)cur;
 280   return ptr;
 281 }
 282 #endif /* unused */
 283
 284 /* free wrapper */
 285 static void
 286 incdep_xfree (struct incdep *cur, void *ptr)
 287 {
 288   /* free() *must* work for the allocation hacks above because
 289      of free_dep_chain. */
 290   free (ptr);
 291   (void)cur;
 292 }
 293
 294 /* alloc a dep structure. These are allocated in bunches to save time. */
 295 struct dep *
 296 incdep_alloc_dep (struct incdep *cur)
 297 {
 298   struct alloccache *cache;
 299   if (cur->worker_tid != -1)
 300     cache = &incdep_dep_caches[cur->worker_tid];
 301   else
 302     cache = &dep_cache;
 303   return alloccache_calloc (cache);
 304 }
 305
 306 /* allocate a record. */
 307 static void *
 308 incdep_alloc_rec (struct incdep *cur)
 309 {
 310   return alloccache_alloc (&incdep_rec_caches[cur->worker_tid]);
 311 }
 312
 313 /* free a record. */
 314 static void
 315 incdep_free_rec (struct incdep *cur, void *rec)
 316 {
 317   /*alloccache_free (&incdep_rec_caches[cur->worker_tid], rec); - doesn't work of course. */
 318 }
 319
 320
 321 /* grow a cache. */
 322 static void *
 323 incdep_cache_allocator (void *thrd, unsigned int size)
 324 {
 325   (void)thrd;
 326 #ifdef __APPLE__
 327   return malloc_zone_malloc (incdep_zone, size);
 328 #else
 329   return xmalloc (size);
 330 #endif
 331 }
 332
 333 /* term a cache. */
 334 static void
 335 incdep_cache_deallocator (void *thrd, void *ptr, unsigned int size)
 336 {
 337   (void)thrd;
 338   (void)size;
 339   free (ptr);
 340 }
 341
 342 /* acquires the lock */
 343 void
 344 incdep_lock(void)
 345 {
 346 #if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
 347   pthread_mutex_lock (&incdep_mtx);
 348 #elif defined (WINDOWS32)
 349   EnterCriticalSection (&incdep_mtx);
 350 #elif defined (__OS2__)
 351   _fmutex_request (&incdep_mtx, 0);
 352 #endif
 353 }
 354
 355 /* releases the lock */
 356 void
 357 incdep_unlock(void)
 358 {
 359 #if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
 360   pthread_mutex_unlock (&incdep_mtx);
 361 #elif defined(WINDOWS32)
 362   LeaveCriticalSection (&incdep_mtx);
 363 #elif defined(__OS2__)
 364   _fmutex_release (&incdep_mtx);
 365 #endif
 366 }
 367
 368 /* signals the main thread that there is stuff todo. caller owns the lock. */
 369 static void
 370 incdep_signal_done (void)
 371 {
 372 #if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
 373   pthread_cond_broadcast (&incdep_cond_done);
 374 #elif defined (WINDOWS32)
 375   if (incdep_hev_done_waiters)
 376     SetEvent (incdep_hev_done);
 377 #elif defined (__OS2__)
 378   if (incdep_hev_done_waiters)
 379     DosPostEventSem (incdep_hev_done);
 380 #endif
 381 }
 382
 383 /* waits for a reader to finish reading. caller owns the lock. */
 384 static void
 385 incdep_wait_done (void)
 386 {
 387 #if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
 388   pthread_cond_wait (&incdep_cond_done, &incdep_mtx);
 389
 390 #elif defined (WINDOWS32)
 391   ResetEvent (incdep_hev_done);
 392   incdep_hev_done_waiters++;
 393   incdep_unlock ();
 394   WaitForSingleObject (incdep_hev_done, INFINITE);
 395   incdep_lock ();
 396   incdep_hev_done_waiters--;
 397
 398 #elif defined (__OS2__)
 399   ULONG ulIgnore;
 400   DosResetEventSem (incdep_hev_done, &ulIgnore);
 401   incdep_hev_done_waiters++;
 402   incdep_unlock ();
 403   DosWaitEventSem (incdep_hev_done, SEM_INDEFINITE_WAIT);
 404   incdep_lock ();
 405   incdep_hev_done_waiters--;
 406 #endif
 407 }
 408
 409 /* signals the worker threads. caller owns the lock. */
 410 static void
 411 incdep_signal_todo (void)
 412 {
 413 #if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
 414   pthread_cond_broadcast (&incdep_cond_todo);
 415 #elif defined(WINDOWS32)
 416   if (incdep_hev_todo_waiters)
 417     SetEvent (incdep_hev_todo);
 418 #elif defined(__OS2__)
 419   if (incdep_hev_todo_waiters)
 420     DosPostEventSem (incdep_hev_todo);
 421 #endif
 422 }
 423
 424 /* waits for stuff to arrive in the todo list. caller owns the lock. */
 425 static void
 426 incdep_wait_todo (void)
 427 {
 428 #if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
 429   pthread_cond_wait (&incdep_cond_todo, &incdep_mtx);
 430
 431 #elif defined (WINDOWS32)
 432   ResetEvent (incdep_hev_todo);
 433   incdep_hev_todo_waiters++;
 434   incdep_unlock ();
 435   WaitForSingleObject (incdep_hev_todo, INFINITE);
 436   incdep_lock ();
 437   incdep_hev_todo_waiters--;
 438
 439 #elif defined (__OS2__)
 440   ULONG ulIgnore;
 441   DosResetEventSem (incdep_hev_todo, &ulIgnore);
 442   incdep_hev_todo_waiters++;
 443   incdep_unlock ();
 444   DosWaitEventSem (incdep_hev_todo, SEM_INDEFINITE_WAIT);
 445   incdep_lock ();
 446   incdep_hev_todo_waiters--;
 447 #endif
 448 }
 449
 450 /* Reads a dep file into memory. */
 451 static int
 452 incdep_read_file (struct incdep *cur, struct floc *f)
 453 {
 454   int fd;
 455   struct stat st;
 456
 457   errno = 0;
 458 #ifdef O_BINARY
 459   fd = open (cur->name, O_RDONLY | O_BINARY, 0);
 460 #else
 461   fd = open (cur->name, O_RDONLY, 0);
 462 #endif
 463   if (fd < 0)
 464     {
 465       /* ignore non-existing dependency files. */
 466       int err = errno;
 467       if (err == ENOENT || stat (cur->name, &st) != 0)
 468         return 1;
 469       error (f, "%s: %s", cur->name, strerror (err));
 470       return -1;
 471     }
 472   if (!fstat (fd, &st))
 473     {
 474       cur->file_base = incdep_xmalloc (cur, st.st_size + 1);
 475       if (read (fd, cur->file_base, st.st_size) == st.st_size)
 476         {
 477           close (fd);
 478           cur->file_end = cur->file_base + st.st_size;
 479           cur->file_base[st.st_size] = '\0';
 480           return 0;
 481         }
 482
 483       /* bail out */
 484
 485       error (f, "%s: read: %s", cur->name, strerror (errno));
 486       incdep_xfree (cur, cur->file_base);
 487     }
 488   else
 489     error (f, "%s: fstat: %s", cur->name, strerror (errno));
 490
 491   close (fd);
 492   cur->file_base = cur->file_end = NULL;
 493   return -1;
 494 }
 495
 496 /* Free the incdep structure. */
 497 static void
 498 incdep_freeit (struct incdep *cur)
 499 {
 500 #ifdef PARSE_IN_WORKER
 501   assert (!cur->recorded_variables_in_set_head);
 502   assert (!cur->recorded_variable_defs_head);
 503   assert (!cur->recorded_files_head);
 504 #endif
 505
 506   incdep_xfree (cur, cur->file_base);
 507   cur->next = NULL;
 508   free (cur);
 509 }
 510
 511 /* A worker thread. */
 512 void
 513 incdep_worker (int thrd)
 514 {
 515   incdep_lock ();
 516
 517   while (!incdep_terminate)
 518    {
 519       /* get job from the todo list. */
 520
 521       struct incdep *cur = incdep_head_todo;
 522       if (!cur)
 523         {
 524           incdep_wait_todo ();
 525           continue;
 526         }
 527       if (cur->next)
 528         incdep_head_todo = cur->next;
 529       else
 530         incdep_head_todo = incdep_tail_todo = NULL;
 531       incdep_num_reading++;
 532
 533       /* read the file. */
 534
 535       incdep_unlock ();
 536       cur->worker_tid = thrd;
 537
 538       incdep_read_file (cur, NILF);
 539 #ifdef PARSE_IN_WORKER
 540       eval_include_dep_file (cur, NILF);
 541 #endif
 542
 543       cur->worker_tid = -1;
 544       incdep_lock ();
 545
 546       /* insert finished job into the done list. */
 547
 548       incdep_num_reading--;
 549       cur->next = NULL;
 550       if (incdep_tail_done)
 551         incdep_tail_done->next = cur;
 552       else
 553         incdep_head_done = cur;
 554       incdep_tail_done = cur;
 555
 556       incdep_signal_done ();
 557    }
 558
 559   incdep_unlock ();
 560 }
 561
 562 /* Thread library specific thread functions wrapping incdep_wroker. */
 563 #ifdef HAVE_PTHREAD
 564 static void *
 565 incdep_worker_pthread (void *thrd)
 566 {
 567   incdep_worker ((size_t)thrd);
 568   return NULL;
 569 }
 570
 571 #elif defined (WINDOWS32)
 572 static unsigned __stdcall
 573 incdep_worker_windows (void *thrd)
 574 {
 575   incdep_worker ((size_t)thrd);
 576   return 0;
 577 }
 578
 579 #elif defined (__OS2__)
 580 static void
 581 incdep_worker_os2 (void *thrd)
 582 {
 583   incdep_worker ((size_t)thrd);
 584 }
 585 #endif
 586
 587 /* Checks if threads are enabled or not.
 588
 589    This is a special hack so that is possible to disable the threads when in a
 590    debian fakeroot environment.  Thus, in addition to the KMK_THREADS_DISABLED
 591    and KMK_THREADS_ENABLED environment variable check we also check for signs
 592    of fakeroot.  */
 593 static int
 594 incdep_are_threads_enabled (void)
 595 {
 596 #if defined (CONFIG_WITHOUT_THREADS)
 597   return 0;
 598 #endif
 599
 600   if (getenv("KMK_THREADS_DISABLED"))
 601     return 0;
 602   if (getenv("KMK_THREADS_ENABLED"))
 603     return 1;
 604 #if defined(__gnu_linux__) || defined(__linux__)
 605   if (getenv("FAKEROOTKEY"))
 606     return 0;
 607   if (getenv("FAKEROOTUID"))
 608     return 0;
 609   if (getenv("FAKEROOTGID"))
 610     return 0;
 611   if (getenv("FAKEROOTEUID"))
 612     return 0;
 613   if (getenv("FAKEROOTEGID"))
 614     return 0;
 615   if (getenv("FAKEROOTSUID"))
 616     return 0;
 617   if (getenv("FAKEROOTSGID"))
 618     return 0;
 619   if (getenv("FAKEROOTFUID"))
 620     return 0;
 621   if (getenv("FAKEROOTFGID"))
 622     return 0;
 623   if (getenv("FAKEROOTDONTTRYCHOWN"))
 624     return 0;
 625   if (getenv("FAKEROOT_FD_BASE"))
 626     return 0;
 627   if (getenv("FAKEROOT_DB_SEARCH_PATHS"))
 628     return 0;
 629 #endif /* GNU/Linux */
 630   return 1;
 631 }
 632
 633 /* Creates the the worker threads. */
 634 static void
 635 incdep_init (struct floc *f)
 636 {
 637   unsigned i;
 638 #if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
 639   int rc;
 640   pthread_attr_t attr;
 641
 642 #elif defined (WINDOWS32)
 643   unsigned tid;
 644   uintptr_t hThread;
 645
 646 #elif defined (__OS2__)
 647   int rc;
 648   int tid;
 649 #endif
 650   (void)f;
 651
 652   /* heap hacks */
 653
 654 #ifdef __APPLE__
 655   incdep_zone = malloc_create_zone (0, 0);
 656   if (!incdep_zone)
 657     incdep_zone = malloc_default_zone ();
 658 #endif
 659
 660
 661   /* create the mutex and two condition variables / event objects. */
 662
 663 #if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
 664   rc = pthread_mutex_init (&incdep_mtx, NULL);
 665   if (rc)
 666     fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
 667   rc = pthread_cond_init (&incdep_cond_todo, NULL);
 668   if (rc)
 669     fatal (f, _("pthread_cond_init failed: err=%d"), rc);
 670   rc = pthread_cond_init (&incdep_cond_done, NULL);
 671   if (rc)
 672     fatal (f, _("pthread_cond_init failed: err=%d"), rc);
 673
 674 #elif defined (WINDOWS32)
 675   InitializeCriticalSection (&incdep_mtx);
 676   incdep_hev_todo = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
 677   if (!incdep_hev_todo)
 678     fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
 679   incdep_hev_done = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
 680   if (!incdep_hev_done)
 681     fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
 682   incdep_hev_todo_waiters = 0;
 683   incdep_hev_done_waiters = 0;
 684
 685 #elif defined (__OS2__)
 686   _fmutex_create (&incdep_mtx, 0);
 687   rc = DosCreateEventSem (NULL, &incdep_hev_todo, 0, FALSE);
 688   if (rc)
 689     fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
 690   rc = DosCreateEventSem (NULL, &incdep_hev_done, 0, FALSE);
 691   if (rc)
 692     fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
 693   incdep_hev_todo_waiters = 0;
 694   incdep_hev_done_waiters = 0;
 695 #endif
 696
 697   /* create the worker threads and associated per thread data. */
 698
 699   incdep_terminate = 0;
 700   if (incdep_are_threads_enabled())
 701     {
 702       incdep_num_threads = sizeof (incdep_threads) / sizeof (incdep_threads[0]);
 703       if (incdep_num_threads + 1 > job_slots)
 704         incdep_num_threads = job_slots <= 1 ? 1 : job_slots - 1;
 705       for (i = 0; i < incdep_num_threads; i++)
 706         {
 707           /* init caches */
 708           unsigned rec_size = sizeof (struct incdep_variable_in_set);
 709           if (rec_size < sizeof (struct incdep_variable_def))
 710             rec_size = sizeof (struct incdep_variable_def);
 711           if (rec_size < sizeof (struct incdep_recorded_files))
 712             rec_size = sizeof (struct incdep_recorded_files);
 713           alloccache_init (&incdep_rec_caches[i], rec_size, "incdep rec",
 714                            incdep_cache_allocator, (void *)(size_t)i);
 715           alloccache_init (&incdep_dep_caches[i], sizeof(struct dep), "incdep dep",
 716                            incdep_cache_allocator, (void *)(size_t)i);
 717           strcache2_init (&incdep_dep_strcaches[i],
 718                           "incdep dep", /* name */
 719                           65536,        /* hash size */
 720                           0,            /* default segment size*/
 721 #ifdef HAVE_CASE_INSENSITIVE_FS
 722                           1,            /* case insensitive */
 723 #else
 724                           0,            /* case insensitive */
 725 #endif
 726                           0);           /* thread safe */
 727
 728           strcache2_init (&incdep_var_strcaches[i],
 729                           "incdep var", /* name */
 730                           32768,        /* hash size */
 731                           0,            /* default segment size*/
 732                           0,            /* case insensitive */
 733                           0);           /* thread safe */
 734
 735           /* create the thread. */
 736 #if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
 737           rc = pthread_attr_init (&attr);
 738           if (rc)
 739             fatal (f, _("pthread_attr_init failed: err=%d"), rc);
 740           /*rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE); */
 741           rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
 742           if (rc)
 743             fatal (f, _("pthread_attr_setdetachstate failed: err=%d"), rc);
 744           rc = pthread_create(&incdep_threads[i], &attr,
 745                                incdep_worker_pthread, (void *)(size_t)i);
 746           if (rc)
 747             fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
 748           pthread_attr_destroy (&attr);
 749
 750 #elif defined (WINDOWS32)
 751           tid = 0;
 752           hThread = _beginthreadex (NULL, 128*1024, incdep_worker_windows,
 753                                     (void *)i, 0, &tid);
 754           if (hThread == 0 || hThread == ~(uintptr_t)0)
 755             fatal (f, _("_beginthreadex failed: err=%d"), errno);
 756           incdep_threads[i] = (HANDLE)hThread;
 757
 758 #elif defined (__OS2__)
 759           tid = _beginthread (incdep_worker_os2, NULL, 128*1024, (void *)i);
 760           if (tid <= 0)
 761             fatal (f, _("_beginthread failed: err=%d"), errno);
 762           incdep_threads[i] = tid;
 763 #endif
 764         }
 765     }
 766   else
 767     incdep_num_threads = 0;
 768
 769   incdep_initialized = 1;
 770 }
 771
 772 /* Flushes outstanding work and terminates the worker threads.
 773    This is called from snap_deps(). */
 774 void
 775 incdep_flush_and_term (void)
 776 {
 777   unsigned i;
 778
 779   if (!incdep_initialized)
 780     return;
 781
 782   /* flush any out standing work */
 783
 784   incdep_flush_it (NILF);
 785
 786   /* tell the threads to terminate */
 787
 788   incdep_lock ();
 789   incdep_terminate = 1;
 790   incdep_signal_todo ();
 791   incdep_unlock ();
 792
 793   /* wait for the threads to quit */
 794
 795   for (i = 0; i < incdep_num_threads; i++)
 796     {
 797       /* more later? */
 798
 799       /* terminate or join up the allocation caches. */
 800       alloccache_term (&incdep_rec_caches[i], incdep_cache_deallocator, (void *)(size_t)i);
 801       alloccache_join (&dep_cache, &incdep_dep_caches[i]);
 802       strcache2_term (&incdep_dep_strcaches[i]);
 803       strcache2_term (&incdep_var_strcaches[i]);
 804     }
 805   incdep_num_threads = 0;
 806
 807   /* destroy the lock and condition variables / event objects. */
 808
 809   /* later */
 810
 811   incdep_initialized = 0;
 812 }
 813
 814 #ifdef PARSE_IN_WORKER
 815 /* Flushes a strcache entry returning the actual string cache entry.
 816    The input is freed! */
 817 static const char *
 818 incdep_flush_strcache_entry (struct strcache2_entry *entry)
 819 {
 820   if (!entry->user)
 821     entry->user = (void *) strcache2_add_hashed_file (&file_strcache,
 822                                                       (const char *)(entry + 1),
 823                                                       entry->length, entry->hash);
 824   return (const char *)entry->user;
 825 }
 826
 827 /* Flushes the recorded instructions. */
 828 static void
 829 incdep_flush_recorded_instructions (struct incdep *cur)
 830 {
 831   struct incdep_variable_in_set *rec_vis;
 832   struct incdep_variable_def *rec_vd;
 833   struct incdep_recorded_files *rec_f;
 834
 835   /* define_variable_in_set */
 836
 837   rec_vis = cur->recorded_variables_in_set_head;
 838   cur->recorded_variables_in_set_head = cur->recorded_variables_in_set_tail = NULL;
 839   if (rec_vis)
 840     do
 841       {
 842         void *free_me = rec_vis;
 843         unsigned int name_length = rec_vis->name_entry->length;
 844         define_variable_in_set (incdep_flush_strcache_entry (rec_vis->name_entry),
 845                                 name_length,
 846                                 rec_vis->value,
 847                                 rec_vis->value_length,
 848                                 rec_vis->duplicate_value,
 849                                 rec_vis->origin,
 850                                 rec_vis->recursive,
 851                                 rec_vis->set,
 852                                 rec_vis->flocp);
 853         rec_vis = rec_vis->next;
 854         incdep_free_rec (cur, free_me);
 855       }
 856     while (rec_vis);
 857
 858   /* do_variable_definition */
 859
 860   rec_vd = cur->recorded_variable_defs_head;
 861   cur->recorded_variable_defs_head = cur->recorded_variable_defs_tail = NULL;
 862   if (rec_vd)
 863     do
 864       {
 865         void *free_me = rec_vd;
 866         do_variable_definition_2 (rec_vd->flocp,
 867                                   incdep_flush_strcache_entry (rec_vd->name_entry),
 868                                   rec_vd->value,
 869                                   rec_vd->value_length,
 870                                   0,
 871                                   rec_vd->value,
 872                                   rec_vd->origin,
 873                                   rec_vd->flavor,
 874                                   rec_vd->target_var);
 875         rec_vd = rec_vd->next;
 876         incdep_free_rec (cur, free_me);
 877       }
 878     while (rec_vd);
 879
 880   /* record_files */
 881
 882   rec_f = cur->recorded_files_head;
 883   cur->recorded_files_head = cur->recorded_files_tail = NULL;
 884   if (rec_f)
 885     do
 886       {
 887         void *free_me = rec_f;
 888         struct dep *dep;
 889         struct nameseq *filenames;
 890
 891         for (dep = rec_f->deps; dep; dep = dep->next)
 892           dep->name = incdep_flush_strcache_entry ((struct strcache2_entry *)dep->name);
 893
 894         filenames = (struct nameseq *) alloccache_alloc (&nameseq_cache);
 895         filenames->next = 0;
 896         filenames->name = incdep_flush_strcache_entry (rec_f->filename_entry);
 897
 898         record_files (filenames,
 899                       rec_f->pattern,
 900                       rec_f->pattern_percent,
 901                       rec_f->deps,
 902                       rec_f->cmds_started,
 903                       rec_f->commands,
 904                       rec_f->commands_idx,
 905                       rec_f->two_colon,
 906                       rec_f->flocp);
 907
 908         rec_f = rec_f->next;
 909         incdep_free_rec (cur, free_me);
 910       }
 911     while (rec_f);
 912 }
 913 #endif /* PARSE_IN_WORKER */
 914
 915 /* Record / issue a warning about a misformed dep file. */
 916 static void
 917 incdep_warn (struct incdep *cur, unsigned int line_no, const char *msg)
 918 {
 919   if (cur->worker_tid == -1)
 920     error (NILF, "%s(%d): %s", cur->name, line_no, msg);
 921 #ifdef PARSE_IN_WORKER
 922   else
 923     {
 924       cur->err_line_no = line_no;
 925       cur->err_msg = msg;
 926     }
 927 #endif
 928 }
 929
 930 /* Dependency or file strcache allocation / recording. */
 931 static const char *
 932 incdep_dep_strcache (struct incdep *cur, const char *str, int len)
 933 {
 934   const char *ret;
 935   if (cur->worker_tid == -1)
 936     {
 937       /* Make sure the string is terminated before we hand it to
 938          strcache_add_len so it does have to make a temporary copy
 939          of it on the stack. */
 940       char ch = str[len];
 941       ((char *)str)[len] = '\0';
 942       ret = strcache_add_len (str, len);
 943       ((char *)str)[len] = ch;
 944     }
 945   else
 946     {
 947       /* Add it out the strcache of the thread. */
 948       ret = strcache2_add (&incdep_dep_strcaches[cur->worker_tid], str, len);
 949       ret = (const char *)strcache2_get_entry(&incdep_dep_strcaches[cur->worker_tid], ret);
 950     }
 951   return ret;
 952 }
 953
 954 /* Variable name allocation / recording. */
 955 static const char *
 956 incdep_var_strcache (struct incdep *cur, const char *str, int len)
 957 {
 958   const char *ret;
 959   if (cur->worker_tid == -1)
 960     {
 961       /* XXX: we're leaking this memory now! This will be fixed later. */
 962       ret = xmalloc (len + 1);
 963       memcpy ((char *)ret, str, len);
 964       ((char *)ret)[len] = '\0';
 965     }
 966   else
 967     {
 968       /* Add it out the strcache of the thread. */
 969       ret = strcache2_add (&incdep_var_strcaches[cur->worker_tid], str, len);
 970       ret = (const char *)strcache2_get_entry(&incdep_var_strcaches[cur->worker_tid], ret);
 971     }
 972   return ret;
 973 }
 974
 975 /* Record / perform a variable definition in a set.
 976    The NAME is in the string cache.
 977    The VALUE is on the heap.
 978    The DUPLICATE_VALUE is always 0. */
 979 static void
 980 incdep_record_variable_in_set (struct incdep *cur,
 981                                const char *name, unsigned int name_length,
 982                                const char *value,
 983                                unsigned int value_length,
 984                                int duplicate_value,
 985                                enum variable_origin origin,
 986                                int recursive,
 987                                struct variable_set *set,
 988                                const struct floc *flocp)
 989 {
 990   assert (!duplicate_value);
 991   if (cur->worker_tid == -1)
 992     define_variable_in_set (name, name_length, value, value_length,
 993                             duplicate_value, origin, recursive, set, flocp);
 994 #ifdef PARSE_IN_WORKER
 995   else
 996     {
 997       struct incdep_variable_in_set *rec =
 998         (struct incdep_variable_in_set *)incdep_alloc_rec (cur);
 999       rec->name_entry = (struct strcache2_entry *)name;
1000       rec->value = value;
1001       rec->value_length = value_length;
1002       rec->duplicate_value = duplicate_value;
1003       rec->origin = origin;
1004       rec->recursive = recursive;
1005       rec->set = set;
1006       rec->flocp = flocp;
1007
1008       rec->next = NULL;
1009       if (cur->recorded_variables_in_set_tail)
1010         cur->recorded_variables_in_set_tail->next = rec;
1011       else
1012         cur->recorded_variables_in_set_head = rec;
1013       cur->recorded_variables_in_set_tail = rec;
1014     }
1015 #endif
1016 }
1017
1018 /* Record / perform a variable definition. The VALUE should be disposed of. */
1019 static void
1020 incdep_record_variable_def (struct incdep *cur,
1021                             const struct floc *flocp,
1022                             const char *name,
1023                             unsigned int name_length,
1024                             char *value,
1025                             unsigned int value_length,
1026                             enum variable_origin origin,
1027                             enum variable_flavor flavor,
1028                             int target_var)
1029 {
1030   if (cur->worker_tid == -1)
1031     do_variable_definition_2 (flocp, name, value, value_length, 0, value,
1032                               origin, flavor, target_var);
1033 #ifdef PARSE_IN_WORKER
1034   else
1035     {
1036       struct incdep_variable_def *rec =
1037         (struct incdep_variable_def *)incdep_alloc_rec (cur);
1038       rec->flocp = flocp;
1039       rec->name_entry = (struct strcache2_entry *)name;
1040       rec->value = value;
1041       rec->value_length = value_length;
1042       rec->origin = origin;
1043       rec->flavor = flavor;
1044       rec->target_var = target_var;
1045
1046       rec->next = NULL;
1047       if (cur->recorded_variable_defs_tail)
1048         cur->recorded_variable_defs_tail->next = rec;
1049       else
1050         cur->recorded_variable_defs_head = rec;
1051       cur->recorded_variable_defs_tail = rec;
1052     }
1053 #else
1054   (void)name_length;
1055 #endif
1056 }
1057
1058 /* Record files.*/
1059 static void
1060 incdep_record_files (struct incdep *cur,
1061                      const char *filename, const char *pattern,
1062                      const char *pattern_percent, struct dep *deps,
1063                      unsigned int cmds_started, char *commands,
1064                      unsigned int commands_idx, int two_colon,
1065                      const struct floc *flocp)
1066 {
1067   if (cur->worker_tid == -1)
1068     {
1069       struct nameseq *filenames = (struct nameseq *) alloccache_alloc (&nameseq_cache);
1070       filenames->next = 0;
1071       filenames->name = filename;
1072       record_files (filenames, pattern, pattern_percent, deps, cmds_started,
1073                     commands, commands_idx, two_colon, flocp);
1074     }
1075 #ifdef PARSE_IN_WORKER
1076   else
1077     {
1078       struct incdep_recorded_files *rec =
1079         (struct incdep_recorded_files *) incdep_alloc_rec (cur);
1080
1081       rec->filename_entry = (struct strcache2_entry *)filename;
1082       rec->pattern = pattern;
1083       rec->pattern_percent = pattern_percent;
1084       rec->deps = deps;
1085       rec->cmds_started = cmds_started;
1086       rec->commands = commands;
1087       rec->commands_idx = commands_idx;
1088       rec->two_colon = two_colon;
1089       rec->flocp = flocp;
1090
1091       rec->next = NULL;
1092       if (cur->recorded_files_tail)
1093         cur->recorded_files_tail->next = rec;
1094       else
1095         cur->recorded_files_head = rec;
1096       cur->recorded_files_tail = rec;
1097     }
1098 #endif
1099 }
1100
1101
1102 /* no nonsense dependency file including.
1103
1104    Because nobody wants bogus dependency files to break their incremental
1105    builds with hard to comprehend error messages, this function does not
1106    use the normal eval routine but does all the parsing itself. This isn't,
1107    as much work as it sounds, because the necessary feature set is very
1108    limited.
1109
1110    eval_include_dep_file groks:
1111
1112    define var
1113    endef
1114
1115    var [|:|?|>]= value [\]
1116
1117    [\]
1118    file: [deps] [\]
1119
1120    */
1121 static void
1122 eval_include_dep_file (struct incdep *curdep, struct floc *f)
1123 {
1124   unsigned line_no = 1;
1125   const char *file_end = curdep->file_end;
1126   const char *cur = curdep->file_base;
1127   const char *endp;
1128
1129   /* if no file data, just return immediately. */
1130   if (!cur)
1131     return;
1132
1133   /* now parse the file. */
1134   while (cur < file_end)
1135     {
1136       /* skip empty lines */
1137       while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1138         ++cur;
1139       if (cur >= file_end)
1140         break;
1141       if (*cur == '#')
1142         {
1143           cur = memchr (cur, '\n', file_end - cur);
1144           if (!cur)
1145             break;
1146         }
1147       if (*cur == '\\')
1148         {
1149           unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1150                            : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1151                            : (file_end - cur == 1) ? 1 : 0;
1152            if (eol_len)
1153              {
1154                cur += eol_len;
1155                line_no++;
1156                continue;
1157              }
1158         }
1159       if (*cur == '\n')
1160         {
1161           cur++;
1162           line_no++;
1163           continue;
1164         }
1165
1166       /* define var
1167          ...
1168          endef */
1169       if (strneq (cur, "define ", 7))
1170         {
1171           const char *var;
1172           unsigned var_len;
1173           const char *value_start;
1174           const char *value_end;
1175           char *value;
1176           unsigned value_len;
1177           int found_endef = 0;
1178
1179           /* extract the variable name. */
1180           cur += 7;
1181           while (isblank ((unsigned char)*cur))
1182             ++cur;
1183           value_start = endp = memchr (cur, '\n', file_end - cur);
1184           if (!endp)
1185               endp = cur;
1186           while (endp > cur && isspace ((unsigned char)endp[-1]))
1187             --endp;
1188           var_len = endp - cur;
1189           if (!var_len)
1190           {
1191               incdep_warn (curdep, line_no, "bogus define statement.");
1192               break;
1193           }
1194           var = incdep_var_strcache (curdep, cur, var_len);
1195
1196           /* find the end of the variable. */
1197           cur = value_end = value_start = value_start + 1;
1198           ++line_no;
1199           while (cur < file_end)
1200             {
1201               /* check for endef, don't bother with skipping leading spaces. */
1202               if (   file_end - cur >= 5
1203                   && strneq (cur, "endef", 5))
1204                 {
1205                   endp = cur + 5;
1206                   while (endp < file_end && isspace ((unsigned char)*endp) && *endp != '\n')
1207                     endp++;
1208                   if (endp >= file_end || *endp == '\n')
1209                     {
1210                       found_endef = 1;
1211                       cur = endp >= file_end ? file_end : endp + 1;
1212                       break;
1213                     }
1214                 }
1215
1216               /* skip a line ahead. */
1217               cur = value_end = memchr (cur, '\n', file_end - cur);
1218               if (cur != NULL)
1219                 ++cur;
1220               else
1221                 cur = value_end = file_end;
1222               ++line_no;
1223             }
1224
1225           if (!found_endef)
1226             {
1227               incdep_warn (curdep, line_no, "missing endef, dropping the rest of the file.");
1228               break;
1229             }
1230           value_len = value_end - value_start;
1231           if (memchr (value_start, '\0', value_len))
1232             {
1233               incdep_warn (curdep, line_no, "'\\0' in define, dropping the rest of the file.");
1234               break;
1235             }
1236
1237           /* make a copy of the value, converting \r\n to \n, and define it. */
1238           value = incdep_xmalloc (curdep, value_len + 1);
1239           endp = memchr (value_start, '\r', value_len);
1240           if (endp)
1241             {
1242               const char *src = value_start;
1243               char *dst = value;
1244               for (;;)
1245                 {
1246                   size_t len = endp - src;
1247                   memcpy (dst, src, len);
1248                   dst += len;
1249                   src = endp;
1250                   if (src + 1 < file_end && src[1] == '\n')
1251                       src++; /* skip the '\r' */
1252                   if (src >= value_end)
1253                     break;
1254                   endp = memchr (endp + 1, '\r', src - value_end);
1255                   if (!endp)
1256                     endp = value_end;
1257                 }
1258               value_len = dst - value;
1259             }
1260           else
1261             memcpy (value, value_start, value_len);
1262           value [value_len] = '\0';
1263
1264           incdep_record_variable_in_set (curdep,
1265                                          var, var_len, value, value_len,
1266                                          0 /* don't duplicate */, o_file,
1267                                          0 /* defines are recursive but this is faster */,
1268                                          NULL /* global set */, f);
1269         }
1270
1271       /* file: deps
1272          OR
1273          variable [:]= value */
1274       else
1275         {
1276           const char *colonp;
1277           const char *equalp;
1278
1279           /* Look for a colon and an equal sign, optimize for colon.
1280              Only one file is support and the colon / equal must be on
1281              the same line. */
1282           colonp = memchr (cur, ':', file_end - cur);
1283 #ifdef HAVE_DOS_PATHS
1284           while (   colonp
1285                  && colonp + 1 < file_end
1286                  && (colonp[1] == '/' || colonp[1] == '\\')
1287                  && colonp > cur
1288                  && isalpha ((unsigned char)colonp[-1])
1289                  && (   colonp == cur + 1
1290                      || strchr (" \t(", colonp[-2]) != 0))
1291               colonp = memchr (colonp + 1, ':', file_end - (colonp + 1));
1292 #endif
1293           endp = NULL;
1294           if (   !colonp
1295               ||  (endp = memchr (cur, '\n', colonp - cur)))
1296             {
1297               colonp = NULL;
1298               equalp = memchr (cur, '=', (endp ? endp : file_end) - cur);
1299               if (   !equalp
1300                   || (!endp && memchr (cur, '\n', equalp - cur)))
1301                 {
1302                   incdep_warn (curdep, line_no, "no colon.");
1303                   break;
1304                 }
1305             }
1306           else
1307             equalp = memchr (cur, '=', (colonp + 2 <= file_end
1308                                         ? colonp + 2 : file_end) - cur);
1309           if (equalp)
1310             {
1311               /* An assignment of some sort. */
1312               const char *var;
1313               unsigned var_len;
1314               const char *value_start;
1315               const char *value_end;
1316               char *value;
1317               unsigned value_len;
1318               unsigned multi_line = 0;
1319               enum variable_flavor flavor;
1320
1321               /* figure the flavor first. */
1322               flavor = f_recursive;
1323               if (equalp > cur)
1324                 {
1325                   if (equalp[-1] == ':')
1326                     flavor = f_simple;
1327                   else if (equalp[-1] == '?')
1328                     flavor = f_conditional;
1329                   else if (equalp[-1] == '+')
1330                     flavor = f_append;
1331                   else if (equalp[-1] == '>')
1332                     flavor = f_prepend;
1333                 }
1334
1335               /* extract the variable name. */
1336               endp = flavor == f_recursive ? equalp : equalp - 1;
1337               while (endp > cur && isblank ((unsigned char)endp[-1]))
1338                 --endp;
1339               var_len = endp - cur;
1340               if (!var_len)
1341                 {
1342                   incdep_warn (curdep, line_no, "empty variable. (includedep)");
1343                   break;
1344                 }
1345               if (   memchr (cur, '$', var_len)
1346                   || memchr (cur, ' ', var_len)
1347                   || memchr (cur, '\t', var_len))
1348                 {
1349                   incdep_warn (curdep, line_no, "fancy variable name. (includedep)");
1350                   break;
1351                 }
1352               var = incdep_var_strcache (curdep, cur, var_len);
1353
1354               /* find the start of the value. */
1355               cur = equalp + 1;
1356               while (cur < file_end && isblank ((unsigned char)*cur))
1357                 cur++;
1358               value_start = cur;
1359
1360               /* find the end of the value / line (this isn't 101% correct). */
1361               value_end = cur;
1362               while (cur < file_end)
1363                 {
1364                   endp = value_end = memchr (cur, '\n', file_end - cur);
1365                   if (!value_end)
1366                     value_end = file_end;
1367                   if (value_end - 1 >= cur && value_end[-1] == '\r')
1368                     --value_end;
1369                   if (value_end - 1 < cur || value_end[-1] != '\\')
1370                     {
1371                       cur = endp ? endp + 1 : file_end;
1372                       break;
1373                     }
1374                   --value_end;
1375                   if (value_end - 1 >= cur && value_end[-1] == '\\')
1376                     {
1377                       incdep_warn (curdep, line_no, "fancy escaping! (includedep)");
1378                       cur = NULL;
1379                       break;
1380                     }
1381                   if (!endp)
1382                     {
1383                       cur = file_end;
1384                       break;
1385                     }
1386
1387                   cur = endp + 1;
1388                   ++multi_line;
1389                   ++line_no;
1390                 }
1391               if (!cur)
1392                 break;
1393               ++line_no;
1394
1395               /* make a copy of the value, converting \r\n to \n, and define it. */
1396               value_len = value_end - value_start;
1397               value = incdep_xmalloc (curdep, value_len + 1);
1398               if (!multi_line)
1399                   memcpy (value, value_start, value_len);
1400               else
1401                 {
1402                   /* unescape it */
1403                   const char *src = value_start;
1404                   char *dst = value;
1405                   while (src < value_end)
1406                     {
1407                       const char *nextp;
1408
1409                       endp = memchr (src, '\n', value_end - src);
1410                       if (!endp)
1411                         nextp = endp = value_end;
1412                       else
1413                         nextp = endp + 1;
1414                       if (endp > src && endp[-1] == '\r')
1415                         --endp;
1416                       if (endp > src && endp[-1] == '\\')
1417                         --endp;
1418
1419                       if (src != value_start)
1420                         *dst++ = ' ';
1421                       memcpy (dst, src, endp - src);
1422                       dst += endp - src;
1423                       src = nextp;
1424                     }
1425                   value_len = dst - value;
1426                 }
1427               value [value_len] = '\0';
1428
1429               /* do the definition */
1430               if (flavor == f_recursive
1431                || (   flavor == f_simple
1432                    && !memchr (value, '$', value_len)))
1433                 incdep_record_variable_in_set (curdep,
1434                                                var, var_len, value, value_len,
1435                                                0 /* don't duplicate */, o_file,
1436                                                flavor == f_recursive /* recursive */,
1437                                                NULL /* global set */, f);
1438               else
1439                 incdep_record_variable_def (curdep,
1440                                             f, var, var_len, value, value_len,
1441                                             o_file, flavor, 0 /* not target var */);
1442             }
1443           else
1444             {
1445               /* file: dependencies */
1446
1447               const char *filename;
1448               struct dep *deps = 0;
1449               struct dep **nextdep = &deps;
1450               struct dep *dep;
1451
1452               /* extract the filename, ASSUME a single one. */
1453               endp = colonp;
1454               while (endp > cur && isblank ((unsigned char)endp[-1]))
1455                 --endp;
1456               if (cur == endp)
1457                 {
1458                   incdep_warn (curdep, line_no, "empty filename.");
1459                   break;
1460                 }
1461               if (   memchr (cur, '$', endp - cur)
1462                   || memchr (cur, ' ', endp - cur)
1463                   || memchr (cur, '\t', endp - cur))
1464                 {
1465                   incdep_warn (curdep, line_no, "multiple / fancy file name. (includedep)");
1466                   break;
1467                 }
1468               filename = incdep_dep_strcache (curdep, cur, endp - cur);
1469
1470               /* parse any dependencies. */
1471               cur = colonp + 1;
1472               while (cur < file_end)
1473                 {
1474                   /* skip blanks and count lines. */
1475                   while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1476                     ++cur;
1477                   if (cur >= file_end)
1478                     break;
1479                   if (*cur == '\n')
1480                     {
1481                       cur++;
1482                       line_no++;
1483                       break;
1484                     }
1485
1486                   /* continuation + eol? */
1487                   if (*cur == '\\')
1488                     {
1489                       unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1490                                        : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1491                                        : (file_end - cur == 1) ? 1 : 0;
1492                       if (eol_len)
1493                         {
1494                           cur += eol_len;
1495                           line_no++;
1496                           continue;
1497                         }
1498                     }
1499
1500                   /* find the end of the filename */
1501                   endp = cur;
1502                   while (endp < file_end && !isspace ((unsigned char)*endp))
1503                     ++endp;
1504
1505                   /* add it to the list. */
1506                   *nextdep = dep = incdep_alloc_dep (curdep);
1507                   dep->name = incdep_dep_strcache (curdep, cur, endp - cur);
1508                   dep->includedep = 1;
1509                   nextdep = &dep->next;
1510
1511                   cur = endp;
1512                 }
1513
1514               /* enter the file with its dependencies. */
1515               incdep_record_files (curdep,
1516                                    filename, NULL, NULL, deps, 0, NULL, 0, 0, f);
1517             }
1518         }
1519     }
1520
1521   /* free the file data */
1522   incdep_xfree (curdep, curdep->file_base);
1523   curdep->file_base = curdep->file_end = NULL;
1524 }
1525
1526 /* Flushes the incdep todo and done lists. */
1527 static void
1528 incdep_flush_it (struct floc *f)
1529 {
1530   incdep_lock ();
1531   for (;;)
1532     {
1533       struct incdep *cur = incdep_head_done;
1534
1535       /* if the done list is empty, grab a todo list entry. */
1536       if (!cur && incdep_head_todo)
1537         {
1538           cur = incdep_head_todo;
1539           if (cur->next)
1540             incdep_head_todo = cur->next;
1541           else
1542             incdep_head_todo = incdep_tail_todo = NULL;
1543           incdep_unlock ();
1544
1545           incdep_read_file (cur, f);
1546           eval_include_dep_file (cur, f);
1547           incdep_freeit (cur);
1548
1549           incdep_lock ();
1550           continue;
1551         }
1552
1553       /* if the todo list and done list are empty we're either done
1554          or will have to wait for the thread(s) to finish. */
1555       if (!cur && !incdep_num_reading)
1556           break; /* done */
1557       if (!cur)
1558         {
1559           while (!incdep_head_done)
1560             incdep_wait_done ();
1561           cur = incdep_head_done;
1562         }
1563
1564       /* we grab the entire done list and work thru it. */
1565       incdep_head_done = incdep_tail_done = NULL;
1566       incdep_unlock ();
1567
1568       while (cur)
1569         {
1570           struct incdep *next = cur->next;
1571 #ifdef PARSE_IN_WORKER
1572           incdep_flush_recorded_instructions (cur);
1573 #else
1574           eval_include_dep_file (cur, f);
1575 #endif
1576           incdep_freeit (cur);
1577           cur = next;
1578         }
1579
1580       incdep_lock ();
1581     } /* outer loop */
1582   incdep_unlock ();
1583 }
1584
1585
1586 /* splits up a list of file names and feeds it to eval_include_dep_file,
1587    employing threads to try speed up the file reading. */
1588 void
1589 eval_include_dep (const char *names, struct floc *f, enum incdep_op op)
1590 {
1591   struct incdep *head = 0;
1592   struct incdep *tail = 0;
1593   struct incdep *cur;
1594   const char *names_iterator = names;
1595   const char *name;
1596   unsigned int name_len;
1597
1598   /* loop through NAMES, creating a todo list out of them. */
1599
1600   while ((name = find_next_token (&names_iterator, &name_len)) != 0)
1601     {
1602        cur = xmalloc (sizeof (*cur) + name_len); /* not incdep_xmalloc here */
1603        cur->file_base = cur->file_end = NULL;
1604        memcpy (cur->name, name, name_len);
1605        cur->name[name_len] = '\0';
1606        cur->worker_tid = -1;
1607 #ifdef PARSE_IN_WORKER
1608        cur->err_line_no = 0;
1609        cur->err_msg = NULL;
1610        cur->recorded_variables_in_set_head = NULL;
1611        cur->recorded_variables_in_set_tail = NULL;
1612        cur->recorded_variable_defs_head = NULL;
1613        cur->recorded_variable_defs_tail = NULL;
1614        cur->recorded_files_head = NULL;
1615        cur->recorded_files_tail = NULL;
1616 #endif
1617
1618        cur->next = NULL;
1619        if (tail)
1620          tail->next = cur;
1621        else
1622          head = cur;
1623        tail = cur;
1624     }
1625
1626 #ifdef ELECTRIC_HEAP
1627   if (1)
1628 #else
1629   if (op == incdep_read_it)
1630 #endif
1631     {
1632       /* work our way thru the files directly */
1633
1634       cur = head;
1635       while (cur)
1636         {
1637           struct incdep *next = cur->next;
1638           incdep_read_file (cur, f);
1639           eval_include_dep_file (cur, f);
1640           incdep_freeit (cur);
1641           cur = next;
1642         }
1643     }
1644   else
1645     {
1646       /* initialize the worker threads and related stuff the first time around. */
1647
1648       if (!incdep_initialized)
1649         incdep_init (f);
1650
1651       /* queue the files and notify the worker threads. */
1652
1653       incdep_lock ();
1654
1655       if (incdep_tail_todo)
1656         incdep_tail_todo->next = head;
1657       else
1658         incdep_head_todo = head;
1659       incdep_tail_todo = tail;
1660
1661       incdep_signal_todo ();
1662       incdep_unlock ();
1663
1664       /* flush the todo queue if we're requested to do so. */
1665
1666       if (op == incdep_flush)
1667         incdep_flush_it (f);
1668     }
1669 }
1670
1671 #endif /* CONFIG_WITH_INCLUDEDEP */
1672