solenv/bin/concat-deps.c

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  *    Copyright (C) 2011 Norbert Thiebaud
   4  *    License: GPLv3
   5  */
   6
   7 /* define to activate stats reporting on hash usage*/
   8 /* #define HASH_STAT */
   9
  10 /* ===============================================
  11  * Set-up: defines to identify the system and system related properties
  12  * ===============================================
  13  */
  14
  15 #ifdef __APPLE__
  16 #ifdef __x86_64__
  17 #undef CORE_BIG_ENDIAN
  18 #define CORE_LITTLE_ENDIAN
  19 #else
  20 #define CORE_BIG_ENDIAN
  21 #undef CORE_LITTLE_ENDIAN
  22 #endif
  23
  24 #endif
  25 #ifdef _AIX
  26 #define CORE_BIG_ENDIAN
  27 #undef CORE_LITTLE_ENDIAN
  28 #endif /* Def _AIX */
  29
  30 #ifdef _MSC_VER
  31 #undef CORE_BIG_ENDIAN
  32 #define CORE_LITTLE_ENDIAN
  33 #endif /* Def _MSC_VER */
  34
  35 #if defined(__linux) || defined(__FreeBSD_kernel__)
  36 #include <sys/param.h>
  37 #if __BYTE_ORDER == __LITTLE_ENDIAN
  38 #undef CORE_BIG_ENDIAN
  39 #define CORE_LITTLE_ENDIAN
  40 #else /* !(__BYTE_ORDER == __LITTLE_ENDIAN) */
  41 #if __BYTE_ORDER == __BIG_ENDIAN
  42 #define CORE_BIG_ENDIAN
  43 #undef CORE_LITTLE_ENDIAN
  44 #endif /* __BYTE_ORDER == __BIG_ENDIAN */
  45 #endif /* !(__BYTE_ORDER == __LITTLE_ENDIAN) */
  46 #endif /* Def __linux */
  47
  48 #if defined(__OpenBSD__) || defined(__FreeBSD__) || \
  49     defined(__NetBSD__) || defined(__DragonFly__)
  50 #include <machine/endian.h>
  51 #if _BYTE_ORDER == _LITTLE_ENDIAN
  52 #undef CORE_BIG_ENDIAN
  53 #define CORE_LITTLE_ENDIAN
  54 #else /* !(_BYTE_ORDER == _LITTLE_ENDIAN) */
  55 #if _BYTE_ORDER == _BIG_ENDIAN
  56 #define CORE_BIG_ENDIAN
  57 #undef CORE_LITTLE_ENDIAN
  58 #endif /* _BYTE_ORDER == _BIG_ENDIAN */
  59 #endif /* !(_BYTE_ORDER == _LITTLE_ENDIAN) */
  60 #endif /* Def *BSD */
  61
  62 #if defined(__HAIKU__)
  63 #include <endian.h>
  64 #if __BYTE_ORDER == __LITTLE_ENDIAN
  65 #undef CORE_BIG_ENDIAN
  66 #define CORE_LITTLE_ENDIAN
  67 #else /* !(__BYTE_ORDER == __LITTLE_ENDIAN) */
  68 #if __BYTE_ORDER == __BIG_ENDIAN
  69 #define CORE_BIG_ENDIAN
  70 #undef CORE_LITTLE_ENDIAN
  71 #endif /* __BYTE_ORDER == __BIG_ENDIAN */
  72 #endif /* !(__BYTE_ORDER == __LITTLE_ENDIAN) */
  73 #endif /* Def __HAIKU__ */
  74
  75 #ifdef __sun
  76 #ifdef __sparc
  77 #define CORE_BIG_ENDIAN
  78 #undef CORE_LITTLE_ENDIAN
  79 #else  /* Ndef __sparc */
  80 #undef CORE_BIG_ENDIAN
  81 #define CORE_LITTLE_ENDIAN
  82 #endif /* Ndef __sparc */
  83 #endif /* Def __sun */
  84
  85 #include <assert.h>
  86 #include <stdio.h>
  87 #include <stdlib.h>
  88 #include <sys/types.h>
  89 #include <sys/stat.h>
  90 #include <errno.h>
  91 #include <fcntl.h>
  92 #include <string.h>
  93 #include <ctype.h>
  94
  95 #ifdef _MSC_VER
  96 #include <io.h>
  97 #else
  98 #include <unistd.h>
  99 #endif
 100
 101 #include <config_options.h>
 102
 103 /* modes */
 104 #ifdef _MSC_VER
 105 #define FILE_O_RDONLY     _O_RDONLY
 106 #define FILE_O_BINARY     _O_BINARY
 107 #define PATHNCMP _strnicmp /* MSVC converts paths to lower-case sometimes? */
 108 #define ssize_t long
 109 #define S_ISREG(mode) (((mode) & _S_IFMT) == (_S_IFREG)) /* MSVC does not have this macro */
 110 #else /* not windaube */
 111 #define FILE_O_RDONLY     O_RDONLY
 112 #define FILE_O_BINARY     0
 113 #define PATHNCMP strncmp
 114 #endif /* not windaube */
 115
 116 #ifndef TRUE
 117 #define TRUE 1
 118 #endif
 119 #ifndef FALSE
 120 #define FALSE 0
 121 #endif
 122
 123 static int internal_boost = 0;
 124 static char* base_dir;
 125 static char* work_dir;
 126 static size_t work_dir_len;
 127
 128 #ifdef __GNUC__
 129 #define clz __builtin_clz
 130 #else
 131 static int clz(unsigned int value)
 132 {
 133     int result = 32;
 134
 135     while(value)
 136     {
 137         value >>= 1;
 138         result -= 1;
 139     }
 140     return result;
 141 }
 142 #endif
 143
 144 static unsigned int get_unaligned_uint(const unsigned char* cursor)
 145 {
 146     unsigned int   result;
 147
 148     memcpy(&result, cursor, sizeof(unsigned int));
 149     return result;
 150 }
 151
 152 /* ===============================================
 153  * memory pool for fast fix-size allocation (non-thread-safe)
 154  * ===============================================
 155  */
 156 struct pool
 157 {
 158     void*    head_free;  /**< head of a linked list of freed element */
 159     char*    fresh;      /**< top of a memory block to dig new element */
 160     char*    tail;       /**< to detect end of extent... when fresh pass tail */
 161     void*    extent;     /**< pointer to the primary extent block */
 162     int      size_elem;  /**< size of an element. */
 163     int      primary;    /**< primary allocation in bytes */
 164     int      secondary;  /**< secondary allocation in bytes */
 165 };
 166 #define POOL_ALIGN_INCREMENT 8 /**< alignment, must be a power of 2 and of size > to sizeof(void*) */
 167
 168
 169 static void* pool_take_extent(struct pool* pool, int allocate)
 170 {
 171     unsigned int size = 0;
 172     void* extent;
 173     void* data = NULL;
 174
 175     if(pool->extent)
 176     {
 177         /* we already have an extent, so this is a secondary */
 178         if(pool->secondary)
 179         {
 180             size = pool->secondary;
 181         }
 182     }
 183     else
 184     {
 185         assert(pool->primary);
 186         size = pool->primary;
 187     }
 188     if(size)
 189     {
 190         extent = malloc(size);
 191         if(extent)
 192         {
 193             *(void**)extent = pool->extent;
 194             pool->extent = extent;
 195             if(allocate)
 196             {
 197                 data = ((char*)extent) + POOL_ALIGN_INCREMENT;
 198                 pool->fresh = ((char*)data) + pool->size_elem;
 199                 pool->tail = pool->fresh + (size - pool->size_elem);
 200             }
 201             else
 202             {
 203                 pool->fresh = ((char*)extent) + POOL_ALIGN_INCREMENT;
 204                 pool->tail = pool->fresh + (size - pool->size_elem);
 205             }
 206         }
 207     }
 208     return data;
 209 }
 210
 211 /* Create a memory pool for fix size objects
 212  * this is a simplified implementation that
 213  * is _not_ thread safe.
 214  */
 215 static struct pool* pool_create(int size_elem, int primary, int secondary)
 216 {
 217     struct pool* pool;
 218
 219     assert(primary > 0);
 220     assert(secondary >= 0);
 221     assert(size_elem > 0);
 222
 223     pool = (struct pool*)calloc(1, sizeof(struct pool));
 224     if(!pool) return NULL;
 225     /* Adjust the element size so that it be aligned, and so that an element could
 226      * at least contain a void*
 227      */
 228     pool->size_elem = size_elem = (size_elem + POOL_ALIGN_INCREMENT - 1) & ~(POOL_ALIGN_INCREMENT - 1);
 229
 230     pool->primary = (size_elem * primary) + POOL_ALIGN_INCREMENT;
 231     pool->secondary = secondary > 0 ? (size_elem * secondary) + POOL_ALIGN_INCREMENT : 0;
 232     pool_take_extent(pool, FALSE);
 233
 234     return pool;
 235
 236 }
 237
 238 static void pool_destroy(struct pool* pool)
 239 {
 240     void* extent;
 241     void* next;
 242
 243     if(pool != NULL)
 244     {
 245         extent = pool->extent;
 246         while(extent)
 247         {
 248             next = *(void**)extent;
 249             free(extent);
 250             extent = next;
 251         }
 252         free(pool);
 253     }
 254 }
 255
 256 static void* pool_alloc(struct pool* pool)
 257 {
 258     void* data;
 259
 260     data = pool->head_free;
 261     if(data == NULL)
 262     {
 263         /* we have no old-freed elem */
 264         if(pool->fresh <= pool->tail)
 265         {
 266             /* pick a slice of the current extent */
 267             data = (void*)pool->fresh;
 268             pool->fresh += pool->size_elem;
 269         }
 270         else
 271         {
 272             /* allocate a new extent */
 273             data = pool_take_extent(pool, TRUE);
 274         }
 275     }
 276     else
 277     {
 278         /* re-used old freed element by chopping the head of the free list */
 279         pool->head_free = *(void**)data;
 280     }
 281
 282     return data;
 283 }
 284
 285
 286 /* ===============================================
 287  * Hash implementation customized to be just tracking
 288  * a unique list of string (i.e no data associated
 289  * with the key, no need for retrieval, etc...
 290  *
 291  * This is tuned for the particular use-case we have here
 292  * measures in tail_build showed that
 293  * we can get north of 4000 distinct values stored in a hash
 294  * the collision rate is at worse around 2%
 295  * the collision needing an expensive memcmp to resolve
 296  * have a rate typically at 1 per 1000
 297  * for tail_build we register 37229 unique key
 298  * with a total of 377 extra memcmp needed
 299  * which is completely negligible compared to the
 300  * number of memcmp required to eliminate duplicate
 301  * entry (north of 2.5 millions for tail_build)
 302  * ===============================================
 303  */
 304
 305 struct hash_elem
 306 {
 307     struct hash_elem* next;
 308     const char*    key;
 309     int      key_len;
 310 };
 311
 312 struct hash
 313 {
 314     struct hash_elem** array;
 315     struct pool* elems_pool;
 316     unsigned int used;
 317     unsigned int size;
 318     unsigned int load_limit;
 319 #ifdef HASH_STAT
 320     int stored;
 321     int collisions;
 322     int cost;
 323     int memcmp;
 324 #endif
 325 };
 326
 327 /* The following hash_compute function was adapted from :
 328  * lookup3.c, by Bob Jenkins, May 2006, Public Domain.
 329  *
 330  * The changes from the original are mostly cosmetic
 331  */
 332 #define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k))))
 333
 334
 335 #if defined CORE_BIG_ENDIAN
 336 #define MASK_C1 0xFFFFFF00
 337 #define MASK_C2 0xFFFF0000
 338 #define MASK_C3 0xFF000000
 339 #elif defined CORE_LITTLE_ENDIAN
 340 #define MASK_C1 0xFFFFFF
 341 #define MASK_C2 0xFFFF
 342 #define MASK_C3 0xFF
 343 #else
 344 #error "Missing Endianness definition"
 345 #endif
 346
 347
 348 #define mix(a,b,c) \
 349 { \
 350   a -= c;  a ^= rot(c, 4);  c += b; \
 351   b -= a;  b ^= rot(a, 6);  a += c; \
 352   c -= b;  c ^= rot(b, 8);  b += a; \
 353   a -= c;  a ^= rot(c,16);  c += b; \
 354   b -= a;  b ^= rot(a,19);  a += c; \
 355   c -= b;  c ^= rot(b, 4);  b += a; \
 356 }
 357 #define final(a,b,c) \
 358 { \
 359   c ^= b; c -= rot(b,14); \
 360   a ^= c; a -= rot(c,11); \
 361   b ^= a; b -= rot(a,25); \
 362   c ^= b; c -= rot(b,16); \
 363   a ^= c; a -= rot(c,4);  \
 364   b ^= a; b -= rot(a,14); \
 365   c ^= b; c -= rot(b,24); \
 366 }
 367
 368 static unsigned int hash_compute( struct hash const * hash, const char* key, int length)
 369 {
 370     unsigned int a;
 371     unsigned int b;
 372     unsigned int c;                                          /* internal state */
 373     const unsigned char* uk = (const unsigned char*)key;
 374
 375     /* Set up the internal state */
 376     a = b = c = 0xdeadbeef + (length << 2);
 377
 378     /* we use this to 'hash' full path with mostly a common root
 379      * let's now waste too much cycles hashing mostly constant stuff
 380      */
 381     if(length > 36)
 382     {
 383         uk += length - 36;
 384         length = 36;
 385     }
 386     /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */
 387     while (length > 12)
 388     {
 389         a += get_unaligned_uint(uk);
 390         b += get_unaligned_uint(uk+4);
 391         c += get_unaligned_uint(uk+8);
 392         mix(a,b,c);
 393         length -= 12;
 394         uk += 12;
 395     }
 396
 397     /*----------------------------- handle the last (probably partial) block */
 398     /* Note: we possibly over-read, which would trigger complaint from VALGRIND
 399      * but we mask the undefined stuff if any, so we are still good, thanks
 400      * to alignment of memory allocation and tail-memory management overhead
 401      * we always can read 3 bytes past the official end without triggering
 402      * a segfault -- if you find a platform/compiler couple for which that postulate
 403      * is false, then you just need to over-allocate by 2 more bytes in file_load()
 404      * file_load already over-allocate by 1 to stick a \0 at the end of the buffer.
 405      */
 406     switch(length)
 407     {
 408     case 12: c+=get_unaligned_uint(uk+8); b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 409     case 11: c+=get_unaligned_uint(uk+8) & MASK_C1; b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 410     case 10: c+=get_unaligned_uint(uk+8) & MASK_C2; b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 411     case 9 : c+=get_unaligned_uint(uk+8) & MASK_C3; b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 412     case 8 : b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 413     case 7 : b+=get_unaligned_uint(uk+4) & MASK_C1; a+=get_unaligned_uint(uk); break;
 414     case 6 : b+=get_unaligned_uint(uk+4) & MASK_C2; a+=get_unaligned_uint(uk); break;
 415     case 5 : b+=get_unaligned_uint(uk+4) & MASK_C3; a+=get_unaligned_uint(uk); break;
 416     case 4 : a+=get_unaligned_uint(uk); break;
 417     case 3 : a+=get_unaligned_uint(uk) & MASK_C1; break;
 418     case 2 : a+=get_unaligned_uint(uk) & MASK_C2; break;
 419     case 1 : a+=get_unaligned_uint(uk) & MASK_C3; break;
 420     case 0 : return c & hash->size;              /* zero length strings require no mixing */
 421     }
 422
 423     final(a,b,c);
 424     return c & hash->size;
 425 }
 426
 427 static void hash_destroy(struct hash* hash)
 428 {
 429     if(hash)
 430     {
 431         if(hash->array)
 432         {
 433             free(hash->array);
 434         }
 435         if(hash->elems_pool)
 436         {
 437             pool_destroy(hash->elems_pool);
 438         }
 439         free(hash);
 440     }
 441 }
 442
 443 static struct hash* hash_create(unsigned int size)
 444 {
 445     struct hash* hash;
 446
 447     assert(size > 0);
 448     hash = calloc(1, sizeof(struct hash));
 449     if(hash)
 450     {
 451         size += (size >> 2) + 1; /* ~ 75% load factor */
 452         if(size >= 15)
 453         {
 454             hash->size = (((unsigned int)0xFFFFFFFF) >> clz((unsigned int)size));
 455         }
 456         else
 457         {
 458             hash->size = size = 15;
 459         }
 460         hash->load_limit = hash->size - (hash->size >> 2);
 461         hash->used = 0;
 462         hash->array = (struct hash_elem**)calloc(hash->size + 1, sizeof(struct hash_elem*));
 463         if(hash->array == NULL)
 464         {
 465             hash_destroy(hash);
 466             hash = NULL;
 467         }
 468     }
 469     if(hash)
 470     {
 471         hash->elems_pool = pool_create(sizeof(struct hash_elem),
 472                                        size, size << 1);
 473         if(!hash->elems_pool)
 474         {
 475             hash_destroy(hash);
 476             hash = NULL;
 477         }
 478     }
 479     return hash;
 480 }
 481
 482 static void hash_resize(struct hash* hash)
 483 {
 484     unsigned int old_size = hash->size;
 485     unsigned int hashed;
 486     struct hash_elem* hash_elem;
 487     struct hash_elem* next;
 488     struct hash_elem** array;
 489     unsigned int i;
 490
 491     hash->size = (old_size << 1) + 1;
 492     /* we really should avoid to get there... so print a message to alert of the condition */
 493     fprintf(stderr, "resize hash %u -> %u\n", old_size, hash->size);
 494     if(hash->size == old_size)
 495     {
 496         return;
 497     }
 498     array = calloc(hash->size + 1, sizeof(struct hash_elem*));
 499     if(array)
 500     {
 501         hash->load_limit = hash->size - (hash->size >> 2);
 502         for(i=0; i <= old_size; i++)
 503         {
 504             hash_elem = (struct hash_elem*)hash->array[i];
 505             while(hash_elem)
 506             {
 507                 next = hash_elem->next;
 508
 509                 hashed = hash_compute(hash, hash_elem->key, hash_elem->key_len);
 510                 hash_elem->next = array[hashed];
 511                 array[hashed] = hash_elem;
 512                 hash_elem = next;
 513             }
 514         }
 515         free(hash->array);
 516         hash->array = (struct hash_elem**)array;
 517     }
 518     else
 519     {
 520         hash->size = old_size;
 521     }
 522 }
 523
 524 static int compare_key(struct hash const * hash, const char* a, const char* b, int len, int const * cost)
 525 {
 526 #ifdef HASH_STAT
 527     *cost += 1;
 528     hash->memcmp += 1;
 529 #else
 530     (void) hash;
 531     (void) cost;
 532 #endif
 533     return memcmp(a,b, len);
 534 }
 535
 536 /* a customized hash_store function that just store the key and return
 537  * TRUE if the key was effectively stored, or FALSE if the key was already there
 538  */
 539 static int hash_store(struct hash* hash, const char* key, int key_len)
 540 {
 541     unsigned int hashed;
 542     struct hash_elem* hash_elem;
 543     int cost = 0;
 544
 545     (void) cost;
 546     hashed = hash_compute(hash, key, key_len);
 547 #ifdef HASH_STAT
 548     hash->stored += 1;
 549 #endif
 550     hash_elem = (struct hash_elem*)hash->array[hashed];
 551     while(hash_elem && (hash_elem->key_len != key_len || compare_key(hash, hash_elem->key, key, key_len, &cost)))
 552     {
 553         hash_elem = hash_elem->next;
 554     }
 555
 556     if(!hash_elem)
 557     {
 558         hash_elem = pool_alloc(hash->elems_pool);
 559         if(hash_elem)
 560         {
 561             hash_elem->key = key;
 562             hash_elem->key_len = key_len;
 563             hash_elem->next = hash->array[hashed];
 564
 565 #ifdef HASH_STAT
 566             if(hash_elem->next)
 567             {
 568                 hash->collisions += 1;
 569                 hash->cost += cost;
 570             }
 571 #endif
 572             hash->array[hashed] = hash_elem;
 573             hash->used += 1;
 574             if(hash->used > hash->load_limit)
 575             {
 576                 hash_resize(hash);
 577             }
 578         }
 579         return TRUE;
 580     }
 581     return FALSE;
 582 }
 583
 584 static int file_stat(const char* name, struct stat* buffer_stat, int* rc)
 585 {
 586     int rc_local = stat(name, buffer_stat);
 587     if (rc_local  < 0)
 588     {
 589         *rc = errno;
 590     }
 591     return rc_local;
 592 }
 593
 594 static off_t file_get_size(const char* name, int* rc)
 595 {
 596     struct stat buffer_stat;
 597     off_t       size = -1;
 598
 599     if (!file_stat(name, &buffer_stat, rc))
 600     {
 601         if(S_ISREG(buffer_stat.st_mode))
 602         {
 603             size = buffer_stat.st_size;
 604         }
 605         else
 606         {
 607             *rc = EINVAL;
 608         }
 609     }
 610     return size;
 611 }
 612
 613 #if !ENABLE_RUNTIME_OPTIMIZATIONS
 614 static void * file_load_buffers[100000];
 615 static size_t file_load_buffer_count = 0;
 616 #endif
 617
 618 static char* file_load(const char* name, off_t* size, int* return_rc)
 619 {
 620     off_t local_size = 0;
 621     int rc = 0;
 622     char* buffer = NULL;
 623     int fd;
 624
 625     assert(name != NULL);
 626
 627     if(!size)
 628     {
 629         size = &local_size;
 630     }
 631     *size = file_get_size(name, &rc);
 632     if (!rc && *size >= 0)
 633     {
 634         fd = open(name, FILE_O_RDONLY | FILE_O_BINARY);
 635         if (!(fd == -1))
 636         {
 637             buffer = malloc((size_t)(*size + 1));
 638 #if !ENABLE_RUNTIME_OPTIMIZATIONS
 639             if (buffer != NULL)
 640             {
 641                 if (file_load_buffer_count == 100000)
 642                 {
 643                     free(buffer);
 644                     buffer = NULL;
 645                 }
 646                 else
 647                 {
 648                     file_load_buffers[file_load_buffer_count++] = buffer;
 649                 }
 650             }
 651 #endif
 652             if (buffer == NULL)
 653             {
 654                 rc = ENOMEM;
 655             }
 656             else
 657             {
 658                 ssize_t i;
 659
 660               REDO:
 661                 i = read(fd, buffer, (size_t)(*size));
 662                 if(i == -1)
 663                 {
 664                     if(errno == EINTR)
 665                     {
 666                         goto REDO;
 667                     }
 668                     else
 669                     {
 670                         rc = errno;
 671                     }
 672                 }
 673                 else
 674                 {
 675                     if (i != *size)
 676                     {
 677                         rc = EIO;
 678                     }
 679                 }
 680                 buffer[*size] = 0;
 681             }
 682             close(fd);
 683         }
 684     }
 685
 686     if(rc && buffer)
 687     {
 688         free(buffer);
 689         buffer = NULL;
 690     }
 691     if(return_rc)
 692     {
 693         *return_rc = rc;
 694     }
 695     return buffer;
 696 }
 697
 698 static void cancel_relative(char const * base, char** ref_cursor, char** ref_cursor_out, char const * end)
 699 {
 700     char* cursor = *ref_cursor;
 701     char* cursor_out = *ref_cursor_out;
 702
 703     do
 704     {
 705         cursor += 3;
 706         while(cursor_out > base && cursor_out[-1] == '/')
 707             cursor_out--;
 708         while(cursor_out > base && *--cursor_out != '/');
 709     }
 710     while(cursor + 3 < end && !memcmp(cursor, "/../", 4));
 711     *ref_cursor = cursor;
 712     *ref_cursor_out = cursor_out;
 713 }
 714
 715 static void eat_space(char ** token)
 716 {
 717     while ((' ' == **token) || ('\t' == **token)) {
 718         ++(*token);
 719     }
 720 }
 721
 722 /*
 723  * Prune LibreOffice specific duplicate dependencies to improve
 724  * gnumake startup time, and shrink the disk-space footprint.
 725  */
 726 static int
 727 elide_dependency(const char* key, int key_len, const char **unpacked_end)
 728 {
 729 #if 0
 730     {
 731         int i;
 732         fprintf (stderr, "elide?%d!: '", internal_boost);
 733         for (i = 0; i < key_len; i++) {
 734             fprintf (stderr, "%c", key[i]);
 735         }
 736         fprintf (stderr, "'\n");
 737     }
 738 #endif
 739
 740     /* boost brings a plague of header files */
 741     int i;
 742     int unpacked = 0;
 743     /* walk down path elements */
 744     for (i = 0; i < key_len - 1; i++)
 745     {
 746         if (key[i] == '/')
 747         {
 748             if (0 == unpacked)
 749             {
 750                 if (!PATHNCMP(key + i + 1, "workdir/", 8))
 751                 {
 752                     unpacked = 1;
 753                     continue;
 754                 }
 755             }
 756             else
 757             {
 758                 if (!PATHNCMP(key + i + 1, "UnpackedTarball/", 16))
 759                 {
 760                     if (unpacked_end)
 761                         *unpacked_end = strchr(key + i + 17, '/');
 762                     return 1;
 763                 }
 764             }
 765         }
 766     }
 767
 768     return 0;
 769 }
 770
 771 /*
 772  * We collapse tens of internal boost headers to the unpacked target, such
 773  * that you can re-compile / install boost and all is well.
 774  */
 775 static void emit_single_boost_header(void)
 776 {
 777 #define BOOST_TARGET "/UnpackedTarball/boost.done"
 778     fprintf(stdout, "%s" BOOST_TARGET " ", work_dir);
 779 }
 780
 781 static void emit_unpacked_target(const char* token, const char* end)
 782 {
 783     fwrite(token, 1, end-token, stdout);
 784     fputs(".done ", stdout);
 785 }
 786
 787 /* prefix paths to absolute */
 788 static void print_fullpaths(char* line)
 789 {
 790     char* token;
 791     char* end;
 792     int boost_count = 0;
 793     int token_len;
 794     const char * unpacked_end = NULL; /* end of UnpackedTarget match (if any) */
 795     /* for UnpackedTarget the target is GenC{,xx}Object, don't mangle! */
 796     int target_seen = 0;
 797
 798     token = line;
 799     eat_space(&token);
 800     while (*token)
 801     {
 802         end = token;
 803         /* hard to believe that in this day and age drive letters still exist */
 804         if (*end && (':' == *(end+1)) &&
 805             (('\\' == *(end+2)) || ('/' == *(end+2))) &&
 806             isalpha((unsigned char)*end))
 807         {
 808             end = end + 3; /* only one cross, err drive letter per filename */
 809         }
 810         while (*end && (' ' != *end) && ('\t' != *end) && (':' != *end)) {
 811             ++end;
 812         }
 813         token_len = end - token;
 814         if (target_seen &&
 815             elide_dependency(token, token_len, &unpacked_end))
 816         {
 817             if (unpacked_end)
 818             {
 819                 if (internal_boost && !PATHNCMP(unpacked_end - 5, "boost", 5))
 820                 {
 821                     ++boost_count;
 822                     if (boost_count == 1)
 823                         emit_single_boost_header();
 824                     else
 825                     {
 826                         /* don't output, and swallow trailing \\\n if any */
 827                         token = end;
 828                         eat_space(&token);
 829                         if (token[0] == '\\' && token[1] == '\n')
 830                             end = token + 2;
 831                     }
 832                 }
 833                 else
 834                 {
 835                     emit_unpacked_target(token, unpacked_end);
 836                 }
 837                 unpacked_end = NULL;
 838             }
 839         }
 840         else
 841         {
 842             if (fwrite(token, token_len, 1, stdout) != 1)
 843                 abort();
 844             fputc(' ', stdout);
 845         }
 846         token = end;
 847         eat_space(&token);
 848         if (!target_seen && ':' == *token)
 849         {
 850             target_seen = 1;
 851             fputc(':', stdout);
 852             ++token;
 853             eat_space(&token);
 854         }
 855     }
 856 }
 857
 858 static char * eat_space_at_end(char * end)
 859 {
 860     char * real_end;
 861     assert('\0' == *end);
 862     real_end = end - 1;
 863     while (' ' == *real_end || '\t' == *real_end || '\n' == *real_end
 864                 || ':' == *real_end)
 865     {    /* eat colon and whitespace at end */
 866          --real_end;
 867     }
 868     return real_end;
 869 }
 870
 871 static char* phony_content_buffer;
 872 static char* generate_phony_line(char const * phony_target, char const * extension)
 873 {
 874     char const * src;
 875     char* dest;
 876     char* last_dot = NULL;
 877     //fprintf(stderr, "generate_phony_line called with phony_target %s and extension %s\n", phony_target, extension);
 878     for(dest = phony_content_buffer+work_dir_len+1, src = phony_target; *src != 0; ++src, ++dest)
 879     {
 880         *dest = *src;
 881         if(*dest == '.')
 882         {
 883             last_dot = dest;
 884         }
 885     }
 886     //fprintf(stderr, "generate_phony_line after phony_target copy: %s\n", phony_content_buffer);
 887     for(dest = last_dot+1, src = extension; *src != 0; ++src, ++dest)
 888     {
 889         *dest = *src;
 890     }
 891     //fprintf(stderr, "generate_phony_line after extension add: %s\n", phony_content_buffer);
 892     strcpy(dest, ": $(gb_Helper_PHONY)\n");
 893     //fprintf(stderr, "generate_phony_line after phony add: %s\n", phony_content_buffer);
 894     return phony_content_buffer;
 895 }
 896
 897 static int generate_phony_file(char* fn, char const * content)
 898 {
 899     FILE* depfile;
 900     depfile = fopen(fn, "w");
 901     if(!depfile)
 902     {
 903         fprintf(stderr, "Could not open '%s' for writing: %s\n", fn, strerror(errno));
 904     }
 905     else
 906     {
 907         fputs(content, depfile);
 908         fclose(depfile);
 909     }
 910     return !depfile;
 911 }
 912
 913 static int process(struct hash* dep_hash, char* fn)
 914 {
 915     int rc;
 916     char* buffer;
 917     char* end;
 918     char* cursor;
 919     char* cursor_out;
 920     char* base;
 921     char* created_line = NULL;
 922     char* src_relative;
 923     int continuation = 0;
 924     char last_ns = 0;
 925     off_t size;
 926
 927     buffer = file_load(fn, &size, &rc);
 928     if(!rc)
 929     {
 930         base = cursor_out = cursor = end = buffer;
 931         end += size;
 932
 933         /* first eat unneeded space at the beginning of file
 934          */
 935         while(cursor < end && (*cursor == ' ' || *cursor == '\\'))
 936             ++cursor;
 937
 938         while(cursor < end)
 939         {
 940             if(*cursor == '\\')
 941             {
 942                 continuation = 1;
 943                 *cursor_out++ = *cursor++;
 944             }
 945             else if(*cursor == '/')
 946             {
 947                 if(cursor + 3 < end)
 948                 {
 949                     if(!memcmp(cursor, "/../", 4))
 950                     {
 951                         cancel_relative(base, &cursor, &cursor_out, end);
 952                     }
 953                 }
 954                 *cursor_out++ = *cursor++;
 955             }
 956             else if(*cursor == '\n')
 957             {
 958                 if(!continuation)
 959                 {
 960                     *cursor_out = 0;
 961                     if(base < cursor)
 962                     {
 963                         /* here we have a complete rule */
 964                         if(last_ns == ':')
 965                         {
 966                             /* if the rule ended in ':' that is a no-dep rule
 967                              * these are the one for which we want to filter
 968                              * duplicate out
 969                              */
 970                             int key_len = eat_space_at_end(cursor_out) - base;
 971                             if (!elide_dependency(base,key_len + 1, NULL)
 972                                 && hash_store(dep_hash, base, key_len))
 973                             {
 974                                 /* DO NOT modify base after it has been added
 975                                    as key by hash_store */
 976                                 print_fullpaths(base);
 977                                 putc('\n', stdout);
 978                             }
 979                         }
 980                         else
 981                         {
 982                             /* rule with dep, just write it */
 983                             print_fullpaths(base);
 984                             putc('\n', stdout);
 985                         }
 986                         last_ns = ' '; // cannot hurt to reset it
 987                     }
 988                     cursor += 1;
 989                     base = cursor_out = cursor;
 990                 }
 991                 else
 992                 {
 993                     /* here we have a '\' followed by \n this is a continuation
 994                      * i.e not a complete rule yet
 995                      */
 996                     *cursor_out++ = *cursor++;
 997                     continuation = 0; // cancel current one (empty lines!)
 998                 }
 999             }
1000             else
1001             {
1002                 continuation = 0;
1003                 /* not using isspace() here save 25% of I refs and 75% of D refs based on cachegrind */
1004                 if(*cursor != ' ' && *cursor != '\n' && *cursor != '\t' )
1005                 {
1006                     last_ns = *cursor;
1007                 }
1008                 *cursor_out++ = *cursor++;
1009             }
1010         }
1011         /* just in case the file did not end with a \n, there may be a pending rule */
1012         if(base < cursor_out)
1013         {
1014             if(last_ns == ':')
1015             {
1016                 int key_len = eat_space_at_end(cursor_out) - base;
1017                 if (!elide_dependency(base,key_len + 1, NULL) &&
1018                     hash_store(dep_hash, base, key_len))
1019                 {
1020                     puts(base);
1021                     putc('\n', stdout);
1022                 }
1023             }
1024             else
1025             {
1026                 puts(base);
1027                 putc('\n', stdout);
1028             }
1029         }
1030     }
1031     else
1032     {
1033         if(strncmp(fn, work_dir, work_dir_len) == 0)
1034         {
1035             if(strncmp(fn+work_dir_len, "/Dep/", 5) == 0)
1036             {
1037                 src_relative = fn+work_dir_len+5;
1038                 // cases ordered by frequency
1039                 if(strncmp(src_relative, "CxxObject/", 10) == 0)
1040                 {
1041                     created_line = generate_phony_line(src_relative, "o");
1042                     rc = generate_phony_file(fn, created_line);
1043                 }
1044                 else if(strncmp(src_relative, "GenCxxObject/", 13) == 0)
1045                 {
1046                     created_line = generate_phony_line(src_relative, "o");
1047                     rc = generate_phony_file(fn, created_line);
1048                 }
1049                 else if(strncmp(src_relative, "CObject/", 8) == 0)
1050                 {
1051                     created_line = generate_phony_line(src_relative, "o");
1052                     rc = generate_phony_file(fn, created_line);
1053                 }
1054                 else if(strncmp(src_relative, "GenCObject/", 11) == 0)
1055                 {
1056                     created_line = generate_phony_line(src_relative, "o");
1057                     rc = generate_phony_file(fn, created_line);
1058                 }
1059                 else if(strncmp(src_relative, "SdiObject/", 10) == 0)
1060                 {
1061                     created_line = generate_phony_line(src_relative, "o");
1062                     rc = generate_phony_file(fn, created_line);
1063                 }
1064                 else if(strncmp(src_relative, "AsmObject/", 10) == 0)
1065                 {
1066                     created_line = generate_phony_line(src_relative, "o");
1067                     rc = generate_phony_file(fn, created_line);
1068                 }
1069                 else if(strncmp(src_relative, "ObjCxxObject/", 13) == 0)
1070                 {
1071                     created_line = generate_phony_line(src_relative, "o");
1072                     rc = generate_phony_file(fn, created_line);
1073                 }
1074                 else if(strncmp(src_relative, "ObjCObject/", 11) == 0)
1075                 {
1076                     created_line = generate_phony_line(src_relative, "o");
1077                     rc = generate_phony_file(fn, created_line);
1078                 }
1079                 else if(strncmp(src_relative, "CxxClrObject/", 13) == 0)
1080                 {
1081                     created_line = generate_phony_line(src_relative, "o");
1082                     rc = generate_phony_file(fn, created_line);
1083                 }
1084                 else if(strncmp(src_relative, "GenCxxClrObject/", 16) == 0)
1085                 {
1086                     created_line = generate_phony_line(src_relative, "o");
1087                     rc = generate_phony_file(fn, created_line);
1088                 }
1089                 else
1090                 {
1091                     fprintf(stderr, "no magic for %s(%s) in %s\n", fn, src_relative, work_dir);
1092                 }
1093             }
1094             if(!rc)
1095             {
1096                 puts(created_line);
1097             }
1098         }
1099     }
1100     /* Note: yes we are going to leak 'buffer'
1101      * this is on purpose, to avoid cloning the 'key' out of it and our special
1102      * 'hash' just store the pointer to the key inside of buffer, hence it need
1103      * to remain allocated
1104      */
1105     // coverity[leaked_storage] - this is on purpose
1106     return rc;
1107 }
1108
1109 static void usage(void)
1110 {
1111     fputs("Usage: concat-deps <file that contains dep_files>\n", stderr);
1112 }
1113
1114 #define kDEFAULT_HASH_SIZE 4096
1115 #define PHONY_TARGET_BUFFER 4096
1116
1117 static int get_var(char **var, const char *name)
1118 {
1119     *var = (char *)getenv(name);
1120     if(!*var)
1121     {
1122         fprintf(stderr,"Error: %s is missing in the environment\n", name);
1123         return 1;
1124     }
1125     return 0;
1126 }
1127
1128 int main(int argc, char** argv)
1129 {
1130     int rc = 0;
1131     off_t in_list_size = 0;
1132     char* in_list;
1133     char* in_list_cursor;
1134     char* in_list_base;
1135     struct hash* dep_hash = NULL;
1136     const char *env_str;
1137
1138     if(argc < 2)
1139     {
1140         usage();
1141         return 1;
1142     }
1143     if(get_var(&base_dir, "SRCDIR") || get_var(&work_dir, "WORKDIR"))
1144         return 1;
1145     work_dir_len = strlen(work_dir);
1146     phony_content_buffer = malloc(PHONY_TARGET_BUFFER);
1147     assert(phony_content_buffer); // Don't handle OOM conditions
1148     strcpy(phony_content_buffer, work_dir);
1149     phony_content_buffer[work_dir_len] = '/';
1150
1151     env_str = getenv("SYSTEM_BOOST");
1152     internal_boost = !env_str || strcmp(env_str,"TRUE");
1153
1154     in_list = file_load(argv[1], &in_list_size, &rc);
1155     if(!rc)
1156     {
1157         dep_hash = hash_create( kDEFAULT_HASH_SIZE);
1158         in_list_base = in_list_cursor = in_list;
1159
1160         /* extract filename of dep file from a 'space' separated list */
1161         while(*in_list_cursor)
1162         {
1163             /* the input here may contain Win32 \r\n EOL */
1164             if(*in_list_cursor == ' '
1165                 || *in_list_cursor == '\n' || *in_list_cursor == '\r')
1166             {
1167                 *in_list_cursor = 0;
1168                 if(in_list_base < in_list_cursor)
1169                 {
1170                     rc = process(dep_hash, in_list_base);
1171                     if(rc)
1172                     {
1173                         break;
1174                     }
1175                 }
1176                 in_list_cursor += 1;
1177                 in_list_base = in_list_cursor;
1178             }
1179             else
1180             {
1181                 in_list_cursor += 1;
1182             }
1183         }
1184         if(!rc)
1185         {
1186             /* catch the last entry in case the input did not terminate with a 'space' */
1187             if(in_list_base < in_list_cursor)
1188             {
1189                 rc = process(dep_hash, in_list_base);
1190             }
1191         }
1192 #ifdef HASH_STAT
1193         fprintf(stderr, "stats: u:%d s:%d l:%d t:%d c:%d m:%d $:%d\n",
1194                 dep_hash->used, dep_hash->size, dep_hash->load_limit, dep_hash->stored,
1195                 dep_hash->collisions, dep_hash->memcmp, dep_hash->cost);
1196 #endif
1197     }
1198 #if !ENABLE_RUNTIME_OPTIMIZATIONS
1199     {
1200         size_t i;
1201         hash_destroy(dep_hash);
1202         for (i = 0; i != file_load_buffer_count; ++i)
1203         {
1204             free(file_load_buffers[i]);
1205         }
1206     }
1207 #endif
1208     return rc;
1209 }
1210
1211 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */