solenv/bin/concat-deps.c

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  *    Copyright (C) 2011 Norbert Thiebaud
   4  *    License: GPLv3
   5  */
   6
   7 /* define to activate stats reporting on hash usage*/
   8 /* #define HASH_STAT */
   9
  10 /* ===============================================
  11  * Set-up: defines to identify the system and system related properties
  12  * ===============================================
  13  */
  14
  15 #ifdef __APPLE__
  16 #ifdef __x86_64__
  17 #undef CORE_BIG_ENDIAN
  18 #define CORE_LITTLE_ENDIAN
  19 #else
  20 #define CORE_BIG_ENDIAN
  21 #undef CORE_LITTLE_ENDIAN
  22 #endif
  23
  24 #endif
  25 #ifdef _AIX
  26 #define CORE_BIG_ENDIAN
  27 #undef CORE_LITTLE_ENDIAN
  28 #endif /* Def _AIX */
  29
  30 #ifdef _MSC_VER
  31 #define __windows
  32 #undef CORE_BIG_ENDIAN
  33 #define CORE_LITTLE_ENDIAN
  34 #endif /* Def _MSC_VER */
  35
  36 #if defined(__linux) || defined(__FreeBSD_kernel__)
  37 #include <sys/param.h>
  38 #if __BYTE_ORDER == __LITTLE_ENDIAN
  39 #undef CORE_BIG_ENDIAN
  40 #define CORE_LITTLE_ENDIAN
  41 #else /* !(__BYTE_ORDER == __LITTLE_ENDIAN) */
  42 #if __BYTE_ORDER == __BIG_ENDIAN
  43 #define CORE_BIG_ENDIAN
  44 #undef CORE_LITTLE_ENDIAN
  45 #endif /* __BYTE_ORDER == __BIG_ENDIAN */
  46 #endif /* !(__BYTE_ORDER == __LITTLE_ENDIAN) */
  47 #endif /* Def __linux */
  48
  49 #if defined(__OpenBSD__) || defined(__FreeBSD__) || \
  50     defined(__NetBSD__) || defined(__DragonFly__)
  51 #include <machine/endian.h>
  52 #if _BYTE_ORDER == _LITTLE_ENDIAN
  53 #undef CORE_BIG_ENDIAN
  54 #define CORE_LITTLE_ENDIAN
  55 #else /* !(_BYTE_ORDER == _LITTLE_ENDIAN) */
  56 #if _BYTE_ORDER == _BIG_ENDIAN
  57 #define CORE_BIG_ENDIAN
  58 #undef CORE_LITTLE_ENDIAN
  59 #endif /* _BYTE_ORDER == _BIG_ENDIAN */
  60 #endif /* !(_BYTE_ORDER == _LITTLE_ENDIAN) */
  61 #endif /* Def *BSD */
  62
  63 #ifdef __sun
  64 #ifdef __sparc
  65 #define CORE_BIG_ENDIAN
  66 #undef CORE_LITTLE_ENDIAN
  67 #else  /* Ndef __sparc */
  68 #undef CORE_BIG_ENDIAN
  69 #define CORE_LITTLE_ENDIAN
  70 #endif /* Ndef __sparc */
  71 #endif /* Def __sun */
  72
  73 #include <assert.h>
  74 #include <stdio.h>
  75 #include <stdlib.h>
  76 #include <sys/types.h>
  77 #include <sys/stat.h>
  78 #include <errno.h>
  79 #include <fcntl.h>
  80 #include <string.h>
  81 #include <ctype.h>
  82
  83 #ifdef __windows
  84 #include <io.h>
  85 #else
  86 #include <unistd.h>
  87 #endif
  88
  89 #include <config_options.h>
  90
  91 /* modes */
  92 #ifdef __windows
  93 #define FILE_O_RDONLY     _O_RDONLY
  94 #define FILE_O_BINARY     _O_BINARY
  95 #define PATHNCMP _strnicmp /* MSVC converts paths to lower-case sometimes? */
  96 #define inline __inline
  97 #define ssize_t long
  98 #define S_ISREG(mode) (((mode) & _S_IFMT) == (_S_IFREG)) /* MSVC does not have this macro */
  99 #else /* not windaube */
 100 #define FILE_O_RDONLY     O_RDONLY
 101 #define FILE_O_BINARY     0
 102 #define PATHNCMP strncmp
 103 #endif /* not windaube */
 104
 105 #ifndef TRUE
 106 #define TRUE 1
 107 #endif
 108 #ifndef FALSE
 109 #define FALSE 0
 110 #endif
 111
 112 int internal_boost = 0;
 113 static char* base_dir;
 114 static char* work_dir;
 115 size_t work_dir_len;
 116
 117 #ifdef __GNUC__
 118 #define clz __builtin_clz
 119 #else
 120 static inline int clz(unsigned int value)
 121 {
 122 int result = 32;
 123
 124     while(value)
 125     {
 126         value >>= 1;
 127         result -= 1;
 128     }
 129     return result;
 130 }
 131 #endif
 132
 133 static inline unsigned int get_unaligned_uint(const unsigned char* cursor)
 134 {
 135 unsigned int   result;
 136
 137     memcpy(&result, cursor, sizeof(unsigned int));
 138     return result;
 139 }
 140
 141 /* ===============================================
 142  * memory pool for fast fix-size allocation (non-thread-safe)
 143  * ===============================================
 144  */
 145 struct pool
 146 {
 147     void*    head_free;  /**< head of a linked list of freed element */
 148     char*    fresh;      /**< top of a memory block to dig new element */
 149     char*    tail;       /**< to detect end of extent... when fresh pass tail */
 150     void*    extent;     /**< pointer to the primary extent block */
 151     int      size_elem;  /**< size of an element. */
 152     int      primary;    /**< primary allocation in bytes */
 153     int      secondary;  /**< secondary allocation in bytes */
 154 };
 155 #define POOL_ALIGN_INCREMENT 8 /**< Alignement, must be a power of 2 and of size > to sizeof(void*) */
 156
 157
 158 static void* pool_take_extent(struct pool* pool, int allocate)
 159 {
 160 unsigned int size = 0;
 161 void* extent;
 162 void* data = NULL;
 163
 164     if(pool->extent)
 165     {
 166         /* we already have an extent, so this is a secondary */
 167         if(pool->secondary)
 168         {
 169             size = pool->secondary;
 170         }
 171     }
 172     else
 173     {
 174         assert(pool->primary);
 175         size = pool->primary;
 176     }
 177     if(size)
 178     {
 179         extent = malloc(size);
 180         if(extent)
 181         {
 182             *(void**)extent = pool->extent;
 183             pool->extent = extent;
 184             if(allocate)
 185             {
 186                 data = ((char*)extent) + POOL_ALIGN_INCREMENT;
 187                 pool->fresh = ((char*)data) + pool->size_elem;
 188                 pool->tail = pool->fresh + (size - pool->size_elem);
 189             }
 190             else
 191             {
 192                 pool->fresh = ((char*)extent) + POOL_ALIGN_INCREMENT;
 193                 pool->tail = pool->fresh + (size - pool->size_elem);
 194             }
 195         }
 196     }
 197     return data;
 198 }
 199
 200 /* Create a memory pool for fix size objects
 201  * this is a simplified implementation that
 202  * is _not_ thread safe.
 203  */
 204 static struct pool* pool_create(int size_elem, int primary, int secondary)
 205 {
 206 struct pool* pool;
 207
 208     assert(primary > 0);
 209     assert(secondary >= 0);
 210     assert(size_elem > 0);
 211
 212     pool = (struct pool*)calloc(1, sizeof(struct pool));
 213     if(!pool) return NULL;
 214     /* Adjust the element size so that it be aligned, and so that an element could
 215      * at least contain a void*
 216      */
 217     pool->size_elem = size_elem = (size_elem + POOL_ALIGN_INCREMENT - 1) & ~(POOL_ALIGN_INCREMENT - 1);
 218
 219     pool->primary = (size_elem * primary) + POOL_ALIGN_INCREMENT;
 220     pool->secondary = secondary > 0 ? (size_elem * secondary) + POOL_ALIGN_INCREMENT : 0;
 221     pool_take_extent(pool, FALSE);
 222
 223     return pool;
 224
 225 }
 226
 227 static void pool_destroy(struct pool* pool)
 228 {
 229 void* extent;
 230 void* next;
 231
 232     if(pool != NULL)
 233     {
 234         extent = pool->extent;
 235         while(extent)
 236         {
 237             next = *(void**)extent;
 238             free(extent);
 239             extent = next;
 240         }
 241         free(pool);
 242     }
 243 }
 244
 245 static inline void* pool_alloc(struct pool* pool)
 246 {
 247 void* data;
 248
 249     data = pool->head_free;
 250     if(data == NULL)
 251     {
 252         /* we have no old-freed elem */
 253         if(pool->fresh <= pool->tail)
 254         {
 255             /* pick a slice of the current extent */
 256             data = (void*)pool->fresh;
 257             pool->fresh += pool->size_elem;
 258         }
 259         else
 260         {
 261             /* allocate a new extent */
 262             data = pool_take_extent(pool, TRUE);
 263         }
 264     }
 265     else
 266     {
 267         /* re-used old freed element by chopipng the head of the free list */
 268         pool->head_free = *(void**)data;
 269     }
 270
 271     return data;
 272 }
 273
 274
 275 /* ===============================================
 276  * Hash implementation customized to be just tracking
 277  * a unique list of string (i.e no data associated
 278  * with the key, no need for retrieval, etc..
 279  *
 280  * This is tuned for the particular use-case we have here
 281  * measures in tail_build showed that
 282  * we can get north of 4000 distinct values stored in a hash
 283  * the collision rate is at worse around 2%
 284  * the collision needing an expensive memcmp to resolve
 285  * have a rate typically at 1 per 1000
 286  * for tail_build we register 37229 unique key
 287  * with a total of 377 extra memcmp needed
 288  * which is completely negligible compared to the
 289  * number of memcmp required to eliminate duplicate
 290  * entry (north of 2.5 millions for tail_build)
 291  * ===============================================
 292  */
 293
 294 struct hash_elem
 295 {
 296     struct hash_elem* next;
 297     const char*    key;
 298     int      key_len;
 299 };
 300
 301 struct hash
 302 {
 303     struct hash_elem** array;
 304     struct pool* elems_pool;
 305     int flags;
 306     unsigned int used;
 307     unsigned int size;
 308     unsigned int load_limit;
 309 #ifdef HASH_STAT
 310     int stored;
 311     int collisions;
 312     int cost;
 313     int memcmp;
 314 #endif
 315 };
 316 #define HASH_F_NO_RESIZE (1<<0)
 317
 318 /* The following hash_compute function was adapted from :
 319  * lookup3.c, by Bob Jenkins, May 2006, Public Domain.
 320  *
 321  * The changes from the original are mostly cosmetic
 322  */
 323 #define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k))))
 324
 325
 326 #if defined CORE_BIG_ENDIAN
 327 #define MASK_C1 0xFFFFFF00
 328 #define MASK_C2 0xFFFF0000
 329 #define MASK_C3 0xFF000000
 330 #elif defined CORE_LITTLE_ENDIAN
 331 #define MASK_C1 0xFFFFFF
 332 #define MASK_C2 0xFFFF
 333 #define MASK_C3 0xFF
 334 #else
 335 #error "Missing Endianness definition"
 336 #endif
 337
 338
 339 #define mix(a,b,c) \
 340 { \
 341   a -= c;  a ^= rot(c, 4);  c += b; \
 342   b -= a;  b ^= rot(a, 6);  a += c; \
 343   c -= b;  c ^= rot(b, 8);  b += a; \
 344   a -= c;  a ^= rot(c,16);  c += b; \
 345   b -= a;  b ^= rot(a,19);  a += c; \
 346   c -= b;  c ^= rot(b, 4);  b += a; \
 347 }
 348 #define final(a,b,c) \
 349 { \
 350   c ^= b; c -= rot(b,14); \
 351   a ^= c; a -= rot(c,11); \
 352   b ^= a; b -= rot(a,25); \
 353   c ^= b; c -= rot(b,16); \
 354   a ^= c; a -= rot(c,4);  \
 355   b ^= a; b -= rot(a,14); \
 356   c ^= b; c -= rot(b,24); \
 357 }
 358
 359 static unsigned int hash_compute( struct hash* hash, const char* key, int length)
 360 {
 361     unsigned int a;
 362     unsigned int b;
 363     unsigned int c;                                          /* internal state */
 364     const unsigned char* uk = (const unsigned char*)key;
 365
 366     /* Set up the internal state */
 367     a = b = c = 0xdeadbeef + (length << 2);
 368
 369     /* we use this to 'hash' full path with mostly a common root
 370      * let's now waste too much cycles hashing mostly constant stuff
 371      */
 372     if(length > 36)
 373     {
 374         uk += length - 36;
 375         length = 36;
 376     }
 377     /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */
 378     while (length > 12)
 379     {
 380         a += get_unaligned_uint(uk);
 381         b += get_unaligned_uint(uk+4);
 382         c += get_unaligned_uint(uk+8);
 383         mix(a,b,c);
 384         length -= 12;
 385         uk += 12;
 386     }
 387
 388     /*----------------------------- handle the last (probably partial) block */
 389     /* Note: we possibly over-read, which would trigger complaint from VALGRIND
 390      * but we mask the undefined stuff if any, so we are still good, thanks
 391      * to alignment of memory allocation and tail-memory management overhead
 392      * we always can read 3 bytes past the official end without triggering
 393      * a segfault -- if you find a platform/compiler couple for which that postulat
 394      * is false, then you just need to over-allocate by 2 more bytes in file_load()
 395      * file_load already over-allocate by 1 to sitck a \0 at the end of the buffer.
 396      */
 397     switch(length)
 398     {
 399     case 12: c+=get_unaligned_uint(uk+8); b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 400     case 11: c+=get_unaligned_uint(uk+8) & MASK_C1; b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 401     case 10: c+=get_unaligned_uint(uk+8) & MASK_C2; b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 402     case 9 : c+=get_unaligned_uint(uk+8) & MASK_C3; b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 403     case 8 : b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 404     case 7 : b+=get_unaligned_uint(uk+4) & MASK_C1; a+=get_unaligned_uint(uk); break;
 405     case 6 : b+=get_unaligned_uint(uk+4) & MASK_C2; a+=get_unaligned_uint(uk); break;
 406     case 5 : b+=get_unaligned_uint(uk+4) & MASK_C3; a+=get_unaligned_uint(uk); break;
 407     case 4 : a+=get_unaligned_uint(uk); break;
 408     case 3 : a+=get_unaligned_uint(uk) & MASK_C1; break;
 409     case 2 : a+=get_unaligned_uint(uk) & MASK_C2; break;
 410     case 1 : a+=get_unaligned_uint(uk) & MASK_C3; break;
 411     case 0 : return c & hash->size;              /* zero length strings require no mixing */
 412     }
 413
 414     final(a,b,c);
 415     return c & hash->size;
 416 }
 417
 418 static void hash_destroy(struct hash* hash)
 419 {
 420     if(hash)
 421     {
 422         if(hash->array)
 423         {
 424             free(hash->array);
 425         }
 426         if(hash->elems_pool)
 427         {
 428             pool_destroy(hash->elems_pool);
 429         }
 430         free(hash);
 431     }
 432 }
 433
 434 static struct hash* hash_create(unsigned int size)
 435 {
 436 struct hash* hash;
 437
 438     assert(size > 0);
 439     hash = calloc(1, sizeof(struct hash));
 440     if(hash)
 441     {
 442         size += (size >> 2) + 1; /* ~ 75% load factor */
 443         if(size >= 15)
 444         {
 445             hash->size = (((unsigned int)0xFFFFFFFF) >> clz((unsigned int)size));
 446         }
 447         else
 448         {
 449             hash->size = size = 15;
 450         }
 451         hash->load_limit = hash->size - (hash->size >> 2);
 452         hash->used = 0;
 453         hash->array = (struct hash_elem**)calloc(hash->size + 1, sizeof(struct hash_elem*));
 454         if(hash->array == NULL)
 455         {
 456             hash_destroy(hash);
 457             hash = NULL;
 458         }
 459     }
 460     if(hash)
 461     {
 462         hash->elems_pool = pool_create(sizeof(struct hash_elem),
 463                                        size, size << 1);
 464         if(!hash->elems_pool)
 465         {
 466             hash_destroy(hash);
 467             hash = NULL;
 468         }
 469     }
 470     return hash;
 471 }
 472
 473 static void hash_resize(struct hash* hash)
 474 {
 475 unsigned int old_size = hash->size;
 476 unsigned int hashed;
 477 struct hash_elem* hash_elem;
 478 struct hash_elem* next;
 479 struct hash_elem** array;
 480 unsigned int i;
 481
 482     hash->size = (old_size << 1) + 1;
 483     /* we really should avoid to get there... so print a message to alert of the condition */
 484     fprintf(stderr, "resize hash %d -> %d\n", old_size, hash->size);
 485     if(hash->size == old_size)
 486     {
 487         hash->flags |= HASH_F_NO_RESIZE;
 488         return;
 489     }
 490     array = calloc(hash->size + 1, sizeof(struct hash_elem*));
 491     if(array)
 492     {
 493         hash->load_limit = hash->size - (hash->size >> 2);
 494         for(i=0; i <= old_size; i++)
 495         {
 496             hash_elem = (struct hash_elem*)hash->array[i];
 497             while(hash_elem)
 498             {
 499                 next = hash_elem->next;
 500
 501                 hashed = hash_compute(hash, hash_elem->key, hash_elem->key_len);
 502                 hash_elem->next = array[hashed];
 503                 array[hashed] = hash_elem;
 504                 hash_elem = next;
 505             }
 506         }
 507         free(hash->array);
 508         hash->array = (struct hash_elem**)array;
 509     }
 510     else
 511     {
 512         hash->size = old_size;
 513         hash->flags |= HASH_F_NO_RESIZE;
 514     }
 515 }
 516
 517 #ifdef HASH_STAT
 518 static inline int compare_key(struct hash* hash, const char* a, const char* b, int len, int* cost)
 519 {
 520     *cost += 1;
 521     hash->memcmp += 1;
 522     return memcmp(a,b, len);
 523 }
 524 #else
 525 #define compare_key(h,a,b,l,c) memcmp(a,b,l)
 526 #endif
 527
 528 /* a customized hash_store function that just store the key and return
 529  * TRUE if the key was effectively stored, or FALSE if the key was already there
 530  */
 531 static int hash_store(struct hash* hash, const char* key, int key_len)
 532 {
 533 unsigned int hashed;
 534 struct hash_elem* hash_elem;
 535 int cost = 0;
 536
 537     (void) cost;
 538     hashed = hash_compute(hash, key, key_len);
 539 #ifdef HASH_STAT
 540     hash->stored += 1;
 541 #endif
 542     hash_elem = (struct hash_elem*)hash->array[hashed];
 543     while(hash_elem && (hash_elem->key_len != key_len || compare_key(hash, hash_elem->key, key, key_len, &cost)))
 544     {
 545         hash_elem = hash_elem->next;
 546     }
 547
 548     if(!hash_elem)
 549     {
 550         hash_elem = pool_alloc(hash->elems_pool);
 551         if(hash_elem)
 552         {
 553             hash_elem->key = key;
 554             hash_elem->key_len = key_len;
 555             hash_elem->next = hash->array[hashed];
 556
 557 #ifdef HASH_STAT
 558             if(hash_elem->next)
 559             {
 560                 hash->collisions += 1;
 561                 hash->cost += cost;
 562             }
 563 #endif
 564             hash->array[hashed] = hash_elem;
 565             hash->used += 1;
 566             if(hash->used > hash->load_limit)
 567             {
 568                 hash_resize(hash);
 569             }
 570         }
 571         return TRUE;
 572     }
 573     return FALSE;
 574 }
 575
 576 static int file_stat(const char* name, struct stat* buffer_stat, int* rc)
 577 {
 578 int rc_local = 0;
 579
 580     rc_local = stat(name, buffer_stat);
 581     if (rc_local  < 0)
 582     {
 583         *rc = errno;
 584     }
 585     return rc_local;
 586 }
 587
 588 static off_t file_get_size(const char* name, int* rc)
 589 {
 590 struct stat buffer_stat;
 591 off_t       size = -1;
 592
 593     if (!file_stat(name, &buffer_stat, rc))
 594     {
 595         if(S_ISREG(buffer_stat.st_mode))
 596         {
 597             size = buffer_stat.st_size;
 598         }
 599         else
 600         {
 601             *rc = EINVAL;
 602         }
 603     }
 604     return size;
 605 }
 606
 607 #if !ENABLE_RUNTIME_OPTIMIZATIONS
 608 static void * file_load_buffers[100000];
 609 static size_t file_load_buffer_count = 0;
 610 #endif
 611
 612 static char* file_load(const char* name, off_t* size, int* return_rc)
 613 {
 614 off_t local_size = 0;
 615 int rc = 0;
 616 char* buffer = NULL;
 617 int fd;
 618
 619     assert(name != NULL);
 620
 621     if(!size)
 622     {
 623         size = &local_size;
 624     }
 625     *size = file_get_size(name, &rc);
 626     if (!rc && *size >= 0)
 627     {
 628         fd = open(name, FILE_O_RDONLY | FILE_O_BINARY);
 629         if (!(fd == -1))
 630         {
 631             buffer = malloc((size_t)(*size + 1));
 632 #if !ENABLE_RUNTIME_OPTIMIZATIONS
 633             if (buffer != NULL)
 634             {
 635                 if (file_load_buffer_count == 100000)
 636                 {
 637                     free(buffer);
 638                     buffer = NULL;
 639                 }
 640                 else
 641                 {
 642                     file_load_buffers[file_load_buffer_count++] = buffer;
 643                 }
 644             }
 645 #endif
 646             if (buffer == NULL)
 647             {
 648                 rc = ENOMEM;
 649             }
 650             else
 651             {
 652             ssize_t i;
 653
 654               REDO:
 655                 i = read(fd, buffer, (size_t)(*size));
 656                 if(i == -1)
 657                 {
 658                     if(errno == EINTR)
 659                     {
 660                         goto REDO;
 661                     }
 662                     else
 663                     {
 664                         rc = errno;
 665                     }
 666                 }
 667                 else
 668                 {
 669                     if (i != *size)
 670                     {
 671                         rc = EIO;
 672                     }
 673                 }
 674                 buffer[*size] = 0;
 675             }
 676             close(fd);
 677         }
 678     }
 679
 680     if(rc && buffer)
 681     {
 682         free(buffer);
 683         buffer = NULL;
 684     }
 685     if(return_rc)
 686     {
 687         *return_rc = rc;
 688     }
 689     return buffer;
 690 }
 691
 692 static void cancel_relative(char const * base, char** ref_cursor, char** ref_cursor_out, char const * end)
 693 {
 694     char* cursor = *ref_cursor;
 695     char* cursor_out = *ref_cursor_out;
 696
 697     do
 698     {
 699         cursor += 3;
 700         while(cursor_out > base && cursor_out[-1] == '/')
 701             cursor_out--;
 702         while(cursor_out > base && *--cursor_out != '/');
 703     }
 704     while(cursor + 3 < end && !memcmp(cursor, "/../", 4));
 705     *ref_cursor = cursor;
 706     *ref_cursor_out = cursor_out;
 707 }
 708
 709 static inline void eat_space(char ** token)
 710 {
 711     while ((' ' == **token) || ('\t' == **token)) {
 712         ++(*token);
 713     }
 714 }
 715
 716 /*
 717  * Prune LibreOffice specific duplicate dependencies to improve
 718  * gnumake startup time, and shrink the disk-space footprint.
 719  */
 720 static inline int
 721 elide_dependency(const char* key, int key_len, const char **unpacked_end)
 722 {
 723 #if 0
 724     {
 725         int i;
 726         fprintf (stderr, "elide?%d!: '", internal_boost);
 727         for (i = 0; i < key_len; i++) {
 728             fprintf (stderr, "%c", key[i]);
 729         }
 730         fprintf (stderr, "'\n");
 731     }
 732 #endif
 733
 734     /* boost brings a plague of header files */
 735     int i;
 736     int unpacked = 0;
 737     /* walk down path elements */
 738     for (i = 0; i < key_len - 1; i++)
 739     {
 740         if (key[i] == '/')
 741         {
 742             if (0 == unpacked)
 743             {
 744                 if (!PATHNCMP(key + i + 1, "workdir/", 8))
 745                 {
 746                     unpacked = 1;
 747                     continue;
 748                 }
 749             }
 750             else
 751             {
 752                 if (!PATHNCMP(key + i + 1, "UnpackedTarball/", 16))
 753                 {
 754                     if (unpacked_end)
 755                         *unpacked_end = strchr(key + i + 17, '/');
 756                     return 1;
 757                 }
 758             }
 759         }
 760     }
 761
 762     return 0;
 763 }
 764
 765 /*
 766  * We collapse tens of internal boost headers to the unpacked target, such
 767  * that you can re-compile / install boost and all is well.
 768  */
 769 static void emit_single_boost_header(void)
 770 {
 771 #define BOOST_TARGET "/UnpackedTarball/boost.done"
 772     fprintf(stdout, "%s" BOOST_TARGET " ", work_dir);
 773 }
 774
 775 static void emit_unpacked_target(const char* token, const char* end)
 776 {
 777     fwrite(token, 1, end-token, stdout);
 778     fputs(".done ", stdout);
 779 }
 780
 781 /* prefix paths to absolute */
 782 static inline void print_fullpaths(char* line)
 783 {
 784     char* token;
 785     char* end;
 786     int boost_count = 0;
 787     int token_len;
 788     const char * unpacked_end = NULL; /* end of UnpackedTarget match (if any) */
 789     /* for UnpackedTarget the target is GenC{,xx}Object, don't mangle! */
 790     int target_seen = 0;
 791
 792     token = line;
 793     eat_space(&token);
 794     while (*token)
 795     {
 796         end = token;
 797         /* hard to believe that in this day and age drive letters still exist */
 798         if (*end && (':' == *(end+1)) &&
 799             (('\\' == *(end+2)) || ('/' == *(end+2))) && isalpha(*end))
 800         {
 801             end = end + 3; /* only one cross, err drive letter per filename */
 802         }
 803         while (*end && (' ' != *end) && ('\t' != *end) && (':' != *end)) {
 804             ++end;
 805         }
 806         token_len = end - token;
 807         if (target_seen &&
 808             elide_dependency(token, token_len, &unpacked_end))
 809         {
 810             if (unpacked_end)
 811             {
 812                 if (internal_boost && !PATHNCMP(unpacked_end - 5, "boost", 5))
 813                 {
 814                     ++boost_count;
 815                     if (boost_count == 1)
 816                         emit_single_boost_header();
 817                     else
 818                     {
 819                         /* don't output, and swallow trailing \\\n if any */
 820                         token = end;
 821                         eat_space(&token);
 822                         if (token[0] == '\\' && token[1] == '\n')
 823                             end = token + 2;
 824                     }
 825                 }
 826                 else
 827                 {
 828                     emit_unpacked_target(token, unpacked_end);
 829                 }
 830                 unpacked_end = NULL;
 831             }
 832         }
 833         else
 834         {
 835             if (fwrite(token, token_len, 1, stdout) != 1)
 836                 abort();
 837             fputc(' ', stdout);
 838         }
 839         token = end;
 840         eat_space(&token);
 841         if (!target_seen)
 842         {
 843             if (':' == *token)
 844             {
 845                 target_seen = 1;
 846                 fputc(':', stdout);
 847                 ++token;
 848                 eat_space(&token);
 849             }
 850         }
 851     }
 852 }
 853
 854 static inline char * eat_space_at_end(char * end)
 855 {
 856     char * real_end;
 857     assert('\0' == *end);
 858     real_end = end - 1;
 859     while (' ' == *real_end || '\t' == *real_end || '\n' == *real_end
 860                 || ':' == *real_end)
 861     {    /* eat colon and whitespace at end */
 862          --real_end;
 863     }
 864     return real_end;
 865 }
 866
 867 static char* phony_content_buffer;
 868 static inline char* generate_phony_line(char const * phony_target, char const * extension)
 869 {
 870 char const * src;
 871 char* dest;
 872 char* last_dot = NULL;
 873     //fprintf(stderr, "generate_phony_line called with phony_target %s and extension %s\n", phony_target, extension);
 874     for(dest = phony_content_buffer+work_dir_len, src = phony_target; *src != 0; ++src, ++dest)
 875     {
 876         *dest = *src;
 877         if(*dest == '.')
 878         {
 879             last_dot = dest;
 880         }
 881     }
 882     //fprintf(stderr, "generate_phony_line after phony_target copy: %s\n", phony_content_buffer);
 883     for(dest = last_dot+1, src = extension; *src != 0; ++src, ++dest)
 884     {
 885         *dest = *src;
 886     }
 887     //fprintf(stderr, "generate_phony_line after extension add: %s\n", phony_content_buffer);
 888     strcpy(dest, ": $(gb_Helper_PHONY)\n");
 889     //fprintf(stderr, "generate_phony_line after phony add: %s\n", phony_content_buffer);
 890     return phony_content_buffer;
 891 }
 892
 893 static inline int generate_phony_file(char* fn, char const * content)
 894 {
 895 FILE* depfile;
 896     depfile = fopen(fn, "w");
 897     if(!depfile)
 898     {
 899         fprintf(stderr, "Could not open '%s' for writing: %s\n", fn, strerror(errno));
 900     }
 901     else
 902     {
 903         fputs(content, depfile);
 904         fclose(depfile);
 905     }
 906     return !depfile;
 907 }
 908
 909 static int process(struct hash* dep_hash, char* fn)
 910 {
 911 int rc;
 912 char* buffer;
 913 char* end;
 914 char* cursor;
 915 char* cursor_out;
 916 char* base;
 917 char* created_line = NULL;
 918 char* src_relative;
 919 int continuation = 0;
 920 char last_ns = 0;
 921 off_t size;
 922
 923     buffer = file_load(fn, &size, &rc);
 924     if(!rc)
 925     {
 926         base = cursor_out = cursor = end = buffer;
 927         end += size;
 928
 929         /* first eat unneeded space at the beginning of file
 930          */
 931         while(cursor < end && (*cursor == ' ' || *cursor == '\\'))
 932             ++cursor;
 933
 934         while(cursor < end)
 935         {
 936             if(*cursor == '\\')
 937             {
 938                 continuation = 1;
 939                 *cursor_out++ = *cursor++;
 940             }
 941             else if(*cursor == '/')
 942             {
 943                 if(cursor + 3 < end)
 944                 {
 945                     if(!memcmp(cursor, "/../", 4))
 946                     {
 947                         cancel_relative(base, &cursor, &cursor_out, end);
 948                     }
 949                 }
 950                 *cursor_out++ = *cursor++;
 951             }
 952             else if(*cursor == '\n')
 953             {
 954                 if(!continuation)
 955                 {
 956                     *cursor_out = 0;
 957                     if(base < cursor)
 958                     {
 959                         /* here we have a complete rule */
 960                         if(last_ns == ':')
 961                         {
 962                             /* if the rule ended in ':' that is a no-dep rule
 963                              * these are the one for which we want to filter
 964                              * duplicate out
 965                              */
 966                             int key_len = eat_space_at_end(cursor_out) - base;
 967                             if (!elide_dependency(base,key_len + 1, NULL)
 968                                 && hash_store(dep_hash, base, key_len))
 969                             {
 970                                 /* DO NOT modify base after it has been added
 971                                    as key by hash_store */
 972                                 print_fullpaths(base);
 973                                 putc('\n', stdout);
 974                             }
 975                         }
 976                         else
 977                         {
 978                             /* rule with dep, just write it */
 979                             print_fullpaths(base);
 980                             putc('\n', stdout);
 981                         }
 982                         last_ns = ' '; // cannot hurt to reset it
 983                     }
 984                     cursor += 1;
 985                     base = cursor_out = cursor;
 986                 }
 987                 else
 988                 {
 989                     /* here we have a '\' followed by \n this is a continuation
 990                      * i.e not a complete rule yet
 991                      */
 992                     *cursor_out++ = *cursor++;
 993                     continuation = 0; // cancel current one (empty lines!)
 994                 }
 995             }
 996             else
 997             {
 998                 continuation = 0;
 999                 /* not using isspace() here save 25% of I refs and 75% of D refs based on cachegrind */
1000                 if(*cursor != ' ' && *cursor != '\n' && *cursor != '\t' )
1001                 {
1002                     last_ns = *cursor;
1003                 }
1004                 *cursor_out++ = *cursor++;
1005             }
1006         }
1007         /* just in case the file did not end with a \n, there may be a pending rule */
1008         if(base < cursor_out)
1009         {
1010             if(last_ns == ':')
1011             {
1012                 int key_len = eat_space_at_end(cursor_out) - base;
1013                 if (!elide_dependency(base,key_len + 1, NULL) &&
1014                     hash_store(dep_hash, base, key_len))
1015                 {
1016                     puts(base);
1017                     putc('\n', stdout);
1018                 }
1019             }
1020             else
1021             {
1022                 puts(base);
1023                 putc('\n', stdout);
1024             }
1025         }
1026     }
1027     else
1028     {
1029         if(strncmp(fn, work_dir, work_dir_len) == 0)
1030         {
1031             if(strncmp(fn+work_dir_len, "/Dep/", 5) == 0)
1032             {
1033                 src_relative = fn+work_dir_len+5;
1034                 // cases ordered by frequency
1035                 if(strncmp(src_relative, "CxxObject/", 10) == 0)
1036                 {
1037                     created_line = generate_phony_line(src_relative+10, "o");
1038                     rc = generate_phony_file(fn, created_line);
1039                 }
1040                 else if(strncmp(fn+work_dir_len+5, "SrsPartTarget/", 14) == 0)
1041                 {
1042                     created_line = generate_phony_line(src_relative+14, "");
1043                     rc = generate_phony_file(fn, created_line);
1044                 }
1045                 else if(strncmp(src_relative, "GenCxxObject/", 13) == 0)
1046                 {
1047                     created_line = generate_phony_line(src_relative+13, "o");
1048                     rc = generate_phony_file(fn, created_line);
1049                 }
1050                 else if(strncmp(src_relative, "CObject/", 8) == 0)
1051                 {
1052                     created_line = generate_phony_line(src_relative+8, "o");
1053                     rc = generate_phony_file(fn, created_line);
1054                 }
1055                 else if(strncmp(src_relative, "GenCObject/", 11) == 0)
1056                 {
1057                     created_line = generate_phony_line(src_relative+11, "o");
1058                     rc = generate_phony_file(fn, created_line);
1059                 }
1060                 else if(strncmp(src_relative, "SdiObject/", 10) == 0)
1061                 {
1062                     created_line = generate_phony_line(src_relative+10, "o");
1063                     rc = generate_phony_file(fn, created_line);
1064                 }
1065                 else if(strncmp(src_relative, "AsmObject/", 10) == 0)
1066                 {
1067                     created_line = generate_phony_line(src_relative+10, "o");
1068                     rc = generate_phony_file(fn, created_line);
1069                 }
1070                 else if(strncmp(src_relative, "ObjCxxObject/", 13) == 0)
1071                 {
1072                     created_line = generate_phony_line(src_relative+13, "o");
1073                     rc = generate_phony_file(fn, created_line);
1074                 }
1075                 else if(strncmp(src_relative, "ObjCObject/", 11) == 0)
1076                 {
1077                     created_line = generate_phony_line(src_relative+11, "o");
1078                     rc = generate_phony_file(fn, created_line);
1079                 }
1080                 else
1081                 {
1082                     fprintf(stderr, "no magic for %s(%s) in %s\n", fn, src_relative, work_dir);
1083                 }
1084             }
1085             if(!rc)
1086             {
1087                 puts(created_line);
1088             }
1089         }
1090     }
1091     /* Note: yes we are going to leak 'buffer'
1092      * this is on purpose, to avoid cloning the 'key' out of it and our special
1093      * 'hash' just store the pointer to the key inside of buffer, hence it need
1094      * to remain allocated
1095      */
1096     // coverity[leaked_storage] - this is on purpose
1097     return rc;
1098 }
1099
1100 static void usage(void)
1101 {
1102     fputs("Usage: concat-deps <file that contains dep_files>\n", stderr);
1103 }
1104
1105 #define kDEFAULT_HASH_SIZE 4096
1106 #define PHONY_TARGET_BUFFER 4096
1107
1108 static int get_var(char **var, const char *name)
1109 {
1110     *var = (char *)getenv(name);
1111     if(!*var)
1112     {
1113         fprintf(stderr,"Error: %s is missing in the environment\n", name);
1114         return 1;
1115     }
1116     return 0;
1117 }
1118
1119 int main(int argc, char** argv)
1120 {
1121 int rc = 0;
1122 off_t in_list_size = 0;
1123 char* in_list;
1124 char* in_list_cursor;
1125 char* in_list_base;
1126 struct hash* dep_hash = NULL;
1127 const char *env_str;
1128
1129     if(argc < 2)
1130     {
1131         usage();
1132         return 1;
1133     }
1134     if(get_var(&base_dir, "SRCDIR") || get_var(&work_dir, "WORKDIR"))
1135         return 1;
1136     work_dir_len = strlen(work_dir);
1137     phony_content_buffer = malloc(PHONY_TARGET_BUFFER);
1138     strcpy(phony_content_buffer, work_dir);
1139     phony_content_buffer[work_dir_len] = '/';
1140
1141     env_str = getenv("SYSTEM_BOOST");
1142     internal_boost = !env_str || strcmp(env_str,"TRUE");
1143
1144     in_list = file_load(argv[1], &in_list_size, &rc);
1145     if(!rc)
1146     {
1147         dep_hash = hash_create( kDEFAULT_HASH_SIZE);
1148         in_list_base = in_list_cursor = in_list;
1149
1150         /* extract filename of dep file from a 'space' separated list */
1151         while(*in_list_cursor)
1152         {
1153             /* the input here may contain Win32 \r\n EOL */
1154             if(*in_list_cursor == ' '
1155                 || *in_list_cursor == '\n' || *in_list_cursor == '\r')
1156             {
1157                 *in_list_cursor = 0;
1158                 if(in_list_base < in_list_cursor)
1159                 {
1160                     rc = process(dep_hash, in_list_base);
1161                     if(rc)
1162                     {
1163                         break;
1164                     }
1165                 }
1166                 in_list_cursor += 1;
1167                 in_list_base = in_list_cursor;
1168             }
1169             else
1170             {
1171                 in_list_cursor += 1;
1172             }
1173         }
1174         if(!rc)
1175         {
1176             /* catch the last entry in case the input did not terminate with a 'space' */
1177             if(in_list_base < in_list_cursor)
1178             {
1179                 rc = process(dep_hash, in_list_base);
1180             }
1181         }
1182 #ifdef HASH_STAT
1183         fprintf(stderr, "stats: u:%d s:%d l:%d t:%d c:%d m:%d $:%d\n",
1184                 dep_hash->used, dep_hash->size, dep_hash->load_limit, dep_hash->stored,
1185                 dep_hash->collisions, dep_hash->memcmp, dep_hash->cost);
1186 #endif
1187     }
1188 #if !ENABLE_RUNTIME_OPTIMIZATIONS
1189     {
1190         size_t i;
1191         hash_destroy(dep_hash);
1192         for (i = 0; i != file_load_buffer_count; ++i)
1193         {
1194             free(file_load_buffers[i]);
1195         }
1196     }
1197 #endif
1198     return rc;
1199 }
1200
1201 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */