solenv/bin/concat-deps.c

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  *    Copyright (C) 2011 Norbert Thiebaud
   4  *    License: GPLv3
   5  */
   6
   7 /* define to activate stats reporting on hash usage*/
   8 /* #define HASH_STAT */
   9
  10 /* ===============================================
  11  * Set-up: defines to identify the system and system related properties
  12  * ===============================================
  13  */
  14
  15 #ifdef __APPLE__
  16 #ifdef __x86_64__
  17 #undef CORE_BIG_ENDIAN
  18 #define CORE_LITTLE_ENDIAN
  19 #else
  20 #define CORE_BIG_ENDIAN
  21 #undef CORE_LITTLE_ENDIAN
  22 #endif
  23
  24 #endif
  25
  26 #ifdef _MSC_VER
  27 #undef CORE_BIG_ENDIAN
  28 #define CORE_LITTLE_ENDIAN
  29 #endif /* Def _MSC_VER */
  30
  31 #if defined(__linux) || defined(__FreeBSD__)
  32 #include <sys/param.h>
  33 #if __BYTE_ORDER == __LITTLE_ENDIAN
  34 #undef CORE_BIG_ENDIAN
  35 #define CORE_LITTLE_ENDIAN
  36 #else /* !(__BYTE_ORDER == __LITTLE_ENDIAN) */
  37 #if __BYTE_ORDER == __BIG_ENDIAN
  38 #define CORE_BIG_ENDIAN
  39 #undef CORE_LITTLE_ENDIAN
  40 #endif /* __BYTE_ORDER == __BIG_ENDIAN */
  41 #endif /* !(__BYTE_ORDER == __LITTLE_ENDIAN) */
  42 #endif /* Def __linux */
  43
  44 #if defined(__OpenBSD__) || defined(__FreeBSD__) || \
  45     defined(__NetBSD__) || defined(__DragonFly__)
  46 #include <machine/endian.h>
  47 #if _BYTE_ORDER == _LITTLE_ENDIAN
  48 #undef CORE_BIG_ENDIAN
  49 #define CORE_LITTLE_ENDIAN
  50 #else /* !(_BYTE_ORDER == _LITTLE_ENDIAN) */
  51 #if _BYTE_ORDER == _BIG_ENDIAN
  52 #define CORE_BIG_ENDIAN
  53 #undef CORE_LITTLE_ENDIAN
  54 #endif /* _BYTE_ORDER == _BIG_ENDIAN */
  55 #endif /* !(_BYTE_ORDER == _LITTLE_ENDIAN) */
  56 #endif /* Def *BSD */
  57
  58 #if defined(__HAIKU__)
  59 #include <endian.h>
  60 #if __BYTE_ORDER == __LITTLE_ENDIAN
  61 #undef CORE_BIG_ENDIAN
  62 #define CORE_LITTLE_ENDIAN
  63 #else /* !(__BYTE_ORDER == __LITTLE_ENDIAN) */
  64 #if __BYTE_ORDER == __BIG_ENDIAN
  65 #define CORE_BIG_ENDIAN
  66 #undef CORE_LITTLE_ENDIAN
  67 #endif /* __BYTE_ORDER == __BIG_ENDIAN */
  68 #endif /* !(__BYTE_ORDER == __LITTLE_ENDIAN) */
  69 #endif /* Def __HAIKU__ */
  70
  71 #ifdef __sun
  72 #ifdef __sparc
  73 #define CORE_BIG_ENDIAN
  74 #undef CORE_LITTLE_ENDIAN
  75 #else  /* Ndef __sparc */
  76 #undef CORE_BIG_ENDIAN
  77 #define CORE_LITTLE_ENDIAN
  78 #endif /* Ndef __sparc */
  79 #endif /* Def __sun */
  80
  81 #include <assert.h>
  82 #include <stdio.h>
  83 #include <stdlib.h>
  84 #include <sys/types.h>
  85 #include <sys/stat.h>
  86 #include <errno.h>
  87 #include <fcntl.h>
  88 #include <string.h>
  89 #include <ctype.h>
  90
  91 #ifdef _MSC_VER
  92 #include <io.h>
  93 #else
  94 #include <unistd.h>
  95 #endif
  96
  97 #include <config_options.h>
  98
  99 /* modes */
 100 #ifdef _MSC_VER
 101 #define FILE_O_RDONLY     _O_RDONLY
 102 #define FILE_O_BINARY     _O_BINARY
 103 #define PATHNCMP _strnicmp /* MSVC converts paths to lower-case sometimes? */
 104 #define ssize_t long
 105 #define S_ISREG(mode) (((mode) & _S_IFMT) == (_S_IFREG)) /* MSVC does not have this macro */
 106 #else /* not windaube */
 107 #define FILE_O_RDONLY     O_RDONLY
 108 #define FILE_O_BINARY     0
 109 #define PATHNCMP strncmp
 110 #endif /* not windaube */
 111
 112 #ifndef TRUE
 113 #define TRUE 1
 114 #endif
 115 #ifndef FALSE
 116 #define FALSE 0
 117 #endif
 118
 119 static int internal_boost = 0;
 120 static char* base_dir;
 121 static char* work_dir;
 122 static size_t work_dir_len;
 123
 124 #ifdef __GNUC__
 125 #define clz __builtin_clz
 126 #else
 127 static int clz(unsigned int value)
 128 {
 129     int result = 32;
 130
 131     while(value)
 132     {
 133         value >>= 1;
 134         result -= 1;
 135     }
 136     return result;
 137 }
 138 #endif
 139
 140 static unsigned int get_unaligned_uint(const unsigned char* cursor)
 141 {
 142     unsigned int   result;
 143
 144     memcpy(&result, cursor, sizeof(unsigned int));
 145     return result;
 146 }
 147
 148 /* ===============================================
 149  * memory pool for fast fix-size allocation (non-thread-safe)
 150  * ===============================================
 151  */
 152 struct pool
 153 {
 154     void*    head_free;  /**< head of a linked list of freed element */
 155     char*    fresh;      /**< top of a memory block to dig new element */
 156     char*    tail;       /**< to detect end of extent... when fresh pass tail */
 157     void*    extent;     /**< pointer to the primary extent block */
 158     int      size_elem;  /**< size of an element. */
 159     int      primary;    /**< primary allocation in bytes */
 160     int      secondary;  /**< secondary allocation in bytes */
 161 };
 162 #define POOL_ALIGN_INCREMENT 8 /**< alignment, must be a power of 2 and of size > to sizeof(void*) */
 163
 164
 165 static void* pool_take_extent(struct pool* pool, int allocate)
 166 {
 167     unsigned int size = 0;
 168     void* extent;
 169     void* data = NULL;
 170
 171     if(pool->extent)
 172     {
 173         /* we already have an extent, so this is a secondary */
 174         if(pool->secondary)
 175         {
 176             size = pool->secondary;
 177         }
 178     }
 179     else
 180     {
 181         assert(pool->primary);
 182         size = pool->primary;
 183     }
 184     if(size)
 185     {
 186         extent = malloc(size);
 187         if(extent)
 188         {
 189             *(void**)extent = pool->extent;
 190             pool->extent = extent;
 191             if(allocate)
 192             {
 193                 data = ((char*)extent) + POOL_ALIGN_INCREMENT;
 194                 pool->fresh = ((char*)data) + pool->size_elem;
 195                 pool->tail = pool->fresh + (size - pool->size_elem);
 196             }
 197             else
 198             {
 199                 pool->fresh = ((char*)extent) + POOL_ALIGN_INCREMENT;
 200                 pool->tail = pool->fresh + (size - pool->size_elem);
 201             }
 202         }
 203     }
 204     return data;
 205 }
 206
 207 /* Create a memory pool for fix size objects
 208  * this is a simplified implementation that
 209  * is _not_ thread safe.
 210  */
 211 static struct pool* pool_create(int size_elem, int primary, int secondary)
 212 {
 213     struct pool* pool;
 214
 215     assert(primary > 0);
 216     assert(secondary >= 0);
 217     assert(size_elem > 0);
 218
 219     pool = (struct pool*)calloc(1, sizeof(struct pool));
 220     if(!pool) return NULL;
 221     /* Adjust the element size so that it be aligned, and so that an element could
 222      * at least contain a void*
 223      */
 224     pool->size_elem = size_elem = (size_elem + POOL_ALIGN_INCREMENT - 1) & ~(POOL_ALIGN_INCREMENT - 1);
 225
 226     pool->primary = (size_elem * primary) + POOL_ALIGN_INCREMENT;
 227     pool->secondary = secondary > 0 ? (size_elem * secondary) + POOL_ALIGN_INCREMENT : 0;
 228     pool_take_extent(pool, FALSE);
 229
 230     return pool;
 231
 232 }
 233
 234 static void pool_destroy(struct pool* pool)
 235 {
 236     void* extent;
 237     void* next;
 238
 239     if(pool != NULL)
 240     {
 241         extent = pool->extent;
 242         while(extent)
 243         {
 244             next = *(void**)extent;
 245             free(extent);
 246             extent = next;
 247         }
 248         free(pool);
 249     }
 250 }
 251
 252 static void* pool_alloc(struct pool* pool)
 253 {
 254     void* data;
 255
 256     data = pool->head_free;
 257     if(data == NULL)
 258     {
 259         /* we have no old-freed elem */
 260         if(pool->fresh <= pool->tail)
 261         {
 262             /* pick a slice of the current extent */
 263             data = (void*)pool->fresh;
 264             pool->fresh += pool->size_elem;
 265         }
 266         else
 267         {
 268             /* allocate a new extent */
 269             data = pool_take_extent(pool, TRUE);
 270         }
 271     }
 272     else
 273     {
 274         /* re-used old freed element by chopping the head of the free list */
 275         pool->head_free = *(void**)data;
 276     }
 277
 278     return data;
 279 }
 280
 281
 282 /* ===============================================
 283  * Hash implementation customized to be just tracking
 284  * a unique list of string (i.e no data associated
 285  * with the key, no need for retrieval, etc...
 286  *
 287  * This is tuned for the particular use-case we have here
 288  * measures in tail_build showed that
 289  * we can get north of 4000 distinct values stored in a hash
 290  * the collision rate is at worse around 2%
 291  * the collision needing an expensive memcmp to resolve
 292  * have a rate typically at 1 per 1000
 293  * for tail_build we register 37229 unique key
 294  * with a total of 377 extra memcmp needed
 295  * which is completely negligible compared to the
 296  * number of memcmp required to eliminate duplicate
 297  * entry (north of 2.5 millions for tail_build)
 298  * ===============================================
 299  */
 300
 301 struct hash_elem
 302 {
 303     struct hash_elem* next;
 304     const char*    key;
 305     int      key_len;
 306 };
 307
 308 struct hash
 309 {
 310     struct hash_elem** array;
 311     struct pool* elems_pool;
 312     unsigned int used;
 313     unsigned int size;
 314     unsigned int load_limit;
 315 #ifdef HASH_STAT
 316     int stored;
 317     int collisions;
 318     int cost;
 319     int memcmp;
 320 #endif
 321 };
 322
 323 /* The following hash_compute function was adapted from :
 324  * lookup3.c, by Bob Jenkins, May 2006, Public Domain.
 325  *
 326  * The changes from the original are mostly cosmetic
 327  */
 328 #define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k))))
 329
 330
 331 #if defined CORE_BIG_ENDIAN
 332 #define MASK_C1 0xFFFFFF00
 333 #define MASK_C2 0xFFFF0000
 334 #define MASK_C3 0xFF000000
 335 #elif defined CORE_LITTLE_ENDIAN
 336 #define MASK_C1 0xFFFFFF
 337 #define MASK_C2 0xFFFF
 338 #define MASK_C3 0xFF
 339 #else
 340 #error "Missing Endianness definition"
 341 #endif
 342
 343
 344 #define mix(a,b,c) \
 345 { \
 346   a -= c;  a ^= rot(c, 4);  c += b; \
 347   b -= a;  b ^= rot(a, 6);  a += c; \
 348   c -= b;  c ^= rot(b, 8);  b += a; \
 349   a -= c;  a ^= rot(c,16);  c += b; \
 350   b -= a;  b ^= rot(a,19);  a += c; \
 351   c -= b;  c ^= rot(b, 4);  b += a; \
 352 }
 353 #define final(a,b,c) \
 354 { \
 355   c ^= b; c -= rot(b,14); \
 356   a ^= c; a -= rot(c,11); \
 357   b ^= a; b -= rot(a,25); \
 358   c ^= b; c -= rot(b,16); \
 359   a ^= c; a -= rot(c,4);  \
 360   b ^= a; b -= rot(a,14); \
 361   c ^= b; c -= rot(b,24); \
 362 }
 363
 364 static unsigned int hash_compute( struct hash const * hash, const char* key, int length)
 365 {
 366     unsigned int a;
 367     unsigned int b;
 368     unsigned int c;                                          /* internal state */
 369     const unsigned char* uk = (const unsigned char*)key;
 370
 371     /* Set up the internal state */
 372     a = b = c = 0xdeadbeef + (length << 2);
 373
 374     /* we use this to 'hash' full path with mostly a common root
 375      * let's now waste too much cycles hashing mostly constant stuff
 376      */
 377     if(length > 36)
 378     {
 379         uk += length - 36;
 380         length = 36;
 381     }
 382     /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */
 383     while (length > 12)
 384     {
 385         a += get_unaligned_uint(uk);
 386         b += get_unaligned_uint(uk+4);
 387         c += get_unaligned_uint(uk+8);
 388         mix(a,b,c);
 389         length -= 12;
 390         uk += 12;
 391     }
 392
 393     /*----------------------------- handle the last (probably partial) block */
 394     /* Note: we possibly over-read, which would trigger complaint from VALGRIND
 395      * but we mask the undefined stuff if any, so we are still good, thanks
 396      * to alignment of memory allocation and tail-memory management overhead
 397      * we always can read 3 bytes past the official end without triggering
 398      * a segfault -- if you find a platform/compiler couple for which that postulate
 399      * is false, then you just need to over-allocate by 2 more bytes in file_load()
 400      * file_load already over-allocate by 1 to stick a \0 at the end of the buffer.
 401      */
 402     switch(length)
 403     {
 404     case 12: c+=get_unaligned_uint(uk+8); b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 405     case 11: c+=get_unaligned_uint(uk+8) & MASK_C1; b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 406     case 10: c+=get_unaligned_uint(uk+8) & MASK_C2; b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 407     case 9 : c+=get_unaligned_uint(uk+8) & MASK_C3; b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 408     case 8 : b+=get_unaligned_uint(uk+4); a+=get_unaligned_uint(uk); break;
 409     case 7 : b+=get_unaligned_uint(uk+4) & MASK_C1; a+=get_unaligned_uint(uk); break;
 410     case 6 : b+=get_unaligned_uint(uk+4) & MASK_C2; a+=get_unaligned_uint(uk); break;
 411     case 5 : b+=get_unaligned_uint(uk+4) & MASK_C3; a+=get_unaligned_uint(uk); break;
 412     case 4 : a+=get_unaligned_uint(uk); break;
 413     case 3 : a+=get_unaligned_uint(uk) & MASK_C1; break;
 414     case 2 : a+=get_unaligned_uint(uk) & MASK_C2; break;
 415     case 1 : a+=get_unaligned_uint(uk) & MASK_C3; break;
 416     case 0 : return c & hash->size;              /* zero length strings require no mixing */
 417     }
 418
 419     final(a,b,c);
 420     return c & hash->size;
 421 }
 422
 423 static void hash_destroy(struct hash* hash)
 424 {
 425     if(hash)
 426     {
 427         if(hash->array)
 428         {
 429             free(hash->array);
 430         }
 431         if(hash->elems_pool)
 432         {
 433             pool_destroy(hash->elems_pool);
 434         }
 435         free(hash);
 436     }
 437 }
 438
 439 static struct hash* hash_create(unsigned int size)
 440 {
 441     struct hash* hash;
 442
 443     assert(size > 0);
 444     hash = (struct hash*)(calloc(1, sizeof(struct hash)));
 445     if(hash)
 446     {
 447         size += (size >> 2) + 1; /* ~ 75% load factor */
 448         if(size >= 15)
 449         {
 450             hash->size = (((unsigned int)0xFFFFFFFF) >> clz((unsigned int)size));
 451         }
 452         else
 453         {
 454             hash->size = size = 15;
 455         }
 456         hash->load_limit = hash->size - (hash->size >> 2);
 457         hash->used = 0;
 458         hash->array = (struct hash_elem**)calloc(hash->size + 1, sizeof(struct hash_elem*));
 459         if(hash->array == NULL)
 460         {
 461             hash_destroy(hash);
 462             hash = NULL;
 463         }
 464     }
 465     if(hash)
 466     {
 467         hash->elems_pool = pool_create(sizeof(struct hash_elem),
 468                                        size, size << 1);
 469         if(!hash->elems_pool)
 470         {
 471             hash_destroy(hash);
 472             hash = NULL;
 473         }
 474     }
 475     return hash;
 476 }
 477
 478 static void hash_resize(struct hash* hash)
 479 {
 480     unsigned int old_size = hash->size;
 481     unsigned int hashed;
 482     struct hash_elem* hash_elem;
 483     struct hash_elem* next;
 484     struct hash_elem** array;
 485     unsigned int i;
 486
 487     hash->size = (old_size << 1) + 1;
 488     /* we really should avoid to get there... so print a message to alert of the condition */
 489     fprintf(stderr, "resize hash %u -> %u\n", old_size, hash->size);
 490     if(hash->size == old_size)
 491     {
 492         return;
 493     }
 494     array = (struct hash_elem**)calloc(hash->size + 1, sizeof(struct hash_elem*));
 495     if(array)
 496     {
 497         hash->load_limit = hash->size - (hash->size >> 2);
 498         for(i=0; i <= old_size; i++)
 499         {
 500             hash_elem = (struct hash_elem*)hash->array[i];
 501             while(hash_elem)
 502             {
 503                 next = hash_elem->next;
 504
 505                 hashed = hash_compute(hash, hash_elem->key, hash_elem->key_len);
 506                 hash_elem->next = array[hashed];
 507                 array[hashed] = hash_elem;
 508                 hash_elem = next;
 509             }
 510         }
 511         free(hash->array);
 512         hash->array = (struct hash_elem**)array;
 513     }
 514     else
 515     {
 516         hash->size = old_size;
 517     }
 518 }
 519
 520 static int compare_key(struct hash const * hash, const char* a, const char* b, int len, int const * cost)
 521 {
 522 #ifdef HASH_STAT
 523     *cost += 1;
 524     hash->memcmp += 1;
 525 #else
 526     (void) hash;
 527     (void) cost;
 528 #endif
 529     return memcmp(a,b, len);
 530 }
 531
 532 /* a customized hash_store function that just store the key and return
 533  * TRUE if the key was effectively stored, or FALSE if the key was already there
 534  */
 535 static int hash_store(struct hash* hash, const char* key, int key_len)
 536 {
 537     unsigned int hashed;
 538     struct hash_elem* hash_elem;
 539     int cost = 0;
 540
 541     (void) cost;
 542     hashed = hash_compute(hash, key, key_len);
 543 #ifdef HASH_STAT
 544     hash->stored += 1;
 545 #endif
 546     hash_elem = (struct hash_elem*)hash->array[hashed];
 547     while(hash_elem && (hash_elem->key_len != key_len || compare_key(hash, hash_elem->key, key, key_len, &cost)))
 548     {
 549         hash_elem = hash_elem->next;
 550     }
 551
 552     if(!hash_elem)
 553     {
 554         hash_elem = (struct hash_elem*)pool_alloc(hash->elems_pool);
 555         if(hash_elem)
 556         {
 557             hash_elem->key = key;
 558             hash_elem->key_len = key_len;
 559             hash_elem->next = hash->array[hashed];
 560
 561 #ifdef HASH_STAT
 562             if(hash_elem->next)
 563             {
 564                 hash->collisions += 1;
 565                 hash->cost += cost;
 566             }
 567 #endif
 568             hash->array[hashed] = hash_elem;
 569             hash->used += 1;
 570             if(hash->used > hash->load_limit)
 571             {
 572                 hash_resize(hash);
 573             }
 574         }
 575         return TRUE;
 576     }
 577     return FALSE;
 578 }
 579
 580 static int file_stat(const char* name, struct stat* buffer_stat, int* rc)
 581 {
 582     int rc_local = stat(name, buffer_stat);
 583     if (rc_local  < 0)
 584     {
 585         *rc = errno;
 586     }
 587     return rc_local;
 588 }
 589
 590 static off_t file_get_size(const char* name, int* rc)
 591 {
 592     struct stat buffer_stat;
 593     off_t       size = -1;
 594
 595     if (!file_stat(name, &buffer_stat, rc))
 596     {
 597         if(S_ISREG(buffer_stat.st_mode))
 598         {
 599             size = buffer_stat.st_size;
 600         }
 601         else
 602         {
 603             *rc = EINVAL;
 604         }
 605     }
 606     return size;
 607 }
 608
 609 #if !ENABLE_RUNTIME_OPTIMIZATIONS
 610 static void * file_load_buffers[100000];
 611 static size_t file_load_buffer_count = 0;
 612 #endif
 613
 614 static char* file_load(const char* name, off_t* size, int* return_rc)
 615 {
 616     off_t local_size = 0;
 617     int rc = 0;
 618     char* buffer = NULL;
 619     int fd;
 620
 621     assert(name != NULL);
 622
 623     if(!size)
 624     {
 625         size = &local_size;
 626     }
 627     *size = file_get_size(name, &rc);
 628     if (!rc && *size >= 0)
 629     {
 630         fd = open(name, FILE_O_RDONLY | FILE_O_BINARY);
 631         if (!(fd == -1))
 632         {
 633             buffer = (char*)malloc((size_t)(*size + 1));
 634 #if !ENABLE_RUNTIME_OPTIMIZATIONS
 635             if (buffer != NULL)
 636             {
 637                 if (file_load_buffer_count == 100000)
 638                 {
 639                     free(buffer);
 640                     buffer = NULL;
 641                 }
 642                 else
 643                 {
 644                     file_load_buffers[file_load_buffer_count++] = buffer;
 645                 }
 646             }
 647 #endif
 648             if (buffer == NULL)
 649             {
 650                 rc = ENOMEM;
 651             }
 652             else
 653             {
 654                 ssize_t i;
 655
 656               REDO:
 657                 i = read(fd, buffer, (size_t)(*size));
 658                 if(i == -1)
 659                 {
 660                     if(errno == EINTR)
 661                     {
 662                         goto REDO;
 663                     }
 664                     else
 665                     {
 666                         rc = errno;
 667                     }
 668                 }
 669                 else
 670                 {
 671                     if (i != *size)
 672                     {
 673                         rc = EIO;
 674                     }
 675                 }
 676                 buffer[*size] = 0;
 677             }
 678             close(fd);
 679         }
 680     }
 681
 682     if(rc && buffer)
 683     {
 684         free(buffer);
 685         buffer = NULL;
 686     }
 687     if(return_rc)
 688     {
 689         *return_rc = rc;
 690     }
 691     return buffer;
 692 }
 693
 694 static void cancel_relative(char const * base, char** ref_cursor, char** ref_cursor_out, char const * end)
 695 {
 696     char* cursor = *ref_cursor;
 697     char* cursor_out = *ref_cursor_out;
 698
 699     do
 700     {
 701         cursor += 3;
 702         while(cursor_out > base && cursor_out[-1] == '/')
 703             cursor_out--;
 704         while(cursor_out > base && *--cursor_out != '/');
 705     }
 706     while(cursor + 3 < end && !memcmp(cursor, "/../", 4));
 707     *ref_cursor = cursor;
 708     *ref_cursor_out = cursor_out;
 709 }
 710
 711 static void eat_space(char ** token)
 712 {
 713     while ((' ' == **token) || ('\t' == **token)) {
 714         ++(*token);
 715     }
 716 }
 717
 718 /*
 719  * Prune LibreOffice specific duplicate dependencies to improve
 720  * gnumake startup time, and shrink the disk-space footprint.
 721  */
 722 static int
 723 elide_dependency(const char* key, int key_len, const char **unpacked_end)
 724 {
 725 #if 0
 726     {
 727         int i;
 728         fprintf (stderr, "elide?%d!: '", internal_boost);
 729         for (i = 0; i < key_len; i++) {
 730             fprintf (stderr, "%c", key[i]);
 731         }
 732         fprintf (stderr, "'\n");
 733     }
 734 #endif
 735
 736     /* boost brings a plague of header files */
 737     int i;
 738     int unpacked = 0;
 739     /* walk down path elements */
 740     for (i = 0; i < key_len - 1; i++)
 741     {
 742         if (key[i] == '/')
 743         {
 744             if (0 == unpacked)
 745             {
 746                 if (!PATHNCMP(key + i + 1, "workdir/", 8))
 747                 {
 748                     unpacked = 1;
 749                     continue;
 750                 }
 751             }
 752             else
 753             {
 754                 if (!PATHNCMP(key + i + 1, "UnpackedTarball/", 16))
 755                 {
 756                     if (unpacked_end)
 757                         *unpacked_end = strchr(key + i + 17, '/');
 758                     return 1;
 759                 }
 760             }
 761         }
 762     }
 763
 764     return 0;
 765 }
 766
 767 /*
 768  * We collapse tens of internal boost headers to the unpacked target, such
 769  * that you can re-compile / install boost and all is well.
 770  */
 771 static void emit_single_boost_header(void)
 772 {
 773 #define BOOST_TARGET "/UnpackedTarball/boost.done"
 774     fprintf(stdout, "%s" BOOST_TARGET " ", work_dir);
 775 }
 776
 777 static void emit_unpacked_target(const char* token, const char* end)
 778 {
 779     fwrite(token, 1, end-token, stdout);
 780     fputs(".done ", stdout);
 781 }
 782
 783 /* prefix paths to absolute */
 784 static void print_fullpaths(char* line)
 785 {
 786     char* token;
 787     char* end;
 788     int boost_count = 0;
 789     int token_len;
 790     const char * unpacked_end = NULL; /* end of UnpackedTarget match (if any) */
 791     /* for UnpackedTarget the target is GenC{,xx}Object, don't mangle! */
 792     int target_seen = 0;
 793
 794     token = line;
 795     eat_space(&token);
 796     while (*token)
 797     {
 798         end = token;
 799         /* hard to believe that in this day and age drive letters still exist */
 800         if (*end && (':' == *(end+1)) &&
 801             (('\\' == *(end+2)) || ('/' == *(end+2))) &&
 802             isalpha((unsigned char)*end))
 803         {
 804             end = end + 3; /* only one cross, err drive letter per filename */
 805         }
 806         while (*end && (' ' != *end) && ('\t' != *end) && (':' != *end)) {
 807             ++end;
 808         }
 809         token_len = end - token;
 810         if (target_seen &&
 811             elide_dependency(token, token_len, &unpacked_end))
 812         {
 813             if (unpacked_end)
 814             {
 815                 if (internal_boost && !PATHNCMP(unpacked_end - 5, "boost", 5))
 816                 {
 817                     ++boost_count;
 818                     if (boost_count == 1)
 819                         emit_single_boost_header();
 820                     else
 821                     {
 822                         /* don't output, and swallow trailing \\\n if any */
 823                         token = end;
 824                         eat_space(&token);
 825                         if (token[0] == '\\' && token[1] == '\n')
 826                             end = token + 2;
 827                     }
 828                 }
 829                 else
 830                 {
 831                     emit_unpacked_target(token, unpacked_end);
 832                 }
 833                 unpacked_end = NULL;
 834             }
 835         }
 836         else
 837         {
 838             if (fwrite(token, token_len, 1, stdout) != 1)
 839                 abort();
 840             fputc(' ', stdout);
 841         }
 842         token = end;
 843         eat_space(&token);
 844         if (!target_seen && ':' == *token)
 845         {
 846             target_seen = 1;
 847             fputc(':', stdout);
 848             ++token;
 849             eat_space(&token);
 850         }
 851     }
 852 }
 853
 854 static char * eat_space_at_end(char * end)
 855 {
 856     char * real_end;
 857     assert('\0' == *end);
 858     real_end = end - 1;
 859     while (' ' == *real_end || '\t' == *real_end || '\n' == *real_end
 860                 || ':' == *real_end)
 861     {    /* eat colon and whitespace at end */
 862          --real_end;
 863     }
 864     return real_end;
 865 }
 866
 867 static char* phony_content_buffer;
 868 static char* generate_phony_line(char const * phony_target, char const * extension)
 869 {
 870     char const * src;
 871     char* dest;
 872     char* last_dot = NULL;
 873     //fprintf(stderr, "generate_phony_line called with phony_target %s and extension %s\n", phony_target, extension);
 874     for(dest = phony_content_buffer+work_dir_len+1, src = phony_target; *src != 0; ++src, ++dest)
 875     {
 876         *dest = *src;
 877         if(*dest == '.')
 878         {
 879             last_dot = dest;
 880         }
 881     }
 882     //fprintf(stderr, "generate_phony_line after phony_target copy: %s\n", phony_content_buffer);
 883     for(dest = last_dot+1, src = extension; *src != 0; ++src, ++dest)
 884     {
 885         *dest = *src;
 886     }
 887     //fprintf(stderr, "generate_phony_line after extension add: %s\n", phony_content_buffer);
 888     strcpy(dest, ": $(gb_Helper_PHONY)\n");
 889     //fprintf(stderr, "generate_phony_line after phony add: %s\n", phony_content_buffer);
 890     return phony_content_buffer;
 891 }
 892
 893 static int generate_phony_file(char* fn, char const * content)
 894 {
 895     FILE* depfile;
 896     depfile = fopen(fn, "w");
 897     if(!depfile)
 898     {
 899         fprintf(stderr, "Could not open '%s' for writing: %s\n", fn, strerror(errno));
 900     }
 901     else
 902     {
 903         fputs(content, depfile);
 904         fclose(depfile);
 905     }
 906     return !depfile;
 907 }
 908
 909 static int process(struct hash* dep_hash, char* fn)
 910 {
 911     int rc;
 912     char* buffer;
 913     char* end;
 914     char* cursor;
 915     char* cursor_out;
 916     char* base;
 917     char* created_line = NULL;
 918     char* src_relative;
 919     int continuation = 0;
 920     char last_ns = 0;
 921     off_t size;
 922
 923     buffer = file_load(fn, &size, &rc);
 924     if(!rc)
 925     {
 926         base = cursor_out = cursor = end = buffer;
 927         end += size;
 928
 929         /* first eat unneeded space at the beginning of file
 930          */
 931         while(cursor < end && (*cursor == ' ' || *cursor == '\\'))
 932             ++cursor;
 933
 934         while(cursor < end)
 935         {
 936             if(*cursor == '\\')
 937             {
 938                 continuation = 1;
 939                 *cursor_out++ = *cursor++;
 940             }
 941             else if(*cursor == '/')
 942             {
 943                 if(cursor + 2 < end)
 944                 {
 945                     if(!memcmp(cursor, "/./", 3))
 946                     {
 947                         cursor += 2;
 948                         continue;
 949                     }
 950                 }
 951                 if(cursor + 3 < end)
 952                 {
 953                     if(!memcmp(cursor, "/../", 4))
 954                     {
 955                         cancel_relative(base, &cursor, &cursor_out, end);
 956                     }
 957                 }
 958                 *cursor_out++ = *cursor++;
 959             }
 960             else if(*cursor == '\n')
 961             {
 962                 if(!continuation)
 963                 {
 964                     *cursor_out = 0;
 965                     if(base < cursor)
 966                     {
 967                         /* here we have a complete rule */
 968                         if(last_ns == ':')
 969                         {
 970                             /* if the rule ended in ':' that is a no-dep rule
 971                              * these are the one for which we want to filter
 972                              * duplicate out
 973                              */
 974                             int key_len = eat_space_at_end(cursor_out) - base;
 975                             if (!elide_dependency(base,key_len + 1, NULL)
 976                                 && hash_store(dep_hash, base, key_len))
 977                             {
 978                                 /* DO NOT modify base after it has been added
 979                                    as key by hash_store */
 980                                 print_fullpaths(base);
 981                                 putc('\n', stdout);
 982                             }
 983                         }
 984                         else
 985                         {
 986                             /* rule with dep, just write it */
 987                             print_fullpaths(base);
 988                             putc('\n', stdout);
 989                         }
 990                         last_ns = ' '; // cannot hurt to reset it
 991                     }
 992                     cursor += 1;
 993                     base = cursor_out = cursor;
 994                 }
 995                 else
 996                 {
 997                     /* here we have a '\' followed by \n this is a continuation
 998                      * i.e not a complete rule yet
 999                      */
1000                     *cursor_out++ = *cursor++;
1001                     continuation = 0; // cancel current one (empty lines!)
1002                 }
1003             }
1004             else
1005             {
1006                 continuation = 0;
1007                 /* not using isspace() here save 25% of I refs and 75% of D refs based on cachegrind */
1008                 if(*cursor != ' ' && *cursor != '\n' && *cursor != '\t' )
1009                 {
1010                     last_ns = *cursor;
1011                 }
1012                 *cursor_out++ = *cursor++;
1013             }
1014         }
1015
1016         /* just in case the file did not end with a \n, there may be a pending rule */
1017         if(base < cursor_out)
1018         {
1019             if(last_ns == ':')
1020             {
1021                 int key_len = eat_space_at_end(cursor_out) - base;
1022                 if (!elide_dependency(base,key_len + 1, NULL) &&
1023                     hash_store(dep_hash, base, key_len))
1024                 {
1025                     puts(base);
1026                     putc('\n', stdout);
1027                 }
1028             }
1029             else
1030             {
1031                 puts(base);
1032                 putc('\n', stdout);
1033             }
1034         }
1035     }
1036     else
1037     {
1038         if(strncmp(fn, work_dir, work_dir_len) == 0)
1039         {
1040             if(strncmp(fn+work_dir_len, "/Dep/", 5) == 0)
1041             {
1042                 src_relative = fn+work_dir_len+5;
1043                 // cases ordered by frequency
1044                 if(strncmp(src_relative, "CxxObject/", 10) == 0)
1045                 {
1046                     created_line = generate_phony_line(src_relative, "o");
1047                     rc = generate_phony_file(fn, created_line);
1048                 }
1049                 else if(strncmp(src_relative, "GenCxxObject/", 13) == 0)
1050                 {
1051                     created_line = generate_phony_line(src_relative, "o");
1052                     rc = generate_phony_file(fn, created_line);
1053                 }
1054                 else if(strncmp(src_relative, "CObject/", 8) == 0)
1055                 {
1056                     created_line = generate_phony_line(src_relative, "o");
1057                     rc = generate_phony_file(fn, created_line);
1058                 }
1059                 else if(strncmp(src_relative, "GenCObject/", 11) == 0)
1060                 {
1061                     created_line = generate_phony_line(src_relative, "o");
1062                     rc = generate_phony_file(fn, created_line);
1063                 }
1064                 else if(strncmp(src_relative, "SdiObject/", 10) == 0)
1065                 {
1066                     created_line = generate_phony_line(src_relative, "o");
1067                     rc = generate_phony_file(fn, created_line);
1068                 }
1069                 else if(strncmp(src_relative, "AsmObject/", 10) == 0)
1070                 {
1071                     created_line = generate_phony_line(src_relative, "o");
1072                     rc = generate_phony_file(fn, created_line);
1073                 }
1074                 else if(strncmp(src_relative, "ObjCxxObject/", 13) == 0)
1075                 {
1076                     created_line = generate_phony_line(src_relative, "o");
1077                     rc = generate_phony_file(fn, created_line);
1078                 }
1079                 else if(strncmp(src_relative, "ObjCObject/", 11) == 0)
1080                 {
1081                     created_line = generate_phony_line(src_relative, "o");
1082                     rc = generate_phony_file(fn, created_line);
1083                 }
1084                 else if(strncmp(src_relative, "GenObjCxxObject/", 16) == 0)
1085                 {
1086                     created_line = generate_phony_line(src_relative, "o");
1087                     rc = generate_phony_file(fn, created_line);
1088                 }
1089                 else if(strncmp(src_relative, "GenObjCObject/", 14) == 0)
1090                 {
1091                     created_line = generate_phony_line(src_relative, "o");
1092                     rc = generate_phony_file(fn, created_line);
1093                 }
1094                 else if(strncmp(src_relative, "GenNasmObject/", 14) == 0)
1095                 {
1096                     created_line = generate_phony_line(src_relative, "o");
1097                     rc = generate_phony_file(fn, created_line);
1098                 }
1099                 else if(strncmp(src_relative, "CxxClrObject/", 13) == 0)
1100                 {
1101                     created_line = generate_phony_line(src_relative, "o");
1102                     rc = generate_phony_file(fn, created_line);
1103                 }
1104                 else if(strncmp(src_relative, "GenCxxClrObject/", 16) == 0)
1105                 {
1106                     created_line = generate_phony_line(src_relative, "o");
1107                     rc = generate_phony_file(fn, created_line);
1108                 }
1109                 else
1110                 {
1111                     fprintf(stderr, "no magic for %s(%s) in %s\n", fn, src_relative, work_dir);
1112                 }
1113             }
1114             if(!rc)
1115             {
1116                 puts(created_line);
1117             }
1118         }
1119     }
1120     /* Note: yes we are going to leak 'buffer'
1121      * this is on purpose, to avoid cloning the 'key' out of it and our special
1122      * 'hash' just store the pointer to the key inside of buffer, hence it need
1123      * to remain allocated
1124      */
1125     // coverity[leaked_storage] - this is on purpose
1126     return rc;
1127 }
1128
1129 static void usage(void)
1130 {
1131     fputs("Usage: concat-deps <file that contains dep_files>\n", stderr);
1132 }
1133
1134 #define kDEFAULT_HASH_SIZE 4096
1135 #define PHONY_TARGET_BUFFER 4096
1136
1137 static int get_var(char **var, const char *name)
1138 {
1139     *var = (char *)getenv(name);
1140     if(!*var)
1141     {
1142         fprintf(stderr,"Error: %s is missing in the environment\n", name);
1143         return 1;
1144     }
1145     return 0;
1146 }
1147
1148 int main(int argc, char** argv)
1149 {
1150     int rc = 0;
1151     off_t in_list_size = 0;
1152     char* in_list;
1153     char* in_list_cursor;
1154     char* in_list_base;
1155     struct hash* dep_hash = NULL;
1156     const char *env_str;
1157
1158     if(argc < 2)
1159     {
1160         usage();
1161         return 1;
1162     }
1163     if(get_var(&base_dir, "SRCDIR") || get_var(&work_dir, "WORKDIR"))
1164         return 1;
1165     work_dir_len = strlen(work_dir);
1166     phony_content_buffer = (char*)malloc(PHONY_TARGET_BUFFER);
1167     assert(phony_content_buffer); // Don't handle OOM conditions
1168     strcpy(phony_content_buffer, work_dir);
1169     phony_content_buffer[work_dir_len] = '/';
1170
1171     env_str = getenv("SYSTEM_BOOST");
1172     internal_boost = !env_str || strcmp(env_str,"TRUE");
1173
1174     in_list = file_load(argv[1], &in_list_size, &rc);
1175     if(!rc)
1176     {
1177         dep_hash = hash_create( kDEFAULT_HASH_SIZE);
1178         in_list_base = in_list_cursor = in_list;
1179
1180         /* extract filename of dep file from a 'space' separated list */
1181         while(*in_list_cursor)
1182         {
1183             /* the input here may contain Win32 \r\n EOL */
1184             if(*in_list_cursor == ' '
1185                 || *in_list_cursor == '\n' || *in_list_cursor == '\r')
1186             {
1187                 *in_list_cursor = 0;
1188                 if(in_list_base < in_list_cursor)
1189                 {
1190                     rc = process(dep_hash, in_list_base);
1191                     if(rc)
1192                     {
1193                         break;
1194                     }
1195                 }
1196                 in_list_cursor += 1;
1197                 in_list_base = in_list_cursor;
1198             }
1199             else
1200             {
1201                 in_list_cursor += 1;
1202             }
1203         }
1204         if(!rc)
1205         {
1206             /* catch the last entry in case the input did not terminate with a 'space' */
1207             if(in_list_base < in_list_cursor)
1208             {
1209                 rc = process(dep_hash, in_list_base);
1210             }
1211         }
1212 #ifdef HASH_STAT
1213         fprintf(stderr, "stats: u:%d s:%d l:%d t:%d c:%d m:%d $:%d\n",
1214                 dep_hash->used, dep_hash->size, dep_hash->load_limit, dep_hash->stored,
1215                 dep_hash->collisions, dep_hash->memcmp, dep_hash->cost);
1216 #endif
1217     }
1218 #if !ENABLE_RUNTIME_OPTIMIZATIONS
1219     {
1220         size_t i;
1221         hash_destroy(dep_hash);
1222         for (i = 0; i != file_load_buffer_count; ++i)
1223         {
1224             free(file_load_buffers[i]);
1225         }
1226     }
1227 #endif
1228     return rc;
1229 }
1230
1231 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */