gcc/lto-ltrans-cache.cc

   1 /* File caching.
   2    Copyright (C) 2023-2025 Free Software Foundation, Inc.
   3
   4 This file is part of GCC.
   5
   6 GCC is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 3, or (at your option) any later
   9 version.
  10
  11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with GCC; see the file COPYING3.  If not see
  18 <http://www.gnu.org/licenses/>.  */
  19
  20 #define INCLUDE_ALGORITHM
  21 #define INCLUDE_STRING
  22 #define INCLUDE_ARRAY
  23 #define INCLUDE_MAP
  24 #define INCLUDE_VECTOR
  25 #include "config.h"
  26 #include "system.h"
  27 #include "sha1.h"
  28 #include "lto-ltrans-cache.h"
  29
  30 static const checksum_t NULL_CHECKSUM = {
  31   0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  32   0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  33 };
  34
  35 /* Computes checksum for given file, returns NULL_CHECKSUM if not
  36    possible.  */
  37 static checksum_t
  38 file_checksum (char const *filename)
  39 {
  40   FILE *file = fopen (filename, "rb");
  41
  42   if (!file)
  43     return NULL_CHECKSUM;
  44
  45   checksum_t result = NULL_CHECKSUM;
  46
  47   int ret = sha1_stream (file, &result);
  48
  49   if (ret)
  50     result = NULL_CHECKSUM;
  51
  52   fclose (file);
  53
  54   return result;
  55 }
  56
  57 /* Checks identity of two files.  */
  58 static bool
  59 files_identical (char const *first_filename, char const *second_filename)
  60 {
  61   bool ret = true;
  62
  63 #if HAVE_MMAP_FILE
  64   struct stat st;
  65   if (stat (first_filename, &st) < 0 || !S_ISREG (st.st_mode))
  66     return false;
  67   size_t len = st.st_size;
  68   if (stat (second_filename, &st) < 0 || !S_ISREG (st.st_mode))
  69     return false;
  70   if (len != (size_t) st.st_size)
  71     return false;
  72
  73   int fd;
  74   void *map[2] = { NULL, NULL };
  75
  76   fd = open (first_filename, O_RDONLY);
  77   if (fd < 0)
  78     goto fail_mmap;
  79   map[0] = mmap (NULL, len, PROT_READ, MAP_PRIVATE, fd, 0);
  80   close (fd);
  81   if (map[0] == MAP_FAILED)
  82     goto fail_mmap;
  83
  84   fd = open (second_filename, O_RDONLY);
  85   if (fd < 0)
  86     goto fail_mmap2;
  87   map[1] = mmap (NULL, len, PROT_READ, MAP_PRIVATE, fd, 0);
  88   close (fd);
  89   if (map[1] == MAP_FAILED)
  90     goto fail_mmap2;
  91
  92   ret = memcmp (map[0], map[1], len) == 0;
  93   munmap (map[1], len);
  94   munmap (map[0], len);
  95   return ret;
  96
  97 fail_mmap2:
  98   munmap (map[0], len);
  99 fail_mmap:
 100 #endif
 101
 102   /* Fallback.  */
 103
 104   FILE *f_first = fopen (first_filename, "rb");
 105   if (!f_first)
 106     return false;
 107
 108   FILE *f_second = fopen (second_filename, "rb");
 109   if (!f_second)
 110     {
 111       fclose (f_first);
 112       return false;
 113     }
 114
 115   const size_t buffer_len = 64 * 1024;
 116   uint8_t *buffer1 = (uint8_t*)xmalloc (buffer_len);
 117   uint8_t *buffer2 = (uint8_t*)xmalloc (buffer_len);
 118
 119   size_t len1, len2;
 120
 121   do
 122     {
 123       len1 = fread (buffer1, 1, buffer_len, f_first);
 124       len2 = fread (buffer2, 1, buffer_len, f_second);
 125
 126       if (len1 != len2 || memcmp (buffer1, buffer2, len1) != 0)
 127         {
 128           ret = false;
 129           break;
 130         }
 131     }
 132   while (len1 != 0);
 133
 134   free (buffer2);
 135   free (buffer1);
 136   fclose (f_first);
 137   fclose (f_second);
 138   return ret;
 139 }
 140
 141 /* Contructor of cache item.  */
 142 ltrans_file_cache::item::item (std::string input, std::string output,
 143                                checksum_t input_checksum,
 144                                uint32_t last_used):
 145   input (std::move (input)), output (std::move (output)),
 146   input_checksum (input_checksum), last_used (last_used)
 147 {
 148   lock = lockfile (this->input + ".lock");
 149 }
 150 /* Destructor of cache item.  */
 151 ltrans_file_cache::item::~item ()
 152 {
 153   lock.unlock ();
 154 }
 155
 156 /* Reads next cache item from cachedata file.
 157    Adds `dir/` prefix to filenames.  */
 158 static ltrans_file_cache::item*
 159 read_cache_item (FILE* f, const char* dir)
 160 {
 161   checksum_t checksum;
 162   uint32_t last_used;
 163
 164   if (fread (&checksum, 1, checksum.size (), f) != checksum.size ())
 165     return NULL;
 166   if (fread (&last_used, sizeof (last_used), 1, f) != 1)
 167     return NULL;
 168
 169   std::string input (dir);
 170   input.push_back ('/');
 171   std::string output = input; /* Copy.  */
 172
 173   int c;
 174   while ((c = getc (f)))
 175     {
 176       if (c == EOF)
 177         return NULL;
 178       input.push_back (c);
 179     }
 180   while ((c = getc (f)))
 181     {
 182       if (c == EOF)
 183         return NULL;
 184       output.push_back (c);
 185     }
 186
 187   return new ltrans_file_cache::item (&input[0], &output[0], checksum,
 188                                       last_used);
 189 }
 190
 191 /* Writes cache item to cachedata file.
 192    Removes `dir/` prefix from filenames.  */
 193 static void
 194 write_cache_item (FILE* f, ltrans_file_cache::item *item, const char* dir)
 195 {
 196   fwrite (&item->input_checksum, 1, item->input_checksum.size (), f);
 197   fwrite (&item->last_used, sizeof (item->last_used), 1, f);
 198
 199   gcc_assert (item->input.size () > strlen (dir));
 200   fputs (item->input.c_str () + strlen (dir) + 1, f);
 201   fputc (0, f);
 202
 203   gcc_assert (item->output.size () > strlen (dir));
 204   fputs (item->output.c_str () + strlen (dir) + 1, f);
 205   fputc (0, f);
 206 }
 207
 208 /* Constructor.  Resulting cache item filenames will be
 209    in format `prefix%d[.ltrans]suffix`.  */
 210 ltrans_file_cache::ltrans_file_cache (const char* dir, const char* prefix,
 211                                       const char* suffix,
 212                                       size_t soft_cache_size):
 213   dir (dir), prefix (prefix), suffix (suffix),
 214   soft_cache_size (soft_cache_size)
 215 {
 216   if (!dir) return;
 217
 218   creation_lock = lockfile (std::string (dir) + "/lockfile_creation");
 219   deletion_lock = lockfile (std::string (dir) + "/lockfile_deletion");
 220
 221   cache_prefix = std::string (dir) + "/" + prefix;
 222   cache_free_idx = 0;
 223
 224   str_buffer = (char *)xmalloc (cache_prefix.size ()
 225                                 + sizeof ("4294967295.ltrans")
 226                                 + strlen (suffix));
 227 }
 228
 229 /* Destructor.  */
 230 ltrans_file_cache::~ltrans_file_cache ()
 231 {
 232   if (!*this)
 233     return;
 234
 235   cleanup ();
 236   free (str_buffer);
 237 }
 238
 239 /* Adds given cache item to all relevant datastructures.  */
 240 void
 241 ltrans_file_cache::add_item (ltrans_file_cache::item* item)
 242 {
 243   items.push_back (item);
 244   map_checksum[item->input_checksum] = item;
 245   map_input[item->input] = item;
 246
 247   usage_counter = std::max (usage_counter, item->last_used);
 248 }
 249
 250 /* Creates cachedata filename for save/load.  */
 251 std::string
 252 ltrans_file_cache::filename_cachedata ()
 253 {
 254   return std::string (dir) + "/cachedata";
 255 }
 256
 257 /* Loads data about previously cached items from cachedata file.
 258    Sorts items by last_used and remaps last_used to small integers.
 259
 260    Must be called with creation_lock or deletion_lock held to
 261    prevent data race.  */
 262 void
 263 ltrans_file_cache::load_cache ()
 264 {
 265   cleanup ();
 266
 267   std::string filename = filename_cachedata ();
 268   FILE *file = fopen (filename.c_str (), "rb");
 269
 270   if (!file)
 271     return;
 272
 273   ltrans_file_cache::item* _item;
 274   while ((_item = read_cache_item (file, dir)))
 275     add_item (_item);
 276
 277   fclose (file);
 278
 279   std::sort (items.begin (), items.end (),
 280              [](item* a, item* b)
 281                {return a->last_used < b->last_used;});
 282
 283   for (size_t i = 0; i < items.size (); ++i)
 284     items[i]->last_used = (uint32_t) i;
 285   usage_counter = (uint32_t) items.size ();
 286 }
 287
 288 /* Rewrites data about cache items into cachedata file.
 289
 290    Must be only called when creation_lock or deletion_lock was held since last
 291    call to load_cache.  */
 292 void
 293 ltrans_file_cache::save_cache ()
 294 {
 295   std::string filename = filename_cachedata ();
 296   FILE *file = fopen (filename.c_str (), "wb");
 297
 298   if (!file)
 299     return;
 300
 301   for (item* _item: items)
 302     write_cache_item (file, _item, dir);
 303
 304   fclose (file);
 305 }
 306
 307 /* Creates new cache item with given checksum.
 308    New input/output files are chosen to not collide with other items.
 309
 310    Must be called with creation_lock held to prevent data race.  */
 311 ltrans_file_cache::item*
 312 ltrans_file_cache::create_item (const checksum_t& checksum)
 313 {
 314   size_t prefix_len = cache_prefix.size ();
 315
 316   strcpy (str_buffer, cache_prefix.c_str ());
 317
 318   for (;;)
 319     {
 320       sprintf (str_buffer + prefix_len, "%04u%s", cache_free_idx, suffix);
 321
 322       if (map_input.find (str_buffer) == map_input.end ())
 323         break;
 324       cache_free_idx++;
 325     }
 326
 327   std::string input = str_buffer;
 328
 329   sprintf (str_buffer + prefix_len, "%04u.ltrans%s", cache_free_idx, suffix);
 330
 331   return new item (std::move (input), str_buffer, checksum, usage_counter++);
 332 }
 333
 334 /* Adds input file into cache.  Cache item with input file identical to
 335    added input file will be returned as _item.
 336    If the file was already cached, `true` is returned, `false` otherwise.
 337    The added input file is deleted (or moved).
 338
 339    Must be called with creation_lock held to prevent data race.  */
 340 bool
 341 ltrans_file_cache::add_to_cache (const char* filename, item*& _item)
 342 {
 343   checksum_t checksum = file_checksum (filename);
 344
 345   auto it = map_checksum.find (checksum);
 346
 347   if (it != map_checksum.end ()
 348       && files_identical (filename, it->second->input.c_str ()))
 349     {
 350       unlink (filename);
 351       _item = it->second;
 352       _item->last_used = usage_counter++;
 353       return true;
 354     }
 355   else
 356     {
 357       /* Cache miss.  Move into cache dir.  */
 358       _item = create_item (checksum);
 359       add_item (_item);
 360
 361       rename (filename, _item->input.c_str ());
 362       return false;
 363     }
 364 }
 365
 366 /* If exists, returns cache item corresponding to cached input file.  */
 367 ltrans_file_cache::item*
 368 ltrans_file_cache::get_item (const char* input)
 369 {
 370   auto it = map_input.find (input);
 371   if (it == map_input.end ())
 372     return NULL;
 373   return it->second;
 374 }
 375
 376 /* If no other process holds the deletion_lock, prunes oldest unused cache
 377    items over limit.  */
 378 void
 379 ltrans_file_cache::try_prune ()
 380 {
 381   if (deletion_lock.try_lock_write () == 0)
 382     {
 383       prune ();
 384       deletion_lock.unlock ();
 385     }
 386 }
 387
 388 /* Returns true if file exists.  */
 389 static int
 390 file_exists (char const *name)
 391 {
 392   return access (name, R_OK) == 0;
 393 }
 394
 395 /* Prunes oldest unused cache items over limit.
 396    Must be called with deletion_lock held to prevent data race.  */
 397 void
 398 ltrans_file_cache::prune ()
 399 {
 400   load_cache ();
 401   if (items.size () > soft_cache_size)
 402     {
 403       std::vector<item*> sorted_items = std::move (items);
 404
 405       cleanup ();
 406
 407       for (size_t i = 0; i < sorted_items.size (); ++i)
 408         {
 409           ltrans_file_cache::item* item = sorted_items[i];
 410           if ((i < sorted_items.size () - soft_cache_size)
 411               || !file_exists (item->input.c_str ())
 412               || !file_exists (item->output.c_str ()))
 413             {
 414               unlink (item->input.c_str ());
 415               unlink (item->output.c_str ());
 416               delete item;
 417             }
 418           else
 419             add_item (item);
 420         }
 421     }
 422   save_cache ();
 423 }
 424
 425 /* Clears cache class, as if only constructor was called.  */
 426 void
 427 ltrans_file_cache::cleanup ()
 428 {
 429   map_checksum.clear ();
 430   map_input.clear ();
 431
 432   for (ltrans_file_cache::item* item: items)
 433     delete item;
 434   items.clear ();
 435
 436   usage_counter = 0;
 437 }