src/backend/backup/walsummary.c

   1 /*-------------------------------------------------------------------------
   2  *
   3  * walsummary.c
   4  *        Functions for accessing and managing WAL summary data.
   5  *
   6  * Portions Copyright (c) 2010-2024, PostgreSQL Global Development Group
   7  *
   8  * src/backend/backup/walsummary.c
   9  *
  10  *-------------------------------------------------------------------------
  11  */
  12
  13 #include "postgres.h"
  14
  15 #include <sys/stat.h>
  16 #include <unistd.h>
  17
  18 #include "access/xlog_internal.h"
  19 #include "backup/walsummary.h"
  20 #include "common/int.h"
  21 #include "utils/wait_event.h"
  22
  23 static bool IsWalSummaryFilename(char *filename);
  24 static int      ListComparatorForWalSummaryFiles(const ListCell *a,
  25                                                                                          const ListCell *b);
  26
  27 /*
  28  * Get a list of WAL summaries.
  29  *
  30  * If tli != 0, only WAL summaries with the indicated TLI will be included.
  31  *
  32  * If start_lsn != InvalidXLogRecPtr, only summaries that end after the
  33  * indicated LSN will be included.
  34  *
  35  * If end_lsn != InvalidXLogRecPtr, only summaries that start before the
  36  * indicated LSN will be included.
  37  *
  38  * The intent is that you can call GetWalSummaries(tli, start_lsn, end_lsn)
  39  * to get all WAL summaries on the indicated timeline that overlap the
  40  * specified LSN range.
  41  */
  42 List *
  43 GetWalSummaries(TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
  44 {
  45         DIR                *sdir;
  46         struct dirent *dent;
  47         List       *result = NIL;
  48
  49         sdir = AllocateDir(XLOGDIR "/summaries");
  50         while ((dent = ReadDir(sdir, XLOGDIR "/summaries")) != NULL)
  51         {
  52                 WalSummaryFile *ws;
  53                 uint32          tmp[5];
  54                 TimeLineID      file_tli;
  55                 XLogRecPtr      file_start_lsn;
  56                 XLogRecPtr      file_end_lsn;
  57
  58                 /* Decode filename, or skip if it's not in the expected format. */
  59                 if (!IsWalSummaryFilename(dent->d_name))
  60                         continue;
  61                 sscanf(dent->d_name, "%08X%08X%08X%08X%08X",
  62                            &tmp[0], &tmp[1], &tmp[2], &tmp[3], &tmp[4]);
  63                 file_tli = tmp[0];
  64                 file_start_lsn = ((uint64) tmp[1]) << 32 | tmp[2];
  65                 file_end_lsn = ((uint64) tmp[3]) << 32 | tmp[4];
  66
  67                 /* Skip if it doesn't match the filter criteria. */
  68                 if (tli != 0 && tli != file_tli)
  69                         continue;
  70                 if (!XLogRecPtrIsInvalid(start_lsn) && start_lsn >= file_end_lsn)
  71                         continue;
  72                 if (!XLogRecPtrIsInvalid(end_lsn) && end_lsn <= file_start_lsn)
  73                         continue;
  74
  75                 /* Add it to the list. */
  76                 ws = palloc(sizeof(WalSummaryFile));
  77                 ws->tli = file_tli;
  78                 ws->start_lsn = file_start_lsn;
  79                 ws->end_lsn = file_end_lsn;
  80                 result = lappend(result, ws);
  81         }
  82         FreeDir(sdir);
  83
  84         return result;
  85 }
  86
  87 /*
  88  * Build a new list of WAL summaries based on an existing list, but filtering
  89  * out summaries that don't match the search parameters.
  90  *
  91  * If tli != 0, only WAL summaries with the indicated TLI will be included.
  92  *
  93  * If start_lsn != InvalidXLogRecPtr, only summaries that end after the
  94  * indicated LSN will be included.
  95  *
  96  * If end_lsn != InvalidXLogRecPtr, only summaries that start before the
  97  * indicated LSN will be included.
  98  */
  99 List *
 100 FilterWalSummaries(List *wslist, TimeLineID tli,
 101                                    XLogRecPtr start_lsn, XLogRecPtr end_lsn)
 102 {
 103         List       *result = NIL;
 104         ListCell   *lc;
 105
 106         /* Loop over input. */
 107         foreach(lc, wslist)
 108         {
 109                 WalSummaryFile *ws = lfirst(lc);
 110
 111                 /* Skip if it doesn't match the filter criteria. */
 112                 if (tli != 0 && tli != ws->tli)
 113                         continue;
 114                 if (!XLogRecPtrIsInvalid(start_lsn) && start_lsn > ws->end_lsn)
 115                         continue;
 116                 if (!XLogRecPtrIsInvalid(end_lsn) && end_lsn < ws->start_lsn)
 117                         continue;
 118
 119                 /* Add it to the result list. */
 120                 result = lappend(result, ws);
 121         }
 122
 123         return result;
 124 }
 125
 126 /*
 127  * Check whether the supplied list of WalSummaryFile objects covers the
 128  * whole range of LSNs from start_lsn to end_lsn. This function ignores
 129  * timelines, so the caller should probably filter using the appropriate
 130  * timeline before calling this.
 131  *
 132  * If the whole range of LSNs is covered, returns true, otherwise false.
 133  * If false is returned, *missing_lsn is set either to InvalidXLogRecPtr
 134  * if there are no WAL summary files in the input list, or to the first LSN
 135  * in the range that is not covered by a WAL summary file in the input list.
 136  */
 137 bool
 138 WalSummariesAreComplete(List *wslist, XLogRecPtr start_lsn,
 139                                                 XLogRecPtr end_lsn, XLogRecPtr *missing_lsn)
 140 {
 141         XLogRecPtr      current_lsn = start_lsn;
 142         ListCell   *lc;
 143
 144         /* Special case for empty list. */
 145         if (wslist == NIL)
 146         {
 147                 *missing_lsn = InvalidXLogRecPtr;
 148                 return false;
 149         }
 150
 151         /* Make a private copy of the list and sort it by start LSN. */
 152         wslist = list_copy(wslist);
 153         list_sort(wslist, ListComparatorForWalSummaryFiles);
 154
 155         /*
 156          * Consider summary files in order of increasing start_lsn, advancing the
 157          * known-summarized range from start_lsn toward end_lsn.
 158          *
 159          * Normally, the summary files should cover non-overlapping WAL ranges,
 160          * but this algorithm is intended to be correct even in case of overlap.
 161          */
 162         foreach(lc, wslist)
 163         {
 164                 WalSummaryFile *ws = lfirst(lc);
 165
 166                 if (ws->start_lsn > current_lsn)
 167                 {
 168                         /* We found a gap. */
 169                         break;
 170                 }
 171                 if (ws->end_lsn > current_lsn)
 172                 {
 173                         /*
 174                          * Next summary extends beyond end of previous summary, so extend
 175                          * the end of the range known to be summarized.
 176                          */
 177                         current_lsn = ws->end_lsn;
 178
 179                         /*
 180                          * If the range we know to be summarized has reached the required
 181                          * end LSN, we have proved completeness.
 182                          */
 183                         if (current_lsn >= end_lsn)
 184                                 return true;
 185                 }
 186         }
 187
 188         /*
 189          * We either ran out of summary files without reaching the end LSN, or we
 190          * hit a gap in the sequence that resulted in us bailing out of the loop
 191          * above.
 192          */
 193         *missing_lsn = current_lsn;
 194         return false;
 195 }
 196
 197 /*
 198  * Open a WAL summary file.
 199  *
 200  * This will throw an error in case of trouble. As an exception, if
 201  * missing_ok = true and the trouble is specifically that the file does
 202  * not exist, it will not throw an error and will return a value less than 0.
 203  */
 204 File
 205 OpenWalSummaryFile(WalSummaryFile *ws, bool missing_ok)
 206 {
 207         char            path[MAXPGPATH];
 208         File            file;
 209
 210         snprintf(path, MAXPGPATH,
 211                          XLOGDIR "/summaries/%08X%08X%08X%08X%08X.summary",
 212                          ws->tli,
 213                          LSN_FORMAT_ARGS(ws->start_lsn),
 214                          LSN_FORMAT_ARGS(ws->end_lsn));
 215
 216         file = PathNameOpenFile(path, O_RDONLY);
 217         if (file < 0 && (errno != EEXIST || !missing_ok))
 218                 ereport(ERROR,
 219                                 (errcode_for_file_access(),
 220                                  errmsg("could not open file \"%s\": %m", path)));
 221
 222         return file;
 223 }
 224
 225 /*
 226  * Remove a WAL summary file if the last modification time precedes the
 227  * cutoff time.
 228  */
 229 void
 230 RemoveWalSummaryIfOlderThan(WalSummaryFile *ws, time_t cutoff_time)
 231 {
 232         char            path[MAXPGPATH];
 233         struct stat statbuf;
 234
 235         snprintf(path, MAXPGPATH,
 236                          XLOGDIR "/summaries/%08X%08X%08X%08X%08X.summary",
 237                          ws->tli,
 238                          LSN_FORMAT_ARGS(ws->start_lsn),
 239                          LSN_FORMAT_ARGS(ws->end_lsn));
 240
 241         if (lstat(path, &statbuf) != 0)
 242         {
 243                 if (errno == ENOENT)
 244                         return;
 245                 ereport(ERROR,
 246                                 (errcode_for_file_access(),
 247                                  errmsg("could not stat file \"%s\": %m", path)));
 248         }
 249         if (statbuf.st_mtime >= cutoff_time)
 250                 return;
 251         if (unlink(path) != 0)
 252                 ereport(ERROR,
 253                                 (errcode_for_file_access(),
 254                                  errmsg("could not stat file \"%s\": %m", path)));
 255         ereport(DEBUG2,
 256                         (errmsg_internal("removing file \"%s\"", path)));
 257 }
 258
 259 /*
 260  * Test whether a filename looks like a WAL summary file.
 261  */
 262 static bool
 263 IsWalSummaryFilename(char *filename)
 264 {
 265         return strspn(filename, "0123456789ABCDEF") == 40 &&
 266                 strcmp(filename + 40, ".summary") == 0;
 267 }
 268
 269 /*
 270  * Data read callback for use with CreateBlockRefTableReader.
 271  */
 272 int
 273 ReadWalSummary(void *wal_summary_io, void *data, int length)
 274 {
 275         WalSummaryIO *io = wal_summary_io;
 276         int                     nbytes;
 277
 278         nbytes = FileRead(io->file, data, length, io->filepos,
 279                                           WAIT_EVENT_WAL_SUMMARY_READ);
 280         if (nbytes < 0)
 281                 ereport(ERROR,
 282                                 (errcode_for_file_access(),
 283                                  errmsg("could not read file \"%s\": %m",
 284                                                 FilePathName(io->file))));
 285
 286         io->filepos += nbytes;
 287         return nbytes;
 288 }
 289
 290 /*
 291  * Data write callback for use with WriteBlockRefTable.
 292  */
 293 int
 294 WriteWalSummary(void *wal_summary_io, void *data, int length)
 295 {
 296         WalSummaryIO *io = wal_summary_io;
 297         int                     nbytes;
 298
 299         nbytes = FileWrite(io->file, data, length, io->filepos,
 300                                            WAIT_EVENT_WAL_SUMMARY_WRITE);
 301         if (nbytes < 0)
 302                 ereport(ERROR,
 303                                 (errcode_for_file_access(),
 304                                  errmsg("could not write file \"%s\": %m",
 305                                                 FilePathName(io->file))));
 306         if (nbytes != length)
 307                 ereport(ERROR,
 308                                 (errcode_for_file_access(),
 309                                  errmsg("could not write file \"%s\": wrote only %d of %d bytes at offset %u",
 310                                                 FilePathName(io->file), nbytes,
 311                                                 length, (unsigned) io->filepos),
 312                                  errhint("Check free disk space.")));
 313
 314         io->filepos += nbytes;
 315         return nbytes;
 316 }
 317
 318 /*
 319  * Error-reporting callback for use with CreateBlockRefTableReader.
 320  */
 321 void
 322 ReportWalSummaryError(void *callback_arg, char *fmt,...)
 323 {
 324         StringInfoData buf;
 325         va_list         ap;
 326         int                     needed;
 327
 328         initStringInfo(&buf);
 329         for (;;)
 330         {
 331                 va_start(ap, fmt);
 332                 needed = appendStringInfoVA(&buf, fmt, ap);
 333                 va_end(ap);
 334                 if (needed == 0)
 335                         break;
 336                 enlargeStringInfo(&buf, needed);
 337         }
 338         ereport(ERROR,
 339                         errcode(ERRCODE_DATA_CORRUPTED),
 340                         errmsg_internal("%s", buf.data));
 341 }
 342
 343 /*
 344  * Comparator to sort a List of WalSummaryFile objects by start_lsn.
 345  */
 346 static int
 347 ListComparatorForWalSummaryFiles(const ListCell *a, const ListCell *b)
 348 {
 349         WalSummaryFile *ws1 = lfirst(a);
 350         WalSummaryFile *ws2 = lfirst(b);
 351
 352         return pg_cmp_u64(ws1->start_lsn, ws2->start_lsn);
 353 }