src/backend/backup/backup_manifest.c

   1 /*-------------------------------------------------------------------------
   2  *
   3  * backup_manifest.c
   4  *        code for generating and sending a backup manifest
   5  *
   6  * Portions Copyright (c) 2010-2024, PostgreSQL Global Development Group
   7  *
   8  * IDENTIFICATION
   9  *        src/backend/backup/backup_manifest.c
  10  *
  11  *-------------------------------------------------------------------------
  12  */
  13 #include "postgres.h"
  14
  15 #include "access/timeline.h"
  16 #include "access/xlog.h"
  17 #include "backup/backup_manifest.h"
  18 #include "backup/basebackup_sink.h"
  19 #include "common/relpath.h"
  20 #include "mb/pg_wchar.h"
  21 #include "utils/builtins.h"
  22 #include "utils/json.h"
  23
  24 static void AppendStringToManifest(backup_manifest_info *manifest, const char *s);
  25
  26 /*
  27  * Does the user want a backup manifest?
  28  *
  29  * It's simplest to always have a manifest_info object, so that we don't need
  30  * checks for NULL pointers in too many places. However, if the user doesn't
  31  * want a manifest, we set manifest->buffile to NULL.
  32  */
  33 static inline bool
  34 IsManifestEnabled(backup_manifest_info *manifest)
  35 {
  36         return (manifest->buffile != NULL);
  37 }
  38
  39 /*
  40  * Convenience macro for appending data to the backup manifest.
  41  */
  42 #define AppendToManifest(manifest, ...) \
  43         { \
  44                 char *_manifest_s = psprintf(__VA_ARGS__);      \
  45                 AppendStringToManifest(manifest, _manifest_s);  \
  46                 pfree(_manifest_s);     \
  47         }
  48
  49 /*
  50  * Initialize state so that we can construct a backup manifest.
  51  *
  52  * NB: Although the checksum type for the data files is configurable, the
  53  * checksum for the manifest itself always uses SHA-256. See comments in
  54  * SendBackupManifest.
  55  */
  56 void
  57 InitializeBackupManifest(backup_manifest_info *manifest,
  58                                                  backup_manifest_option want_manifest,
  59                                                  pg_checksum_type manifest_checksum_type)
  60 {
  61         memset(manifest, 0, sizeof(backup_manifest_info));
  62         manifest->checksum_type = manifest_checksum_type;
  63
  64         if (want_manifest == MANIFEST_OPTION_NO)
  65                 manifest->buffile = NULL;
  66         else
  67         {
  68                 manifest->buffile = BufFileCreateTemp(false);
  69                 manifest->manifest_ctx = pg_cryptohash_create(PG_SHA256);
  70                 if (pg_cryptohash_init(manifest->manifest_ctx) < 0)
  71                         elog(ERROR, "failed to initialize checksum of backup manifest: %s",
  72                                  pg_cryptohash_error(manifest->manifest_ctx));
  73         }
  74
  75         manifest->manifest_size = UINT64CONST(0);
  76         manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
  77         manifest->first_file = true;
  78         manifest->still_checksumming = true;
  79
  80         if (want_manifest != MANIFEST_OPTION_NO)
  81                 AppendToManifest(manifest,
  82                                                  "{ \"PostgreSQL-Backup-Manifest-Version\": 2,\n"
  83                                                  "\"System-Identifier\": " UINT64_FORMAT ",\n"
  84                                                  "\"Files\": [",
  85                                                  GetSystemIdentifier());
  86 }
  87
  88 /*
  89  * Free resources assigned to a backup manifest constructed.
  90  */
  91 void
  92 FreeBackupManifest(backup_manifest_info *manifest)
  93 {
  94         pg_cryptohash_free(manifest->manifest_ctx);
  95         manifest->manifest_ctx = NULL;
  96 }
  97
  98 /*
  99  * Add an entry to the backup manifest for a file.
 100  */
 101 void
 102 AddFileToBackupManifest(backup_manifest_info *manifest, Oid spcoid,
 103                                                 const char *pathname, size_t size, pg_time_t mtime,
 104                                                 pg_checksum_context *checksum_ctx)
 105 {
 106         char            pathbuf[MAXPGPATH];
 107         int                     pathlen;
 108         StringInfoData buf;
 109
 110         if (!IsManifestEnabled(manifest))
 111                 return;
 112
 113         /*
 114          * If this file is part of a tablespace, the pathname passed to this
 115          * function will be relative to the tar file that contains it. We want the
 116          * pathname relative to the data directory (ignoring the intermediate
 117          * symlink traversal).
 118          */
 119         if (OidIsValid(spcoid))
 120         {
 121                 snprintf(pathbuf, sizeof(pathbuf), "%s/%u/%s", PG_TBLSPC_DIR, spcoid,
 122                                  pathname);
 123                 pathname = pathbuf;
 124         }
 125
 126         /*
 127          * Each file's entry needs to be separated from any entry that follows by
 128          * a comma, but there's no comma before the first one or after the last
 129          * one. To make that work, adding a file to the manifest starts by
 130          * terminating the most recently added line, with a comma if appropriate,
 131          * but does not terminate the line inserted for this file.
 132          */
 133         initStringInfo(&buf);
 134         if (manifest->first_file)
 135         {
 136                 appendStringInfoChar(&buf, '\n');
 137                 manifest->first_file = false;
 138         }
 139         else
 140                 appendStringInfoString(&buf, ",\n");
 141
 142         /*
 143          * Write the relative pathname to this file out to the manifest. The
 144          * manifest is always stored in UTF-8, so we have to encode paths that are
 145          * not valid in that encoding.
 146          */
 147         pathlen = strlen(pathname);
 148         if (!manifest->force_encode &&
 149                 pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
 150         {
 151                 appendStringInfoString(&buf, "{ \"Path\": ");
 152                 escape_json_with_len(&buf, pathname, pathlen);
 153                 appendStringInfoString(&buf, ", ");
 154         }
 155         else
 156         {
 157                 appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
 158                 enlargeStringInfo(&buf, 2 * pathlen);
 159                 buf.len += hex_encode(pathname, pathlen,
 160                                                           &buf.data[buf.len]);
 161                 appendStringInfoString(&buf, "\", ");
 162         }
 163
 164         appendStringInfo(&buf, "\"Size\": %zu, ", size);
 165
 166         /*
 167          * Convert last modification time to a string and append it to the
 168          * manifest. Since it's not clear what time zone to use and since time
 169          * zone definitions can change, possibly causing confusion, use GMT
 170          * always.
 171          */
 172         appendStringInfoString(&buf, "\"Last-Modified\": \"");
 173         enlargeStringInfo(&buf, 128);
 174         buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
 175                                                    pg_gmtime(&mtime));
 176         appendStringInfoChar(&buf, '"');
 177
 178         /* Add checksum information. */
 179         if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
 180         {
 181                 uint8           checksumbuf[PG_CHECKSUM_MAX_LENGTH];
 182                 int                     checksumlen;
 183
 184                 checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
 185                 if (checksumlen < 0)
 186                         elog(ERROR, "could not finalize checksum of file \"%s\"",
 187                                  pathname);
 188
 189                 appendStringInfo(&buf,
 190                                                  ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
 191                                                  pg_checksum_type_name(checksum_ctx->type));
 192                 enlargeStringInfo(&buf, 2 * checksumlen);
 193                 buf.len += hex_encode((char *) checksumbuf, checksumlen,
 194                                                           &buf.data[buf.len]);
 195                 appendStringInfoChar(&buf, '"');
 196         }
 197
 198         /* Close out the object. */
 199         appendStringInfoString(&buf, " }");
 200
 201         /* OK, add it to the manifest. */
 202         AppendStringToManifest(manifest, buf.data);
 203
 204         /* Avoid leaking memory. */
 205         pfree(buf.data);
 206 }
 207
 208 /*
 209  * Add information about the WAL that will need to be replayed when restoring
 210  * this backup to the manifest.
 211  */
 212 void
 213 AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr,
 214                                                    TimeLineID starttli, XLogRecPtr endptr,
 215                                                    TimeLineID endtli)
 216 {
 217         List       *timelines;
 218         ListCell   *lc;
 219         bool            first_wal_range = true;
 220         bool            found_start_timeline = false;
 221
 222         if (!IsManifestEnabled(manifest))
 223                 return;
 224
 225         /* Terminate the list of files. */
 226         AppendStringToManifest(manifest, "\n],\n");
 227
 228         /* Read the timeline history for the ending timeline. */
 229         timelines = readTimeLineHistory(endtli);
 230
 231         /* Start a list of LSN ranges. */
 232         AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
 233
 234         foreach(lc, timelines)
 235         {
 236                 TimeLineHistoryEntry *entry = lfirst(lc);
 237                 XLogRecPtr      tl_beginptr;
 238
 239                 /*
 240                  * We only care about timelines that were active during the backup.
 241                  * Skip any that ended before the backup started. (Note that if
 242                  * entry->end is InvalidXLogRecPtr, it means that the timeline has not
 243                  * yet ended.)
 244                  */
 245                 if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
 246                         continue;
 247
 248                 /*
 249                  * Because the timeline history file lists newer timelines before
 250                  * older ones, the first timeline we encounter that is new enough to
 251                  * matter ought to match the ending timeline of the backup.
 252                  */
 253                 if (first_wal_range && endtli != entry->tli)
 254                         ereport(ERROR,
 255                                         errmsg("expected end timeline %u but found timeline %u",
 256                                                    starttli, entry->tli));
 257
 258                 /*
 259                  * If this timeline entry matches with the timeline on which the
 260                  * backup started, WAL needs to be checked from the start LSN of the
 261                  * backup.  If this entry refers to a newer timeline, WAL needs to be
 262                  * checked since the beginning of this timeline, so use the LSN where
 263                  * the timeline began.
 264                  */
 265                 if (starttli == entry->tli)
 266                         tl_beginptr = startptr;
 267                 else
 268                 {
 269                         tl_beginptr = entry->begin;
 270
 271                         /*
 272                          * If we reach a TLI that has no valid beginning LSN, there can't
 273                          * be any more timelines in the history after this point, so we'd
 274                          * better have arrived at the expected starting TLI. If not,
 275                          * something's gone horribly wrong.
 276                          */
 277                         if (XLogRecPtrIsInvalid(entry->begin))
 278                                 ereport(ERROR,
 279                                                 errmsg("expected start timeline %u but found timeline %u",
 280                                                            starttli, entry->tli));
 281                 }
 282
 283                 AppendToManifest(manifest,
 284                                                  "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
 285                                                  first_wal_range ? "" : ",\n",
 286                                                  entry->tli,
 287                                                  LSN_FORMAT_ARGS(tl_beginptr),
 288                                                  LSN_FORMAT_ARGS(endptr));
 289
 290                 if (starttli == entry->tli)
 291                 {
 292                         found_start_timeline = true;
 293                         break;
 294                 }
 295
 296                 endptr = entry->begin;
 297                 first_wal_range = false;
 298         }
 299
 300         /*
 301          * The last entry in the timeline history for the ending timeline should
 302          * be the ending timeline itself. Verify that this is what we observed.
 303          */
 304         if (!found_start_timeline)
 305                 ereport(ERROR,
 306                                 errmsg("start timeline %u not found in history of timeline %u",
 307                                            starttli, endtli));
 308
 309         /* Terminate the list of WAL ranges. */
 310         AppendStringToManifest(manifest, "\n],\n");
 311 }
 312
 313 /*
 314  * Finalize the backup manifest, and send it to the client.
 315  */
 316 void
 317 SendBackupManifest(backup_manifest_info *manifest, bbsink *sink)
 318 {
 319         uint8           checksumbuf[PG_SHA256_DIGEST_LENGTH];
 320         char            checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
 321         size_t          manifest_bytes_done = 0;
 322
 323         if (!IsManifestEnabled(manifest))
 324                 return;
 325
 326         /*
 327          * Append manifest checksum, so that the problems with the manifest itself
 328          * can be detected.
 329          *
 330          * We always use SHA-256 for this, regardless of what algorithm is chosen
 331          * for checksumming the files.  If we ever want to make the checksum
 332          * algorithm used for the manifest file variable, the client will need a
 333          * way to figure out which algorithm to use as close to the beginning of
 334          * the manifest file as possible, to avoid having to read the whole thing
 335          * twice.
 336          */
 337         manifest->still_checksumming = false;
 338         if (pg_cryptohash_final(manifest->manifest_ctx, checksumbuf,
 339                                                         sizeof(checksumbuf)) < 0)
 340                 elog(ERROR, "failed to finalize checksum of backup manifest: %s",
 341                          pg_cryptohash_error(manifest->manifest_ctx));
 342         AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
 343
 344         hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
 345         checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
 346
 347         AppendStringToManifest(manifest, checksumstringbuf);
 348         AppendStringToManifest(manifest, "\"}\n");
 349
 350         /*
 351          * We've written all the data to the manifest file.  Rewind the file so
 352          * that we can read it all back.
 353          */
 354         if (BufFileSeek(manifest->buffile, 0, 0, SEEK_SET))
 355                 ereport(ERROR,
 356                                 (errcode_for_file_access(),
 357                                  errmsg("could not rewind temporary file")));
 358
 359
 360         /*
 361          * Send the backup manifest.
 362          */
 363         bbsink_begin_manifest(sink);
 364         while (manifest_bytes_done < manifest->manifest_size)
 365         {
 366                 size_t          bytes_to_read;
 367
 368                 bytes_to_read = Min(sink->bbs_buffer_length,
 369                                                         manifest->manifest_size - manifest_bytes_done);
 370                 BufFileReadExact(manifest->buffile, sink->bbs_buffer, bytes_to_read);
 371                 bbsink_manifest_contents(sink, bytes_to_read);
 372                 manifest_bytes_done += bytes_to_read;
 373         }
 374         bbsink_end_manifest(sink);
 375
 376         /* Release resources */
 377         BufFileClose(manifest->buffile);
 378 }
 379
 380 /*
 381  * Append a cstring to the manifest.
 382  */
 383 static void
 384 AppendStringToManifest(backup_manifest_info *manifest, const char *s)
 385 {
 386         int                     len = strlen(s);
 387
 388         Assert(manifest != NULL);
 389         if (manifest->still_checksumming)
 390         {
 391                 if (pg_cryptohash_update(manifest->manifest_ctx, (uint8 *) s, len) < 0)
 392                         elog(ERROR, "failed to update checksum of backup manifest: %s",
 393                                  pg_cryptohash_error(manifest->manifest_ctx));
 394         }
 395         BufFileWrite(manifest->buffile, s, len);
 396         manifest->manifest_size += len;
 397 }