src/bin/pg_dump/pg_backup_files.c

   1 /*-------------------------------------------------------------------------
   2  *
   3  * pg_backup_files.c
   4  *
   5  *      This file is copied from the 'custom' format file, but dumps data into
   6  *      separate files, and the TOC into the 'main' file.
   7  *
   8  *      IT IS FOR DEMONSTRATION PURPOSES ONLY.
   9  *
  10  *      (and could probably be used as a basis for writing a tar file)
  11  *
  12  *      See the headers to pg_restore for more details.
  13  *
  14  * Copyright (c) 2000, Philip Warner
  15  *              Rights are granted to use this software in any way so long
  16  *              as this notice is not removed.
  17  *
  18  *      The author is not responsible for loss or damages that may
  19  *      result from it's use.
  20  *
  21  *
  22  * IDENTIFICATION
  23  *              $PostgreSQL$
  24  *
  25  *-------------------------------------------------------------------------
  26  */
  27
  28 #include "pg_backup_archiver.h"
  29
  30 static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
  31 static void _StartData(ArchiveHandle *AH, TocEntry *te);
  32 static size_t _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
  33 static void _EndData(ArchiveHandle *AH, TocEntry *te);
  34 static int      _WriteByte(ArchiveHandle *AH, const int i);
  35 static int      _ReadByte(ArchiveHandle *);
  36 static size_t _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
  37 static size_t _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
  38 static void _CloseArchive(ArchiveHandle *AH);
  39 static void _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt);
  40 static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
  41 static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
  42 static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
  43
  44 static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
  45 static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
  46 static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
  47 static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
  48
  49 #define K_STD_BUF_SIZE 1024
  50
  51 typedef struct
  52 {
  53         int                     hasSeek;
  54         pgoff_t         filePos;
  55         FILE       *blobToc;
  56 } lclContext;
  57
  58 typedef struct
  59 {
  60 #ifdef HAVE_LIBZ
  61         gzFile     *FH;
  62 #else
  63         FILE       *FH;
  64 #endif
  65         char       *filename;
  66 } lclTocEntry;
  67
  68 static const char *modulename = gettext_noop("file archiver");
  69 static void _LoadBlobs(ArchiveHandle *AH, RestoreOptions *ropt);
  70 static void _getBlobTocEntry(ArchiveHandle *AH, Oid *oid, char *fname);
  71
  72 /*
  73  *      Initializer
  74  */
  75 void
  76 InitArchiveFmt_Files(ArchiveHandle *AH)
  77 {
  78         lclContext *ctx;
  79
  80         /* Assuming static functions, this can be copied for each format. */
  81         AH->ArchiveEntryPtr = _ArchiveEntry;
  82         AH->StartDataPtr = _StartData;
  83         AH->WriteDataPtr = _WriteData;
  84         AH->EndDataPtr = _EndData;
  85         AH->WriteBytePtr = _WriteByte;
  86         AH->ReadBytePtr = _ReadByte;
  87         AH->WriteBufPtr = _WriteBuf;
  88         AH->ReadBufPtr = _ReadBuf;
  89         AH->ClosePtr = _CloseArchive;
  90         AH->PrintTocDataPtr = _PrintTocData;
  91         AH->ReadExtraTocPtr = _ReadExtraToc;
  92         AH->WriteExtraTocPtr = _WriteExtraToc;
  93         AH->PrintExtraTocPtr = _PrintExtraToc;
  94
  95         AH->StartBlobsPtr = _StartBlobs;
  96         AH->StartBlobPtr = _StartBlob;
  97         AH->EndBlobPtr = _EndBlob;
  98         AH->EndBlobsPtr = _EndBlobs;
  99
 100         /*
 101          * Set up some special context used in compressing data.
 102          */
 103         ctx = (lclContext *) calloc(1, sizeof(lclContext));
 104         AH->formatData = (void *) ctx;
 105         ctx->filePos = 0;
 106
 107         /* Initialize LO buffering */
 108         AH->lo_buf_size = LOBBUFSIZE;
 109         AH->lo_buf = (void *) malloc(LOBBUFSIZE);
 110         if (AH->lo_buf == NULL)
 111                 die_horribly(AH, modulename, "out of memory\n");
 112
 113         /*
 114          * Now open the TOC file
 115          */
 116         if (AH->mode == archModeWrite)
 117         {
 118
 119                 write_msg(modulename, "WARNING:\n"
 120                                   "  This format is for demonstration purposes; it is not intended for\n"
 121                                   "  normal use. Files will be written in the current working directory.\n");
 122
 123                 if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
 124                 {
 125                         AH->FH = fopen(AH->fSpec, PG_BINARY_W);
 126                         if (AH->FH == NULL)
 127                                 die_horribly(NULL, modulename, "could not open output file \"%s\": %s\n",
 128                                                          AH->fSpec, strerror(errno));
 129                 }
 130                 else
 131                 {
 132                         AH->FH = stdout;
 133                         if (AH->FH == NULL)
 134                                 die_horribly(NULL, modulename, "could not open output file: %s\n",
 135                                                          strerror(errno));
 136                 }
 137
 138                 ctx->hasSeek = checkSeek(AH->FH);
 139
 140                 if (AH->compression < 0 || AH->compression > 9)
 141                         AH->compression = Z_DEFAULT_COMPRESSION;
 142
 143
 144         }
 145         else
 146         {                                                       /* Read Mode */
 147
 148                 if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
 149                 {
 150                         AH->FH = fopen(AH->fSpec, PG_BINARY_R);
 151                         if (AH->FH == NULL)
 152                                 die_horribly(NULL, modulename, "could not open input file \"%s\": %s\n",
 153                                                          AH->fSpec, strerror(errno));
 154                 }
 155                 else
 156                 {
 157                         AH->FH = stdin;
 158                         if (AH->FH == NULL)
 159                                 die_horribly(NULL, modulename, "could not open input file: %s\n",
 160                                                          strerror(errno));
 161                 }
 162
 163                 ctx->hasSeek = checkSeek(AH->FH);
 164
 165                 ReadHead(AH);
 166                 ReadToc(AH);
 167                 /* Nothing else in the file... */
 168                 if (fclose(AH->FH) != 0)
 169                         die_horribly(AH, modulename, "could not close TOC file: %s\n", strerror(errno));
 170         }
 171 }
 172
 173 /*
 174  * - Start a new TOC entry
 175  *       Setup the output file name.
 176  */
 177 static void
 178 _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
 179 {
 180         lclTocEntry *ctx;
 181         char            fn[K_STD_BUF_SIZE];
 182
 183         ctx = (lclTocEntry *) calloc(1, sizeof(lclTocEntry));
 184         if (te->dataDumper)
 185         {
 186 #ifdef HAVE_LIBZ
 187                 if (AH->compression == 0)
 188                         sprintf(fn, "%d.dat", te->dumpId);
 189                 else
 190                         sprintf(fn, "%d.dat.gz", te->dumpId);
 191 #else
 192                 sprintf(fn, "%d.dat", te->dumpId);
 193 #endif
 194                 ctx->filename = strdup(fn);
 195         }
 196         else
 197         {
 198                 ctx->filename = NULL;
 199                 ctx->FH = NULL;
 200         }
 201         te->formatData = (void *) ctx;
 202 }
 203
 204 static void
 205 _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
 206 {
 207         lclTocEntry *ctx = (lclTocEntry *) te->formatData;
 208
 209         if (ctx->filename)
 210                 WriteStr(AH, ctx->filename);
 211         else
 212                 WriteStr(AH, "");
 213 }
 214
 215 static void
 216 _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
 217 {
 218         lclTocEntry *ctx = (lclTocEntry *) te->formatData;
 219
 220         if (ctx == NULL)
 221         {
 222                 ctx = (lclTocEntry *) calloc(1, sizeof(lclTocEntry));
 223                 te->formatData = (void *) ctx;
 224         }
 225
 226         ctx->filename = ReadStr(AH);
 227         if (strlen(ctx->filename) == 0)
 228         {
 229                 free(ctx->filename);
 230                 ctx->filename = NULL;
 231         }
 232         ctx->FH = NULL;
 233 }
 234
 235 static void
 236 _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
 237 {
 238         lclTocEntry *ctx = (lclTocEntry *) te->formatData;
 239
 240         if (AH->public.verbose)
 241                 ahprintf(AH, "-- File: %s\n", ctx->filename);
 242 }
 243
 244 static void
 245 _StartData(ArchiveHandle *AH, TocEntry *te)
 246 {
 247         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 248         char            fmode[10];
 249
 250         sprintf(fmode, "wb%d", AH->compression);
 251
 252 #ifdef HAVE_LIBZ
 253         tctx->FH = gzopen(tctx->filename, fmode);
 254 #else
 255         tctx->FH = fopen(tctx->filename, PG_BINARY_W);
 256 #endif
 257
 258         if (tctx->FH == NULL)
 259                 die_horribly(AH, modulename, "could not open output file \"%s\": %s\n",
 260                                          tctx->filename, strerror(errno));
 261 }
 262
 263 static size_t
 264 _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
 265 {
 266         lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
 267
 268         GZWRITE((void *) data, 1, dLen, tctx->FH);
 269
 270         return dLen;
 271 }
 272
 273 static void
 274 _EndData(ArchiveHandle *AH, TocEntry *te)
 275 {
 276         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 277
 278         /* Close the file */
 279         if (GZCLOSE(tctx->FH) != 0)
 280                 die_horribly(AH, modulename, "could not close data file\n");
 281
 282         tctx->FH = NULL;
 283 }
 284
 285 /*
 286  * Print data for a given file
 287  */
 288 static void
 289 _PrintFileData(ArchiveHandle *AH, char *filename, RestoreOptions *ropt)
 290 {
 291         char            buf[4096];
 292         size_t          cnt;
 293
 294         if (!filename)
 295                 return;
 296
 297 #ifdef HAVE_LIBZ
 298         AH->FH = gzopen(filename, "rb");
 299 #else
 300         AH->FH = fopen(filename, PG_BINARY_R);
 301 #endif
 302
 303         if (AH->FH == NULL)
 304                 die_horribly(AH, modulename, "could not open input file \"%s\": %s\n",
 305                                          filename, strerror(errno));
 306
 307         while ((cnt = GZREAD(buf, 1, 4095, AH->FH)) > 0)
 308         {
 309                 buf[cnt] = '\0';
 310                 ahwrite(buf, 1, cnt, AH);
 311         }
 312
 313         if (GZCLOSE(AH->FH) != 0)
 314                 die_horribly(AH, modulename, "could not close data file after reading\n");
 315 }
 316
 317
 318 /*
 319  * Print data for a given TOC entry
 320 */
 321 static void
 322 _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
 323 {
 324         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 325
 326         if (!tctx->filename)
 327                 return;
 328
 329         if (strcmp(te->desc, "BLOBS") == 0)
 330                 _LoadBlobs(AH, ropt);
 331         else
 332                 _PrintFileData(AH, tctx->filename, ropt);
 333 }
 334
 335 static void
 336 _getBlobTocEntry(ArchiveHandle *AH, Oid *oid, char fname[K_STD_BUF_SIZE])
 337 {
 338         lclContext *ctx = (lclContext *) AH->formatData;
 339         char            blobTe[K_STD_BUF_SIZE];
 340
 341         if (fgets(blobTe, sizeof(blobTe), ctx->blobToc) != NULL)
 342         {
 343                 size_t          fpos;
 344                 size_t          eos;
 345
 346                 *oid = atooid(blobTe);
 347
 348                 fpos = strcspn(blobTe, " ");
 349
 350                 strlcpy(fname, &blobTe[fpos + 1], K_STD_BUF_SIZE);
 351
 352                 eos = strlen(fname) - 1;
 353
 354                 if (fname[eos] == '\n')
 355                         fname[eos] = '\0';
 356         }
 357         else
 358         {
 359                 *oid = 0;
 360                 fname[0] = '\0';
 361         }
 362 }
 363
 364 static void
 365 _LoadBlobs(ArchiveHandle *AH, RestoreOptions *ropt)
 366 {
 367         Oid                     oid;
 368         lclContext *ctx = (lclContext *) AH->formatData;
 369         char            fname[K_STD_BUF_SIZE];
 370
 371         StartRestoreBlobs(AH);
 372
 373         ctx->blobToc = fopen("blobs.toc", PG_BINARY_R);
 374
 375         if (ctx->blobToc == NULL)
 376                 die_horribly(AH, modulename, "could not open large object TOC for input: %s\n", strerror(errno));
 377
 378         _getBlobTocEntry(AH, &oid, fname);
 379
 380         while (oid != 0)
 381         {
 382                 StartRestoreBlob(AH, oid);
 383                 _PrintFileData(AH, fname, ropt);
 384                 EndRestoreBlob(AH, oid);
 385                 _getBlobTocEntry(AH, &oid, fname);
 386         }
 387
 388         if (fclose(ctx->blobToc) != 0)
 389                 die_horribly(AH, modulename, "could not close large object TOC file: %s\n", strerror(errno));
 390
 391         EndRestoreBlobs(AH);
 392 }
 393
 394
 395 static int
 396 _WriteByte(ArchiveHandle *AH, const int i)
 397 {
 398         lclContext *ctx = (lclContext *) AH->formatData;
 399
 400         if (fputc(i, AH->FH) == EOF)
 401                 die_horribly(AH, modulename, "could not write byte\n");
 402
 403         ctx->filePos += 1;
 404
 405         return 1;
 406 }
 407
 408 static int
 409 _ReadByte(ArchiveHandle *AH)
 410 {
 411         lclContext *ctx = (lclContext *) AH->formatData;
 412         int                     res;
 413
 414         res = getc(AH->FH);
 415         if (res == EOF)
 416                 die_horribly(AH, modulename, "unexpected end of file\n");
 417         ctx->filePos += 1;
 418         return res;
 419 }
 420
 421 static size_t
 422 _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
 423 {
 424         lclContext *ctx = (lclContext *) AH->formatData;
 425         size_t          res;
 426
 427         res = fwrite(buf, 1, len, AH->FH);
 428         if (res != len)
 429                 die_horribly(AH, modulename, "could not write to output file: %s\n", strerror(errno));
 430
 431         ctx->filePos += res;
 432         return res;
 433 }
 434
 435 static size_t
 436 _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
 437 {
 438         lclContext *ctx = (lclContext *) AH->formatData;
 439         size_t          res;
 440
 441         res = fread(buf, 1, len, AH->FH);
 442         ctx->filePos += res;
 443         return res;
 444 }
 445
 446 static void
 447 _CloseArchive(ArchiveHandle *AH)
 448 {
 449         if (AH->mode == archModeWrite)
 450         {
 451                 WriteHead(AH);
 452                 WriteToc(AH);
 453                 if (fclose(AH->FH) != 0)
 454                         die_horribly(AH, modulename, "could not close TOC file: %s\n", strerror(errno));
 455                 WriteDataChunks(AH);
 456         }
 457
 458         AH->FH = NULL;
 459 }
 460
 461
 462
 463 /*
 464  * BLOB support
 465  */
 466
 467 /*
 468  * Called by the archiver when starting to save all BLOB DATA (not schema).
 469  * This routine should save whatever format-specific information is needed
 470  * to read the BLOBs back into memory.
 471  *
 472  * It is called just prior to the dumper's DataDumper routine.
 473  *
 474  * Optional, but strongly recommended.
 475  */
 476 static void
 477 _StartBlobs(ArchiveHandle *AH, TocEntry *te)
 478 {
 479         lclContext *ctx = (lclContext *) AH->formatData;
 480         char            fname[K_STD_BUF_SIZE];
 481
 482         sprintf(fname, "blobs.toc");
 483         ctx->blobToc = fopen(fname, PG_BINARY_W);
 484
 485         if (ctx->blobToc == NULL)
 486                 die_horribly(AH, modulename,
 487                 "could not open large object TOC for output: %s\n", strerror(errno));
 488 }
 489
 490 /*
 491  * Called by the archiver when the dumper calls StartBlob.
 492  *
 493  * Mandatory.
 494  *
 495  * Must save the passed OID for retrieval at restore-time.
 496  */
 497 static void
 498 _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
 499 {
 500         lclContext *ctx = (lclContext *) AH->formatData;
 501         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 502         char            fmode[10];
 503         char            fname[255];
 504         char       *sfx;
 505
 506         if (oid == 0)
 507                 die_horribly(AH, modulename, "invalid OID for large object (%u)\n", oid);
 508
 509         if (AH->compression != 0)
 510                 sfx = ".gz";
 511         else
 512                 sfx = "";
 513
 514         sprintf(fmode, "wb%d", AH->compression);
 515         sprintf(fname, "blob_%u.dat%s", oid, sfx);
 516
 517         fprintf(ctx->blobToc, "%u %s\n", oid, fname);
 518
 519 #ifdef HAVE_LIBZ
 520         tctx->FH = gzopen(fname, fmode);
 521 #else
 522         tctx->FH = fopen(fname, PG_BINARY_W);
 523 #endif
 524
 525         if (tctx->FH == NULL)
 526                 die_horribly(AH, modulename, "could not open large object file \"%s\" for input: %s\n",
 527                                          fname, strerror(errno));
 528 }
 529
 530 /*
 531  * Called by the archiver when the dumper calls EndBlob.
 532  *
 533  * Optional.
 534  */
 535 static void
 536 _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
 537 {
 538         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 539
 540         if (GZCLOSE(tctx->FH) != 0)
 541                 die_horribly(AH, modulename, "could not close large object file\n");
 542 }
 543
 544 /*
 545  * Called by the archiver when finishing saving all BLOB DATA.
 546  *
 547  * Optional.
 548  */
 549 static void
 550 _EndBlobs(ArchiveHandle *AH, TocEntry *te)
 551 {
 552         lclContext *ctx = (lclContext *) AH->formatData;
 553
 554         /* Write out a fake zero OID to mark end-of-blobs. */
 555         /* WriteInt(AH, 0); */
 556
 557         if (fclose(ctx->blobToc) != 0)
 558                 die_horribly(AH, modulename, "could not close large object TOC file: %s\n", strerror(errno));
 559 }