src/bin/pg_dump/pg_backup_files.c

   1 /*-------------------------------------------------------------------------
   2  *
   3  * pg_backup_files.c
   4  *
   5  *      This file is copied from the 'custom' format file, but dumps data into
   6  *      separate files, and the TOC into the 'main' file.
   7  *
   8  *      IT IS FOR DEMONSTRATION PURPOSES ONLY.
   9  *
  10  *      (and could probably be used as a basis for writing a tar file)
  11  *
  12  *      See the headers to pg_restore for more details.
  13  *
  14  * Copyright (c) 2000, Philip Warner
  15  *              Rights are granted to use this software in any way so long
  16  *              as this notice is not removed.
  17  *
  18  *      The author is not responsible for loss or damages that may
  19  *      result from it's use.
  20  *
  21  *
  22  * IDENTIFICATION
  23  *              $PostgreSQL$
  24  *
  25  *-------------------------------------------------------------------------
  26  */
  27
  28 #include "pg_backup_archiver.h"
  29
  30 static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
  31 static void _StartData(ArchiveHandle *AH, TocEntry *te);
  32 static size_t _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
  33 static void _EndData(ArchiveHandle *AH, TocEntry *te);
  34 static int      _WriteByte(ArchiveHandle *AH, const int i);
  35 static int      _ReadByte(ArchiveHandle *);
  36 static size_t _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
  37 static size_t _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
  38 static void _CloseArchive(ArchiveHandle *AH);
  39 static void _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt);
  40 static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
  41 static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
  42 static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
  43
  44 static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
  45 static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
  46 static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
  47 static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
  48
  49 #define K_STD_BUF_SIZE 1024
  50
  51 typedef struct
  52 {
  53         int                     hasSeek;
  54         pgoff_t         filePos;
  55         FILE       *blobToc;
  56 } lclContext;
  57
  58 typedef struct
  59 {
  60 #ifdef HAVE_LIBZ
  61         gzFile     *FH;
  62 #else
  63         FILE       *FH;
  64 #endif
  65         char       *filename;
  66 } lclTocEntry;
  67
  68 static const char *modulename = gettext_noop("file archiver");
  69 static void _LoadBlobs(ArchiveHandle *AH, RestoreOptions *ropt);
  70 static void _getBlobTocEntry(ArchiveHandle *AH, Oid *oid, char *fname);
  71
  72 /*
  73  *      Initializer
  74  */
  75 void
  76 InitArchiveFmt_Files(ArchiveHandle *AH)
  77 {
  78         lclContext *ctx;
  79
  80         /* Assuming static functions, this can be copied for each format. */
  81         AH->ArchiveEntryPtr = _ArchiveEntry;
  82         AH->StartDataPtr = _StartData;
  83         AH->WriteDataPtr = _WriteData;
  84         AH->EndDataPtr = _EndData;
  85         AH->WriteBytePtr = _WriteByte;
  86         AH->ReadBytePtr = _ReadByte;
  87         AH->WriteBufPtr = _WriteBuf;
  88         AH->ReadBufPtr = _ReadBuf;
  89         AH->ClosePtr = _CloseArchive;
  90         AH->ReopenPtr = NULL;
  91         AH->PrintTocDataPtr = _PrintTocData;
  92         AH->ReadExtraTocPtr = _ReadExtraToc;
  93         AH->WriteExtraTocPtr = _WriteExtraToc;
  94         AH->PrintExtraTocPtr = _PrintExtraToc;
  95
  96         AH->StartBlobsPtr = _StartBlobs;
  97         AH->StartBlobPtr = _StartBlob;
  98         AH->EndBlobPtr = _EndBlob;
  99         AH->EndBlobsPtr = _EndBlobs;
 100         AH->ClonePtr = NULL;
 101         AH->DeClonePtr = NULL;
 102
 103         /*
 104          * Set up some special context used in compressing data.
 105          */
 106         ctx = (lclContext *) calloc(1, sizeof(lclContext));
 107         AH->formatData = (void *) ctx;
 108         ctx->filePos = 0;
 109
 110         /* Initialize LO buffering */
 111         AH->lo_buf_size = LOBBUFSIZE;
 112         AH->lo_buf = (void *) malloc(LOBBUFSIZE);
 113         if (AH->lo_buf == NULL)
 114                 die_horribly(AH, modulename, "out of memory\n");
 115
 116         /*
 117          * Now open the TOC file
 118          */
 119         if (AH->mode == archModeWrite)
 120         {
 121
 122                 write_msg(modulename, "WARNING:\n"
 123                                   "  This format is for demonstration purposes; it is not intended for\n"
 124                                   "  normal use. Files will be written in the current working directory.\n");
 125
 126                 if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
 127                 {
 128                         AH->FH = fopen(AH->fSpec, PG_BINARY_W);
 129                         if (AH->FH == NULL)
 130                                 die_horribly(NULL, modulename, "could not open output file \"%s\": %s\n",
 131                                                          AH->fSpec, strerror(errno));
 132                 }
 133                 else
 134                 {
 135                         AH->FH = stdout;
 136                         if (AH->FH == NULL)
 137                                 die_horribly(NULL, modulename, "could not open output file: %s\n",
 138                                                          strerror(errno));
 139                 }
 140
 141                 ctx->hasSeek = checkSeek(AH->FH);
 142
 143                 if (AH->compression < 0 || AH->compression > 9)
 144                         AH->compression = Z_DEFAULT_COMPRESSION;
 145
 146
 147         }
 148         else
 149         {                                                       /* Read Mode */
 150
 151                 if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
 152                 {
 153                         AH->FH = fopen(AH->fSpec, PG_BINARY_R);
 154                         if (AH->FH == NULL)
 155                                 die_horribly(NULL, modulename, "could not open input file \"%s\": %s\n",
 156                                                          AH->fSpec, strerror(errno));
 157                 }
 158                 else
 159                 {
 160                         AH->FH = stdin;
 161                         if (AH->FH == NULL)
 162                                 die_horribly(NULL, modulename, "could not open input file: %s\n",
 163                                                          strerror(errno));
 164                 }
 165
 166                 ctx->hasSeek = checkSeek(AH->FH);
 167
 168                 ReadHead(AH);
 169                 ReadToc(AH);
 170                 /* Nothing else in the file... */
 171                 if (fclose(AH->FH) != 0)
 172                         die_horribly(AH, modulename, "could not close TOC file: %s\n", strerror(errno));
 173         }
 174 }
 175
 176 /*
 177  * - Start a new TOC entry
 178  *       Setup the output file name.
 179  */
 180 static void
 181 _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
 182 {
 183         lclTocEntry *ctx;
 184         char            fn[K_STD_BUF_SIZE];
 185
 186         ctx = (lclTocEntry *) calloc(1, sizeof(lclTocEntry));
 187         if (te->dataDumper)
 188         {
 189 #ifdef HAVE_LIBZ
 190                 if (AH->compression == 0)
 191                         sprintf(fn, "%d.dat", te->dumpId);
 192                 else
 193                         sprintf(fn, "%d.dat.gz", te->dumpId);
 194 #else
 195                 sprintf(fn, "%d.dat", te->dumpId);
 196 #endif
 197                 ctx->filename = strdup(fn);
 198         }
 199         else
 200         {
 201                 ctx->filename = NULL;
 202                 ctx->FH = NULL;
 203         }
 204         te->formatData = (void *) ctx;
 205 }
 206
 207 static void
 208 _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
 209 {
 210         lclTocEntry *ctx = (lclTocEntry *) te->formatData;
 211
 212         if (ctx->filename)
 213                 WriteStr(AH, ctx->filename);
 214         else
 215                 WriteStr(AH, "");
 216 }
 217
 218 static void
 219 _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
 220 {
 221         lclTocEntry *ctx = (lclTocEntry *) te->formatData;
 222
 223         if (ctx == NULL)
 224         {
 225                 ctx = (lclTocEntry *) calloc(1, sizeof(lclTocEntry));
 226                 te->formatData = (void *) ctx;
 227         }
 228
 229         ctx->filename = ReadStr(AH);
 230         if (strlen(ctx->filename) == 0)
 231         {
 232                 free(ctx->filename);
 233                 ctx->filename = NULL;
 234         }
 235         ctx->FH = NULL;
 236 }
 237
 238 static void
 239 _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
 240 {
 241         lclTocEntry *ctx = (lclTocEntry *) te->formatData;
 242
 243         if (AH->public.verbose)
 244                 ahprintf(AH, "-- File: %s\n", ctx->filename);
 245 }
 246
 247 static void
 248 _StartData(ArchiveHandle *AH, TocEntry *te)
 249 {
 250         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 251         char            fmode[10];
 252
 253         sprintf(fmode, "wb%d", AH->compression);
 254
 255 #ifdef HAVE_LIBZ
 256         tctx->FH = gzopen(tctx->filename, fmode);
 257 #else
 258         tctx->FH = fopen(tctx->filename, PG_BINARY_W);
 259 #endif
 260
 261         if (tctx->FH == NULL)
 262                 die_horribly(AH, modulename, "could not open output file \"%s\": %s\n",
 263                                          tctx->filename, strerror(errno));
 264 }
 265
 266 static size_t
 267 _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
 268 {
 269         lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
 270
 271         GZWRITE((void *) data, 1, dLen, tctx->FH);
 272
 273         return dLen;
 274 }
 275
 276 static void
 277 _EndData(ArchiveHandle *AH, TocEntry *te)
 278 {
 279         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 280
 281         /* Close the file */
 282         if (GZCLOSE(tctx->FH) != 0)
 283                 die_horribly(AH, modulename, "could not close data file\n");
 284
 285         tctx->FH = NULL;
 286 }
 287
 288 /*
 289  * Print data for a given file
 290  */
 291 static void
 292 _PrintFileData(ArchiveHandle *AH, char *filename, RestoreOptions *ropt)
 293 {
 294         char            buf[4096];
 295         size_t          cnt;
 296
 297         if (!filename)
 298                 return;
 299
 300 #ifdef HAVE_LIBZ
 301         AH->FH = gzopen(filename, "rb");
 302 #else
 303         AH->FH = fopen(filename, PG_BINARY_R);
 304 #endif
 305
 306         if (AH->FH == NULL)
 307                 die_horribly(AH, modulename, "could not open input file \"%s\": %s\n",
 308                                          filename, strerror(errno));
 309
 310         while ((cnt = GZREAD(buf, 1, 4095, AH->FH)) > 0)
 311         {
 312                 buf[cnt] = '\0';
 313                 ahwrite(buf, 1, cnt, AH);
 314         }
 315
 316         if (GZCLOSE(AH->FH) != 0)
 317                 die_horribly(AH, modulename, "could not close data file after reading\n");
 318 }
 319
 320
 321 /*
 322  * Print data for a given TOC entry
 323 */
 324 static void
 325 _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
 326 {
 327         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 328
 329         if (!tctx->filename)
 330                 return;
 331
 332         if (strcmp(te->desc, "BLOBS") == 0)
 333                 _LoadBlobs(AH, ropt);
 334         else
 335                 _PrintFileData(AH, tctx->filename, ropt);
 336 }
 337
 338 static void
 339 _getBlobTocEntry(ArchiveHandle *AH, Oid *oid, char fname[K_STD_BUF_SIZE])
 340 {
 341         lclContext *ctx = (lclContext *) AH->formatData;
 342         char            blobTe[K_STD_BUF_SIZE];
 343
 344         if (fgets(blobTe, sizeof(blobTe), ctx->blobToc) != NULL)
 345         {
 346                 size_t          fpos;
 347                 size_t          eos;
 348
 349                 *oid = atooid(blobTe);
 350
 351                 fpos = strcspn(blobTe, " ");
 352
 353                 strlcpy(fname, &blobTe[fpos + 1], K_STD_BUF_SIZE);
 354
 355                 eos = strlen(fname) - 1;
 356
 357                 if (fname[eos] == '\n')
 358                         fname[eos] = '\0';
 359         }
 360         else
 361         {
 362                 *oid = 0;
 363                 fname[0] = '\0';
 364         }
 365 }
 366
 367 static void
 368 _LoadBlobs(ArchiveHandle *AH, RestoreOptions *ropt)
 369 {
 370         Oid                     oid;
 371         lclContext *ctx = (lclContext *) AH->formatData;
 372         char            fname[K_STD_BUF_SIZE];
 373
 374         StartRestoreBlobs(AH);
 375
 376         ctx->blobToc = fopen("blobs.toc", PG_BINARY_R);
 377
 378         if (ctx->blobToc == NULL)
 379                 die_horribly(AH, modulename, "could not open large object TOC for input: %s\n", strerror(errno));
 380
 381         _getBlobTocEntry(AH, &oid, fname);
 382
 383         while (oid != 0)
 384         {
 385                 StartRestoreBlob(AH, oid);
 386                 _PrintFileData(AH, fname, ropt);
 387                 EndRestoreBlob(AH, oid);
 388                 _getBlobTocEntry(AH, &oid, fname);
 389         }
 390
 391         if (fclose(ctx->blobToc) != 0)
 392                 die_horribly(AH, modulename, "could not close large object TOC file: %s\n", strerror(errno));
 393
 394         EndRestoreBlobs(AH);
 395 }
 396
 397
 398 static int
 399 _WriteByte(ArchiveHandle *AH, const int i)
 400 {
 401         lclContext *ctx = (lclContext *) AH->formatData;
 402
 403         if (fputc(i, AH->FH) == EOF)
 404                 die_horribly(AH, modulename, "could not write byte\n");
 405
 406         ctx->filePos += 1;
 407
 408         return 1;
 409 }
 410
 411 static int
 412 _ReadByte(ArchiveHandle *AH)
 413 {
 414         lclContext *ctx = (lclContext *) AH->formatData;
 415         int                     res;
 416
 417         res = getc(AH->FH);
 418         if (res == EOF)
 419                 die_horribly(AH, modulename, "unexpected end of file\n");
 420         ctx->filePos += 1;
 421         return res;
 422 }
 423
 424 static size_t
 425 _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
 426 {
 427         lclContext *ctx = (lclContext *) AH->formatData;
 428         size_t          res;
 429
 430         res = fwrite(buf, 1, len, AH->FH);
 431         if (res != len)
 432                 die_horribly(AH, modulename, "could not write to output file: %s\n", strerror(errno));
 433
 434         ctx->filePos += res;
 435         return res;
 436 }
 437
 438 static size_t
 439 _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
 440 {
 441         lclContext *ctx = (lclContext *) AH->formatData;
 442         size_t          res;
 443
 444         res = fread(buf, 1, len, AH->FH);
 445         ctx->filePos += res;
 446         return res;
 447 }
 448
 449 static void
 450 _CloseArchive(ArchiveHandle *AH)
 451 {
 452         if (AH->mode == archModeWrite)
 453         {
 454                 WriteHead(AH);
 455                 WriteToc(AH);
 456                 if (fclose(AH->FH) != 0)
 457                         die_horribly(AH, modulename, "could not close TOC file: %s\n", strerror(errno));
 458                 WriteDataChunks(AH);
 459         }
 460
 461         AH->FH = NULL;
 462 }
 463
 464
 465
 466 /*
 467  * BLOB support
 468  */
 469
 470 /*
 471  * Called by the archiver when starting to save all BLOB DATA (not schema).
 472  * This routine should save whatever format-specific information is needed
 473  * to read the BLOBs back into memory.
 474  *
 475  * It is called just prior to the dumper's DataDumper routine.
 476  *
 477  * Optional, but strongly recommended.
 478  */
 479 static void
 480 _StartBlobs(ArchiveHandle *AH, TocEntry *te)
 481 {
 482         lclContext *ctx = (lclContext *) AH->formatData;
 483         char            fname[K_STD_BUF_SIZE];
 484
 485         sprintf(fname, "blobs.toc");
 486         ctx->blobToc = fopen(fname, PG_BINARY_W);
 487
 488         if (ctx->blobToc == NULL)
 489                 die_horribly(AH, modulename,
 490                 "could not open large object TOC for output: %s\n", strerror(errno));
 491 }
 492
 493 /*
 494  * Called by the archiver when the dumper calls StartBlob.
 495  *
 496  * Mandatory.
 497  *
 498  * Must save the passed OID for retrieval at restore-time.
 499  */
 500 static void
 501 _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
 502 {
 503         lclContext *ctx = (lclContext *) AH->formatData;
 504         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 505         char            fmode[10];
 506         char            fname[255];
 507         char       *sfx;
 508
 509         if (oid == 0)
 510                 die_horribly(AH, modulename, "invalid OID for large object (%u)\n", oid);
 511
 512         if (AH->compression != 0)
 513                 sfx = ".gz";
 514         else
 515                 sfx = "";
 516
 517         sprintf(fmode, "wb%d", AH->compression);
 518         sprintf(fname, "blob_%u.dat%s", oid, sfx);
 519
 520         fprintf(ctx->blobToc, "%u %s\n", oid, fname);
 521
 522 #ifdef HAVE_LIBZ
 523         tctx->FH = gzopen(fname, fmode);
 524 #else
 525         tctx->FH = fopen(fname, PG_BINARY_W);
 526 #endif
 527
 528         if (tctx->FH == NULL)
 529                 die_horribly(AH, modulename, "could not open large object file \"%s\" for input: %s\n",
 530                                          fname, strerror(errno));
 531 }
 532
 533 /*
 534  * Called by the archiver when the dumper calls EndBlob.
 535  *
 536  * Optional.
 537  */
 538 static void
 539 _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
 540 {
 541         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 542
 543         if (GZCLOSE(tctx->FH) != 0)
 544                 die_horribly(AH, modulename, "could not close large object file\n");
 545 }
 546
 547 /*
 548  * Called by the archiver when finishing saving all BLOB DATA.
 549  *
 550  * Optional.
 551  */
 552 static void
 553 _EndBlobs(ArchiveHandle *AH, TocEntry *te)
 554 {
 555         lclContext *ctx = (lclContext *) AH->formatData;
 556
 557         /* Write out a fake zero OID to mark end-of-blobs. */
 558         /* WriteInt(AH, 0); */
 559
 560         if (fclose(ctx->blobToc) != 0)
 561                 die_horribly(AH, modulename, "could not close large object TOC file: %s\n", strerror(errno));
 562 }