src/bin/pg_dump/pg_backup_tar.c

   1 /*-------------------------------------------------------------------------
   2  *
   3  * pg_backup_tar.c
   4  *
   5  *      This file is copied from the 'files' format file, but dumps data into
   6  *      one temp file then sends it to the output TAR archive.
   7  *
   8  *      The tar format also includes a 'restore.sql' script which is there for
   9  *      the benefit of humans. This script is never used by pg_restore.
  10  *
  11  *      NOTE: If you untar the created 'tar' file, the resulting files are
  12  *      compatible with the 'directory' format. Please keep the two formats in
  13  *      sync.
  14  *
  15  *      See the headers to pg_backup_directory & pg_restore for more details.
  16  *
  17  * Copyright (c) 2000, Philip Warner
  18  *              Rights are granted to use this software in any way so long
  19  *              as this notice is not removed.
  20  *
  21  *      The author is not responsible for loss or damages that may
  22  *      result from its use.
  23  *
  24  *
  25  * IDENTIFICATION
  26  *              src/bin/pg_dump/pg_backup_tar.c
  27  *
  28  *-------------------------------------------------------------------------
  29  */
  30 #include "postgres_fe.h"
  31
  32 #include <sys/stat.h>
  33 #include <ctype.h>
  34 #include <limits.h>
  35 #include <unistd.h>
  36
  37 #include "common/file_utils.h"
  38 #include "fe_utils/string_utils.h"
  39 #include "pg_backup_archiver.h"
  40 #include "pg_backup_tar.h"
  41 #include "pg_backup_utils.h"
  42 #include "pgtar.h"
  43
  44 static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
  45 static void _StartData(ArchiveHandle *AH, TocEntry *te);
  46 static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
  47 static void _EndData(ArchiveHandle *AH, TocEntry *te);
  48 static int      _WriteByte(ArchiveHandle *AH, const int i);
  49 static int      _ReadByte(ArchiveHandle *AH);
  50 static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
  51 static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
  52 static void _CloseArchive(ArchiveHandle *AH);
  53 static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
  54 static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
  55 static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
  56 static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
  57
  58 static void _StartLOs(ArchiveHandle *AH, TocEntry *te);
  59 static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
  60 static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
  61 static void _EndLOs(ArchiveHandle *AH, TocEntry *te);
  62
  63 #define K_STD_BUF_SIZE 1024
  64
  65
  66 typedef struct
  67 {
  68         FILE       *nFH;
  69         FILE       *tarFH;
  70         FILE       *tmpFH;
  71         char       *targetFile;
  72         char            mode;
  73         pgoff_t         pos;
  74         pgoff_t         fileLen;
  75         ArchiveHandle *AH;
  76 } TAR_MEMBER;
  77
  78 typedef struct
  79 {
  80         int                     hasSeek;
  81         pgoff_t         filePos;
  82         TAR_MEMBER *loToc;
  83         FILE       *tarFH;
  84         pgoff_t         tarFHpos;
  85         pgoff_t         tarNextMember;
  86         TAR_MEMBER *FH;
  87         int                     isSpecialScript;
  88         TAR_MEMBER *scriptTH;
  89 } lclContext;
  90
  91 typedef struct
  92 {
  93         TAR_MEMBER *TH;
  94         char       *filename;
  95 } lclTocEntry;
  96
  97 static void _LoadLOs(ArchiveHandle *AH, TocEntry *te);
  98
  99 static TAR_MEMBER *tarOpen(ArchiveHandle *AH, const char *filename, char mode);
 100 static void tarClose(ArchiveHandle *AH, TAR_MEMBER *th);
 101
 102 #ifdef __NOT_USED__
 103 static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
 104 #endif
 105 static int      tarPrintf(TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(2, 3);
 106
 107 static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
 108 static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
 109 static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
 110 static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
 111 static void _tarWriteHeader(TAR_MEMBER *th);
 112 static int      _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th);
 113 static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh);
 114
 115 static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len);
 116
 117 /*
 118  *      Initializer
 119  */
 120 void
 121 InitArchiveFmt_Tar(ArchiveHandle *AH)
 122 {
 123         lclContext *ctx;
 124
 125         /* Assuming static functions, this can be copied for each format. */
 126         AH->ArchiveEntryPtr = _ArchiveEntry;
 127         AH->StartDataPtr = _StartData;
 128         AH->WriteDataPtr = _WriteData;
 129         AH->EndDataPtr = _EndData;
 130         AH->WriteBytePtr = _WriteByte;
 131         AH->ReadBytePtr = _ReadByte;
 132         AH->WriteBufPtr = _WriteBuf;
 133         AH->ReadBufPtr = _ReadBuf;
 134         AH->ClosePtr = _CloseArchive;
 135         AH->ReopenPtr = NULL;
 136         AH->PrintTocDataPtr = _PrintTocData;
 137         AH->ReadExtraTocPtr = _ReadExtraToc;
 138         AH->WriteExtraTocPtr = _WriteExtraToc;
 139         AH->PrintExtraTocPtr = _PrintExtraToc;
 140
 141         AH->StartLOsPtr = _StartLOs;
 142         AH->StartLOPtr = _StartLO;
 143         AH->EndLOPtr = _EndLO;
 144         AH->EndLOsPtr = _EndLOs;
 145         AH->ClonePtr = NULL;
 146         AH->DeClonePtr = NULL;
 147
 148         AH->WorkerJobDumpPtr = NULL;
 149         AH->WorkerJobRestorePtr = NULL;
 150
 151         /*
 152          * Set up some special context used in compressing data.
 153          */
 154         ctx = pg_malloc0_object(lclContext);
 155         AH->formatData = ctx;
 156         ctx->filePos = 0;
 157         ctx->isSpecialScript = 0;
 158
 159         /*
 160          * Now open the tar file, and load the TOC if we're in read mode.
 161          */
 162         if (AH->mode == archModeWrite)
 163         {
 164                 if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
 165                 {
 166                         ctx->tarFH = fopen(AH->fSpec, PG_BINARY_W);
 167                         if (ctx->tarFH == NULL)
 168                                 pg_fatal("could not open TOC file \"%s\" for output: %m",
 169                                                  AH->fSpec);
 170                 }
 171                 else
 172                 {
 173                         ctx->tarFH = stdout;
 174                         if (ctx->tarFH == NULL)
 175                                 pg_fatal("could not open TOC file for output: %m");
 176                 }
 177
 178                 ctx->tarFHpos = 0;
 179
 180                 /*
 181                  * Make unbuffered since we will dup() it, and the buffers screw each
 182                  * other
 183                  */
 184                 /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
 185
 186                 ctx->hasSeek = checkSeek(ctx->tarFH);
 187
 188                 /*
 189                  * We don't support compression because reading the files back is not
 190                  * possible since gzdopen uses buffered IO which totally screws file
 191                  * positioning.
 192                  */
 193                 if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
 194                         pg_fatal("compression is not supported by tar archive format");
 195         }
 196         else
 197         {                                                       /* Read Mode */
 198                 if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
 199                 {
 200                         ctx->tarFH = fopen(AH->fSpec, PG_BINARY_R);
 201                         if (ctx->tarFH == NULL)
 202                                 pg_fatal("could not open TOC file \"%s\" for input: %m",
 203                                                  AH->fSpec);
 204                 }
 205                 else
 206                 {
 207                         ctx->tarFH = stdin;
 208                         if (ctx->tarFH == NULL)
 209                                 pg_fatal("could not open TOC file for input: %m");
 210                 }
 211
 212                 /*
 213                  * Make unbuffered since we will dup() it, and the buffers screw each
 214                  * other
 215                  */
 216                 /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
 217
 218                 ctx->tarFHpos = 0;
 219
 220                 ctx->hasSeek = checkSeek(ctx->tarFH);
 221
 222                 ctx->FH = tarOpen(AH, "toc.dat", 'r');
 223                 ReadHead(AH);
 224                 ReadToc(AH);
 225                 tarClose(AH, ctx->FH);  /* Nothing else in the file... */
 226         }
 227 }
 228
 229 /*
 230  * - Start a new TOC entry
 231  *       Setup the output file name.
 232  */
 233 static void
 234 _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
 235 {
 236         lclTocEntry *ctx;
 237         char            fn[K_STD_BUF_SIZE];
 238
 239         ctx = pg_malloc0_object(lclTocEntry);
 240         if (te->dataDumper != NULL)
 241         {
 242                 snprintf(fn, sizeof(fn), "%d.dat", te->dumpId);
 243                 ctx->filename = pg_strdup(fn);
 244         }
 245         else
 246         {
 247                 ctx->filename = NULL;
 248                 ctx->TH = NULL;
 249         }
 250         te->formatData = ctx;
 251 }
 252
 253 static void
 254 _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
 255 {
 256         lclTocEntry *ctx = (lclTocEntry *) te->formatData;
 257
 258         if (ctx->filename)
 259                 WriteStr(AH, ctx->filename);
 260         else
 261                 WriteStr(AH, "");
 262 }
 263
 264 static void
 265 _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
 266 {
 267         lclTocEntry *ctx = (lclTocEntry *) te->formatData;
 268
 269         if (ctx == NULL)
 270         {
 271                 ctx = pg_malloc0_object(lclTocEntry);
 272                 te->formatData = ctx;
 273         }
 274
 275         ctx->filename = ReadStr(AH);
 276         if (strlen(ctx->filename) == 0)
 277         {
 278                 free(ctx->filename);
 279                 ctx->filename = NULL;
 280         }
 281         ctx->TH = NULL;
 282 }
 283
 284 static void
 285 _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
 286 {
 287         lclTocEntry *ctx = (lclTocEntry *) te->formatData;
 288
 289         if (AH->public.verbose && ctx->filename != NULL)
 290                 ahprintf(AH, "-- File: %s\n", ctx->filename);
 291 }
 292
 293 static void
 294 _StartData(ArchiveHandle *AH, TocEntry *te)
 295 {
 296         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 297
 298         tctx->TH = tarOpen(AH, tctx->filename, 'w');
 299 }
 300
 301 static TAR_MEMBER *
 302 tarOpen(ArchiveHandle *AH, const char *filename, char mode)
 303 {
 304         lclContext *ctx = (lclContext *) AH->formatData;
 305         TAR_MEMBER *tm;
 306
 307         if (mode == 'r')
 308         {
 309                 tm = _tarPositionTo(AH, filename);
 310                 if (!tm)                                /* Not found */
 311                 {
 312                         if (filename)
 313                         {
 314                                 /*
 315                                  * Couldn't find the requested file. Future: do SEEK(0) and
 316                                  * retry.
 317                                  */
 318                                 pg_fatal("could not find file \"%s\" in archive", filename);
 319                         }
 320                         else
 321                         {
 322                                 /* Any file OK, none left, so return NULL */
 323                                 return NULL;
 324                         }
 325                 }
 326
 327                 if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
 328                         tm->nFH = ctx->tarFH;
 329                 else
 330                         pg_fatal("compression is not supported by tar archive format");
 331         }
 332         else
 333         {
 334                 int                     old_umask;
 335
 336                 tm = pg_malloc0_object(TAR_MEMBER);
 337
 338                 /*
 339                  * POSIX does not require, but permits, tmpfile() to restrict file
 340                  * permissions.  Given an OS crash after we write data, the filesystem
 341                  * might retain the data but forget tmpfile()'s unlink().  If so, the
 342                  * file mode protects confidentiality of the data written.
 343                  */
 344                 old_umask = umask(S_IRWXG | S_IRWXO);
 345
 346 #ifndef WIN32
 347                 tm->tmpFH = tmpfile();
 348 #else
 349
 350                 /*
 351                  * On WIN32, tmpfile() generates a filename in the root directory,
 352                  * which requires administrative permissions on certain systems. Loop
 353                  * until we find a unique file name we can create.
 354                  */
 355                 while (1)
 356                 {
 357                         char       *name;
 358                         int                     fd;
 359
 360                         name = _tempnam(NULL, "pg_temp_");
 361                         if (name == NULL)
 362                                 break;
 363                         fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY |
 364                                           O_TEMPORARY, S_IRUSR | S_IWUSR);
 365                         free(name);
 366
 367                         if (fd != -1)           /* created a file */
 368                         {
 369                                 tm->tmpFH = fdopen(fd, "w+b");
 370                                 break;
 371                         }
 372                         else if (errno != EEXIST)       /* failure other than file exists */
 373                                 break;
 374                 }
 375 #endif
 376
 377                 if (tm->tmpFH == NULL)
 378                         pg_fatal("could not generate temporary file name: %m");
 379
 380                 umask(old_umask);
 381
 382                 if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
 383                         tm->nFH = tm->tmpFH;
 384                 else
 385                         pg_fatal("compression is not supported by tar archive format");
 386
 387                 tm->AH = AH;
 388                 tm->targetFile = pg_strdup(filename);
 389         }
 390
 391         tm->mode = mode;
 392         tm->tarFH = ctx->tarFH;
 393
 394         return tm;
 395 }
 396
 397 static void
 398 tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
 399 {
 400         if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
 401                 pg_fatal("compression is not supported by tar archive format");
 402
 403         if (th->mode == 'w')
 404                 _tarAddFile(AH, th);    /* This will close the temp file */
 405
 406         /*
 407          * else Nothing to do for normal read since we don't dup() normal file
 408          * handle, and we don't use temp files.
 409          */
 410
 411         free(th->targetFile);
 412
 413         th->nFH = NULL;
 414 }
 415
 416 #ifdef __NOT_USED__
 417 static char *
 418 tarGets(char *buf, size_t len, TAR_MEMBER *th)
 419 {
 420         char       *s;
 421         size_t          cnt = 0;
 422         char            c = ' ';
 423         int                     eof = 0;
 424
 425         /* Can't read past logical EOF */
 426         if (len > (th->fileLen - th->pos))
 427                 len = th->fileLen - th->pos;
 428
 429         while (cnt < len && c != '\n')
 430         {
 431                 if (_tarReadRaw(th->AH, &c, 1, th, NULL) <= 0)
 432                 {
 433                         eof = 1;
 434                         break;
 435                 }
 436                 buf[cnt++] = c;
 437         }
 438
 439         if (eof && cnt == 0)
 440                 s = NULL;
 441         else
 442         {
 443                 buf[cnt++] = '\0';
 444                 s = buf;
 445         }
 446
 447         if (s)
 448         {
 449                 len = strlen(s);
 450                 th->pos += len;
 451         }
 452
 453         return s;
 454 }
 455 #endif
 456
 457 /*
 458  * Just read bytes from the archive. This is the low level read routine
 459  * that is used for ALL reads on a tar file.
 460  */
 461 static size_t
 462 _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
 463 {
 464         lclContext *ctx = (lclContext *) AH->formatData;
 465         size_t          avail;
 466         size_t          used = 0;
 467         size_t          res = 0;
 468
 469         Assert(th || fh);
 470
 471         avail = AH->lookaheadLen - AH->lookaheadPos;
 472         if (avail > 0)
 473         {
 474                 /* We have some lookahead bytes to use */
 475                 if (avail >= len)               /* Just use the lookahead buffer */
 476                         used = len;
 477                 else
 478                         used = avail;
 479
 480                 /* Copy, and adjust buffer pos */
 481                 memcpy(buf, AH->lookahead + AH->lookaheadPos, used);
 482                 AH->lookaheadPos += used;
 483
 484                 /* Adjust required length */
 485                 len -= used;
 486         }
 487
 488         /* Read the file if len > 0 */
 489         if (len > 0)
 490         {
 491                 if (fh)
 492                 {
 493                         res = fread(&((char *) buf)[used], 1, len, fh);
 494                         if (res != len && !feof(fh))
 495                                 READ_ERROR_EXIT(fh);
 496                 }
 497                 else if (th)
 498                 {
 499                         res = fread(&((char *) buf)[used], 1, len, th->nFH);
 500                         if (res != len && !feof(th->nFH))
 501                                 READ_ERROR_EXIT(th->nFH);
 502                 }
 503         }
 504
 505         ctx->tarFHpos += res + used;
 506
 507         return (res + used);
 508 }
 509
 510 static size_t
 511 tarRead(void *buf, size_t len, TAR_MEMBER *th)
 512 {
 513         size_t          res;
 514
 515         if (th->pos + len > th->fileLen)
 516                 len = th->fileLen - th->pos;
 517
 518         if (len <= 0)
 519                 return 0;
 520
 521         res = _tarReadRaw(th->AH, buf, len, th, NULL);
 522
 523         th->pos += res;
 524
 525         return res;
 526 }
 527
 528 static size_t
 529 tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
 530 {
 531         size_t          res;
 532
 533         res = fwrite(buf, 1, len, th->nFH);
 534
 535         th->pos += res;
 536         return res;
 537 }
 538
 539 static void
 540 _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
 541 {
 542         lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
 543
 544         if (tarWrite(data, dLen, tctx->TH) != dLen)
 545                 WRITE_ERROR_EXIT;
 546 }
 547
 548 static void
 549 _EndData(ArchiveHandle *AH, TocEntry *te)
 550 {
 551         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 552
 553         /* Close the file */
 554         tarClose(AH, tctx->TH);
 555         tctx->TH = NULL;
 556 }
 557
 558 /*
 559  * Print data for a given file
 560  */
 561 static void
 562 _PrintFileData(ArchiveHandle *AH, char *filename)
 563 {
 564         lclContext *ctx = (lclContext *) AH->formatData;
 565         char            buf[4096];
 566         size_t          cnt;
 567         TAR_MEMBER *th;
 568
 569         if (!filename)
 570                 return;
 571
 572         th = tarOpen(AH, filename, 'r');
 573         ctx->FH = th;
 574
 575         while ((cnt = tarRead(buf, 4095, th)) > 0)
 576         {
 577                 buf[cnt] = '\0';
 578                 ahwrite(buf, 1, cnt, AH);
 579         }
 580
 581         tarClose(AH, th);
 582 }
 583
 584
 585 /*
 586  * Print data for a given TOC entry
 587 */
 588 static void
 589 _PrintTocData(ArchiveHandle *AH, TocEntry *te)
 590 {
 591         lclContext *ctx = (lclContext *) AH->formatData;
 592         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 593         int                     pos1;
 594
 595         if (!tctx->filename)
 596                 return;
 597
 598         /*
 599          * If we're writing the special restore.sql script, emit a suitable
 600          * command to include each table's data from the corresponding file.
 601          *
 602          * In the COPY case this is a bit klugy because the regular COPY command
 603          * was already printed before we get control.
 604          */
 605         if (ctx->isSpecialScript)
 606         {
 607                 if (te->copyStmt)
 608                 {
 609                         /* Abort the COPY FROM stdin */
 610                         ahprintf(AH, "\\.\n");
 611
 612                         /*
 613                          * The COPY statement should look like "COPY ... FROM stdin;\n",
 614                          * see dumpTableData().
 615                          */
 616                         pos1 = (int) strlen(te->copyStmt) - 13;
 617                         if (pos1 < 6 || strncmp(te->copyStmt, "COPY ", 5) != 0 ||
 618                                 strcmp(te->copyStmt + pos1, " FROM stdin;\n") != 0)
 619                                 pg_fatal("unexpected COPY statement syntax: \"%s\"",
 620                                                  te->copyStmt);
 621
 622                         /* Emit all but the FROM part ... */
 623                         ahwrite(te->copyStmt, 1, pos1, AH);
 624                         /* ... and insert modified FROM */
 625                         ahprintf(AH, " FROM '$$PATH$$/%s';\n\n", tctx->filename);
 626                 }
 627                 else
 628                 {
 629                         /* --inserts mode, no worries, just include the data file */
 630                         ahprintf(AH, "\\i $$PATH$$/%s\n\n", tctx->filename);
 631                 }
 632
 633                 return;
 634         }
 635
 636         if (strcmp(te->desc, "BLOBS") == 0)
 637                 _LoadLOs(AH, te);
 638         else
 639                 _PrintFileData(AH, tctx->filename);
 640 }
 641
 642 static void
 643 _LoadLOs(ArchiveHandle *AH, TocEntry *te)
 644 {
 645         Oid                     oid;
 646         lclContext *ctx = (lclContext *) AH->formatData;
 647         TAR_MEMBER *th;
 648         size_t          cnt;
 649         bool            foundLO = false;
 650         char            buf[4096];
 651
 652         StartRestoreLOs(AH);
 653
 654         /*
 655          * The blobs_NNN.toc or blobs.toc file is fairly useless to us because it
 656          * will appear only after the associated blob_NNN.dat files.  For archive
 657          * versions >= 16 we can look at the BLOBS entry's te->tag to discover the
 658          * OID of the first blob we want to restore, and then search forward to
 659          * find the appropriate blob_<oid>.dat file.  For older versions we rely
 660          * on the knowledge that there was only one BLOBS entry and just search
 661          * for the first blob_<oid>.dat file.  Once we find the first blob file to
 662          * restore, restore all blobs until we reach the blobs[_NNN].toc file.
 663          */
 664         if (AH->version >= K_VERS_1_16)
 665         {
 666                 /* We rely on atooid to not complain about nnnn..nnnn tags */
 667                 oid = atooid(te->tag);
 668                 snprintf(buf, sizeof(buf), "blob_%u.dat", oid);
 669                 th = tarOpen(AH, buf, 'r'); /* Advance to first desired file */
 670         }
 671         else
 672                 th = tarOpen(AH, NULL, 'r');    /* Open next file */
 673
 674         while (th != NULL)
 675         {
 676                 ctx->FH = th;
 677
 678                 if (strncmp(th->targetFile, "blob_", 5) == 0)
 679                 {
 680                         oid = atooid(&th->targetFile[5]);
 681                         if (oid != 0)
 682                         {
 683                                 pg_log_info("restoring large object with OID %u", oid);
 684
 685                                 StartRestoreLO(AH, oid, AH->public.ropt->dropSchema);
 686
 687                                 while ((cnt = tarRead(buf, 4095, th)) > 0)
 688                                 {
 689                                         buf[cnt] = '\0';
 690                                         ahwrite(buf, 1, cnt, AH);
 691                                 }
 692                                 EndRestoreLO(AH, oid);
 693                                 foundLO = true;
 694                         }
 695                         tarClose(AH, th);
 696                 }
 697                 else
 698                 {
 699                         tarClose(AH, th);
 700
 701                         /*
 702                          * Once we have found the first LO, stop at the first non-LO entry
 703                          * (which will be 'blobs[_NNN].toc').  This coding would eat all
 704                          * the rest of the archive if there are no LOs ... but this
 705                          * function shouldn't be called at all in that case.
 706                          */
 707                         if (foundLO)
 708                                 break;
 709                 }
 710
 711                 th = tarOpen(AH, NULL, 'r');
 712         }
 713         EndRestoreLOs(AH);
 714 }
 715
 716
 717 static int
 718 _WriteByte(ArchiveHandle *AH, const int i)
 719 {
 720         lclContext *ctx = (lclContext *) AH->formatData;
 721         char            b = i;                  /* Avoid endian problems */
 722
 723         if (tarWrite(&b, 1, ctx->FH) != 1)
 724                 WRITE_ERROR_EXIT;
 725
 726         ctx->filePos += 1;
 727         return 1;
 728 }
 729
 730 static int
 731 _ReadByte(ArchiveHandle *AH)
 732 {
 733         lclContext *ctx = (lclContext *) AH->formatData;
 734         size_t          res;
 735         unsigned char c;
 736
 737         res = tarRead(&c, 1, ctx->FH);
 738         if (res != 1)
 739                 /* We already would have exited for errors on reads, must be EOF */
 740                 pg_fatal("could not read from input file: end of file");
 741         ctx->filePos += 1;
 742         return c;
 743 }
 744
 745 static void
 746 _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
 747 {
 748         lclContext *ctx = (lclContext *) AH->formatData;
 749
 750         if (tarWrite(buf, len, ctx->FH) != len)
 751                 WRITE_ERROR_EXIT;
 752
 753         ctx->filePos += len;
 754 }
 755
 756 static void
 757 _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
 758 {
 759         lclContext *ctx = (lclContext *) AH->formatData;
 760
 761         if (tarRead(buf, len, ctx->FH) != len)
 762                 /* We already would have exited for errors on reads, must be EOF */
 763                 pg_fatal("could not read from input file: end of file");
 764
 765         ctx->filePos += len;
 766 }
 767
 768 static void
 769 _CloseArchive(ArchiveHandle *AH)
 770 {
 771         lclContext *ctx = (lclContext *) AH->formatData;
 772         TAR_MEMBER *th;
 773         RestoreOptions *ropt;
 774         RestoreOptions *savRopt;
 775         DumpOptions *savDopt;
 776         int                     savVerbose,
 777                                 i;
 778
 779         if (AH->mode == archModeWrite)
 780         {
 781                 /*
 782                  * Write the Header & TOC to the archive FIRST
 783                  */
 784                 th = tarOpen(AH, "toc.dat", 'w');
 785                 ctx->FH = th;
 786                 WriteHead(AH);
 787                 WriteToc(AH);
 788                 tarClose(AH, th);               /* Not needed any more */
 789
 790                 /*
 791                  * Now send the data (tables & LOs)
 792                  */
 793                 WriteDataChunks(AH, NULL);
 794
 795                 /*
 796                  * Now this format wants to append a script which does a full restore
 797                  * if the files have been extracted.
 798                  */
 799                 th = tarOpen(AH, "restore.sql", 'w');
 800
 801                 tarPrintf(th, "--\n"
 802                                   "-- NOTE:\n"
 803                                   "--\n"
 804                                   "-- File paths need to be edited. Search for $$PATH$$ and\n"
 805                                   "-- replace it with the path to the directory containing\n"
 806                                   "-- the extracted data files.\n"
 807                                   "--\n");
 808
 809                 AH->CustomOutPtr = _scriptOut;
 810
 811                 ctx->isSpecialScript = 1;
 812                 ctx->scriptTH = th;
 813
 814                 ropt = NewRestoreOptions();
 815                 memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
 816                 ropt->filename = NULL;
 817                 ropt->dropSchema = 1;
 818                 ropt->superuser = NULL;
 819                 ropt->suppressDumpWarnings = true;
 820
 821                 savDopt = AH->public.dopt;
 822                 savRopt = AH->public.ropt;
 823
 824                 SetArchiveOptions((Archive *) AH, NULL, ropt);
 825
 826                 savVerbose = AH->public.verbose;
 827                 AH->public.verbose = 0;
 828
 829                 RestoreArchive((Archive *) AH);
 830
 831                 SetArchiveOptions((Archive *) AH, savDopt, savRopt);
 832
 833                 AH->public.verbose = savVerbose;
 834
 835                 tarClose(AH, th);
 836
 837                 ctx->isSpecialScript = 0;
 838
 839                 /*
 840                  * EOF marker for tar files is two blocks of NULLs.
 841                  */
 842                 for (i = 0; i < TAR_BLOCK_SIZE * 2; i++)
 843                 {
 844                         if (fputc(0, ctx->tarFH) == EOF)
 845                                 WRITE_ERROR_EXIT;
 846                 }
 847
 848                 /* Sync the output file if one is defined */
 849                 if (AH->dosync && AH->fSpec)
 850                         (void) fsync_fname(AH->fSpec, false);
 851         }
 852
 853         AH->FH = NULL;
 854 }
 855
 856 static size_t
 857 _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
 858 {
 859         lclContext *ctx = (lclContext *) AH->formatData;
 860
 861         return tarWrite(buf, len, ctx->scriptTH);
 862 }
 863
 864 /*
 865  * Large Object support
 866  */
 867
 868 /*
 869  * Called by the archiver when starting to save BLOB DATA (not schema).
 870  * This routine should save whatever format-specific information is needed
 871  * to read the LOs back into memory.
 872  *
 873  * It is called just prior to the dumper's DataDumper routine.
 874  *
 875  * Optional, but strongly recommended.
 876  *
 877  */
 878 static void
 879 _StartLOs(ArchiveHandle *AH, TocEntry *te)
 880 {
 881         lclContext *ctx = (lclContext *) AH->formatData;
 882         char            fname[K_STD_BUF_SIZE];
 883
 884         sprintf(fname, "blobs_%d.toc", te->dumpId);
 885         ctx->loToc = tarOpen(AH, fname, 'w');
 886 }
 887
 888 /*
 889  * Called by the archiver when the dumper calls StartLO.
 890  *
 891  * Mandatory.
 892  *
 893  * Must save the passed OID for retrieval at restore-time.
 894  */
 895 static void
 896 _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
 897 {
 898         lclContext *ctx = (lclContext *) AH->formatData;
 899         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 900         char            fname[255];
 901
 902         if (oid == 0)
 903                 pg_fatal("invalid OID for large object (%u)", oid);
 904
 905         if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
 906                 pg_fatal("compression is not supported by tar archive format");
 907
 908         sprintf(fname, "blob_%u.dat", oid);
 909
 910         tarPrintf(ctx->loToc, "%u %s\n", oid, fname);
 911
 912         tctx->TH = tarOpen(AH, fname, 'w');
 913 }
 914
 915 /*
 916  * Called by the archiver when the dumper calls EndLO.
 917  *
 918  * Optional.
 919  *
 920  */
 921 static void
 922 _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
 923 {
 924         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
 925
 926         tarClose(AH, tctx->TH);
 927 }
 928
 929 /*
 930  * Called by the archiver when finishing saving BLOB DATA.
 931  *
 932  * Optional.
 933  *
 934  */
 935 static void
 936 _EndLOs(ArchiveHandle *AH, TocEntry *te)
 937 {
 938         lclContext *ctx = (lclContext *) AH->formatData;
 939
 940         /* Write out a fake zero OID to mark end-of-LOs. */
 941         /* WriteInt(AH, 0); */
 942
 943         tarClose(AH, ctx->loToc);
 944 }
 945
 946
 947
 948 /*------------
 949  * TAR Support
 950  *------------
 951  */
 952
 953 static int
 954 tarPrintf(TAR_MEMBER *th, const char *fmt,...)
 955 {
 956         int                     save_errno = errno;
 957         char       *p;
 958         size_t          len = 128;              /* initial assumption about buffer size */
 959         size_t          cnt;
 960
 961         for (;;)
 962         {
 963                 va_list         args;
 964
 965                 /* Allocate work buffer. */
 966                 p = (char *) pg_malloc(len);
 967
 968                 /* Try to format the data. */
 969                 errno = save_errno;
 970                 va_start(args, fmt);
 971                 cnt = pvsnprintf(p, len, fmt, args);
 972                 va_end(args);
 973
 974                 if (cnt < len)
 975                         break;                          /* success */
 976
 977                 /* Release buffer and loop around to try again with larger len. */
 978                 free(p);
 979                 len = cnt;
 980         }
 981
 982         cnt = tarWrite(p, cnt, th);
 983         free(p);
 984         return (int) cnt;
 985 }
 986
 987 bool
 988 isValidTarHeader(char *header)
 989 {
 990         int                     sum;
 991         int                     chk = tarChecksum(header);
 992
 993         sum = read_tar_number(&header[TAR_OFFSET_CHECKSUM], 8);
 994
 995         if (sum != chk)
 996                 return false;
 997
 998         /* POSIX tar format */
 999         if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar\0", 6) == 0 &&
1000                 memcmp(&header[TAR_OFFSET_VERSION], "00", 2) == 0)
1001                 return true;
1002         /* GNU tar format */
1003         if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar  \0", 8) == 0)
1004                 return true;
1005         /* not-quite-POSIX format written by pre-9.3 pg_dump */
1006         if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar00\0", 8) == 0)
1007                 return true;
1008
1009         return false;
1010 }
1011
1012 /* Given the member, write the TAR header & copy the file */
1013 static void
1014 _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
1015 {
1016         lclContext *ctx = (lclContext *) AH->formatData;
1017         FILE       *tmp = th->tmpFH;    /* Grab it for convenience */
1018         char            buf[32768];
1019         size_t          cnt;
1020         pgoff_t         len = 0;
1021         size_t          res;
1022         size_t          i,
1023                                 pad;
1024
1025         /*
1026          * Find file len & go back to start.
1027          */
1028         if (fseeko(tmp, 0, SEEK_END) != 0)
1029                 pg_fatal("error during file seek: %m");
1030         th->fileLen = ftello(tmp);
1031         if (th->fileLen < 0)
1032                 pg_fatal("could not determine seek position in archive file: %m");
1033         if (fseeko(tmp, 0, SEEK_SET) != 0)
1034                 pg_fatal("error during file seek: %m");
1035
1036         _tarWriteHeader(th);
1037
1038         while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
1039         {
1040                 if ((res = fwrite(buf, 1, cnt, th->tarFH)) != cnt)
1041                         WRITE_ERROR_EXIT;
1042                 len += res;
1043         }
1044         if (!feof(tmp))
1045                 READ_ERROR_EXIT(tmp);
1046
1047         if (fclose(tmp) != 0)           /* This *should* delete it... */
1048                 pg_fatal("could not close temporary file: %m");
1049
1050         if (len != th->fileLen)
1051                 pg_fatal("actual file length (%lld) does not match expected (%lld)",
1052                                  (long long) len, (long long) th->fileLen);
1053
1054         pad = tarPaddingBytesRequired(len);
1055         for (i = 0; i < pad; i++)
1056         {
1057                 if (fputc('\0', th->tarFH) == EOF)
1058                         WRITE_ERROR_EXIT;
1059         }
1060
1061         ctx->tarFHpos += len + pad;
1062 }
1063
1064 /* Locate the file in the archive, read header and position to data */
1065 static TAR_MEMBER *
1066 _tarPositionTo(ArchiveHandle *AH, const char *filename)
1067 {
1068         lclContext *ctx = (lclContext *) AH->formatData;
1069         TAR_MEMBER *th = pg_malloc0_object(TAR_MEMBER);
1070         char            c;
1071         char            header[TAR_BLOCK_SIZE];
1072         size_t          i,
1073                                 len,
1074                                 blks;
1075         int                     id;
1076
1077         th->AH = AH;
1078
1079         /* Go to end of current file, if any */
1080         if (ctx->tarFHpos != 0)
1081         {
1082                 pg_log_debug("moving from position %lld to next member at file position %lld",
1083                                          (long long) ctx->tarFHpos, (long long) ctx->tarNextMember);
1084
1085                 while (ctx->tarFHpos < ctx->tarNextMember)
1086                         _tarReadRaw(AH, &c, 1, NULL, ctx->tarFH);
1087         }
1088
1089         pg_log_debug("now at file position %lld", (long long) ctx->tarFHpos);
1090
1091         /* We are at the start of the file, or at the next member */
1092
1093         /* Get the header */
1094         if (!_tarGetHeader(AH, th))
1095         {
1096                 if (filename)
1097                         pg_fatal("could not find header for file \"%s\" in tar archive", filename);
1098                 else
1099                 {
1100                         /*
1101                          * We're just scanning the archive for the next file, so return
1102                          * null
1103                          */
1104                         free(th);
1105                         return NULL;
1106                 }
1107         }
1108
1109         while (filename != NULL && strcmp(th->targetFile, filename) != 0)
1110         {
1111                 pg_log_debug("skipping tar member %s", th->targetFile);
1112
1113                 id = atoi(th->targetFile);
1114                 if ((TocIDRequired(AH, id) & REQ_DATA) != 0)
1115                         pg_fatal("restoring data out of order is not supported in this archive format: "
1116                                          "\"%s\" is required, but comes before \"%s\" in the archive file.",
1117                                          th->targetFile, filename);
1118
1119                 /* Header doesn't match, so read to next header */
1120                 len = th->fileLen;
1121                 len += tarPaddingBytesRequired(th->fileLen);
1122                 blks = len / TAR_BLOCK_SIZE;    /* # of tar blocks */
1123
1124                 for (i = 0; i < blks; i++)
1125                         _tarReadRaw(AH, &header[0], TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1126
1127                 if (!_tarGetHeader(AH, th))
1128                         pg_fatal("could not find header for file \"%s\" in tar archive", filename);
1129         }
1130
1131         ctx->tarNextMember = ctx->tarFHpos + th->fileLen
1132                 + tarPaddingBytesRequired(th->fileLen);
1133         th->pos = 0;
1134
1135         return th;
1136 }
1137
1138 /* Read & verify a header */
1139 static int
1140 _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
1141 {
1142         lclContext *ctx = (lclContext *) AH->formatData;
1143         char            h[TAR_BLOCK_SIZE];
1144         char            tag[100 + 1];
1145         int                     sum,
1146                                 chk;
1147         pgoff_t         len;
1148         pgoff_t         hPos;
1149         bool            gotBlock = false;
1150
1151         while (!gotBlock)
1152         {
1153                 /* Save the pos for reporting purposes */
1154                 hPos = ctx->tarFHpos;
1155
1156                 /* Read the next tar block, return EOF, exit if short */
1157                 len = _tarReadRaw(AH, h, TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1158                 if (len == 0)                   /* EOF */
1159                         return 0;
1160
1161                 if (len != TAR_BLOCK_SIZE)
1162                         pg_fatal(ngettext("incomplete tar header found (%lu byte)",
1163                                                           "incomplete tar header found (%lu bytes)",
1164                                                           len),
1165                                          (unsigned long) len);
1166
1167                 /* Calc checksum */
1168                 chk = tarChecksum(h);
1169                 sum = read_tar_number(&h[TAR_OFFSET_CHECKSUM], 8);
1170
1171                 /*
1172                  * If the checksum failed, see if it is a null block. If so, silently
1173                  * continue to the next block.
1174                  */
1175                 if (chk == sum)
1176                         gotBlock = true;
1177                 else
1178                 {
1179                         int                     i;
1180
1181                         for (i = 0; i < TAR_BLOCK_SIZE; i++)
1182                         {
1183                                 if (h[i] != 0)
1184                                 {
1185                                         gotBlock = true;
1186                                         break;
1187                                 }
1188                         }
1189                 }
1190         }
1191
1192         /* Name field is 100 bytes, might not be null-terminated */
1193         strlcpy(tag, &h[TAR_OFFSET_NAME], 100 + 1);
1194
1195         len = read_tar_number(&h[TAR_OFFSET_SIZE], 12);
1196
1197         pg_log_debug("TOC Entry %s at %llu (length %llu, checksum %d)",
1198                                  tag, (unsigned long long) hPos, (unsigned long long) len, sum);
1199
1200         if (chk != sum)
1201                 pg_fatal("corrupt tar header found in %s (expected %d, computed %d) file position %llu",
1202                                  tag, sum, chk, (unsigned long long) ftello(ctx->tarFH));
1203
1204         th->targetFile = pg_strdup(tag);
1205         th->fileLen = len;
1206
1207         return 1;
1208 }
1209
1210
1211 static void
1212 _tarWriteHeader(TAR_MEMBER *th)
1213 {
1214         char            h[TAR_BLOCK_SIZE];
1215
1216         tarCreateHeader(h, th->targetFile, NULL, th->fileLen,
1217                                         0600, 04000, 02000, time(NULL));
1218
1219         /* Now write the completed header. */
1220         if (fwrite(h, 1, TAR_BLOCK_SIZE, th->tarFH) != TAR_BLOCK_SIZE)
1221                 WRITE_ERROR_EXIT;
1222 }