src/bin/pg_dump/pg_backup_archiver.h

   1 /*-------------------------------------------------------------------------
   2  *
   3  * pg_backup_archiver.h
   4  *
   5  *      Private interface to the pg_dump archiver routines.
   6  *      It is NOT intended that these routines be called by any
   7  *      dumper directly.
   8  *
   9  *      See the headers to pg_restore for more details.
  10  *
  11  * Copyright (c) 2000, Philip Warner
  12  *              Rights are granted to use this software in any way so long
  13  *              as this notice is not removed.
  14  *
  15  *      The author is not responsible for loss or damages that may
  16  *      result from its use.
  17  *
  18  *
  19  * IDENTIFICATION
  20  *              src/bin/pg_dump/pg_backup_archiver.h
  21  *
  22  *-------------------------------------------------------------------------
  23  */
  24 #ifndef __PG_BACKUP_ARCHIVE__
  25 #define __PG_BACKUP_ARCHIVE__
  26
  27 #include <time.h>
  28
  29 #include "libpq-fe.h"
  30 #include "pg_backup.h"
  31 #include "pqexpbuffer.h"
  32
  33 #define LOBBUFSIZE 16384
  34
  35 /* Data block types */
  36 #define BLK_DATA 1
  37 #define BLK_BLOBS 3
  38
  39 /* Encode version components into a convenient integer <maj><min><rev> */
  40 #define MAKE_ARCHIVE_VERSION(major, minor, rev) (((major) * 256 + (minor)) * 256 + (rev))
  41
  42 #define ARCHIVE_MAJOR(version) (((version) >> 16) & 255)
  43 #define ARCHIVE_MINOR(version) (((version) >>  8) & 255)
  44 #define ARCHIVE_REV(version)   (((version)              ) & 255)
  45
  46 /* Historical version numbers (checked in code) */
  47 #define K_VERS_1_0      MAKE_ARCHIVE_VERSION(1, 0, 0)
  48 #define K_VERS_1_2      MAKE_ARCHIVE_VERSION(1, 2, 0)   /* Allow No ZLIB */
  49 #define K_VERS_1_3      MAKE_ARCHIVE_VERSION(1, 3, 0)   /* BLOBS */
  50 #define K_VERS_1_4      MAKE_ARCHIVE_VERSION(1, 4, 0)   /* Date & name in header */
  51 #define K_VERS_1_5      MAKE_ARCHIVE_VERSION(1, 5, 0)   /* Handle dependencies */
  52 #define K_VERS_1_6      MAKE_ARCHIVE_VERSION(1, 6, 0)   /* Schema field in TOCs */
  53 #define K_VERS_1_7      MAKE_ARCHIVE_VERSION(1, 7, 0)   /* File Offset size in
  54                                                                                                          * header */
  55 #define K_VERS_1_8      MAKE_ARCHIVE_VERSION(1, 8, 0)   /* change interpretation
  56                                                                                                          * of ID numbers and
  57                                                                                                          * dependencies */
  58 #define K_VERS_1_9      MAKE_ARCHIVE_VERSION(1, 9, 0)   /* add default_with_oids
  59                                                                                                          * tracking */
  60 #define K_VERS_1_10 MAKE_ARCHIVE_VERSION(1, 10, 0)      /* add tablespace */
  61 #define K_VERS_1_11 MAKE_ARCHIVE_VERSION(1, 11, 0)      /* add toc section
  62                                                                                                          * indicator */
  63 #define K_VERS_1_12 MAKE_ARCHIVE_VERSION(1, 12, 0)      /* add separate BLOB
  64                                                                                                          * entries */
  65 #define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0)      /* change search_path
  66                                                                                                          * behavior */
  67 #define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0)      /* add tableam */
  68 #define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0)      /* add
  69                                                                                                          * compression_algorithm
  70                                                                                                          * in header */
  71 #define K_VERS_1_16 MAKE_ARCHIVE_VERSION(1, 16, 0)      /* BLOB METADATA entries
  72                                                                                                          * and multiple BLOBS,
  73                                                                                                          * relkind */
  74
  75 /* Current archive version number (the format we can output) */
  76 #define K_VERS_MAJOR 1
  77 #define K_VERS_MINOR 16
  78 #define K_VERS_REV 0
  79 #define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
  80
  81 /* Newest format we can read */
  82 #define K_VERS_MAX      MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, 255)
  83
  84
  85 /* Flags to indicate disposition of offsets stored in files */
  86 #define K_OFFSET_POS_NOT_SET 1
  87 #define K_OFFSET_POS_SET 2
  88 #define K_OFFSET_NO_DATA 3
  89
  90 /*
  91  * Special exit values from worker children.  We reserve 0 for normal
  92  * success; 1 and other small values should be interpreted as crashes.
  93  */
  94 #define WORKER_OK                                         0
  95 #define WORKER_CREATE_DONE                        10
  96 #define WORKER_INHIBIT_DATA                       11
  97 #define WORKER_IGNORED_ERRORS             12
  98
  99 typedef struct _archiveHandle ArchiveHandle;
 100 typedef struct _tocEntry TocEntry;
 101 struct ParallelState;
 102
 103 #define READ_ERROR_EXIT(fd) \
 104         do { \
 105                 if (feof(fd)) \
 106                         pg_fatal("could not read from input file: end of file"); \
 107                 else \
 108                         pg_fatal("could not read from input file: %m"); \
 109         } while (0)
 110
 111 #define WRITE_ERROR_EXIT \
 112         do { \
 113                 pg_fatal("could not write to output file: %m"); \
 114         } while (0)
 115
 116 typedef enum T_Action
 117 {
 118         ACT_DUMP,
 119         ACT_RESTORE,
 120 } T_Action;
 121
 122 typedef void (*ClosePtrType) (ArchiveHandle *AH);
 123 typedef void (*ReopenPtrType) (ArchiveHandle *AH);
 124 typedef void (*ArchiveEntryPtrType) (ArchiveHandle *AH, TocEntry *te);
 125
 126 typedef void (*StartDataPtrType) (ArchiveHandle *AH, TocEntry *te);
 127 typedef void (*WriteDataPtrType) (ArchiveHandle *AH, const void *data, size_t dLen);
 128 typedef void (*EndDataPtrType) (ArchiveHandle *AH, TocEntry *te);
 129
 130 typedef void (*StartLOsPtrType) (ArchiveHandle *AH, TocEntry *te);
 131 typedef void (*StartLOPtrType) (ArchiveHandle *AH, TocEntry *te, Oid oid);
 132 typedef void (*EndLOPtrType) (ArchiveHandle *AH, TocEntry *te, Oid oid);
 133 typedef void (*EndLOsPtrType) (ArchiveHandle *AH, TocEntry *te);
 134
 135 typedef int (*WriteBytePtrType) (ArchiveHandle *AH, const int i);
 136 typedef int (*ReadBytePtrType) (ArchiveHandle *AH);
 137 typedef void (*WriteBufPtrType) (ArchiveHandle *AH, const void *c, size_t len);
 138 typedef void (*ReadBufPtrType) (ArchiveHandle *AH, void *buf, size_t len);
 139 typedef void (*WriteExtraTocPtrType) (ArchiveHandle *AH, TocEntry *te);
 140 typedef void (*ReadExtraTocPtrType) (ArchiveHandle *AH, TocEntry *te);
 141 typedef void (*PrintExtraTocPtrType) (ArchiveHandle *AH, TocEntry *te);
 142 typedef void (*PrintTocDataPtrType) (ArchiveHandle *AH, TocEntry *te);
 143
 144 typedef void (*PrepParallelRestorePtrType) (ArchiveHandle *AH);
 145 typedef void (*ClonePtrType) (ArchiveHandle *AH);
 146 typedef void (*DeClonePtrType) (ArchiveHandle *AH);
 147
 148 typedef int (*WorkerJobDumpPtrType) (ArchiveHandle *AH, TocEntry *te);
 149 typedef int (*WorkerJobRestorePtrType) (ArchiveHandle *AH, TocEntry *te);
 150
 151 typedef size_t (*CustomOutPtrType) (ArchiveHandle *AH, const void *buf, size_t len);
 152
 153 typedef enum
 154 {
 155         SQL_SCAN = 0,                           /* normal */
 156         SQL_IN_SINGLE_QUOTE,            /* '...' literal */
 157         SQL_IN_DOUBLE_QUOTE,            /* "..." identifier */
 158 } sqlparseState;
 159
 160 typedef struct
 161 {
 162         sqlparseState state;            /* see above */
 163         bool            backSlash;              /* next char is backslash quoted? */
 164         PQExpBuffer curCmd;                     /* incomplete line (NULL if not created) */
 165 } sqlparseInfo;
 166
 167 typedef enum
 168 {
 169         STAGE_NONE = 0,
 170         STAGE_INITIALIZING,
 171         STAGE_PROCESSING,
 172         STAGE_FINALIZING,
 173 } ArchiverStage;
 174
 175 typedef enum
 176 {
 177         OUTPUT_SQLCMDS = 0,                     /* emitting general SQL commands */
 178         OUTPUT_COPYDATA,                        /* writing COPY data */
 179         OUTPUT_OTHERDATA,                       /* writing data as INSERT commands */
 180 } ArchiverOutput;
 181
 182 /*
 183  * For historical reasons, ACL items are interspersed with everything else in
 184  * a dump file's TOC; typically they're right after the object they're for.
 185  * However, we need to restore data before ACLs, as otherwise a read-only
 186  * table (ie one where the owner has revoked her own INSERT privilege) causes
 187  * data restore failures.  On the other hand, matview REFRESH commands should
 188  * come out after ACLs, as otherwise non-superuser-owned matviews might not
 189  * be able to execute.  (If the permissions at the time of dumping would not
 190  * allow a REFRESH, too bad; we won't fix that for you.)  We also want event
 191  * triggers to be restored after ACLs, so that they can't mess those up.
 192  *
 193  * These considerations force us to make three passes over the TOC,
 194  * restoring the appropriate subset of items in each pass.  We assume that
 195  * the dependency sort resulted in an appropriate ordering of items within
 196  * each subset.
 197  *
 198  * XXX This mechanism should be superseded by tracking dependencies on ACLs
 199  * properly; but we'll still need it for old dump files even after that.
 200  */
 201 typedef enum
 202 {
 203         RESTORE_PASS_MAIN = 0,          /* Main pass (most TOC item types) */
 204         RESTORE_PASS_ACL,                       /* ACL item types */
 205         RESTORE_PASS_POST_ACL,          /* Event trigger and matview refresh items */
 206
 207 #define RESTORE_PASS_LAST RESTORE_PASS_POST_ACL
 208 } RestorePass;
 209
 210 #define REQ_SCHEMA      0x01            /* want schema */
 211 #define REQ_DATA        0x02            /* want data */
 212 #define REQ_SPECIAL     0x04            /* for special TOC entries */
 213
 214 struct _archiveHandle
 215 {
 216         Archive         public;                 /* Public part of archive */
 217         int                     version;                /* Version of file */
 218
 219         char       *archiveRemoteVersion;       /* When reading an archive, the
 220                                                                                  * version of the dumped DB */
 221         char       *archiveDumpVersion; /* When reading an archive, the version of
 222                                                                          * the dumper */
 223
 224         size_t          intSize;                /* Size of an integer in the archive */
 225         size_t          offSize;                /* Size of a file offset in the archive -
 226                                                                  * Added V1.7 */
 227         ArchiveFormat format;           /* Archive format */
 228
 229         sqlparseInfo sqlparse;          /* state for parsing INSERT data */
 230
 231         time_t          createDate;             /* Date archive created */
 232
 233         /*
 234          * Fields used when discovering archive format.  For tar format, we load
 235          * the first block into the lookahead buffer, and verify that it looks
 236          * like a tar header.  The tar module must then consume bytes from the
 237          * lookahead buffer before reading any more from the file.  For custom
 238          * format, we load only the "PGDMP" marker into the buffer, and then set
 239          * readHeader after confirming it matches.  The buffer is vestigial in
 240          * this case, as the subsequent code just checks readHeader and doesn't
 241          * examine the buffer.
 242          */
 243         int                     readHeader;             /* Set if we already read "PGDMP" marker */
 244         char       *lookahead;          /* Buffer used when reading header to discover
 245                                                                  * format */
 246         size_t          lookaheadSize;  /* Allocated size of buffer */
 247         size_t          lookaheadLen;   /* Length of valid data in lookahead */
 248         size_t          lookaheadPos;   /* Current read position in lookahead buffer */
 249
 250         ArchiveEntryPtrType ArchiveEntryPtr;    /* Called for each metadata object */
 251         StartDataPtrType StartDataPtr;  /* Called when table data is about to be
 252                                                                          * dumped */
 253         WriteDataPtrType WriteDataPtr;  /* Called to send some table data to the
 254                                                                          * archive */
 255         EndDataPtrType EndDataPtr;      /* Called when table data dump is finished */
 256         WriteBytePtrType WriteBytePtr;  /* Write a byte to output */
 257         ReadBytePtrType ReadBytePtr;    /* Read a byte from an archive */
 258         WriteBufPtrType WriteBufPtr;    /* Write a buffer of output to the archive */
 259         ReadBufPtrType ReadBufPtr;      /* Read a buffer of input from the archive */
 260         ClosePtrType ClosePtr;          /* Close the archive */
 261         ReopenPtrType ReopenPtr;        /* Reopen the archive */
 262         WriteExtraTocPtrType WriteExtraTocPtr;  /* Write extra TOC entry data
 263                                                                                          * associated with the current
 264                                                                                          * archive format */
 265         ReadExtraTocPtrType ReadExtraTocPtr;    /* Read extra info associated with
 266                                                                                          * archive format */
 267         PrintExtraTocPtrType PrintExtraTocPtr;  /* Extra TOC info for format */
 268         PrintTocDataPtrType PrintTocDataPtr;
 269
 270         StartLOsPtrType StartLOsPtr;
 271         EndLOsPtrType EndLOsPtr;
 272         StartLOPtrType StartLOPtr;
 273         EndLOPtrType EndLOPtr;
 274
 275         SetupWorkerPtrType SetupWorkerPtr;
 276         WorkerJobDumpPtrType WorkerJobDumpPtr;
 277         WorkerJobRestorePtrType WorkerJobRestorePtr;
 278
 279         PrepParallelRestorePtrType PrepParallelRestorePtr;
 280         ClonePtrType ClonePtr;          /* Clone format-specific fields */
 281         DeClonePtrType DeClonePtr;      /* Clean up cloned fields */
 282
 283         CustomOutPtrType CustomOutPtr;  /* Alternative script output routine */
 284
 285         /* Stuff for direct DB connection */
 286         char       *archdbname;         /* DB name *read* from archive */
 287         char       *savedPassword;      /* password for ropt->username, if known */
 288         char       *use_role;
 289         PGconn     *connection;
 290         /* If connCancel isn't NULL, SIGINT handler will send a cancel */
 291         PGcancel   *volatile connCancel;
 292
 293         int                     connectToDB;    /* Flag to indicate if direct DB connection is
 294                                                                  * required */
 295         ArchiverOutput outputKind;      /* Flag for what we're currently writing */
 296         bool            pgCopyIn;               /* Currently in libpq 'COPY IN' mode. */
 297
 298         int                     loFd;
 299         bool            writingLO;
 300         int                     loCount;                /* # of LOs restored */
 301
 302         char       *fSpec;                      /* Archive File Spec */
 303         FILE       *FH;                         /* General purpose file handle */
 304         void       *OF;                         /* Output file */
 305
 306         struct _tocEntry *toc;          /* Header of circular list of TOC entries */
 307         int                     tocCount;               /* Number of TOC entries */
 308         DumpId          maxDumpId;              /* largest DumpId among all TOC entries */
 309
 310         /* arrays created after the TOC list is complete: */
 311         struct _tocEntry **tocsByDumpId;        /* TOCs indexed by dumpId */
 312         DumpId     *tableDataId;        /* TABLE DATA ids, indexed by table dumpId */
 313
 314         struct _tocEntry *currToc;      /* Used when dumping data */
 315         pg_compress_specification compression_spec; /* Requested specification for
 316                                                                                                  * compression */
 317         bool            dosync;                 /* data requested to be synced on sight */
 318         DataDirSyncMethod sync_method;
 319         ArchiveMode mode;                       /* File mode - r or w */
 320         void       *formatData;         /* Header data specific to file format */
 321
 322         /* these vars track state to avoid sending redundant SET commands */
 323         char       *currUser;           /* current username, or NULL if unknown */
 324         char       *currSchema;         /* current schema, or NULL */
 325         char       *currTablespace; /* current tablespace, or NULL */
 326         char       *currTableAm;        /* current table access method, or NULL */
 327
 328         /* in --transaction-size mode, this counts objects emitted in cur xact */
 329         int                     txnCount;
 330
 331         void       *lo_buf;
 332         size_t          lo_buf_used;
 333         size_t          lo_buf_size;
 334
 335         int                     noTocComments;
 336         ArchiverStage stage;
 337         ArchiverStage lastErrorStage;
 338         RestorePass restorePass;        /* used only during parallel restore */
 339         struct _tocEntry *currentTE;
 340         struct _tocEntry *lastErrorTE;
 341 };
 342
 343 struct _tocEntry
 344 {
 345         struct _tocEntry *prev;
 346         struct _tocEntry *next;
 347         CatalogId       catalogId;
 348         DumpId          dumpId;
 349         teSection       section;
 350         bool            hadDumper;              /* Archiver was passed a dumper routine (used
 351                                                                  * in restore) */
 352         char       *tag;                        /* index tag */
 353         char       *namespace;          /* null or empty string if not in a schema */
 354         char       *tablespace;         /* null if not in a tablespace; empty string
 355                                                                  * means use database default */
 356         char       *tableam;            /* table access method, only for TABLE tags */
 357         char            relkind;                /* relation kind, only for TABLE tags */
 358         char       *owner;
 359         char       *desc;
 360         char       *defn;
 361         char       *dropStmt;
 362         char       *copyStmt;
 363         DumpId     *dependencies;       /* dumpIds of objects this one depends on */
 364         int                     nDeps;                  /* number of dependencies */
 365
 366         DataDumperPtr dataDumper;       /* Routine to dump data for object */
 367         const void *dataDumperArg;      /* Arg for above routine */
 368         void       *formatData;         /* TOC Entry data specific to file format */
 369
 370         /* working state while dumping/restoring */
 371         pgoff_t         dataLength;             /* item's data size; 0 if none or unknown */
 372         int                     reqs;                   /* do we need schema and/or data of object
 373                                                                  * (REQ_* bit mask) */
 374         bool            created;                /* set for DATA member if TABLE was created */
 375
 376         /* working state (needed only for parallel restore) */
 377         struct _tocEntry *pending_prev; /* list links for pending-items list; */
 378         struct _tocEntry *pending_next; /* NULL if not in that list */
 379         int                     depCount;               /* number of dependencies not yet restored */
 380         DumpId     *revDeps;            /* dumpIds of objects depending on this one */
 381         int                     nRevDeps;               /* number of such dependencies */
 382         DumpId     *lockDeps;           /* dumpIds of objects this one needs lock on */
 383         int                     nLockDeps;              /* number of such dependencies */
 384 };
 385
 386 extern int      parallel_restore(ArchiveHandle *AH, TocEntry *te);
 387 extern void on_exit_close_archive(Archive *AHX);
 388
 389 extern void warn_or_exit_horribly(ArchiveHandle *AH, const char *fmt,...) pg_attribute_printf(2, 3);
 390
 391 /* Options for ArchiveEntry */
 392 typedef struct _archiveOpts
 393 {
 394         const char *tag;
 395         const char *namespace;
 396         const char *tablespace;
 397         const char *tableam;
 398         char            relkind;
 399         const char *owner;
 400         const char *description;
 401         teSection       section;
 402         const char *createStmt;
 403         const char *dropStmt;
 404         const char *copyStmt;
 405         const DumpId *deps;
 406         int                     nDeps;
 407         DataDumperPtr dumpFn;
 408         const void *dumpArg;
 409 } ArchiveOpts;
 410 #define ARCHIVE_OPTS(...) &(ArchiveOpts){__VA_ARGS__}
 411 /* Called to add a TOC entry */
 412 extern TocEntry *ArchiveEntry(Archive *AHX, CatalogId catalogId,
 413                                                           DumpId dumpId, ArchiveOpts *opts);
 414
 415 extern void WriteHead(ArchiveHandle *AH);
 416 extern void ReadHead(ArchiveHandle *AH);
 417 extern void WriteToc(ArchiveHandle *AH);
 418 extern void ReadToc(ArchiveHandle *AH);
 419 extern void WriteDataChunks(ArchiveHandle *AH, struct ParallelState *pstate);
 420 extern void WriteDataChunksForTocEntry(ArchiveHandle *AH, TocEntry *te);
 421 extern ArchiveHandle *CloneArchive(ArchiveHandle *AH);
 422 extern void DeCloneArchive(ArchiveHandle *AH);
 423
 424 extern int      TocIDRequired(ArchiveHandle *AH, DumpId id);
 425 TocEntry   *getTocEntryByDumpId(ArchiveHandle *AH, DumpId id);
 426 extern bool checkSeek(FILE *fp);
 427
 428 #define appendStringLiteralAHX(buf,str,AH) \
 429         appendStringLiteral(buf, str, (AH)->public.encoding, (AH)->public.std_strings)
 430
 431 #define appendByteaLiteralAHX(buf,str,len,AH) \
 432         appendByteaLiteral(buf, str, len, (AH)->public.std_strings)
 433
 434 /*
 435  * Mandatory routines for each supported format
 436  */
 437
 438 extern size_t WriteInt(ArchiveHandle *AH, int i);
 439 extern int      ReadInt(ArchiveHandle *AH);
 440 extern char *ReadStr(ArchiveHandle *AH);
 441 extern size_t WriteStr(ArchiveHandle *AH, const char *c);
 442
 443 int                     ReadOffset(ArchiveHandle *, pgoff_t *);
 444 size_t          WriteOffset(ArchiveHandle *, pgoff_t, int);
 445
 446 extern void StartRestoreLOs(ArchiveHandle *AH);
 447 extern void StartRestoreLO(ArchiveHandle *AH, Oid oid, bool drop);
 448 extern void EndRestoreLO(ArchiveHandle *AH, Oid oid);
 449 extern void EndRestoreLOs(ArchiveHandle *AH);
 450
 451 extern void InitArchiveFmt_Custom(ArchiveHandle *AH);
 452 extern void InitArchiveFmt_Null(ArchiveHandle *AH);
 453 extern void InitArchiveFmt_Directory(ArchiveHandle *AH);
 454 extern void InitArchiveFmt_Tar(ArchiveHandle *AH);
 455
 456 extern bool isValidTarHeader(char *header);
 457
 458 extern void ReconnectToServer(ArchiveHandle *AH, const char *dbname);
 459 extern void IssueCommandPerBlob(ArchiveHandle *AH, TocEntry *te,
 460                                                                 const char *cmdBegin, const char *cmdEnd);
 461 extern void IssueACLPerBlob(ArchiveHandle *AH, TocEntry *te);
 462 extern void DropLOIfExists(ArchiveHandle *AH, Oid oid);
 463
 464 void            ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH);
 465 int                     ahprintf(ArchiveHandle *AH, const char *fmt,...) pg_attribute_printf(2, 3);
 466
 467 #endif