src/bin/pg_dump/pg_backup_archiver.h

   1 /*-------------------------------------------------------------------------
   2  *
   3  * pg_backup_archiver.h
   4  *
   5  *      Private interface to the pg_dump archiver routines.
   6  *      It is NOT intended that these routines be called by any
   7  *      dumper directly.
   8  *
   9  *      See the headers to pg_restore for more details.
  10  *
  11  * Copyright (c) 2000, Philip Warner
  12  *              Rights are granted to use this software in any way so long
  13  *              as this notice is not removed.
  14  *
  15  *      The author is not responsible for loss or damages that may
  16  *      result from it's use.
  17  *
  18  *
  19  * IDENTIFICATION
  20  *              $PostgreSQL$
  21  *
  22  *-------------------------------------------------------------------------
  23  */
  24
  25 #ifndef __PG_BACKUP_ARCHIVE__
  26 #define __PG_BACKUP_ARCHIVE__
  27
  28 #include "postgres_fe.h"
  29
  30 #include <time.h>
  31
  32 #include "pg_backup.h"
  33
  34 #include "libpq-fe.h"
  35 #include "pqexpbuffer.h"
  36
  37 #define LOBBUFSIZE 16384
  38
  39 /*
  40  * Note: zlib.h must be included *after* libpq-fe.h, because the latter may
  41  * include ssl.h, which has a naming conflict with zlib.h.
  42  */
  43 #ifdef HAVE_LIBZ
  44 #include <zlib.h>
  45 #define GZCLOSE(fh) gzclose(fh)
  46 #define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
  47 #define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
  48 #else
  49 #define GZCLOSE(fh) fclose(fh)
  50 #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
  51 #define GZREAD(p, s, n, fh) fread(p, s, n, fh)
  52 #define Z_DEFAULT_COMPRESSION (-1)
  53
  54 typedef struct _z_stream
  55 {
  56         void       *next_in;
  57         void       *next_out;
  58         size_t          avail_in;
  59         size_t          avail_out;
  60 } z_stream;
  61 typedef z_stream *z_streamp;
  62 #endif
  63
  64 #define K_VERS_MAJOR 1
  65 #define K_VERS_MINOR 11
  66 #define K_VERS_REV 0
  67
  68 /* Data block types */
  69 #define BLK_DATA 1
  70 #define BLK_BLOB 2
  71 #define BLK_BLOBS 3
  72
  73 /* Some important version numbers (checked in code) */
  74 #define K_VERS_1_0 (( (1 * 256 + 0) * 256 + 0) * 256 + 0)
  75 #define K_VERS_1_2 (( (1 * 256 + 2) * 256 + 0) * 256 + 0)               /* Allow No ZLIB */
  76 #define K_VERS_1_3 (( (1 * 256 + 3) * 256 + 0) * 256 + 0)               /* BLOBs */
  77 #define K_VERS_1_4 (( (1 * 256 + 4) * 256 + 0) * 256 + 0)               /* Date & name in header */
  78 #define K_VERS_1_5 (( (1 * 256 + 5) * 256 + 0) * 256 + 0)               /* Handle dependencies */
  79 #define K_VERS_1_6 (( (1 * 256 + 6) * 256 + 0) * 256 + 0)               /* Schema field in TOCs */
  80 #define K_VERS_1_7 (( (1 * 256 + 7) * 256 + 0) * 256 + 0)               /* File Offset size in
  81                                                                                                                                  * header */
  82 #define K_VERS_1_8 (( (1 * 256 + 8) * 256 + 0) * 256 + 0)               /* change interpretation
  83                                                                                                                                  * of ID numbers and
  84                                                                                                                                  * dependencies */
  85 #define K_VERS_1_9 (( (1 * 256 + 9) * 256 + 0) * 256 + 0)               /* add default_with_oids
  86                                                                                                                                  * tracking */
  87 #define K_VERS_1_10 (( (1 * 256 + 10) * 256 + 0) * 256 + 0)             /* add tablespace */
  88 #define K_VERS_1_11 (( (1 * 256 + 11) * 256 + 0) * 256 + 0)             /* add toc section
  89                                                                                                                                  * indicator */
  90
  91 #define K_VERS_MAX (( (1 * 256 + 11) * 256 + 255) * 256 + 0)
  92
  93
  94 /* Flags to indicate disposition of offsets stored in files */
  95 #define K_OFFSET_POS_NOT_SET 1
  96 #define K_OFFSET_POS_SET 2
  97 #define K_OFFSET_NO_DATA 3
  98
  99 struct _archiveHandle;
 100 struct _tocEntry;
 101 struct _restoreList;
 102
 103 typedef void (*ClosePtr) (struct _archiveHandle * AH);
 104 typedef void (*ReopenPtr) (struct _archiveHandle * AH);
 105 typedef void (*ArchiveEntryPtr) (struct _archiveHandle * AH, struct _tocEntry * te);
 106
 107 typedef void (*StartDataPtr) (struct _archiveHandle * AH, struct _tocEntry * te);
 108 typedef size_t (*WriteDataPtr) (struct _archiveHandle * AH, const void *data, size_t dLen);
 109 typedef void (*EndDataPtr) (struct _archiveHandle * AH, struct _tocEntry * te);
 110
 111 typedef void (*StartBlobsPtr) (struct _archiveHandle * AH, struct _tocEntry * te);
 112 typedef void (*StartBlobPtr) (struct _archiveHandle * AH, struct _tocEntry * te, Oid oid);
 113 typedef void (*EndBlobPtr) (struct _archiveHandle * AH, struct _tocEntry * te, Oid oid);
 114 typedef void (*EndBlobsPtr) (struct _archiveHandle * AH, struct _tocEntry * te);
 115
 116 typedef int (*WriteBytePtr) (struct _archiveHandle * AH, const int i);
 117 typedef int (*ReadBytePtr) (struct _archiveHandle * AH);
 118 typedef size_t (*WriteBufPtr) (struct _archiveHandle * AH, const void *c, size_t len);
 119 typedef size_t (*ReadBufPtr) (struct _archiveHandle * AH, void *buf, size_t len);
 120 typedef void (*SaveArchivePtr) (struct _archiveHandle * AH);
 121 typedef void (*WriteExtraTocPtr) (struct _archiveHandle * AH, struct _tocEntry * te);
 122 typedef void (*ReadExtraTocPtr) (struct _archiveHandle * AH, struct _tocEntry * te);
 123 typedef void (*PrintExtraTocPtr) (struct _archiveHandle * AH, struct _tocEntry * te);
 124 typedef void (*PrintTocDataPtr) (struct _archiveHandle * AH, struct _tocEntry * te, RestoreOptions *ropt);
 125
 126 typedef void (*ClonePtr) (struct _archiveHandle * AH);
 127 typedef void (*DeClonePtr) (struct _archiveHandle * AH);
 128
 129 typedef size_t (*CustomOutPtr) (struct _archiveHandle * AH, const void *buf, size_t len);
 130
 131 typedef struct _outputContext
 132 {
 133         void       *OF;
 134         int                     gzOut;
 135 } OutputContext;
 136
 137 typedef enum
 138 {
 139         SQL_SCAN = 0,                           /* normal */
 140         SQL_IN_SQL_COMMENT,                     /* -- comment */
 141         SQL_IN_EXT_COMMENT,                     /* slash-star comment */
 142         SQL_IN_SINGLE_QUOTE,            /* '...' literal */
 143         SQL_IN_E_QUOTE,                         /* E'...' literal */
 144         SQL_IN_DOUBLE_QUOTE,            /* "..." identifier */
 145         SQL_IN_DOLLAR_TAG,                      /* possible dollar-quote starting tag */
 146         SQL_IN_DOLLAR_QUOTE                     /* body of dollar quote */
 147 } sqlparseState;
 148
 149 typedef struct
 150 {
 151         sqlparseState state;            /* see above */
 152         char            lastChar;               /* preceding char, or '\0' initially */
 153         bool            backSlash;              /* next char is backslash quoted? */
 154         int                     braceDepth;             /* parenthesis nesting depth */
 155         PQExpBuffer tagBuf;                     /* dollar quote tag (NULL if not created) */
 156         int                     minTagEndPos;   /* first possible end position of $-quote */
 157 } sqlparseInfo;
 158
 159 typedef enum
 160 {
 161         STAGE_NONE = 0,
 162         STAGE_INITIALIZING,
 163         STAGE_PROCESSING,
 164         STAGE_FINALIZING
 165 } ArchiverStage;
 166
 167 typedef enum
 168 {
 169         REQ_SCHEMA = 1,
 170         REQ_DATA = 2,
 171         REQ_ALL = REQ_SCHEMA + REQ_DATA
 172 } teReqs;
 173
 174 typedef struct _archiveHandle
 175 {
 176         Archive         public;                 /* Public part of archive */
 177         char            vmaj;                   /* Version of file */
 178         char            vmin;
 179         char            vrev;
 180         int                     version;                /* Conveniently formatted version */
 181
 182         char       *archiveRemoteVersion;       /* When reading an archive, the
 183                                                                                  * version of the dumped DB */
 184         char       *archiveDumpVersion;         /* When reading an archive, the
 185                                                                                  * version of the dumper */
 186
 187         int                     debugLevel;             /* Used for logging (currently only by
 188                                                                  * --verbose) */
 189         size_t          intSize;                /* Size of an integer in the archive */
 190         size_t          offSize;                /* Size of a file offset in the archive -
 191                                                                  * Added V1.7 */
 192         ArchiveFormat format;           /* Archive format */
 193
 194         sqlparseInfo sqlparse;
 195         PQExpBuffer sqlBuf;
 196
 197         time_t          createDate;             /* Date archive created */
 198
 199         /*
 200          * Fields used when discovering header. A format can always get the
 201          * previous read bytes from here...
 202          */
 203         int                     readHeader;             /* Used if file header has been read already */
 204         char       *lookahead;          /* Buffer used when reading header to discover
 205                                                                  * format */
 206         size_t          lookaheadSize;  /* Size of allocated buffer */
 207         size_t          lookaheadLen;   /* Length of data in lookahead */
 208         pgoff_t         lookaheadPos;   /* Current read position in lookahead buffer */
 209
 210         ArchiveEntryPtr ArchiveEntryPtr;        /* Called for each metadata object */
 211         StartDataPtr StartDataPtr;      /* Called when table data is about to be
 212                                                                  * dumped */
 213         WriteDataPtr WriteDataPtr;      /* Called to send some table data to the
 214                                                                  * archive */
 215         EndDataPtr EndDataPtr;          /* Called when table data dump is finished */
 216         WriteBytePtr WriteBytePtr;      /* Write a byte to output */
 217         ReadBytePtr ReadBytePtr;        /* Read a byte from an archive */
 218         WriteBufPtr WriteBufPtr;        /* Write a buffer of output to the archive */
 219         ReadBufPtr ReadBufPtr;          /* Read a buffer of input from the archive */
 220         ClosePtr ClosePtr;                      /* Close the archive */
 221         ReopenPtr ReopenPtr;            /* Reopen the archive */
 222         WriteExtraTocPtr WriteExtraTocPtr;      /* Write extra TOC entry data
 223                                                                                  * associated with the current archive
 224                                                                                  * format */
 225         ReadExtraTocPtr ReadExtraTocPtr;        /* Read extr info associated with
 226                                                                                  * archie format */
 227         PrintExtraTocPtr PrintExtraTocPtr;      /* Extra TOC info for format */
 228         PrintTocDataPtr PrintTocDataPtr;
 229
 230         StartBlobsPtr StartBlobsPtr;
 231         EndBlobsPtr EndBlobsPtr;
 232         StartBlobPtr StartBlobPtr;
 233         EndBlobPtr EndBlobPtr;
 234
 235         ClonePtr ClonePtr;                      /* Clone format-specific fields */
 236         DeClonePtr DeClonePtr;          /* Clean up cloned fields */
 237
 238         CustomOutPtr CustomOutPtr;      /* Alternative script output routine */
 239
 240         /* Stuff for direct DB connection */
 241         char       *archdbname;         /* DB name *read* from archive */
 242         enum trivalue promptPassword;
 243         char       *savedPassword;      /* password for ropt->username, if known */
 244         PGconn     *connection;
 245         int                     connectToDB;    /* Flag to indicate if direct DB connection is
 246                                                                  * required */
 247         bool            writingCopyData;        /* True when we are sending COPY data */
 248         bool            pgCopyIn;               /* Currently in libpq 'COPY IN' mode. */
 249         PQExpBuffer pgCopyBuf;          /* Left-over data from incomplete lines in
 250                                                                  * COPY IN */
 251
 252         int                     loFd;                   /* BLOB fd */
 253         int                     writingBlob;    /* Flag */
 254         int                     blobCount;              /* # of blobs restored */
 255
 256         char       *fSpec;                      /* Archive File Spec */
 257         FILE       *FH;                         /* General purpose file handle */
 258         void       *OF;
 259         int                     gzOut;                  /* Output file */
 260
 261         struct _tocEntry *toc;          /* List of TOC entries */
 262         int                     tocCount;               /* Number of TOC entries */
 263         DumpId          maxDumpId;              /* largest DumpId among all TOC entries */
 264
 265         struct _tocEntry *currToc;      /* Used when dumping data */
 266         int                     compression;    /* Compression requested on open */
 267         ArchiveMode mode;                       /* File mode - r or w */
 268         void       *formatData;         /* Header data specific to file format */
 269
 270         RestoreOptions *ropt;           /* Used to check restore options in ahwrite
 271                                                                  * etc */
 272
 273         /* these vars track state to avoid sending redundant SET commands */
 274         char       *currUser;           /* current username, or NULL if unknown */
 275         char       *currSchema;         /* current schema, or NULL */
 276         char       *currTablespace; /* current tablespace, or NULL */
 277         bool            currWithOids;   /* current default_with_oids setting */
 278
 279         void       *lo_buf;
 280         size_t          lo_buf_used;
 281         size_t          lo_buf_size;
 282
 283         int                     noTocComments;
 284         ArchiverStage stage;
 285         ArchiverStage lastErrorStage;
 286         struct _tocEntry *currentTE;
 287         struct _tocEntry *lastErrorTE;
 288 } ArchiveHandle;
 289
 290 typedef struct _tocEntry
 291 {
 292         struct _tocEntry *prev;
 293         struct _tocEntry *next;
 294         CatalogId       catalogId;
 295         DumpId          dumpId;
 296         teSection       section;
 297         bool            hadDumper;              /* Archiver was passed a dumper routine (used
 298                                                                  * in restore) */
 299         char       *tag;                        /* index tag */
 300         char       *namespace;          /* null or empty string if not in a schema */
 301         char       *tablespace;         /* null if not in a tablespace; empty string
 302                                                                  * means use database default */
 303         char       *owner;
 304         bool            withOids;               /* Used only by "TABLE" tags */
 305         char       *desc;
 306         char       *defn;
 307         char       *dropStmt;
 308         char       *copyStmt;
 309         DumpId     *dependencies;       /* dumpIds of objects this one depends on */
 310         int                     nDeps;                  /* number of dependencies */
 311
 312         DataDumperPtr dataDumper;       /* Routine to dump data for object */
 313         void       *dataDumperArg;      /* Arg for above routine */
 314         void       *formatData;         /* TOC Entry data specific to file format */
 315
 316         /* working state (needed only for parallel restore) */
 317         bool            restored;               /* item is in progress or done */
 318         bool            created;                /* set for DATA member if TABLE was created */
 319         int                     depCount;               /* number of dependencies not yet restored */
 320         DumpId     *lockDeps;           /* dumpIds of objects this one needs lock on */
 321         int                     nLockDeps;              /* number of such dependencies */
 322 } TocEntry;
 323
 324 /* Used everywhere */
 325 extern const char *progname;
 326
 327 extern void die_horribly(ArchiveHandle *AH, const char *modulename, const char *fmt,...) __attribute__((format(printf, 3, 4)));
 328 extern void warn_or_die_horribly(ArchiveHandle *AH, const char *modulename, const char *fmt,...) __attribute__((format(printf, 3, 4)));
 329 extern void write_msg(const char *modulename, const char *fmt,...) __attribute__((format(printf, 2, 3)));
 330
 331 extern void WriteTOC(ArchiveHandle *AH);
 332 extern void ReadTOC(ArchiveHandle *AH);
 333 extern void WriteHead(ArchiveHandle *AH);
 334 extern void ReadHead(ArchiveHandle *AH);
 335 extern void WriteToc(ArchiveHandle *AH);
 336 extern void ReadToc(ArchiveHandle *AH);
 337 extern void WriteDataChunks(ArchiveHandle *AH);
 338
 339 extern teReqs TocIDRequired(ArchiveHandle *AH, DumpId id, RestoreOptions *ropt);
 340 extern bool checkSeek(FILE *fp);
 341
 342 #define appendStringLiteralAHX(buf,str,AH) \
 343         appendStringLiteral(buf, str, (AH)->public.encoding, (AH)->public.std_strings)
 344
 345 /*
 346  * Mandatory routines for each supported format
 347  */
 348
 349 extern size_t WriteInt(ArchiveHandle *AH, int i);
 350 extern int      ReadInt(ArchiveHandle *AH);
 351 extern char *ReadStr(ArchiveHandle *AH);
 352 extern size_t WriteStr(ArchiveHandle *AH, const char *s);
 353
 354 int                     ReadOffset(ArchiveHandle *, pgoff_t *);
 355 size_t          WriteOffset(ArchiveHandle *, pgoff_t, int);
 356
 357 extern void StartRestoreBlobs(ArchiveHandle *AH);
 358 extern void StartRestoreBlob(ArchiveHandle *AH, Oid oid);
 359 extern void EndRestoreBlob(ArchiveHandle *AH, Oid oid);
 360 extern void EndRestoreBlobs(ArchiveHandle *AH);
 361
 362 extern void InitArchiveFmt_Custom(ArchiveHandle *AH);
 363 extern void InitArchiveFmt_Files(ArchiveHandle *AH);
 364 extern void InitArchiveFmt_Null(ArchiveHandle *AH);
 365 extern void InitArchiveFmt_Tar(ArchiveHandle *AH);
 366
 367 extern bool isValidTarHeader(char *header);
 368
 369 extern int      ReconnectToServer(ArchiveHandle *AH, const char *dbname, const char *newUser);
 370
 371 int                     ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH);
 372 int                     ahprintf(ArchiveHandle *AH, const char *fmt,...) __attribute__((format(printf, 2, 3)));
 373
 374 void            ahlog(ArchiveHandle *AH, int level, const char *fmt,...) __attribute__((format(printf, 3, 4)));
 375
 376 #endif