contrib/pg_standby/pg_standby.c

   1 /*
   2  * $PostgreSQL$
   3  *
   4  *
   5  * pg_standby.c
   6  *
   7  * Production-ready example of how to create a Warm Standby
   8  * database server using continuous archiving as a
   9  * replication mechanism
  10  *
  11  * We separate the parameters for archive and nextWALfile
  12  * so that we can check the archive exists, even if the
  13  * WAL file doesn't (yet).
  14  *
  15  * This program will be executed once in full for each file
  16  * requested by the warm standby server.
  17  *
  18  * It is designed to cater to a variety of needs, as well
  19  * providing a customizable section.
  20  *
  21  * Original author:             Simon Riggs  simon@2ndquadrant.com
  22  * Current maintainer:  Simon Riggs
  23  */
  24 #include "postgres_fe.h"
  25
  26 #include <ctype.h>
  27 #include <dirent.h>
  28 #include <sys/stat.h>
  29 #include <signal.h>
  30
  31 #ifdef WIN32
  32 int                     getopt(int argc, char *const argv[], const char *optstring);
  33 #else
  34 #include <sys/time.h>
  35 #include <unistd.h>
  36
  37 #ifdef HAVE_GETOPT_H
  38 #include <getopt.h>
  39 #endif
  40 #endif   /* ! WIN32 */
  41
  42 extern char *optarg;
  43 extern int      optind;
  44
  45 /* Options and defaults */
  46 int                     sleeptime = 5;          /* amount of time to sleep between file checks */
  47 int                     waittime = -1;          /* how long we have been waiting, -1 no wait
  48                                                                  * yet */
  49 int                     maxwaittime = 0;        /* how long are we prepared to wait for? */
  50 int                     keepfiles = 0;          /* number of WAL files to keep, 0 keep all */
  51 int                     maxretries = 3;         /* number of retries on restore command */
  52 bool            debug = false;          /* are we debugging? */
  53 bool            triggered = false;      /* have we been triggered? */
  54 bool            need_cleanup = false;           /* do we need to remove files from
  55                                                                                  * archive? */
  56
  57 static volatile sig_atomic_t signaled = false;
  58
  59 char       *archiveLocation;    /* where to find the archive? */
  60 char       *triggerPath;                /* where to find the trigger file? */
  61 char       *xlogFilePath;               /* where we are going to restore to */
  62 char       *nextWALFileName;    /* the file we need to get from archive */
  63 char       *restartWALFileName; /* the file from which we can restart restore */
  64 char       *priorWALFileName;   /* the file we need to get from archive */
  65 char            WALFilePath[MAXPGPATH];         /* the file path including archive */
  66 char            restoreCommand[MAXPGPATH];      /* run this to restore */
  67 char            exclusiveCleanupFileName[MAXPGPATH];            /* the file we need to
  68                                                                                                                  * get from archive */
  69
  70 #define RESTORE_COMMAND_COPY 0
  71 #define RESTORE_COMMAND_LINK 1
  72 int                     restoreCommandType;
  73
  74 #define XLOG_DATA                        0
  75 #define XLOG_HISTORY             1
  76 #define XLOG_BACKUP_LABEL        2
  77 int                     nextWALFileType;
  78
  79 #define SET_RESTORE_COMMAND(cmd, arg1, arg2) \
  80         snprintf(restoreCommand, MAXPGPATH, cmd " \"%s\" \"%s\"", arg1, arg2)
  81
  82 struct stat stat_buf;
  83
  84 /* =====================================================================
  85  *
  86  *                Customizable section
  87  *
  88  * =====================================================================
  89  *
  90  *      Currently, this section assumes that the Archive is a locally
  91  *      accessible directory. If you want to make other assumptions,
  92  *      such as using a vendor-specific archive and access API, these
  93  *      routines are the ones you'll need to change. You're
  94  *      enouraged to submit any changes to pgsql-patches@postgresql.org
  95  *      or personally to the current maintainer. Those changes may be
  96  *      folded in to later versions of this program.
  97  */
  98
  99 #define XLOG_DATA_FNAME_LEN             24
 100 /* Reworked from access/xlog_internal.h */
 101 #define XLogFileName(fname, tli, log, seg)      \
 102         snprintf(fname, XLOG_DATA_FNAME_LEN + 1, "%08X%08X%08X", tli, log, seg)
 103
 104 /*
 105  *      Initialize allows customized commands into the warm standby program.
 106  *
 107  *      As an example, and probably the common case, we use either
 108  *      cp/ln commands on *nix, or copy/move command on Windows.
 109  *
 110  */
 111 static void
 112 CustomizableInitialize(void)
 113 {
 114 #ifdef WIN32
 115         snprintf(WALFilePath, MAXPGPATH, "%s\\%s", archiveLocation, nextWALFileName);
 116         switch (restoreCommandType)
 117         {
 118                 case RESTORE_COMMAND_LINK:
 119                         SET_RESTORE_COMMAND("mklink", WALFilePath, xlogFilePath);
 120                 case RESTORE_COMMAND_COPY:
 121                 default:
 122                         SET_RESTORE_COMMAND("copy", WALFilePath, xlogFilePath);
 123                         break;
 124         }
 125 #else
 126         snprintf(WALFilePath, MAXPGPATH, "%s/%s", archiveLocation, nextWALFileName);
 127         switch (restoreCommandType)
 128         {
 129                 case RESTORE_COMMAND_LINK:
 130 #if HAVE_WORKING_LINK
 131                         SET_RESTORE_COMMAND("ln -s -f", WALFilePath, xlogFilePath);
 132                         break;
 133 #endif
 134                 case RESTORE_COMMAND_COPY:
 135                 default:
 136                         SET_RESTORE_COMMAND("cp", WALFilePath, xlogFilePath);
 137                         break;
 138         }
 139 #endif
 140
 141         /*
 142          * This code assumes that archiveLocation is a directory You may wish to
 143          * add code to check for tape libraries, etc.. So, since it is a
 144          * directory, we use stat to test if its accessible
 145          */
 146         if (stat(archiveLocation, &stat_buf) != 0)
 147         {
 148                 fprintf(stderr, "pg_standby: archiveLocation \"%s\" does not exist\n", archiveLocation);
 149                 fflush(stderr);
 150                 exit(2);
 151         }
 152 }
 153
 154 /*
 155  * CustomizableNextWALFileReady()
 156  *
 157  *        Is the requested file ready yet?
 158  */
 159 static bool
 160 CustomizableNextWALFileReady()
 161 {
 162         if (stat(WALFilePath, &stat_buf) == 0)
 163         {
 164                 /*
 165                  * If its a backup file, return immediately If its a regular file
 166                  * return only if its the right size already
 167                  */
 168                 if (strlen(nextWALFileName) > 24 &&
 169                         strspn(nextWALFileName, "0123456789ABCDEF") == 24 &&
 170                 strcmp(nextWALFileName + strlen(nextWALFileName) - strlen(".backup"),
 171                            ".backup") == 0)
 172                 {
 173                         nextWALFileType = XLOG_BACKUP_LABEL;
 174                         return true;
 175                 }
 176                 else if (stat_buf.st_size == XLOG_SEG_SIZE)
 177                 {
 178 #ifdef WIN32
 179
 180                         /*
 181                          * Windows reports that the file has the right number of bytes
 182                          * even though the file is still being copied and cannot be opened
 183                          * by pg_standby yet. So we wait for sleeptime secs before
 184                          * attempting to restore. If that is not enough, we will rely on
 185                          * the retry/holdoff mechanism.
 186                          */
 187                         pg_usleep(sleeptime * 1000000L);
 188 #endif
 189                         nextWALFileType = XLOG_DATA;
 190                         return true;
 191                 }
 192
 193                 /*
 194                  * If still too small, wait until it is the correct size
 195                  */
 196                 if (stat_buf.st_size > XLOG_SEG_SIZE)
 197                 {
 198                         if (debug)
 199                         {
 200                                 fprintf(stderr, "file size greater than expected\n");
 201                                 fflush(stderr);
 202                         }
 203                         exit(3);
 204                 }
 205         }
 206
 207         return false;
 208 }
 209
 210 #define MaxSegmentsPerLogFile ( 0xFFFFFFFF / XLOG_SEG_SIZE )
 211
 212 static void
 213 CustomizableCleanupPriorWALFiles(void)
 214 {
 215         /*
 216          * Work out name of prior file from current filename
 217          */
 218         if (nextWALFileType == XLOG_DATA)
 219         {
 220                 int                     rc;
 221                 DIR                *xldir;
 222                 struct dirent *xlde;
 223
 224                 /*
 225                  * Assume its OK to keep failing. The failure situation may change
 226                  * over time, so we'd rather keep going on the main processing than
 227                  * fail because we couldnt clean up yet.
 228                  */
 229                 if ((xldir = opendir(archiveLocation)) != NULL)
 230                 {
 231                         while ((xlde = readdir(xldir)) != NULL)
 232                         {
 233                                 /*
 234                                  * We ignore the timeline part of the XLOG segment identifiers
 235                                  * in deciding whether a segment is still needed.  This
 236                                  * ensures that we won't prematurely remove a segment from a
 237                                  * parent timeline. We could probably be a little more
 238                                  * proactive about removing segments of non-parent timelines,
 239                                  * but that would be a whole lot more complicated.
 240                                  *
 241                                  * We use the alphanumeric sorting property of the filenames
 242                                  * to decide which ones are earlier than the
 243                                  * exclusiveCleanupFileName file. Note that this means files
 244                                  * are not removed in the order they were originally written,
 245                                  * in case this worries you.
 246                                  */
 247                                 if (strlen(xlde->d_name) == XLOG_DATA_FNAME_LEN &&
 248                                         strspn(xlde->d_name, "0123456789ABCDEF") == XLOG_DATA_FNAME_LEN &&
 249                                   strcmp(xlde->d_name + 8, exclusiveCleanupFileName + 8) < 0)
 250                                 {
 251 #ifdef WIN32
 252                                         snprintf(WALFilePath, MAXPGPATH, "%s\\%s", archiveLocation, xlde->d_name);
 253 #else
 254                                         snprintf(WALFilePath, MAXPGPATH, "%s/%s", archiveLocation, xlde->d_name);
 255 #endif
 256
 257                                         if (debug)
 258                                                 fprintf(stderr, "\nremoving \"%s\"", WALFilePath);
 259
 260                                         rc = unlink(WALFilePath);
 261                                         if (rc != 0)
 262                                         {
 263                                                 fprintf(stderr, "\npg_standby: ERROR failed to remove \"%s\": %s",
 264                                                                 WALFilePath, strerror(errno));
 265                                                 break;
 266                                         }
 267                                 }
 268                         }
 269                         if (debug)
 270                                 fprintf(stderr, "\n");
 271                 }
 272                 else
 273                         fprintf(stderr, "pg_standby: archiveLocation \"%s\" open error\n", archiveLocation);
 274
 275                 closedir(xldir);
 276                 fflush(stderr);
 277         }
 278 }
 279
 280 /* =====================================================================
 281  *                End of Customizable section
 282  * =====================================================================
 283  */
 284
 285 /*
 286  * SetWALFileNameForCleanup()
 287  *
 288  *        Set the earliest WAL filename that we want to keep on the archive
 289  *        and decide whether we need_cleanup
 290  */
 291 static bool
 292 SetWALFileNameForCleanup(void)
 293 {
 294         uint32          tli = 1,
 295                                 log = 0,
 296                                 seg = 0;
 297         uint32          log_diff = 0,
 298                                 seg_diff = 0;
 299         bool            cleanup = false;
 300
 301         if (restartWALFileName)
 302         {
 303                 /*
 304                  * Don't do cleanup if the restartWALFileName provided
 305                  * is later than the xlog file requested. This is an error
 306                  * and we must not remove these files from archive.
 307                  * This shouldn't happen, but better safe than sorry.
 308                  */
 309                 if (strcmp(restartWALFileName, nextWALFileName) > 0)
 310                         return false;
 311
 312                 strcpy(exclusiveCleanupFileName, restartWALFileName);
 313                 return true;
 314         }
 315
 316         if (keepfiles > 0)
 317         {
 318                 sscanf(nextWALFileName, "%08X%08X%08X", &tli, &log, &seg);
 319                 if (tli > 0 && log >= 0 && seg > 0)
 320                 {
 321                         log_diff = keepfiles / MaxSegmentsPerLogFile;
 322                         seg_diff = keepfiles % MaxSegmentsPerLogFile;
 323                         if (seg_diff > seg)
 324                         {
 325                                 log_diff++;
 326                                 seg = MaxSegmentsPerLogFile - (seg_diff - seg);
 327                         }
 328                         else
 329                                 seg -= seg_diff;
 330
 331                         if (log >= log_diff)
 332                         {
 333                                 log -= log_diff;
 334                                 cleanup = true;
 335                         }
 336                         else
 337                         {
 338                                 log = 0;
 339                                 seg = 0;
 340                         }
 341                 }
 342         }
 343
 344         XLogFileName(exclusiveCleanupFileName, tli, log, seg);
 345
 346         return cleanup;
 347 }
 348
 349 /*
 350  * CheckForExternalTrigger()
 351  *
 352  *        Is there a trigger file?
 353  */
 354 static bool
 355 CheckForExternalTrigger(void)
 356 {
 357         int                     rc;
 358
 359         /*
 360          * Look for a trigger file, if that option has been selected
 361          *
 362          * We use stat() here because triggerPath is always a file rather than
 363          * potentially being in an archive
 364          */
 365         if (triggerPath && stat(triggerPath, &stat_buf) == 0)
 366         {
 367                 fprintf(stderr, "trigger file found\n");
 368                 fflush(stderr);
 369
 370                 /*
 371                  * If trigger file found, we *must* delete it. Here's why: When
 372                  * recovery completes, we will be asked again for the same file from
 373                  * the archive using pg_standby so must remove trigger file so we can
 374                  * reload file again and come up correctly.
 375                  */
 376                 rc = unlink(triggerPath);
 377                 if (rc != 0)
 378                 {
 379                         fprintf(stderr, "\n ERROR: could not remove \"%s\": %s", triggerPath, strerror(errno));
 380                         fflush(stderr);
 381                         exit(rc);
 382                 }
 383                 return true;
 384         }
 385
 386         return false;
 387 }
 388
 389 /*
 390  * RestoreWALFileForRecovery()
 391  *
 392  *        Perform the action required to restore the file from archive
 393  */
 394 static bool
 395 RestoreWALFileForRecovery(void)
 396 {
 397         int                     rc = 0;
 398         int                     numretries = 0;
 399
 400         if (debug)
 401         {
 402                 fprintf(stderr, "\nrunning restore              :");
 403                 fflush(stderr);
 404         }
 405
 406         while (numretries < maxretries)
 407         {
 408                 rc = system(restoreCommand);
 409                 if (rc == 0)
 410                 {
 411                         if (debug)
 412                         {
 413                                 fprintf(stderr, " OK");
 414                                 fflush(stderr);
 415                         }
 416                         return true;
 417                 }
 418                 pg_usleep(numretries++ * sleeptime * 1000000L);
 419         }
 420
 421         /*
 422          * Allow caller to add additional info
 423          */
 424         if (debug)
 425                 fprintf(stderr, "not restored           : ");
 426         return false;
 427 }
 428
 429 static void
 430 usage(void)
 431 {
 432         fprintf(stderr, "\npg_standby allows Warm Standby servers to be configured\n");
 433         fprintf(stderr, "Usage:\n");
 434         fprintf(stderr, "  pg_standby [OPTION]... ARCHIVELOCATION NEXTWALFILE XLOGFILEPATH [RESTARTWALFILE]\n");
 435         fprintf(stderr, "                               note space between ARCHIVELOCATION and NEXTWALFILE\n");
 436         fprintf(stderr, "with main intended use as a restore_command in the recovery.conf\n");
 437         fprintf(stderr, "        restore_command = 'pg_standby [OPTION]... ARCHIVELOCATION %%f %%p %%r'\n");
 438         fprintf(stderr, "e.g. restore_command = 'pg_standby -l /mnt/server/archiverdir %%f %%p %%r'\n");
 439         fprintf(stderr, "\nOptions:\n");
 440         fprintf(stderr, "  -c                   copies file from archive (default)\n");
 441         fprintf(stderr, "  -d                   generate lots of debugging output (testing only)\n");
 442         fprintf(stderr, "  -k NUMFILESTOKEEP    if RESTARTWALFILE not used, removes files prior to limit (0 keeps all)\n");
 443         fprintf(stderr, "  -l                   links into archive (leaves file in archive)\n");
 444         fprintf(stderr, "  -r MAXRETRIES                max number of times to retry, with progressive wait (default=3)\n");
 445         fprintf(stderr, "  -s SLEEPTIME         seconds to wait between file checks (min=1, max=60, default=5)\n");
 446         fprintf(stderr, "  -t TRIGGERFILE       defines a trigger file to initiate failover (no default)\n");
 447         fprintf(stderr, "  -w MAXWAITTIME       max seconds to wait for a file (0=no limit)(default=0)\n");
 448         fflush(stderr);
 449 }
 450
 451 static void
 452 sighandler(int sig)
 453 {
 454         signaled = true;
 455 }
 456
 457 /*------------ MAIN ----------------------------------------*/
 458 int
 459 main(int argc, char **argv)
 460 {
 461         int                     c;
 462
 463         (void) signal(SIGINT, sighandler);
 464         (void) signal(SIGQUIT, sighandler);
 465
 466         while ((c = getopt(argc, argv, "cdk:lr:s:t:w:")) != -1)
 467         {
 468                 switch (c)
 469                 {
 470                         case 'c':                       /* Use copy */
 471                                 restoreCommandType = RESTORE_COMMAND_COPY;
 472                                 break;
 473                         case 'd':                       /* Debug mode */
 474                                 debug = true;
 475                                 break;
 476                         case 'k':                       /* keepfiles */
 477                                 keepfiles = atoi(optarg);
 478                                 if (keepfiles < 0)
 479                                 {
 480                                         fprintf(stderr, "usage: pg_standby -k keepfiles must be >= 0\n");
 481                                         usage();
 482                                         exit(2);
 483                                 }
 484                                 break;
 485                         case 'l':                       /* Use link */
 486                                 restoreCommandType = RESTORE_COMMAND_LINK;
 487                                 break;
 488                         case 'r':                       /* Retries */
 489                                 maxretries = atoi(optarg);
 490                                 if (maxretries < 0)
 491                                 {
 492                                         fprintf(stderr, "usage: pg_standby -r maxretries must be >= 0\n");
 493                                         usage();
 494                                         exit(2);
 495                                 }
 496                                 break;
 497                         case 's':                       /* Sleep time */
 498                                 sleeptime = atoi(optarg);
 499                                 if (sleeptime <= 0 || sleeptime > 60)
 500                                 {
 501                                         fprintf(stderr, "usage: pg_standby -s sleeptime incorrectly set\n");
 502                                         usage();
 503                                         exit(2);
 504                                 }
 505                                 break;
 506                         case 't':                       /* Trigger file */
 507                                 triggerPath = optarg;
 508                                 if (CheckForExternalTrigger())
 509                                         exit(1);        /* Normal exit, with non-zero */
 510                                 break;
 511                         case 'w':                       /* Max wait time */
 512                                 maxwaittime = atoi(optarg);
 513                                 if (maxwaittime < 0)
 514                                 {
 515                                         fprintf(stderr, "usage: pg_standby -w maxwaittime incorrectly set\n");
 516                                         usage();
 517                                         exit(2);
 518                                 }
 519                                 break;
 520                         default:
 521                                 usage();
 522                                 exit(2);
 523                                 break;
 524                 }
 525         }
 526
 527         /*
 528          * Parameter checking - after checking to see if trigger file present
 529          */
 530         if (argc == 1)
 531         {
 532                 usage();
 533                 exit(2);
 534         }
 535
 536         /*
 537          * We will go to the archiveLocation to get nextWALFileName.
 538          * nextWALFileName may not exist yet, which would not be an error, so we
 539          * separate the archiveLocation and nextWALFileName so we can check
 540          * separately whether archiveLocation exists, if not that is an error
 541          */
 542         if (optind < argc)
 543         {
 544                 archiveLocation = argv[optind];
 545                 optind++;
 546         }
 547         else
 548         {
 549                 fprintf(stderr, "pg_standby: must specify archiveLocation\n");
 550                 usage();
 551                 exit(2);
 552         }
 553
 554         if (optind < argc)
 555         {
 556                 nextWALFileName = argv[optind];
 557                 optind++;
 558         }
 559         else
 560         {
 561                 fprintf(stderr, "pg_standby: use %%f to specify nextWALFileName\n");
 562                 usage();
 563                 exit(2);
 564         }
 565
 566         if (optind < argc)
 567         {
 568                 xlogFilePath = argv[optind];
 569                 optind++;
 570         }
 571         else
 572         {
 573                 fprintf(stderr, "pg_standby: use %%p to specify xlogFilePath\n");
 574                 usage();
 575                 exit(2);
 576         }
 577
 578         if (optind < argc)
 579         {
 580                 restartWALFileName = argv[optind];
 581                 optind++;
 582         }
 583
 584         CustomizableInitialize();
 585
 586         need_cleanup = SetWALFileNameForCleanup();
 587
 588         if (debug)
 589         {
 590                 fprintf(stderr, "\nTrigger file                 : %s", triggerPath ? triggerPath : "<not set>");
 591                 fprintf(stderr, "\nWaiting for WAL file : %s", nextWALFileName);
 592                 fprintf(stderr, "\nWAL file path                : %s", WALFilePath);
 593                 fprintf(stderr, "\nRestoring to...              : %s", xlogFilePath);
 594                 fprintf(stderr, "\nSleep interval               : %d second%s",
 595                                 sleeptime, (sleeptime > 1 ? "s" : " "));
 596                 fprintf(stderr, "\nMax wait interval    : %d %s",
 597                                 maxwaittime, (maxwaittime > 0 ? "seconds" : "forever"));
 598                 fprintf(stderr, "\nCommand for restore  : %s", restoreCommand);
 599                 fprintf(stderr, "\nKeep archive history : ");
 600                 if (need_cleanup)
 601                         fprintf(stderr, "%s and later", exclusiveCleanupFileName);
 602                 else
 603                         fprintf(stderr, "No cleanup required");
 604                 fflush(stderr);
 605         }
 606
 607         /*
 608          * Check for initial history file: always the first file to be requested
 609          * It's OK if the file isn't there - all other files need to wait
 610          */
 611         if (strlen(nextWALFileName) > 8 &&
 612                 strspn(nextWALFileName, "0123456789ABCDEF") == 8 &&
 613                 strcmp(nextWALFileName + strlen(nextWALFileName) - strlen(".history"),
 614                            ".history") == 0)
 615         {
 616                 nextWALFileType = XLOG_HISTORY;
 617                 if (RestoreWALFileForRecovery())
 618                         exit(0);
 619                 else
 620                 {
 621                         if (debug)
 622                         {
 623                                 fprintf(stderr, "history file not found\n");
 624                                 fflush(stderr);
 625                         }
 626                         exit(1);
 627                 }
 628         }
 629
 630         /*
 631          * Main wait loop
 632          */
 633         while (!CustomizableNextWALFileReady() && !triggered)
 634         {
 635                 if (sleeptime <= 60)
 636                         pg_usleep(sleeptime * 1000000L);
 637
 638                 if (signaled)
 639                 {
 640                         triggered = true;
 641                         if (debug)
 642                         {
 643                                 fprintf(stderr, "\nsignaled to exit\n");
 644                                 fflush(stderr);
 645                         }
 646                 }
 647                 else
 648                 {
 649
 650                         if (debug)
 651                         {
 652                                 fprintf(stderr, "\nWAL file not present yet.");
 653                                 if (triggerPath)
 654                                         fprintf(stderr, " Checking for trigger file...");
 655                                 fflush(stderr);
 656                         }
 657
 658                         waittime += sleeptime;
 659
 660                         if (!triggered && (CheckForExternalTrigger() || (waittime >= maxwaittime && maxwaittime > 0)))
 661                         {
 662                                 triggered = true;
 663                                 if (debug && waittime >= maxwaittime && maxwaittime > 0)
 664                                         fprintf(stderr, "\nTimed out after %d seconds\n", waittime);
 665                         }
 666                 }
 667         }
 668
 669         /*
 670          * Action on exit
 671          */
 672         if (triggered)
 673                 exit(1);                                /* Normal exit, with non-zero */
 674
 675         /*
 676          * Once we have restored this file successfully we can remove some prior
 677          * WAL files. If this restore fails we musn't remove any file because some
 678          * of them will be requested again immediately after the failed restore,
 679          * or when we restart recovery.
 680          */
 681         if (RestoreWALFileForRecovery() && need_cleanup)
 682                 CustomizableCleanupPriorWALFiles();
 683
 684         return 0;
 685 }