src/bin/pg_archivecleanup/pg_archivecleanup.c

   1 /*
   2  * pg_archivecleanup.c
   3  *
   4  * To be used as archive_cleanup_command to clean an archive when using
   5  * standby mode.
   6  *
   7  * src/bin/pg_archivecleanup/pg_archivecleanup.c
   8  */
   9 #include "postgres_fe.h"
  10
  11 #include <ctype.h>
  12 #include <dirent.h>
  13 #include <sys/stat.h>
  14 #include <fcntl.h>
  15 #include <signal.h>
  16 #include <sys/time.h>
  17
  18 #include "access/xlog_internal.h"
  19 #include "common/logging.h"
  20 #include "getopt_long.h"
  21
  22 static const char *progname;
  23
  24 /* Options and defaults */
  25 static bool dryrun = false;             /* are we performing a dry-run operation? */
  26 static bool cleanBackupHistory = false; /* remove files including backup
  27                                                                                  * history files */
  28 static char *additional_ext = NULL; /* Extension to remove from filenames */
  29
  30 static char *archiveLocation;   /* where to find the archive? */
  31 static char *restartWALFileName;        /* the file from which we can restart
  32                                                                          * restore */
  33 static char exclusiveCleanupFileName[MAXFNAMELEN];      /* the oldest file we want
  34                                                                                                          * to remain in archive */
  35
  36
  37 /* =====================================================================
  38  *
  39  *                Customizable section
  40  *
  41  * =====================================================================
  42  *
  43  *      Currently, this section assumes that the Archive is a locally
  44  *      accessible directory. If you want to make other assumptions,
  45  *      such as using a vendor-specific archive and access API, these
  46  *      routines are the ones you'll need to change. You're
  47  *      encouraged to submit any changes to pgsql-hackers@lists.postgresql.org
  48  *      or personally to the current maintainer. Those changes may be
  49  *      folded in to later versions of this program.
  50  */
  51
  52 /*
  53  *      Initialize allows customized commands into the archive cleanup program.
  54  *
  55  *      You may wish to add code to check for tape libraries, etc..
  56  */
  57 static void
  58 Initialize(void)
  59 {
  60         /*
  61          * This code assumes that archiveLocation is a directory, so we use stat
  62          * to test if it's accessible.
  63          */
  64         struct stat stat_buf;
  65
  66         if (stat(archiveLocation, &stat_buf) != 0 ||
  67                 !S_ISDIR(stat_buf.st_mode))
  68         {
  69                 pg_log_error("archive location \"%s\" does not exist",
  70                                          archiveLocation);
  71                 exit(2);
  72         }
  73 }
  74
  75 static void
  76 TrimExtension(char *filename, char *extension)
  77 {
  78         int                     flen;
  79         int                     elen;
  80
  81         if (extension == NULL)
  82                 return;
  83
  84         elen = strlen(extension);
  85         flen = strlen(filename);
  86
  87         if (flen > elen && strcmp(filename + flen - elen, extension) == 0)
  88                 filename[flen - elen] = '\0';
  89 }
  90
  91 static void
  92 CleanupPriorWALFiles(void)
  93 {
  94         int                     rc;
  95         DIR                *xldir;
  96         struct dirent *xlde;
  97         char            walfile[MAXPGPATH];
  98
  99         xldir = opendir(archiveLocation);
 100         if (xldir == NULL)
 101                 pg_fatal("could not open archive location \"%s\": %m",
 102                                  archiveLocation);
 103
 104         while (errno = 0, (xlde = readdir(xldir)) != NULL)
 105         {
 106                 char            WALFilePath[MAXPGPATH * 2]; /* the file path including
 107                                                                                                  * archive */
 108
 109                 /*
 110                  * Truncation is essentially harmless, because we skip files whose
 111                  * format is different from WAL files and backup history files. (In
 112                  * principle, one could use a 1000-character additional_ext and get
 113                  * trouble.)
 114                  */
 115                 strlcpy(walfile, xlde->d_name, MAXPGPATH);
 116                 TrimExtension(walfile, additional_ext);
 117
 118                 /*
 119                  * Ignore anything does that not look like a WAL segment, a .partial
 120                  * WAL segment or a backup history file (if requested).
 121                  */
 122                 if (!IsXLogFileName(walfile) && !IsPartialXLogFileName(walfile) &&
 123                         !(cleanBackupHistory && IsBackupHistoryFileName(walfile)))
 124                         continue;
 125
 126                 /*
 127                  * We ignore the timeline part of the XLOG segment identifiers in
 128                  * deciding whether a segment is still needed.  This ensures that we
 129                  * won't prematurely remove a segment from a parent timeline. We could
 130                  * probably be a little more proactive about removing segments of
 131                  * non-parent timelines, but that would be a whole lot more
 132                  * complicated.
 133                  *
 134                  * We use the alphanumeric sorting property of the filenames to decide
 135                  * which ones are earlier than the exclusiveCleanupFileName file. Note
 136                  * that this means files are not removed in the order they were
 137                  * originally written, in case this worries you.
 138                  */
 139                 if (strcmp(walfile + 8, exclusiveCleanupFileName + 8) >= 0)
 140                         continue;
 141
 142                 /*
 143                  * Use the original file name again now, including any extension that
 144                  * might have been chopped off before testing the sequence.
 145                  */
 146                 snprintf(WALFilePath, sizeof(WALFilePath), "%s/%s",
 147                                  archiveLocation, xlde->d_name);
 148
 149                 if (dryrun)
 150                 {
 151                         /*
 152                          * Prints the name of the file to be removed and skips the actual
 153                          * removal.  The regular printout is so that the user can pipe the
 154                          * output into some other program.
 155                          */
 156                         printf("%s\n", WALFilePath);
 157                         pg_log_debug("file \"%s\" would be removed", WALFilePath);
 158                         continue;
 159                 }
 160
 161                 pg_log_debug("removing file \"%s\"", WALFilePath);
 162
 163                 rc = unlink(WALFilePath);
 164                 if (rc != 0)
 165                         pg_fatal("could not remove file \"%s\": %m",
 166                                          WALFilePath);
 167         }
 168
 169         if (errno)
 170                 pg_fatal("could not read archive location \"%s\": %m",
 171                                  archiveLocation);
 172         if (closedir(xldir))
 173                 pg_fatal("could not close archive location \"%s\": %m",
 174                                  archiveLocation);
 175 }
 176
 177 /*
 178  * SetWALFileNameForCleanup()
 179  *
 180  *        Set the earliest WAL filename that we want to keep on the archive
 181  *        and decide whether we need cleanup
 182  */
 183 static void
 184 SetWALFileNameForCleanup(void)
 185 {
 186         bool            fnameOK = false;
 187
 188         TrimExtension(restartWALFileName, additional_ext);
 189
 190         /*
 191          * If restartWALFileName is a WAL file name then just use it directly. If
 192          * restartWALFileName is a .partial or .backup filename, make sure we use
 193          * the prefix of the filename, otherwise we will remove wrong files since
 194          * 000000010000000000000010.partial and
 195          * 000000010000000000000010.00000020.backup are after
 196          * 000000010000000000000010.
 197          */
 198         if (IsXLogFileName(restartWALFileName))
 199         {
 200                 strcpy(exclusiveCleanupFileName, restartWALFileName);
 201                 fnameOK = true;
 202         }
 203         else if (IsPartialXLogFileName(restartWALFileName))
 204         {
 205                 int                     args;
 206                 uint32          tli = 1,
 207                                         log = 0,
 208                                         seg = 0;
 209
 210                 args = sscanf(restartWALFileName, "%08X%08X%08X.partial",
 211                                           &tli, &log, &seg);
 212                 if (args == 3)
 213                 {
 214                         fnameOK = true;
 215
 216                         /*
 217                          * Use just the prefix of the filename, ignore everything after
 218                          * first period
 219                          */
 220                         XLogFileNameById(exclusiveCleanupFileName, tli, log, seg);
 221                 }
 222         }
 223         else if (IsBackupHistoryFileName(restartWALFileName))
 224         {
 225                 int                     args;
 226                 uint32          tli = 1,
 227                                         log = 0,
 228                                         seg = 0,
 229                                         offset = 0;
 230
 231                 args = sscanf(restartWALFileName, "%08X%08X%08X.%08X.backup", &tli, &log, &seg, &offset);
 232                 if (args == 4)
 233                 {
 234                         fnameOK = true;
 235
 236                         /*
 237                          * Use just the prefix of the filename, ignore everything after
 238                          * first period
 239                          */
 240                         XLogFileNameById(exclusiveCleanupFileName, tli, log, seg);
 241                 }
 242         }
 243
 244         if (!fnameOK)
 245         {
 246                 pg_log_error("invalid file name argument");
 247                 pg_log_error_hint("Try \"%s --help\" for more information.", progname);
 248                 exit(2);
 249         }
 250 }
 251
 252 /* =====================================================================
 253  *                End of Customizable section
 254  * =====================================================================
 255  */
 256
 257 static void
 258 usage(void)
 259 {
 260         printf(_("%s removes older WAL files from PostgreSQL archives.\n\n"), progname);
 261         printf(_("Usage:\n"));
 262         printf(_("  %s [OPTION]... ARCHIVELOCATION OLDESTKEPTWALFILE\n"), progname);
 263         printf(_("\nOptions:\n"));
 264         printf(_("  -b, --clean-backup-history  clean up files including backup history files\n"));
 265         printf(_("  -d, --debug                 generate debug output (verbose mode)\n"));
 266         printf(_("  -n, --dry-run               dry run, show the names of the files that would be\n"
 267                          "                              removed\n"));
 268         printf(_("  -V, --version               output version information, then exit\n"));
 269         printf(_("  -x, --strip-extension=EXT   strip this extension before identifying files for\n"
 270                          "                              clean up\n"));
 271         printf(_("  -?, --help                  show this help, then exit\n"));
 272         printf(_("\n"
 273                          "For use as \"archive_cleanup_command\" in postgresql.conf:\n"
 274                          "  archive_cleanup_command = 'pg_archivecleanup [OPTION]... ARCHIVELOCATION %%r'\n"
 275                          "e.g.\n"
 276                          "  archive_cleanup_command = 'pg_archivecleanup /mnt/server/archiverdir %%r'\n"));
 277         printf(_("\n"
 278                          "Or for use as a standalone archive cleaner:\n"
 279                          "e.g.\n"
 280                          "  pg_archivecleanup /mnt/server/archiverdir 000000010000000000000010.00000020.backup\n"));
 281         printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
 282         printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
 283 }
 284
 285 /*------------ MAIN ----------------------------------------*/
 286 int
 287 main(int argc, char **argv)
 288 {
 289         static struct option long_options[] = {
 290                 {"clean-backup-history", no_argument, NULL, 'b'},
 291                 {"debug", no_argument, NULL, 'd'},
 292                 {"dry-run", no_argument, NULL, 'n'},
 293                 {"strip-extension", required_argument, NULL, 'x'},
 294                 {NULL, 0, NULL, 0}
 295         };
 296         int                     c;
 297
 298         pg_logging_init(argv[0]);
 299         set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_archivecleanup"));
 300         progname = get_progname(argv[0]);
 301
 302         if (argc > 1)
 303         {
 304                 if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
 305                 {
 306                         usage();
 307                         exit(0);
 308                 }
 309                 if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
 310                 {
 311                         puts("pg_archivecleanup (PostgreSQL) " PG_VERSION);
 312                         exit(0);
 313                 }
 314         }
 315
 316         while ((c = getopt_long(argc, argv, "bdnx:", long_options, NULL)) != -1)
 317         {
 318                 switch (c)
 319                 {
 320                         case 'b':                       /* Remove backup history files as well */
 321                                 cleanBackupHistory = true;
 322                                 break;
 323                         case 'd':                       /* Debug mode */
 324                                 pg_logging_increase_verbosity();
 325                                 break;
 326                         case 'n':                       /* Dry-Run mode */
 327                                 dryrun = true;
 328                                 break;
 329                         case 'x':
 330                                 additional_ext = pg_strdup(optarg); /* Extension to remove
 331                                                                                                          * from xlogfile names */
 332                                 break;
 333                         default:
 334                                 /* getopt already emitted a complaint */
 335                                 pg_log_error_hint("Try \"%s --help\" for more information.", progname);
 336                                 exit(2);
 337                 }
 338         }
 339
 340         /*
 341          * We will go to the archiveLocation to check restartWALFileName.
 342          * restartWALFileName may not exist anymore, which would not be an error,
 343          * so we separate the archiveLocation and restartWALFileName so we can
 344          * check separately whether archiveLocation exists, if not that is an
 345          * error
 346          */
 347         if (optind < argc)
 348         {
 349                 archiveLocation = argv[optind];
 350                 optind++;
 351         }
 352         else
 353         {
 354                 pg_log_error("must specify archive location");
 355                 pg_log_error_hint("Try \"%s --help\" for more information.", progname);
 356                 exit(2);
 357         }
 358
 359         if (optind < argc)
 360         {
 361                 restartWALFileName = argv[optind];
 362                 optind++;
 363         }
 364         else
 365         {
 366                 pg_log_error("must specify oldest kept WAL file");
 367                 pg_log_error_hint("Try \"%s --help\" for more information.", progname);
 368                 exit(2);
 369         }
 370
 371         if (optind < argc)
 372         {
 373                 pg_log_error("too many command-line arguments");
 374                 pg_log_error_hint("Try \"%s --help\" for more information.", progname);
 375                 exit(2);
 376         }
 377
 378         /*
 379          * Check archive exists and other initialization if required.
 380          */
 381         Initialize();
 382
 383         /*
 384          * Check filename is a valid name, then process to find cut-off
 385          */
 386         SetWALFileNameForCleanup();
 387
 388         pg_log_debug("keeping WAL file \"%s/%s\" and later",
 389                                  archiveLocation, exclusiveCleanupFileName);
 390
 391         /*
 392          * Remove WAL files older than cut-off
 393          */
 394         CleanupPriorWALFiles();
 395
 396         exit(0);
 397 }