src/bin/pg_basebackup/t/020_pg_receivewal.pl

   1
   2 # Copyright (c) 2021-2025, PostgreSQL Global Development Group
   3
   4 use strict;
   5 use warnings FATAL => 'all';
   6 use PostgreSQL::Test::Utils;
   7 use PostgreSQL::Test::Cluster;
   8 use Test::More;
   9
  10 program_help_ok('pg_receivewal');
  11 program_version_ok('pg_receivewal');
  12 program_options_handling_ok('pg_receivewal');
  13
  14 # Set umask so test directories and files are created with default permissions
  15 umask(0077);
  16
  17 my $primary = PostgreSQL::Test::Cluster->new('primary');
  18 $primary->init(allows_streaming => 1, extra => ['--wal-segsize=1']);
  19 $primary->start;
  20
  21 my $stream_dir = $primary->basedir . '/archive_wal';
  22 mkdir($stream_dir);
  23
  24 # Sanity checks for command line options.
  25 $primary->command_fails(['pg_receivewal'],
  26         'pg_receivewal needs target directory specified');
  27 $primary->command_fails(
  28         [
  29                 'pg_receivewal',
  30                 '--directory' => $stream_dir,
  31                 '--create-slot',
  32                 '--drop-slot',
  33         ],
  34         'failure if both --create-slot and --drop-slot specified');
  35 $primary->command_fails(
  36         [ 'pg_receivewal', '--directory' => $stream_dir, '--create-slot' ],
  37         'failure if --create-slot specified without --slot');
  38 $primary->command_fails(
  39         [
  40                 'pg_receivewal',
  41                 '--directory' => $stream_dir,
  42                 '--synchronous',
  43                 '--no-sync',
  44         ],
  45         'failure if --synchronous specified with --no-sync');
  46 $primary->command_fails_like(
  47         [
  48                 'pg_receivewal',
  49                 '--directory' => $stream_dir,
  50                 '--compress' => 'none:1',
  51         ],
  52         qr/\Qpg_receivewal: error: invalid compression specification: compression algorithm "none" does not accept a compression level/,
  53         'failure if --compress none:N (where N > 0)');
  54
  55 # Slot creation and drop
  56 my $slot_name = 'test';
  57 $primary->command_ok(
  58         [ 'pg_receivewal', '--slot' => $slot_name, '--create-slot' ],
  59         'creating a replication slot');
  60 my $slot = $primary->slot($slot_name);
  61 is($slot->{'slot_type'}, 'physical', 'physical replication slot was created');
  62 is($slot->{'restart_lsn'}, '', 'restart LSN of new slot is null');
  63 $primary->command_ok(
  64         [ 'pg_receivewal', '--slot' => $slot_name, '--drop-slot' ],
  65         'dropping a replication slot');
  66 is($primary->slot($slot_name)->{'slot_type'},
  67         '', 'replication slot was removed');
  68
  69 # Generate some WAL.  Use --synchronous at the same time to add more
  70 # code coverage.  Switch to the next segment first so that subsequent
  71 # restarts of pg_receivewal will see this segment as full..
  72 $primary->psql('postgres', 'CREATE TABLE test_table(x integer PRIMARY KEY);');
  73 $primary->psql('postgres', 'SELECT pg_switch_wal();');
  74 my $nextlsn =
  75   $primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
  76 chomp($nextlsn);
  77 $primary->psql('postgres', 'INSERT INTO test_table VALUES (1);');
  78
  79 # Stream up to the given position.  This is necessary to have a fixed
  80 # started point for the next commands done in this test, with or without
  81 # compression involved.
  82 $primary->command_ok(
  83         [
  84                 'pg_receivewal',
  85                 '--directory' => $stream_dir,
  86                 '--verbose',
  87                 '--endpos' => $nextlsn,
  88                 '--synchronous',
  89                 '--no-loop',
  90         ],
  91         'streaming some WAL with --synchronous');
  92
  93 # Verify that one partial file was generated and keep track of it
  94 my @partial_wals = glob "$stream_dir/*\.partial";
  95 is(scalar(@partial_wals), 1, "one partial WAL segment was created");
  96
  97 note "Testing pg_receivewal with compression methods";
  98
  99 # Check ZLIB compression if available.
 100 SKIP:
 101 {
 102         skip "postgres was not built with ZLIB support", 5
 103           if (!check_pg_config("#define HAVE_LIBZ 1"));
 104
 105         # Generate more WAL worth one completed, compressed, segment.
 106         $primary->psql('postgres', 'SELECT pg_switch_wal();');
 107         $nextlsn =
 108           $primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
 109         chomp($nextlsn);
 110         $primary->psql('postgres', 'INSERT INTO test_table VALUES (2);');
 111
 112         $primary->command_ok(
 113                 [
 114                         'pg_receivewal',
 115                         '--directory' => $stream_dir,
 116                         '--verbose',
 117                         '--endpos' => $nextlsn,
 118                         '--compress' => 'gzip:1',
 119                         '--no-loop'
 120                 ],
 121                 "streaming some WAL using ZLIB compression");
 122
 123         # Verify that the stored files are generated with their expected
 124         # names.
 125         my @zlib_wals = glob "$stream_dir/*.gz";
 126         is(scalar(@zlib_wals), 1,
 127                 "one WAL segment compressed with ZLIB was created");
 128         my @zlib_partial_wals = glob "$stream_dir/*.gz.partial";
 129         is(scalar(@zlib_partial_wals),
 130                 1, "one partial WAL segment compressed with ZLIB was created");
 131
 132         # Verify that the start streaming position is computed correctly by
 133         # comparing it with the partial file generated previously.  The name
 134         # of the previous partial, now-completed WAL segment is updated, keeping
 135         # its base number.
 136         $partial_wals[0] =~ s/\.partial$/.gz/;
 137         is($zlib_wals[0] eq $partial_wals[0],
 138                 1, "one partial WAL segment is now completed");
 139         # Update the list of partial wals with the current one.
 140         @partial_wals = @zlib_partial_wals;
 141
 142         # Check the integrity of the completed segment, if gzip is a command
 143         # available.
 144         my $gzip = $ENV{GZIP_PROGRAM};
 145         skip "program gzip is not found in your system", 1
 146           if (!defined $gzip
 147                 || $gzip eq '');
 148
 149         my $gzip_is_valid = system_log($gzip, '--test', @zlib_wals);
 150         is($gzip_is_valid, 0,
 151                 "gzip verified the integrity of compressed WAL segments");
 152 }
 153
 154 # Check LZ4 compression if available
 155 SKIP:
 156 {
 157         skip "postgres was not built with LZ4 support", 5
 158           if (!check_pg_config("#define USE_LZ4 1"));
 159
 160         # Generate more WAL including one completed, compressed segment.
 161         $primary->psql('postgres', 'SELECT pg_switch_wal();');
 162         $nextlsn =
 163           $primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
 164         chomp($nextlsn);
 165         $primary->psql('postgres', 'INSERT INTO test_table VALUES (3);');
 166
 167         # Stream up to the given position.
 168         $primary->command_ok(
 169                 [
 170                         'pg_receivewal',
 171                         '--directory' => $stream_dir,
 172                         '--verbose',
 173                         '--endpos' => $nextlsn,
 174                         '--no-loop',
 175                         '--compress' => 'lz4'
 176                 ],
 177                 'streaming some WAL using --compress=lz4');
 178
 179         # Verify that the stored files are generated with their expected
 180         # names.
 181         my @lz4_wals = glob "$stream_dir/*.lz4";
 182         is(scalar(@lz4_wals), 1,
 183                 "one WAL segment compressed with LZ4 was created");
 184         my @lz4_partial_wals = glob "$stream_dir/*.lz4.partial";
 185         is(scalar(@lz4_partial_wals),
 186                 1, "one partial WAL segment compressed with LZ4 was created");
 187
 188         # Verify that the start streaming position is computed correctly by
 189         # comparing it with the partial file generated previously.  The name
 190         # of the previous partial, now-completed WAL segment is updated, keeping
 191         # its base number.
 192         $partial_wals[0] =~ s/(\.gz)?\.partial$/.lz4/;
 193         is($lz4_wals[0] eq $partial_wals[0],
 194                 1, "one partial WAL segment is now completed");
 195         # Update the list of partial wals with the current one.
 196         @partial_wals = @lz4_partial_wals;
 197
 198         # Check the integrity of the completed segment, if LZ4 is an available
 199         # command.
 200         my $lz4 = $ENV{LZ4};
 201         skip "program lz4 is not found in your system", 1
 202           if (!defined $lz4
 203                 || $lz4 eq '');
 204
 205         my $lz4_is_valid = system_log($lz4, '-t', @lz4_wals);
 206         is($lz4_is_valid, 0,
 207                 "lz4 verified the integrity of compressed WAL segments");
 208 }
 209
 210 # Verify that the start streaming position is computed and that the value is
 211 # correct regardless of whether any compression is available.
 212 $primary->psql('postgres', 'SELECT pg_switch_wal();');
 213 $nextlsn =
 214   $primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
 215 chomp($nextlsn);
 216 $primary->psql('postgres', 'INSERT INTO test_table VALUES (4);');
 217 $primary->command_ok(
 218         [
 219                 'pg_receivewal',
 220                 '--directory' => $stream_dir,
 221                 '--verbose',
 222                 '--endpos' => $nextlsn,
 223                 '--no-loop'
 224         ],
 225         "streaming some WAL");
 226
 227 $partial_wals[0] =~ s/(\.gz|\.lz4)?.partial//;
 228 ok(-e $partial_wals[0], "check that previously partial WAL is now complete");
 229
 230 # Permissions on WAL files should be default
 231 SKIP:
 232 {
 233         skip "unix-style permissions not supported on Windows", 1
 234           if ($windows_os);
 235
 236         ok(check_mode_recursive($stream_dir, 0700, 0600),
 237                 "check stream dir permissions");
 238 }
 239
 240 note "Testing pg_receivewal with slot as starting streaming point";
 241
 242 # When using a replication slot, archiving should be resumed from the slot's
 243 # restart LSN.  Use a new archive location and new slot for this test.
 244 my $slot_dir = $primary->basedir . '/slot_wal';
 245 mkdir($slot_dir);
 246 $slot_name = 'archive_slot';
 247
 248 # Setup the slot, reserving WAL at creation (corresponding to the
 249 # last redo LSN here, actually, so use a checkpoint to reduce the
 250 # number of segments archived).
 251 $primary->psql('postgres', 'checkpoint;');
 252 $primary->psql('postgres',
 253         "SELECT pg_create_physical_replication_slot('$slot_name', true);");
 254
 255 # Get the segment name associated with the slot's restart LSN, that should
 256 # be archived.
 257 my $walfile_streamed = $primary->safe_psql(
 258         'postgres',
 259         "SELECT pg_walfile_name(restart_lsn)
 260   FROM pg_replication_slots
 261   WHERE slot_name = '$slot_name';");
 262
 263 # Switch to a new segment, to make sure that the segment retained by the
 264 # slot is still streamed.  This may not be necessary, but play it safe.
 265 $primary->psql('postgres', 'INSERT INTO test_table VALUES (5);');
 266 $primary->psql('postgres', 'SELECT pg_switch_wal();');
 267 $nextlsn =
 268   $primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
 269 chomp($nextlsn);
 270
 271 # Add a bit more data to accelerate the end of the next pg_receivewal
 272 # commands.
 273 $primary->psql('postgres', 'INSERT INTO test_table VALUES (6);');
 274
 275 # Check case where the slot does not exist.
 276 $primary->command_fails_like(
 277         [
 278                 'pg_receivewal',
 279                 '--directory' => $slot_dir,
 280                 '--slot' => 'nonexistentslot',
 281                 '--no-loop',
 282                 '--no-sync',
 283                 '--verbose',
 284                 '--endpos' => $nextlsn
 285         ],
 286         qr/pg_receivewal: error: replication slot "nonexistentslot" does not exist/,
 287         'pg_receivewal fails with non-existing slot');
 288 $primary->command_ok(
 289         [
 290                 'pg_receivewal',
 291                 '--directory' => $slot_dir,
 292                 '--slot' => $slot_name,
 293                 '--no-loop',
 294                 '--no-sync',
 295                 '--verbose',
 296                 '--endpos' => $nextlsn
 297         ],
 298         "WAL streamed from the slot's restart_lsn");
 299 ok(-e "$slot_dir/$walfile_streamed",
 300         "WAL from the slot's restart_lsn has been archived");
 301
 302 # Test timeline switch using a replication slot, requiring a promoted
 303 # standby.
 304 my $backup_name = "basebackup";
 305 $primary->backup($backup_name);
 306 my $standby = PostgreSQL::Test::Cluster->new("standby");
 307 $standby->init_from_backup($primary, $backup_name, has_streaming => 1);
 308 $standby->start;
 309
 310 # Create a replication slot on this new standby
 311 my $archive_slot = "archive_slot";
 312 $standby->psql(
 313         '',
 314         "CREATE_REPLICATION_SLOT $archive_slot PHYSICAL (RESERVE_WAL)",
 315         replication => 1);
 316 # Wait for standby catchup
 317 $primary->wait_for_catchup($standby);
 318 # Get a walfilename from before the promotion to make sure it is archived
 319 # after promotion
 320 my $standby_slot = $standby->slot($archive_slot);
 321 my $replication_slot_lsn = $standby_slot->{'restart_lsn'};
 322
 323 # pg_walfile_name() is not supported while in recovery, so use the primary
 324 # to build the segment name.  Both nodes are on the same timeline, so this
 325 # produces a segment name with the timeline we are switching from.
 326 my $walfile_before_promotion =
 327   $primary->safe_psql('postgres',
 328         "SELECT pg_walfile_name('$replication_slot_lsn');");
 329 # Everything is setup, promote the standby to trigger a timeline switch.
 330 $standby->promote;
 331
 332 # Force a segment switch to make sure at least one full WAL is archived
 333 # on the new timeline.
 334 my $walfile_after_promotion = $standby->safe_psql('postgres',
 335         "SELECT pg_walfile_name(pg_current_wal_insert_lsn());");
 336 $standby->psql('postgres', 'INSERT INTO test_table VALUES (7);');
 337 $standby->psql('postgres', 'SELECT pg_switch_wal();');
 338 $nextlsn =
 339   $standby->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
 340 chomp($nextlsn);
 341 # This speeds up the operation.
 342 $standby->psql('postgres', 'INSERT INTO test_table VALUES (8);');
 343
 344 # Now try to resume from the slot after the promotion.
 345 my $timeline_dir = $primary->basedir . '/timeline_wal';
 346 mkdir($timeline_dir);
 347
 348 $standby->command_ok(
 349         [
 350                 'pg_receivewal',
 351                 '--directory' => $timeline_dir,
 352                 '--verbose',
 353                 '--endpos' => $nextlsn,
 354                 '--slot' => $archive_slot,
 355                 '--no-sync',
 356                 '--no-loop'
 357         ],
 358         "Stream some wal after promoting, resuming from the slot's position");
 359 ok(-e "$timeline_dir/$walfile_before_promotion",
 360         "WAL segment $walfile_before_promotion archived after timeline jump");
 361 ok(-e "$timeline_dir/$walfile_after_promotion",
 362         "WAL segment $walfile_after_promotion archived after timeline jump");
 363 ok(-e "$timeline_dir/00000002.history",
 364         "timeline history file archived after timeline jump");
 365
 366 done_testing();