| blob | 9c270e7d466ceec01958225af83a8698562f015f |
1 /*-------------------------------------------------------------------------
2 *
3 * xlog.c
4 * PostgreSQL write-ahead log manager
5 *
6 * The Write-Ahead Log (WAL) functionality is split into several source
7 * files, in addition to this one:
8 *
9 * xloginsert.c - Functions for constructing WAL records
10 * xlogrecovery.c - WAL recovery and standby code
11 * xlogreader.c - Facility for reading WAL files and parsing WAL records
12 * xlogutils.c - Helper functions for WAL redo routines
13 *
14 * This file contains functions for coordinating database startup and
15 * checkpointing, and managing the write-ahead log buffers when the
16 * system is running.
17 *
18 * StartupXLOG() is the main entry point of the startup process. It
19 * coordinates database startup, performing WAL recovery, and the
20 * transition from WAL recovery into normal operations.
21 *
22 * XLogInsertRecord() inserts a WAL record into the WAL buffers. Most
23 * callers should not call this directly, but use the functions in
24 * xloginsert.c to construct the WAL record. XLogFlush() can be used
25 * to force the WAL to disk.
26 *
27 * In addition to those, there are many other functions for interrogating
28 * the current system state, and for starting/stopping backups.
29 *
30 *
31 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
32 * Portions Copyright (c) 1994, Regents of the University of California
33 *
34 * src/backend/access/transam/xlog.c
35 *
36 *-------------------------------------------------------------------------
37 */
41 #include <ctype.h>
42 #include <math.h>
43 #include <time.h>
44 #include <fcntl.h>
45 #include <sys/stat.h>
46 #include <sys/time.h>
47 #include <unistd.h>
106 #ifdef WAL_DEBUG
108 #endif
110 /* timeline ID to be used when bootstrapping */
111 #define BootstrapTimeLineID 1
113 /* User-settable parameters */
139 #ifdef WAL_DEBUG
141 #endif
145 /*
146 * Number of WAL insertion locks to use. A higher value allows more insertions
147 * to happen concurrently, but adds some CPU overhead to flushing the WAL,
148 * which needs to iterate all the locks.
149 */
150 #define NUM_XLOGINSERT_LOCKS 8
152 /*
153 * Max distance from last checkpoint, before triggering a new xlog-based
154 * checkpoint.
155 */
158 /* Estimated distance between checkpoints, in bytes */
162 /*
163 * Track whether there were any deferred checks for custom resource managers
164 * specified in wal_consistency_checking.
165 */
168 /*
169 * GUC support
170 */
173 #ifdef HAVE_FSYNC_WRITETHROUGH
175 #endif
177 #ifdef O_SYNC
179 #endif
180 #ifdef O_DSYNC
182 #endif
184 };
187 /*
188 * Although only "on", "off", and "always" are documented,
189 * we accept all the likely variants of "on" and "off".
190 */
202 };
204 /*
205 * Statistics for current checkpoint are collected in this global struct.
206 * Because only the checkpointer or a stand-alone backend can perform
207 * checkpoints, this will be unused in normal backends.
208 */
209 CheckpointStatsData CheckpointStats;
211 /*
212 * During recovery, lastFullPageWrites keeps track of full_page_writes that
213 * the replayed WAL records indicate. It's initialized with full_page_writes
214 * that the recovery starting checkpoint record indicates, and then updated
215 * each time XLOG_FPW_CHANGE record is replayed.
216 */
219 /*
220 * Local copy of the state tracked by SharedRecoveryState in shared memory,
221 * It is false if SharedRecoveryState is RECOVERY_STATE_DONE. True actually
222 * means "not known, need to check the shared state".
223 */
226 /*
227 * Local state for XLogInsertAllowed():
228 * 1: unconditionally allowed to insert XLOG
229 * 0: unconditionally not allowed to insert XLOG
230 * -1: must check RecoveryInProgress(); disallow until it is false
231 * Most processes start with -1 and transition to 1 after seeing that recovery
232 * is not in progress. But we can also force the value for special cases.
233 * The coding in XLogInsertAllowed() depends on the first two of these states
234 * being numerically the same as bool true and false.
235 */
238 /*
239 * ProcLastRecPtr points to the start of the last XLOG record inserted by the
240 * current backend. It is updated for all inserts. XactLastRecEnd points to
241 * end+1 of the last record, and is reset when we end a top-level transaction,
242 * or start a new one; so it can be used to tell if the current transaction has
243 * created any XLOG records.
244 *
245 * While in parallel mode, this may not be fully up to date. When committing,
246 * a transaction can assume this covers all xlog records written either by the
247 * user backend or by any parallel worker which was present at any point during
248 * the transaction. But when aborting, or when still in parallel mode, other
249 * parallel backends may have written WAL records at later LSNs than the value
250 * stored here. The parallel leader advances its own copy, when necessary,
251 * in WaitForParallelWorkersToFinish.
252 */
257 /*
258 * RedoRecPtr is this backend's local copy of the REDO record pointer
259 * (which is almost but not quite the same as a pointer to the most recent
260 * CHECKPOINT record). We update this from the shared-memory copy,
261 * XLogCtl->Insert.RedoRecPtr, whenever we can safely do so (ie, when we
262 * hold an insertion lock). See XLogInsertRecord for details. We are also
263 * allowed to update from XLogCtl->RedoRecPtr if we hold the info_lck;
264 * see GetRedoRecPtr.
265 *
266 * NB: Code that uses this variable must be prepared not only for the
267 * possibility that it may be arbitrarily out of date, but also for the
268 * possibility that it might be set to InvalidXLogRecPtr. We used to
269 * initialize it as a side effect of the first call to RecoveryInProgress(),
270 * which meant that most code that might use it could assume that it had a
271 * real if perhaps stale value. That's no longer the case.
272 */
275 /*
276 * doPageWrites is this backend's local copy of (fullPageWrites ||
277 * runningBackups > 0). It is used together with RedoRecPtr to decide whether
278 * a full-page image of a page need to be taken.
279 *
280 * NB: Initially this is false, and there's no guarantee that it will be
281 * initialized to any other value before it is first used. Any code that
282 * makes use of it must recheck the value after obtaining a WALInsertLock,
283 * and respond appropriately if it turns out that the previous value wasn't
284 * accurate.
285 */
288 /*----------
289 * Shared-memory data structures for XLOG control
290 *
291 * LogwrtRqst indicates a byte position that we need to write and/or fsync
292 * the log up to (all records before that point must be written or fsynced).
293 * The positions already written/fsynced are maintained in logWriteResult
294 * and logFlushResult using atomic access.
295 * In addition to the shared variable, each backend has a private copy of
296 * both in LogwrtResult, which is updated when convenient.
297 *
298 * The request bookkeeping is simpler: there is a shared XLogCtl->LogwrtRqst
299 * (protected by info_lck), but we don't need to cache any copies of it.
300 *
301 * info_lck is only held long enough to read/update the protected variables,
302 * so it's a plain spinlock. The other locks are held longer (potentially
303 * over I/O operations), so we use LWLocks for them. These locks are:
304 *
305 * WALBufMappingLock: must be held to replace a page in the WAL buffer cache.
306 * It is only held while initializing and changing the mapping. If the
307 * contents of the buffer being replaced haven't been written yet, the mapping
308 * lock is released while the write is done, and reacquired afterwards.
309 *
310 * WALWriteLock: must be held to write WAL buffers to disk (XLogWrite or
311 * XLogFlush).
312 *
313 * ControlFileLock: must be held to read/update control file or create
314 * new log file.
315 *
316 *----------
317 */
320 {
326 {
331 /*
332 * Inserting to WAL is protected by a small fixed number of WAL insertion
333 * locks. To insert to the WAL, you must hold one of the locks - it doesn't
334 * matter which one. To lock out other concurrent insertions, you must hold
335 * of them. Each WAL insertion lock consists of a lightweight lock, plus an
336 * indicator of how far the insertion has progressed (insertingAt).
337 *
338 * The insertingAt values are read when a process wants to flush WAL from
339 * the in-memory buffers to disk, to check that all the insertions to the
340 * region the process is about to write out have finished. You could simply
341 * wait for all currently in-progress insertions to finish, but the
342 * insertingAt indicator allows you to ignore insertions to later in the WAL,
343 * so that you only wait for the insertions that are modifying the buffers
344 * you're about to write out.
345 *
346 * This isn't just an optimization. If all the WAL buffers are dirty, an
347 * inserter that's holding a WAL insert lock might need to evict an old WAL
348 * buffer, which requires flushing the WAL. If it's possible for an inserter
349 * to block on another inserter unnecessarily, deadlock can arise when two
350 * inserters holding a WAL insert lock wait for each other to finish their
351 * insertion.
352 *
353 * Small WAL records that don't cross a page boundary never update the value,
354 * the WAL record is just copied to the page and the lock is released. But
355 * to avoid the deadlock-scenario explained above, the indicator is always
356 * updated before sleeping while holding an insertion lock.
357 *
358 * lastImportantAt contains the LSN of the last important WAL record inserted
359 * using a given lock. This value is used to detect if there has been
360 * important WAL activity since the last time some action, like a checkpoint,
361 * was performed - allowing to not repeat the action if not. The LSN is
362 * updated for all insertions, unless the XLOG_MARK_UNIMPORTANT flag was
363 * set. lastImportantAt is never cleared, only overwritten by the LSN of newer
364 * records. Tracking the WAL activity directly in WALInsertLock has the
365 * advantage of not needing any additional locks to update the value.
366 */
368 {
369 LWLock lock;
370 pg_atomic_uint64 insertingAt;
371 XLogRecPtr lastImportantAt;
374 /*
375 * All the WAL insertion locks are allocated as an array in shared memory. We
376 * force the array stride to be a power of 2, which saves a few cycles in
377 * indexing, but more importantly also ensures that individual slots don't
378 * cross cache line boundaries. (Of course, we have to also ensure that the
379 * array start address is suitably aligned.)
380 */
382 {
383 WALInsertLock l;
387 /*
388 * Session status of running backup, used for sanity checks in SQL-callable
389 * functions to start and stop backups.
390 */
393 /*
394 * Shared state data for WAL insertion.
395 */
397 {
400 /*
401 * CurrBytePos is the end of reserved WAL. The next record will be
402 * inserted at that position. PrevBytePos is the start position of the
403 * previously inserted (or rather, reserved) record - it is copied to the
404 * prev-link of the next record. These are stored as "usable byte
405 * positions" rather than XLogRecPtrs (see XLogBytePosToRecPtr()).
406 */
407 uint64 CurrBytePos;
408 uint64 PrevBytePos;
410 /*
411 * Make sure the above heavily-contended spinlock and byte positions are
412 * on their own cache line. In particular, the RedoRecPtr and full page
413 * write variables below should be on a different cache line. They are
414 * read on every WAL insertion, but updated rarely, and we don't want
415 * those reads to steal the cache line containing Curr/PrevBytePos.
416 */
419 /*
420 * fullPageWrites is the authoritative value used by all backends to
421 * determine whether to write full-page image to WAL. This shared value,
422 * instead of the process-local fullPageWrites, is required because, when
423 * full_page_writes is changed by SIGHUP, we must WAL-log it before it
424 * actually affects WAL-logging by backends. Checkpointer sets at startup
425 * or after SIGHUP.
426 *
427 * To read these fields, you must hold an insertion lock. To modify them,
428 * you must hold ALL the locks.
429 */
433 /*
434 * runningBackups is a counter indicating the number of backups currently
435 * in progress. lastBackupStart is the latest checkpoint redo location
436 * used as a starting point for an online backup.
437 */
439 XLogRecPtr lastBackupStart;
441 /*
442 * WAL insertion locks.
443 */
447 /*
448 * Total shared-memory state for XLOG.
449 */
451 {
452 XLogCtlInsert Insert;
454 /* Protected by info_lck: */
455 XLogwrtRqst LogwrtRqst;
463 /* Fake LSN counter, for unlogged relations. */
464 pg_atomic_uint64 unloggedLSN;
466 /* Time and LSN of last xlog segment switch. Protected by WALWriteLock. */
467 pg_time_t lastSegSwitchTime;
468 XLogRecPtr lastSegSwitchLSN;
470 /* These are accessed using atomics -- info_lck not needed */
475 /*
476 * Latest initialized page in the cache (last byte position + 1).
477 *
478 * To change the identity of a buffer (and InitializedUpTo), you need to
479 * hold WALBufMappingLock. To change the identity of a buffer that's
480 * still dirty, the old page needs to be written out first, and for that
481 * you need WALWriteLock, and you need to ensure that there are no
482 * in-progress insertions to the page by calling
483 * WaitXLogInsertionsToFinish().
484 */
485 XLogRecPtr InitializedUpTo;
487 /*
488 * These values do not change after startup, although the pointed-to pages
489 * and xlblocks values certainly do. xlblocks values are protected by
490 * WALBufMappingLock.
491 */
496 /*
497 * InsertTimeLineID is the timeline into which new WAL is being inserted
498 * and flushed. It is zero during recovery, and does not change once set.
499 *
500 * If we create a new timeline when the system was started up,
501 * PrevTimeLineID is the old timeline's ID that we forked off from.
502 * Otherwise it's equal to InsertTimeLineID.
503 *
504 * We set these fields while holding info_lck. Most that reads these
505 * values knows that recovery is no longer in progress and so can safely
506 * read the value without a lock, but code that could be run either during
507 * or after recovery can take info_lck while reading these values.
508 */
509 TimeLineID InsertTimeLineID;
510 TimeLineID PrevTimeLineID;
512 /*
513 * SharedRecoveryState indicates if we're still in crash or archive
514 * recovery. Protected by info_lck.
515 */
516 RecoveryState SharedRecoveryState;
518 /*
519 * InstallXLogFileSegmentActive indicates whether the checkpointer should
520 * arrange for future segments by recycling and/or PreallocXlogFiles().
521 * Protected by ControlFileLock. Only the startup process changes it. If
522 * true, anyone can use InstallXLogFileSegment(). If false, the startup
523 * process owns the exclusive right to install segments, by reading from
524 * the archive and possibly replacing existing files.
525 */
528 /*
529 * WalWriterSleeping indicates whether the WAL writer is currently in
530 * low-power mode (and hence should be nudged if an async commit occurs).
531 * Protected by info_lck.
532 */
535 /*
536 * During recovery, we keep a copy of the latest checkpoint record here.
537 * lastCheckPointRecPtr points to start of checkpoint record and
538 * lastCheckPointEndPtr points to end+1 of checkpoint record. Used by the
539 * checkpointer when it wants to create a restartpoint.
540 *
541 * Protected by info_lck.
542 */
543 XLogRecPtr lastCheckPointRecPtr;
544 XLogRecPtr lastCheckPointEndPtr;
545 CheckPoint lastCheckPoint;
547 /*
548 * lastFpwDisableRecPtr points to the start of the last replayed
549 * XLOG_FPW_CHANGE record that instructs full_page_writes is disabled.
550 */
551 XLogRecPtr lastFpwDisableRecPtr;
556 /*
557 * Classification of XLogInsertRecord operations.
558 */
560 {
561 WALINSERT_NORMAL,
562 WALINSERT_SPECIAL_SWITCH,
563 WALINSERT_SPECIAL_CHECKPOINT
568 /* a private copy of XLogCtl->Insert.WALInsertLocks, for convenience */
571 /*
572 * We maintain an image of pg_control in shared memory.
573 */
576 /*
577 * Calculate the amount of space left on the page after 'endptr'. Beware
578 * multiple evaluation!
579 */
580 #define INSERT_FREESPACE(endptr) \
581 (((endptr) % XLOG_BLCKSZ == 0) ? 0 : (XLOG_BLCKSZ - (endptr) % XLOG_BLCKSZ))
583 /* Macro to advance to next buffer index. */
584 #define NextBufIdx(idx) \
585 (((idx) == XLogCtl->XLogCacheBlck) ? 0 : ((idx) + 1))
587 /*
588 * XLogRecPtrToBufIdx returns the index of the WAL buffer that holds, or
589 * would hold if it was in cache, the page containing 'recptr'.
590 */
591 #define XLogRecPtrToBufIdx(recptr) \
592 (((recptr) / XLOG_BLCKSZ) % (XLogCtl->XLogCacheBlck + 1))
594 /*
595 * These are the number of bytes in a WAL page usable for WAL data.
596 */
597 #define UsableBytesInPage (XLOG_BLCKSZ - SizeOfXLogShortPHD)
599 /*
600 * Convert values of GUCs measured in megabytes to equiv. segment count.
601 * Rounds down.
602 */
603 #define ConvertToXSegs(x, segsize) XLogMBVarToSegs((x), (segsize))
605 /* The number of bytes in a WAL segment usable for WAL data. */
608 /*
609 * Private, possibly out-of-date copy of shared LogwrtResult.
610 * See discussion above.
611 */
614 /*
615 * Update local copy of shared XLogCtl->log{Write,Flush}Result
616 *
617 * It's critical that Flush always trails Write, so the order of the reads is
618 * important, as is the barrier. See also XLogWrite.
619 */
620 #define RefreshXLogWriteResult(_target) \
621 do { \
622 _target.Flush = pg_atomic_read_u64(&XLogCtl->logFlushResult); \
623 pg_read_barrier(); \
624 _target.Write = pg_atomic_read_u64(&XLogCtl->logWriteResult); \
625 } while (0)
627 /*
628 * openLogFile is -1 or a kernel FD for an open log file segment.
629 * openLogSegNo identifies the segment, and openLogTLI the corresponding TLI.
630 * These variables are only used to write the XLOG, and so will normally refer
631 * to the active segment.
632 *
633 * Note: call Reserve/ReleaseExternalFD to track consumption of this FD.
634 */
639 /*
640 * Local copies of equivalent fields in the control file. When running
641 * crash recovery, LocalMinRecoveryPoint is set to InvalidXLogRecPtr as we
642 * expect to replay all the WAL available, and updateMinRecoveryPoint is
643 * switched to false to prevent any updates while replaying records.
644 * Those values are kept consistent as long as crash recovery runs.
645 */
650 /* For WALInsertLockAcquire/Release functions */
654 #ifdef WAL_DEBUG
656 #endif
659 XLogRecPtr EndOfLog,
660 TimeLineID newTLI);
666 XLogRecPtr pagePtr,
667 TimeLineID newTLI);
677 TimeLineID tli);
685 TimeLineID insertTLI);
702 TimeLineID tli);
718 /*
719 * Insert an XLOG record represented by an already-constructed chain of data
720 * chunks. This is a low-level routine; to construct the WAL record header
721 * and data, use the higher-level routines in xloginsert.c.
722 *
723 * If 'fpw_lsn' is valid, it is the oldest LSN among the pages that this
724 * WAL record applies to, that were not included in the record as full page
725 * images. If fpw_lsn <= RedoRecPtr, the function does not perform the
726 * insertion and returns InvalidXLogRecPtr. The caller can then recalculate
727 * which pages need a full-page image, and retry. If fpw_lsn is invalid, the
728 * record is always inserted.
729 *
730 * 'flags' gives more in-depth control on the record being inserted. See
731 * XLogSetRecordFlags() for details.
732 *
733 * 'topxid_included' tells whether the top-transaction id is logged along with
734 * current subtransaction. See XLogRecordAssemble().
735 *
736 * The first XLogRecData in the chain must be for the record header, and its
737 * data must be MAXALIGNed. XLogInsertRecord fills in the xl_prev and
738 * xl_crc fields in the header, the rest of the header must already be filled
739 * by the caller.
740 *
741 * Returns XLOG pointer to end of record (beginning of next record).
742 * This can be used as LSN for data pages affected by the logged action.
743 * (LSN is the XLOG point up to which the XLOG must be flushed to disk
744 * before the data page can be written out. This implements the basic
745 * WAL rule "write the log before the data".)
746 */
747 XLogRecPtr
749 XLogRecPtr fpw_lsn,
750 uint8 flags,
753 {
755 pg_crc32c rdata_crc;
760 XLogRecPtr StartPos;
761 XLogRecPtr EndPos;
763 TimeLineID insertTLI;
765 /* Does this record type require special handling? */
767 {
772 }
774 /* we assume that all of the record header is in the first chunk */
777 /* cross-check on whether we should be here or not */
781 /*
782 * Given that we're not in recovery, InsertTimeLineID is set and can't
783 * change, so we can read it without a lock.
784 */
787 /*----------
788 *
789 * We have now done all the preparatory work we can without holding a
790 * lock or modifying shared state. From here on, inserting the new WAL
791 * record to the shared WAL buffer cache is a two-step process:
792 *
793 * 1. Reserve the right amount of space from the WAL. The current head of
794 * reserved space is kept in Insert->CurrBytePos, and is protected by
795 * insertpos_lck.
796 *
797 * 2. Copy the record to the reserved WAL space. This involves finding the
798 * correct WAL buffer containing the reserved space, and copying the
799 * record in place. This can be done concurrently in multiple processes.
800 *
801 * To keep track of which insertions are still in-progress, each concurrent
802 * inserter acquires an insertion lock. In addition to just indicating that
803 * an insertion is in progress, the lock tells others how far the inserter
804 * has progressed. There is a small fixed number of insertion locks,
805 * determined by NUM_XLOGINSERT_LOCKS. When an inserter crosses a page
806 * boundary, it updates the value stored in the lock to the how far it has
807 * inserted, to allow the previous buffer to be flushed.
808 *
809 * Holding onto an insertion lock also protects RedoRecPtr and
810 * fullPageWrites from changing until the insertion is finished.
811 *
812 * Step 2 can usually be done completely in parallel. If the required WAL
813 * page is not initialized yet, you have to grab WALBufMappingLock to
814 * initialize it, but the WAL writer tries to do that ahead of insertions
815 * to avoid that from happening in the critical path.
816 *
817 *----------
818 */
822 {
825 /*
826 * Check to see if my copy of RedoRecPtr is out of date. If so, may
827 * have to go back and have the caller recompute everything. This can
828 * only happen just after a checkpoint, so it's better to be slow in
829 * this case and fast otherwise.
830 *
831 * Also check to see if fullPageWrites was just turned on or there's a
832 * running backup (which forces full-page writes); if we weren't
833 * already doing full-page writes then go back and recompute.
834 *
835 * If we aren't doing full-page writes then RedoRecPtr doesn't
836 * actually affect the contents of the XLOG record, so we'll update
837 * our local copy but not force a recomputation. (If doPageWrites was
838 * just turned off, we could recompute the record without full pages,
839 * but we choose not to bother.)
840 */
842 {
845 }
851 {
852 /*
853 * Oops, some buffer now needs to be backed up that the caller
854 * didn't back up. Start over.
855 */
859 }
861 /*
862 * Reserve space for the record in the WAL. This also sets the xl_prev
863 * pointer.
864 */
868 /* Normal records are always inserted. */
870 }
872 {
873 /*
874 * In order to insert an XLOG_SWITCH record, we need to hold all of
875 * the WAL insertion locks, not just one, so that no one else can
876 * begin inserting a record until we've figured out how much space
877 * remains in the current WAL segment and claimed all of it.
878 *
879 * Nonetheless, this case is simpler than the normal cases handled
880 * below, which must check for changes in doPageWrites and RedoRecPtr.
881 * Those checks are only needed for records that can contain buffer
882 * references, and an XLOG_SWITCH record never does.
883 */
887 }
888 else
889 {
892 /*
893 * We need to update both the local and shared copies of RedoRecPtr,
894 * which means that we need to hold all the WAL insertion locks.
895 * However, there can't be any buffer references, so as above, we need
896 * not check RedoRecPtr before inserting the record; we just need to
897 * update it afterwards.
898 */
905 }
908 {
909 /*
910 * Now that xl_prev has been filled in, calculate CRC of the record
911 * header.
912 */
918 /*
919 * All the record data, including the header, is now ready to be
920 * inserted. Copy the record in the space reserved.
921 */
926 /*
927 * Unless record is flagged as not important, update LSN of last
928 * important record in the current slot. When holding all locks, just
929 * update the first one.
930 */
932 {
936 }
937 }
938 else
939 {
940 /*
941 * This was an xlog-switch record, but the current insert location was
942 * already exactly at the beginning of a segment, so there was no need
943 * to do anything.
944 */
945 }
947 /*
948 * Done! Let others know that we're finished.
949 */
956 /*
957 * Mark top transaction id is logged (if needed) so that we should not try
958 * to log it again with the next WAL record in the current subtransaction.
959 */
963 /*
964 * Update shared LogwrtRqst.Write, if we crossed page boundary.
965 */
967 {
969 /* advance global request to include new block(s) */
974 }
976 /*
977 * If this was an XLOG_SWITCH record, flush the record and the empty
978 * padding space that fills the rest of the segment, and perform
979 * end-of-segment actions (eg, notifying archiver).
980 */
982 {
986 /*
987 * Even though we reserved the rest of the segment for us, which is
988 * reflected in EndPos, we return a pointer to just the end of the
989 * xlog-switch record.
990 */
992 {
995 {
1000 else
1002 }
1003 }
1004 }
1006 #ifdef WAL_DEBUG
1008 {
1012 StringInfoData buf;
1013 StringInfoData recordBuf;
1015 MemoryContext oldCxt;
1022 /*
1023 * We have to piece together the WAL record data from the XLogRecData
1024 * entries, so that we can pass it to the rm_desc function as one
1025 * contiguous chunk.
1026 */
1031 /* We also need temporary space to decode the record. */
1041 NULL);
1043 {
1044 appendStringInfoString(&buf, "error decoding record: out of memory while allocating a WAL reading processor");
1045 }
1047 decoded,
1048 record,
1049 EndPos,
1051 {
1054 }
1055 else
1056 {
1062 }
1069 }
1070 #endif
1072 /*
1073 * Update our global variables
1074 */
1078 /* Report WAL traffic to the instrumentation. */
1080 {
1084 }
1087 }
1089 /*
1090 * Reserves the right amount of space for a record of given size from the WAL.
1091 * *StartPos is set to the beginning of the reserved section, *EndPos to
1092 * its end+1. *PrevPtr is set to the beginning of the previous record; it is
1093 * used to set the xl_prev of this record.
1094 *
1095 * This is the performance critical part of XLogInsert that must be serialized
1096 * across backends. The rest can happen mostly in parallel. Try to keep this
1097 * section as short as possible, insertpos_lck can be heavily contended on a
1098 * busy system.
1099 *
1100 * NB: The space calculation here must match the code in CopyXLogRecordToWAL,
1101 * where we actually copy the record to the reserved space.
1102 *
1103 * NB: Testing shows that XLogInsertRecord runs faster if this code is inlined;
1104 * however, because there are two call sites, the compiler is reluctant to
1105 * inline. We use pg_attribute_always_inline here to try to convince it.
1106 */
1110 {
1112 uint64 startbytepos;
1113 uint64 endbytepos;
1114 uint64 prevbytepos;
1118 /* All (non xlog-switch) records should contain data. */
1121 /*
1122 * The duration the spinlock needs to be held is minimized by minimizing
1123 * the calculations that have to be done while holding the lock. The
1124 * current tip of reserved WAL is kept in CurrBytePos, as a byte position
1125 * that only counts "usable" bytes in WAL, that is, it excludes all WAL
1126 * page headers. The mapping between "usable" byte positions and physical
1127 * positions (XLogRecPtrs) can be done outside the locked region, and
1128 * because the usable byte position doesn't include any headers, reserving
1129 * X bytes from WAL is almost as simple as "CurrBytePos += X".
1130 */
1145 /*
1146 * Check that the conversions between "usable byte positions" and
1147 * XLogRecPtrs work consistently in both directions.
1148 */
1152 }
1154 /*
1155 * Like ReserveXLogInsertLocation(), but for an xlog-switch record.
1156 *
1157 * A log-switch record is handled slightly differently. The rest of the
1158 * segment will be reserved for this insertion, as indicated by the returned
1159 * *EndPos value. However, if we are already at the beginning of the current
1160 * segment, *StartPos and *EndPos are set to the current location without
1161 * reserving any space, and the function returns false.
1162 */
1163 static bool
1165 {
1167 uint64 startbytepos;
1168 uint64 endbytepos;
1169 uint64 prevbytepos;
1171 XLogRecPtr ptr;
1172 uint32 segleft;
1174 /*
1175 * These calculations are a bit heavy-weight to be done while holding a
1176 * spinlock, but since we're holding all the WAL insertion locks, there
1177 * are no other inserters competing for it. GetXLogInsertRecPtr() does
1178 * compete for it, but that's not called very frequently.
1179 */
1186 {
1190 }
1200 {
1201 /* consume the rest of the segment */
1204 }
1218 }
1220 /*
1221 * Subroutine of XLogInsertRecord. Copies a WAL record to an already-reserved
1222 * area in the WAL.
1223 */
1224 static void
1227 {
1231 XLogRecPtr CurrPos;
1232 XLogPageHeader pagehdr;
1234 /*
1235 * Get a pointer to the right place in the right WAL buffer to start
1236 * inserting to.
1237 */
1242 /*
1243 * there should be enough space for at least the first field (xl_tot_len)
1244 * on this page.
1245 */
1248 /* Copy record data */
1251 {
1256 {
1257 /*
1258 * Write what fits on this page, and continue on the next page.
1259 */
1267 /*
1268 * Get pointer to beginning of next page, and set the xlp_rem_len
1269 * in the page header. Set XLP_FIRST_IS_CONTRECORD.
1270 *
1271 * It's safe to set the contrecord flag and xlp_rem_len without a
1272 * lock on the page. All the other flags were already set when the
1273 * page was initialized, in AdvanceXLInsertBuffer, and we're the
1274 * only backend that needs to set the contrecord flag.
1275 */
1281 /* skip over the page header */
1283 {
1286 }
1287 else
1288 {
1291 }
1293 }
1303 }
1306 /*
1307 * If this was an xlog-switch, it's not enough to write the switch record,
1308 * we also have to consume all the remaining space in the WAL segment. We
1309 * have already reserved that space, but we need to actually fill it.
1310 */
1312 {
1313 /* An xlog-switch record doesn't contain any data besides the header */
1316 /* Assert that we did reserve the right amount of space */
1319 /* Use up all the remaining space on the current page */
1322 /*
1323 * Cause all remaining pages in the segment to be flushed, leaving the
1324 * XLog position where it should be, at the start of the next segment.
1325 * We do this one page at a time, to make sure we don't deadlock
1326 * against ourselves if wal_buffers < wal_segment_size.
1327 */
1329 {
1330 /*
1331 * The minimal action to flush the page would be to call
1332 * WALInsertLockUpdateInsertingAt(CurrPos) followed by
1333 * AdvanceXLInsertBuffer(...). The page would be left initialized
1334 * mostly to zeros, except for the page header (always the short
1335 * variant, as this is never a segment's first page).
1336 *
1337 * The large vistas of zeros are good for compressibility, but the
1338 * headers interrupting them every XLOG_BLCKSZ (with values that
1339 * differ from page to page) are not. The effect varies with
1340 * compression tool, but bzip2 for instance compresses about an
1341 * order of magnitude worse if those headers are left in place.
1342 *
1343 * Rather than complicating AdvanceXLInsertBuffer itself (which is
1344 * called in heavily-loaded circumstances as well as this lightly-
1345 * loaded one) with variant behavior, we just use GetXLogBuffer
1346 * (which itself calls the two methods we need) to get the pointer
1347 * and zero most of the page. Then we just zero the page header.
1348 */
1353 }
1354 }
1355 else
1356 {
1357 /* Align the end position, so that the next record starts aligned */
1359 }
1365 }
1367 /*
1368 * Acquire a WAL insertion lock, for inserting to WAL.
1369 */
1370 static void
1372 {
1375 /*
1376 * It doesn't matter which of the WAL insertion locks we acquire, so try
1377 * the one we used last time. If the system isn't particularly busy, it's
1378 * a good bet that it's still available, and it's good to have some
1379 * affinity to a particular lock so that you don't unnecessarily bounce
1380 * cache lines between processes when there's no contention.
1381 *
1382 * If this is the first time through in this backend, pick a lock
1383 * (semi-)randomly. This allows the locks to be used evenly if you have a
1384 * lot of very short connections.
1385 */
1392 /*
1393 * The insertingAt value is initially set to 0, as we don't know our
1394 * insert location yet.
1395 */
1398 {
1399 /*
1400 * If we couldn't get the lock immediately, try another lock next
1401 * time. On a system with more insertion locks than concurrent
1402 * inserters, this causes all the inserters to eventually migrate to a
1403 * lock that no-one else is using. On a system with more inserters
1404 * than locks, it still helps to distribute the inserters evenly
1405 * across the locks.
1406 */
1408 }
1409 }
1411 /*
1412 * Acquire all WAL insertion locks, to prevent other backends from inserting
1413 * to WAL.
1414 */
1415 static void
1417 {
1420 /*
1421 * When holding all the locks, all but the last lock's insertingAt
1422 * indicator is set to 0xFFFFFFFFFFFFFFFF, which is higher than any real
1423 * XLogRecPtr value, to make sure that no-one blocks waiting on those.
1424 */
1426 {
1430 PG_UINT64_MAX);
1431 }
1432 /* Variable value reset to 0 at release */
1436 }
1438 /*
1439 * Release our insertion lock (or locks, if we're holding them all).
1440 *
1441 * NB: Reset all variables to 0, so they cause LWLockWaitForVar to block the
1442 * next time the lock is acquired.
1443 */
1444 static void
1446 {
1448 {
1457 }
1458 else
1459 {
1463 }
1464 }
1466 /*
1467 * Update our insertingAt value, to let others know that we've finished
1468 * inserting up to that point.
1469 */
1470 static void
1472 {
1474 {
1475 /*
1476 * We use the last lock to mark our actual position, see comments in
1477 * WALInsertLockAcquireExclusive.
1478 */
1481 insertingAt);
1482 }
1483 else
1486 insertingAt);
1487 }
1489 /*
1490 * Wait for any WAL insertions < upto to finish.
1491 *
1492 * Returns the location of the oldest insertion that is still in-progress.
1493 * Any WAL prior to that point has been fully copied into WAL buffers, and
1494 * can be flushed out to disk. Because this waits for any insertions older
1495 * than 'upto' to finish, the return value is always >= 'upto'.
1496 *
1497 * Note: When you are about to write out WAL, you must call this function
1498 * *before* acquiring WALWriteLock, to avoid deadlocks. This function might
1499 * need to wait for an insertion to finish (or at least advance to next
1500 * uninitialized page), and the inserter might need to evict an old WAL buffer
1501 * to make room for a new one, which in turn requires WALWriteLock.
1502 */
1503 static XLogRecPtr
1505 {
1506 uint64 bytepos;
1507 XLogRecPtr inserted;
1508 XLogRecPtr reservedUpto;
1509 XLogRecPtr finishedUpto;
1516 /*
1517 * Check if there's any work to do. Use a barrier to ensure we get the
1518 * freshest value.
1519 */
1524 /* Read the current insert position */
1530 /*
1531 * No-one should request to flush a piece of WAL that hasn't even been
1532 * reserved yet. However, it can happen if there is a block with a bogus
1533 * LSN on disk, for example. XLogFlush checks for that situation and
1534 * complains, but only after the flush. Here we just assume that to mean
1535 * that all WAL that has been reserved needs to be finished. In this
1536 * corner-case, the return value can be smaller than 'upto' argument.
1537 */
1539 {
1544 }
1546 /*
1547 * Loop through all the locks, sleeping on any in-progress insert older
1548 * than 'upto'.
1549 *
1550 * finishedUpto is our return value, indicating the point upto which all
1551 * the WAL insertions have been finished. Initialize it to the head of
1552 * reserved WAL, and as we iterate through the insertion locks, back it
1553 * out for any insertion that's still in progress.
1554 */
1557 {
1560 do
1561 {
1562 /*
1563 * See if this insertion is in progress. LWLockWaitForVar will
1564 * wait for the lock to be released, or for the 'value' to be set
1565 * by a LWLockUpdateVar call. When a lock is initially acquired,
1566 * its value is 0 (InvalidXLogRecPtr), which means that we don't
1567 * know where it's inserting yet. We will have to wait for it. If
1568 * it's a small insertion, the record will most likely fit on the
1569 * same page and the inserter will release the lock without ever
1570 * calling LWLockUpdateVar. But if it has to sleep, it will
1571 * advertise the insertion point with LWLockUpdateVar before
1572 * sleeping.
1573 *
1574 * In this loop we are only waiting for insertions that started
1575 * before WaitXLogInsertionsToFinish was called. The lack of
1576 * memory barriers in the loop means that we might see locks as
1577 * "unused" that have since become used. This is fine because
1578 * they only can be used for later insertions that we would not
1579 * want to wait on anyway. Not taking a lock to acquire the
1580 * current insertingAt value means that we might see older
1581 * insertingAt values. This is also fine, because if we read a
1582 * value too old, we will add ourselves to the wait queue, which
1583 * contains atomic operations.
1584 */
1588 {
1589 /* the lock was free, so no insertion in progress */
1592 }
1594 /*
1595 * This insertion is still in progress. Have to wait, unless the
1596 * inserter has proceeded past 'upto'.
1597 */
1602 }
1604 /*
1605 * Advance the limit we know to have been inserted and return the freshest
1606 * value we know of, which might be beyond what we requested if somebody
1607 * is concurrently doing this with an 'upto' pointer ahead of us.
1608 */
1610 finishedUpto);
1613 }
1615 /*
1616 * Get a pointer to the right location in the WAL buffer containing the
1617 * given XLogRecPtr.
1618 *
1619 * If the page is not initialized yet, it is initialized. That might require
1620 * evicting an old dirty buffer from the buffer cache, which means I/O.
1621 *
1622 * The caller must ensure that the page containing the requested location
1623 * isn't evicted yet, and won't be evicted. The way to ensure that is to
1624 * hold onto a WAL insertion lock with the insertingAt position set to
1625 * something <= ptr. GetXLogBuffer() will update insertingAt if it needs
1626 * to evict an old page from the buffer. (This means that once you call
1627 * GetXLogBuffer() with a given 'ptr', you must not access anything before
1628 * that point anymore, and must not call GetXLogBuffer() with an older 'ptr'
1629 * later, because older buffers might be recycled already)
1630 */
1633 {
1635 XLogRecPtr endptr;
1638 XLogRecPtr expectedEndPtr;
1640 /*
1641 * Fast path for the common case that we need to access again the same
1642 * page as last time.
1643 */
1645 {
1649 }
1651 /*
1652 * The XLog buffer cache is organized so that a page is always loaded to a
1653 * particular buffer. That way we can easily calculate the buffer a given
1654 * page must be loaded into, from the XLogRecPtr alone.
1655 */
1658 /*
1659 * See what page is loaded in the buffer at the moment. It could be the
1660 * page we're looking for, or something older. It can't be anything newer
1661 * - that would imply the page we're looking for has already been written
1662 * out to disk and evicted, and the caller is responsible for making sure
1663 * that doesn't happen.
1664 *
1665 * We don't hold a lock while we read the value. If someone is just about
1666 * to initialize or has just initialized the page, it's possible that we
1667 * get InvalidXLogRecPtr. That's ok, we'll grab the mapping lock (in
1668 * AdvanceXLInsertBuffer) and retry if we see anything other than the page
1669 * we're looking for.
1670 */
1676 {
1677 XLogRecPtr initializedUpto;
1679 /*
1680 * Before calling AdvanceXLInsertBuffer(), which can block, let others
1681 * know how far we're finished with inserting the record.
1682 *
1683 * NB: If 'ptr' points to just after the page header, advertise a
1684 * position at the beginning of the page rather than 'ptr' itself. If
1685 * there are no other insertions running, someone might try to flush
1686 * up to our advertised location. If we advertised a position after
1687 * the page header, someone might try to flush the page header, even
1688 * though page might actually not be initialized yet. As the first
1689 * inserter on the page, we are effectively responsible for making
1690 * sure that it's initialized, before we let insertingAt to move past
1691 * the page header.
1692 */
1699 else
1710 }
1711 else
1712 {
1713 /*
1714 * Make sure the initialization of the page is visible to us, and
1715 * won't arrive later to overwrite the WAL data we write on the page.
1716 */
1718 }
1720 /*
1721 * Found the buffer holding this page. Return a pointer to the right
1722 * offset within the page.
1723 */
1731 }
1733 /*
1734 * Read WAL data directly from WAL buffers, if available. Returns the number
1735 * of bytes read successfully.
1736 *
1737 * Fewer than 'count' bytes may be read if some of the requested WAL data has
1738 * already been evicted.
1739 *
1740 * No locks are taken.
1741 *
1742 * Caller should ensure that it reads no further than LogwrtResult.Write
1743 * (which should have been updated by the caller when determining how far to
1744 * read). The 'tli' argument is only used as a convenient safety check so that
1745 * callers do not read from WAL buffers on a historical timeline.
1746 */
1747 Size
1749 TimeLineID tli)
1750 {
1753 XLogRecPtr inserted;
1761 /*
1762 * Caller should ensure that the requested data has been inserted into WAL
1763 * buffers before we try to read it.
1764 */
1772 /*
1773 * Loop through the buffers without a lock. For each buffer, atomically
1774 * read and verify the end pointer, then copy the data out, and finally
1775 * re-read and re-verify the end pointer.
1776 *
1777 * Once a page is evicted, it never returns to the WAL buffers, so if the
1778 * end pointer matches the expected end pointer before and after we copy
1779 * the data, then the right page must have been present during the data
1780 * copy. Read barriers are necessary to ensure that the data copy actually
1781 * happens between the two verification steps.
1782 *
1783 * If either verification fails, we simply terminate the loop and return
1784 * with the data that had been already copied out successfully.
1785 */
1787 {
1790 XLogRecPtr expectedEndPtr;
1791 XLogRecPtr endptr;
1794 Size npagebytes;
1796 /*
1797 * Calculate the end pointer we expect in the xlblocks array if the
1798 * correct page is present.
1799 */
1802 /*
1803 * First verification step: check that the correct page is present in
1804 * the WAL buffers.
1805 */
1810 /*
1811 * The correct page is present (or was at the time the endptr was
1812 * read; must re-verify later). Calculate pointer to source data and
1813 * determine how much data to read from this page.
1814 */
1819 /*
1820 * Ensure that the data copy and the first verification step are not
1821 * reordered.
1822 */
1825 /* data copy */
1828 /*
1829 * Ensure that the data copy and the second verification step are not
1830 * reordered.
1831 */
1834 /*
1835 * Second verification step: check that the page we read from wasn't
1836 * evicted while we were copying the data.
1837 */
1845 }
1850 }
1852 /*
1853 * Converts a "usable byte position" to XLogRecPtr. A usable byte position
1854 * is the position starting from the beginning of WAL, excluding all WAL
1855 * page headers.
1856 */
1857 static XLogRecPtr
1859 {
1860 uint64 fullsegs;
1861 uint64 fullpages;
1862 uint64 bytesleft;
1863 uint32 seg_offset;
1864 XLogRecPtr result;
1870 {
1871 /* fits on first page of segment */
1873 }
1874 else
1875 {
1876 /* account for the first page on segment with long header */
1884 }
1889 }
1891 /*
1892 * Like XLogBytePosToRecPtr, but if the position is at a page boundary,
1893 * returns a pointer to the beginning of the page (ie. before page header),
1894 * not to where the first xlog record on that page would go to. This is used
1895 * when converting a pointer to the end of a record.
1896 */
1897 static XLogRecPtr
1899 {
1900 uint64 fullsegs;
1901 uint64 fullpages;
1902 uint64 bytesleft;
1903 uint32 seg_offset;
1904 XLogRecPtr result;
1910 {
1911 /* fits on first page of segment */
1914 else
1916 }
1917 else
1918 {
1919 /* account for the first page on segment with long header */
1928 else
1930 }
1935 }
1937 /*
1938 * Convert an XLogRecPtr to a "usable byte position".
1939 */
1940 static uint64
1942 {
1943 uint64 fullsegs;
1944 uint32 fullpages;
1945 uint32 offset;
1946 uint64 result;
1954 {
1957 {
1960 }
1961 }
1962 else
1963 {
1968 {
1971 }
1972 }
1975 }
1977 /*
1978 * Initialize XLOG buffers, writing out old buffers if they still contain
1979 * unwritten data, upto the page containing 'upto'. Or if 'opportunistic' is
1980 * true, initialize as many pages as we can without having to write out
1981 * unwritten data. Any new pages are initialized to zeros, with pages headers
1982 * initialized properly.
1983 */
1984 static void
1986 {
1989 XLogRecPtr OldPageRqstPtr;
1990 XLogwrtRqst WriteRqst;
1992 XLogRecPtr NewPageBeginPtr;
1993 XLogPageHeader NewPage;
1998 /*
1999 * Now that we have the lock, check if someone initialized the page
2000 * already.
2001 */
2003 {
2006 /*
2007 * Get ending-offset of the buffer page we need to replace (this may
2008 * be zero if the buffer hasn't been used yet). Fall through if it's
2009 * already written out.
2010 */
2013 {
2014 /*
2015 * Nope, got work to do. If we just want to pre-initialize as much
2016 * as we can without flushing, give up now.
2017 */
2021 /* Advance shared memory write request position */
2027 /*
2028 * Acquire an up-to-date LogwrtResult value and see if we still
2029 * need to write it or if someone else already did.
2030 */
2033 {
2034 /*
2035 * Must acquire write lock. Release WALBufMappingLock first,
2036 * to make sure that all insertions that we need to wait for
2037 * can finish (up to this same position). Otherwise we risk
2038 * deadlock.
2039 */
2048 {
2049 /* OK, someone wrote it already */
2051 }
2052 else
2053 {
2054 /* Have to write it ourselves */
2062 }
2063 /* Re-acquire WALBufMappingLock and retry */
2066 }
2067 }
2069 /*
2070 * Now the next buffer slot is free and we can set it up to be the
2071 * next output page.
2072 */
2080 /*
2081 * Mark the xlblock with InvalidXLogRecPtr and issue a write barrier
2082 * before initializing. Otherwise, the old page may be partially
2083 * zeroed but look valid.
2084 */
2088 /*
2089 * Be sure to re-zero the buffer so that bytes beyond what we've
2090 * written will look like zeroes and not valid XLOG records...
2091 */
2094 /*
2095 * Fill the new page's header
2096 */
2105 /*
2106 * If online backup is not in progress, mark the header to indicate
2107 * that WAL records beginning in this page have removable backup
2108 * blocks. This allows the WAL archiver to know whether it is safe to
2109 * compress archived WAL data by transforming full-block records into
2110 * the non-full-block format. It is sufficient to record this at the
2111 * page level because we force a page switch (in fact a segment
2112 * switch) when starting a backup, so the flag will be off before any
2113 * records can be written during the backup. At the end of a backup,
2114 * the last page will be marked as all unsafe when perhaps only part
2115 * is unsafe, but at worst the archiver would miss the opportunity to
2116 * compress a few records.
2117 */
2121 /*
2122 * If first page of an XLOG segment file, make it a long header.
2123 */
2125 {
2132 }
2134 /*
2135 * Make sure the initialization of the page becomes visible to others
2136 * before the xlblocks update. GetXLogBuffer() reads xlblocks without
2137 * holding a lock.
2138 */
2144 npages++;
2145 }
2148 #ifdef WAL_DEBUG
2150 {
2153 }
2154 #endif
2155 }
2157 /*
2158 * Calculate CheckPointSegments based on max_wal_size_mb and
2159 * checkpoint_completion_target.
2160 */
2161 static void
2163 {
2166 /*-------
2167 * Calculate the distance at which to trigger a checkpoint, to avoid
2168 * exceeding max_wal_size_mb. This is based on two assumptions:
2169 *
2170 * a) we keep WAL for only one checkpoint cycle (prior to PG11 we kept
2171 * WAL for two checkpoint cycles to allow us to recover from the
2172 * secondary checkpoint if the first checkpoint failed, though we
2173 * only did this on the primary anyway, not on standby. Keeping just
2174 * one checkpoint simplifies processing and reduces disk space in
2175 * many smaller databases.)
2176 * b) during checkpoint, we consume checkpoint_completion_target *
2177 * number of segments consumed between checkpoints.
2178 *-------
2179 */
2183 /* round down */
2188 }
2190 void
2192 {
2195 }
2197 void
2199 {
2202 }
2204 bool
2206 {
2208 {
2211 }
2214 }
2216 /*
2217 * GUC check_hook for max_slot_wal_keep_size
2218 *
2219 * We don't allow the value of max_slot_wal_keep_size other than -1 during the
2220 * binary upgrade. See start_postmaster() in pg_upgrade for more details.
2221 */
2222 bool
2224 {
2226 {
2230 }
2233 }
2235 /*
2236 * At a checkpoint, how many WAL segments to recycle as preallocated future
2237 * XLOG segments? Returns the highest segment that should be preallocated.
2238 */
2239 static XLogSegNo
2241 {
2242 XLogSegNo minSegNo;
2243 XLogSegNo maxSegNo;
2245 XLogSegNo recycleSegNo;
2247 /*
2248 * Calculate the segment numbers that min_wal_size_mb and max_wal_size_mb
2249 * correspond to. Always recycle enough segments to meet the minimum, and
2250 * remove enough segments to stay below the maximum.
2251 */
2257 /*
2258 * Between those limits, recycle enough segments to get us through to the
2259 * estimated end of next checkpoint.
2260 *
2261 * To estimate where the next checkpoint will finish, assume that the
2262 * system runs steadily consuming CheckPointDistanceEstimate bytes between
2263 * every checkpoint.
2264 */
2266 /* add 10% for good measure. */
2270 wal_segment_size);
2278 }
2280 /*
2281 * Check whether we've consumed enough xlog space that a checkpoint is needed.
2282 *
2283 * new_segno indicates a log file that has just been filled up (or read
2284 * during recovery). We measure the distance from RedoRecPtr to new_segno
2285 * and see if that exceeds CheckPointSegments.
2286 *
2287 * Note: it is caller's responsibility that RedoRecPtr is up-to-date.
2288 */
2289 bool
2291 {
2292 XLogSegNo old_segno;
2299 }
2301 /*
2302 * Write and/or fsync the log at least as far as WriteRqst indicates.
2303 *
2304 * If flexible == true, we don't have to write as far as WriteRqst, but
2305 * may stop at any convenient boundary (such as a cache or logfile boundary).
2306 * This option allows us to avoid uselessly issuing multiple writes when a
2307 * single one would do.
2308 *
2309 * Must be called with WALWriteLock held. WaitXLogInsertionsToFinish(WriteRqst)
2310 * must be called before grabbing the lock, to make sure the data is ready to
2311 * write.
2312 */
2313 static void
2315 {
2322 uint32 startoffset;
2324 /* We should always be inside a critical section here */
2327 /*
2328 * Update local LogwrtResult (caller probably did this already, but...)
2329 */
2332 /*
2333 * Since successive pages in the xlog cache are consecutively allocated,
2334 * we can usually gather multiple pages together and issue just one
2335 * write() call. npages is the number of pages we have determined can be
2336 * written together; startidx is the cache block index of the first one,
2337 * and startoffset is the file offset at which it should go. The latter
2338 * two variables are only valid when npages > 0, but we must initialize
2339 * all of them to keep the compiler quiet.
2340 */
2345 /*
2346 * Within the loop, curridx is the cache block index of the page to
2347 * consider writing. Begin at the buffer containing the next unwritten
2348 * page, or last partially written page.
2349 */
2353 {
2354 /*
2355 * Make sure we're not ahead of the insert process. This could happen
2356 * if we're passed a bogus WriteRqst.Write that is past the end of the
2357 * last page that's been initialized by AdvanceXLInsertBuffer.
2358 */
2366 /* Advance LogwrtResult.Write to end of current buffer page */
2371 wal_segment_size))
2372 {
2373 /*
2374 * Switch to new logfile segment. We cannot have any pending
2375 * pages here (since we dump what we have at segment end).
2376 */
2381 wal_segment_size);
2384 /* create/use new log file */
2387 }
2389 /* Make sure we have the current logfile open */
2391 {
2393 wal_segment_size);
2397 }
2399 /* Add current page to the set of pending pages-to-dump */
2401 {
2402 /* first of group */
2405 wal_segment_size);
2406 }
2407 npages++;
2409 /*
2410 * Dump the set if this will be the last loop iteration, or if we are
2411 * at the last page of the cache area (since the next page won't be
2412 * contiguous in memory), or if we are at the end of the logfile
2413 * segment.
2414 */
2422 finishing_seg)
2423 {
2425 Size nbytes;
2426 Size nleft;
2427 ssize_t written;
2428 instr_time start;
2430 /* OK to write the page(s) */
2434 do
2435 {
2438 /*
2439 * Measure I/O timing to write WAL data, for pg_stat_io and/or
2440 * pg_stat_wal.
2441 */
2451 /*
2452 * Increment the I/O timing and the number of times WAL data
2453 * were written out to disk.
2454 */
2456 {
2457 instr_time end;
2461 }
2466 {
2475 wal_segment_size);
2481 }
2489 /*
2490 * If we just wrote the whole last page of a logfile segment,
2491 * fsync the segment immediately. This avoids having to go back
2492 * and re-open prior segments when an fsync request comes along
2493 * later. Doing it here ensures that one and only one backend will
2494 * perform this fsync.
2495 *
2496 * This is also the right place to notify the Archiver that the
2497 * segment is ready to copy to archival storage, and to update the
2498 * timer for archive_timeout, and to signal for a checkpoint if
2499 * too many logfile segments have been used since the last
2500 * checkpoint.
2501 */
2503 {
2506 /* signal that we need to wakeup walsenders later */
2517 /*
2518 * Request a checkpoint if we've consumed too much xlog since
2519 * the last one. For speed, we first check using the local
2520 * copy of RedoRecPtr, which might be out of date; if it looks
2521 * like a checkpoint is needed, forcibly update RedoRecPtr and
2522 * recheck.
2523 */
2525 {
2529 }
2530 }
2531 }
2534 {
2535 /* Only asked to write a partial page */
2538 }
2541 /* If flexible, break out of loop as soon as we wrote something */
2544 }
2548 /*
2549 * If asked to flush, do so
2550 */
2553 {
2554 /*
2555 * Could get here without iterating above loop, in which case we might
2556 * have no open file or the wrong one. However, we do not need to
2557 * fsync more than one file.
2558 */
2561 {
2564 wal_segment_size))
2567 {
2569 wal_segment_size);
2573 }
2576 }
2578 /* signal that we need to wakeup walsenders later */
2582 }
2584 /*
2585 * Update shared-memory status
2586 *
2587 * We make sure that the shared 'request' values do not fall behind the
2588 * 'result' values. This is not absolutely essential, but it saves some
2589 * code in a couple of places.
2590 */
2598 /*
2599 * We write Write first, bar, then Flush. When reading, the opposite must
2600 * be done (with a matching barrier in between), so that we always see a
2601 * Flush value that trails behind the Write value seen.
2602 */
2607 #ifdef USE_ASSERT_CHECKING
2608 {
2609 XLogRecPtr Flush;
2610 XLogRecPtr Write;
2611 XLogRecPtr Insert;
2619 /* WAL written to disk is always ahead of WAL flushed */
2622 /* WAL inserted to buffers is always ahead of WAL written */
2624 }
2625 #endif
2626 }
2628 /*
2629 * Record the LSN for an asynchronous transaction commit/abort
2630 * and nudge the WALWriter if there is work for it to do.
2631 * (This should not be called for synchronous commits.)
2632 */
2633 void
2635 {
2639 XLogRecPtr prevAsyncXactLSN;
2648 /*
2649 * If somebody else already called this function with a more aggressive
2650 * LSN, they will have done what we needed (and perhaps more).
2651 */
2655 /*
2656 * If the WALWriter is sleeping, kick it to make it come out of low-power
2657 * mode, so that this async commit will reach disk within the expected
2658 * amount of time. Otherwise, determine whether it has enough WAL
2659 * available to flush, the same way that XLogBackgroundFlush() does.
2660 */
2663 else
2664 {
2669 flushblocks =
2674 }
2677 {
2683 }
2684 }
2686 /*
2687 * Record the LSN up to which we can remove WAL because it's not required by
2688 * any replication slot.
2689 */
2690 void
2692 {
2696 }
2699 /*
2700 * Return the oldest LSN we must retain to satisfy the needs of some
2701 * replication slot.
2702 */
2703 static XLogRecPtr
2705 {
2706 XLogRecPtr retval;
2713 }
2715 /*
2716 * Advance minRecoveryPoint in control file.
2717 *
2718 * If we crash during recovery, we must reach this point again before the
2719 * database is consistent.
2720 *
2721 * If 'force' is true, 'lsn' argument is ignored. Otherwise, minRecoveryPoint
2722 * is only updated if it's not already greater than or equal to 'lsn'.
2723 */
2724 static void
2726 {
2727 /* Quick check using our local copy of the variable */
2731 /*
2732 * An invalid minRecoveryPoint means that we need to recover all the WAL,
2733 * i.e., we're doing crash recovery. We never modify the control file's
2734 * value in that case, so we can short-circuit future checks here too. The
2735 * local values of minRecoveryPoint and minRecoveryPointTLI should not be
2736 * updated until crash recovery finishes. We only do this for the startup
2737 * process as it should not update its own reference of minRecoveryPoint
2738 * until it has finished crash recovery to make sure that all WAL
2739 * available is replayed in this case. This also saves from extra locks
2740 * taken on the control file from the startup process.
2741 */
2743 {
2746 }
2750 /* update local copy */
2757 {
2758 XLogRecPtr newMinRecoveryPoint;
2759 TimeLineID newMinRecoveryPointTLI;
2761 /*
2762 * To avoid having to update the control file too often, we update it
2763 * all the way to the last record being replayed, even though 'lsn'
2764 * would suffice for correctness. This also allows the 'force' case
2765 * to not need a valid 'lsn' value.
2766 *
2767 * Another important reason for doing it this way is that the passed
2768 * 'lsn' value could be bogus, i.e., past the end of available WAL, if
2769 * the caller got it from a corrupted heap page. Accepting such a
2770 * value as the min recovery point would prevent us from coming up at
2771 * all. Instead, we just log a warning and continue with recovery.
2772 * (See also the comments about corrupt LSNs in XLogFlush.)
2773 */
2780 /* update control file */
2782 {
2792 newMinRecoveryPointTLI)));
2793 }
2794 }
2796 }
2798 /*
2799 * Ensure that all XLOG data through the given position is flushed to disk.
2800 *
2801 * NOTE: this differs from XLogWrite mainly in that the WALWriteLock is not
2802 * already held, and we try to avoid acquiring it if possible.
2803 */
2804 void
2806 {
2807 XLogRecPtr WriteRqstPtr;
2808 XLogwrtRqst WriteRqst;
2811 /*
2812 * During REDO, we are reading not writing WAL. Therefore, instead of
2813 * trying to flush the WAL, we should update minRecoveryPoint instead. We
2814 * test XLogInsertAllowed(), not InRecovery, because we need checkpointer
2815 * to act this way too, and because when it tries to write the
2816 * end-of-recovery checkpoint, it should indeed flush.
2817 */
2819 {
2822 }
2824 /* Quick exit if already known flushed */
2828 #ifdef WAL_DEBUG
2834 #endif
2838 /*
2839 * Since fsync is usually a horribly expensive operation, we try to
2840 * piggyback as much data as we can on each fsync: if we see any more data
2841 * entered into the xlog buffer, we'll write and fsync that too, so that
2842 * the final value of LogwrtResult.Flush is as large as possible. This
2843 * gives us some chance of avoiding another fsync immediately after.
2844 */
2846 /* initialize to given target; may increase below */
2849 /*
2850 * Now wait until we get the write lock, or someone else does the flush
2851 * for us.
2852 */
2854 {
2855 XLogRecPtr insertpos;
2857 /* done already? */
2862 /*
2863 * Before actually performing the write, wait for all in-flight
2864 * insertions to the pages we're about to write to finish.
2865 */
2872 /*
2873 * Try to get the write lock. If we can't get it immediately, wait
2874 * until it's released, and recheck if we still need to do the flush
2875 * or if the backend that held the lock did it for us already. This
2876 * helps to maintain a good rate of group committing when the system
2877 * is bottlenecked by the speed of fsyncing.
2878 */
2880 {
2881 /*
2882 * The lock is now free, but we didn't acquire it yet. Before we
2883 * do, loop back to check if someone else flushed the record for
2884 * us already.
2885 */
2887 }
2889 /* Got the lock; recheck whether request is satisfied */
2892 {
2895 }
2897 /*
2898 * Sleep before flush! By adding a delay here, we may give further
2899 * backends the opportunity to join the backlog of group commit
2900 * followers; this can significantly improve transaction throughput,
2901 * at the risk of increasing transaction latency.
2902 *
2903 * We do not sleep if enableFsync is not turned on, nor if there are
2904 * fewer than CommitSiblings other backends with active transactions.
2905 */
2908 {
2911 /*
2912 * Re-check how far we can now flush the WAL. It's generally not
2913 * safe to call WaitXLogInsertionsToFinish while holding
2914 * WALWriteLock, because an in-progress insertion might need to
2915 * also grab WALWriteLock to make progress. But we know that all
2916 * the insertions up to insertpos have already finished, because
2917 * that's what the earlier WaitXLogInsertionsToFinish() returned.
2918 * We're only calling it again to allow insertpos to be moved
2919 * further forward, not to actually wait for anyone.
2920 */
2922 }
2924 /* try to write/flush later additions to XLOG as well */
2931 /* done */
2933 }
2937 /* wake up walsenders now that we've released heavily contended locks */
2940 /*
2941 * If we still haven't flushed to the request point then we have a
2942 * problem; most likely, the requested flush point is past end of XLOG.
2943 * This has been seen to occur when a disk page has a corrupted LSN.
2944 *
2945 * Formerly we treated this as a PANIC condition, but that hurts the
2946 * system's robustness rather than helping it: we do not want to take down
2947 * the whole system due to corruption on one data page. In particular, if
2948 * the bad page is encountered again during recovery then we would be
2949 * unable to restart the database at all! (This scenario actually
2950 * happened in the field several times with 7.1 releases.) As of 8.4, bad
2951 * LSNs encountered during recovery are UpdateMinRecoveryPoint's problem;
2952 * the only time we can reach here during recovery is while flushing the
2953 * end-of-recovery checkpoint record, and we don't expect that to have a
2954 * bad LSN.
2955 *
2956 * Note that for calls from xact.c, the ERROR will be promoted to PANIC
2957 * since xact.c calls this routine inside a critical section. However,
2958 * calls from bufmgr.c are not within critical sections and so we will not
2959 * force a restart for a bad LSN on a data page.
2960 */
2966 }
2968 /*
2969 * Write & flush xlog, but without specifying exactly where to.
2970 *
2971 * We normally write only completed blocks; but if there is nothing to do on
2972 * that basis, we check for unwritten async commits in the current incomplete
2973 * block, and write through the latest one of those. Thus, if async commits
2974 * are not being used, we will write complete blocks only.
2975 *
2976 * If, based on the above, there's anything to write we do so immediately. But
2977 * to avoid calling fsync, fdatasync et. al. at a rate that'd impact
2978 * concurrent IO, we only flush WAL every wal_writer_delay ms, or if there's
2979 * more than wal_writer_flush_after unflushed blocks.
2980 *
2981 * We can guarantee that async commits reach disk after at most three
2982 * wal_writer_delay cycles. (When flushing complete blocks, we allow XLogWrite
2983 * to write "flexibly", meaning it can stop at the end of the buffer ring;
2984 * this makes a difference only with very high load or long wal_writer_delay,
2985 * but imposes one extra cycle for the worst case for async commits.)
2986 *
2987 * This routine is invoked periodically by the background walwriter process.
2988 *
2989 * Returns true if there was any work to do, even if we skipped flushing due
2990 * to wal_writer_delay/wal_writer_flush_after.
2991 */
2992 bool
2994 {
2995 XLogwrtRqst WriteRqst;
2998 TimestampTz now;
3000 TimeLineID insertTLI;
3002 /* XLOG doesn't need flushing during recovery */
3006 /*
3007 * Since we're not in recovery, InsertTimeLineID is set and can't change,
3008 * so we can read it without a lock.
3009 */
3012 /* read updated LogwrtRqst */
3017 /* back off to last completed page boundary */
3020 /* if we have already flushed that far, consider async commit records */
3023 {
3028 }
3030 /*
3031 * If already known flushed, we're done. Just need to check if we are
3032 * holding an open file handle to a logfile that's no longer in use,
3033 * preventing the file from being deleted.
3034 */
3036 {
3038 {
3040 wal_segment_size))
3041 {
3043 }
3044 }
3046 }
3048 /*
3049 * Determine how far to flush WAL, based on the wal_writer_delay and
3050 * wal_writer_flush_after GUCs.
3051 *
3052 * Note that XLogSetAsyncXactLSN() performs similar calculation based on
3053 * wal_writer_flush_after, to decide when to wake us up. Make sure the
3054 * logic is the same in both places if you change this.
3055 */
3057 flushblocks =
3061 {
3062 /* first call, or block based limits disabled */
3065 }
3067 {
3068 /*
3069 * Flush the writes at least every WalWriterDelay ms. This is
3070 * important to bound the amount of time it takes for an asynchronous
3071 * commit to hit disk.
3072 */
3075 }
3077 {
3078 /* exceeded wal_writer_flush_after blocks, flush */
3081 }
3082 else
3083 {
3084 /* no flushing, this time round */
3086 }
3088 #ifdef WAL_DEBUG
3090 elog(LOG, "xlog bg flush request write %X/%X; flush: %X/%X, current is write %X/%X; flush %X/%X",
3095 #endif
3099 /* now wait for any in-progress insertions to finish and get write lock */
3105 {
3107 }
3112 /* wake up walsenders now that we've released heavily contended locks */
3115 /*
3116 * Great, done. To take some work off the critical path, try to initialize
3117 * as many of the no-longer-needed WAL buffers for future use as we can.
3118 */
3121 /*
3122 * If we determined that we need to write data, but somebody else
3123 * wrote/flushed already, it should be considered as being active, to
3124 * avoid hibernating too early.
3125 */
3127 }
3129 /*
3130 * Test whether XLOG data has been flushed up to (at least) the given position.
3131 *
3132 * Returns true if a flush is still needed. (It may be that someone else
3133 * is already in process of flushing that far, however.)
3134 */
3135 bool
3137 {
3138 /*
3139 * During recovery, we don't flush WAL but update minRecoveryPoint
3140 * instead. So "needs flush" is taken to mean whether minRecoveryPoint
3141 * would need to be updated.
3142 */
3144 {
3145 /*
3146 * An invalid minRecoveryPoint means that we need to recover all the
3147 * WAL, i.e., we're doing crash recovery. We never modify the control
3148 * file's value in that case, so we can short-circuit future checks
3149 * here too. This triggers a quick exit path for the startup process,
3150 * which cannot update its local copy of minRecoveryPoint as long as
3151 * it has not replayed all WAL available when doing crash recovery.
3152 */
3156 /* Quick exit if already known to be updated or cannot be updated */
3160 /*
3161 * Update local copy of minRecoveryPoint. But if the lock is busy,
3162 * just return a conservative guess.
3163 */
3170 /*
3171 * Check minRecoveryPoint for any other process than the startup
3172 * process doing crash recovery, which should not update the control
3173 * file value if crash recovery is still running.
3174 */
3178 /* check again */
3181 else
3183 }
3185 /* Quick exit if already known flushed */
3189 /* read LogwrtResult and update local state */
3192 /* check again */
3197 }
3199 /*
3200 * Try to make a given XLOG file segment exist.
3201 *
3202 * logsegno: identify segment.
3203 *
3204 * *added: on return, true if this call raised the number of extant segments.
3205 *
3206 * path: on return, this char[MAXPGPATH] has the path to the logsegno file.
3207 *
3208 * Returns -1 or FD of opened file. A -1 here is not an error; a caller
3209 * wanting an open segment should attempt to open "path", which usually will
3210 * succeed. (This is weird, but it's efficient for the callers.)
3211 */
3212 static int
3215 {
3217 XLogSegNo installed_segno;
3218 XLogSegNo max_segno;
3222 instr_time io_start;
3228 /*
3229 * Try to use existent file (checkpoint maker may have created it already)
3230 */
3235 {
3240 }
3241 else
3244 /*
3245 * Initialize an empty (all zeroes) segment. NOTE: it is possible that
3246 * another process is doing the same thing. If so, we will end up
3247 * pre-creating an extra log segment. That seems OK, and better than
3248 * holding the lock throughout this lengthy process.
3249 */
3259 /* do not use get_sync_bit() here --- want to fsync only at end of fill */
3266 /* Measure I/O timing when initializing segment */
3272 {
3273 ssize_t rc;
3275 /*
3276 * Zero-fill the file. With this setting, we do this the hard way to
3277 * ensure that all the file space has really been allocated. On
3278 * platforms that allow "holes" in files, just seeking to the end
3279 * doesn't allocate intermediate space. This way, we know that we
3280 * have all the space and (after the fsync below) that all the
3281 * indirect blocks are down on disk. Therefore, fdatasync(2) or
3282 * O_DSYNC will be sufficient to sync future writes to the log file.
3283 */
3288 }
3289 else
3290 {
3291 /*
3292 * Otherwise, seeking to the end and writing a solitary byte is
3293 * enough.
3294 */
3297 {
3298 /* if write didn't set errno, assume no disk space */
3300 }
3301 }
3304 /*
3305 * A full segment worth of data is written when using wal_init_zero. One
3306 * byte is written when not using it.
3307 */
3313 {
3314 /*
3315 * If we fail to make the file, delete it to release disk space
3316 */
3326 }
3328 /* Measure I/O timing when flushing segment */
3333 {
3340 }
3351 /*
3352 * Now move the segment into place with its final name. Cope with
3353 * possibility that someone else has created the file while we were
3354 * filling ours: if so, use ours to pre-create a future log segment.
3355 */
3358 /*
3359 * XXX: What should we use as max_segno? We used to use XLOGfileslop when
3360 * that was a constant, but that was always a bit dubious: normally, at a
3361 * checkpoint, XLOGfileslop was the offset from the checkpoint record, but
3362 * here, it was the offset from the insert location. We can't do the
3363 * normal XLOGfileslop calculation here because we don't have access to
3364 * the prior checkpoint's redo location. So somewhat arbitrarily, just use
3365 * CheckPointSegments.
3366 */
3369 logtli))
3370 {
3373 }
3374 else
3375 {
3376 /*
3377 * No need for any more future segments, or InstallXLogFileSegment()
3378 * failed to rename the file into place. If the rename failed, a
3379 * caller opening the file may fail.
3380 */
3383 }
3386 }
3388 /*
3389 * Create a new XLOG file segment, or open a pre-existing one.
3390 *
3391 * logsegno: identify segment to be created/opened.
3392 *
3393 * Returns FD of opened file.
3394 *
3395 * Note: errors here are ERROR not PANIC because we might or might not be
3396 * inside a critical section (eg, during checkpoint there is no reason to
3397 * take down the system on failure). They will promote to PANIC if we are
3398 * in a critical section.
3399 */
3400 int
3402 {
3413 /* Now open original target segment (might not be file I just made) */
3421 }
3423 /*
3424 * Create a new XLOG file segment by copying a pre-existing one.
3425 *
3426 * destsegno: identify segment to be created.
3427 *
3428 * srcTLI, srcsegno: identify segment to be copied (could be from
3429 * a different timeline)
3430 *
3431 * upto: how much of the source file to copy (the rest is filled with
3432 * zeros)
3433 *
3434 * Currently this is only used during recovery, and so there are no locking
3435 * considerations. But we should be just as tense as XLogFileInit to avoid
3436 * emplacing a bogus file.
3437 */
3438 static void
3442 {
3445 PGAlignedXLogBlock buffer;
3450 /*
3451 * Open the source file
3452 */
3460 /*
3461 * Copy into a temp file name.
3462 */
3467 /* do not use get_sync_bit() here --- want to fsync only at end of fill */
3474 /*
3475 * Do the data copying.
3476 */
3478 {
3483 /*
3484 * The part that is not read from the source file is filled with
3485 * zeros.
3486 */
3491 {
3499 {
3504 path)));
3505 else
3510 }
3512 }
3516 {
3519 /*
3520 * If we fail to make the file, delete it to release disk space
3521 */
3523 /* if write didn't set errno, assume problem is no disk space */
3529 }
3531 }
3550 /*
3551 * Now move the segment into place with its final name.
3552 */
3555 }
3557 /*
3558 * Install a new XLOG segment file as a current or future log segment.
3559 *
3560 * This is used both to install a newly-created segment (which has a temp
3561 * filename while it's being created) and to recycle an old segment.
3562 *
3563 * *segno: identify segment to install as (or first possible target).
3564 * When find_free is true, this is modified on return to indicate the
3565 * actual installation location or last segment searched.
3566 *
3567 * tmppath: initial name of file to install. It will be renamed into place.
3568 *
3569 * find_free: if true, install the new segment at the first empty segno
3570 * number at or after the passed numbers. If false, install the new segment
3571 * exactly where specified, deleting any existing segment file there.
3572 *
3573 * max_segno: maximum segment number to install the new file as. Fail if no
3574 * free slot is found between *segno and max_segno. (Ignored when find_free
3575 * is false.)
3576 *
3577 * tli: The timeline on which the new segment should be installed.
3578 *
3579 * Returns true if the file was installed successfully. false indicates that
3580 * max_segno limit was exceeded, the startup process has disabled this
3581 * function for now, or an error occurred while renaming the file into place.
3582 */
3583 static bool
3586 {
3596 {
3599 }
3602 {
3603 /* Force installation: get rid of any pre-existing segment file */
3605 }
3606 else
3607 {
3608 /* Find a free slot to put it in */
3610 {
3612 {
3613 /* Failed to find a free slot within specified range */
3616 }
3619 }
3620 }
3624 {
3626 /* durable_rename already emitted log message */
3628 }
3633 }
3635 /*
3636 * Open a pre-existing logfile segment for writing.
3637 */
3638 int
3640 {
3654 }
3656 /*
3657 * Close the current logfile segment for writing.
3658 */
3659 static void
3661 {
3664 /*
3665 * WAL segment files will not be re-read in normal operation, so we advise
3666 * the OS to release any cached pages. But do not do so if WAL archiving
3667 * or streaming is active, because archiver and walsender process could
3668 * use the cache to read the WAL segment.
3669 */
3670 #if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
3673 #endif
3676 {
3685 }
3689 }
3691 /*
3692 * Preallocate log files beyond the specified log endpoint.
3693 *
3694 * XXX this is currently extremely conservative, since it forces only one
3695 * future log segment to exist, and even that only if we are 75% done with
3696 * the current one. This is only appropriate for very low-WAL-volume systems.
3697 * High-volume systems will be OK once they've built up a sufficient set of
3698 * recycled log segments, but the startup transient is likely to include
3699 * a lot of segment creations by foreground processes, which is not so good.
3700 *
3701 * XLogFileInitInternal() can ereport(ERROR). All known causes indicate big
3702 * trouble; for example, a full filesystem is one cause. The checkpoint WAL
3703 * and/or ControlFile updates already completed. If a RequestCheckpoint()
3704 * initiated the present checkpoint and an ERROR ends this function, the
3705 * command that called RequestCheckpoint() fails. That's not ideal, but it's
3706 * not worth contorting more functions to use caller-specified elevel values.
3707 * (With or without RequestCheckpoint(), an ERROR forestalls some inessential
3708 * reporting and resource reclamation.)
3709 */
3710 static void
3712 {
3713 XLogSegNo _logSegNo;
3717 uint64 offset;
3725 {
3726 _logSegNo++;
3732 }
3733 }
3735 /*
3736 * Throws an error if the given log segment has already been removed or
3737 * recycled. The caller should only pass a segment that it knows to have
3738 * existed while the server has been running, as this function always
3739 * succeeds if no WAL segments have been removed since startup.
3740 * 'tli' is only used in the error message.
3741 *
3742 * Note: this function guarantees to keep errno unchanged on return.
3743 * This supports callers that use this to possibly deliver a better
3744 * error message about a missing file, while still being able to throw
3745 * a normal file-access error afterwards, if this does return.
3746 */
3747 void
3749 {
3751 XLogSegNo lastRemovedSegNo;
3758 {
3766 filename)));
3767 }
3769 }
3771 /*
3772 * Return the last WAL segment removed, or 0 if no segment has been removed
3773 * since startup.
3774 *
3775 * NB: the result can be out of date arbitrarily fast, the caller has to deal
3776 * with that.
3777 */
3778 XLogSegNo
3780 {
3781 XLogSegNo lastRemovedSegNo;
3788 }
3790 /*
3791 * Return the oldest WAL segment on the given TLI that still exists in
3792 * XLOGDIR, or 0 if none.
3793 */
3794 XLogSegNo
3796 {
3803 {
3804 TimeLineID file_tli;
3805 XLogSegNo file_segno;
3807 /* Ignore files that are not XLOG segments. */
3811 /* Parse filename to get TLI and segno. */
3813 wal_segment_size);
3815 /* Ignore anything that's not from the TLI of interest. */
3819 /* If it's the oldest so far, update oldest_segno. */
3822 }
3826 }
3828 /*
3829 * Update the last removed segno pointer in shared memory, to reflect that the
3830 * given XLOG file has been removed.
3831 */
3832 static void
3834 {
3835 uint32 tli;
3836 XLogSegNo segno;
3844 }
3846 /*
3847 * Remove all temporary log files in pg_wal
3848 *
3849 * This is called at the beginning of recovery after a previous crash,
3850 * at a point where no other processes write fresh WAL data.
3851 */
3852 static void
3854 {
3862 {
3871 }
3873 }
3875 /*
3876 * Recycle or remove all log files older or equal to passed segno.
3877 *
3878 * endptr is current (or recent) end of xlog, and lastredoptr is the
3879 * redo pointer of the last checkpoint. These are used to determine
3880 * whether we want to recycle rather than delete no-longer-wanted log files.
3881 *
3882 * insertTLI is the current timeline for XLOG insertion. Any recycled
3883 * segments should be reused for this timeline.
3884 */
3885 static void
3887 TimeLineID insertTLI)
3888 {
3892 XLogSegNo endlogSegNo;
3893 XLogSegNo recycleSegNo;
3895 /* Initialize info about where to try to recycle to */
3899 /*
3900 * Construct a filename of the last segment to be kept. The timeline ID
3901 * doesn't matter, we ignore that in the comparison. (During recovery,
3902 * InsertTimeLineID isn't set, so we can't use that.)
3903 */
3907 lastoff);
3912 {
3913 /* Ignore files that are not XLOG segments */
3918 /*
3919 * We ignore the timeline part of the XLOG segment identifiers in
3920 * deciding whether a segment is still needed. This ensures that we
3921 * won't prematurely remove a segment from a parent timeline. We could
3922 * probably be a little more proactive about removing segments of
3923 * non-parent timelines, but that would be a whole lot more
3924 * complicated.
3925 *
3926 * We use the alphanumeric sorting property of the filenames to decide
3927 * which ones are earlier than the lastoff segment.
3928 */
3930 {
3932 {
3933 /* Update the last removed location in shared memory first */
3937 }
3938 }
3939 }
3942 }
3944 /*
3945 * Recycle or remove WAL files that are not part of the given timeline's
3946 * history.
3947 *
3948 * This is called during recovery, whenever we switch to follow a new
3949 * timeline, and at the end of recovery when we create a new timeline. We
3950 * wouldn't otherwise care about extra WAL files lying in pg_wal, but they
3951 * might be leftover pre-allocated or recycled WAL segments on the old timeline
3952 * that we haven't used yet, and contain garbage. If we just leave them in
3953 * pg_wal, they will eventually be archived, and we can't let that happen.
3954 * Files that belong to our timeline history are valid, because we have
3955 * successfully replayed them, but from others we can't be sure.
3956 *
3957 * 'switchpoint' is the current point in WAL where we switch to new timeline,
3958 * and 'newTLI' is the new timeline we switch to.
3959 */
3960 void
3962 {
3966 XLogSegNo endLogSegNo;
3967 XLogSegNo switchLogSegNo;
3968 XLogSegNo recycleSegNo;
3970 /*
3971 * Initialize info about where to begin the work. This will recycle,
3972 * somewhat arbitrarily, 10 future segments.
3973 */
3978 /*
3979 * Construct a filename of the last segment to be kept.
3980 */
3984 switchseg);
3989 {
3990 /* Ignore files that are not XLOG segments */
3994 /*
3995 * Remove files that are on a timeline older than the new one we're
3996 * switching to, but with a segment number >= the first segment on the
3997 * new timeline.
3998 */
4001 {
4002 /*
4003 * If the file has already been marked as .ready, however, don't
4004 * remove it yet. It should be OK to remove it - files that are
4005 * not part of our timeline history are not required for recovery
4006 * - but seems safer to let them be archived and removed later.
4007 */
4010 }
4011 }
4014 }
4016 /*
4017 * Recycle or remove a log file that's no longer needed.
4018 *
4019 * segment_de is the dirent structure of the segment to recycle or remove.
4020 * recycleSegNo is the segment number to recycle up to. endlogSegNo is
4021 * the segment number of the current (or recent) end of WAL.
4022 *
4023 * endlogSegNo gets incremented if the segment is recycled so as it is not
4024 * checked again with future callers of this function.
4025 *
4026 * insertTLI is the current timeline for XLOG insertion. Any recycled segments
4027 * should be used for this timeline.
4028 */
4029 static void
4032 TimeLineID insertTLI)
4033 {
4035 #ifdef WIN32
4037 #endif
4042 /*
4043 * Before deleting the file, see if it can be recycled as a future log
4044 * segment. Only recycle normal files, because we don't want to recycle
4045 * symbolic links pointing to a separate archive directory.
4046 */
4053 {
4056 segname)));
4058 /* Needn't recheck that slot on future iterations */
4060 }
4061 else
4062 {
4063 /* No need for any more future segments, or recycling failed ... */
4068 segname)));
4070 #ifdef WIN32
4072 /*
4073 * On Windows, if another process (e.g another backend) holds the file
4074 * open in FILE_SHARE_DELETE mode, unlink will succeed, but the file
4075 * will still show up in directory listing until the last handle is
4076 * closed. To avoid confusing the lingering deleted file for a live
4077 * WAL file that needs to be archived, rename it before deleting it.
4078 *
4079 * If another process holds the file open without FILE_SHARE_DELETE
4080 * flag, rename will fail. We'll try again at the next checkpoint.
4081 */
4084 {
4088 path)));
4090 }
4092 #else
4094 #endif
4096 {
4097 /* Message already logged by durable_unlink() */
4099 }
4101 }
4104 }
4106 /*
4107 * Verify whether pg_wal, pg_wal/archive_status, and pg_wal/summaries exist.
4108 * If the latter do not exist, recreate them.
4109 *
4110 * It is not the goal of this function to verify the contents of these
4111 * directories, but to help in cases where someone has performed a cluster
4112 * copy for PITR purposes but omitted pg_wal from the copy.
4113 *
4114 * We could also recreate pg_wal if it doesn't exist, but a deliberate
4115 * policy decision was made not to. It is fairly common for pg_wal to be
4116 * a symlink, and if that was the DBA's intent then automatically making a
4117 * plain directory would result in degraded performance with no notice.
4118 */
4119 static void
4121 {
4125 /* Check for pg_wal; if it doesn't exist, error out */
4131 XLOGDIR)));
4133 /* Check for archive_status */
4136 {
4137 /* Check for weird cases where it exists but isn't a directory */
4142 path)));
4143 }
4144 else
4145 {
4152 path)));
4153 }
4155 /* Check for summaries */
4158 {
4159 /* Check for weird cases where it exists but isn't a directory */
4163 path)));
4164 }
4165 else
4166 {
4172 path)));
4173 }
4174 }
4176 /*
4177 * Remove previous backup history files. This also retries creation of
4178 * .ready files for any backup history files for which XLogArchiveNotify
4179 * failed earlier.
4180 */
4181 static void
4183 {
4191 {
4193 {
4195 {
4201 }
4202 }
4203 }
4206 }
4208 /*
4209 * I/O routines for pg_control
4210 *
4211 * *ControlFile is a buffer in shared memory that holds an image of the
4212 * contents of pg_control. WriteControlFile() initializes pg_control
4213 * given a preloaded buffer, ReadControlFile() loads the buffer from
4214 * the pg_control file (during postmaster or standalone-backend startup),
4215 * and UpdateControlFile() rewrites pg_control after we modify xlog state.
4216 * InitControlFile() fills the buffer with initial values.
4217 *
4218 * For simplicity, WriteControlFile() initializes the fields of pg_control
4219 * that are related to checking backend/database compatibility, and
4220 * ReadControlFile() verifies they are correct. We could split out the
4221 * I/O and compatibility-check functions, but there seems no need currently.
4222 */
4224 static void
4226 {
4229 /*
4230 * Generate a random nonce. This is used for authentication requests that
4231 * will fail because the user does not exist. The nonce is used to create
4232 * a genuine-looking password challenge for the non-existent user, in lieu
4233 * of an actual stored password.
4234 */
4241 /* Initialize pg_control status fields */
4247 /* Set important parameter values for use when replaying WAL */
4257 }
4259 static void
4261 {
4265 /*
4266 * Initialize version and compatibility-check fields
4267 */
4287 /* Contents are protected with a CRC */
4294 /*
4295 * We write out PG_CONTROL_FILE_SIZE bytes into pg_control, zero-padding
4296 * the excess over sizeof(ControlFileData). This reduces the odds of
4297 * premature-EOF errors when reading pg_control. We'll still fail when we
4298 * check the contents of the file, but hopefully with a more specific
4299 * error than "couldn't read pg_control".
4300 */
4310 XLOG_CONTROL_FILE)));
4315 {
4316 /* if write didn't set errno, assume problem is no disk space */
4322 XLOG_CONTROL_FILE)));
4323 }
4331 XLOG_CONTROL_FILE)));
4338 XLOG_CONTROL_FILE)));
4339 }
4341 static void
4343 {
4344 pg_crc32c crc;
4349 /*
4350 * Read data...
4351 */
4358 XLOG_CONTROL_FILE)));
4363 {
4368 XLOG_CONTROL_FILE)));
4369 else
4374 }
4379 /*
4380 * Check for expected pg_control format version. If this is wrong, the
4381 * CRC check will likely fail because we'll be checking the wrong number
4382 * of bytes. Complaining about wrong version will probably be more
4383 * enlightening than complaining about wrong CRC.
4384 */
4386 if (ControlFile->pg_control_version != PG_CONTROL_VERSION && ControlFile->pg_control_version % 65536 == 0 && ControlFile->pg_control_version / 65536 != 0)
4394 errhint("This could be a problem of mismatched byte ordering. It looks like you need to initdb.")));
4405 /* Now check the CRC. */
4417 /*
4418 * Do compatibility checking immediately. If the database isn't
4419 * compatible with the backend executable, we want to abort before we can
4420 * possibly do any damage.
4421 */
4426 /* translator: %s is a variable name and %d is its value */
4436 /* translator: %s is a variable name and %d is its value */
4446 errdetail("The database cluster appears to use a different floating-point number format than the server executable."),
4452 /* translator: %s is a variable name and %d is its value */
4462 /* translator: %s is a variable name and %d is its value */
4472 /* translator: %s is a variable name and %d is its value */
4482 /* translator: %s is a variable name and %d is its value */
4492 /* translator: %s is a variable name and %d is its value */
4502 /* translator: %s is a variable name and %d is its value */
4512 /* translator: %s is a variable name and %d is its value */
4519 #ifdef USE_FLOAT8_BYVAL
4527 #else
4535 #endif
4543 wal_segment_size,
4544 wal_segment_size),
4549 PGC_S_DYNAMIC_DEFAULT);
4551 /* check and update variables dependent on wal_segment_size */
4554 /* translator: both %s are GUC names */
4560 /* translator: both %s are GUC names */
4564 UsableBytesInSegment =
4570 /* Make the initdb settings visible as GUC variables, too */
4573 }
4575 /*
4576 * Utility wrapper to update the control file. Note that the control
4577 * file gets flushed.
4578 */
4579 static void
4581 {
4583 }
4585 /*
4586 * Returns the unique system identifier from control file.
4587 */
4588 uint64
4590 {
4593 }
4595 /*
4596 * Returns the random nonce from control file.
4597 */
4600 {
4603 }
4605 /*
4606 * Are checksums enabled for data pages?
4607 */
4608 bool
4610 {
4613 }
4615 /*
4616 * Returns a fake LSN for unlogged relations.
4617 *
4618 * Each call generates an LSN that is greater than any previous value
4619 * returned. The current counter value is saved and restored across clean
4620 * shutdowns, but like unlogged relations, does not survive a crash. This can
4621 * be used in lieu of real LSN values returned by XLogInsert, if you need an
4622 * LSN-like increasing sequence of numbers without writing any WAL.
4623 */
4624 XLogRecPtr
4626 {
4628 }
4630 /*
4631 * Auto-tune the number of XLOG buffers.
4632 *
4633 * The preferred setting for wal_buffers is about 3% of shared_buffers, with
4634 * a maximum of one XLOG segment (there is little reason to think that more
4635 * is helpful, at least so long as we force an fsync when switching log files)
4636 * and a minimum of 8 blocks (which was the default value prior to PostgreSQL
4637 * 9.1, when auto-tuning was added).
4638 *
4639 * This should not be called until NBuffers has received its final value.
4640 */
4641 static int
4643 {
4652 }
4654 /*
4655 * GUC check_hook for wal_buffers
4656 */
4657 bool
4659 {
4660 /*
4661 * -1 indicates a request for auto-tune.
4662 */
4664 {
4665 /*
4666 * If we haven't yet changed the boot_val default of -1, just let it
4667 * be. We'll fix it when XLOGShmemSize is called.
4668 */
4672 /* Otherwise, substitute the auto-tune value */
4674 }
4676 /*
4677 * We clamp manually-set values to at least 4 blocks. Prior to PostgreSQL
4678 * 9.1, a minimum of 4 was enforced by guc.c, but since that is no longer
4679 * the case, we just silently treat such values as a request for the
4680 * minimum. (We could throw an error instead, but that doesn't seem very
4681 * helpful.)
4682 */
4687 }
4689 /*
4690 * GUC check_hook for wal_consistency_checking
4691 */
4692 bool
4694 {
4700 /* Initialize the array */
4703 /* Need a modifiable copy of string */
4706 /* Parse string into list of identifiers */
4708 {
4709 /* syntax error in list */
4714 }
4717 {
4721 /* Check for 'all'. */
4723 {
4727 }
4728 else
4729 {
4730 /* Check if the token matches any known resource manager. */
4734 {
4737 {
4741 }
4742 }
4744 {
4745 /*
4746 * During startup, it might be a not-yet-loaded custom
4747 * resource manager. Defer checking until
4748 * InitializeWalConsistencyChecking().
4749 */
4751 {
4753 }
4754 else
4755 {
4760 }
4761 }
4762 }
4763 }
4768 /* assign new value */
4772 }
4774 /*
4775 * GUC assign_hook for wal_consistency_checking
4776 */
4777 void
4779 {
4780 /*
4781 * If some checks were deferred, it's possible that the checks will fail
4782 * later during InitializeWalConsistencyChecking(). But in that case, the
4783 * postmaster will exit anyway, so it's safe to proceed with the
4784 * assignment.
4785 *
4786 * Any built-in resource managers specified are assigned immediately,
4787 * which affects WAL created before shared_preload_libraries are
4788 * processed. Any custom resource managers specified won't be assigned
4789 * until after shared_preload_libraries are processed, but that's OK
4790 * because WAL for a custom resource manager can't be written before the
4791 * module is loaded anyway.
4792 */
4794 }
4796 /*
4797 * InitializeWalConsistencyChecking: run after loading custom resource managers
4798 *
4799 * If any unknown resource managers were specified in the
4800 * wal_consistency_checking GUC, processing was deferred. Now that
4801 * shared_preload_libraries have been loaded, process wal_consistency_checking
4802 * again.
4803 */
4804 void
4806 {
4810 {
4818 wal_consistency_checking_string,
4822 /* checking should not be deferred again */
4824 }
4825 }
4827 /*
4828 * GUC show_hook for archive_command
4829 */
4832 {
4835 else
4837 }
4839 /*
4840 * GUC show_hook for in_hot_standby
4841 */
4844 {
4845 /*
4846 * We display the actual state based on shared memory, so that this GUC
4847 * reports up-to-date state if examined intra-query. The underlying
4848 * variable (in_hot_standby_guc) changes only when we transmit a new value
4849 * to the client.
4850 */
4852 }
4854 /*
4855 * Read the control file, set respective GUCs.
4856 *
4857 * This is to be called during startup, including a crash recovery cycle,
4858 * unless in bootstrap mode, where no control file yet exists. As there's no
4859 * usable shared memory yet (its sizing can depend on the contents of the
4860 * control file!), first store the contents in local memory. XLOGShmemInit()
4861 * will then copy it to shared memory later.
4862 *
4863 * reset just controls whether previous contents are to be expected (in the
4864 * reset case, there's a dangling pointer into old shared memory), or not.
4865 */
4866 void
4868 {
4872 }
4874 /*
4875 * Get the wal_level from the control file. For a standby, this value should be
4876 * considered as its active wal_level, because it may be different from what
4877 * was originally configured on standby.
4878 */
4879 WalLevel
4881 {
4883 }
4885 /*
4886 * Initialization of shared memory for XLOG
4887 */
4888 Size
4890 {
4891 Size size;
4893 /*
4894 * If the value of wal_buffers is -1, use the preferred auto-tune value.
4895 * This isn't an amazingly clean place to do this, but we must wait till
4896 * NBuffers has received its final value, and must do it before using the
4897 * value of XLOGbuffers to do anything important.
4898 *
4899 * We prefer to report this value's source as PGC_S_DYNAMIC_DEFAULT.
4900 * However, if the DBA explicitly set wal_buffers = -1 in the config file,
4901 * then PGC_S_DYNAMIC_DEFAULT will fail to override that and we must force
4902 * the matter with PGC_S_OVERRIDE.
4903 */
4905 {
4910 PGC_S_DYNAMIC_DEFAULT);
4913 PGC_S_OVERRIDE);
4914 }
4917 /* XLogCtl */
4920 /* WAL insertion locks, plus alignment */
4922 /* xlblocks array */
4924 /* extra alignment padding for XLOG I/O buffers */
4926 /* and the buffers themselves */
4929 /*
4930 * Note: we don't count ControlFileData, it comes out of the "slop factor"
4931 * added by CreateSharedMemoryAndSemaphores. This lets us use this
4932 * routine again below to compute the actual allocation size.
4933 */
4936 }
4938 void
4940 {
4942 foundXLog;
4947 #ifdef WAL_DEBUG
4949 /*
4950 * Create a memory context for WAL debugging that's exempt from the normal
4951 * "no pallocs in critical section" rule. Yes, that can lead to a PANIC if
4952 * an allocation fails, but wal_debug is not for production use anyway.
4953 */
4955 {
4958 ALLOCSET_DEFAULT_SIZES);
4960 }
4961 #endif
4972 {
4973 /* both should be present or neither */
4976 /* Initialize local copy of WALInsertLocks */
4982 }
4985 /*
4986 * Already have read control file locally, unless in bootstrap mode. Move
4987 * contents into shared memory.
4988 */
4990 {
4993 }
4995 /*
4996 * Since XLogCtlData contains XLogRecPtr fields, its sizeof should be a
4997 * multiple of the alignment for same, so no extra alignment padding is
4998 * needed here.
4999 */
5005 {
5007 }
5009 /* WAL insertion locks. Ensure they're aligned to the full padded size */
5017 {
5021 }
5023 /*
5024 * Align the start of the page buffers to a full xlog block size boundary.
5025 * This simplifies some calculations in XLOG insertion. It is also
5026 * required for O_DIRECT.
5027 */
5032 /*
5033 * Do basic initialization of XLogCtl shared data. (StartupXLOG will fill
5034 * in additional info.)
5035 */
5047 }
5049 /*
5050 * This func must be called ONCE on system install. It creates pg_control
5051 * and the initial XLOG segment.
5052 */
5053 void
5055 {
5056 CheckPoint checkPoint;
5058 XLogPageHeader page;
5059 XLogLongPageHeader longpage;
5062 uint64 sysidentifier;
5064 pg_crc32c crc;
5066 /* allow ordinary WAL segment creation, like StartupXLOG() would */
5069 /*
5070 * Select a hopefully-unique system identifier code for this installation.
5071 * We use the result of gettimeofday(), including the fractional seconds
5072 * field, as being about as unique as we can easily get. (Think not to
5073 * use random(), since it hasn't been seeded and there's no portable way
5074 * to seed it other than the system clock value...) The upper half of the
5075 * uint64 value is just the tv_sec part, while the lower half contains the
5076 * tv_usec part (which must fit in 20 bits), plus 12 bits from our current
5077 * PID for a little extra uniqueness. A person knowing this encoding can
5078 * determine the initialization time of the installation, which could
5079 * perhaps be useful sometimes.
5080 */
5086 /* page buffer must be aligned suitably for O_DIRECT */
5091 /*
5092 * Set up information for the initial checkpoint record
5093 *
5094 * The initial checkpoint record is written to the beginning of the WAL
5095 * segment with logid=0 logseg=1. The very first WAL segment, 0/0, is not
5096 * used, so that we can use 0/0 to mean "before any valid WAL segment".
5097 */
5126 /* Set up the XLOG page header */
5136 /* Insert the initial checkpoint record */
5145 /* fill the XLogRecordDataHeaderShort struct */
5158 /* Create first XLOG segment file */
5162 /*
5163 * We needn't bother with Reserve/ReleaseExternalFD here, since we'll
5164 * close the file again in a moment.
5165 */
5167 /* Write the first page with the initial record */
5171 {
5172 /* if write didn't set errno, assume problem is no disk space */
5178 }
5195 /* Now create pg_control */
5201 /* some additional ControlFile fields are set in WriteControlFile() */
5204 /* Bootstrap the commit log, too */
5212 /*
5213 * Force control file to be read - in contrast to normal processing we'd
5214 * otherwise never run the checks and GUC related initializations therein.
5215 */
5217 }
5221 {
5229 }
5231 /*
5232 * Initialize the first WAL segment on new timeline.
5233 */
5234 static void
5236 {
5238 XLogSegNo endLogSegNo;
5239 XLogSegNo startLogSegNo;
5241 /* we always switch to a new timeline after archive recovery */
5244 /*
5245 * Update min recovery point one last time.
5246 */
5249 /*
5250 * Calculate the last segment on the old timeline, and the first segment
5251 * on the new timeline. If the switch happens in the middle of a segment,
5252 * they are the same, but if the switch happens exactly at a segment
5253 * boundary, startLogSegNo will be endLogSegNo + 1.
5254 */
5258 /*
5259 * Initialize the starting WAL segment for the new timeline. If the switch
5260 * happens in the middle of a segment, copy data from the last WAL segment
5261 * of the old timeline up to the switch point, to the starting WAL segment
5262 * on the new timeline.
5263 */
5265 {
5266 /*
5267 * Make a copy of the file on the new timeline.
5268 *
5269 * Writing WAL isn't allowed yet, so there are no locking
5270 * considerations. But we should be just as tense as XLogFileInit to
5271 * avoid emplacing a bogus file.
5272 */
5275 }
5276 else
5277 {
5278 /*
5279 * The switch happened at a segment boundary, so just create the next
5280 * segment on the new timeline.
5281 */
5287 {
5295 }
5296 }
5298 /*
5299 * Let's just make real sure there are not .ready or .done flags posted
5300 * for the new segment.
5301 */
5304 }
5306 /*
5307 * Perform cleanup actions at the conclusion of archive recovery.
5308 */
5309 static void
5311 TimeLineID newTLI)
5312 {
5313 /*
5314 * Execute the recovery_end_command, if any.
5315 */
5320 WAIT_EVENT_RECOVERY_END_COMMAND);
5322 /*
5323 * We switched to a new timeline. Clean up segments on the old timeline.
5324 *
5325 * If there are any higher-numbered segments on the old timeline, remove
5326 * them. They might contain valid WAL, but they might also be
5327 * pre-allocated files containing garbage. In any case, they are not part
5328 * of the new timeline's history so we don't need them.
5329 */
5332 /*
5333 * If the switch happened in the middle of a segment, what to do with the
5334 * last, partial segment on the old timeline? If we don't archive it, and
5335 * the server that created the WAL never archives it either (e.g. because
5336 * it was hit by a meteor), it will never make it to the archive. That's
5337 * OK from our point of view, because the new segment that we created with
5338 * the new TLI contains all the WAL from the old timeline up to the switch
5339 * point. But if you later try to do PITR to the "missing" WAL on the old
5340 * timeline, recovery won't find it in the archive. It's physically
5341 * present in the new file with new TLI, but recovery won't look there
5342 * when it's recovering to the older timeline. On the other hand, if we
5343 * archive the partial segment, and the original server on that timeline
5344 * is still running and archives the completed version of the same segment
5345 * later, it will fail. (We used to do that in 9.4 and below, and it
5346 * caused such problems).
5347 *
5348 * As a compromise, we rename the last segment with the .partial suffix,
5349 * and archive it. Archive recovery will never try to read .partial
5350 * segments, so they will normally go unused. But in the odd PITR case,
5351 * the administrator can copy them manually to the pg_wal directory
5352 * (removing the suffix). They can be useful in debugging, too.
5353 *
5354 * If a .done or .ready file already exists for the old timeline, however,
5355 * we had already determined that the segment is complete, so we can let
5356 * it be archived normally. (In particular, if it was restored from the
5357 * archive to begin with, it's expected to have a .done file).
5358 */
5361 {
5363 XLogSegNo endLogSegNo;
5369 {
5374 /*
5375 * If we're summarizing WAL, we can't rename the partial file
5376 * until the summarizer finishes with it, else it will fail.
5377 */
5385 /*
5386 * Make sure there's no .done or .ready file for the .partial
5387 * file.
5388 */
5393 }
5394 }
5395 }
5397 /*
5398 * Check to see if required parameters are set high enough on this server
5399 * for various aspects of recovery operation.
5400 *
5401 * Note that all the parameters which this function tests need to be
5402 * listed in Administrator's Overview section in high-availability.sgml.
5403 * If you change them, don't forget to update the list.
5404 */
5405 static void
5407 {
5408 /*
5409 * For archive recovery, the WAL must be generated with at least 'replica'
5410 * wal_level.
5411 */
5413 {
5419 }
5421 /*
5422 * For Hot Standby, the WAL must be generated with 'replica' mode, and we
5423 * must have at least as many backend slots as the primary.
5424 */
5426 {
5427 /* We ignore autovacuum_worker_slots when we make this test. */
5429 MaxConnections,
5432 max_worker_processes,
5435 max_wal_senders,
5438 max_prepared_xacts,
5441 max_locks_per_xact,
5443 }
5444 }
5446 /*
5447 * This must be called ONCE during postmaster or standalone-backend startup
5448 */
5449 void
5451 {
5453 CheckPoint checkPoint;
5458 XLogRecPtr EndOfLog;
5459 TimeLineID EndOfLogTLI;
5460 TimeLineID newTLI;
5463 XLogRecPtr abortedRecPtr;
5464 XLogRecPtr missingContrecPtr;
5465 TransactionId oldestActiveXID;
5468 /*
5469 * We should have an aux process resource owner to use, and we should not
5470 * be in a transaction that's installed some other resowner.
5471 */
5477 /*
5478 * Check that contents look valid.
5479 */
5486 {
5489 /*
5490 * This is the expected case, so don't be chatty in standalone
5491 * mode
5492 */
5536 }
5538 /* This is just to allow attaching to startup process with a debugger */
5539 #ifdef XLOG_REPLAY_DELAY
5542 #endif
5544 /*
5545 * Verify that pg_wal, pg_wal/archive_status, and pg_wal/summaries exist.
5546 * In cases where someone has performed a copy for PITR, these directories
5547 * may have been excluded and need to be re-created.
5548 */
5551 /* Set up timeout handler needed to report startup progress. */
5554 startup_progress_timeout_handler);
5556 /*----------
5557 * If we previously crashed, perform a couple of actions:
5558 *
5559 * - The pg_wal directory may still include some temporary WAL segments
5560 * used when creating a new segment, so perform some clean up to not
5561 * bloat this path. This is done first as there is no point to sync
5562 * this temporary data.
5563 *
5564 * - There might be data which we had written, intending to fsync it, but
5565 * which we had not actually fsync'd yet. Therefore, a power failure in
5566 * the near future might cause earlier unflushed writes to be lost, even
5567 * though more recent data written to disk from here on would be
5568 * persisted. To avoid that, fsync the entire data directory.
5569 */
5572 {
5576 }
5577 else
5580 /*
5581 * Prepare for WAL recovery if needed.
5582 *
5583 * InitWalRecovery analyzes the control file and the backup label file, if
5584 * any. It updates the in-memory ControlFile buffer according to the
5585 * starting checkpoint, and sets InRecovery and ArchiveRecoveryRequested.
5586 * It also applies the tablespace map file, if any.
5587 */
5592 /* initialize shared memory variables from the checkpoint record */
5604 /*
5605 * Clear out any old relcache cache files. This is *necessary* if we do
5606 * any WAL replay, since that would probably result in the cache files
5607 * being out of sync with database reality. In theory we could leave them
5608 * in place if the database had been cleanly shut down, but it seems
5609 * safest to just remove them always and let them be rebuilt during the
5610 * first backend startup. These files needs to be removed from all
5611 * directories including pg_tblspc, however the symlinks are created only
5612 * after reading tablespace_map file in case of archive recovery from
5613 * backup, so needs to clear old relcache files here after creating
5614 * symlinks.
5615 */
5618 /*
5619 * Initialize replication slots, before there's a chance to remove
5620 * required resources.
5621 */
5624 /*
5625 * Startup logical state, needs to be setup now so we have proper data
5626 * during crash recovery.
5627 */
5630 /*
5631 * Startup CLOG. This must be done after TransamVariables->nextXid has
5632 * been initialized and before we accept connections or begin WAL replay.
5633 */
5636 /*
5637 * Startup MultiXact. We need to do this early to be able to replay
5638 * truncations.
5639 */
5642 /*
5643 * Ditto for commit timestamps. Activate the facility if the setting is
5644 * enabled in the control file, as there should be no tracking of commit
5645 * timestamps done when the setting was disabled. This facility can be
5646 * started or stopped when replaying a XLOG_PARAMETER_CHANGE record.
5647 */
5651 /*
5652 * Recover knowledge about replay progress of known replication partners.
5653 */
5656 /*
5657 * Initialize unlogged LSN. On a clean shutdown, it's restored from the
5658 * control file. On recovery, all unlogged relations are blown away, so
5659 * the unlogged LSN counter can be reset too.
5660 */
5664 else
5666 FirstNormalUnloggedLSN);
5668 /*
5669 * Copy any missing timeline history files between 'now' and the recovery
5670 * target timeline from archive to pg_wal. While we don't need those files
5671 * ourselves - the history file of the recovery target timeline covers all
5672 * the previous timelines in the history too - a cascading standby server
5673 * might be interested in them. Or, if you archive the WAL from this
5674 * server to a different archive than the primary, it'd be good for all
5675 * the history files to get archived there after failover, so that you can
5676 * use one of the old timelines as a PITR target. Timeline history files
5677 * are small, so it's better to copy them unnecessarily than not copy them
5678 * and regret later.
5679 */
5682 /*
5683 * Before running in recovery, scan pg_twophase and fill in its status to
5684 * be able to work on entries generated by redo. Doing a scan before
5685 * taking any recovery action has the merit to discard any 2PC files that
5686 * are newer than the first record to replay, saving from any conflicts at
5687 * replay. This avoids as well any subsequent scans when doing recovery
5688 * of the on-disk two-phase data.
5689 */
5692 /*
5693 * When starting with crash recovery, reset pgstat data - it might not be
5694 * valid. Otherwise restore pgstat data. It's safe to do this here,
5695 * because postmaster will not yet have started any other processes.
5696 *
5697 * NB: Restoring replication slot stats relies on slot state to have
5698 * already been restored from disk.
5699 *
5700 * TODO: With a bit of extra work we could just start with a pgstat file
5701 * associated with the checkpoint redo location we're starting from.
5702 */
5705 else
5713 /* REDO */
5715 {
5716 /* Initialize state for RecoveryInProgress() */
5720 else
5724 /*
5725 * Update pg_control to show that we are recovering and to show the
5726 * selected checkpoint as the place we are starting from. We also mark
5727 * pg_control with any minimum recovery stop point obtained from a
5728 * backup history file.
5729 *
5730 * No need to hold ControlFileLock yet, we aren't up far enough.
5731 */
5734 /*
5735 * If there was a backup label file, it's done its job and the info
5736 * has now been propagated into pg_control. We must get rid of the
5737 * label file so that if we crash during recovery, we'll pick up at
5738 * the latest recovery restartpoint instead of going all the way back
5739 * to the backup start point. It seems prudent though to just rename
5740 * the file out of the way rather than delete it completely.
5741 */
5743 {
5746 }
5748 /*
5749 * If there was a tablespace_map file, it's done its job and the
5750 * symlinks have been created. We must get rid of the map file so
5751 * that if we crash during recovery, we don't create symlinks again.
5752 * It seems prudent though to just rename the file out of the way
5753 * rather than delete it completely.
5754 */
5756 {
5759 }
5761 /*
5762 * Initialize our local copy of minRecoveryPoint. When doing crash
5763 * recovery we want to replay up to the end of WAL. Particularly, in
5764 * the case of a promoted standby minRecoveryPoint value in the
5765 * control file is only updated after the first checkpoint. However,
5766 * if the instance crashes before the first post-recovery checkpoint
5767 * is completed then recovery will use a stale location causing the
5768 * startup process to think that there are still invalid page
5769 * references when checking for data consistency.
5770 */
5772 {
5775 }
5776 else
5777 {
5780 }
5782 /* Check that the GUCs used to generate the WAL allow recovery */
5785 /*
5786 * We're in recovery, so unlogged relations may be trashed and must be
5787 * reset. This should be done BEFORE allowing Hot Standby
5788 * connections, so that read-only backends don't try to read whatever
5789 * garbage is left over from before.
5790 */
5793 /*
5794 * Likewise, delete any saved transaction snapshot files that got left
5795 * behind by crashed backends.
5796 */
5799 /*
5800 * Initialize for Hot Standby, if enabled. We won't let backends in
5801 * yet, not until we've reached the min recovery point specified in
5802 * control file and we've established a recovery snapshot from a
5803 * running-xacts WAL record.
5804 */
5806 {
5817 else
5821 /* Tell procarray about the range of xids it has to deal with */
5824 /*
5825 * Startup subtrans only. CLOG, MultiXact and commit timestamp
5826 * have already been started up and other SLRUs are not maintained
5827 * during recovery and need not be started yet.
5828 */
5831 /*
5832 * If we're beginning at a shutdown checkpoint, we know that
5833 * nothing was running on the primary at this point. So fake-up an
5834 * empty running-xacts record and use that here and now. Recover
5835 * additional standby state for prepared transactions.
5836 */
5838 {
5839 RunningTransactionsData running;
5840 TransactionId latestCompletedXid;
5842 /* Update pg_subtrans entries for any prepared transactions */
5845 /*
5846 * Construct a RunningTransactions snapshot representing a
5847 * shut down server, with only prepared transactions still
5848 * alive. We're never overflowed at this point because all
5849 * subxids are listed with their parent prepared transactions.
5850 */
5863 }
5864 }
5866 /*
5867 * We're all set for replaying the WAL now. Do it.
5868 */
5871 }
5872 else
5875 /*
5876 * Finish WAL recovery.
5877 */
5884 /*
5885 * Reset ps status display, so as no information related to recovery shows
5886 * up.
5887 */
5890 /*
5891 * When recovering from a backup (we are in recovery, and archive recovery
5892 * was requested), complain if we did not roll forward far enough to reach
5893 * the point where the database is consistent. For regular online
5894 * backup-from-primary, that means reaching the end-of-backup WAL record
5895 * (at which point we reset backupStartPoint to be Invalid), for
5896 * backup-from-replica (which can't inject records into the WAL stream),
5897 * that point is when we reach the minRecoveryPoint in pg_control (which
5898 * we purposefully copy last when backing up from a replica). For
5899 * pg_rewind (which creates a backup_label with a method of "pg_rewind")
5900 * or snapshot-style backups (which don't), backupEndRequired will be set
5901 * to false.
5902 *
5903 * Note: it is indeed okay to look at the local variable
5904 * LocalMinRecoveryPoint here, even though ControlFile->minRecoveryPoint
5905 * might be further ahead --- ControlFile->minRecoveryPoint cannot have
5906 * been advanced beyond the WAL we processed.
5907 */
5911 {
5912 /*
5913 * Ran off end of WAL before reaching end-of-backup WAL record, or
5914 * minRecoveryPoint. That's a bad sign, indicating that you tried to
5915 * recover from an online backup but never called pg_backup_stop(), or
5916 * you didn't archive all the WAL needed.
5917 */
5919 {
5925 else
5929 }
5930 }
5932 /*
5933 * Reset unlogged relations to the contents of their INIT fork. This is
5934 * done AFTER recovery is complete so as to include any unlogged relations
5935 * created during recovery, but BEFORE recovery is marked as having
5936 * completed successfully. Otherwise we'd not retry if any of the post
5937 * end-of-recovery steps fail.
5938 */
5942 /*
5943 * Pre-scan prepared transactions to find out the range of XIDs present.
5944 * This information is not quite needed yet, but it is positioned here so
5945 * as potential problems are detected before any on-disk change is done.
5946 */
5949 /*
5950 * Allow ordinary WAL segment creation before possibly switching to a new
5951 * timeline, which creates a new segment, and after the last ReadRecord().
5952 */
5955 /*
5956 * Consider whether we need to assign a new timeline ID.
5957 *
5958 * If we did archive recovery, we always assign a new ID. This handles a
5959 * couple of issues. If we stopped short of the end of WAL during
5960 * recovery, then we are clearly generating a new timeline and must assign
5961 * it a unique new ID. Even if we ran to the end, modifying the current
5962 * last segment is problematic because it may result in trying to
5963 * overwrite an already-archived copy of that segment, and we encourage
5964 * DBAs to make their archive_commands reject that. We can dodge the
5965 * problem by making the new active segment have a new timeline ID.
5966 *
5967 * In a normal crash recovery, we can just extend the timeline we were in.
5968 */
5971 {
5976 /*
5977 * Make a writable copy of the last WAL segment. (Note that we also
5978 * have a copy of the last block of the old WAL in
5979 * endOfRecovery->lastPage; we will use that below.)
5980 */
5983 /*
5984 * Remove the signal files out of the way, so that we don't
5985 * accidentally re-enter archive recovery mode in a subsequent crash.
5986 */
5993 /*
5994 * Write the timeline history file, and have it archived. After this
5995 * point (or rather, as soon as the file is archived), the timeline
5996 * will appear as "taken" in the WAL archive and to any standby
5997 * servers. If we crash before actually switching to the new
5998 * timeline, standby servers will nevertheless think that we switched
5999 * to the new timeline, and will try to connect to the new timeline.
6000 * To minimize the window for that, try to do as little as possible
6001 * between here and writing the end-of-recovery record.
6002 */
6008 }
6010 /* Save the selected TimeLineID in shared memory, too */
6016 /*
6017 * Actually, if WAL ended in an incomplete record, skip the parts that
6018 * made it through and start writing after the portion that persisted.
6019 * (It's critical to first write an OVERWRITE_CONTRECORD message, which
6020 * we'll do as soon as we're open for writing new WAL.)
6021 */
6023 {
6024 /*
6025 * We should only have a missingContrecPtr if we're not switching to a
6026 * new timeline. When a timeline switch occurs, WAL is copied from the
6027 * old timeline to the new only up to the end of the last complete
6028 * record, so there can't be an incomplete WAL record that we need to
6029 * disregard.
6030 */
6034 }
6036 /*
6037 * Prepare to write WAL starting at EndOfLog location, and init xlog
6038 * buffer cache using the block containing the last record from the
6039 * previous incarnation.
6040 */
6045 /*
6046 * Tricky point here: lastPage contains the *last* block that the LastRec
6047 * record spans, not the one it starts in. The last block is indeed the
6048 * one we want to use.
6049 */
6051 {
6060 /* Copy the valid part of the last block, and zero the rest */
6065 pg_atomic_write_u64(&XLogCtl->xlblocks[firstIdx], endOfRecoveryInfo->lastPageBeginPtr + XLOG_BLCKSZ);
6067 }
6068 else
6069 {
6070 /*
6071 * There is no partial block to copy. Just set InitializedUpTo, and
6072 * let the first attempt to insert a log record to initialize the next
6073 * buffer.
6074 */
6076 }
6078 /*
6079 * Update local and shared status. This is OK to do without any locks
6080 * because no other process can be reading or writing WAL yet.
6081 */
6089 /*
6090 * Preallocate additional log files, if wanted.
6091 */
6094 /*
6095 * Okay, we're officially UP.
6096 */
6099 /* start the archive_timeout timer and LSN running */
6103 /* also initialize latestCompletedXid, to nextXid - 1 */
6109 /*
6110 * Start up subtrans, if not already done for hot standby. (commit
6111 * timestamps are started below, if necessary.)
6112 */
6116 /*
6117 * Perform end of recovery actions for any SLRUs that need it.
6118 */
6122 /*
6123 * Reload shared-memory state for prepared transactions. This needs to
6124 * happen before renaming the last partial segment of the old timeline as
6125 * it may be possible that we have to recover some transactions from it.
6126 */
6129 /* Shut down xlogreader */
6132 /* Enable WAL writes for this backend only. */
6135 /* If necessary, write overwrite-contrecord before doing anything else */
6137 {
6140 }
6142 /*
6143 * Update full_page_writes in shared memory and write an XLOG_FPW_CHANGE
6144 * record before resource manager writes cleanup WAL records or checkpoint
6145 * record is written.
6146 */
6150 /*
6151 * Emit checkpoint or end-of-recovery record in XLOG, if required.
6152 */
6156 /*
6157 * If any of the critical GUCs have changed, log them before we allow
6158 * backends to write WAL.
6159 */
6162 /* If this is archive recovery, perform post-recovery cleanup actions. */
6166 /*
6167 * Local WAL inserts enabled, so it's time to finish initialization of
6168 * commit timestamp.
6169 */
6172 /*
6173 * All done with end-of-recovery actions.
6174 *
6175 * Now allow backends to write WAL and update the control file status in
6176 * consequence. SharedRecoveryState, that controls if backends can write
6177 * WAL, is updated while holding ControlFileLock to prevent other backends
6178 * to look at an inconsistent state of the control file in shared memory.
6179 * There is still a small window during which backends can write WAL and
6180 * the control file is still referring to a system not in DB_IN_PRODUCTION
6181 * state while looking at the on-disk control file.
6182 *
6183 * Also, we use info_lck to update SharedRecoveryState to ensure that
6184 * there are no race conditions concerning visibility of other recent
6185 * updates to shared memory.
6186 */
6197 /*
6198 * Shutdown the recovery environment. This must occur after
6199 * RecoverPreparedTransactions() (see notes in lock_twophase_recover())
6200 * and after switching SharedRecoveryState to RECOVERY_STATE_DONE so as
6201 * any session building a snapshot will not rely on KnownAssignedXids as
6202 * RecoveryInProgress() would return false at this stage. This is
6203 * particularly critical for prepared 2PC transactions, that would still
6204 * need to be included in snapshots once recovery has ended.
6205 */
6209 /*
6210 * If there were cascading standby servers connected to us, nudge any wal
6211 * sender processes to notice that we've been promoted.
6212 */
6215 /*
6216 * If this was a promotion, request an (online) checkpoint now. This isn't
6217 * required for consistency, but the last restartpoint might be far back,
6218 * and in case of a crash, recovering from it might take a longer than is
6219 * appropriate now that we're not in standby mode anymore.
6220 */
6223 }
6225 /*
6226 * Callback from PerformWalRecovery(), called when we switch from crash
6227 * recovery to archive recovery mode. Updates the control file accordingly.
6228 */
6229 void
6231 {
6232 /* initialize minRecoveryPoint to this record */
6236 {
6239 }
6240 /* update local copy */
6244 /*
6245 * The startup process can update its local copy of minRecoveryPoint from
6246 * this point.
6247 */
6252 /*
6253 * We update SharedRecoveryState while holding the lock on ControlFileLock
6254 * so both states are consistent in shared memory.
6255 */
6261 }
6263 /*
6264 * Callback from PerformWalRecovery(), called when we reach the end of backup.
6265 * Updates the control file accordingly.
6266 */
6267 void
6269 {
6270 /*
6271 * We have reached the end of base backup, as indicated by pg_control. The
6272 * data on disk is now consistent (unless minRecoveryPoint is further
6273 * ahead, which can happen if we crashed during previous recovery). Reset
6274 * backupStartPoint and backupEndPoint, and update minRecoveryPoint to
6275 * make sure we don't allow starting up at an earlier point even if
6276 * recovery is stopped and restarted soon after this.
6277 */
6281 {
6284 }
6292 }
6294 /*
6295 * Perform whatever XLOG actions are necessary at end of REDO.
6296 *
6297 * The goal here is to make sure that we'll be able to recover properly if
6298 * we crash again. If we choose to write a checkpoint, we'll write a shutdown
6299 * checkpoint rather than an on-line one. This is not particularly critical,
6300 * but since we may be assigning a new TLI, using a shutdown checkpoint allows
6301 * us to have the rule that TLI only changes in shutdown checkpoints, which
6302 * allows some extra error checking in xlog_redo.
6303 */
6304 static bool
6306 {
6309 /*
6310 * Perform a checkpoint to update all our recovery activity to disk.
6311 *
6312 * Note that we write a shutdown checkpoint rather than an on-line one.
6313 * This is not particularly critical, but since we may be assigning a new
6314 * TLI, using a shutdown checkpoint allows us to have the rule that TLI
6315 * only changes in shutdown checkpoints, which allows some extra error
6316 * checking in xlog_redo.
6317 *
6318 * In promotion, only create a lightweight end-of-recovery record instead
6319 * of a full checkpoint. A checkpoint is requested later, after we're
6320 * fully out of recovery mode and already accepting queries.
6321 */
6324 {
6327 /*
6328 * Insert a special WAL record to mark the end of recovery, since we
6329 * aren't doing a checkpoint. That means that the checkpointer process
6330 * may likely be in the middle of a time-smoothed restartpoint and
6331 * could continue to be for minutes after this. That sounds strange,
6332 * but the effect is roughly the same and it would be stranger to try
6333 * to come out of the restartpoint and then checkpoint. We request a
6334 * checkpoint later anyway, just for safety.
6335 */
6337 }
6338 else
6339 {
6341 CHECKPOINT_IMMEDIATE |
6342 CHECKPOINT_WAIT);
6343 }
6346 }
6348 /*
6349 * Is the system still in recovery?
6350 *
6351 * Unlike testing InRecovery, this works in any process that's connected to
6352 * shared memory.
6353 */
6354 bool
6356 {
6357 /*
6358 * We check shared state each time only until we leave recovery mode. We
6359 * can't re-enter recovery, so there's no need to keep checking after the
6360 * shared variable has once been seen false.
6361 */
6364 else
6365 {
6366 /*
6367 * use volatile pointer to make sure we make a fresh read of the
6368 * shared variable.
6369 */
6374 /*
6375 * Note: We don't need a memory barrier when we're still in recovery.
6376 * We might exit recovery immediately after return, so the caller
6377 * can't rely on 'true' meaning that we're still in recovery anyway.
6378 */
6381 }
6382 }
6384 /*
6385 * Returns current recovery state from shared memory.
6386 *
6387 * This returned state is kept consistent with the contents of the control
6388 * file. See details about the possible values of RecoveryState in xlog.h.
6389 */
6390 RecoveryState
6392 {
6393 RecoveryState retval;
6400 }
6402 /*
6403 * Is this process allowed to insert new WAL records?
6404 *
6405 * Ordinarily this is essentially equivalent to !RecoveryInProgress().
6406 * But we also have provisions for forcing the result "true" or "false"
6407 * within specific processes regardless of the global state.
6408 */
6409 bool
6411 {
6412 /*
6413 * If value is "unconditionally true" or "unconditionally false", just
6414 * return it. This provides the normal fast path once recovery is known
6415 * done.
6416 */
6420 /*
6421 * Else, must check to see if we're still in recovery.
6422 */
6426 /*
6427 * On exit from recovery, reset to "unconditionally true", since there is
6428 * no need to keep checking.
6429 */
6432 }
6434 /*
6435 * Make XLogInsertAllowed() return true in the current process only.
6436 *
6437 * Note: it is allowed to switch LocalXLogInsertAllowed back to -1 later,
6438 * and even call LocalSetXLogInsertAllowed() again after that.
6439 *
6440 * Returns the previous value of LocalXLogInsertAllowed.
6441 */
6442 static int
6444 {
6450 }
6452 /*
6453 * Return the current Redo pointer from shared memory.
6454 *
6455 * As a side-effect, the local RedoRecPtr copy is updated.
6456 */
6457 XLogRecPtr
6459 {
6460 XLogRecPtr ptr;
6462 /*
6463 * The possibly not up-to-date copy in XlogCtl is enough. Even if we
6464 * grabbed a WAL insertion lock to read the authoritative value in
6465 * Insert->RedoRecPtr, someone might update it just after we've released
6466 * the lock.
6467 */
6476 }
6478 /*
6479 * Return information needed to decide whether a modified block needs a
6480 * full-page image to be included in the WAL record.
6481 *
6482 * The returned values are cached copies from backend-private memory, and
6483 * possibly out-of-date or, indeed, uninitialized, in which case they will
6484 * be InvalidXLogRecPtr and false, respectively. XLogInsertRecord will
6485 * re-check them against up-to-date values, while holding the WAL insert lock.
6486 */
6487 void
6489 {
6492 }
6494 /*
6495 * GetInsertRecPtr -- Returns the current insert position.
6496 *
6497 * NOTE: The value *actually* returned is the position of the last full
6498 * xlog page. It lags behind the real insert position by at most 1 page.
6499 * For that, we don't need to scan through WAL insertion locks, and an
6500 * approximation is enough for the current usage of this function.
6501 */
6502 XLogRecPtr
6504 {
6505 XLogRecPtr recptr;
6512 }
6514 /*
6515 * GetFlushRecPtr -- Returns the current flush position, ie, the last WAL
6516 * position known to be fsync'd to disk. This should only be used on a
6517 * system that is known not to be in recovery.
6518 */
6519 XLogRecPtr
6521 {
6526 /*
6527 * If we're writing and flushing WAL, the time line can't be changing, so
6528 * no lock is required.
6529 */
6534 }
6536 /*
6537 * GetWALInsertionTimeLine -- Returns the current timeline of a system that
6538 * is not in recovery.
6539 */
6540 TimeLineID
6542 {
6545 /* Since the value can't be changing, no lock is required. */
6547 }
6549 /*
6550 * GetWALInsertionTimeLineIfSet -- If the system is not in recovery, returns
6551 * the WAL insertion timeline; else, returns 0. Wherever possible, use
6552 * GetWALInsertionTimeLine() instead, since it's cheaper. Note that this
6553 * function decides recovery has ended as soon as the insert TLI is set, which
6554 * happens before we set XLogCtl->SharedRecoveryState to RECOVERY_STATE_DONE.
6555 */
6556 TimeLineID
6558 {
6559 TimeLineID insertTLI;
6566 }
6568 /*
6569 * GetLastImportantRecPtr -- Returns the LSN of the last important record
6570 * inserted. All records not explicitly marked as unimportant are considered
6571 * important.
6572 *
6573 * The LSN is determined by computing the maximum of
6574 * WALInsertLocks[i].lastImportantAt.
6575 */
6576 XLogRecPtr
6578 {
6583 {
6584 XLogRecPtr last_important;
6586 /*
6587 * Need to take a lock to prevent torn reads of the LSN, which are
6588 * possible on some of the supported platforms. WAL insert locks only
6589 * support exclusive mode, so we have to use that.
6590 */
6597 }
6600 }
6602 /*
6603 * Get the time and LSN of the last xlog segment switch
6604 */
6605 pg_time_t
6607 {
6608 pg_time_t result;
6610 /* Need WALWriteLock, but shared lock is sufficient */
6617 }
6619 /*
6620 * This must be called ONCE during postmaster or standalone-backend shutdown
6621 */
6622 void
6624 {
6625 /*
6626 * We should have an aux process resource owner to use, and we should not
6627 * be in a transaction that's installed some other resowner.
6628 */
6634 /* Don't be chatty in standalone mode */
6638 /*
6639 * Signal walsenders to move to stopping state.
6640 */
6643 /*
6644 * Wait for WAL senders to be in stopping state. This prevents commands
6645 * from writing new WAL.
6646 */
6651 else
6652 {
6653 /*
6654 * If archiving is enabled, rotate the last XLOG file so that all the
6655 * remaining records are archived (postmaster wakes up the archiver
6656 * process one more time at the end of shutdown). The checkpoint
6657 * record will go to the next XLOG file and won't be archived (yet).
6658 */
6663 }
6664 }
6666 /*
6667 * Log start of a checkpoint.
6668 */
6669 static void
6671 {
6674 /* translator: the placeholders show checkpoint options */
6684 else
6686 /* translator: the placeholders show checkpoint options */
6696 }
6698 /*
6699 * Log end of a checkpoint.
6700 */
6701 static void
6703 {
6705 sync_msecs,
6706 total_msecs,
6707 longest_msecs,
6708 average_msecs;
6709 uint64 average_sync_time;
6719 /* Accumulate checkpoint timing summary data, in milliseconds. */
6723 /*
6724 * All of the published timing statistics are accounted for. Only
6725 * continue if a log message is to be written.
6726 */
6733 /*
6734 * Timing values returned from CheckpointStats are in microseconds.
6735 * Convert to milliseconds for consistent printing.
6736 */
6745 /*
6746 * ControlFileLock is not required to see ControlFile->checkPoint and
6747 * ->checkPointCopy here as we are the only updator of those variables at
6748 * this moment.
6749 */
6753 "wrote %d SLRU buffers; %d WAL file(s) added, "
6754 "%d removed, %d recycled; write=%ld.%03d s, "
6755 "sync=%ld.%03d s, total=%ld.%03d s; sync files=%d, "
6756 "longest=%ld.%03d s, average=%ld.%03d s; distance=%d kB, "
6774 else
6777 "wrote %d SLRU buffers; %d WAL file(s) added, "
6778 "%d removed, %d recycled; write=%ld.%03d s, "
6779 "sync=%ld.%03d s, total=%ld.%03d s; sync files=%d, "
6780 "longest=%ld.%03d s, average=%ld.%03d s; distance=%d kB, "
6798 }
6800 /*
6801 * Update the estimate of distance between checkpoints.
6802 *
6803 * The estimate is used to calculate the number of WAL segments to keep
6804 * preallocated, see XLOGfileslop().
6805 */
6806 static void
6808 {
6809 /*
6810 * To estimate the number of segments consumed between checkpoints, keep a
6811 * moving average of the amount of WAL generated in previous checkpoint
6812 * cycles. However, if the load is bursty, with quiet periods and busy
6813 * periods, we want to cater for the peak load. So instead of a plain
6814 * moving average, let the average decline slowly if the previous cycle
6815 * used less WAL than estimated, but bump it up immediately if it used
6816 * more.
6817 *
6818 * When checkpoints are triggered by max_wal_size, this should converge to
6819 * CheckpointSegments * wal_segment_size,
6820 *
6821 * Note: This doesn't pay any attention to what caused the checkpoint.
6822 * Checkpoints triggered manually with CHECKPOINT command, or by e.g.
6823 * starting a base backup, are counted the same as those created
6824 * automatically. The slow-decline will largely mask them out, if they are
6825 * not frequent. If they are frequent, it seems reasonable to count them
6826 * in as any others; if you issue a manual checkpoint every 5 minutes and
6827 * never let a timed checkpoint happen, it makes sense to base the
6828 * preallocation on that 5 minute interval rather than whatever
6829 * checkpoint_timeout is set to.
6830 */
6834 else
6835 CheckPointDistanceEstimate =
6837 }
6839 /*
6840 * Update the ps display for a process running a checkpoint. Note that
6841 * this routine should not do any allocations so as it can be called
6842 * from a critical section.
6843 */
6844 static void
6846 {
6847 /*
6848 * The status is reported only for end-of-recovery and shutdown
6849 * checkpoints or shutdown restartpoints. Updating the ps display is
6850 * useful in those situations as it may not be possible to rely on
6851 * pg_stat_activity to see the status of the checkpointer or the startup
6852 * process.
6853 */
6859 else
6860 {
6868 }
6869 }
6872 /*
6873 * Perform a checkpoint --- either during shutdown, or on-the-fly
6874 *
6875 * flags is a bitwise OR of the following:
6876 * CHECKPOINT_IS_SHUTDOWN: checkpoint is for database shutdown.
6877 * CHECKPOINT_END_OF_RECOVERY: checkpoint is for end of WAL recovery.
6878 * CHECKPOINT_IMMEDIATE: finish the checkpoint ASAP,
6879 * ignoring checkpoint_completion_target parameter.
6880 * CHECKPOINT_FORCE: force a checkpoint even if no XLOG activity has occurred
6881 * since the last one (implied by CHECKPOINT_IS_SHUTDOWN or
6882 * CHECKPOINT_END_OF_RECOVERY).
6883 * CHECKPOINT_FLUSH_ALL: also flush buffers of unlogged tables.
6884 *
6885 * Note: flags contains other bits, of interest here only for logging purposes.
6886 * In particular note that this routine is synchronous and does not pay
6887 * attention to CHECKPOINT_WAIT.
6888 *
6889 * If !shutdown then we are writing an online checkpoint. An XLOG_CHECKPOINT_REDO
6890 * record is inserted into WAL at the logical location of the checkpoint, before
6891 * flushing anything to disk, and when the checkpoint is eventually completed,
6892 * and it is from this point that WAL replay will begin in the case of a recovery
6893 * from this checkpoint. Once everything is written to disk, an
6894 * XLOG_CHECKPOINT_ONLINE record is written to complete the checkpoint, and
6895 * points back to the earlier XLOG_CHECKPOINT_REDO record. This mechanism allows
6896 * other write-ahead log records to be written while the checkpoint is in
6897 * progress, but we must be very careful about order of operations. This function
6898 * may take many minutes to execute on a busy system.
6899 *
6900 * On the other hand, when shutdown is true, concurrent insertion into the
6901 * write-ahead log is impossible, so there is no need for two separate records.
6902 * In this case, we only insert an XLOG_CHECKPOINT_SHUTDOWN record, and it's
6903 * both the record marking the completion of the checkpoint and the location
6904 * from which WAL replay would begin if needed.
6905 *
6906 * Returns true if a new checkpoint was performed, or false if it was skipped
6907 * because the system was idle.
6908 */
6909 bool
6911 {
6913 CheckPoint checkPoint;
6914 XLogRecPtr recptr;
6915 XLogSegNo _logSegNo;
6917 uint32 freespace;
6918 XLogRecPtr PriorRedoPtr;
6919 XLogRecPtr last_important_lsn;
6924 /*
6925 * An end-of-recovery checkpoint is really a shutdown checkpoint, just
6926 * issued at a different time.
6927 */
6930 else
6933 /* sanity check */
6937 /*
6938 * Prepare to accumulate statistics.
6939 *
6940 * Note: because it is possible for log_checkpoints to change while a
6941 * checkpoint proceeds, we always accumulate stats, even if
6942 * log_checkpoints is currently off.
6943 */
6947 /*
6948 * Let smgr prepare for checkpoint; this has to happen outside the
6949 * critical section and before we determine the REDO pointer. Note that
6950 * smgr must not do anything that'd have to be undone if we decide no
6951 * checkpoint is needed.
6952 */
6955 /*
6956 * Use a critical section to force system panic if we have trouble.
6957 */
6961 {
6966 }
6968 /* Begin filling in the checkpoint WAL record */
6972 /*
6973 * For Hot Standby, derive the oldestActiveXid before we fix the redo
6974 * pointer. This allows us to begin accumulating changes to assemble our
6975 * starting snapshot of locks and transactions.
6976 */
6979 else
6982 /*
6983 * Get location of last important record before acquiring insert locks (as
6984 * GetLastImportantRecPtr() also locks WAL locks).
6985 */
6988 /*
6989 * If this isn't a shutdown or forced checkpoint, and if there has been no
6990 * WAL activity requiring a checkpoint, skip it. The idea here is to
6991 * avoid inserting duplicate checkpoints when the system is idle.
6992 */
6995 {
6997 {
7002 }
7003 }
7005 /*
7006 * An end-of-recovery checkpoint is created before anyone is allowed to
7007 * write WAL. To allow us to write the checkpoint record, temporarily
7008 * enable XLogInsertAllowed.
7009 */
7016 else
7019 /*
7020 * We must block concurrent insertions while examining insert state.
7021 */
7028 {
7031 /*
7032 * Compute new REDO record ptr = location of next XLOG record.
7033 *
7034 * Since this is a shutdown checkpoint, there can't be any concurrent
7035 * WAL insertion.
7036 */
7039 {
7042 else
7044 }
7047 /*
7048 * Here we update the shared RedoRecPtr for future XLogInsert calls;
7049 * this must be done while holding all the insertion locks.
7050 *
7051 * Note: if we fail to complete the checkpoint, RedoRecPtr will be
7052 * left pointing past where it really needs to point. This is okay;
7053 * the only consequence is that XLogInsert might back up whole buffers
7054 * that it didn't really need to. We can't postpone advancing
7055 * RedoRecPtr because XLogInserts that happen while we are dumping
7056 * buffers must assume that their buffer changes are not included in
7057 * the checkpoint.
7058 */
7060 }
7062 /*
7063 * Now we can release the WAL insertion locks, allowing other xacts to
7064 * proceed while we are flushing disk buffers.
7065 */
7068 /*
7069 * If this is an online checkpoint, we have not yet determined the redo
7070 * point. We do so now by inserting the special XLOG_CHECKPOINT_REDO
7071 * record; the LSN at which it starts becomes the new redo pointer. We
7072 * don't do this for a shutdown checkpoint, because in that case no WAL
7073 * can be written between the redo point and the insertion of the
7074 * checkpoint record itself, so the checkpoint record itself serves to
7075 * mark the redo point.
7076 */
7078 {
7079 /* Include WAL level in record for WAL summarizer's benefit. */
7084 /*
7085 * XLogInsertRecord will have updated XLogCtl->Insert.RedoRecPtr in
7086 * shared memory and RedoRecPtr in backend-local memory, but we need
7087 * to copy that into the record that will be inserted when the
7088 * checkpoint is complete.
7089 */
7091 }
7093 /* Update the info_lck-protected copy of RedoRecPtr as well */
7098 /*
7099 * If enabled, log checkpoint start. We postpone this until now so as not
7100 * to log anything if we decided to skip the checkpoint.
7101 */
7105 /* Update the process title */
7110 /*
7111 * Get the other info we need for the checkpoint record.
7112 *
7113 * We don't need to save oldestClogXid in the checkpoint, it only matters
7114 * for the short period in which clog is being truncated, and if we crash
7115 * during that we'll redo the clog truncation and fix up oldestClogXid
7116 * there.
7117 */
7141 /*
7142 * Having constructed the checkpoint record, ensure all shmem disk buffers
7143 * and commit-log buffers are flushed to disk.
7144 *
7145 * This I/O could fail for various reasons. If so, we will fail to
7146 * complete the checkpoint, but there is no reason to force a system
7147 * panic. Accordingly, exit critical section while doing it.
7148 */
7151 /*
7152 * In some cases there are groups of actions that must all occur on one
7153 * side or the other of a checkpoint record. Before flushing the
7154 * checkpoint record we must explicitly wait for any backend currently
7155 * performing those groups of actions.
7156 *
7157 * One example is end of transaction, so we must wait for any transactions
7158 * that are currently in commit critical sections. If an xact inserted
7159 * its commit record into XLOG just before the REDO point, then a crash
7160 * restart from the REDO point would not replay that record, which means
7161 * that our flushing had better include the xact's update of pg_xact. So
7162 * we wait till he's out of his commit critical section before proceeding.
7163 * See notes in RecordTransactionCommit().
7164 *
7165 * Because we've already released the insertion locks, this test is a bit
7166 * fuzzy: it is possible that we will wait for xacts we didn't really need
7167 * to wait for. But the delay should be short and it seems better to make
7168 * checkpoint take a bit longer than to hold off insertions longer than
7169 * necessary. (In fact, the whole reason we have this issue is that xact.c
7170 * does commit record XLOG insertion and clog update as two separate steps
7171 * protected by different locks, but again that seems best on grounds of
7172 * minimizing lock contention.)
7173 *
7174 * A transaction that has not yet set delayChkptFlags when we look cannot
7175 * be at risk, since it has not inserted its commit record yet; and one
7176 * that's already cleared it is not at risk either, since it's done fixing
7177 * clog and we will correctly flush the update below. So we cannot miss
7178 * any xacts we need to wait for.
7179 */
7182 {
7183 do
7184 {
7185 /*
7186 * Keep absorbing fsync requests while we wait. There could even
7187 * be a deadlock if we don't, if the process that prevents the
7188 * checkpoint is trying to add a request to the queue.
7189 */
7196 DELAY_CHKPT_START));
7197 }
7204 {
7205 do
7206 {
7213 DELAY_CHKPT_COMPLETE));
7214 }
7217 /*
7218 * Take a snapshot of running transactions and write this to WAL. This
7219 * allows us to reconstruct the state of running transactions during
7220 * archive recovery, if required. Skip, if this info disabled.
7221 *
7222 * If we are shutting down, or Startup process is completing crash
7223 * recovery we don't need to write running xact data.
7224 */
7230 /*
7231 * Now insert the checkpoint record into XLOG.
7232 */
7237 XLOG_CHECKPOINT_ONLINE);
7241 /*
7242 * We mustn't write any new WAL after a shutdown checkpoint, or it will be
7243 * overwritten at next startup. No-one should even try, this just allows
7244 * sanity-checking. In the case of an end-of-recovery checkpoint, we want
7245 * to just temporarily disable writing until the system has exited
7246 * recovery.
7247 */
7249 {
7252 else
7254 }
7256 /*
7257 * We now have ProcLastRecPtr = start of actual checkpoint record, recptr
7258 * = end of actual checkpoint record.
7259 */
7264 /*
7265 * Remember the prior checkpoint's redo ptr for
7266 * UpdateCheckPointDistanceEstimate()
7267 */
7270 /*
7271 * Update the control file.
7272 */
7278 /* crash recovery should always recover to the end of WAL */
7282 /*
7283 * Persist unloggedLSN value. It's reset on crash recovery, so this goes
7284 * unused on non-shutdown checkpoints, but seems useful to store it always
7285 * for debugging purposes.
7286 */
7292 /* Update shared-memory copy of checkpoint XID/epoch */
7297 /*
7298 * We are now done with critical updates; no need for system panic if we
7299 * have trouble while fooling with old log segments.
7300 */
7303 /*
7304 * WAL summaries end when the next XLOG_CHECKPOINT_REDO or
7305 * XLOG_CHECKPOINT_SHUTDOWN record is reached. This is the first point
7306 * where (a) we're not inside of a critical section and (b) we can be
7307 * certain that the relevant record has been flushed to disk, which must
7308 * happen before it can be summarized.
7309 *
7310 * If this is a shutdown checkpoint, then this happens reasonably
7311 * promptly: we've only just inserted and flushed the
7312 * XLOG_CHECKPOINT_SHUTDOWN record. If this is not a shutdown checkpoint,
7313 * then this might not be very prompt at all: the XLOG_CHECKPOINT_REDO
7314 * record was written before we began flushing data to disk, and that
7315 * could be many minutes ago at this point. However, we don't XLogFlush()
7316 * after inserting that record, so we're not guaranteed that it's on disk
7317 * until after the above call that flushes the XLOG_CHECKPOINT_ONLINE
7318 * record.
7319 */
7322 /*
7323 * Let smgr do post-checkpoint cleanup (eg, deleting old files).
7324 */
7327 /*
7328 * Update the average distance between checkpoints if the prior checkpoint
7329 * exists.
7330 */
7334 /*
7335 * Delete old log files, those no longer needed for last checkpoint to
7336 * prevent the disk holding the xlog from growing full.
7337 */
7342 InvalidTransactionId))
7343 {
7344 /*
7345 * Some slots have been invalidated; recalculate the old-segment
7346 * horizon, starting again from RedoRecPtr.
7347 */
7350 }
7351 _logSegNo--;
7355 /*
7356 * Make more log segments if needed. (Do this after recycling old log
7357 * segments, since that may supply some of the needed files.)
7358 */
7362 /*
7363 * Truncate pg_subtrans if possible. We can throw away all data before
7364 * the oldest XMIN of any running transaction. No future transaction will
7365 * attempt to reference any pg_subtrans entry older than that (see Asserts
7366 * in subtrans.c). During recovery, though, we mustn't do this because
7367 * StartupSUBTRANS hasn't been called yet.
7368 */
7372 /* Real work is done; log and update stats. */
7375 /* Reset the process title */
7379 NBuffers,
7385 }
7387 /*
7388 * Mark the end of recovery in WAL though without running a full checkpoint.
7389 * We can expect that a restartpoint is likely to be in progress as we
7390 * do this, though we are unwilling to wait for it to complete.
7391 *
7392 * CreateRestartPoint() allows for the case where recovery may end before
7393 * the restartpoint completes so there is no concern of concurrent behaviour.
7394 */
7395 static void
7397 {
7398 xl_end_of_recovery xlrec;
7399 XLogRecPtr recptr;
7401 /* sanity check */
7421 /*
7422 * Update the control file so that crash recovery can follow the timeline
7423 * changes to this point.
7424 */
7432 }
7434 /*
7435 * Write an OVERWRITE_CONTRECORD message.
7436 *
7437 * When on WAL replay we expect a continuation record at the start of a page
7438 * that is not there, recovery ends and WAL writing resumes at that point.
7439 * But it's wrong to resume writing new WAL back at the start of the record
7440 * that was broken, because downstream consumers of that WAL (physical
7441 * replicas) are not prepared to "rewind". So the first action after
7442 * finishing replay of all valid WAL must be to write a record of this type
7443 * at the point where the contrecord was missing; to support xlogreader
7444 * detecting the special case, XLP_FIRST_IS_OVERWRITE_CONTRECORD is also added
7445 * to the page header where the record occurs. xlogreader has an ad-hoc
7446 * mechanism to report metadata about the broken record, which is what we
7447 * use here.
7448 *
7449 * At replay time, XLP_FIRST_IS_OVERWRITE_CONTRECORD instructs xlogreader to
7450 * skip the record it was reading, and pass back the LSN of the skipped
7451 * record, so that its caller can verify (on "replay" of that record) that the
7452 * XLOG_OVERWRITE_CONTRECORD matches what was effectively overwritten.
7453 *
7454 * 'aborted_lsn' is the beginning position of the record that was incomplete.
7455 * It is included in the WAL record. 'pagePtr' and 'newTLI' point to the
7456 * beginning of the XLOG page where the record is to be inserted. They must
7457 * match the current WAL insert position, they're passed here just so that we
7458 * can verify that.
7459 */
7460 static XLogRecPtr
7462 TimeLineID newTLI)
7463 {
7464 xl_overwrite_contrecord xlrec;
7465 XLogRecPtr recptr;
7466 XLogPageHeader pagehdr;
7467 XLogRecPtr startPos;
7469 /* sanity checks */
7476 /* The current WAL insert position should be right after the page header */
7480 else
7489 /*
7490 * Initialize the XLOG page header (by GetXLogBuffer), and set the
7491 * XLP_FIRST_IS_OVERWRITE_CONTRECORD flag.
7492 *
7493 * No other backend is allowed to write WAL yet, so acquiring the WAL
7494 * insertion lock is just pro forma.
7495 */
7501 /*
7502 * Insert the XLOG_OVERWRITE_CONTRECORD record as the first record on the
7503 * page. We know it becomes the first record, because no other backend is
7504 * allowed to write WAL yet.
7505 */
7512 /* check that the record was inserted to the right place */
7522 }
7524 /*
7525 * Flush all data in shared memory to disk, and fsync
7526 *
7527 * This is the common code shared between regular checkpoints and
7528 * recovery restartpoints.
7529 */
7530 static void
7532 {
7539 /* Write out all dirty data in SLRUs and the main buffer pool */
7549 /* Perform all queued up fsyncs */
7556 /* We deliberately delay 2PC checkpointing as long as possible */
7558 }
7560 /*
7561 * Save a checkpoint for recovery restart if appropriate
7562 *
7563 * This function is called each time a checkpoint record is read from XLOG.
7564 * It must determine whether the checkpoint represents a safe restartpoint or
7565 * not. If so, the checkpoint record is stashed in shared memory so that
7566 * CreateRestartPoint can consult it. (Note that the latter function is
7567 * executed by the checkpointer, while this one will be executed by the
7568 * startup process.)
7569 */
7570 static void
7572 {
7573 /*
7574 * Also refrain from creating a restartpoint if we have seen any
7575 * references to non-existent pages. Restarting recovery from the
7576 * restartpoint would not see the references, so we would lose the
7577 * cross-check that the pages belonged to a relation that was dropped
7578 * later.
7579 */
7581 {
7583 "could not record restart point at %X/%X because there "
7587 }
7589 /*
7590 * Copy the checkpoint record to shared memory, so that checkpointer can
7591 * work out the next time it wants to perform a restartpoint.
7592 */
7598 }
7600 /*
7601 * Establish a restartpoint if possible.
7602 *
7603 * This is similar to CreateCheckPoint, but is used during WAL recovery
7604 * to establish a point from which recovery can roll forward without
7605 * replaying the entire recovery log.
7606 *
7607 * Returns true if a new restartpoint was established. We can only establish
7608 * a restartpoint if we have replayed a safe checkpoint record since last
7609 * restartpoint.
7610 */
7611 bool
7613 {
7614 XLogRecPtr lastCheckPointRecPtr;
7615 XLogRecPtr lastCheckPointEndPtr;
7616 CheckPoint lastCheckPoint;
7617 XLogRecPtr PriorRedoPtr;
7618 XLogRecPtr receivePtr;
7619 XLogRecPtr replayPtr;
7620 TimeLineID replayTLI;
7621 XLogRecPtr endptr;
7622 XLogSegNo _logSegNo;
7623 TimestampTz xtime;
7625 /* Concurrent checkpoint/restartpoint cannot happen */
7628 /* Get a local copy of the last safe checkpoint record. */
7635 /*
7636 * Check that we're still in recovery mode. It's ok if we exit recovery
7637 * mode after this check, the restart point is valid anyway.
7638 */
7640 {
7644 }
7646 /*
7647 * If the last checkpoint record we've replayed is already our last
7648 * restartpoint, we can't perform a new restart point. We still update
7649 * minRecoveryPoint in that case, so that if this is a shutdown restart
7650 * point, we won't start up earlier than before. That's not strictly
7651 * necessary, but when hot standby is enabled, it would be rather weird if
7652 * the database opened up for read-only connections at a point-in-time
7653 * before the last shutdown. Such time travel is still possible in case of
7654 * immediate shutdown, though.
7655 *
7656 * We don't explicitly advance minRecoveryPoint when we do create a
7657 * restartpoint. It's assumed that flushing the buffers will do that as a
7658 * side-effect.
7659 */
7662 {
7669 {
7674 }
7676 }
7678 /*
7679 * Update the shared RedoRecPtr so that the startup process can calculate
7680 * the number of segments replayed since last restartpoint, and request a
7681 * restartpoint if it exceeds CheckPointSegments.
7682 *
7683 * Like in CreateCheckPoint(), hold off insertions to update it, although
7684 * during recovery this is just pro forma, because no WAL insertions are
7685 * happening.
7686 */
7691 /* Also update the info_lck-protected copy */
7696 /*
7697 * Prepare to accumulate statistics.
7698 *
7699 * Note: because it is possible for log_checkpoints to change while a
7700 * checkpoint proceeds, we always accumulate stats, even if
7701 * log_checkpoints is currently off.
7702 */
7709 /* Update the process title */
7714 /*
7715 * This location needs to be after CheckPointGuts() to ensure that some
7716 * work has already happened during this checkpoint.
7717 */
7720 /*
7721 * Remember the prior checkpoint's redo ptr for
7722 * UpdateCheckPointDistanceEstimate()
7723 */
7726 /*
7727 * Update pg_control, using current time. Check that it still shows an
7728 * older checkpoint, else do nothing; this is a quick hack to make sure
7729 * nothing really bad happens if somehow we get here after the
7730 * end-of-recovery checkpoint.
7731 */
7734 {
7735 /*
7736 * Update the checkpoint information. We do this even if the cluster
7737 * does not show DB_IN_ARCHIVE_RECOVERY to match with the set of WAL
7738 * segments recycled below.
7739 */
7743 /*
7744 * Ensure minRecoveryPoint is past the checkpoint record and update it
7745 * if the control file still shows DB_IN_ARCHIVE_RECOVERY. Normally,
7746 * this will have happened already while writing out dirty buffers,
7747 * but not necessarily - e.g. because no buffers were dirtied. We do
7748 * this because a backup performed in recovery uses minRecoveryPoint
7749 * to determine which WAL files must be included in the backup, and
7750 * the file (or files) containing the checkpoint record must be
7751 * included, at a minimum. Note that for an ordinary restart of
7752 * recovery there's no value in having the minimum recovery point any
7753 * earlier than this anyway, because redo will begin just after the
7754 * checkpoint record.
7755 */
7757 {
7759 {
7763 /* update local copy */
7766 }
7769 }
7771 }
7774 /*
7775 * Update the average distance between checkpoints/restartpoints if the
7776 * prior checkpoint exists.
7777 */
7781 /*
7782 * Delete old log files, those no longer needed for last restartpoint to
7783 * prevent the disk holding the xlog from growing full.
7784 */
7787 /*
7788 * Retreat _logSegNo using the current end of xlog replayed or received,
7789 * whichever is later.
7790 */
7797 InvalidTransactionId))
7798 {
7799 /*
7800 * Some slots have been invalidated; recalculate the old-segment
7801 * horizon, starting again from RedoRecPtr.
7802 */
7805 }
7806 _logSegNo--;
7808 /*
7809 * Try to recycle segments on a useful timeline. If we've been promoted
7810 * since the beginning of this restartpoint, use the new timeline chosen
7811 * at end of recovery. If we're still in recovery, use the timeline we're
7812 * currently replaying.
7813 *
7814 * There is no guarantee that the WAL segments will be useful on the
7815 * current timeline; if recovery proceeds to a new timeline right after
7816 * this, the pre-allocated WAL segments on this timeline will not be used,
7817 * and will go wasted until recycled on the next restartpoint. We'll live
7818 * with that.
7819 */
7825 /*
7826 * Make more log segments if needed. (Do this after recycling old log
7827 * segments, since that may supply some of the needed files.)
7828 */
7831 /*
7832 * Truncate pg_subtrans if possible. We can throw away all data before
7833 * the oldest XMIN of any running transaction. No future transaction will
7834 * attempt to reference any pg_subtrans entry older than that (see Asserts
7835 * in subtrans.c). When hot standby is disabled, though, we mustn't do
7836 * this because StartupSUBTRANS hasn't been called yet.
7837 */
7841 /* Real work is done; log and update stats. */
7844 /* Reset the process title */
7854 /*
7855 * Finally, execute archive_cleanup_command, if any.
7856 */
7861 WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND);
7864 }
7866 /*
7867 * Report availability of WAL for the given target LSN
7868 * (typically a slot's restart_lsn)
7869 *
7870 * Returns one of the following enum values:
7871 *
7872 * * WALAVAIL_RESERVED means targetLSN is available and it is in the range of
7873 * max_wal_size.
7874 *
7875 * * WALAVAIL_EXTENDED means it is still available by preserving extra
7876 * segments beyond max_wal_size. If max_slot_wal_keep_size is smaller
7877 * than max_wal_size, this state is not returned.
7878 *
7879 * * WALAVAIL_UNRESERVED means it is being lost and the next checkpoint will
7880 * remove reserved segments. The walsender using this slot may return to the
7881 * above.
7882 *
7883 * * WALAVAIL_REMOVED means it has been removed. A replication stream on
7884 * a slot with this LSN cannot continue. (Any associated walsender
7885 * processes should have been terminated already.)
7886 *
7887 * * WALAVAIL_INVALID_LSN means the slot hasn't been set to reserve WAL.
7888 */
7889 WALAvailability
7891 {
7898 uint64 keepSegs;
7900 /*
7901 * slot does not reserve WAL. Either deactivated, or has never been active
7902 */
7906 /*
7907 * Calculate the oldest segment currently reserved by all slots,
7908 * considering wal_keep_size and max_slot_wal_keep_size. Initialize
7909 * oldestSlotSeg to the current segment.
7910 */
7915 /*
7916 * Find the oldest extant segment file. We get 1 until checkpoint removes
7917 * the first WAL segment file since startup, which causes the status being
7918 * wrong under certain abnormal conditions but that doesn't actually harm.
7919 */
7922 /* calculate oldest segment by max_wal_size */
7928 else
7931 /* the segment we care about */
7934 /*
7935 * No point in returning reserved or extended status values if the
7936 * targetSeg is known to be lost.
7937 */
7939 {
7940 /* show "reserved" when targetSeg is within max_wal_size */
7944 /* being retained by slots exceeding max_wal_size */
7946 }
7948 /* WAL segments are no longer retained but haven't been removed yet */
7952 /* Definitely lost */
7954 }
7957 /*
7958 * Retreat *logSegNo to the last segment that we need to retain because of
7959 * either wal_keep_size or replication slots.
7960 *
7961 * This is calculated by subtracting wal_keep_size from the given xlog
7962 * location, recptr and by making sure that that result is below the
7963 * requirement of replication slots. For the latter criterion we do consider
7964 * the effects of max_slot_wal_keep_size: reserve at most that much space back
7965 * from recptr.
7966 *
7967 * Note about replication slots: if this function calculates a value
7968 * that's further ahead than what slots need reserved, then affected
7969 * slots need to be invalidated and this function invoked again.
7970 * XXX it might be a good idea to rewrite this function so that
7971 * invalidation is optionally done here, instead.
7972 */
7973 static void
7975 {
7976 XLogSegNo currSegNo;
7977 XLogSegNo segno;
7978 XLogRecPtr keep;
7983 /*
7984 * Calculate how many segments are kept by slots first, adjusting for
7985 * max_slot_wal_keep_size.
7986 */
7989 {
7992 /* Cap by max_slot_wal_keep_size ... */
7994 {
7995 uint64 slot_keep_segs;
7997 slot_keep_segs =
8002 }
8003 }
8005 /*
8006 * If WAL summarization is in use, don't remove WAL that has yet to be
8007 * summarized.
8008 */
8011 {
8012 XLogSegNo unsummarized_segno;
8017 }
8019 /* but, keep at least wal_keep_size if that's set */
8021 {
8022 uint64 keep_segs;
8026 {
8027 /* avoid underflow, don't go below 1 */
8030 else
8032 }
8033 }
8035 /* don't delete WAL segments newer than the calculated segment */
8038 }
8040 /*
8041 * Write a NEXTOID log record
8042 */
8043 void
8045 {
8050 /*
8051 * We need not flush the NEXTOID record immediately, because any of the
8052 * just-allocated OIDs could only reach disk as part of a tuple insert or
8053 * update that would have its own XLOG record that must follow the NEXTOID
8054 * record. Therefore, the standard buffer LSN interlock applied to those
8055 * records will ensure no such OID reaches disk before the NEXTOID record
8056 * does.
8057 *
8058 * Note, however, that the above statement only covers state "within" the
8059 * database. When we use a generated OID as a file or directory name, we
8060 * are in a sense violating the basic WAL rule, because that filesystem
8061 * change may reach disk before the NEXTOID WAL record does. The impact
8062 * of this is that if a database crash occurs immediately afterward, we
8063 * might after restart re-generate the same OID and find that it conflicts
8064 * with the leftover file or directory. But since for safety's sake we
8065 * always loop until finding a nonconflicting filename, this poses no real
8066 * problem in practice. See pgsql-hackers discussion 27-Sep-2006.
8067 */
8068 }
8070 /*
8071 * Write an XLOG SWITCH record.
8072 *
8073 * Here we just blindly issue an XLogInsert request for the record.
8074 * All the magic happens inside XLogInsert.
8075 *
8076 * The return value is either the end+1 address of the switch record,
8077 * or the end+1 address of the prior segment if we did not need to
8078 * write a switch record because we are already at segment start.
8079 */
8080 XLogRecPtr
8082 {
8083 XLogRecPtr RecPtr;
8085 /* XLOG SWITCH has no data */
8093 }
8095 /*
8096 * Write a RESTORE POINT record
8097 */
8098 XLogRecPtr
8100 {
8101 XLogRecPtr RecPtr;
8102 xl_restore_point xlrec;
8117 }
8119 /*
8120 * Check if any of the GUC parameters that are critical for hot standby
8121 * have changed, and update the value in pg_control file if necessary.
8122 */
8123 static void
8125 {
8134 {
8135 /*
8136 * The change in number of backend slots doesn't need to be WAL-logged
8137 * if archiving is not enabled, as you can't start archive recovery
8138 * with wal_level=minimal anyway. We don't really care about the
8139 * values in pg_control either if wal_level=minimal, but seems better
8140 * to keep them up-to-date to avoid confusion.
8141 */
8143 {
8144 xl_parameter_change xlrec;
8145 XLogRecPtr recptr;
8161 }
8176 }
8177 }
8179 /*
8180 * Update full_page_writes in shared memory, and write an
8181 * XLOG_FPW_CHANGE record if necessary.
8182 *
8183 * Note: this function assumes there is no other process running
8184 * concurrently that could update it.
8185 */
8186 void
8188 {
8192 /*
8193 * Do nothing if full_page_writes has not been changed.
8194 *
8195 * It's safe to check the shared full_page_writes without the lock,
8196 * because we assume that there is no concurrently running process which
8197 * can update it.
8198 */
8202 /*
8203 * Perform this outside critical section so that the WAL insert
8204 * initialization done by RecoveryInProgress() doesn't trigger an
8205 * assertion failure.
8206 */
8211 /*
8212 * It's always safe to take full page images, even when not strictly
8213 * required, but not the other round. So if we're setting full_page_writes
8214 * to true, first set it true and then write the WAL record. If we're
8215 * setting it to false, first write the WAL record and then set the global
8216 * flag.
8217 */
8219 {
8223 }
8225 /*
8226 * Write an XLOG_FPW_CHANGE record. This allows us to keep track of
8227 * full_page_writes during archive recovery, if required.
8228 */
8230 {
8235 }
8238 {
8242 }
8244 }
8246 /*
8247 * XLOG resource manager's routines
8248 *
8249 * Definitions of info values are in include/catalog/pg_control.h, though
8250 * not all record types are related to control file updates.
8251 *
8252 * NOTE: Some XLOG record types that are directly related to WAL recovery
8253 * are handled in xlogrecovery_redo().
8254 */
8255 void
8257 {
8261 /*
8262 * In XLOG rmgr, backup blocks are only used by XLOG_FPI and
8263 * XLOG_FPI_FOR_HINT records.
8264 */
8269 {
8270 Oid nextOid;
8272 /*
8273 * We used to try to take the maximum of TransamVariables->nextOid and
8274 * the recorded nextOid, but that fails if the OID counter wraps
8275 * around. Since no OID allocation should be happening during replay
8276 * anyway, better to just believe the record exactly. We still take
8277 * OidGenLock while setting the variable, just in case.
8278 */
8284 }
8286 {
8287 CheckPoint checkPoint;
8288 TimeLineID replayTLI;
8291 /* In a SHUTDOWN checkpoint, believe the counters exactly */
8305 /*
8306 * No need to set oldestClogXid here as well; it'll be set when we
8307 * redo an xl_clog_truncate if it changed since initialization.
8308 */
8311 /*
8312 * If we see a shutdown checkpoint while waiting for an end-of-backup
8313 * record, the backup was canceled and the end-of-backup record will
8314 * never arrive.
8315 */
8322 /*
8323 * If we see a shutdown checkpoint, we know that nothing was running
8324 * on the primary at this point. So fake-up an empty running-xacts
8325 * record and use that here and now. Recover additional standby state
8326 * for prepared transactions.
8327 */
8329 {
8332 TransactionId oldestActiveXID;
8333 TransactionId latestCompletedXid;
8334 RunningTransactionsData running;
8338 /* Update pg_subtrans entries for any prepared transactions */
8341 /*
8342 * Construct a RunningTransactions snapshot representing a shut
8343 * down server, with only prepared transactions still alive. We're
8344 * never overflowed at this point because all subxids are listed
8345 * with their parent prepared transactions.
8346 */
8359 }
8361 /* ControlFile->checkPointCopy always tracks the latest ckpt XID */
8366 /* Update shared-memory copy of checkpoint XID/epoch */
8371 /*
8372 * We should've already switched to the new TLI before replaying this
8373 * record.
8374 */
8382 }
8384 {
8385 CheckPoint checkPoint;
8386 TimeLineID replayTLI;
8389 /* In an ONLINE checkpoint, treat the XID counter as a minimum */
8396 /*
8397 * We ignore the nextOid counter in an ONLINE checkpoint, preferring
8398 * to track OID assignment through XLOG_NEXTOID records. The nextOid
8399 * counter is from the start of the checkpoint and might well be stale
8400 * compared to later XLOG_NEXTOID records. We could try to take the
8401 * maximum of the nextOid counter and our latest value, but since
8402 * there's no particular guarantee about the speed with which the OID
8403 * counter wraps around, that's a risky thing to do. In any case,
8404 * users of the nextOid counter are required to avoid assignment of
8405 * duplicates, so that a somewhat out-of-date value should be safe.
8406 */
8408 /* Handle multixact */
8412 /*
8413 * NB: This may perform multixact truncation when replaying WAL
8414 * generated by an older primary.
8415 */
8422 /* ControlFile->checkPointCopy always tracks the latest ckpt XID */
8427 /* Update shared-memory copy of checkpoint XID/epoch */
8432 /* TLI should not change in an on-line checkpoint */
8440 }
8442 {
8443 /* nothing to do here, handled in xlogrecovery_redo() */
8444 }
8446 {
8447 xl_end_of_recovery xlrec;
8448 TimeLineID replayTLI;
8452 /*
8453 * For Hot Standby, we could treat this like a Shutdown Checkpoint,
8454 * but this case is rarer and harder to test, so the benefit doesn't
8455 * outweigh the potential extra cost of maintenance.
8456 */
8458 /*
8459 * We should've already switched to the new TLI before replaying this
8460 * record.
8461 */
8467 }
8469 {
8470 /* nothing to do here */
8471 }
8473 {
8474 /* nothing to do here */
8475 }
8477 {
8478 /* nothing to do here, handled in xlogrecovery.c */
8479 }
8481 {
8482 /*
8483 * XLOG_FPI records contain nothing else but one or more block
8484 * references. Every block reference must include a full-page image
8485 * even if full_page_writes was disabled when the record was generated
8486 * - otherwise there would be no point in this record.
8487 *
8488 * XLOG_FPI_FOR_HINT records are generated when a page needs to be
8489 * WAL-logged because of a hint bit update. They are only generated
8490 * when checksums and/or wal_log_hints are enabled. They may include
8491 * no full-page images if full_page_writes was disabled when they were
8492 * generated. In this case there is nothing to do here.
8493 *
8494 * No recovery conflicts are generated by these generic records - if a
8495 * resource manager needs to generate conflicts, it has to define a
8496 * separate WAL record type and redo routine.
8497 */
8499 {
8500 Buffer buffer;
8503 {
8507 }
8512 }
8513 }
8515 {
8516 /* nothing to do here, handled in xlogrecovery_redo() */
8517 }
8519 {
8520 xl_parameter_change xlrec;
8522 /* Update our copy of the parameters in pg_control */
8525 /*
8526 * Invalidate logical slots if we are in hot standby and the primary
8527 * does not have a WAL level sufficient for logical decoding. No need
8528 * to search for potentially conflicting logically slots if standby is
8529 * running with wal_level lower than logical, because in that case, we
8530 * would have either disallowed creation of logical slots or
8531 * invalidated existing ones.
8532 */
8538 InvalidTransactionId);
8549 /*
8550 * Update minRecoveryPoint to ensure that if recovery is aborted, we
8551 * recover back up to this point before allowing hot standby again.
8552 * This is important if the max_* settings are decreased, to ensure
8553 * you don't run queries against the WAL preceding the change. The
8554 * local copies cannot be updated as long as crash recovery is
8555 * happening and we expect all the WAL to be replayed.
8556 */
8558 {
8561 }
8563 {
8564 TimeLineID replayTLI;
8569 }
8578 /* Check to see if any parameter change gives a problem on recovery */
8580 }
8582 {
8587 /*
8588 * Update the LSN of the last replayed XLOG_FPW_CHANGE record so that
8589 * do_pg_backup_start() and do_pg_backup_stop() can check whether
8590 * full_page_writes has been disabled during online backup.
8591 */
8593 {
8598 }
8600 /* Keep track of full_page_writes */
8602 }
8604 {
8605 /* nothing to do here, just for informational purposes */
8606 }
8607 }
8609 /*
8610 * Return the extra open flags used for opening a file, depending on the
8611 * value of the GUCs wal_sync_method, fsync and debug_io_direct.
8612 */
8613 static int
8615 {
8618 /*
8619 * Use O_DIRECT if requested, except in walreceiver process. The WAL
8620 * written by walreceiver is normally read by the startup process soon
8621 * after it's written. Also, walreceiver performs unaligned writes, which
8622 * don't work with O_DIRECT, so it is required for correctness too.
8623 */
8627 /* If fsync is disabled, never open in sync mode */
8632 {
8633 /*
8634 * enum values for all sync options are defined even if they are
8635 * not supported on the current platform. But if not, they are
8636 * not included in the enum option array, and therefore will never
8637 * be seen here.
8638 */
8643 #ifdef O_SYNC
8646 #endif
8647 #ifdef O_DSYNC
8650 #endif
8652 /* can't happen (unless we are out of sync with option array) */
8655 }
8656 }
8658 /*
8659 * GUC support
8660 */
8661 void
8663 {
8665 {
8666 /*
8667 * To ensure that no blocks escape unsynced, force an fsync on the
8668 * currently open log segment (if any). Also, if the open flag is
8669 * changing, close the log file so it will be reopened (with new flag
8670 * bit) at next use.
8671 */
8673 {
8676 {
8682 wal_segment_size);
8687 }
8692 }
8693 }
8694 }
8697 /*
8698 * Issue appropriate kind of fsync (if any) for an XLOG output file.
8699 *
8700 * 'fd' is a file descriptor for the XLOG file to be fsync'd.
8701 * 'segno' is for error reporting purposes.
8702 */
8703 void
8705 {
8707 instr_time start;
8711 /*
8712 * Quick exit if fsync is disabled or write() has already synced the WAL
8713 * file.
8714 */
8720 /*
8721 * Measure I/O timing to sync the WAL file for pg_stat_io and/or
8722 * pg_stat_wal.
8723 */
8728 {
8733 #ifdef HAVE_FSYNC_WRITETHROUGH
8738 #endif
8745 /* not reachable */
8753 }
8755 /* PANIC if failed to fsync */
8757 {
8766 }
8770 /*
8771 * Increment the I/O timing and the number of times WAL files were synced.
8772 */
8774 {
8775 instr_time end;
8779 }
8785 }
8787 /*
8788 * do_pg_backup_start is the workhorse of the user-visible pg_backup_start()
8789 * function. It creates the necessary starting checkpoint and constructs the
8790 * backup state and tablespace map.
8791 *
8792 * Input parameters are "state" (the backup state), "fast" (if true, we do
8793 * the checkpoint in immediate mode to make it faster), and "tablespaces"
8794 * (if non-NULL, indicates a list of tablespaceinfo structs describing the
8795 * cluster's tablespaces.).
8796 *
8797 * The tablespace map contents are appended to passed-in parameter
8798 * tablespace_map and the caller is responsible for including it in the backup
8799 * archive as 'tablespace_map'. The tablespace_map file is required mainly for
8800 * tar format in windows as native windows utilities are not able to create
8801 * symlinks while extracting files from tar. However for consistency and
8802 * platform-independence, we do it the same way everywhere.
8803 *
8804 * It fills in "state" with the information required for the backup, such
8805 * as the minimum WAL location that must be present to restore from this
8806 * backup (starttli) and the corresponding timeline ID (starttli).
8807 *
8808 * Every successfully started backup must be stopped by calling
8809 * do_pg_backup_stop() or do_pg_abort_backup(). There can be many
8810 * backups active at the same time.
8811 *
8812 * It is the responsibility of the caller of this function to verify the
8813 * permissions of the calling user!
8814 */
8815 void
8818 {
8824 /*
8825 * During recovery, we don't need to check WAL level. Because, if WAL
8826 * level is not sufficient, it's impossible to get here during recovery.
8827 */
8838 MAXPGPATH)));
8842 /*
8843 * Mark backup active in shared memory. We must do full-page WAL writes
8844 * during an on-line backup even if not doing so at other times, because
8845 * it's quite possible for the backup dump to obtain a "torn" (partially
8846 * written) copy of a database page if it reads the page concurrently with
8847 * our write to the same page. This can be fixed as long as the first
8848 * write to the page in the WAL sequence is a full-page write. Hence, we
8849 * increment runningBackups then force a CHECKPOINT, to ensure there are
8850 * no dirty pages in shared memory that might get dumped while the backup
8851 * is in progress without having a corresponding WAL record. (Once the
8852 * backup is complete, we need not force full-page writes anymore, since
8853 * we expect that any pages not modified during the backup interval must
8854 * have been correctly captured by the backup.)
8855 *
8856 * Note that forcing full-page writes has no effect during an online
8857 * backup from the standby.
8858 *
8859 * We must hold all the insertion locks to change the value of
8860 * runningBackups, to ensure adequate interlocking against
8861 * XLogInsertRecord().
8862 */
8867 /*
8868 * Ensure we decrement runningBackups if we fail below. NB -- for this to
8869 * work correctly, it is critical that sessionBackupState is only updated
8870 * after this block is over.
8871 */
8873 {
8880 /*
8881 * Force an XLOG file switch before the checkpoint, to ensure that the
8882 * WAL segment the checkpoint is written to doesn't contain pages with
8883 * old timeline IDs. That would otherwise happen if you called
8884 * pg_backup_start() right after restoring from a PITR archive: the
8885 * first WAL segment containing the startup checkpoint has pages in
8886 * the beginning with the old timeline ID. That can cause trouble at
8887 * recovery: we won't have a history file covering the old timeline if
8888 * pg_wal directory was not included in the base backup and the WAL
8889 * archive was cleared too before starting the backup.
8890 *
8891 * This also ensures that we have emitted a WAL page header that has
8892 * XLP_BKP_REMOVABLE off before we emit the checkpoint record.
8893 * Therefore, if a WAL archiver (such as pglesslog) is trying to
8894 * compress out removable backup blocks, it won't remove any that
8895 * occur after this point.
8896 *
8897 * During recovery, we skip forcing XLOG file switch, which means that
8898 * the backup taken during recovery is not available for the special
8899 * recovery case described above.
8900 */
8904 do
8905 {
8908 /*
8909 * Force a CHECKPOINT. Aside from being necessary to prevent torn
8910 * page problems, this guarantees that two successive backup runs
8911 * will have different checkpoint positions and hence different
8912 * history file names, even if nothing happened in between.
8913 *
8914 * During recovery, establish a restartpoint if possible. We use
8915 * the last restartpoint as the backup starting checkpoint. This
8916 * means that two successive backup runs can have same checkpoint
8917 * positions.
8918 *
8919 * Since the fact that we are executing do_pg_backup_start()
8920 * during recovery means that checkpointer is running, we can use
8921 * RequestCheckpoint() to establish a restartpoint.
8922 *
8923 * We use CHECKPOINT_IMMEDIATE only if requested by user (via
8924 * passing fast = true). Otherwise this can take awhile.
8925 */
8929 /*
8930 * Now we need to fetch the checkpoint record location, and also
8931 * its REDO pointer. The oldest point in WAL that would be needed
8932 * to restore starting from the checkpoint is precisely the REDO
8933 * pointer.
8934 */
8943 {
8944 XLogRecPtr recptr;
8946 /*
8947 * Check to see if all WAL replayed during online backup
8948 * (i.e., since last restartpoint used as backup starting
8949 * checkpoint) contain full-page writes.
8950 */
8961 "is corrupt and should not be used. "
8965 /*
8966 * During recovery, since we don't use the end-of-backup WAL
8967 * record and don't write the backup history file, the
8968 * starting WAL location doesn't need to be unique. This means
8969 * that two base backups started at the same time might use
8970 * the same checkpoint as starting locations.
8971 */
8973 }
8975 /*
8976 * If two base backups are started at the same time (in WAL sender
8977 * processes), we need to make sure that they use different
8978 * checkpoints as starting locations, because we use the starting
8979 * WAL location as a unique identifier for the base backup in the
8980 * end-of-backup WAL record and when we write the backup history
8981 * file. Perhaps it would be better generate a separate unique ID
8982 * for each backup instead of forcing another checkpoint, but
8983 * taking a checkpoint right after another is not that expensive
8984 * either because only few buffers have been dirtied yet.
8985 */
8988 {
8991 }
8995 /*
8996 * Construct tablespace_map file.
8997 */
9000 /* Collect information about all tablespaces */
9003 {
9008 PGFileType de_type;
9010 Oid tsoid;
9012 /*
9013 * Try to parse the directory name as an unsigned integer.
9014 *
9015 * Tablespace directories should be positive integers that can be
9016 * represented in 32 bits, with no leading zeroes or trailing
9017 * garbage. If we come across a name that doesn't meet those
9018 * criteria, skip it.
9019 */
9032 {
9033 StringInfoData escapedpath;
9038 {
9041 fullpath)));
9043 }
9045 {
9048 fullpath)));
9050 }
9053 /*
9054 * Relpath holds the relative path of the tablespace directory
9055 * when it's located within PGDATA, or NULL if it's located
9056 * elsewhere.
9057 */
9063 /*
9064 * Add a backslash-escaped version of the link path to the
9065 * tablespace map file.
9066 */
9069 {
9073 }
9077 }
9079 {
9080 /*
9081 * It's possible to use allow_in_place_tablespaces to create
9082 * directories directly under pg_tblspc, for testing purposes
9083 * only.
9084 *
9085 * In this case, we store a relative path rather than an
9086 * absolute path into the tablespaceinfo.
9087 */
9091 }
9092 else
9093 {
9094 /* Skip any other file type that appears here. */
9096 }
9106 }
9110 }
9115 /*
9116 * Mark that the start phase has correctly finished for the backup.
9117 */
9119 }
9121 /*
9122 * Utility routine to fetch the session-level status of a backup running.
9123 */
9124 SessionBackupState
9126 {
9128 }
9130 /*
9131 * do_pg_backup_stop
9132 *
9133 * Utility function called at the end of an online backup. It creates history
9134 * file (if required), resets sessionBackupState and so on. It can optionally
9135 * wait for WAL segments to be archived.
9136 *
9137 * "state" is filled with the information necessary to restore from this
9138 * backup with its stop LSN (stoppoint), its timeline ID (stoptli), etc.
9139 *
9140 * It is the responsibility of the caller of this function to verify the
9141 * permissions of the calling user!
9142 */
9143 void
9145 {
9150 XLogSegNo _logSegNo;
9160 /*
9161 * During recovery, we don't need to check WAL level. Because, if WAL
9162 * level is not sufficient, it's impossible to get here during recovery.
9163 */
9170 /*
9171 * OK to update backup counter and session-level lock.
9172 *
9173 * Note that CHECK_FOR_INTERRUPTS() must not occur while updating them,
9174 * otherwise they can be updated inconsistently, which might cause
9175 * do_pg_abort_backup() to fail.
9176 */
9179 /*
9180 * It is expected that each do_pg_backup_start() call is matched by
9181 * exactly one do_pg_backup_stop() call.
9182 */
9186 /*
9187 * Clean up session-level lock.
9188 *
9189 * You might think that WALInsertLockRelease() can be called before
9190 * cleaning up session-level lock because session-level lock doesn't need
9191 * to be protected with WAL insertion lock. But since
9192 * CHECK_FOR_INTERRUPTS() can occur in it, session-level lock must be
9193 * cleaned up before it.
9194 */
9199 /*
9200 * If we are taking an online backup from the standby, we confirm that the
9201 * standby has not been promoted during the backup.
9202 */
9208 "and should not be used. "
9211 /*
9212 * During recovery, we don't write an end-of-backup record. We assume that
9213 * pg_control was backed up last and its minimum recovery point can be
9214 * available as the backup end location. Since we don't have an
9215 * end-of-backup record, we use the pg_control value to check whether
9216 * we've reached the end of backup when starting recovery from this
9217 * backup. We have no way of checking if pg_control wasn't backed up last
9218 * however.
9219 *
9220 * We don't force a switch to new WAL file but it is still possible to
9221 * wait for all the required files to be archived if waitforarchive is
9222 * true. This is okay if we use the backup to start a standby and fetch
9223 * the missing WAL using streaming replication. But in the case of an
9224 * archive recovery, a user should set waitforarchive to true and wait for
9225 * them to be archived to ensure that all the required files are
9226 * available.
9227 *
9228 * We return the current minimum recovery point as the backup end
9229 * location. Note that it can be greater than the exact backup end
9230 * location if the minimum recovery point is updated after the backup of
9231 * pg_control. This is harmless for current uses.
9232 *
9233 * XXX currently a backup history file is for informational and debug
9234 * purposes only. It's not essential for an online backup. Furthermore,
9235 * even if it's created, it will not be archived during recovery because
9236 * an archiver is not invoked. So it doesn't seem worthwhile to write a
9237 * backup history file during recovery.
9238 */
9240 {
9241 XLogRecPtr recptr;
9243 /*
9244 * Check to see if all WAL replayed during online backup contain
9245 * full-page writes.
9246 */
9257 "is corrupt and should not be used. "
9266 }
9267 else
9268 {
9271 /*
9272 * Write the backup-end xlog record
9273 */
9279 /*
9280 * Given that we're not in recovery, InsertTimeLineID is set and can't
9281 * change, so we can read it without a lock.
9282 */
9285 /*
9286 * Force a switch to a new xlog segment file, so that the backup is
9287 * valid as soon as archiver moves out the current segment file.
9288 */
9293 /*
9294 * Write the backup history file
9295 */
9304 histfilepath)));
9306 /* Build and save the contents of the backup history file */
9315 histfilepath)));
9317 /*
9318 * Clean out any no-longer-needed history files. As a side effect,
9319 * this will post a .ready file for the newly created history file,
9320 * notifying the archiver that history file may be archived
9321 * immediately.
9322 */
9324 }
9326 /*
9327 * If archiving is enabled, wait for all the required WAL files to be
9328 * archived before returning. If archiving isn't enabled, the required WAL
9329 * needs to be transported via streaming replication (hopefully with
9330 * wal_keep_size set high enough), or some more exotic mechanism like
9331 * polling and copying files from pg_wal with script. We have no knowledge
9332 * of those mechanisms, so it's up to the user to ensure that he gets all
9333 * the required WAL.
9334 *
9335 * We wait until both the last WAL file filled during backup and the
9336 * history file have been archived, and assume that the alphabetic sorting
9337 * property of the WAL files ensures any earlier WAL files are safely
9338 * archived as well.
9339 *
9340 * We wait forever, since archive_command is supposed to work and we
9341 * assume the admin wanted his backup to work completely. If you don't
9342 * wish to wait, then either waitforarchive should be passed in as false,
9343 * or you can set statement_timeout. Also, some notices are issued to
9344 * clue in anyone who might be doing this interactively.
9345 */
9350 {
9353 wal_segment_size);
9364 {
9368 {
9372 }
9377 WAIT_EVENT_BACKUP_WAIT_WAL_ARCHIVE);
9381 {
9385 waits),
9387 "You can safely cancel this backup, "
9389 }
9390 }
9394 }
9397 (errmsg("WAL archiving is not enabled; you must ensure that all required WAL segments are copied through other means to complete the backup")));
9398 }
9401 /*
9402 * do_pg_abort_backup: abort a running backup
9403 *
9404 * This does just the most basic steps of do_pg_backup_stop(), by taking the
9405 * system out of backup mode, thus making it a lot more safe to call from
9406 * an error handler.
9407 *
9408 * 'arg' indicates that it's being called during backup setup; so
9409 * sessionBackupState has not been modified yet, but runningBackups has
9410 * already been incremented. When it's false, then it's invoked as a
9411 * before_shmem_exit handler, and therefore we must not change state
9412 * unless sessionBackupState indicates that a backup is actually running.
9413 *
9414 * NB: This gets used as a PG_ENSURE_ERROR_CLEANUP callback and
9415 * before_shmem_exit handler, hence the odd-looking signature.
9416 */
9417 void
9419 {
9422 /* If called during backup start, there shouldn't be one already running */
9426 {
9437 }
9438 }
9440 /*
9441 * Register a handler that will warn about unterminated backups at end of
9442 * session, unless this has already been done.
9443 */
9444 void
9446 {
9453 }
9455 /*
9456 * Get latest WAL insert pointer
9457 */
9458 XLogRecPtr
9460 {
9462 uint64 current_bytepos;
9469 }
9471 /*
9472 * Get latest WAL write pointer
9473 */
9474 XLogRecPtr
9476 {
9480 }
9482 /*
9483 * Returns the redo pointer of the last checkpoint or restartpoint. This is
9484 * the oldest point in WAL that we still need, if we have to restart recovery.
9485 */
9486 void
9488 {
9493 }
9495 /* Thin wrapper around ShutdownWalRcv(). */
9496 void
9498 {
9504 }
9506 /* Enable WAL file recycling and preallocation. */
9507 void
9509 {
9513 }
9515 bool
9517 {
9525 }
9527 /*
9528 * Update the WalWriterSleeping flag.
9529 */
9530 void
9532 {
9536 }