From 05da9001b9edc954c009be19a881626b2a763146 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 5 Apr 2024 17:12:40 +1100 Subject: [PATCH] ctdb-scripts: Add support for backing up persistent TDBs Signed-off-by: Vinit Agnihotri Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs --- ctdb/config/ctdb-backup-persistent-tdbs.sh | 123 +++++++++++++++++++++++++++ ctdb/config/events/legacy/95.database.script | 35 ++++++++ ctdb/doc/ctdb-script.options.5.xml | 53 +++++++++++- ctdb/doc/examples/ctdb.spec.in | 1 + ctdb/wscript | 1 + 5 files changed, 212 insertions(+), 1 deletion(-) create mode 100755 ctdb/config/ctdb-backup-persistent-tdbs.sh diff --git a/ctdb/config/ctdb-backup-persistent-tdbs.sh b/ctdb/config/ctdb-backup-persistent-tdbs.sh new file mode 100755 index 00000000000..9610eefeb38 --- /dev/null +++ b/ctdb/config/ctdb-backup-persistent-tdbs.sh @@ -0,0 +1,123 @@ +#!/bin/sh + +# Backup persistent CTDB TDBs into the given directory. + +# Copyright: DataDirect Networks, 2024 +# Authors: Vinit Agnihotri +# Martin Schwenke + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +# Options: +# +# -l: Only do the backup if this node is the leader node, otherwise +# exit with 0. +# +# -L : Only do the backup if this node is the leader node, otherwise +# exit with . + +# +# Option/argument handling +# + +die() +{ + echo "ERROR: $1" + exit 1 +} + +usage() +{ + die "usage: $0 [-l | -L ] " +} + +leader_only=false +leader_only_rc=0 +dir="" + +while getopts "L:lh?" opt; do + case "$opt" in + L) + leader_only=true + leader_only_rc="$OPTARG" + ;; + l) + leader_only=true + ;; + \? | h) + usage + ;; + esac +done +shift $((OPTIND - 1)) + +if [ $# -ne 1 ]; then + usage +fi + +dir="$1" + +if [ ! -d "$dir" ]; then + die "No such directory ${dir}" +fi + +if $leader_only; then + this_node=$(ctdb pnn) + leader_node=$(ctdb leader) + if [ "$this_node" != "$leader_node" ]; then + exit "$leader_only_rc" + fi +fi + +# +# Backups TDBs in timestamped subdirectory +# + +dt=$(date "+%Y%m%d%H%M%S") +prefix="ctdb-persistent-db-backup-${dt}" +outdir="${dir}/${prefix}" + +# Clean up temporary directory on failure" +trap 'rm -rf ${outdir}' 0 + +mkdir -p "$outdir" + +if ! db_map=$(ctdb getdbmap -X); then + die "Failed to list databases" +fi +db_list=$(echo "$db_map" | awk -F '|' '$5 == "1" { print $3 }') + +cd "$outdir" || die "Failed to change directory to ${dir}" + +for db in $db_list; do + if ! ctdb backupdb "$db" "${db}.backup"; then + die "Failed to backup ${db}" + fi +done + +# +# Create tarball +# + +cd "$dir" || die "Failed to change directory to ${dir}" + +tarball="${prefix}.tgz" + +if ! tar -c -z -f "$tarball" "$prefix"; then + die "Failed to create tarball" +fi + +echo "Created backup tarball ${dir}/${tarball}" + +exit 0 diff --git a/ctdb/config/events/legacy/95.database.script b/ctdb/config/events/legacy/95.database.script index dab4125806d..e2627c6c1d0 100755 --- a/ctdb/config/events/legacy/95.database.script +++ b/ctdb/config/events/legacy/95.database.script @@ -101,6 +101,38 @@ EOF done } +maybe_backup_persistent_tdbs() +{ + _dir="${CTDB_PERSISTENT_DB_BACKUP_DIR:-}" + if [ -z "$_dir" ]; then + return 0 + fi + + if [ ! -d "$_dir" ]; then + echo "Creating CTDB_PERSISTENT_DB_BACKUP_DIR=${_dir}" + if ! mkdir -p "$_dir"; then + die "ERROR: unable to create ${_dir}" + fi + fi + + # Don't backup if there are backup files from within the past day + _out=$(find "$_dir" -type f -mtime -1) + if [ -n "$_out" ]; then + return 0 + fi + + # Script will ignore if this isn't leader node, so don't + # double-check that here... + "${CTDB_BASE}/ctdb-backup-persistent-tdbs.sh" -l "$_dir" + + # Remove backups beyond the limit (default 14) + _limit="${CTDB_PERSISTENT_DB_BACKUP_LIMIT:-14}" + _offset=$((_limit + 1)) + # Can't sort by time using find instead of ls + # shellcheck disable=SC2012 + ls -t "$_dir"/* 2>/dev/null | tail -n "+${_offset}" | xargs rm -f +} + ############################################################ ctdb_check_args "$@" @@ -115,6 +147,9 @@ init) check_non_persistent_databases fi ;; +monitor) + maybe_backup_persistent_tdbs + ;; esac # all OK diff --git a/ctdb/doc/ctdb-script.options.5.xml b/ctdb/doc/ctdb-script.options.5.xml index 1fcf9c47228..11597097a04 100644 --- a/ctdb/doc/ctdb-script.options.5.xml +++ b/ctdb/doc/ctdb-script.options.5.xml @@ -1000,7 +1000,8 @@ CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=9000 - CTDB checks the consistency of databases during startup. + CTDB checks the consistency of databases during startup and + provides a facility to backup persistent databases. @@ -1024,6 +1025,56 @@ CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=9000 + + CTDB_PERSISTENT_DB_BACKUP_DIR=DIRECTORY + + + Create a daily backup tarball for all persistent TDBs + in DIRECTORY. Note that DIRECTORY must exist or no + backups will be created. + + + Given that persistent databases are fully replicated, + duplication is avoid by only creating backups on the + current leader node. To maintain a complete, single + set of backups, it makes sense for DIRECTORY to be in + a cluster filesystem. + + + This creates the backup from the + monitor event, which should be fine + because backing up persistent databases is a local + operation. Users who do not wish do create backups + during the monitor event can choose + not to use this option and instead run + /usr/local/etc/ctdb/ctdb-backup-persistent-tdbs.sh + -l DIRECTORY on all + nodes using a + cron + 8 job, which + will also need to manually manage backup pruning. + + + No default. No daily backups are created. + + + + + + CTDB_PERSISTENT_DB_BACKUP_LIMIT=COUNT + + + Keep at most COUNT backups in + CTDB_PERSISTENT_DB_BACKUP_DIR. Note that if + additional manual backups are created in this + directory then these will count towards the limit. + + + Default is 14. + + + + diff --git a/ctdb/doc/examples/ctdb.spec.in b/ctdb/doc/examples/ctdb.spec.in index 83c28b49559..23b0008d83b 100644 --- a/ctdb/doc/examples/ctdb.spec.in +++ b/ctdb/doc/examples/ctdb.spec.in @@ -199,6 +199,7 @@ fi %doc doc/examples %dir %{_sysconfdir}/ctdb %{_sysconfdir}/ctdb/functions +%{_sysconfdir}/ctdb/ctdb-backup-persistent-tdbs.sh %dir %{_sysconfdir}/ctdb/events %{_sysconfdir}/ctdb/events/* %dir %{_sysconfdir}/ctdb/nfs-checks.d diff --git a/ctdb/wscript b/ctdb/wscript index 34a7a212de1..e89f6548af3 100644 --- a/ctdb/wscript +++ b/ctdb/wscript @@ -877,6 +877,7 @@ def build(bld): bld.INSTALL_FILES(bld.env.CTDB_ETCDIR, 'functions', destname='functions') etc_scripts = [ + 'ctdb-backup-persistent-tdbs.sh', 'ctdb-crash-cleanup.sh', 'debug-hung-script.sh', 'debug_locks.sh', -- 2.11.4.GIT