Implement Consul backup and disaster recovery with automated snapshots and restoration

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
CONSUL_BACKUP_USER="consul-backup"
BACKUP_BASE_DIR="/opt/consul-backup"
GPG_PASSPHRASE="${GPG_PASSPHRASE:-ConsulBackup$(date +%s)!}"
CONSUL_DATACENTER="${CONSUL_DATACENTER:-dc1}"
S3_BUCKET="${S3_BUCKET:-}"

# Usage message
usage() {
    echo "Usage: $0 [options]"
    echo "Options:"
    echo "  -b, --s3-bucket BUCKET    S3 bucket for remote backups (optional)"
    echo "  -d, --datacenter NAME     Consul datacenter name (default: dc1)"
    echo "  -p, --passphrase PHRASE   GPG passphrase (default: auto-generated)"
    echo "  -h, --help               Show this help message"
    exit 1
}

# Parse command line arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -b|--s3-bucket)
            S3_BUCKET="$2"
            shift 2
            ;;
        -d|--datacenter)
            CONSUL_DATACENTER="$2"
            shift 2
            ;;
        -p|--passphrase)
            GPG_PASSPHRASE="$2"
            shift 2
            ;;
        -h|--help)
            usage
            ;;
        *)
            echo -e "${RED}Unknown option: $1${NC}"
            usage
            ;;
    esac
done

# Logging functions
log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

log_step() {
    echo -e "${BLUE}$1${NC}"
}

# Cleanup function
cleanup() {
    local exit_code=$?
    if [[ $exit_code -ne 0 ]]; then
        log_error "Installation failed. Cleaning up..."
        systemctl stop consul-backup.timer 2>/dev/null || true
        systemctl disable consul-backup.timer 2>/dev/null || true
        rm -f /etc/systemd/system/consul-backup.service
        rm -f /etc/systemd/system/consul-backup.timer
        systemctl daemon-reload
    fi
    exit $exit_code
}

trap cleanup ERR

# Check if running as root
if [[ $EUID -ne 0 ]]; then
    log_error "This script must be run as root"
    exit 1
fi

# Detect OS distribution
log_step "[1/12] Detecting operating system..."
if [[ -f /etc/os-release ]]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_UPDATE="apt update"
            PKG_INSTALL="apt install -y"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf check-update || true"
            PKG_INSTALL="dnf install -y"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_UPDATE="yum check-update || true"
            PKG_INSTALL="yum install -y"
            ;;
        *)
            log_error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
    log_info "Detected OS: $PRETTY_NAME"
else
    log_error "Cannot detect operating system"
    exit 1
fi

# Check prerequisites
log_step "[2/12] Checking prerequisites..."
if ! command -v consul &> /dev/null; then
    log_error "Consul is not installed or not in PATH"
    exit 1
fi

if ! consul members &> /dev/null; then
    log_error "Cannot connect to Consul cluster"
    exit 1
fi

log_info "Prerequisites check passed"

# Install required packages
log_step "[3/12] Installing required packages..."
$PKG_UPDATE
$PKG_INSTALL gnupg2 jq curl
if [[ -n "$S3_BUCKET" ]]; then
    if [[ "$PKG_MGR" == "apt" ]]; then
        $PKG_INSTALL awscli
    else
        $PKG_INSTALL awscli2
    fi
fi
log_info "Required packages installed"

# Create backup user and directories
log_step "[4/12] Creating backup user and directories..."
if ! id "$CONSUL_BACKUP_USER" &>/dev/null; then
    useradd -r -s /bin/bash -d "$BACKUP_BASE_DIR" "$CONSUL_BACKUP_USER"
fi

mkdir -p "$BACKUP_BASE_DIR"/{scripts,backups,logs,keys}
chown -R "$CONSUL_BACKUP_USER:$CONSUL_BACKUP_USER" "$BACKUP_BASE_DIR"
chmod 755 "$BACKUP_BASE_DIR"
chmod 700 "$BACKUP_BASE_DIR"/{backups,keys}
chmod 755 "$BACKUP_BASE_DIR"/{scripts,logs}
log_info "Backup user and directories created"

# Generate GPG encryption key
log_step "[5/12] Generating GPG encryption key..."
sudo -u "$CONSUL_BACKUP_USER" gpg --batch --gen-key <<EOF
Key-Type: RSA
Key-Length: 4096
Name-Real: Consul Backup
Name-Email: consul-backup@$(hostname -f)
Expire-Date: 0
Passphrase: $GPG_PASSPHRASE
%commit
EOF

GPG_KEY_ID=$(sudo -u "$CONSUL_BACKUP_USER" gpg --list-keys --with-colons | grep fpr | head -1 | cut -d: -f10)
echo "CONSUL_GPG_KEY=$GPG_KEY_ID" > "$BACKUP_BASE_DIR/keys/gpg-key-id"
echo "GPG_PASSPHRASE=$GPG_PASSPHRASE" >> "$BACKUP_BASE_DIR/keys/gpg-key-id"
chown "$CONSUL_BACKUP_USER:$CONSUL_BACKUP_USER" "$BACKUP_BASE_DIR/keys/gpg-key-id"
chmod 600 "$BACKUP_BASE_DIR/keys/gpg-key-id"
log_info "GPG key generated with ID: ${GPG_KEY_ID:0:16}..."

# Create Consul ACL policy and token
log_step "[6/12] Creating Consul ACL policy and token..."
consul acl policy create -name "consul-backup" -rules - <<EOF || true
acl = "read"
key_prefix "" {
  policy = "read"
}
node_prefix "" {
  policy = "read"
}
operator = "read"
service_prefix "" {
  policy = "read"
}
session_prefix "" {
  policy = "read"
}
EOF

BACKUP_TOKEN=$(consul acl token create -policy-name "consul-backup" -description "Consul backup token" -format json | jq -r '.SecretID')
echo "CONSUL_HTTP_TOKEN=$BACKUP_TOKEN" > "$BACKUP_BASE_DIR/keys/consul-token"
chown "$CONSUL_BACKUP_USER:$CONSUL_BACKUP_USER" "$BACKUP_BASE_DIR/keys/consul-token"
chmod 600 "$BACKUP_BASE_DIR/keys/consul-token"
log_info "Consul ACL token created"

# Create backup script
log_step "[7/12] Creating backup script..."
cat > "$BACKUP_BASE_DIR/scripts/consul-backup.sh" <<'SCRIPT_EOF'
#!/bin/bash
set -euo pipefail

BACKUP_DIR="/opt/consul-backup/backups"
LOG_FILE="/opt/consul-backup/logs/backup.log"
RETENTION_DAYS=30
S3_BUCKET="${S3_BUCKET:-}"
CONSUL_ADDR="http://localhost:8500"
GPG_KEY_FILE="/opt/consul-backup/keys/gpg-key-id"
TOKEN_FILE="/opt/consul-backup/keys/consul-token"

[[ -f "$GPG_KEY_FILE" ]] && source "$GPG_KEY_FILE"
[[ -f "$TOKEN_FILE" ]] && source "$TOKEN_FILE"

log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

TIMESTAMP=$(date '+%Y%m%d_%H%M%S')
SNAPSHOT_FILE="consul-snapshot-$TIMESTAMP.snap"
ENCRYPTED_FILE="$SNAPSHOT_FILE.gpg"

log "Starting Consul backup process"

if ! consul members &>/dev/null; then
    log "ERROR: Cannot connect to Consul cluster"
    exit 1
fi

log "Creating Consul snapshot"
if consul snapshot save -http-addr="$CONSUL_ADDR" "$BACKUP_DIR/$SNAPSHOT_FILE"; then
    log "Snapshot created successfully: $SNAPSHOT_FILE"
else
    log "ERROR: Failed to create snapshot"
    exit 1
fi

log "Encrypting snapshot with GPG"
if echo "$GPG_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --trust-model always --encrypt -r "$CONSUL_GPG_KEY" --cipher-algo AES256 --output "$BACKUP_DIR/$ENCRYPTED_FILE" "$BACKUP_DIR/$SNAPSHOT_FILE"; then
    log "Snapshot encrypted successfully"
    rm -f "$BACKUP_DIR/$SNAPSHOT_FILE"
else
    log "ERROR: Failed to encrypt snapshot"
    exit 1
fi

if [[ -n "$S3_BUCKET" ]] && command -v aws &>/dev/null; then
    log "Uploading encrypted snapshot to S3"
    if aws s3 cp "$BACKUP_DIR/$ENCRYPTED_FILE" "s3://$S3_BUCKET/consul-backups/$(date +%Y/%m)/"; then
        log "Snapshot uploaded to S3 successfully"
    else
        log "WARNING: Failed to upload snapshot to S3"
    fi
fi

log "Cleaning up old backups (older than $RETENTION_DAYS days)"
find "$BACKUP_DIR" -name "consul-snapshot-*.snap.gpg" -mtime +$RETENTION_DAYS -delete
find "$LOG_FILE" -mtime +$RETENTION_DAYS -delete 2>/dev/null || true

log "Backup process completed successfully"
SCRIPT_EOF

chown "$CONSUL_BACKUP_USER:$CONSUL_BACKUP_USER" "$BACKUP_BASE_DIR/scripts/consul-backup.sh"
chmod 750 "$BACKUP_BASE_DIR/scripts/consul-backup.sh"
log_info "Backup script created"

# Create systemd service
log_step "[8/12] Creating systemd service..."
cat > /etc/systemd/system/consul-backup.service <<EOF
[Unit]
Description=Consul Backup Service
After=consul.service

[Service]
Type=oneshot
User=$CONSUL_BACKUP_USER
Group=$CONSUL_BACKUP_USER
ExecStart=$BACKUP_BASE_DIR/scripts/consul-backup.sh
Environment=S3_BUCKET=$S3_BUCKET
StandardOutput=journal
StandardError=journal
EOF

log_info "Systemd service created"

# Create systemd timer
log_step "[9/12] Creating systemd timer..."
cat > /etc/systemd/system/consul-backup.timer <<EOF
[Unit]
Description=Run Consul backup every 6 hours
Requires=consul-backup.service

[Timer]
OnCalendar=*-*-* 00,06,12,18:00:00
Persistent=true

[Install]
WantedBy=timers.target
EOF

systemctl daemon-reload
systemctl enable consul-backup.timer
systemctl start consul-backup.timer
log_info "Systemd timer configured and started"

# Create restore script
log_step "[10/12] Creating restore script..."
cat > "$BACKUP_BASE_DIR/scripts/consul-restore.sh" <<'RESTORE_EOF'
#!/bin/bash
set -euo pipefail

if [[ $# -ne 1 ]]; then
    echo "Usage: $0 <encrypted_snapshot_file>"
    exit 1
fi

ENCRYPTED_FILE="$1"
GPG_KEY_FILE="/opt/consul-backup/keys/gpg-key-id"
TOKEN_FILE="/opt/consul-backup/keys/consul-token"

[[ -f "$GPG_KEY_FILE" ]] && source "$GPG_KEY_FILE"
[[ -f "$TOKEN_FILE" ]] && source "$TOKEN_FILE"

if [[ ! -f "$ENCRYPTED_FILE" ]]; then
    echo "ERROR: Encrypted snapshot file not found: $ENCRYPTED_FILE"
    exit 1
fi

TEMP_SNAPSHOT="/tmp/consul-restore-$(date +%s).snap"

echo "Decrypting snapshot..."
if echo "$GPG_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --decrypt --output "$TEMP_SNAPSHOT" "$ENCRYPTED_FILE"; then
    echo "Snapshot decrypted successfully"
else
    echo "ERROR: Failed to decrypt snapshot"
    exit 1
fi

echo "Restoring Consul snapshot..."
if consul snapshot restore "$TEMP_SNAPSHOT"; then
    echo "Snapshot restored successfully"
    rm -f "$TEMP_SNAPSHOT"
else
    echo "ERROR: Failed to restore snapshot"
    rm -f "$TEMP_SNAPSHOT"
    exit 1
fi
RESTORE_EOF

chown "$CONSUL_BACKUP_USER:$CONSUL_BACKUP_USER" "$BACKUP_BASE_DIR/scripts/consul-restore.sh"
chmod 750 "$BACKUP_BASE_DIR/scripts/consul-restore.sh"
log_info "Restore script created"

# Configure log rotation
log_step "[11/12] Configuring log rotation..."
cat > /etc/logrotate.d/consul-backup <<EOF
$BACKUP_BASE_DIR/logs/*.log {
    daily
    rotate 30
    compress
    delaycompress
    missingok
    notifempty
    create 644 $CONSUL_BACKUP_USER $CONSUL_BACKUP_USER
}
EOF

log_info "Log rotation configured"

# Verify installation
log_step "[12/12] Verifying installation..."
if systemctl is-active consul-backup.timer &>/dev/null; then
    log_info "Consul backup timer is active"
else
    log_error "Consul backup timer is not active"
    exit 1
fi

# Test backup script
log_info "Running test backup..."
sudo -u "$CONSUL_BACKUP_USER" "$BACKUP_BASE_DIR/scripts/consul-backup.sh"

if [[ -n "$(find "$BACKUP_BASE_DIR/backups" -name "consul-snapshot-*.snap.gpg" -mtime -1)" ]]; then
    log_info "Test backup completed successfully"
else
    log_error "Test backup failed"
    exit 1
fi

echo
log_info "Consul backup and disaster recovery setup completed successfully!"
echo
echo -e "${BLUE}Configuration Summary:${NC}"
echo "- Backup user: $CONSUL_BACKUP_USER"
echo "- Backup directory: $BACKUP_BASE_DIR"
echo "- GPG key ID: ${GPG_KEY_ID:0