Backup and restore Redis Sentinel cluster data with automated strategies and monitoring

Install required backup tools

Install Redis tools and backup utilities needed for cluster data management.

sudo apt update
sudo apt install -y redis-tools awscli s3cmd gzip pigz cron

sudo dnf install -y redis awscli s3cmd gzip pigz cronie
sudo systemctl enable --now crond

Create backup directory structure

Set up organized directories for local backups with proper permissions and ownership.

sudo mkdir -p /var/backups/redis/{daily,weekly,monthly,snapshots}
sudo mkdir -p /var/backups/redis/logs
sudo useradd -r -s /bin/false redis-backup
sudo chown -R redis-backup:redis-backup /var/backups/redis
sudo chmod 750 /var/backups/redis
sudo chmod 755 /var/backups/redis/logs

Configure Redis nodes for backup

Enable RDB snapshots and configure backup-friendly settings on all Redis nodes.

# Enable RDB snapshots
save 900 1
save 300 10
save 60 10000

# RDB configuration
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /var/lib/redis

# Enable AOF for point-in-time recovery
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec

# Backup-friendly settings
stop-writes-on-bgsave-error yes
rdb-save-incremental-fsync yes

Restart Redis services

Apply the new configuration by restarting all Redis nodes in the cluster.

sudo systemctl restart redis-server
sudo systemctl status redis-server

Create backup script for Sentinel cluster

Develop a comprehensive backup script that handles multiple Redis nodes and Sentinel configuration.

#!/bin/bash

# Redis Sentinel Backup Script
# Supports multiple Redis nodes and Sentinel instances

set -euo pipefail

# Configuration
BACKUP_DIR="/var/backups/redis"
LOG_FILE="$BACKUP_DIR/logs/backup-$(date +%Y%m%d-%H%M%S).log"
RETENTION_DAYS=30
S3_BUCKET="your-redis-backups"
NOTIFY_EMAIL="admin@example.com"

# Redis Sentinel configuration
SENTINEL_HOSTS=("127.0.0.1:26379" "127.0.0.1:26380" "127.0.0.1:26381")
MASTER_NAME="mymaster"

# Logging function
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

# Error handling
error_exit() {
    log "ERROR: $1"
    echo "Redis backup failed: $1" | mail -s "Redis Backup Alert" "$NOTIFY_EMAIL"
    exit 1
}

# Get master info from Sentinel
get_master_info() {
    local sentinel_host="$1"
    local host port
    
    for attempt in {1..3}; do
        if master_info=$(redis-cli -h "${sentinel_host%:*}" -p "${sentinel_host#*:}" \
                        SENTINEL get-master-addr-by-name "$MASTER_NAME" 2>/dev/null); then
            host=$(echo "$master_info" | head -n1)
            port=$(echo "$master_info" | tail -n1)
            echo "$host:$port"
            return 0
        fi
        log "Attempt $attempt failed for sentinel $sentinel_host"
        sleep 2
    done
    return 1
}

# Get all Redis nodes (master + slaves)
get_all_nodes() {
    local master_addr
    local nodes=()
    
    # Get master address
    for sentinel in "${SENTINEL_HOSTS[@]}"; do
        if master_addr=$(get_master_info "$sentinel"); then
            nodes+=("$master_addr")
            break
        fi
    done
    
    [[ ${#nodes[@]} -eq 0 ]] && error_exit "Could not determine master address"
    
    # Get slave addresses
    local master_host="${master_addr%:*}"
    local master_port="${master_addr#*:}"
    
    if slaves=$(redis-cli -h "$master_host" -p "$master_port" \
               INFO replication | grep "slave[0-9]" | cut -d: -f1 --complement); then
        while IFS= read -r slave_line; do
            if [[ -n "$slave_line" ]]; then
                local slave_ip=$(echo "$slave_line" | cut -d, -f1 | cut -d= -f2)
                local slave_port=$(echo "$slave_line" | cut -d, -f2 | cut -d= -f2)
                nodes+=("$slave_ip:$slave_port")
            fi
        done <<< "$slaves"
    fi
    
    printf '%s\n' "${nodes[@]}"
}

# Backup single Redis instance
backup_redis_instance() {
    local host="$1"
    local port="$2"
    local backup_path="$3"
    local timestamp="$4"
    
    log "Starting backup for Redis instance $host:$port"
    
    # Create instance backup directory
    local instance_dir="$backup_path/redis-${host//\./-}-$port"
    mkdir -p "$instance_dir"
    
    # Save current dataset
    if ! redis-cli -h "$host" -p "$port" BGSAVE; then
        error_exit "Failed to initiate BGSAVE on $host:$port"
    fi
    
    # Wait for background save to complete
    local save_status
    while true; do
        save_status=$(redis-cli -h "$host" -p "$port" LASTSAVE)
        sleep 2
        local current_save=$(redis-cli -h "$host" -p "$port" LASTSAVE)
        [[ "$save_status" != "$current_save" ]] && break
        sleep 3
    done
    
    # Get Redis data directory
    local redis_dir
    redis_dir=$(redis-cli -h "$host" -p "$port" CONFIG GET dir | tail -n1)
    
    # Copy RDB file
    if [[ -f "$redis_dir/dump.rdb" ]]; then
        cp "$redis_dir/dump.rdb" "$instance_dir/dump-$timestamp.rdb"
        pigz -9 "$instance_dir/dump-$timestamp.rdb"
        log "RDB backup completed for $host:$port"
    else
        log "WARNING: No RDB file found for $host:$port"
    fi
    
    # Copy AOF file if enabled
    if [[ -f "$redis_dir/appendonly.aof" ]]; then
        cp "$redis_dir/appendonly.aof" "$instance_dir/appendonly-$timestamp.aof"
        pigz -9 "$instance_dir/appendonly-$timestamp.aof"
        log "AOF backup completed for $host:$port"
    fi
    
    # Save instance configuration
    redis-cli -h "$host" -p "$port" CONFIG GET '*' > "$instance_dir/config-$timestamp.txt"
    
    # Save instance info
    redis-cli -h "$host" -p "$port" INFO ALL > "$instance_dir/info-$timestamp.txt"
}

# Backup Sentinel configuration
backup_sentinel_config() {
    local backup_path="$1"
    local timestamp="$2"
    
    log "Backing up Sentinel configuration"
    
    local sentinel_dir="$backup_path/sentinel"
    mkdir -p "$sentinel_dir"
    
    # Backup Sentinel configuration files
    for config_file in /etc/redis/sentinel*.conf; do
        if [[ -f "$config_file" ]]; then
            cp "$config_file" "$sentinel_dir/$(basename "$config_file")-$timestamp"
        fi
    done
    
    # Get runtime Sentinel configuration
    for sentinel in "${SENTINEL_HOSTS[@]}"; do
        local host="${sentinel%:*}"
        local port="${sentinel#*:}"
        
        if redis-cli -h "$host" -p "$port" ping >/dev/null 2>&1; then
            redis-cli -h "$host" -p "$port" SENTINEL masters > \
                "$sentinel_dir/masters-${host//\./-}-$port-$timestamp.txt"
            redis-cli -h "$host" -p "$port" SENTINEL slaves "$MASTER_NAME" > \
                "$sentinel_dir/slaves-${host//\./-}-$port-$timestamp.txt"
        fi
    done
}

# Upload to S3 (if configured)
upload_to_s3() {
    local backup_path="$1"
    
    if [[ -n "$S3_BUCKET" ]] && command -v aws >/dev/null 2>&1; then
        log "Uploading backup to S3"
        if aws s3 sync "$backup_path" "s3://$S3_BUCKET/$(basename "$backup_path")/" \
           --storage-class STANDARD_IA; then
            log "S3 upload completed successfully"
        else
            log "WARNING: S3 upload failed"
        fi
    fi
}

# Clean old backups
cleanup_old_backups() {
    log "Cleaning up backups older than $RETENTION_DAYS days"
    find "$BACKUP_DIR" -type f -name "*.gz" -mtime +"$RETENTION_DAYS" -delete
    find "$BACKUP_DIR" -type f -name "*.txt" -mtime +"$RETENTION_DAYS" -delete
    find "$BACKUP_DIR" -empty -type d -delete
}

# Main backup function
main() {
    local timestamp
    timestamp=$(date +%Y%m%d-%H%M%S)
    local backup_type="${1:-daily}"
    local backup_path="$BACKUP_DIR/$backup_type/$timestamp"
    
    log "Starting Redis Sentinel cluster backup ($backup_type)"
    
    # Create backup directory
    mkdir -p "$backup_path"
    
    # Get all Redis nodes
    local nodes
    if ! nodes=$(get_all_nodes); then
        error_exit "Failed to discover Redis nodes"
    fi
    
    # Backup each Redis instance
    while IFS= read -r node; do
        local host="${node%:*}"
        local port="${node#*:}"
        backup_redis_instance "$host" "$port" "$backup_path" "$timestamp"
    done <<< "$nodes"
    
    # Backup Sentinel configuration
    backup_sentinel_config "$backup_path" "$timestamp"
    
    # Create backup manifest
    {
        echo "Redis Sentinel Backup Manifest"
        echo "Timestamp: $timestamp"
        echo "Backup Type: $backup_type"
        echo "Nodes Backed Up:"
        echo "$nodes"
        echo "Files:"
        find "$backup_path" -type f -exec ls -lh {} \;
    } > "$backup_path/manifest.txt"
    
    # Upload to S3
    upload_to_s3 "$backup_path"
    
    # Cleanup old backups
    cleanup_old_backups
    
    log "Backup completed successfully"
    
    # Send success notification
    echo "Redis Sentinel backup completed successfully at $timestamp" | \
        mail -s "Redis Backup Success" "$NOTIFY_EMAIL"
}

# Execute main function
main "$@"

Make backup script executable

Set proper permissions for the backup script and create a secure wrapper.

sudo chmod 750 /usr/local/bin/redis-sentinel-backup.sh
sudo chown root:redis-backup /usr/local/bin/redis-sentinel-backup.sh

Create restore script

Implement point-in-time recovery capabilities with validation and rollback options.

#!/bin/bash

# Redis Sentinel Restore Script
# Supports point-in-time recovery and validation

set -euo pipefail

# Configuration
BACKUP_DIR="/var/backups/redis"
LOG_FILE="/var/log/redis-restore-$(date +%Y%m%d-%H%M%S).log"
REDIS_DATA_DIR="/var/lib/redis"
NOTIFY_EMAIL="admin@example.com"

# Logging function
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

# Error handling
error_exit() {
    log "ERROR: $1"
    exit 1
}

# List available backups
list_backups() {
    echo "Available backups:"
    find "$BACKUP_DIR" -name "manifest.txt" | while read -r manifest; do
        local backup_dir
        backup_dir=$(dirname "$manifest")
        local backup_name
        backup_name=$(basename "$backup_dir")
        local backup_type
        backup_type=$(basename "$(dirname "$backup_dir")")
        
        echo "  $backup_type/$backup_name"
        echo "    $(grep 'Timestamp:' "$manifest")"
        echo "    $(grep 'Nodes Backed Up:' -A 10 "$manifest" | tail -n +2 | head -5)"
        echo ""
    done
}

# Validate backup integrity
validate_backup() {
    local backup_path="$1"
    
    log "Validating backup integrity: $backup_path"
    
    if [[ ! -f "$backup_path/manifest.txt" ]]; then
        error_exit "Backup manifest not found"
    fi
    
    # Check for RDB files
    local rdb_files
    rdb_files=$(find "$backup_path" -name "*.rdb.gz" | wc -l)
    
    if [[ "$rdb_files" -eq 0 ]]; then
        error_exit "No RDB backup files found"
    fi
    
    log "Found $rdb_files RDB backup files"
    
    # Validate compressed files
    find "$backup_path" -name "*.gz" | while read -r compressed_file; do
        if ! pigz -t "$compressed_file" >/dev/null 2>&1; then
            error_exit "Corrupted backup file: $compressed_file"
        fi
    done
    
    log "Backup validation completed successfully"
}

# Stop Redis services
stop_redis_services() {
    log "Stopping Redis services"
    
    sudo systemctl stop redis-server redis-sentinel || true
    sleep 5
    
    # Ensure processes are stopped
    if pgrep -x redis-server >/dev/null || pgrep -x redis-sentinel >/dev/null; then
        log "Force killing remaining Redis processes"
        sudo pkill -TERM redis-server redis-sentinel || true
        sleep 3
        sudo pkill -KILL redis-server redis-sentinel || true
    fi
}

# Start Redis services
start_redis_services() {
    log "Starting Redis services"
    
    sudo systemctl start redis-server
    sleep 5
    sudo systemctl start redis-sentinel
    
    # Wait for services to be ready
    local attempts=0
    while [[ $attempts -lt 30 ]]; do
        if redis-cli ping >/dev/null 2>&1; then
            log "Redis services started successfully"
            return 0
        fi
        sleep 2
        ((attempts++))
    done
    
    error_exit "Redis services failed to start"
}

# Backup current data
backup_current_data() {
    local backup_dir="/tmp/redis-restore-backup-$(date +%Y%m%d-%H%M%S)"
    
    log "Creating safety backup of current data: $backup_dir"
    
    mkdir -p "$backup_dir"
    
    if [[ -f "$REDIS_DATA_DIR/dump.rdb" ]]; then
        cp "$REDIS_DATA_DIR/dump.rdb" "$backup_dir/"
    fi
    
    if [[ -f "$REDIS_DATA_DIR/appendonly.aof" ]]; then
        cp "$REDIS_DATA_DIR/appendonly.aof" "$backup_dir/"
    fi
    
    echo "$backup_dir"
}

# Restore Redis instance
restore_redis_instance() {
    local instance_backup_dir="$1"
    local target_host="${2:-127.0.0.1}"
    local target_port="${3:-6379}"
    
    log "Restoring Redis instance from: $instance_backup_dir"
    
    # Find and extract RDB file
    local rdb_file
    rdb_file=$(find "$instance_backup_dir" -name "dump-*.rdb.gz" | head -n1)
    
    if [[ -n "$rdb_file" ]]; then
        log "Restoring RDB file: $(basename "$rdb_file")"
        
        # Extract to temporary location first
        local temp_rdb="/tmp/$(basename "${rdb_file%.gz}")"
        pigz -dc "$rdb_file" > "$temp_rdb"
        
        # Validate RDB file
        if redis-check-rdb "$temp_rdb"; then
            sudo cp "$temp_rdb" "$REDIS_DATA_DIR/dump.rdb"
            sudo chown redis:redis "$REDIS_DATA_DIR/dump.rdb"
            sudo chmod 660 "$REDIS_DATA_DIR/dump.rdb"
            rm "$temp_rdb"
            log "RDB file restored successfully"
        else
            rm "$temp_rdb"
            error_exit "Invalid RDB file"
        fi
    fi
    
    # Find and extract AOF file if available
    local aof_file
    aof_file=$(find "$instance_backup_dir" -name "appendonly-*.aof.gz" | head -n1)
    
    if [[ -n "$aof_file" ]]; then
        log "Restoring AOF file: $(basename "$aof_file")"
        
        # Extract to temporary location first
        local temp_aof="/tmp/$(basename "${aof_file%.gz}")"
        pigz -dc "$aof_file" > "$temp_aof"
        
        # Validate AOF file
        if redis-check-aof "$temp_aof"; then
            sudo cp "$temp_aof" "$REDIS_DATA_DIR/appendonly.aof"
            sudo chown redis:redis "$REDIS_DATA_DIR/appendonly.aof"
            sudo chmod 660 "$REDIS_DATA_DIR/appendonly.aof"
            rm "$temp_aof"
            log "AOF file restored successfully"
        else
            log "WARNING: Invalid AOF file, skipping"
            rm "$temp_aof"
        fi
    fi
}

# Restore Sentinel configuration
restore_sentinel_config() {
    local backup_path="$1"
    
    log "Restoring Sentinel configuration"
    
    local sentinel_backup="$backup_path/sentinel"
    
    if [[ -d "$sentinel_backup" ]]; then
        # Backup current Sentinel configs
        sudo cp /etc/redis/sentinel*.conf /tmp/ 2>/dev/null || true
        
        # Restore Sentinel configuration files
        find "$sentinel_backup" -name "sentinel*.conf-*" | while read -r config_backup; do
            local original_name
            original_name=$(basename "$config_backup" | sed 's/-[0-9]*-[0-9]*$//')
            sudo cp "$config_backup" "/etc/redis/$original_name"
            sudo chown redis:redis "/etc/redis/$original_name"
            sudo chmod 640 "/etc/redis/$original_name"
        done
        
        log "Sentinel configuration restored"
    fi
}

# Main restore function
main() {
    if [[ $# -lt 1 ]]; then
        echo "Usage: $0 

Set restore script permissions

Configure secure permissions for the restore script.

sudo chmod 750 /usr/local/bin/redis-sentinel-restore.sh
sudo chown root:redis-backup /usr/local/bin/redis-sentinel-restore.sh

Configure automated backup scheduling

Set up cron jobs for automated daily, weekly, and monthly backups.

sudo -u redis-backup crontab -e

# Redis Sentinel Automated Backups
# Daily backup at 2:30 AM
30 2 * * * /usr/local/bin/redis-sentinel-backup.sh daily

# Weekly backup on Sunday at 3:30 AM
30 3 * * 0 /usr/local/bin/redis-sentin