Set up GitLab backup and disaster recovery with automated restoration

#!/usr/bin/env bash

set -euo pipefail

# GitLab Backup and Disaster Recovery Setup Script
# Version: 1.0

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Configuration
GITLAB_USER="git"
BACKUP_BASE_DIR="/var/opt/gitlab/backups"
CONFIG_BACKUP_DIR="/var/opt/gitlab/config-backups"
SCRIPT_DIR="/etc/gitlab/backup-scripts"
LOG_DIR="/var/log/gitlab-backup"
RETENTION_DAYS="${RETENTION_DAYS:-7}"

# Functions
log() {
    echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] $1${NC}"
}

warn() {
    echo -e "${YELLOW}[WARNING] $1${NC}"
}

error() {
    echo -e "${RED}[ERROR] $1${NC}"
}

cleanup() {
    if [ $? -ne 0 ]; then
        error "Installation failed. Cleaning up..."
        rm -rf "$SCRIPT_DIR/gitlab-backup.sh" 2>/dev/null || true
    fi
}

trap cleanup ERR

usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -r, --retention-days DAYS    Set backup retention period (default: 7)"
    echo "  -h, --help                   Show this help message"
    echo ""
    echo "Environment variables:"
    echo "  AWS_ACCESS_KEY_ID            AWS access key for S3 uploads"
    echo "  AWS_SECRET_ACCESS_KEY        AWS secret key for S3 uploads"
    echo "  AWS_REGION                   AWS region (default: us-east-1)"
    echo "  S3_BUCKET                    S3 bucket name for backups"
    exit 1
}

check_prerequisites() {
    log "[1/10] Checking prerequisites..."
    
    if [ "$EUID" -ne 0 ]; then
        error "This script must be run as root or with sudo"
        exit 1
    fi

    if ! command -v gitlab-ctl &> /dev/null; then
        error "GitLab is not installed or gitlab-ctl is not in PATH"
        exit 1
    fi

    if [ ! -f /etc/gitlab/gitlab.rb ]; then
        error "GitLab configuration file not found at /etc/gitlab/gitlab.rb"
        exit 1
    fi
}

detect_distro() {
    log "[2/10] Detecting distribution..."
    
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        case "$ID" in
            ubuntu|debian) 
                PKG_MGR="apt"
                PKG_UPDATE="apt update -y"
                PKG_INSTALL="apt install -y"
                PKG_UPGRADE="apt upgrade -y"
                ;;
            almalinux|rocky|centos|rhel|ol|fedora) 
                PKG_MGR="dnf"
                PKG_UPDATE="dnf check-update || true"
                PKG_INSTALL="dnf install -y"
                PKG_UPGRADE="dnf upgrade -y"
                ;;
            amzn) 
                PKG_MGR="yum"
                PKG_UPDATE="yum check-update || true"
                PKG_INSTALL="yum install -y"
                PKG_UPGRADE="yum update -y"
                ;;
            *) 
                error "Unsupported distribution: $ID"
                exit 1
                ;;
        esac
        log "Detected: $PRETTY_NAME"
    else
        error "Cannot detect distribution. /etc/os-release not found."
        exit 1
    fi
}

install_dependencies() {
    log "[3/10] Installing dependencies..."
    
    $PKG_UPDATE
    $PKG_INSTALL rsync awscli
    
    # Install GPG based on distro
    if [ "$PKG_MGR" = "apt" ]; then
        $PKG_INSTALL gpg
    else
        $PKG_INSTALL gnupg2
    fi
    
    log "Dependencies installed successfully"
}

create_directories() {
    log "[4/10] Creating backup directories..."
    
    mkdir -p "$BACKUP_BASE_DIR"
    mkdir -p "$CONFIG_BACKUP_DIR"
    mkdir -p "$SCRIPT_DIR"
    mkdir -p "$LOG_DIR"
    
    # Set proper ownership and permissions
    chown -R $GITLAB_USER:$GITLAB_USER "$BACKUP_BASE_DIR"
    chown -R $GITLAB_USER:$GITLAB_USER "$CONFIG_BACKUP_DIR"
    chown -R root:root "$SCRIPT_DIR"
    chown -R $GITLAB_USER:$GITLAB_USER "$LOG_DIR"
    
    chmod 755 "$BACKUP_BASE_DIR"
    chmod 755 "$CONFIG_BACKUP_DIR"
    chmod 755 "$SCRIPT_DIR"
    chmod 755 "$LOG_DIR"
    
    log "Directories created with proper permissions"
}

configure_gitlab_backups() {
    log "[5/10] Configuring GitLab backup settings..."
    
    # Backup GitLab configuration
    cp /etc/gitlab/gitlab.rb /etc/gitlab/gitlab.rb.backup.$(date +%Y%m%d_%H%M%S)
    
    # Add backup configuration
    cat >> /etc/gitlab/gitlab.rb << 'EOF'

# GitLab Backup Configuration
gitlab_rails['backup_path'] = '/var/opt/gitlab/backups'
gitlab_rails['backup_keep_time'] = 604800
gitlab_rails['backup_archive_permissions'] = 0644
gitlab_rails['backup_pg_schema'] = 'public'
EOF

    if [ -n "${AWS_ACCESS_KEY_ID:-}" ] && [ -n "${AWS_SECRET_ACCESS_KEY:-}" ]; then
        cat >> /etc/gitlab/gitlab.rb << EOF

# S3 Backup Upload Configuration
gitlab_rails['backup_upload_connection'] = {
  'provider' => 'AWS',
  'region' => '${AWS_REGION:-us-east-1}',
  'aws_access_key_id' => '${AWS_ACCESS_KEY_ID}',
  'aws_secret_access_key' => '${AWS_SECRET_ACCESS_KEY}'
}
gitlab_rails['backup_upload_remote_directory'] = '${S3_BUCKET:-gitlab-backups}'
gitlab_rails['backup_multipart_chunk_size'] = 104857600
gitlab_rails['backup_encryption'] = 'AES256'
EOF
        log "S3 backup upload configured"
    else
        warn "AWS credentials not provided. S3 upload not configured."
    fi
    
    log "GitLab configuration updated"
}

create_backup_script() {
    log "[6/10] Creating automated backup script..."
    
    cat > "$SCRIPT_DIR/gitlab-backup.sh" << 'EOF'
#!/usr/bin/env bash

set -euo pipefail

# GitLab Automated Backup Script
# Configuration
BACKUP_DIR="/var/opt/gitlab/backups"
CONFIG_BACKUP_DIR="/var/opt/gitlab/config-backups"
LOG_FILE="/var/log/gitlab-backup/backup.log"
RETENTION_DAYS="${RETENTION_DAYS:-7}"

# Create log directory
mkdir -p "$(dirname "$LOG_FILE")"

# Logging function
log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

# Error handling
handle_error() {
    log "ERROR: Backup failed at step: $1"
    # Restart GitLab services if they were stopped
    gitlab-ctl start 2>/dev/null || true
    exit 1
}

log "Starting GitLab backup process"

# Create GitLab data backup
log "Creating GitLab data backup"
gitlab-backup create BACKUP=gitlab_$(date +%Y%m%d_%H%M%S) || handle_error "creating data backup"

# Backup GitLab configuration files
log "Backing up GitLab configuration"
mkdir -p "$CONFIG_BACKUP_DIR"
CONFIG_BACKUP_FILE="$CONFIG_BACKUP_DIR/gitlab-config-$(date +%Y%m%d_%H%M%S).tar.gz"

tar -czf "$CONFIG_BACKUP_FILE" \
    /etc/gitlab/gitlab.rb \
    /etc/gitlab/gitlab-secrets.json 2>/dev/null || handle_error "backing up configuration"

# Set proper permissions
chown git:git "$CONFIG_BACKUP_FILE"
chmod 644 "$CONFIG_BACKUP_FILE"

# Clean up old backups
log "Cleaning up old backups (keeping last $RETENTION_DAYS days)"
find "$BACKUP_DIR" -name "*.tar" -type f -mtime +$RETENTION_DAYS -delete 2>/dev/null || true
find "$CONFIG_BACKUP_DIR" -name "*.tar.gz" -type f -mtime +$RETENTION_DAYS -delete 2>/dev/null || true

log "GitLab backup completed successfully"
EOF

    chmod 755 "$SCRIPT_DIR/gitlab-backup.sh"
    chown root:root "$SCRIPT_DIR/gitlab-backup.sh"
    
    log "Backup script created"
}

create_restore_script() {
    log "[7/10] Creating disaster recovery restore script..."
    
    cat > "$SCRIPT_DIR/gitlab-restore.sh" << 'EOF'
#!/usr/bin/env bash

set -euo pipefail

# GitLab Restore Script

if [ $# -ne 1 ]; then
    echo "Usage: $0 <backup_timestamp>"
    echo "Example: $0 20240115_142300"
    echo ""
    echo "Available backups:"
    ls -la /var/opt/gitlab/backups/gitlab_*_gitlab_backup.tar 2>/dev/null | awk '{print $9}' | sed 's/.*gitlab_\(.*\)_gitlab_backup.tar/\1/' || echo "No backups found"
    exit 1
fi

BACKUP_TIMESTAMP="$1"
BACKUP_FILE="/var/opt/gitlab/backups/gitlab_${BACKUP_TIMESTAMP}_gitlab_backup.tar"
CONFIG_BACKUP_DIR="/var/opt/gitlab/config-backups"

if [ ! -f "$BACKUP_FILE" ]; then
    echo "ERROR: Backup file not found: $BACKUP_FILE"
    exit 1
fi

echo "WARNING: This will restore GitLab from backup and overwrite current data!"
echo "Backup file: $BACKUP_FILE"
echo "Press Ctrl+C to cancel or Enter to continue..."
read

echo "Stopping GitLab services..."
gitlab-ctl stop unicorn 2>/dev/null || true
gitlab-ctl stop puma 2>/dev/null || true
gitlab-ctl stop sidekiq

echo "Restoring GitLab from backup..."
gitlab-backup restore BACKUP=$BACKUP_TIMESTAMP

echo "Starting GitLab services..."
gitlab-ctl start

echo "Reconfiguring GitLab..."
gitlab-ctl reconfigure

echo "Restore completed successfully!"
EOF

    chmod 755 "$SCRIPT_DIR/gitlab-restore.sh"
    chown root:root "$SCRIPT_DIR/gitlab-restore.sh"
    
    log "Restore script created"
}

setup_cron_job() {
    log "[8/10] Setting up automated backup schedule..."
    
    # Create cron job for daily backups at 2 AM
    cat > /etc/cron.d/gitlab-backup << EOF
# GitLab automated backup - runs daily at 2 AM
0 2 * * * root $SCRIPT_DIR/gitlab-backup.sh
EOF
    
    chmod 644 /etc/cron.d/gitlab-backup
    
    log "Cron job scheduled for daily backups at 2 AM"
}

apply_gitlab_config() {
    log "[9/10] Applying GitLab configuration changes..."
    
    gitlab-ctl reconfigure
    
    log "GitLab configuration applied"
}

verify_setup() {
    log "[10/10] Verifying backup setup..."
    
    # Check if backup directory exists and has correct permissions
    if [ ! -d "$BACKUP_BASE_DIR" ]; then
        error "Backup directory not created"
        exit 1
    fi
    
    # Check if scripts exist and are executable
    if [ ! -x "$SCRIPT_DIR/gitlab-backup.sh" ] || [ ! -x "$SCRIPT_DIR/gitlab-restore.sh" ]; then
        error "Backup scripts not properly created"
        exit 1
    fi
    
    # Check cron job
    if [ ! -f /etc/cron.d/gitlab-backup ]; then
        error "Cron job not created"
        exit 1
    fi
    
    # Test backup creation (without actually running full backup)
    if ! gitlab-backup create SKIP=uploads,builds,artifacts,lfs,registry,pages DRY_RUN=true &>/dev/null; then
        warn "Backup test failed, but setup is complete. Check GitLab status."
    fi
    
    log "Setup verification completed successfully!"
}

show_summary() {
    echo ""
    echo -e "${BLUE}=== GitLab Backup and Disaster Recovery Setup Complete ===${NC}"
    echo ""
    echo "Backup script location: $SCRIPT_DIR/gitlab-backup.sh"
    echo "Restore script location: $SCRIPT_DIR/gitlab-restore.sh"
    echo "Backup directory: $BACKUP_BASE_DIR"
    echo "Log directory: $LOG_DIR"
    echo ""
    echo "Manual backup: sudo $SCRIPT_DIR/gitlab-backup.sh"
    echo "Restore backup: sudo $SCRIPT_DIR/gitlab-restore.sh <timestamp>"
    echo ""
    echo "Automated backups run daily at 2 AM via cron"
    echo "Backup retention: $RETENTION_DAYS days"
    echo ""
    if [ -n "${S3_BUCKET:-}" ]; then
        echo "S3 bucket configured: $S3_BUCKET"
    else
        echo "Note: Configure AWS credentials for S3 uploads if desired"
    fi
}

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -r|--retention-days)
            RETENTION_DAYS="$2"
            shift 2
            ;;
        -h|--help)
            usage
            ;;
        *)
            error "Unknown option: $1"
            usage
            ;;
    esac
done

# Main execution
main() {
    check_prerequisites
    detect_distro
    install_dependencies
    create_directories
    configure_gitlab_backups
    create_backup_script
    create_restore_script
    setup_cron_job
    apply_gitlab_config
    verify_setup
    show_summary
}

main "$@"