Set up GitLab backup and disaster recovery with automated restoration

Update system packages

Start by updating your package manager to ensure you get the latest versions of backup tools.

sudo apt update && sudo apt upgrade -y
sudo apt install -y rsync gpg awscli

sudo dnf update -y
sudo dnf install -y rsync gnupg2 awscli

Configure GitLab backup settings

Configure GitLab's built-in backup system with retention policies and custom backup paths.

gitlab_rails['backup_path'] = '/var/opt/gitlab/backups'
gitlab_rails['backup_keep_time'] = 604800
gitlab_rails['backup_gitaly_backup_path'] = '/opt/gitlab/embedded/bin/gitaly-backup'
gitlab_rails['backup_archive_permissions'] = 0644
gitlab_rails['backup_pg_schema'] = 'public'
gitlab_rails['backup_upload_connection'] = {
  'provider' => 'AWS',
  'region' => 'eu-west-1',
  'aws_access_key_id' => 'your-access-key',
  'aws_secret_access_key' => 'your-secret-key'
}
gitlab_rails['backup_upload_remote_directory'] = 'gitlab-backups'
gitlab_rails['backup_multipart_chunk_size'] = 104857600
gitlab_rails['backup_encryption'] = 'AES256'

Create backup directories and set permissions

Create the backup directory structure with proper ownership and permissions for GitLab.

sudo mkdir -p /var/opt/gitlab/backups
sudo mkdir -p /etc/gitlab/backup-scripts
sudo mkdir -p /var/log/gitlab-backup
sudo chown -R git:git /var/opt/gitlab/backups
sudo chmod 755 /var/opt/gitlab/backups
sudo chown -R root:root /etc/gitlab/backup-scripts
sudo chmod 755 /etc/gitlab/backup-scripts

Create automated backup script

Create a comprehensive backup script that handles GitLab data, configuration files, and SSL certificates.

#!/bin/bash

GitLab Automated Backup Script
Version: 1.0

set -euo pipefail

Configuration
BACKUP_DIR="/var/opt/gitlab/backups"
CONFIG_BACKUP_DIR="/var/opt/gitlab/config-backups"
LOG_FILE="/var/log/gitlab-backup/backup.log"
RETENTION_DAYS=7
S3_BUCKET="gitlab-backups"
ENCRYPTION_KEY="/etc/gitlab/backup-encryption.key"
SLACK_WEBHOOK="https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK"

Create directories
mkdir -p "$CONFIG_BACKUP_DIR"
mkdir -p "$(dirname "$LOG_FILE")"

Logging function
log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

Error handling
handle_error() {
    log "ERROR: Backup failed at step: $1"
    curl -X POST -H 'Content-type: application/json' \
        --data '{"text":"🔴 GitLab backup failed: '"$1"'"}' \
        "$SLACK_WEBHOOK" 2>/dev/null || true
    exit 1
}

Start backup process
log "Starting GitLab backup process"

Stop GitLab services temporarily for consistent backup
log "Stopping GitLab services for consistent backup"
sudo gitlab-ctl stop unicorn || handle_error "stopping unicorn"
sudo gitlab-ctl stop puma || handle_error "stopping puma"
sudo gitlab-ctl stop sidekiq || handle_error "stopping sidekiq"

Create GitLab data backup
log "Creating GitLab data backup"
sudo gitlab-backup create BACKUP=gitlab_$(date +%Y%m%d_%H%M%S) || handle_error "creating data backup"

Backup GitLab configuration files
log "Backing up GitLab configuration"
CONFIG_BACKUP_FILE="$CONFIG_BACKUP_DIR/gitlab-config-$(date +%Y%m%d_%H%M%S).tar.gz"
sudo tar -czf "$CONFIG_BACKUP_FILE" \
    /etc/gitlab/gitlab.rb \
    /etc/gitlab/gitlab-secrets.json \
    /var/opt/gitlab/nginx/conf \
    /etc/ssl/certs/gitlab* \
    /etc/ssl/private/gitlab* 2>/dev/null || true

Encrypt backups
log "Encrypting backups"
for backup_file in "$BACKUP_DIR"/*.tar; do
    if [[ -f "$backup_file" && ! -f "${backup_file}.gpg" ]]; then
        gpg --cipher-algo AES256 --compress-algo 1 --s2k-cipher-algo AES256 \
            --s2k-digest-algo SHA512 --s2k-mode 3 --s2k-count 65536 \
            --symmetric --output "${backup_file}.gpg" "$backup_file" < "$ENCRYPTION_KEY"
        rm "$backup_file"
        log "Encrypted backup: $(basename "${backup_file}.gpg")"
    fi
done

Upload to S3
log "Uploading backups to S3"
aws s3 sync "$BACKUP_DIR" "s3://$S3_BUCKET/data/" --exclude "" --include ".gpg"
aws s3 sync "$CONFIG_BACKUP_DIR" "s3://$S3_BUCKET/config/"

Clean up old backups
log "Cleaning up old local backups"
find "$BACKUP_DIR" -name "*.gpg" -mtime +$RETENTION_DAYS -delete
find "$CONFIG_BACKUP_DIR" -name "*.tar.gz" -mtime +$RETENTION_DAYS -delete

Start GitLab services
log "Starting GitLab services"
sudo gitlab-ctl start || handle_error "starting services"

Verify backup integrity
log "Verifying backup integrity"
latest_backup=$(ls -t "$BACKUP_DIR"/*.gpg 2>/dev/null | head -1 || echo "")
if [[ -n "$latest_backup" ]]; then
    gpg --list-packets "$latest_backup" >/dev/null 2>&1 || handle_error "backup integrity check"
    log "Backup integrity verified: $(basename "$latest_backup")"
else
    handle_error "no backup files found"
fi

Send success notification
curl -X POST -H 'Content-type: application/json' \
    --data '{"text":"✅ GitLab backup completed successfully"}' \
    "$SLACK_WEBHOOK" 2>/dev/null || true

log "GitLab backup process completed successfully"

Create disaster recovery restoration script

Create an automated restoration script for disaster recovery scenarios.

#!/bin/bash

GitLab Automated Restoration Script
Version: 1.0

set -euo pipefail

Configuration
BACKUP_DIR="/var/opt/gitlab/backups"
CONFIG_BACKUP_DIR="/var/opt/gitlab/config-backups"
LOG_FILE="/var/log/gitlab-backup/restore.log"
S3_BUCKET="gitlab-backups"
ENCRYPTION_KEY="/etc/gitlab/backup-encryption.key"

Logging function
log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

Error handling
handle_error() {
    log "ERROR: Restoration failed at step: $1"
    exit 1
}

Check if backup timestamp provided
if [[ $# -eq 0 ]]; then
    log "Usage: $0 "
    log "Available backups:"
    aws s3 ls "s3://$S3_BUCKET/data/" | grep ".gpg" | awk '{print $4}' | sed 's/gitlab_//g' | sed 's/_.*\.tar\.gpg//g' | sort -u
    exit 1
fi

BACKUP_TIMESTAMP=$1
BACKUP_FILE="gitlab_${BACKUP_TIMESTAMP}_gitlab_backup.tar"

log "Starting GitLab restoration process for backup: $BACKUP_TIMESTAMP"

Download backup from S3
log "Downloading backup from S3"
aws s3 cp "s3://$S3_BUCKET/data/${BACKUP_FILE}.gpg" "$BACKUP_DIR/${BACKUP_FILE}.gpg" || handle_error "downloading backup"

Download latest config backup
log "Downloading configuration backup"
latest_config=$(aws s3 ls "s3://$S3_BUCKET/config/" | sort | tail -n 1 | awk '{print $4}')
aws s3 cp "s3://$S3_BUCKET/config/$latest_config" "$CONFIG_BACKUP_DIR/$latest_config" || handle_error "downloading config"

Decrypt backup
log "Decrypting backup file"
gpg --batch --yes --passphrase-file "$ENCRYPTION_KEY" \
    --output "$BACKUP_DIR/$BACKUP_FILE" \
    --decrypt "$BACKUP_DIR/${BACKUP_FILE}.gpg" || handle_error "decrypting backup"

Stop GitLab services
log "Stopping GitLab services"
sudo gitlab-ctl stop || handle_error "stopping GitLab services"

Restore configuration files
log "Restoring GitLab configuration"
sudo tar -xzf "$CONFIG_BACKUP_DIR/$latest_config" -C / || handle_error "restoring configuration"

Reconfigure GitLab with restored config
log "Reconfiguring GitLab"
sudo gitlab-ctl reconfigure || handle_error "reconfiguring GitLab"

Restore GitLab data
log "Restoring GitLab data"
BACKUP_NAME=$(basename "$BACKUP_FILE" .tar)
sudo gitlab-backup restore BACKUP="$BACKUP_NAME" || handle_error "restoring data"

Start GitLab services
log "Starting GitLab services"
sudo gitlab-ctl start || handle_error "starting GitLab services"

Run GitLab check
log "Running GitLab system check"
sudo gitlab-rake gitlab:check SANITIZE=true || handle_error "GitLab system check"

Clean up temporary files
rm -f "$BACKUP_DIR/${BACKUP_FILE}.gpg" "$BACKUP_DIR/$BACKUP_FILE"

log "GitLab restoration completed successfully for backup: $BACKUP_TIMESTAMP"

Set up backup encryption

Create an encryption key for securing backups at rest and in transit.

sudo openssl rand -base64 32 > /etc/gitlab/backup-encryption.key
sudo chmod 600 /etc/gitlab/backup-encryption.key
sudo chown root:root /etc/gitlab/backup-encryption.key

Make backup scripts executable

Set proper permissions for the backup and restoration scripts.

sudo chmod +x /etc/gitlab/backup-scripts/gitlab-backup.sh
sudo chmod +x /etc/gitlab/backup-scripts/gitlab-restore.sh
sudo chown root:root /etc/gitlab/backup-scripts/*.sh

Configure automated backup scheduling

Set up systemd timers for automated daily backups with proper service definition.

[Unit]
Description=GitLab Backup Service
After=network.target
Wants=network-online.target

[Service]
Type=oneshot
User=root
ExecStart=/etc/gitlab/backup-scripts/gitlab-backup.sh
TimeoutSec=7200
StandardOutput=journal
StandardError=journal

Create backup timer

Configure the systemd timer to run backups daily at 2 AM.

[Unit]
Description=GitLab Backup Timer
Requires=gitlab-backup.service

[Timer]
OnCalendar=daily
Persistent=true
RandomizedDelaySec=300
AccuracySec=1min

[Install]
WantedBy=timers.target

Enable and start backup automation

Enable the systemd timer and reload GitLab configuration with backup settings.

sudo systemctl daemon-reload
sudo systemctl enable gitlab-backup.timer
sudo systemctl start gitlab-backup.timer
sudo gitlab-ctl reconfigure

Set up backup monitoring script

Create a monitoring script that checks backup health and sends alerts.

#!/bin/bash

GitLab Backup Monitoring Script

set -euo pipefail

BACKUP_DIR="/var/opt/gitlab/backups"
LOG_FILE="/var/log/gitlab-backup/monitor.log"
SLACK_WEBHOOK="https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK"
MAX_AGE_HOURS=26

Logging function
log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

Check latest backup age
latest_backup=$(find "$BACKUP_DIR" -name "*.gpg" -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -d' ' -f2-)

if [[ -z "$latest_backup" ]]; then
    log "WARNING: No backup files found"
    curl -X POST -H 'Content-type: application/json' \
        --data '{"text":"⚠️  GitLab backup monitoring: No backup files found"}' \
        "$SLACK_WEBHOOK" 2>/dev/null || true
    exit 1
fi

Calculate backup age
backup_time=$(stat -c %Y "$latest_backup")
current_time=$(date +%s)
age_hours=$(( (current_time - backup_time) / 3600 ))

log "Latest backup: $(basename "$latest_backup"), Age: ${age_hours} hours"

if [[ $age_hours -gt $MAX_AGE_HOURS ]]; then
    log "ALERT: Backup is older than $MAX_AGE_HOURS hours"
    curl -X POST -H 'Content-type: application/json' \
        --data '{"text":"🔴 GitLab backup is '"$age_hours"' hours old (max: '"$MAX_AGE_HOURS"')"}' \
        "$SLACK_WEBHOOK" 2>/dev/null || true
    exit 1
fi

Test backup integrity
if gpg --list-packets "$latest_backup" >/dev/null 2>&1; then
    log "Backup integrity check passed"
else
    log "ALERT: Backup integrity check failed"
    curl -X POST -H 'Content-type: application/json' \
        --data '{"text":"🔴 GitLab backup integrity check failed"}' \
        "$SLACK_WEBHOOK" 2>/dev/null || true
    exit 1
fi

Check disk space
available_space=$(df "$BACKUP_DIR" | awk 'NR==2 {print $4}')
if [[ $available_space -lt 10485760 ]]; then  # 10GB in KB
    log "ALERT: Low disk space in backup directory"
    curl -X POST -H 'Content-type: application/json' \
        --data '{"text":"⚠️  GitLab backup directory has less than 10GB free space"}' \
        "$SLACK_WEBHOOK" 2>/dev/null || true
fi

log "Backup monitoring completed successfully"

Set up monitoring automation

Create systemd service and timer for backup monitoring every 4 hours.

sudo chmod +x /etc/gitlab/backup-scripts/backup-monitor.sh

Create monitoring service
sudo tee /etc/systemd/system/gitlab-backup-monitor.service > /dev/null <Create monitoring timer
sudo tee /etc/systemd/system/gitlab-backup-monitor.timer > /dev/null <--* 00,06,12,18:30:00
Persistent=true
RandomizedDelaySec=300

[Install]
WantedBy=timers.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable gitlab-backup-monitor.timer
sudo systemctl start gitlab-backup-monitor.timer

Create disaster recovery documentation

Document the disaster recovery procedures for your team.

# GitLab Disaster Recovery Procedures

Quick Recovery Steps

Identify backup timestamp to restore:
   
aws s3 ls s3://gitlab-backups/data/ | grep .gpg

Run restoration script:
   
sudo /etc/gitlab/backup-scripts/gitlab-restore.sh YYYYMMDD_HHMMSS

Verify restoration:
   
sudo gitlab-rake gitlab:check

Emergency Contacts
Primary Admin: admin@example.com
Secondary Admin: backup-admin@example.com
Cloud Provider Support: +1-xxx-xxx-xxxx
Recovery Time Objectives
RTO: 4 hours
RPO: 24 hours
Infrastructure Requirements
Minimum 4 CPU cores
8GB RAM
100GB storage
Ubuntu 24.04/Debian 12/AlmaLinux 9/Rocky Linux 9
Testing Schedule
Monthly restoration tests
Quarterly full disaster recovery drills