Configure AWX backup and disaster recovery procedures with automated PostgreSQL snapshots and restoration

Identify AWX components to backup

AWX stores critical data in multiple locations that require different backup strategies.

sudo docker exec awx_postgres pg_dump -U awx awx > /dev/null
sudo ls -la /var/lib/docker/volumes/awx_projects/_data/
sudo ls -la /var/lib/docker/volumes/awx_redis/_data/

Create backup directory structure

Organize backup locations with proper permissions and retention policies.

sudo mkdir -p /opt/awx-backup/{database,projects,configs,logs}
sudo mkdir -p /opt/awx-backup/database/{daily,weekly,monthly}
sudo chown -R $(whoami):$(whoami) /opt/awx-backup
chmod -R 750 /opt/awx-backup

Install backup dependencies

Install PostgreSQL client tools and compression utilities for efficient backups.

sudo apt update
sudo apt install -y postgresql-client-15 gzip rsync gpg

sudo dnf install -y postgresql gzip rsync gnupg2

Create PostgreSQL backup script

This script creates compressed database dumps with timestamps and implements retention policies.

#!/bin/bash
set -euo pipefail

Configuration
BACKUP_DIR="/opt/awx-backup/database"
PG_CONTAINER="awx_postgres"
DB_NAME="awx"
DB_USER="awx"
DATE=$(date +%Y%m%d_%H%M%S)
DAILY_RETENTION=7
WEEKLY_RETENTION=4
MONTHLY_RETENTION=12

Create timestamp-based backup
echo "Starting AWX database backup at $(date)"
docker exec $PG_CONTAINER pg_dump -U $DB_USER -d $DB_NAME --verbose --no-password | gzip > "$BACKUP_DIR/daily/awx_db_$DATE.sql.gz"

Verify backup integrity
if [ -f "$BACKUP_DIR/daily/awx_db_$DATE.sql.gz" ] && [ -s "$BACKUP_DIR/daily/awx_db_$DATE.sql.gz" ]; then
    echo "Database backup completed successfully: awx_db_$DATE.sql.gz"
    
    # Create checksums
    cd "$BACKUP_DIR/daily"
    sha256sum "awx_db_$DATE.sql.gz" > "awx_db_$DATE.sql.gz.sha256"
else
    echo "ERROR: Database backup failed or is empty"
    exit 1
fi

Weekly backup (Sundays)
if [ $(date +%u) -eq 7 ]; then
    cp "$BACKUP_DIR/daily/awx_db_$DATE.sql.gz" "$BACKUP_DIR/weekly/awx_db_weekly_$DATE.sql.gz"
    cp "$BACKUP_DIR/daily/awx_db_$DATE.sql.gz.sha256" "$BACKUP_DIR/weekly/awx_db_weekly_$DATE.sql.gz.sha256"
fi

Monthly backup (1st of month)
if [ $(date +%d) -eq 01 ]; then
    cp "$BACKUP_DIR/daily/awx_db_$DATE.sql.gz" "$BACKUP_DIR/monthly/awx_db_monthly_$DATE.sql.gz"
    cp "$BACKUP_DIR/daily/awx_db_$DATE.sql.gz.sha256" "$BACKUP_DIR/monthly/awx_db_monthly_$DATE.sql.gz.sha256"
fi

Cleanup old backups
find "$BACKUP_DIR/daily" -name "awx_db_*.sql.gz" -mtime +$DAILY_RETENTION -delete
find "$BACKUP_DIR/daily" -name "awx_db_*.sql.gz.sha256" -mtime +$DAILY_RETENTION -delete
find "$BACKUP_DIR/weekly" -name "awx_db_weekly_.sql.gz" -mtime +$((WEEKLY_RETENTION  7)) -delete
find "$BACKUP_DIR/weekly" -name "awx_db_weekly_.sql.gz.sha256" -mtime +$((WEEKLY_RETENTION  7)) -delete
find "$BACKUP_DIR/monthly" -name "awx_db_monthly_.sql.gz" -mtime +$((MONTHLY_RETENTION  30)) -delete
find "$BACKUP_DIR/monthly" -name "awx_db_monthly_.sql.gz.sha256" -mtime +$((MONTHLY_RETENTION  30)) -delete

echo "AWX database backup completed at $(date)"

Make backup script executable

Set proper permissions and test the backup script functionality.

chmod +x /opt/awx-backup/backup-awx-db.sh
sudo /opt/awx-backup/backup-awx-db.sh

Configure automated scheduling

Schedule daily database backups using systemd timers for reliable execution.

[Unit]
Description=AWX Database Backup Service
Wants=awx-backup.timer

[Service]
Type=oneshot
User=root
ExecStart=/opt/awx-backup/backup-awx-db.sh
StandardOutput=journal
StandardError=journal

Create systemd timer

Configure the backup timer to run daily at 2 AM with randomized delays to avoid system load spikes.

[Unit]
Description=AWX Database Backup Timer
Requires=awx-backup.service

[Timer]
OnCalendar=--* 02:00:00
RandomizedDelaySec=300
Persistent=true

[Install]
WantedBy=timers.target

Enable and start backup timer

Activate the systemd timer and verify it's scheduled correctly.

sudo systemctl daemon-reload
sudo systemctl enable awx-backup.timer
sudo systemctl start awx-backup.timer
sudo systemctl status awx-backup.timer

Create comprehensive backup script

This script backs up AWX projects, inventories, and configuration files in addition to the database.

#!/bin/bash
set -euo pipefail

Configuration
BACKUP_BASE="/opt/awx-backup"
DATE=$(date +%Y%m%d_%H%M%S)
AWX_PROJECTS_VOLUME="awx_projects"
AWX_CONFIG_DIR="/var/lib/docker/volumes"
LOG_FILE="$BACKUP_BASE/logs/backup_$DATE.log"

Start logging
exec 1> >(tee -a "$LOG_FILE")
exec 2> >(tee -a "$LOG_FILE" >&2)

echo "Starting full AWX backup at $(date)"

Backup database first
echo "Backing up PostgreSQL database..."
/opt/awx-backup/backup-awx-db.sh

Backup AWX projects
echo "Backing up AWX projects and playbooks..."
if docker volume inspect $AWX_PROJECTS_VOLUME >/dev/null 2>&1; then
    PROJECT_MOUNT=$(docker volume inspect $AWX_PROJECTS_VOLUME | jq -r '.[0].Mountpoint')
    tar -czf "$BACKUP_BASE/projects/awx_projects_$DATE.tar.gz" -C "$PROJECT_MOUNT" .
    echo "Projects backup completed: awx_projects_$DATE.tar.gz"
else
    echo "WARNING: AWX projects volume not found"
fi

Backup AWX configuration
echo "Backing up AWX configuration..."
tar -czf "$BACKUP_BASE/configs/awx_config_$DATE.tar.gz" \
    --exclude='*.sock' \
    --exclude='*.pid' \
    --exclude='*.log' \
    -C /var/lib/docker/volumes/ awx_redis awx_receptor

Backup Docker Compose configuration if exists
if [ -f "/opt/awx/docker-compose.yml" ]; then
    cp "/opt/awx/docker-compose.yml" "$BACKUP_BASE/configs/docker-compose_$DATE.yml"
fi

if [ -f "/opt/awx/.env" ]; then
    cp "/opt/awx/.env" "$BACKUP_BASE/configs/env_$DATE.txt"
fi

Create backup manifest
echo "Creating backup manifest..."
cat > "$BACKUP_BASE/configs/backup_manifest_$DATE.txt" << EOF
AWX Full Backup Manifest
Date: $(date)
Backup ID: $DATE

Database Backup:
File: awx_db_$DATE.sql.gz
Size: $(du -h $BACKUP_BASE/database/daily/awx_db_$DATE.sql.gz | cut -f1)
Checksum: $(cat $BACKUP_BASE/database/daily/awx_db_$DATE.sql.gz.sha256 | cut -d' ' -f1)
Projects Backup:
File: awx_projects_$DATE.tar.gz
Size: $(du -h $BACKUP_BASE/projects/awx_projects_$DATE.tar.gz 2>/dev/null | cut -f1 || echo "N/A")
Configuration Backup:
File: awx_config_$DATE.tar.gz
Size: $(du -h $BACKUP_BASE/configs/awx_config_$DATE.tar.gz | cut -f1)
Total Backup Size: $(du -sh $BACKUP_BASE | cut -f1)
EOF

Cleanup old project and config backups (keep 30 days)
find "$BACKUP_BASE/projects" -name "awx_projects_*.tar.gz" -mtime +30 -delete
find "$BACKUP_BASE/configs" -name "awx_config_*.tar.gz" -mtime +30 -delete
find "$BACKUP_BASE/configs" -name "backup_manifest_*.txt" -mtime +30 -delete
find "$BACKUP_BASE/logs" -name "backup_*.log" -mtime +30 -delete

echo "Full AWX backup completed at $(date)"
echo "Backup manifest: $BACKUP_BASE/configs/backup_manifest_$DATE.txt"

Install jq for JSON parsing

Install jq utility needed by the backup script to parse Docker volume information.

sudo apt install -y jq

sudo dnf install -y jq

Make full backup script executable and test

Set permissions and run a test backup to verify all components work correctly.

chmod +x /opt/awx-backup/backup-awx-full.sh
sudo /opt/awx-backup/backup-awx-full.sh

Configure weekly full backup

Schedule comprehensive backups weekly to capture all AWX components.

[Unit]
Description=AWX Full Backup Service
Wants=awx-full-backup.timer

[Service]
Type=oneshot
User=root
ExecStart=/opt/awx-backup/backup-awx-full.sh
StandardOutput=journal
StandardError=journal

Create weekly backup timer

Schedule full backups every Sunday at 1 AM to avoid conflicts with daily database backups.

[Unit]
Description=AWX Full Backup Timer
Requires=awx-full-backup.service

[Timer]
OnCalendar=Sun --* 01:00:00
RandomizedDelaySec=600
Persistent=true

[Install]
WantedBy=timers.target

Enable weekly backup timer

Activate the weekly full backup schedule.

sudo systemctl daemon-reload
sudo systemctl enable awx-full-backup.timer
sudo systemctl start awx-full-backup.timer
sudo systemctl list-timers | grep awx

Create disaster recovery script

This script automates the restoration process for testing disaster recovery procedures.

#!/bin/bash
set -euo pipefail

Configuration
BACKUP_BASE="/opt/awx-backup"
RESTORE_LOG="/opt/awx-backup/logs/restore_$(date +%Y%m%d_%H%M%S).log"

Function to show usage
show_usage() {
    echo "Usage: $0 [backup_date] [--dry-run]"
    echo "Example: $0 20241201_140000"
    echo "Available backups:"
    ls -1 $BACKUP_BASE/database/daily/awx_db_*.sql.gz | head -10
    exit 1
}

Parse arguments
BACKUP_DATE="$1"
DRY_RUN="${2:-}"

if [ -z "$BACKUP_DATE" ]; then
    show_usage
fi

Start logging
exec 1> >(tee -a "$RESTORE_LOG")
exec 2> >(tee -a "$RESTORE_LOG" >&2)

echo "Starting AWX disaster recovery for backup: $BACKUP_DATE"
echo "Dry run mode: $([ "$DRY_RUN" = "--dry-run" ] && echo "YES" || echo "NO")"

Verify backup files exist
DB_BACKUP="$BACKUP_BASE/database/daily/awx_db_$BACKUP_DATE.sql.gz"
PROJECTS_BACKUP="$BACKUP_BASE/projects/awx_projects_$BACKUP_DATE.tar.gz"
CONFIG_BACKUP="$BACKUP_BASE/configs/awx_config_$BACKUP_DATE.tar.gz"
CHECKSUM_FILE="$BACKUP_BASE/database/daily/awx_db_$BACKUP_DATE.sql.gz.sha256"

if [ ! -f "$DB_BACKUP" ]; then
    echo "ERROR: Database backup not found: $DB_BACKUP"
    exit 1
fi

echo "Verifying backup integrity..."
if [ -f "$CHECKSUM_FILE" ]; then
    cd "$(dirname "$DB_BACKUP")"
    if sha256sum -c "$(basename "$CHECKSUM_FILE")"; then
        echo "Database backup checksum verified"
    else
        echo "ERROR: Database backup checksum verification failed"
        exit 1
    fi
fi

if [ "$DRY_RUN" = "--dry-run" ]; then
    echo "DRY RUN - Would perform the following actions:"
    echo "1. Stop AWX containers"
    echo "2. Restore database from: $DB_BACKUP"
    [ -f "$PROJECTS_BACKUP" ] && echo "3. Restore projects from: $PROJECTS_BACKUP"
    [ -f "$CONFIG_BACKUP" ] && echo "4. Restore configuration from: $CONFIG_BACKUP"
    echo "5. Start AWX containers"
    echo "6. Verify AWX functionality"
    exit 0
fi

Stop AWX services
echo "Stopping AWX containers..."
cd /opt/awx
docker-compose down

Restore database
echo "Restoring AWX database..."
docker-compose up -d postgres
sleep 10

Drop and recreate database
docker exec awx_postgres psql -U postgres -c "DROP DATABASE IF EXISTS awx;"
docker exec awx_postgres psql -U postgres -c "CREATE DATABASE awx OWNER awx;"

Restore database content
zcat "$DB_BACKUP" | docker exec -i awx_postgres psql -U awx -d awx

echo "Database restoration completed"

Restore projects if available
if [ -f "$PROJECTS_BACKUP" ]; then
    echo "Restoring AWX projects..."
    PROJECT_MOUNT=$(docker volume inspect awx_projects | jq -r '.[0].Mountpoint')
    sudo rm -rf "$PROJECT_MOUNT"/*
    sudo tar -xzf "$PROJECTS_BACKUP" -C "$PROJECT_MOUNT"
    echo "Projects restoration completed"
fi

Restore configuration if available
if [ -f "$CONFIG_BACKUP" ]; then
    echo "Restoring AWX configuration..."
    sudo tar -xzf "$CONFIG_BACKUP" -C /var/lib/docker/volumes/
    echo "Configuration restoration completed"
fi

Start all AWX services
echo "Starting AWX services..."
docker-compose up -d

Wait for services to be ready
echo "Waiting for AWX to become available..."
sleep 30

for i in {1..12}; do
    if curl -k -s https://localhost/api/v2/ping/ | grep -q '"version"'; then
        echo "AWX is responding to API requests"
        break
    fi
    echo "Waiting for AWX... ($i/12)"
    sleep 10
done

echo "AWX disaster recovery completed at $(date)"
echo "Restoration log: $RESTORE_LOG"
echo "Please verify AWX functionality manually"

Make restore script executable

Set proper permissions for the disaster recovery script.

chmod +x /opt/awx-backup/restore-awx.sh

Test disaster recovery procedure

Perform a dry-run test to validate the restoration process without making changes.

# Find a recent backup date
ls -la /opt/awx-backup/database/daily/

Test with dry-run (replace with actual backup date)
sudo /opt/awx-backup/restore-awx.sh 20241201_020000 --dry-run

Create backup monitoring script

Monitor backup health and send alerts if backups fail or become outdated.

#!/bin/bash
set -euo pipefail

Configuration
BACKUP_BASE="/opt/awx-backup"
MAX_AGE_HOURS=25  # Alert if no backup in 25 hours
ALERT_EMAIL="admin@example.com"  # Configure your email

Check if recent backup exists
LATEST_BACKUP=$(find "$BACKUP_BASE/database/daily" -name "awx_db_*.sql.gz" -mtime -1 | sort | tail -1)

if [ -z "$LATEST_BACKUP" ]; then
    echo "CRITICAL: No AWX database backup found in the last 24 hours"
    # Send email alert (configure mail server)
    # echo "No AWX backup in 24 hours" | mail -s "AWX Backup Alert" $ALERT_EMAIL
    exit 1
fi

Check backup file size (should be > 1MB for a typical AWX database)
BACKUP_SIZE=$(stat -c%s "$LATEST_BACKUP")
if [ "$BACKUP_SIZE" -lt 1048576 ]; then
    echo "WARNING: Latest backup file is suspiciously small: $(du -h "$LATEST_BACKUP")"
    exit 1
fi

Check backup age
BACKUP_AGE=$(stat -c%Y "$LATEST_BACKUP")
CURRENT_TIME=$(date +%s)
AGE_HOURS=$(( (CURRENT_TIME - BACKUP_AGE) / 3600 ))

if [ "$AGE_HOURS" -gt "$MAX_AGE_HOURS" ]; then
    echo "WARNING: Latest backup is $AGE_HOURS hours old"
    exit 1
fi

echo "OK: Latest backup is $(basename "$LATEST_BACKUP"), size $(du -h "$LATEST_BACKUP" | cut -f1), $AGE_HOURS hours old"
echo "Backup directory usage: $(du -sh $BACKUP_BASE | cut -f1)"

Show backup counts by type
echo "Backup inventory:"
echo "- Daily: $(find "$BACKUP_BASE/database/daily" -name "*.sql.gz" | wc -l) backups"
echo "- Weekly: $(find "$BACKUP_BASE/database/weekly" -name "*.sql.gz" | wc -l) backups"
echo "- Monthly: $(find "$BACKUP_BASE/database/monthly" -name "*.sql.gz" | wc -l) backups"
echo "- Projects: $(find "$BACKUP_BASE/projects" -name "*.tar.gz" | wc -l) backups"

Make monitoring script executable and schedule

Set up automated backup monitoring to catch issues early.

chmod +x /opt/awx-backup/monitor-backups.sh

Test the monitoring script
sudo /opt/awx-backup/monitor-backups.sh

Add monitoring to cron

Schedule backup monitoring to run every 6 hours for proactive alerting.

echo "0 /6    /opt/awx-backup/monitor-backups.sh > /var/log/awx-backup-monitor.log 2>&1" | sudo crontab -