Implement Grafana high availability clustering with PostgreSQL backend and load balancing

#!/usr/bin/env bash
set -euo pipefail

# Grafana High Availability Cluster Installation Script
# Supports: Ubuntu, Debian, AlmaLinux, Rocky Linux, CentOS, RHEL

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Global variables
SCRIPT_NAME=$(basename "$0")
GRAFANA_DB_PASS=""
GRAFANA_ADMIN_PASS=""
PG_PRIMARY_IP=""
NODE_TYPE=""
TOTAL_STEPS=8

print_usage() {
    cat << EOF
Usage: $SCRIPT_NAME [OPTIONS]
Install Grafana HA cluster with PostgreSQL backend

Options:
    -t, --type TYPE         Node type: primary-db, standby-db, or grafana
    -p, --primary IP        PostgreSQL primary server IP
    -d, --db-pass PASS      Grafana database password
    -a, --admin-pass PASS   Grafana admin password
    -h, --help              Show this help

Example:
    $SCRIPT_NAME -t primary-db -d SecurePass123 -a AdminPass123
    $SCRIPT_NAME -t standby-db -p 10.0.1.10 -d SecurePass123
    $SCRIPT_NAME -t grafana -p 10.0.1.10 -d SecurePass123 -a AdminPass123
EOF
}

log() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

error() {
    echo -e "${RED}[ERROR]${NC} $1" >&2
}

cleanup() {
    error "Installation failed. Check logs for details."
    exit 1
}

trap cleanup ERR

# Parse command line arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -t|--type)
            NODE_TYPE="$2"
            shift 2
            ;;
        -p|--primary)
            PG_PRIMARY_IP="$2"
            shift 2
            ;;
        -d|--db-pass)
            GRAFANA_DB_PASS="$2"
            shift 2
            ;;
        -a|--admin-pass)
            GRAFANA_ADMIN_PASS="$2"
            shift 2
            ;;
        -h|--help)
            print_usage
            exit 0
            ;;
        *)
            error "Unknown option: $1"
            print_usage
            exit 1
            ;;
    esac
done

# Validate required parameters
if [[ -z "$NODE_TYPE" || ! "$NODE_TYPE" =~ ^(primary-db|standby-db|grafana)$ ]]; then
    error "Node type (-t) is required: primary-db, standby-db, or grafana"
    print_usage
    exit 1
fi

if [[ "$NODE_TYPE" =~ ^(standby-db|grafana)$ && -z "$PG_PRIMARY_IP" ]]; then
    error "Primary IP (-p) is required for standby-db and grafana nodes"
    print_usage
    exit 1
fi

if [[ -z "$GRAFANA_DB_PASS" ]]; then
    error "Database password (-d) is required"
    print_usage
    exit 1
fi

if [[ "$NODE_TYPE" =~ ^(primary-db|grafana)$ && -z "$GRAFANA_ADMIN_PASS" ]]; then
    error "Admin password (-a) is required for primary-db and grafana nodes"
    print_usage
    exit 1
fi

# Check if running as root
if [[ $EUID -ne 0 ]]; then
    error "This script must be run as root"
    exit 1
fi

# Detect OS and package manager
if [[ -f /etc/os-release ]]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_UPDATE="apt update && apt upgrade -y"
            PKG_INSTALL="apt install -y"
            PG_VERSION="16"
            PG_CONFIG_DIR="/etc/postgresql/16/main"
            PG_DATA_DIR="/var/lib/postgresql/16/main"
            PG_SERVICE="postgresql"
            ;;
        almalinux|rocky|centos|rhel|ol)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            PG_VERSION="16"
            PG_CONFIG_DIR="/var/lib/pgsql/16/data"
            PG_DATA_DIR="/var/lib/pgsql/16/data"
            PG_SERVICE="postgresql-16"
            ;;
        fedora)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            PG_VERSION="16"
            PG_CONFIG_DIR="/var/lib/pgsql/data"
            PG_DATA_DIR="/var/lib/pgsql/data"
            PG_SERVICE="postgresql"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_UPDATE="yum update -y"
            PKG_INSTALL="yum install -y"
            PG_VERSION="16"
            PG_CONFIG_DIR="/var/lib/pgsql/data"
            PG_DATA_DIR="/var/lib/pgsql/data"
            PG_SERVICE="postgresql"
            ;;
        *)
            error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
else
    error "Cannot detect OS distribution"
    exit 1
fi

log "Starting Grafana HA cluster installation on $PRETTY_NAME"
log "Node type: $NODE_TYPE"

# Step 1: Update system packages
echo -e "${GREEN}[1/$TOTAL_STEPS]${NC} Updating system packages..."
$PKG_UPDATE

# Step 2: Install PostgreSQL (for DB nodes)
if [[ "$NODE_TYPE" =~ ^(primary-db|standby-db)$ ]]; then
    echo -e "${GREEN}[2/$TOTAL_STEPS]${NC} Installing PostgreSQL..."
    
    if [[ "$PKG_MGR" == "apt" ]]; then
        $PKG_INSTALL postgresql-16 postgresql-contrib-16
        systemctl enable --now postgresql
    else
        $PKG_INSTALL postgresql16-server postgresql16-contrib
        if [[ ! -f "$PG_DATA_DIR/PG_VERSION" ]]; then
            /usr/pgsql-16/bin/postgresql-16-setup initdb
        fi
        systemctl enable --now postgresql-16
    fi
    
    # Configure PostgreSQL for primary node
    if [[ "$NODE_TYPE" == "primary-db" ]]; then
        echo -e "${GREEN}[3/$TOTAL_STEPS]${NC} Configuring PostgreSQL primary server..."
        
        # Create Grafana database and user
        sudo -u postgres createdb grafana || true
        sudo -u postgres psql -c "CREATE USER grafana WITH PASSWORD '$GRAFANA_DB_PASS';" || true
        sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE grafana TO grafana;"
        
        # Configure PostgreSQL for replication
        cat >> "$PG_CONFIG_DIR/postgresql.conf" << EOF

# Grafana HA Configuration
listen_addresses = '*'
wal_level = replica
max_wal_senders = 10
max_replication_slots = 10
archive_mode = on
archive_command = '/bin/true'
log_replication_commands = on
EOF
        
        cat >> "$PG_CONFIG_DIR/pg_hba.conf" << EOF

# Grafana connections
host    grafana         grafana         0.0.0.0/0               md5
# Replication connections
host    replication     postgres        0.0.0.0/0               md5
EOF
        
        systemctl restart $PG_SERVICE
        
        # Configure firewall
        if command -v firewall-cmd &> /dev/null; then
            firewall-cmd --permanent --add-port=5432/tcp
            firewall-cmd --reload
        elif command -v ufw &> /dev/null; then
            ufw allow 5432/tcp
        fi
    fi
    
    # Configure standby server
    if [[ "$NODE_TYPE" == "standby-db" ]]; then
        echo -e "${GREEN}[3/$TOTAL_STEPS]${NC} Configuring PostgreSQL standby server..."
        
        systemctl stop $PG_SERVICE
        rm -rf "$PG_DATA_DIR"/*
        
        sudo -u postgres pg_basebackup -h "$PG_PRIMARY_IP" -D "$PG_DATA_DIR" -U postgres -v -P -R
        
        chown -R postgres:postgres "$PG_DATA_DIR"
        chmod 750 "$PG_DATA_DIR"
        
        systemctl start $PG_SERVICE
    fi
else
    echo -e "${GREEN}[2/$TOTAL_STEPS]${NC} Skipping PostgreSQL installation (Grafana node)..."
    echo -e "${GREEN}[3/$TOTAL_STEPS]${NC} Skipping PostgreSQL configuration (Grafana node)..."
fi

# Step 4: Install Grafana (for Grafana nodes)
if [[ "$NODE_TYPE" == "grafana" ]]; then
    echo -e "${GREEN}[4/$TOTAL_STEPS]${NC} Installing Grafana Enterprise..."
    
    if [[ "$PKG_MGR" == "apt" ]]; then
        $PKG_INSTALL software-properties-common wget gnupg
        wget -q -O - https://packages.grafana.com/gpg.key | apt-key add -
        echo "deb https://packages.grafana.com/enterprise/deb stable main" > /etc/apt/sources.list.d/grafana.list
        apt update
        $PKG_INSTALL grafana-enterprise
    else
        $PKG_INSTALL wget
        rpm --import https://packages.grafana.com/gpg.key
        cat > /etc/yum.repos.d/grafana.repo << EOF
[grafana]
name=grafana
baseurl=https://packages.grafana.com/enterprise/rpm
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://packages.grafana.com/gpg.key
EOF
        $PKG_INSTALL grafana-enterprise
    fi
    
    echo -e "${GREEN}[5/$TOTAL_STEPS]${NC} Configuring Grafana for high availability..."
    
    # Generate secret key
    SECRET_KEY=$(openssl rand -base64 32)
    
    # Configure Grafana
    cat > /etc/grafana/grafana.ini << EOF
[database]
type = postgres
host = $PG_PRIMARY_IP:5432
name = grafana
user = grafana
password = $GRAFANA_DB_PASS
ssl_mode = require
max_idle_conn = 25
max_open_conn = 300
conn_max_lifetime = 14400

[server]
http_port = 3000
domain = $(hostname -f)
root_url = http://$(hostname -f):3000/

[security]
admin_user = admin
admin_password = $GRAFANA_ADMIN_PASS
secret_key = $SECRET_KEY
disable_gravatar = true

[users]
allow_sign_up = false
allow_org_create = false

[auth.anonymous]
enabled = false

[log]
mode = console file
level = info
EOF
    
    chown root:grafana /etc/grafana/grafana.ini
    chmod 640 /etc/grafana/grafana.ini
    
    systemctl enable --now grafana-server
    
    # Configure firewall for Grafana
    if command -v firewall-cmd &> /dev/null; then
        firewall-cmd --permanent --add-port=3000/tcp
        firewall-cmd --reload
    elif command -v ufw &> /dev/null; then
        ufw allow 3000/tcp
    fi
else
    echo -e "${GREEN}[4/$TOTAL_STEPS]${NC} Skipping Grafana installation (Database node)..."
    echo -e "${GREEN}[5/$TOTAL_STEPS]${NC} Skipping Grafana configuration (Database node)..."
fi

# Step 6: Verify installation
echo -e "${GREEN}[6/$TOTAL_STEPS]${NC} Verifying installation..."

if [[ "$NODE_TYPE" =~ ^(primary-db|standby-db)$ ]]; then
    if systemctl is-active --quiet $PG_SERVICE; then
        log "PostgreSQL is running successfully"
    else
        error "PostgreSQL failed to start"
        exit 1
    fi
fi

if [[ "$NODE_TYPE" == "grafana" ]]; then
    if systemctl is-active --quiet grafana-server; then
        log "Grafana is running successfully"
    else
        error "Grafana failed to start"
        exit 1
    fi
    
    # Test database connection
    if sudo -u grafana grafana-cli admin reset-admin-password "$GRAFANA_ADMIN_PASS" &>/dev/null; then
        log "Database connection verified"
    else
        warn "Database connection could not be verified"
    fi
fi

# Step 7: Display summary
echo -e "${GREEN}[7/$TOTAL_STEPS]${NC} Installation summary..."

cat << EOF

${GREEN}Grafana HA Cluster Node Installation Complete!${NC}

Node Type: $NODE_TYPE
EOF

if [[ "$NODE_TYPE" == "grafana" ]]; then
    cat << EOF
Grafana URL: http://$(hostname -f):3000
Admin User: admin
Admin Password: $GRAFANA_ADMIN_PASS
Database: PostgreSQL at $PG_PRIMARY_IP
EOF
fi

if [[ "$NODE_TYPE" =~ ^(primary-db|standby-db)$ ]]; then
    cat << EOF
PostgreSQL Status: Running
Service: $PG_SERVICE
Data Directory: $PG_DATA_DIR
EOF
fi

# Step 8: Next steps
echo -e "${GREEN}[8/$TOTAL_STEPS]${NC} Next steps..."

if [[ "$NODE_TYPE" == "primary-db" ]]; then
    log "Set up standby database server with: $SCRIPT_NAME -t standby-db -p $(hostname -I | awk '{print $1}') -d $GRAFANA_DB_PASS"
elif [[ "$NODE_TYPE" == "standby-db" ]]; then
    log "Install Grafana nodes with: $SCRIPT_NAME -t grafana -p $PG_PRIMARY_IP -d $GRAFANA_DB_PASS -a <admin_password>"
elif [[ "$NODE_TYPE" == "grafana" ]]; then
    log "Configure load balancer to distribute traffic across Grafana instances"
    log "Access Grafana at http://$(hostname -f):3000"
fi

log "Installation completed successfully!"