Configure Consul multi-datacenter WAN federation for geographic redundancy

#!/usr/bin/env bash

set -euo pipefail

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Default values
DATACENTER=""
PRIMARY_DC=""
NODE_ROLE=""
BIND_ADDR=""
JOIN_ADDRS=""
WAN_JOIN_ADDRS=""

usage() {
    echo "Usage: $0 --datacenter <dc> --primary-dc <primary> --role <server|client> --bind-addr <ip> [options]"
    echo "Options:"
    echo "  --datacenter <dc>        Datacenter name (e.g., dc1, dc2)"
    echo "  --primary-dc <primary>   Primary datacenter name"
    echo "  --role <server|client>   Node role"
    echo "  --bind-addr <ip>         Bind address for this node"
    echo "  --join-addrs <ips>       Comma-separated list of local cluster IPs"
    echo "  --wan-join-addrs <ips>   Comma-separated list of primary DC IPs (for secondary DC)"
    exit 1
}

cleanup() {
    if [ $? -ne 0 ]; then
        echo -e "${RED}[ERROR] Installation failed. Cleaning up...${NC}"
        systemctl stop consul 2>/dev/null || true
        systemctl disable consul 2>/dev/null || true
        rm -f /etc/consul.d/consul.hcl
        userdel consul 2>/dev/null || true
    fi
}

trap cleanup ERR

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --datacenter) DATACENTER="$2"; shift 2 ;;
        --primary-dc) PRIMARY_DC="$2"; shift 2 ;;
        --role) NODE_ROLE="$2"; shift 2 ;;
        --bind-addr) BIND_ADDR="$2"; shift 2 ;;
        --join-addrs) JOIN_ADDRS="$2"; shift 2 ;;
        --wan-join-addrs) WAN_JOIN_ADDRS="$2"; shift 2 ;;
        *) usage ;;
    esac
done

# Validate required arguments
if [[ -z "$DATACENTER" || -z "$PRIMARY_DC" || -z "$NODE_ROLE" || -z "$BIND_ADDR" ]]; then
    usage
fi

if [[ "$NODE_ROLE" != "server" && "$NODE_ROLE" != "client" ]]; then
    echo -e "${RED}[ERROR] Role must be 'server' or 'client'${NC}"
    exit 1
fi

echo -e "${GREEN}[1/10] Checking prerequisites...${NC}"
if [[ $EUID -ne 0 ]]; then
    echo -e "${RED}[ERROR] This script must be run as root${NC}"
    exit 1
fi

# Detect OS
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update"
            FIREWALL_CMD="ufw"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf check-update || true"
            FIREWALL_CMD="firewall-cmd"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum check-update || true"
            FIREWALL_CMD="firewall-cmd"
            ;;
        *)
            echo -e "${RED}[ERROR] Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
else
    echo -e "${RED}[ERROR] Cannot detect OS version${NC}"
    exit 1
fi

echo -e "${GREEN}[2/10] Installing HashiCorp repository...${NC}"
case "$PKG_MGR" in
    apt)
        $PKG_INSTALL wget gpg lsb-release
        wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
        echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" > /etc/apt/sources.list.d/hashicorp.list
        $PKG_UPDATE
        ;;
    dnf|yum)
        $PKG_INSTALL dnf-plugins-core 2>/dev/null || $PKG_INSTALL yum-utils
        if command -v dnf >/dev/null 2>&1; then
            dnf config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
        else
            yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
        fi
        ;;
esac

echo -e "${GREEN}[3/10] Installing Consul...${NC}"
$PKG_INSTALL consul

echo -e "${GREEN}[4/10] Creating consul user and directories...${NC}"
if ! id consul >/dev/null 2>&1; then
    useradd --system --home /etc/consul.d --shell /bin/false consul
fi

mkdir -p /opt/consul /etc/consul.d/tls
chown -R consul:consul /opt/consul /etc/consul.d
chmod 755 /opt/consul /etc/consul.d
chmod 700 /etc/consul.d/tls

echo -e "${GREEN}[5/10] Generating encryption key and certificates...${NC}"
if [[ ! -f /etc/consul.d/encrypt.key ]]; then
    ENCRYPT_KEY=$(consul keygen)
    echo "$ENCRYPT_KEY" > /etc/consul.d/encrypt.key
    chmod 600 /etc/consul.d/encrypt.key
    chown consul:consul /etc/consul.d/encrypt.key
else
    ENCRYPT_KEY=$(cat /etc/consul.d/encrypt.key)
fi

if [[ ! -f /etc/consul.d/tls/consul-agent-ca.pem ]]; then
    cd /etc/consul.d/tls
    consul tls ca create
    consul tls cert create -server -dc "$DATACENTER"
    chown -R consul:consul /etc/consul.d/tls
    chmod 600 /etc/consul.d/tls/*.pem
fi

echo -e "${GREEN}[6/10] Creating Consul configuration...${NC}"
cat > /etc/consul.d/consul.hcl << EOF
datacenter = "$DATACENTER"
primary_datacenter = "$PRIMARY_DC"
data_dir = "/opt/consul"
log_level = "INFO"
server = $([ "$NODE_ROLE" = "server" ] && echo "true" || echo "false")
bind_addr = "$BIND_ADDR"
client_addr = "0.0.0.0"

EOF

if [[ "$NODE_ROLE" = "server" ]]; then
    cat >> /etc/consul.d/consul.hcl << EOF
bootstrap_expect = 3
ui_config {
  enabled = true
}

EOF
fi

if [[ -n "$JOIN_ADDRS" ]]; then
    IFS=',' read -ra ADDR_ARRAY <<< "$JOIN_ADDRS"
    echo "retry_join = [" >> /etc/consul.d/consul.hcl
    for addr in "${ADDR_ARRAY[@]}"; do
        echo "  \"$addr\"," >> /etc/consul.d/consul.hcl
    done
    echo "]" >> /etc/consul.d/consul.hcl
    echo "" >> /etc/consul.d/consul.hcl
fi

if [[ -n "$WAN_JOIN_ADDRS" && "$DATACENTER" != "$PRIMARY_DC" ]]; then
    IFS=',' read -ra WAN_ADDR_ARRAY <<< "$WAN_JOIN_ADDRS"
    echo "retry_join_wan = [" >> /etc/consul.d/consul.hcl
    for addr in "${WAN_ADDR_ARRAY[@]}"; do
        echo "  \"$addr\"," >> /etc/consul.d/consul.hcl
    done
    echo "]" >> /etc/consul.d/consul.hcl
    echo "" >> /etc/consul.d/consul.hcl
fi

cat >> /etc/consul.d/consul.hcl << EOF
connect {
  enabled = true
}

ports {
  grpc = 8502
}

encrypt = "$ENCRYPT_KEY"

tls {
  defaults {
    ca_file = "/etc/consul.d/tls/consul-agent-ca.pem"
    cert_file = "/etc/consul.d/tls/$DATACENTER-server-consul-0.pem"
    key_file = "/etc/consul.d/tls/$DATACENTER-server-consul-0-key.pem"
    verify_incoming = true
    verify_outgoing = true
  }
  internal_rpc {
    verify_server_hostname = true
  }
}

acl = {
  enabled = true
  default_policy = "deny"
  enable_token_persistence = true
EOF

if [[ "$DATACENTER" != "$PRIMARY_DC" ]]; then
    echo "  enable_token_replication = true" >> /etc/consul.d/consul.hcl
fi

echo "}" >> /etc/consul.d/consul.hcl

chown consul:consul /etc/consul.d/consul.hcl
chmod 640 /etc/consul.d/consul.hcl

echo -e "${GREEN}[7/10] Creating systemd service...${NC}"
cat > /etc/systemd/system/consul.service << EOF
[Unit]
Description=Consul
Documentation=https://www.consul.io/
Requires=network-online.target
After=network-online.target
ConditionFileNotEmpty=/etc/consul.d/consul.hcl

[Service]
Type=notify
User=consul
Group=consul
ExecStart=/usr/bin/consul agent -config-dir=/etc/consul.d/
ExecReload=/bin/kill -HUP \$MAINPID
KillMode=process
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable consul

echo -e "${GREEN}[8/10] Configuring firewall...${NC}"
case "$FIREWALL_CMD" in
    ufw)
        if systemctl is-active --quiet ufw; then
            ufw allow 8300/tcp
            ufw allow 8301/tcp
            ufw allow 8301/udp
            ufw allow 8302/tcp
            ufw allow 8302/udp
            ufw allow 8500/tcp
            ufw allow 8502/tcp
            ufw allow 8600/tcp
            ufw allow 8600/udp
        fi
        ;;
    firewall-cmd)
        if systemctl is-active --quiet firewalld; then
            firewall-cmd --permanent --add-port=8300/tcp
            firewall-cmd --permanent --add-port=8301/tcp
            firewall-cmd --permanent --add-port=8301/udp
            firewall-cmd --permanent --add-port=8302/tcp
            firewall-cmd --permanent --add-port=8302/udp
            firewall-cmd --permanent --add-port=8500/tcp
            firewall-cmd --permanent --add-port=8502/tcp
            firewall-cmd --permanent --add-port=8600/tcp
            firewall-cmd --permanent --add-port=8600/udp
            firewall-cmd --reload
        fi
        ;;
esac

echo -e "${GREEN}[9/10] Starting Consul service...${NC}"
systemctl start consul

echo -e "${GREEN}[10/10] Verifying installation...${NC}"
sleep 5

if systemctl is-active --quiet consul; then
    echo -e "${GREEN}✓ Consul service is running${NC}"
else
    echo -e "${RED}✗ Consul service failed to start${NC}"
    exit 1
fi

if consul members >/dev/null 2>&1; then
    echo -e "${GREEN}✓ Consul cluster is accessible${NC}"
else
    echo -e "${YELLOW}⚠ Consul cluster not yet accessible (this is normal for initial setup)${NC}"
fi

echo -e "${GREEN}Installation completed successfully!${NC}"
echo -e "${YELLOW}Next steps:${NC}"
echo "1. Copy the encryption key to all nodes: $(cat /etc/consul.d/encrypt.key)"
echo "2. Copy TLS certificates to other nodes"
echo "3. Bootstrap ACL system: consul acl bootstrap"
echo "4. Access UI at: https://$BIND_ADDR:8500"