Install and configure Nomad for container orchestration with Consul integration

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Configuration variables
DATACENTER="${1:-dc1}"
BIND_IP="${2:-0.0.0.0}"

# Global variables
CONSUL_ENCRYPT_KEY=""
NOMAD_ENCRYPT_KEY=""

usage() {
    echo "Usage: $0 [datacenter] [bind_ip]"
    echo "  datacenter: Nomad/Consul datacenter name (default: dc1)"
    echo "  bind_ip: IP address to bind services (default: 0.0.0.0)"
    exit 1
}

log() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

error() {
    echo -e "${RED}[ERROR]${NC} $1" >&2
}

cleanup() {
    error "Installation failed. Cleaning up..."
    systemctl stop nomad consul docker 2>/dev/null || true
    systemctl disable nomad consul 2>/dev/null || true
    userdel nomad 2>/dev/null || true
    userdel consul 2>/dev/null || true
    rm -rf /opt/nomad /opt/consul /etc/nomad.d /etc/consul.d 2>/dev/null || true
}

trap cleanup ERR

detect_distro() {
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        case "$ID" in
            ubuntu|debian) 
                PKG_MGR="apt"
                PKG_UPDATE="apt update && apt upgrade -y"
                PKG_INSTALL="apt install -y"
                DOCKER_PKG="docker.io"
                ;;
            almalinux|rocky|centos|rhel|ol|fedora) 
                PKG_MGR="dnf"
                PKG_UPDATE="dnf update -y"
                PKG_INSTALL="dnf install -y"
                DOCKER_PKG="docker"
                ;;
            amzn) 
                PKG_MGR="yum"
                PKG_UPDATE="yum update -y"
                PKG_INSTALL="yum install -y"
                DOCKER_PKG="docker"
                ;;
            *) 
                error "Unsupported distribution: $ID"
                exit 1 
                ;;
        esac
    else
        error "Cannot detect distribution. /etc/os-release not found."
        exit 1
    fi
}

check_prerequisites() {
    if [[ $EUID -ne 0 ]]; then
        error "This script must be run as root or with sudo"
        exit 1
    fi

    if ! command -v curl &> /dev/null; then
        error "curl is required but not installed"
        exit 1
    fi
}

install_hashicorp_repo() {
    log "[2/10] Installing HashiCorp repository..."
    
    case "$PKG_MGR" in
        apt)
            curl -fsSL https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
            echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" > /etc/apt/sources.list.d/hashicorp.list
            apt update
            ;;
        dnf)
            $PKG_INSTALL dnf-plugins-core
            dnf config-manager --add-repo https://rpm.releases.hashicorp.com/fedora/hashicorp.repo
            ;;
        yum)
            $PKG_INSTALL yum-utils
            yum-config-manager --add-repo https://rpm.releases.hashicorp.com/fedora/hashicorp.repo
            ;;
    esac
}

create_users_and_directories() {
    log "[4/10] Creating system users and directories..."
    
    # Create system users
    useradd --system --home /etc/nomad.d --shell /bin/false nomad 2>/dev/null || true
    useradd --system --home /etc/consul.d --shell /bin/false consul 2>/dev/null || true
    
    # Create directories
    mkdir -p /opt/nomad/data /opt/consul/data
    mkdir -p /etc/nomad.d /etc/consul.d
    
    # Set ownership and permissions
    chown -R nomad:nomad /opt/nomad /etc/nomad.d
    chown -R consul:consul /opt/consul /etc/consul.d
    chmod 755 /opt/nomad /opt/consul
    chmod 750 /etc/nomad.d /etc/consul.d
}

generate_encryption_keys() {
    log "[6/10] Generating encryption keys..."
    
    CONSUL_ENCRYPT_KEY=$(consul keygen)
    NOMAD_ENCRYPT_KEY=$(nomad operator keygen)
    
    warn "IMPORTANT: Save these keys securely!"
    echo "Consul encryption key: $CONSUL_ENCRYPT_KEY"
    echo "Nomad encryption key: $NOMAD_ENCRYPT_KEY"
    
    # Save keys to temporary secure files
    echo "$CONSUL_ENCRYPT_KEY" > /tmp/consul_key
    echo "$NOMAD_ENCRYPT_KEY" > /tmp/nomad_key
    chmod 600 /tmp/consul_key /tmp/nomad_key
}

configure_consul() {
    log "[7/10] Configuring Consul server..."
    
    cat > /etc/consul.d/consul.hcl << EOF
datacenter = "$DATACENTER"
data_dir = "/opt/consul/data"
log_level = "INFO"
server = true
bootstrap_expect = 1
bind_addr = "$BIND_IP"
client_addr = "$BIND_IP"
retry_join = ["127.0.0.1"]
ui_config {
  enabled = true
}
connect {
  enabled = true
}
encrypt = "$CONSUL_ENCRYPT_KEY"
acl = {
  enabled = true
  default_policy = "deny"
  enable_token_persistence = true
}
ports {
  grpc = 8502
}
EOF
    
    chmod 640 /etc/consul.d/consul.hcl
    chown consul:consul /etc/consul.d/consul.hcl
}

configure_nomad() {
    log "[8/10] Configuring Nomad server..."
    
    cat > /etc/nomad.d/nomad.hcl << EOF
datacenter = "$DATACENTER"
data_dir = "/opt/nomad/data"
log_level = "INFO"
bind_addr = "$BIND_IP"

server {
  enabled = true
  bootstrap_expect = 1
  encrypt = "$NOMAD_ENCRYPT_KEY"
}

client {
  enabled = true
  servers = ["127.0.0.1:4647"]
}

consul {
  address = "127.0.0.1:8500"
  server_service_name = "nomad"
  client_service_name = "nomad-client"
  auto_advertise = true
  server_auto_join = true
  client_auto_join = true
}

acl {
  enabled = true
}

plugin "docker" {
  config {
    allow_privileged = false
    allow_caps = ["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot"]
  }
}
EOF
    
    chmod 640 /etc/nomad.d/nomad.hcl
    chown nomad:nomad /etc/nomad.d/nomad.hcl
}

start_services() {
    log "[9/10] Starting and enabling services..."
    
    systemctl enable --now docker
    systemctl enable --now consul
    sleep 5
    systemctl enable --now nomad
    
    # Wait for services to start
    sleep 10
}

verify_installation() {
    log "[10/10] Verifying installation..."
    
    if systemctl is-active --quiet consul; then
        log "✓ Consul is running"
    else
        error "✗ Consul is not running"
        return 1
    fi
    
    if systemctl is-active --quiet nomad; then
        log "✓ Nomad is running"
    else
        error "✗ Nomad is not running"
        return 1
    fi
    
    if systemctl is-active --quiet docker; then
        log "✓ Docker is running"
    else
        error "✗ Docker is not running"
        return 1
    fi
    
    # Test basic connectivity
    if curl -s http://localhost:8500/v1/status/leader > /dev/null; then
        log "✓ Consul API is responding"
    else
        warn "⚠ Consul API may not be ready yet"
    fi
    
    if curl -s http://localhost:4646/v1/status/leader > /dev/null; then
        log "✓ Nomad API is responding"
    else
        warn "⚠ Nomad API may not be ready yet"
    fi
}

main() {
    log "[1/10] Checking prerequisites and detecting distribution..."
    check_prerequisites
    detect_distro
    
    log "Installing on $ID $VERSION_ID using $PKG_MGR"
    log "Updating system packages..."
    eval "$PKG_UPDATE"
    eval "$PKG_INSTALL curl wget unzip $DOCKER_PKG jq"
    
    install_hashicorp_repo
    
    log "[3/10] Installing Nomad and Consul..."
    eval "$PKG_INSTALL nomad consul"
    
    create_users_and_directories
    
    log "[5/10] Configuring Docker for Nomad..."
    systemctl enable --now docker
    usermod -aG docker nomad
    
    generate_encryption_keys
    configure_consul
    configure_nomad
    start_services
    verify_installation
    
    # Cleanup temporary files
    rm -f /tmp/consul_key /tmp/nomad_key
    
    log "Installation completed successfully!"
    log "Consul UI: http://$(hostname -I | awk '{print $1}'):8500"
    log "Nomad UI: http://$(hostname -I | awk '{print $1}'):4646"
    warn "Remember to configure ACL tokens before using in production!"
}

# Validate arguments
if [[ $# -gt 2 ]]; then
    usage
fi

main "$@"