Configure InfluxDB 2.7 clustering for high availability with data replication and automated failover

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Global variables
SCRIPT_NAME=$(basename "$0")
CLUSTER_NODES=()
NODE_ID=""
LICENSE_KEY=""
DATA_DIR="/var/lib/influxdb"
CONFIG_DIR="/etc/influxdb"

# Usage information
usage() {
    cat << EOF
Usage: $SCRIPT_NAME [OPTIONS]

Configure InfluxDB 2.7 Enterprise clustering for high availability

OPTIONS:
    -n, --nodes         Comma-separated list of cluster node IPs
    -i, --node-id       Unique node ID (1-based integer)
    -l, --license       InfluxDB Enterprise license key
    -d, --data-dir      Data directory (default: /var/lib/influxdb)
    -h, --help          Show this help message

EXAMPLE:
    $SCRIPT_NAME --nodes "10.0.1.10,10.0.1.11,10.0.1.12" --node-id 1 --license "your-license-key"
EOF
}

# Logging functions
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; }
log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

# Cleanup function for rollback
cleanup() {
    local exit_code=$?
    if [ $exit_code -ne 0 ]; then
        log_error "Installation failed. Cleaning up..."
        systemctl stop influxdb || true
        systemctl stop chronograf || true
        systemctl stop kapacitor || true
        systemctl disable influxdb || true
        systemctl disable chronograf || true
        systemctl disable kapacitor || true
    fi
    exit $exit_code
}

trap cleanup ERR

# Parse command line arguments
parse_args() {
    while [[ $# -gt 0 ]]; do
        case $1 in
            -n|--nodes)
                IFS=',' read -ra CLUSTER_NODES <<< "$2"
                shift 2
                ;;
            -i|--node-id)
                NODE_ID="$2"
                shift 2
                ;;
            -l|--license)
                LICENSE_KEY="$2"
                shift 2
                ;;
            -d|--data-dir)
                DATA_DIR="$2"
                shift 2
                ;;
            -h|--help)
                usage
                exit 0
                ;;
            *)
                log_error "Unknown option: $1"
                usage
                exit 1
                ;;
        esac
    done

    if [[ ${#CLUSTER_NODES[@]} -eq 0 || -z "$NODE_ID" || -z "$LICENSE_KEY" ]]; then
        log_error "Missing required arguments"
        usage
        exit 1
    fi
}

# Check prerequisites
check_prerequisites() {
    log_info "[1/8] Checking prerequisites..."
    
    if [[ $EUID -ne 0 ]]; then
        log_error "This script must be run as root"
        exit 1
    fi

    if ! command -v curl &> /dev/null; then
        log_error "curl is required but not installed"
        exit 1
    fi

    if ! command -v gpg &> /dev/null; then
        log_error "gpg is required but not installed"
        exit 1
    fi

    log_success "Prerequisites check passed"
}

# Detect Linux distribution
detect_distro() {
    log_info "[2/8] Detecting Linux distribution..."
    
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        case "$ID" in
            ubuntu|debian)
                PKG_MGR="apt"
                PKG_UPDATE="apt update"
                PKG_INSTALL="apt install -y"
                FIREWALL_CMD="ufw"
                ;;
            almalinux|rocky|centos|rhel|ol)
                PKG_MGR="dnf"
                PKG_UPDATE="dnf makecache"
                PKG_INSTALL="dnf install -y"
                FIREWALL_CMD="firewall-cmd"
                ;;
            fedora)
                PKG_MGR="dnf"
                PKG_UPDATE="dnf makecache"
                PKG_INSTALL="dnf install -y"
                FIREWALL_CMD="firewall-cmd"
                ;;
            amzn)
                PKG_MGR="yum"
                PKG_UPDATE="yum makecache"
                PKG_INSTALL="yum install -y"
                FIREWALL_CMD="firewall-cmd"
                ;;
            *)
                log_error "Unsupported distribution: $ID"
                exit 1
                ;;
        esac
    else
        log_error "Cannot detect Linux distribution"
        exit 1
    fi
    
    log_success "Detected distribution: $PRETTY_NAME (Package manager: $PKG_MGR)"
}

# Setup InfluxData repository
setup_repository() {
    log_info "[3/8] Setting up InfluxData repository..."
    
    case "$PKG_MGR" in
        apt)
            wget -q https://repos.influxdata.com/influxdata-archive_compat.key -O /tmp/influxdata-archive_compat.key
            echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c6b  /tmp/influxdata-archive_compat.key' | sha256sum -c
            cat /tmp/influxdata-archive_compat.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null
            chmod 644 /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg
            echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' > /etc/apt/sources.list.d/influxdata.list
            chmod 644 /etc/apt/sources.list.d/influxdata.list
            ;;
        dnf|yum)
            cat > /etc/yum.repos.d/influxdata.repo << 'EOF'
[influxdata]
name = InfluxData Repository
baseurl = https://repos.influxdata.com/rhel/\$releasever/\$basearch/stable/
enabled = 1
gpgcheck = 1
gpgkey = https://repos.influxdata.com/influxdata-archive_compat.key
EOF
            chmod 644 /etc/yum.repos.d/influxdata.repo
            ;;
    esac
    
    rm -f /tmp/influxdata-archive_compat.key
    log_success "Repository setup completed"
}

# Install InfluxDB Enterprise components
install_influxdb_enterprise() {
    log_info "[4/8] Installing InfluxDB Enterprise components..."
    
    $PKG_UPDATE
    $PKG_INSTALL influxdb2-enterprise chronograf kapacitor
    
    log_success "InfluxDB Enterprise components installed"
}

# Configure InfluxDB clustering
configure_clustering() {
    log_info "[5/8] Configuring InfluxDB clustering..."
    
    mkdir -p "$CONFIG_DIR" "$DATA_DIR"
    chown influxdb:influxdb "$DATA_DIR"
    chmod 755 "$DATA_DIR"
    
    # Generate cluster configuration
    cat > "$CONFIG_DIR/influxdb.conf" << EOF
# InfluxDB Enterprise cluster configuration
[enterprise]
  license-key = "$LICENSE_KEY"
  
[meta]
  dir = "$DATA_DIR/meta"
  bind-address = ":8089"
  http-bind-address = ":8091"
  
[data]
  dir = "$DATA_DIR/data"
  wal-dir = "$DATA_DIR/wal"
  
[cluster]
  shard-writer-timeout = "10s"
  write-timeout = "10s"
  max-concurrent-queries = 0
  query-timeout = "0s"
  max-select-point = 0
  max-select-series = 0
  max-select-buckets = 0

[retention]
  enabled = true
  check-interval = "30m0s"

[http]
  enabled = true
  bind-address = ":8086"
  auth-enabled = false
  log-enabled = true
  write-tracing = false
  pprof-enabled = true
  https-enabled = false
  max-row-limit = 0
  max-connection-limit = 0
  shared-secret = ""
  realm = "InfluxDB"

[logging]
  level = "info"
  suppress-logo = false
EOF

    chown influxdb:influxdb "$CONFIG_DIR/influxdb.conf"
    chmod 644 "$CONFIG_DIR/influxdb.conf"
    
    log_success "Clustering configuration completed"
}

# Configure firewall
configure_firewall() {
    log_info "[6/8] Configuring firewall..."
    
    case "$FIREWALL_CMD" in
        ufw)
            if command -v ufw &> /dev/null && ufw status | grep -q "Status: active"; then
                ufw allow 8086/tcp comment "InfluxDB HTTP API"
                ufw allow 8089/tcp comment "InfluxDB Meta"
                ufw allow 8091/tcp comment "InfluxDB Meta HTTP"
                ufw allow 8888/tcp comment "Chronograf"
                ufw allow 9092/tcp comment "Kapacitor"
            fi
            ;;
        firewall-cmd)
            if systemctl is-active --quiet firewalld; then
                firewall-cmd --permanent --add-port=8086/tcp
                firewall-cmd --permanent --add-port=8089/tcp
                firewall-cmd --permanent --add-port=8091/tcp
                firewall-cmd --permanent --add-port=8888/tcp
                firewall-cmd --permanent --add-port=9092/tcp
                firewall-cmd --reload
            fi
            ;;
    esac
    
    log_success "Firewall configuration completed"
}

# Start and enable services
start_services() {
    log_info "[7/8] Starting and enabling InfluxDB services..."
    
    systemctl daemon-reload
    systemctl enable influxdb chronograf kapacitor
    systemctl start influxdb
    
    # Wait for InfluxDB to start
    sleep 10
    
    # Initialize cluster if this is node 1
    if [[ "$NODE_ID" == "1" ]]; then
        log_info "Initializing InfluxDB cluster..."
        influxd-ctl add-meta "$(hostname):8091"
        influxd-ctl add-data "$(hostname):8088"
        
        # Add other nodes to cluster
        for i in "${!CLUSTER_NODES[@]}"; do
            if [[ $((i + 1)) -ne $NODE_ID ]]; then
                node_ip="${CLUSTER_NODES[$i]}"
                influxd-ctl add-meta "$node_ip:8091" || log_warning "Failed to add meta node $node_ip"
                influxd-ctl add-data "$node_ip:8088" || log_warning "Failed to add data node $node_ip"
            fi
        done
    fi
    
    systemctl start chronograf kapacitor
    
    log_success "Services started and enabled"
}

# Verify installation
verify_installation() {
    log_info "[8/8] Verifying installation..."
    
    # Check service status
    for service in influxdb chronograf kapacitor; do
        if systemctl is-active --quiet "$service"; then
            log_success "$service is running"
        else
            log_error "$service is not running"
            exit 1
        fi
    done
    
    # Test InfluxDB connection
    if curl -s http://localhost:8086/ping > /dev/null; then
        log_success "InfluxDB HTTP API is responding"
    else
        log_error "InfluxDB HTTP API is not responding"
        exit 1
    fi
    
    # Check cluster status (if node 1)
    if [[ "$NODE_ID" == "1" ]]; then
        if influxd-ctl show > /dev/null 2>&1; then
            log_success "Cluster status check passed"
        else
            log_warning "Cluster status check failed - may need manual configuration"
        fi
    fi
    
    log_success "Installation verification completed"
}

# Main function
main() {
    parse_args "$@"
    check_prerequisites
    detect_distro
    setup_repository
    install_influxdb_enterprise
    configure_clustering
    configure_firewall
    start_services
    verify_installation
    
    log_success "InfluxDB Enterprise cluster installation completed!"
    log_info "Access Chronograf at: http://$(hostname -I | awk '{print $1}'):8888"
    log_info "InfluxDB API endpoint: http://$(hostname -I | awk '{print $1}'):8086"
    log_info "Data directory: $DATA_DIR"
    log_info "Config file: $CONFIG_DIR/influxdb.conf"
}

main "$@"