Setup centralized log aggregation with Elasticsearch 8, Logstash 8, and Kibana 8 (ELK Stack)

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Function to print colored output
print_status() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

print_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

print_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Usage function
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -h, --help     Show this help message"
    echo "  -m, --memory   Set Elasticsearch heap size in GB (default: auto-detect)"
    echo ""
    echo "Example: $0 --memory 4"
    exit 1
}

# Cleanup function for rollback
cleanup() {
    print_error "Installation failed. Cleaning up..."
    systemctl stop elasticsearch || true
    systemctl stop logstash || true
    systemctl stop kibana || true
    exit 1
}

trap cleanup ERR

# Parse arguments
HEAP_SIZE=""
while [[ $# -gt 0 ]]; do
    case $1 in
        -h|--help)
            usage
            ;;
        -m|--memory)
            HEAP_SIZE="$2"
            shift 2
            ;;
        *)
            print_error "Unknown option: $1"
            usage
            ;;
    esac
done

# Check if running as root or with sudo
if [[ $EUID -ne 0 ]]; then
    print_error "This script must be run as root or with sudo"
    exit 1
fi

# Detect distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update"
            PKG_UPGRADE="apt upgrade -y"
            JAVA_PKG="openjdk-17-jdk"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf check-update || true"
            PKG_UPGRADE="dnf update -y"
            JAVA_PKG="java-17-openjdk java-17-openjdk-devel"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum check-update || true"
            PKG_UPGRADE="yum update -y"
            JAVA_PKG="java-17-openjdk java-17-openjdk-devel"
            ;;
        *)
            print_error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
else
    print_error "Cannot detect distribution. /etc/os-release not found."
    exit 1
fi

print_status "Detected distribution: $ID"

# Auto-detect memory for heap size
if [[ -z "$HEAP_SIZE" ]]; then
    TOTAL_MEM_KB=$(grep MemTotal /proc/meminfo | awk '{print $2}')
    TOTAL_MEM_GB=$((TOTAL_MEM_KB / 1024 / 1024))
    HEAP_SIZE=$((TOTAL_MEM_GB / 2))
    if [[ $HEAP_SIZE -gt 32 ]]; then
        HEAP_SIZE=32
    fi
    if [[ $HEAP_SIZE -lt 1 ]]; then
        HEAP_SIZE=1
    fi
fi

print_status "Using ${HEAP_SIZE}GB heap size for Elasticsearch"

print_status "[1/10] Updating system packages..."
$PKG_UPDATE
$PKG_UPGRADE

print_status "[2/10] Installing prerequisites..."
if [[ "$PKG_MGR" == "apt" ]]; then
    $PKG_INSTALL curl gnupg2 software-properties-common apt-transport-https ca-certificates
else
    $PKG_INSTALL curl gnupg2 ca-certificates
fi

print_status "[3/10] Installing Java OpenJDK 17..."
$PKG_INSTALL $JAVA_PKG

java -version

print_status "[4/10] Adding Elasticsearch repository..."
curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg

if [[ "$PKG_MGR" == "apt" ]]; then
    echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" > /etc/apt/sources.list.d/elastic-8.x.list
    apt update
else
    cat > /etc/yum.repos.d/elasticsearch.repo << EOF
[elasticsearch]
name=Elasticsearch repository for 8.x packages
baseurl=https://artifacts.elastic.co/packages/8.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF
fi

print_status "[5/10] Installing Elasticsearch..."
if [[ "$PKG_MGR" == "apt" ]]; then
    $PKG_INSTALL elasticsearch
else
    $PKG_INSTALL elasticsearch
fi

# Save the elastic password
print_warning "Please save the elastic user password shown above!"

print_status "[6/10] Configuring Elasticsearch..."
cat > /etc/elasticsearch/elasticsearch.yml << EOF
cluster.name: elk-cluster
node.name: elk-node-1
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: localhost
http.port: 9200
discovery.type: single-node

# Security settings
xpack.security.enabled: true
xpack.security.enrollment.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: certs/http.p12
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.client_authentication: required
xpack.security.transport.ssl.keystore.path: certs/transport.p12
xpack.security.transport.ssl.truststore.path: certs/transport.p12
EOF

# Configure JVM heap
cat > /etc/elasticsearch/jvm.options.d/heap.options << EOF
-Xms${HEAP_SIZE}g
-Xmx${HEAP_SIZE}g
EOF

chown root:elasticsearch /etc/elasticsearch/elasticsearch.yml
chmod 660 /etc/elasticsearch/elasticsearch.yml

print_status "[7/10] Starting Elasticsearch..."
systemctl daemon-reload
systemctl enable elasticsearch
systemctl start elasticsearch

print_status "[8/10] Installing Kibana..."
if [[ "$PKG_MGR" == "apt" ]]; then
    $PKG_INSTALL kibana
else
    $PKG_INSTALL kibana
fi

print_status "[9/10] Installing Logstash..."
if [[ "$PKG_MGR" == "apt" ]]; then
    $PKG_INSTALL logstash
else
    $PKG_INSTALL logstash
fi

# Basic Logstash configuration
cat > /etc/logstash/conf.d/basic.conf << EOF
input {
  beats {
    port => 5044
  }
}

filter {
  if [fileset][module] == "system" {
    if [fileset][name] == "auth" {
      grok {
        match => { "message" => ["%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{IPORHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} %{DATA:[system][auth][ssh][method]} for (invalid user )?%{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]} port %{INT:[system][auth][ssh][port]} ssh2"] }
      }
    }
  }
}

output {
  elasticsearch {
    hosts => ["localhost:9200"]
    ssl => true
    ssl_certificate_verification => false
    user => "elastic"
    password => "${ELASTIC_PASSWORD:-changeme}"
  }
}
EOF

chown root:logstash /etc/logstash/conf.d/basic.conf
chmod 644 /etc/logstash/conf.d/basic.conf

print_status "[10/10] Starting services..."
systemctl enable kibana
systemctl enable logstash
systemctl start logstash

# Wait for Elasticsearch to be ready
print_status "Waiting for Elasticsearch to be ready..."
sleep 30

# Configure firewall if firewalld is active
if systemctl is-active --quiet firewalld 2>/dev/null; then
    print_status "Configuring firewall..."
    firewall-cmd --permanent --add-port=9200/tcp --add-port=5601/tcp --add-port=5044/tcp
    firewall-cmd --reload
fi

# Configure UFW if it's active
if command -v ufw >/dev/null 2>&1 && ufw status | grep -q "Status: active"; then
    print_status "Configuring UFW firewall..."
    ufw allow 9200/tcp
    ufw allow 5601/tcp
    ufw allow 5044/tcp
fi

print_status "Verifying installation..."
systemctl is-active elasticsearch || print_error "Elasticsearch is not running"
systemctl is-active logstash || print_error "Logstash is not running"

print_status "ELK Stack installation completed successfully!"
print_warning "Next steps:"
echo "1. Get the Kibana enrollment token: /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana"
echo "2. Configure Kibana: /usr/share/kibana/bin/kibana-setup --enrollment-token <token>"
echo "3. Start Kibana: systemctl start kibana"
echo "4. Access Kibana at: http://localhost:5601"
echo "5. Default elastic user password was shown during Elasticsearch installation"