Install and configure Jaeger for distributed tracing with Elasticsearch backend

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Default values
JAEGER_VERSION="1.51.0"
ELASTICSEARCH_URL="http://localhost:9200"

# Usage function
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -v VERSION    Jaeger version (default: $JAEGER_VERSION)"
    echo "  -e URL        Elasticsearch URL (default: $ELASTICSEARCH_URL)"
    echo "  -h            Show this help"
    exit 1
}

# Parse arguments
while getopts "v:e:h" opt; do
    case $opt in
        v) JAEGER_VERSION="$OPTARG" ;;
        e) ELASTICSEARCH_URL="$OPTARG" ;;
        h) usage ;;
        *) usage ;;
    esac
done

# Logging functions
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

# Cleanup function
cleanup() {
    log_error "Installation failed. Cleaning up..."
    systemctl stop jaeger-collector jaeger-query 2>/dev/null || true
    systemctl disable jaeger-collector jaeger-query 2>/dev/null || true
    rm -f /etc/systemd/system/jaeger-*.service
    rm -rf /etc/jaeger /var/lib/jaeger /var/log/jaeger
    userdel jaeger 2>/dev/null || true
    rm -f /usr/local/bin/jaeger-*
    systemctl daemon-reload
}

trap cleanup ERR

# Check prerequisites
if [[ $EUID -ne 0 ]]; then
    log_error "This script must be run as root"
    exit 1
fi

# Detect distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update && apt upgrade -y"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf update -y"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum update -y"
            ;;
        *) 
            log_error "Unsupported distro: $ID"
            exit 1
            ;;
    esac
else
    log_error "Cannot detect distribution"
    exit 1
fi

log_info "Detected distribution: $ID"
log_info "Installing Jaeger $JAEGER_VERSION with Elasticsearch backend"

# Step 1: Update system
echo "[1/8] Updating system packages..."
$PKG_UPDATE

# Step 2: Install dependencies
echo "[2/8] Installing required dependencies..."
$PKG_INSTALL curl wget unzip systemd

# Step 3: Create jaeger user and directories
echo "[3/8] Creating Jaeger user and directories..."
useradd --system --shell /bin/false --home-dir /var/lib/jaeger jaeger 2>/dev/null || true
mkdir -p /var/lib/jaeger /var/log/jaeger /etc/jaeger
chown jaeger:jaeger /var/lib/jaeger /var/log/jaeger
chmod 755 /var/lib/jaeger /var/log/jaeger /etc/jaeger

# Step 4: Download Jaeger binaries
echo "[4/8] Downloading Jaeger binaries..."
cd /tmp
wget -q "https://github.com/jaegertracing/jaeger/releases/download/v${JAEGER_VERSION}/jaeger-${JAEGER_VERSION}-linux-amd64.tar.gz"
tar -xzf "jaeger-${JAEGER_VERSION}-linux-amd64.tar.gz"
cp "jaeger-${JAEGER_VERSION}-linux-amd64"/jaeger-* /usr/local/bin/
chmod 755 /usr/local/bin/jaeger-*
chown root:root /usr/local/bin/jaeger-*

# Step 5: Configure Jaeger collector
echo "[5/8] Configuring Jaeger collector..."
cat > /etc/jaeger/collector.yaml << EOF
es:
  server-urls: $ELASTICSEARCH_URL
  index-prefix: jaeger
  create-index-templates: true
  version: 8
  num-shards: 5
  num-replicas: 1
  bulk:
    size: 5000000
    workers: 1
    flush-interval: 200ms
  tags-as-fields:
    all: false
    dot-replacement: "@"
    include: "http.status_code,error"
  archive:
    enabled: true
    index-prefix: jaeger-archive

collector:
  grpc-server:
    host-port: 0.0.0.0:14250
  http-server:
    host-port: 0.0.0.0:14268
  zipkin:
    host-port: 0.0.0.0:9411
  otlp:
    grpc:
      host-port: 0.0.0.0:4317
    http:
      host-port: 0.0.0.0:4318

log-level: info
EOF

# Step 6: Create collector systemd service
echo "[6/8] Creating Jaeger collector service..."
cat > /etc/systemd/system/jaeger-collector.service << 'EOF'
[Unit]
Description=Jaeger Collector
Documentation=https://www.jaegertracing.io/
After=network.target elasticsearch.service
Wants=elasticsearch.service

[Service]
Type=simple
User=jaeger
Group=jaeger
ExecStart=/usr/local/bin/jaeger-collector --config-file=/etc/jaeger/collector.yaml
Restart=always
RestartSec=5
StandardOutput=journal
StandardError=journal
SyslogIdentifier=jaeger-collector
KillMode=mixed
KillSignal=SIGTERM

# Security settings
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/log/jaeger /var/lib/jaeger

# Resource limits
LimitNOFILE=1000000
LimitNPROC=1000000

[Install]
WantedBy=multi-user.target
EOF

# Step 7: Configure Jaeger query service
echo "[7/8] Configuring Jaeger query service..."
cat > /etc/jaeger/query.yaml << EOF
es:
  server-urls: $ELASTICSEARCH_URL
  index-prefix: jaeger
  version: 8
  tags-as-fields:
    all: false
    dot-replacement: "@"
    include: "http.status_code,error"
  archive:
    enabled: true
    index-prefix: jaeger-archive

query:
  base-path: /
  ui-config: /etc/jaeger/ui.json
  log-file: /var/log/jaeger/query.log
  host-port: 0.0.0.0:16686
  grpc-server:
    host-port: 0.0.0.0:16685
  max-clock-skew-adjustment: 0s

log-level: info
EOF

# Create UI configuration
cat > /etc/jaeger/ui.json << 'EOF'
{
  "monitor": {
    "menuEnabled": true
  },
  "dependencies": {
    "menuEnabled": true
  },
  "archiveEnabled": true,
  "tracking": {
    "gaID": "",
    "trackErrors": true
  }
}
EOF

# Create query systemd service
cat > /etc/systemd/system/jaeger-query.service << 'EOF'
[Unit]
Description=Jaeger Query
Documentation=https://www.jaegertracing.io/
After=network.target jaeger-collector.service
Wants=jaeger-collector.service

[Service]
Type=simple
User=jaeger
Group=jaeger
ExecStart=/usr/local/bin/jaeger-query --config-file=/etc/jaeger/query.yaml
Restart=always
RestartSec=5
StandardOutput=journal
StandardError=journal
SyslogIdentifier=jaeger-query
KillMode=mixed
KillSignal=SIGTERM

# Security settings
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/log/jaeger /var/lib/jaeger

[Install]
WantedBy=multi-user.target
EOF

# Set proper ownership and permissions
chown jaeger:jaeger /etc/jaeger/*.yaml /etc/jaeger/*.json
chmod 644 /etc/jaeger/*.yaml /etc/jaeger/*.json

# Step 8: Enable and start services
echo "[8/8] Enabling and starting services..."
systemctl daemon-reload
systemctl enable jaeger-collector jaeger-query
systemctl start jaeger-collector
sleep 3
systemctl start jaeger-query

# Configure firewall if available
if command -v ufw >/dev/null 2>&1; then
    log_info "Configuring UFW firewall..."
    ufw allow 14250/tcp comment "Jaeger gRPC"
    ufw allow 14268/tcp comment "Jaeger HTTP"
    ufw allow 16686/tcp comment "Jaeger UI"
    ufw allow 9411/tcp comment "Zipkin"
    ufw allow 4317/tcp comment "OTLP gRPC"
    ufw allow 4318/tcp comment "OTLP HTTP"
elif command -v firewall-cmd >/dev/null 2>&1; then
    log_info "Configuring firewalld..."
    firewall-cmd --permanent --add-port=14250/tcp --add-port=14268/tcp --add-port=16686/tcp --add-port=9411/tcp --add-port=4317/tcp --add-port=4318/tcp
    firewall-cmd --reload
fi

# Verification
echo
log_info "Verifying installation..."
sleep 5

if systemctl is-active --quiet jaeger-collector; then
    log_info "✓ Jaeger collector is running"
else
    log_error "✗ Jaeger collector failed to start"
    exit 1
fi

if systemctl is-active --quiet jaeger-query; then
    log_info "✓ Jaeger query service is running"
else
    log_error "✗ Jaeger query service failed to start"
    exit 1
fi

if curl -s -f http://localhost:16686/api/services >/dev/null 2>&1; then
    log_info "✓ Jaeger API is responding"
else
    log_warn "! Jaeger API not yet responding (may need more time)"
fi

echo
log_info "Jaeger installation completed successfully!"
log_info "Jaeger UI: http://$(hostname -I | awk '{print $1}'):16686"
log_info "Collector endpoints:"
log_info "  - gRPC: :14250"
log_info "  - HTTP: :14268"
log_info "  - Zipkin: :9411"
log_info "  - OTLP gRPC: :4317"
log_info "  - OTLP HTTP: :4318"
log_info
log_info "Service management:"
log_info "  systemctl {start|stop|restart|status} jaeger-collector"
log_info "  systemctl {start|stop|restart|status} jaeger-query"
log_info
log_info "Logs:"
log_info "  journalctl -u jaeger-collector -f"
log_info "  journalctl -u jaeger-query -f"