Set up ClickHouse monitoring with Prometheus and Grafana dashboards

#!/usr/bin/env bash

set -euo pipefail

# ClickHouse Monitoring Setup with Prometheus and Grafana
# Usage: ./clickhouse-monitoring-setup.sh [clickhouse_host] [monitoring_password]

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Configuration variables
CLICKHOUSE_HOST="${1:-localhost}"
MONITORING_PASSWORD="${2:-$(openssl rand -base64 32)}"
PROMETHEUS_PORT="9090"
GRAFANA_PORT="3000"
CLICKHOUSE_METRICS_PORT="9363"
CLICKHOUSE_HTTP_PORT="8123"

usage() {
    echo "Usage: $0 [clickhouse_host] [monitoring_password]"
    echo "  clickhouse_host: ClickHouse server hostname/IP (default: localhost)"
    echo "  monitoring_password: Password for monitoring user (default: auto-generated)"
    exit 1
}

log() {
    echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] $1${NC}"
}

warn() {
    echo -e "${YELLOW}[WARNING] $1${NC}"
}

error() {
    echo -e "${RED}[ERROR] $1${NC}"
    exit 1
}

cleanup() {
    warn "Script failed. Cleaning up..."
    systemctl stop prometheus grafana-server 2>/dev/null || true
}

trap cleanup ERR

# Detect OS and set package manager
detect_os() {
    if [ ! -f /etc/os-release ]; then
        error "Cannot detect OS. /etc/os-release not found."
    fi
    
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_UPDATE="apt update"
            PKG_INSTALL="apt install -y"
            PROMETHEUS_CONFIG="/etc/prometheus/prometheus.yml"
            GRAFANA_CONFIG="/etc/grafana/grafana.ini"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf check-update || true"
            PKG_INSTALL="dnf install -y"
            PROMETHEUS_CONFIG="/etc/prometheus/prometheus.yml"
            GRAFANA_CONFIG="/etc/grafana/grafana.ini"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_UPDATE="yum check-update || true"
            PKG_INSTALL="yum install -y"
            PROMETHEUS_CONFIG="/etc/prometheus/prometheus.yml"
            GRAFANA_CONFIG="/etc/grafana/grafana.ini"
            ;;
        *)
            error "Unsupported distribution: $ID"
            ;;
    esac
    log "Detected OS: $ID using $PKG_MGR"
}

check_prerequisites() {
    log "[1/8] Checking prerequisites..."
    
    if [ "$EUID" -ne 0 ]; then
        error "Please run as root or with sudo"
    fi
    
    command -v systemctl >/dev/null 2>&1 || error "systemd is required"
    command -v openssl >/dev/null 2>&1 || error "openssl is required"
    
    # Check if ClickHouse is accessible
    if ! command -v clickhouse-client >/dev/null 2>&1; then
        warn "clickhouse-client not found. Ensure ClickHouse is installed."
    fi
}

install_prometheus() {
    log "[2/8] Installing and configuring Prometheus..."
    
    $PKG_UPDATE
    
    case "$PKG_MGR" in
        apt)
            $PKG_INSTALL prometheus
            ;;
        dnf|yum)
            $PKG_INSTALL prometheus2
            # Create symlink for consistency
            [ -L /usr/bin/prometheus ] || ln -s /usr/bin/prometheus2 /usr/bin/prometheus 2>/dev/null || true
            ;;
    esac
    
    systemctl enable prometheus
}

configure_clickhouse_metrics() {
    log "[3/8] Configuring ClickHouse metrics endpoint..."
    
    CLICKHOUSE_CONFIG_DIR="/etc/clickhouse-server/config.d"
    mkdir -p "$CLICKHOUSE_CONFIG_DIR"
    
    cat > "$CLICKHOUSE_CONFIG_DIR/prometheus_metrics.xml" << EOF
<clickhouse>
    <prometheus>
        <endpoint>/metrics</endpoint>
        <port>$CLICKHOUSE_METRICS_PORT</port>
        <metrics>true</metrics>
        <events>true</events>
        <asynchronous_metrics>true</asynchronous_metrics>
    </prometheus>
</clickhouse>
EOF
    
    chown clickhouse:clickhouse "$CLICKHOUSE_CONFIG_DIR/prometheus_metrics.xml" 2>/dev/null || true
    chmod 644 "$CLICKHOUSE_CONFIG_DIR/prometheus_metrics.xml"
    
    # Restart ClickHouse if running
    if systemctl is-active --quiet clickhouse-server; then
        systemctl restart clickhouse-server
        sleep 5
    fi
}

configure_prometheus() {
    log "[4/8] Configuring Prometheus scrape targets..."
    
    mkdir -p /etc/prometheus/rules
    chown prometheus:prometheus /etc/prometheus/rules 2>/dev/null || true
    chmod 755 /etc/prometheus/rules
    
    cat > "$PROMETHEUS_CONFIG" << EOF
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "/etc/prometheus/rules/*.yml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:$PROMETHEUS_PORT']

  - job_name: 'clickhouse-metrics'
    static_configs:
      - targets: ['$CLICKHOUSE_HOST:$CLICKHOUSE_METRICS_PORT']
    scrape_interval: 10s
    metrics_path: /metrics

  - job_name: 'clickhouse-system'
    static_configs:
      - targets: ['$CLICKHOUSE_HOST:$CLICKHOUSE_HTTP_PORT']
    scrape_interval: 30s
    metrics_path: /
    params:
      query: ['SELECT concat(metric, \' \', toString(value)) FROM system.metrics FORMAT LineAsString']
    basic_auth:
      username: monitoring
      password: '$MONITORING_PASSWORD'
EOF
    
    chown prometheus:prometheus "$PROMETHEUS_CONFIG" 2>/dev/null || true
    chmod 644 "$PROMETHEUS_CONFIG"
}

create_monitoring_user() {
    log "[5/8] Creating ClickHouse monitoring user..."
    
    if command -v clickhouse-client >/dev/null 2>&1; then
        clickhouse-client --host "$CLICKHOUSE_HOST" --query "CREATE USER IF NOT EXISTS monitoring IDENTIFIED BY '$MONITORING_PASSWORD'" || warn "Failed to create monitoring user"
        clickhouse-client --host "$CLICKHOUSE_HOST" --query "GRANT SELECT ON system.* TO monitoring" || warn "Failed to grant system permissions"
        clickhouse-client --host "$CLICKHOUSE_HOST" --query "GRANT SELECT ON INFORMATION_SCHEMA.* TO monitoring" || warn "Failed to grant information schema permissions"
    else
        warn "ClickHouse client not available. Please manually create monitoring user."
    fi
}

install_grafana() {
    log "[6/8] Installing Grafana..."
    
    case "$PKG_MGR" in
        apt)
            $PKG_INSTALL apt-transport-https software-properties-common wget
            wget -q -O /usr/share/keyrings/grafana.key https://apt.grafana.com/gpg.key
            echo "deb [signed-by=/usr/share/keyrings/grafana.key] https://apt.grafana.com stable main" | tee /etc/apt/sources.list.d/grafana.list
            apt update
            $PKG_INSTALL grafana
            ;;
        dnf)
            cat > /etc/yum.repos.d/grafana.repo << 'EOF'
[grafana]
name=grafana
baseurl=https://rpm.grafana.com
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://rpm.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
EOF
            $PKG_INSTALL grafana
            ;;
        yum)
            cat > /etc/yum.repos.d/grafana.repo << 'EOF'
[grafana]
name=grafana
baseurl=https://rpm.grafana.com
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://rpm.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
EOF
            $PKG_INSTALL grafana
            ;;
    esac
    
    systemctl enable grafana-server
}

configure_grafana() {
    log "[7/8] Configuring Grafana data source..."
    
    mkdir -p /etc/grafana/provisioning/datasources
    mkdir -p /etc/grafana/provisioning/dashboards
    
    cat > /etc/grafana/provisioning/datasources/prometheus.yml << EOF
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://localhost:$PROMETHEUS_PORT
    isDefault: true
    editable: true
    jsonData:
      timeInterval: "10s"
      queryTimeout: "60s"
EOF
    
    chown -R grafana:grafana /etc/grafana/provisioning 2>/dev/null || true
    chmod -R 644 /etc/grafana/provisioning/datasources/*.yml
    chmod 755 /etc/grafana/provisioning/datasources /etc/grafana/provisioning/dashboards
}

start_services() {
    log "[8/8] Starting services..."
    
    systemctl start prometheus
    systemctl start grafana-server
    
    # Wait for services to start
    sleep 10
    
    # Configure firewall if available
    if command -v ufw >/dev/null 2>&1 && ufw status | grep -q "Status: active"; then
        ufw allow $PROMETHEUS_PORT/tcp
        ufw allow $GRAFANA_PORT/tcp
        ufw allow $CLICKHOUSE_METRICS_PORT/tcp
    elif command -v firewall-cmd >/dev/null 2>&1 && systemctl is-active --quiet firewalld; then
        firewall-cmd --permanent --add-port=$PROMETHEUS_PORT/tcp
        firewall-cmd --permanent --add-port=$GRAFANA_PORT/tcp
        firewall-cmd --permanent --add-port=$CLICKHOUSE_METRICS_PORT/tcp
        firewall-cmd --reload
    fi
}

verify_installation() {
    log "Verifying installation..."
    
    # Check if services are running
    if systemctl is-active --quiet prometheus; then
        log "✓ Prometheus is running"
    else
        error "✗ Prometheus is not running"
    fi
    
    if systemctl is-active --quiet grafana-server; then
        log "✓ Grafana is running"
    else
        error "✗ Grafana is not running"
    fi
    
    # Test endpoints
    if curl -s "http://localhost:$PROMETHEUS_PORT/api/v1/query?query=up" >/dev/null; then
        log "✓ Prometheus API is responding"
    else
        warn "✗ Prometheus API is not responding"
    fi
    
    log "Installation completed successfully!"
    log "Access URLs:"
    log "  Prometheus: http://localhost:$PROMETHEUS_PORT"
    log "  Grafana: http://localhost:$GRAFANA_PORT (admin/admin)"
    log "ClickHouse monitoring user password: $MONITORING_PASSWORD"
}

main() {
    detect_os
    check_prerequisites
    install_prometheus
    configure_clickhouse_metrics
    configure_prometheus
    create_monitoring_user
    install_grafana
    configure_grafana
    start_services
    verify_installation
}

main "$@"