Configure Netdata alerts with Slack and Microsoft Teams for real-time monitoring notifications

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Global variables
SCRIPT_NAME=$(basename "$0")
SLACK_WEBHOOK=""
TEAMS_WEBHOOK=""
SLACK_CHANNEL="#monitoring"
TEAMS_CHANNEL="General"

# Usage function
usage() {
    echo "Usage: $SCRIPT_NAME [OPTIONS]"
    echo "Options:"
    echo "  --slack-webhook URL    Slack webhook URL"
    echo "  --teams-webhook URL    Microsoft Teams webhook URL"
    echo "  --slack-channel NAME   Slack channel (default: #monitoring)"
    echo "  --teams-channel NAME   Teams channel (default: General)"
    echo "  -h, --help            Show this help message"
    echo ""
    echo "Example:"
    echo "  $SCRIPT_NAME --slack-webhook 'https://hooks.slack.com/services/...' --teams-webhook 'https://outlook.office.com/webhook/...'"
    exit 1
}

# Parse arguments
parse_args() {
    while [[ $# -gt 0 ]]; do
        case $1 in
            --slack-webhook)
                SLACK_WEBHOOK="$2"
                shift 2
                ;;
            --teams-webhook)
                TEAMS_WEBHOOK="$2"
                shift 2
                ;;
            --slack-channel)
                SLACK_CHANNEL="$2"
                shift 2
                ;;
            --teams-channel)
                TEAMS_CHANNEL="$2"
                shift 2
                ;;
            -h|--help)
                usage
                ;;
            *)
                echo -e "${RED}Unknown option: $1${NC}"
                usage
                ;;
        esac
    done
}

# Error handling and cleanup
cleanup() {
    echo -e "${RED}Installation failed. Cleaning up...${NC}"
    systemctl stop netdata 2>/dev/null || true
}

trap cleanup ERR

# Logging functions
log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Check prerequisites
check_prerequisites() {
    if [[ $EUID -ne 0 ]]; then
        log_error "This script must be run as root or with sudo"
        exit 1
    fi

    if [[ -z "$SLACK_WEBHOOK" && -z "$TEAMS_WEBHOOK" ]]; then
        log_error "At least one webhook URL must be provided"
        usage
    fi
}

# Detect distribution
detect_distro() {
    if [[ ! -f /etc/os-release ]]; then
        log_error "/etc/os-release not found. Cannot detect distribution."
        exit 1
    fi

    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_UPDATE="apt update"
            PKG_INSTALL="apt install -y"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf check-update || true"
            PKG_INSTALL="dnf install -y"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_UPDATE="yum check-update || true"
            PKG_INSTALL="yum install -y"
            ;;
        *)
            log_error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
}

# Update system packages
update_packages() {
    echo "[1/8] Updating system packages..."
    $PKG_UPDATE
    if [[ "$PKG_MGR" == "apt" ]]; then
        apt upgrade -y
    fi
    log_info "System packages updated"
}

# Install dependencies
install_dependencies() {
    echo "[2/8] Installing required dependencies..."
    $PKG_INSTALL curl jq wget gnupg
    log_info "Dependencies installed"
}

# Install Netdata
install_netdata() {
    echo "[3/8] Installing Netdata..."
    if command -v netdata >/dev/null 2>&1; then
        log_warn "Netdata is already installed"
        return
    fi

    bash <(curl -Ss https://my-netdata.io/kickstart.sh) --dont-wait --disable-telemetry
    systemctl enable netdata
    systemctl start netdata
    log_info "Netdata installed and started"
}

# Configure notification directories
setup_notification_config() {
    echo "[4/8] Setting up notification configuration..."
    mkdir -p /etc/netdata/health_alarm_notify.conf.d
    
    if [[ ! -f /etc/netdata/health_alarm_notify.conf ]]; then
        if [[ -f /usr/lib/netdata/conf.d/health_alarm_notify.conf ]]; then
            cp /usr/lib/netdata/conf.d/health_alarm_notify.conf /etc/netdata/health_alarm_notify.conf
        else
            # Create basic config if template doesn't exist
            touch /etc/netdata/health_alarm_notify.conf
        fi
    fi
    
    chown netdata:netdata /etc/netdata/health_alarm_notify.conf
    chmod 644 /etc/netdata/health_alarm_notify.conf
    log_info "Notification configuration directory created"
}

# Configure Slack notifications
configure_slack() {
    if [[ -z "$SLACK_WEBHOOK" ]]; then
        return
    fi

    echo "[5/8] Configuring Slack notifications..."
    cat >> /etc/netdata/health_alarm_notify.conf << EOF

# Slack notifications configuration
SEND_SLACK="YES"
SLACK_WEBHOOK_URL="$SLACK_WEBHOOK"
DEFAULT_RECIPIENT_SLACK="$SLACK_CHANNEL"
SLACK_CHANNEL="${SLACK_CHANNEL#\#}"
SLACK_USERNAME="netdata"
SLACK_ICON_EMOJI=":warning:"

# Role-based Slack recipients
role_recipients_slack[sysadmin]="$SLACK_CHANNEL"
role_recipients_slack[domainadmin]="$SLACK_CHANNEL"
role_recipients_slack[dba]="$SLACK_CHANNEL"
EOF
    log_info "Slack notifications configured"
}

# Configure Microsoft Teams notifications
configure_teams() {
    if [[ -z "$TEAMS_WEBHOOK" ]]; then
        return
    fi

    echo "[6/8] Configuring Microsoft Teams notifications..."
    cat >> /etc/netdata/health_alarm_notify.conf << EOF

# Microsoft Teams notifications configuration
SEND_MSTEAMS="YES"
MSTEAMS_WEBHOOK_URL="$TEAMS_WEBHOOK"
DEFAULT_RECIPIENT_MSTEAMS="$TEAMS_CHANNEL"
MSTEAMS_ICON_DEFAULT="https://registry.my-netdata.io/api/v1/badge.svg?chart=system.cpu"
MSTEAMS_COLOR_DEFAULT="0076D7"

# Role-based Teams recipients
role_recipients_msteams[sysadmin]="$TEAMS_CHANNEL"
role_recipients_msteams[domainadmin]="$TEAMS_CHANNEL"
EOF
    log_info "Microsoft Teams notifications configured"
}

# Create custom alert rules
create_alert_rules() {
    echo "[7/8] Creating custom alert rules..."
    cat > /etc/netdata/health.d/custom-alerts.conf << 'EOF'
# High CPU usage alert
template: cpu_usage_high
      on: system.cpu
    calc: $user + $system + $softirq + $irq + $guest
   units: %
   every: 10s
    warn: $this > 80
    crit: $this > 95
   delay: down 5m multiplier 1.5 max 1h
    info: CPU utilization is high
      to: sysadmin

# Memory usage alert
template: ram_usage_high
      on: system.ram
    calc: ($used - $buffers - $cached) * 100 / $used
   units: %
   every: 10s
    warn: $this > 80
    crit: $this > 90
   delay: down 5m multiplier 1.5 max 1h
    info: RAM usage is high
      to: sysadmin

# Disk space alert
template: disk_space_usage
      on: disk.space
    calc: $used * 100 / ($avail + $used)
   units: %
   every: 1m
    warn: $this > 80
    crit: $this > 90
   delay: up 1m down 5m multiplier 1.5 max 1h
    info: Disk space usage is high
      to: sysadmin
EOF

    chown netdata:netdata /etc/netdata/health.d/custom-alerts.conf
    chmod 644 /etc/netdata/health.d/custom-alerts.conf
    log_info "Custom alert rules created"
}

# Verify installation
verify_installation() {
    echo "[8/8] Verifying installation..."
    
    # Check if Netdata is running
    if ! systemctl is-active --quiet netdata; then
        log_error "Netdata service is not running"
        exit 1
    fi

    # Check if Netdata is accessible
    if ! curl -s http://localhost:19999 > /dev/null; then
        log_error "Netdata web interface is not accessible"
        exit 1
    fi

    # Test notification configuration
    if ! /usr/libexec/netdata/plugins.d/alarm-notify.sh test 2>/dev/null; then
        log_warn "Notification test failed - please check webhook URLs"
    fi

    log_info "Installation verification completed"
}

# Main execution
main() {
    parse_args "$@"
    check_prerequisites
    detect_distro
    
    update_packages
    install_dependencies
    install_netdata
    setup_notification_config
    configure_slack
    configure_teams
    create_alert_rules
    
    # Restart Netdata to apply configuration
    systemctl restart netdata
    
    verify_installation
    
    echo -e "${GREEN}Netdata with Slack/Teams notifications has been successfully installed!${NC}"
    echo "Access Netdata at: http://$(hostname -I | awk '{print $1}'):19999"
    [[ -n "$SLACK_WEBHOOK" ]] && echo "Slack notifications: Configured for $SLACK_CHANNEL"
    [[ -n "$TEAMS_WEBHOOK" ]] && echo "Teams notifications: Configured for $TEAMS_CHANNEL"
}

main "$@"