Set up Telegraf custom plugins for application monitoring with Prometheus and InfluxDB integration

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
readonly RED='\033[0;31m'
readonly GREEN='\033[0;32m'
readonly YELLOW='\033[1;33m'
readonly BLUE='\033[0;34m'
readonly NC='\033[0m' # No Color

# Global variables
SCRIPT_NAME=$(basename "$0")
TEMP_FILES=()
SERVICES_STARTED=()

# Cleanup function
cleanup() {
    local exit_code=$?
    echo -e "\n${YELLOW}[CLEANUP] Cleaning up temporary files...${NC}"
    
    for file in "${TEMP_FILES[@]}"; do
        [ -f "$file" ] && rm -f "$file"
    done
    
    if [ $exit_code -ne 0 ]; then
        echo -e "${RED}[ERROR] Installation failed. Stopping started services...${NC}"
        for service in "${SERVICES_STARTED[@]}"; do
            systemctl stop "$service" 2>/dev/null || true
            systemctl disable "$service" 2>/dev/null || true
        done
    fi
    
    exit $exit_code
}

# Set up error handling
trap cleanup ERR EXIT

usage() {
    cat << EOF
Usage: $SCRIPT_NAME [OPTIONS]

Install and configure Telegraf with custom plugin support for application monitoring

OPTIONS:
    -h, --help          Show this help message
    -p, --prometheus    Enable Prometheus output (default: disabled)
    -i, --influxdb      Enable InfluxDB output (default: disabled)
    --influx-url URL    InfluxDB URL (default: http://localhost:8086)
    --influx-db NAME    InfluxDB database name (default: telegraf)

Examples:
    $SCRIPT_NAME --prometheus --influxdb
    $SCRIPT_NAME -p -i --influx-url http://influxdb:8086 --influx-db metrics

EOF
}

log_step() {
    echo -e "${BLUE}[$1] $2${NC}"
}

log_success() {
    echo -e "${GREEN}[SUCCESS] $1${NC}"
}

log_error() {
    echo -e "${RED}[ERROR] $1${NC}"
}

log_warning() {
    echo -e "${YELLOW}[WARNING] $1${NC}"
}

# Parse command line arguments
ENABLE_PROMETHEUS=false
ENABLE_INFLUXDB=false
INFLUX_URL="http://localhost:8086"
INFLUX_DB="telegraf"

while [[ $# -gt 0 ]]; do
    case $1 in
        -h|--help)
            usage
            exit 0
            ;;
        -p|--prometheus)
            ENABLE_PROMETHEUS=true
            shift
            ;;
        -i|--influxdb)
            ENABLE_INFLUXDB=true
            shift
            ;;
        --influx-url)
            INFLUX_URL="$2"
            shift 2
            ;;
        --influx-db)
            INFLUX_DB="$2"
            shift 2
            ;;
        *)
            log_error "Unknown option: $1"
            usage
            exit 1
            ;;
    esac
done

# Check if at least one output is enabled
if [ "$ENABLE_PROMETHEUS" = false ] && [ "$ENABLE_INFLUXDB" = false ]; then
    log_warning "No outputs enabled. Enabling InfluxDB output by default."
    ENABLE_INFLUXDB=true
fi

# Check prerequisites
log_step "1/8" "Checking prerequisites..."

if [[ $EUID -ne 0 ]]; then
    log_error "This script must be run as root or with sudo"
    exit 1
fi

# Detect distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf check-update || true"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum check-update || true"
            ;;
        *)
            log_error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
else
    log_error "Cannot detect distribution. /etc/os-release not found."
    exit 1
fi

log_success "Running on $PRETTY_NAME with $PKG_MGR package manager"

# Check for required tools
for tool in curl wget systemctl; do
    if ! command -v "$tool" &> /dev/null; then
        log_error "$tool is required but not installed"
        exit 1
    fi
done

# Install prerequisites
log_step "2/8" "Installing prerequisites..."

case "$PKG_MGR" in
    apt)
        $PKG_UPDATE
        $PKG_INSTALL curl wget gnupg2 lsb-release
        ;;
    dnf|yum)
        $PKG_UPDATE
        $PKG_INSTALL curl wget gnupg2
        ;;
esac

log_success "Prerequisites installed"

# Add InfluxData repository
log_step "3/8" "Adding InfluxData repository..."

case "$PKG_MGR" in
    apt)
        # Add GPG key
        curl -sL https://repos.influxdata.com/influxdata-archive_compat.key | apt-key add -
        
        # Add repository
        echo "deb https://repos.influxdata.com/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/influxdata.list
        
        # Update package list
        $PKG_UPDATE
        ;;
    dnf|yum)
        # Create repository file
        cat > /etc/yum.repos.d/influxdata.repo << 'EOF'
[influxdata]
name = InfluxData Repository - Stable
baseurl = https://repos.influxdata.com/stable/\$basearch/main
enabled = 1
gpgcheck = 1
gpgkey = https://repos.influxdata.com/influxdata-archive_compat.key
EOF
        ;;
esac

log_success "InfluxData repository added"

# Install Telegraf
log_step "4/8" "Installing Telegraf..."

$PKG_INSTALL telegraf

log_success "Telegraf installed"

# Create custom plugins directory
log_step "5/8" "Setting up custom plugins directory..."

PLUGINS_DIR="/etc/telegraf/plugins"
mkdir -p "$PLUGINS_DIR"
chown telegraf:telegraf "$PLUGINS_DIR"
chmod 755 "$PLUGINS_DIR"

# Create example custom plugin script
cat > "$PLUGINS_DIR/example_app_metrics.sh" << 'EOF'
#!/bin/bash
# Example custom plugin for application metrics
# This script should output metrics in InfluxDB line protocol format

# Example: Application response time metric
RESPONSE_TIME=$(curl -o /dev/null -s -w '%{time_total}' http://localhost:8080/health 2>/dev/null || echo "0")
echo "app_metrics,service=example_app response_time=${RESPONSE_TIME}"

# Example: Application memory usage
if command -v pgrep &> /dev/null; then
    APP_PID=$(pgrep -f "example_app" | head -1)
    if [ -n "$APP_PID" ] && [ -f "/proc/$APP_PID/status" ]; then
        MEMORY_KB=$(grep VmRSS /proc/$APP_PID/status | awk '{print $2}')
        MEMORY_BYTES=$((MEMORY_KB * 1024))
        echo "app_metrics,service=example_app memory_usage=${MEMORY_BYTES}"
    fi
fi
EOF

chmod 755 "$PLUGINS_DIR/example_app_metrics.sh"
chown telegraf:telegraf "$PLUGINS_DIR/example_app_metrics.sh"

log_success "Custom plugins directory created"

# Backup original configuration
log_step "6/8" "Configuring Telegraf..."

cp /etc/telegraf/telegraf.conf /etc/telegraf/telegraf.conf.backup

# Create new Telegraf configuration
cat > /etc/telegraf/telegraf.conf << EOF
# Telegraf Configuration with Custom Plugins

[global_tags]
  environment = "production"

[agent]
  interval = "10s"
  round_interval = true
  metric_batch_size = 1000
  metric_buffer_limit = 10000
  collection_jitter = "0s"
  flush_interval = "10s"
  flush_jitter = "0s"
  precision = ""
  hostname = ""
  omit_hostname = false

###############################################################################
#                            OUTPUT PLUGINS                                   #
###############################################################################

EOF

# Add InfluxDB output if enabled
if [ "$ENABLE_INFLUXDB" = true ]; then
    cat >> /etc/telegraf/telegraf.conf << EOF
# InfluxDB Output Plugin
[[outputs.influxdb]]
  urls = ["${INFLUX_URL}"]
  database = "${INFLUX_DB}"
  timeout = "5s"

EOF
fi

# Add Prometheus output if enabled
if [ "$ENABLE_PROMETHEUS" = true ]; then
    cat >> /etc/telegraf/telegraf.conf << EOF
# Prometheus Output Plugin
[[outputs.prometheus_client]]
  listen = ":9273"
  metric_version = 2

EOF
fi

# Add input plugins configuration
cat >> /etc/telegraf/telegraf.conf << EOF
###############################################################################
#                            INPUT PLUGINS                                    #
###############################################################################

# System metrics
[[inputs.cpu]]
  percpu = true
  totalcpu = true
  collect_cpu_time = false
  report_active = false

[[inputs.disk]]
  ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"]

[[inputs.diskio]]

[[inputs.kernel]]

[[inputs.mem]]

[[inputs.processes]]

[[inputs.swap]]

[[inputs.system]]

[[inputs.net]]

# Custom application metrics via exec plugin
[[inputs.exec]]
  commands = [
    "${PLUGINS_DIR}/example_app_metrics.sh"
  ]
  timeout = "5s"
  data_format = "influx"
  interval = "30s"

# HTTP response monitoring
[[inputs.http_response]]
  urls = ["http://localhost:8080/health"]
  method = "GET"
  response_timeout = "5s"
  follow_redirects = false
  [inputs.http_response.tags]
    service = "example_app"
EOF

# Set proper permissions for configuration
chown telegraf:telegraf /etc/telegraf/telegraf.conf
chmod 644 /etc/telegraf/telegraf.conf

log_success "Telegraf configuration created"

# Configure firewall if Prometheus output is enabled
log_step "7/8" "Configuring firewall..."

if [ "$ENABLE_PROMETHEUS" = true ]; then
    if command -v firewall-cmd &> /dev/null; then
        # RHEL-based systems with firewalld
        if systemctl is-active --quiet firewalld; then
            firewall-cmd --permanent --add-port=9273/tcp
            firewall-cmd --reload
            log_success "Firewall configured for Prometheus (port 9273)"
        fi
    elif command -v ufw &> /dev/null; then
        # Ubuntu/Debian with ufw
        if ufw status | grep -q "Status: active"; then
            ufw allow 9273/tcp
            log_success "Firewall configured for Prometheus (port 9273)"
        fi
    fi
else
    log_success "Firewall configuration skipped (Prometheus not enabled)"
fi

# Start and enable Telegraf service
log_step "8/8" "Starting Telegraf service..."

systemctl enable telegraf
systemctl start telegraf
SERVICES_STARTED+=("telegraf")

# Wait a moment for service to initialize
sleep 3

log_success "Telegraf service started and enabled"

# Verification checks
echo -e "\n${BLUE}[VERIFICATION] Running verification checks...${NC}"

# Check service status
if systemctl is-active --quiet telegraf; then
    log_success "Telegraf service is running"
else
    log_error "Telegraf service is not running"
    systemctl status telegraf
    exit 1
fi

# Check configuration syntax
if telegraf --test --config /etc/telegraf/telegraf.conf &>/dev/null; then
    log_success "Telegraf configuration is valid"
else
    log_error "Telegraf configuration has errors"
    telegraf --test --config /etc/telegraf/telegraf.conf
    exit 1
fi

# Check if Prometheus endpoint is accessible (if enabled)
if [ "$ENABLE_PROMETHEUS" = true ]; then
    if curl -sf http://localhost:9273/metrics > /dev/null; then
        log_success "Prometheus metrics endpoint is accessible"
    else
        log_warning "Prometheus metrics endpoint is not accessible yet (may need more time to initialize)"
    fi
fi

# Display final information
echo -e "\n${GREEN}[INSTALLATION COMPLETE]${NC}"
echo "=============================="
echo "Telegraf has been successfully installed and configured!"
echo
echo "Configuration file: /etc/telegraf/telegraf.conf"
echo "Custom plugins directory: $PLUGINS_DIR"
echo "Service status: $(systemctl is-active telegraf)"
echo
if [ "$ENABLE_PROMETHEUS" = true ]; then
    echo "Prometheus metrics: http://$(hostname -I | awk '{print $1}'):9273/metrics"
fi
if [ "$ENABLE_INFLUXDB" = true ]; then
    echo "InfluxDB output: $INFLUX_URL (database: $INFLUX_DB)"
fi
echo
echo "To add custom metrics:"
echo "1. Create executable scripts in $PLUGINS_DIR"
echo "2. Add [[inputs.exec]] sections to telegraf.conf"
echo "3. Restart telegraf: systemctl restart telegraf"
echo
echo "Logs: journalctl -u telegraf -f"