Configure Linux performance monitoring with collectd and InfluxDB 1.8 for real-time metrics collection

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Global variables
SCRIPT_NAME=$(basename "$0")
INFLUXDB_USER="collectd"
INFLUXDB_DB="collectd"
INFLUXDB_PASS=$(openssl rand -base64 12)

# Usage function
usage() {
    echo "Usage: $SCRIPT_NAME [OPTIONS]"
    echo "Configure Linux performance monitoring with collectd and InfluxDB 1.8"
    echo ""
    echo "Options:"
    echo "  -h, --help     Show this help message"
    echo "  -p PASSWORD    Set InfluxDB password (default: auto-generated)"
    exit 1
}

# Cleanup function for rollback
cleanup() {
    echo -e "${RED}[ERROR]${NC} Installation failed. Rolling back changes..."
    systemctl stop collectd influxdb 2>/dev/null || true
    systemctl disable collectd influxdb 2>/dev/null || true
    if [[ "$PKG_MGR" == "apt" ]]; then
        apt remove -y collectd influxdb 2>/dev/null || true
    else
        $PKG_INSTALL remove collectd influxdb 2>/dev/null || true
    fi
    echo -e "${YELLOW}[WARNING]${NC} Rollback completed"
    exit 1
}

trap cleanup ERR

# Check prerequisites
check_prerequisites() {
    echo -e "${YELLOW}[INFO]${NC} Checking prerequisites..."
    
    if [[ $EUID -ne 0 ]]; then
        echo -e "${RED}[ERROR]${NC} This script must be run as root or with sudo"
        exit 1
    fi
    
    if ! command -v curl &> /dev/null && ! command -v wget &> /dev/null; then
        echo -e "${RED}[ERROR]${NC} Either curl or wget is required"
        exit 1
    fi
    
    if ! command -v openssl &> /dev/null; then
        echo -e "${RED}[ERROR]${NC} OpenSSL is required for password generation"
        exit 1
    fi
}

# Detect distribution
detect_distro() {
    if [[ ! -f /etc/os-release ]]; then
        echo -e "${RED}[ERROR]${NC} Cannot detect distribution"
        exit 1
    fi
    
    . /etc/os-release
    
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update"
            PKG_UPGRADE="apt upgrade -y"
            COLLECTD_CONFIG="/etc/collectd/collectd.conf"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf update -y"
            PKG_UPGRADE="dnf upgrade -y"
            COLLECTD_CONFIG="/etc/collectd.conf"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum update -y"
            PKG_UPGRADE="yum upgrade -y"
            COLLECTD_CONFIG="/etc/collectd.conf"
            ;;
        *)
            echo -e "${RED}[ERROR]${NC} Unsupported distribution: $ID"
            exit 1
            ;;
    esac
    
    echo -e "${GREEN}[SUCCESS]${NC} Detected $PRETTY_NAME"
}

# Parse command line arguments
parse_args() {
    while [[ $# -gt 0 ]]; do
        case $1 in
            -h|--help)
                usage
                ;;
            -p)
                INFLUXDB_PASS="$2"
                shift 2
                ;;
            *)
                echo -e "${RED}[ERROR]${NC} Unknown option: $1"
                usage
                ;;
        esac
    done
}

# Update system packages
update_system() {
    echo "[1/6] Updating system packages..."
    $PKG_UPDATE
    $PKG_UPGRADE
    echo -e "${GREEN}[SUCCESS]${NC} System packages updated"
}

# Install InfluxDB 1.8
install_influxdb() {
    echo "[2/6] Installing InfluxDB 1.8..."
    
    if [[ "$PKG_MGR" == "apt" ]]; then
        wget -qO- https://repos.influxdata.com/influxdb.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/influxdb.gpg > /dev/null
        echo "deb [signed-by=/etc/apt/trusted.gpg.d/influxdb.gpg] https://repos.influxdata.com/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/influxdb.list
        apt update
        $PKG_INSTALL influxdb
    else
        cat > /etc/yum.repos.d/influxdb.repo << 'EOF'
[influxdb]
name = InfluxDB Repository - RHEL
baseurl = https://repos.influxdata.com/rhel/\$releasever/\$basearch/stable/
enabled = 1
gpgcheck = 1
gpgkey = https://repos.influxdata.com/influxdb.key
EOF
        $PKG_INSTALL influxdb
    fi
    
    systemctl enable influxdb
    systemctl start influxdb
    
    # Wait for InfluxDB to start
    sleep 5
    
    echo -e "${GREEN}[SUCCESS]${NC} InfluxDB installed and started"
}

# Install and configure collectd
install_collectd() {
    echo "[3/6] Installing collectd..."
    
    if [[ "$PKG_MGR" == "apt" ]]; then
        $PKG_INSTALL collectd collectd-utils
    else
        $PKG_INSTALL collectd
    fi
    
    echo -e "${GREEN}[SUCCESS]${NC} Collectd installed"
}

# Configure InfluxDB
configure_influxdb() {
    echo "[4/6] Configuring InfluxDB..."
    
    # Create database and user
    influx -execute "CREATE DATABASE $INFLUXDB_DB"
    influx -execute "CREATE USER $INFLUXDB_USER WITH PASSWORD '$INFLUXDB_PASS'"
    influx -execute "GRANT ALL ON $INFLUXDB_DB TO $INFLUXDB_USER"
    
    # Configure retention policy
    influx -execute "CREATE RETENTION POLICY \"30_days\" ON \"$INFLUXDB_DB\" DURATION 30d REPLICATION 1 DEFAULT"
    
    echo -e "${GREEN}[SUCCESS]${NC} InfluxDB configured"
}

# Configure collectd
configure_collectd() {
    echo "[5/6] Configuring collectd..."
    
    # Backup original config
    cp "$COLLECTD_CONFIG" "$COLLECTD_CONFIG.backup"
    
    # Create new collectd configuration
    cat > "$COLLECTD_CONFIG" << EOF
# Global settings
Hostname "$(hostname)"
FQDNLookup false
BaseDir "/var/lib/collectd"
PluginDir "/usr/lib/collectd"
TypesDB "/usr/share/collectd/types.db"
AutoLoadPlugin false
CollectInternalStats true

# Logging
LoadPlugin syslog
<Plugin syslog>
    LogLevel info
</Plugin>

# Core plugins
LoadPlugin cpu
LoadPlugin df
LoadPlugin disk
LoadPlugin interface
LoadPlugin load
LoadPlugin memory
LoadPlugin processes
LoadPlugin swap
LoadPlugin uptime

# CPU plugin configuration
<Plugin cpu>
    ReportByCpu true
    ReportByState true
    ValuesPercentage true
</Plugin>

# Disk plugin configuration  
<Plugin df>
    MountPoint "/"
    MountPoint "/boot"
    MountPoint "/home"
    IgnoreSelected false
    ReportByDevice false
    ReportInodes true
    ValuesAbsolute true
    ValuesPercentage true
</Plugin>

# Memory plugin configuration
<Plugin memory>
    ValuesAbsolute true
    ValuesPercentage true
</Plugin>

# Network plugin
LoadPlugin network

# InfluxDB output plugin
LoadPlugin network
<Plugin network>
    <Server "127.0.0.1" "25826">
        SecurityLevel None
    </Server>
</Plugin>

# Write to InfluxDB
LoadPlugin write_http
<Plugin write_http>
    <Node "influxdb">
        URL "http://127.0.0.1:8086/write?db=$INFLUXDB_DB&u=$INFLUXDB_USER&p=$INFLUXDB_PASS"
        Format "JSON"
        StoreRates true
    </Node>
</Plugin>

# Include additional configurations
Include "/etc/collectd/collectd.conf.d"
EOF
    
    # Create include directory
    mkdir -p /etc/collectd/collectd.conf.d
    chown -R collectd:collectd /etc/collectd 2>/dev/null || chown -R root:root /etc/collectd
    chmod 755 /etc/collectd
    chmod 644 "$COLLECTD_CONFIG"
    
    # Enable and start collectd
    systemctl enable collectd
    systemctl start collectd
    
    echo -e "${GREEN}[SUCCESS]${NC} Collectd configured and started"
}

# Verify installation
verify_installation() {
    echo "[6/6] Verifying installation..."
    
    # Check services
    if ! systemctl is-active --quiet influxdb; then
        echo -e "${RED}[ERROR]${NC} InfluxDB service is not running"
        exit 1
    fi
    
    if ! systemctl is-active --quiet collectd; then
        echo -e "${RED}[ERROR]${NC} Collectd service is not running"
        exit 1
    fi
    
    # Wait for data collection
    echo "Waiting 30 seconds for data collection..."
    sleep 30
    
    # Verify data in InfluxDB
    if influx -database "$INFLUXDB_DB" -execute "SHOW MEASUREMENTS" | grep -q cpu; then
        echo -e "${GREEN}[SUCCESS]${NC} Data collection verified"
    else
        echo -e "${YELLOW}[WARNING]${NC} No data found yet, this may be normal for new installations"
    fi
    
    echo -e "${GREEN}[SUCCESS]${NC} Installation completed successfully!"
    echo ""
    echo "=== Configuration Details ==="
    echo "InfluxDB Database: $INFLUXDB_DB"
    echo "InfluxDB User: $INFLUXDB_USER"
    echo "InfluxDB Password: $INFLUXDB_PASS"
    echo "InfluxDB URL: http://localhost:8086"
    echo "Collectd Config: $COLLECTD_CONFIG"
    echo ""
    echo "You can query data using: influx -database $INFLUXDB_DB"
}

# Main execution
main() {
    echo -e "${GREEN}Linux Performance Monitoring Setup${NC}"
    echo "Configuring collectd and InfluxDB 1.8..."
    echo ""
    
    parse_args "$@"
    check_prerequisites
    detect_distro
    update_system
    install_influxdb
    install_collectd
    configure_influxdb
    configure_collectd
    verify_installation
}

main "$@"