Set up NGINX monitoring with Prometheus and Grafana for web server observability

#!/usr/bin/env bash
set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Script variables
PROMETHEUS_VERSION="2.48.0"
NGINX_EXPORTER_VERSION="0.11.0"
GRAFANA_PORT="${1:-3000}"

# Usage function
usage() {
    echo "Usage: $0 [grafana_port]"
    echo "  grafana_port: Port for Grafana (default: 3000)"
    exit 1
}

# Logging functions
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

# Cleanup function for rollback
cleanup() {
    log_warn "Installation failed. Cleaning up..."
    systemctl stop prometheus nginx-exporter grafana-server 2>/dev/null || true
    systemctl disable prometheus nginx-exporter grafana-server 2>/dev/null || true
    rm -f /etc/systemd/system/prometheus.service /etc/systemd/system/nginx-exporter.service
    systemctl daemon-reload
    log_warn "Cleanup completed"
}

# Set trap for cleanup on error
trap cleanup ERR

# Check if running as root
if [[ $EUID -ne 0 ]]; then
    log_error "This script must be run as root"
    exit 1
fi

# Validate arguments
if [[ $# -gt 1 ]]; then
    usage
fi

if [[ $# -eq 1 ]] && ! [[ "$1" =~ ^[0-9]+$ ]] || [[ "$1" -lt 1024 ]] || [[ "$1" -gt 65535 ]]; then
    log_error "Port must be a number between 1024 and 65535"
    exit 1
fi

# Detect distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_UPDATE="apt update"
            PKG_INSTALL="apt install -y"
            NGINX_CONF_DIR="/etc/nginx/sites-available"
            NGINX_CONF_ENABLE_DIR="/etc/nginx/sites-enabled"
            NGINX_CONF_FILE="nginx-monitoring"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            NGINX_CONF_DIR="/etc/nginx/conf.d"
            NGINX_CONF_ENABLE_DIR=""
            NGINX_CONF_FILE="nginx-monitoring.conf"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_UPDATE="yum update -y"
            PKG_INSTALL="yum install -y"
            NGINX_CONF_DIR="/etc/nginx/conf.d"
            NGINX_CONF_ENABLE_DIR=""
            NGINX_CONF_FILE="nginx-monitoring.conf"
            ;;
        *) 
            log_error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
else
    log_error "Cannot detect distribution"
    exit 1
fi

log_info "[1/10] Updating system packages..."
$PKG_UPDATE
$PKG_INSTALL wget curl gnupg2 nginx

log_info "[2/10] Installing Prometheus..."
cd /tmp
wget -q "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz"
tar xzf "prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz"
mv "prometheus-${PROMETHEUS_VERSION}.linux-amd64" /opt/prometheus
useradd --no-create-home --shell /bin/false prometheus 2>/dev/null || true
mkdir -p /etc/prometheus /var/lib/prometheus
chown -R prometheus:prometheus /etc/prometheus /var/lib/prometheus /opt/prometheus

log_info "[3/10] Configuring Prometheus systemd service..."
cat > /etc/systemd/system/prometheus.service << 'EOF'
[Unit]
Description=Prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/opt/prometheus/prometheus \
    --config.file /etc/prometheus/prometheus.yml \
    --storage.tsdb.path /var/lib/prometheus/ \
    --web.console.templates=/opt/prometheus/consoles \
    --web.console.libraries=/opt/prometheus/console_libraries \
    --web.listen-address=0.0.0.0:9090 \
    --web.enable-lifecycle
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

log_info "[4/10] Configuring NGINX stub status..."
cat > "${NGINX_CONF_DIR}/${NGINX_CONF_FILE}" << 'EOF'
server {
    listen 8080;
    server_name localhost;
    
    location /nginx_status {
        stub_status on;
        access_log off;
        allow 127.0.0.1;
        allow ::1;
        deny all;
    }
}
EOF

# Enable site for Debian-based systems
if [[ -n "$NGINX_CONF_ENABLE_DIR" ]]; then
    ln -sf "${NGINX_CONF_DIR}/${NGINX_CONF_FILE}" "${NGINX_CONF_ENABLE_DIR}/${NGINX_CONF_FILE}"
fi

nginx -t
systemctl enable nginx
systemctl restart nginx

log_info "[5/10] Installing NGINX Prometheus Exporter..."
cd /tmp
wget -q "https://github.com/nginxinc/nginx-prometheus-exporter/releases/download/v${NGINX_EXPORTER_VERSION}/nginx-prometheus-exporter_${NGINX_EXPORTER_VERSION}_linux_amd64.tar.gz"
tar xzf "nginx-prometheus-exporter_${NGINX_EXPORTER_VERSION}_linux_amd64.tar.gz"
mv nginx-prometheus-exporter /usr/local/bin/
chmod 755 /usr/local/bin/nginx-prometheus-exporter
useradd --no-create-home --shell /bin/false nginx-exporter 2>/dev/null || true

log_info "[6/10] Configuring NGINX Exporter service..."
cat > /etc/systemd/system/nginx-exporter.service << 'EOF'
[Unit]
Description=NGINX Prometheus Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=nginx-exporter
Group=nginx-exporter
Type=simple
ExecStart=/usr/local/bin/nginx-prometheus-exporter \
    -nginx.scrape-uri=http://localhost:8080/nginx_status \
    -web.listen-address=:9113
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

log_info "[7/10] Configuring Prometheus..."
cat > /etc/prometheus/prometheus.yml << 'EOF'
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'nginx'
    static_configs:
      - targets: ['localhost:9113']
    scrape_interval: 10s
    metrics_path: /metrics
EOF
chown prometheus:prometheus /etc/prometheus/prometheus.yml

log_info "[8/10] Installing Grafana..."
if [[ "$PKG_MGR" == "apt" ]]; then
    wget -q -O - https://packages.grafana.com/gpg.key | gpg --dearmor > /usr/share/keyrings/grafana.gpg
    echo "deb [signed-by=/usr/share/keyrings/grafana.gpg] https://packages.grafana.com/oss/deb stable main" > /etc/apt/sources.list.d/grafana.list
    apt update
    $PKG_INSTALL grafana
else
    cat > /etc/yum.repos.d/grafana.repo << 'EOF'
[grafana]
name=grafana
baseurl=https://packages.grafana.com/oss/rpm
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://packages.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
EOF
    $PKG_INSTALL grafana
fi

# Configure Grafana port
sed -i "s/;http_port = 3000/http_port = ${GRAFANA_PORT}/" /etc/grafana/grafana.ini

log_info "[9/10] Starting services..."
systemctl daemon-reload
systemctl enable prometheus nginx-exporter grafana-server
systemctl start prometheus nginx-exporter grafana-server

log_info "[10/10] Configuring firewall..."
if command -v ufw >/dev/null 2>&1 && ufw status | grep -q "Status: active"; then
    ufw allow 9090/tcp
    ufw allow 9113/tcp
    ufw allow "${GRAFANA_PORT}/tcp"
elif command -v firewall-cmd >/dev/null 2>&1 && systemctl is-active firewalld >/dev/null 2>&1; then
    firewall-cmd --permanent --add-port=9090/tcp
    firewall-cmd --permanent --add-port=9113/tcp
    firewall-cmd --permanent --add-port="${GRAFANA_PORT}/tcp"
    firewall-cmd --reload
fi

# Remove trap
trap - ERR

log_info "Verifying installation..."
sleep 5

# Verify services
for service in prometheus nginx-exporter grafana-server nginx; do
    if systemctl is-active --quiet "$service"; then
        log_info "✓ $service is running"
    else
        log_error "✗ $service is not running"
        exit 1
    fi
done

# Verify endpoints
if curl -s http://localhost:9090 >/dev/null; then
    log_info "✓ Prometheus is accessible on port 9090"
else
    log_error "✗ Prometheus is not accessible"
    exit 1
fi

if curl -s http://localhost:9113/metrics >/dev/null; then
    log_info "✓ NGINX Exporter is accessible on port 9113"
else
    log_error "✗ NGINX Exporter is not accessible"
    exit 1
fi

if curl -s "http://localhost:${GRAFANA_PORT}" >/dev/null; then
    log_info "✓ Grafana is accessible on port ${GRAFANA_PORT}"
else
    log_error "✗ Grafana is not accessible"
    exit 1
fi

log_info ""
log_info "Installation completed successfully!"
log_info "Access points:"
log_info "  Prometheus: http://$(hostname -I | awk '{print $1}'):9090"
log_info "  Grafana: http://$(hostname -I | awk '{print $1}'):${GRAFANA_PORT} (admin/admin)"
log_info "  NGINX Exporter: http://$(hostname -I | awk '{print $1}'):9113/metrics"