Monitor container performance with Prometheus and cAdvisor for comprehensive metrics collection

#!/usr/bin/env bash
set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Global variables
INSTALL_DIR="${HOME}/container-monitoring"
COMPOSE_VERSION="v2.24.0"

# Error handling and cleanup
cleanup() {
    if [ $? -ne 0 ]; then
        echo -e "${RED}[ERROR] Installation failed. Cleaning up...${NC}"
        if [ -d "$INSTALL_DIR" ]; then
            cd "$INSTALL_DIR" && docker compose down 2>/dev/null || true
            rm -rf "$INSTALL_DIR"
        fi
    fi
}
trap cleanup EXIT

# Usage message
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -h, --help     Show this help message"
    echo "  -d, --dir DIR  Installation directory (default: ~/container-monitoring)"
    exit 1
}

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -h|--help) usage ;;
        -d|--dir) INSTALL_DIR="$2"; shift ;;
        *) echo -e "${RED}Unknown option: $1${NC}"; usage ;;
    esac
    shift
done

# Detect distribution and package manager
detect_distro() {
    if [ ! -f /etc/os-release ]; then
        echo -e "${RED}Cannot detect Linux distribution${NC}"
        exit 1
    fi
    
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_UPDATE="apt update && apt upgrade -y"
            PKG_INSTALL="apt install -y"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_UPDATE="yum update -y"
            PKG_INSTALL="yum install -y"
            ;;
        *) 
            echo -e "${RED}Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
}

# Check prerequisites
check_prerequisites() {
    echo -e "${BLUE}[1/8] Checking prerequisites...${NC}"
    
    if [ "$EUID" -eq 0 ]; then
        echo -e "${RED}Please don't run this script as root${NC}"
        exit 1
    fi
    
    if ! command -v sudo &> /dev/null; then
        echo -e "${RED}sudo is required but not installed${NC}"
        exit 1
    fi
    
    detect_distro
    echo -e "${GREEN}Prerequisites check passed${NC}"
}

# Update system packages
update_system() {
    echo -e "${BLUE}[2/8] Updating system packages...${NC}"
    sudo bash -c "$PKG_UPDATE"
    sudo $PKG_INSTALL curl wget git
    echo -e "${GREEN}System packages updated${NC}"
}

# Install Docker
install_docker() {
    echo -e "${BLUE}[3/8] Installing Docker...${NC}"
    
    if command -v docker &> /dev/null; then
        echo -e "${YELLOW}Docker already installed${NC}"
        return
    fi
    
    curl -fsSL https://get.docker.com -o get-docker.sh
    sudo sh get-docker.sh
    sudo usermod -aG docker "$USER"
    rm -f get-docker.sh
    
    # Install docker-compose plugin
    case "$PKG_MGR" in
        apt) sudo $PKG_INSTALL docker-compose-plugin ;;
        dnf|yum) sudo $PKG_INSTALL docker-compose-plugin ;;
    esac
    
    sudo systemctl enable --now docker
    echo -e "${GREEN}Docker installed successfully${NC}"
    echo -e "${YELLOW}Please log out and back in, or run 'newgrp docker' for group changes to take effect${NC}"
}

# Create directory structure
create_directories() {
    echo -e "${BLUE}[4/8] Creating directory structure...${NC}"
    
    mkdir -p "$INSTALL_DIR"/{prometheus,grafana,configs}
    mkdir -p "$INSTALL_DIR"/grafana/{dashboards,provisioning/{dashboards,datasources}}
    
    echo -e "${GREEN}Directory structure created${NC}"
}

# Create configuration files
create_configs() {
    echo -e "${BLUE}[5/8] Creating configuration files...${NC}"
    
    # Prometheus configuration
    cat > "$INSTALL_DIR/prometheus/prometheus.yml" << 'EOF'
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "/etc/prometheus/alerts.yml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'cadvisor'
    static_configs:
      - targets: ['cadvisor:8080']
    scrape_interval: 5s
    metrics_path: /metrics

  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']
EOF

    # Prometheus alerts
    cat > "$INSTALL_DIR/prometheus/alerts.yml" << 'EOF'
groups:
  - name: container.rules
    rules:
      - alert: ContainerHighCpuUsage
        expr: rate(container_cpu_usage_seconds_total{name!=""}[1m]) * 100 > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container {{ $labels.name }} high CPU usage"
          description: "Container {{ $labels.name }} CPU usage is above 80% for more than 5 minutes."

      - alert: ContainerHighMemoryUsage
        expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""}) * 100 > 90
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Container {{ $labels.name }} high memory usage"
          description: "Container {{ $labels.name }} memory usage is above 90% for more than 5 minutes."

      - alert: ContainerDown
        expr: up{job="cadvisor"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "cAdvisor is down"
          description: "cAdvisor has been down for more than 1 minute."
EOF

    # Grafana datasource
    cat > "$INSTALL_DIR/grafana/provisioning/datasources/prometheus.yml" << 'EOF'
apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
EOF

    # Set proper permissions
    chmod -R 755 "$INSTALL_DIR"
    find "$INSTALL_DIR" -type f -exec chmod 644 {} \;
    
    echo -e "${GREEN}Configuration files created${NC}"
}

# Create docker-compose file
create_compose() {
    echo -e "${BLUE}[6/8] Creating Docker Compose configuration...${NC}"
    
    cat > "$INSTALL_DIR/docker-compose.yml" << 'EOF'
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    restart: unless-stopped
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - ./prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--storage.tsdb.retention.time=200h'
      - '--web.enable-lifecycle'

  cadvisor:
    image: gcr.io/cadvisor/cadvisor:latest
    container_name: cadvisor
    restart: unless-stopped
    ports:
      - "8080:8080"
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:rw
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /dev/disk/:/dev/disk:ro
    privileged: true
    devices:
      - /dev/kmsg:/dev/kmsg

  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    restart: unless-stopped
    ports:
      - "9100:9100"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.rootfs=/rootfs'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_USERS_ALLOW_SIGN_UP=false

volumes:
  prometheus_data:
  grafana_data:
EOF

    chmod 644 "$INSTALL_DIR/docker-compose.yml"
    echo -e "${GREEN}Docker Compose configuration created${NC}"
}

# Start services
start_services() {
    echo -e "${BLUE}[7/8] Starting monitoring services...${NC}"
    
    cd "$INSTALL_DIR"
    
    # Ensure user can run docker
    if ! groups | grep -q docker; then
        echo -e "${YELLOW}Adding user to docker group temporarily...${NC}"
        newgrp docker << EONG
        docker compose up -d
EONG
    else
        docker compose up -d
    fi
    
    echo -e "${GREEN}Services started successfully${NC}"
}

# Verify installation
verify_installation() {
    echo -e "${BLUE}[8/8] Verifying installation...${NC}"
    
    cd "$INSTALL_DIR"
    
    # Wait for services to start
    sleep 10
    
    # Check service status
    if docker compose ps | grep -q "Up"; then
        echo -e "${GREEN}✓ Docker containers are running${NC}"
    else
        echo -e "${RED}✗ Some containers failed to start${NC}"
        return 1
    fi
    
    # Check endpoints
    local endpoints=("http://localhost:9090" "http://localhost:8080" "http://localhost:3000" "http://localhost:9100")
    for endpoint in "${endpoints[@]}"; do
        if curl -s "$endpoint" > /dev/null; then
            echo -e "${GREEN}✓ $endpoint is accessible${NC}"
        else
            echo -e "${YELLOW}⚠ $endpoint is not yet ready${NC}"
        fi
    done
    
    echo -e "${GREEN}Installation completed successfully!${NC}"
    echo -e "${BLUE}Access points:${NC}"
    echo -e "  Prometheus: http://localhost:9090"
    echo -e "  cAdvisor: http://localhost:8080"
    echo -e "  Grafana: http://localhost:3000 (admin/admin)"
    echo -e "  Node Exporter: http://localhost:9100"
}

# Main execution
main() {
    check_prerequisites
    update_system
    install_docker
    create_directories
    create_configs
    create_compose
    start_services
    verify_installation
}

main "$@"