Optimize Varnish 7 cache storage with memory tuning and persistence configuration

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Configuration
CACHE_DIR="/var/cache/varnish"
STORAGE_SIZE="${VARNISH_STORAGE_SIZE:-4G}"
MEMORY_SIZE="${VARNISH_MEMORY_SIZE:-2G}"
LISTEN_PORT="${VARNISH_PORT:-80}"
BACKEND_HOST="${BACKEND_HOST:-127.0.0.1:8080}"

# Functions
log() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

error() {
    echo -e "${RED}[ERROR]${NC} $1"
    exit 1
}

usage() {
    echo "Usage: $0 [options]"
    echo "Options:"
    echo "  --storage-size SIZE    Cache storage size (default: 4G)"
    echo "  --memory-size SIZE     Memory allocation (default: 2G)"
    echo "  --port PORT           Listen port (default: 80)"
    echo "  --backend HOST:PORT   Backend server (default: 127.0.0.1:8080)"
    echo "  --help                Show this help"
    exit 0
}

cleanup() {
    if [[ "${VARNISH_STOPPED:-}" == "true" ]]; then
        warn "Failure detected, restarting Varnish with original configuration"
        systemctl start varnish || true
    fi
}

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --storage-size) STORAGE_SIZE="$2"; shift 2 ;;
        --memory-size) MEMORY_SIZE="$2"; shift 2 ;;
        --port) LISTEN_PORT="$2"; shift 2 ;;
        --backend) BACKEND_HOST="$2"; shift 2 ;;
        --help) usage ;;
        *) error "Unknown option: $1" ;;
    esac
done

# Trap for cleanup on error
trap cleanup ERR

# Check prerequisites
[[ $EUID -eq 0 ]] || error "This script must be run as root"

echo "[1/10] Detecting distribution..."
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update"
            VARNISH_CONFIG="/etc/varnish/default.vcl"
            SYSTEMD_DIR="/etc/systemd/system"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf makecache"
            VARNISH_CONFIG="/etc/varnish/default.vcl"
            SYSTEMD_DIR="/etc/systemd/system"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum makecache"
            VARNISH_CONFIG="/etc/varnish/default.vcl"
            SYSTEMD_DIR="/etc/systemd/system"
            ;;
        *)
            error "Unsupported distribution: $ID"
            ;;
    esac
    log "Detected: $PRETTY_NAME"
else
    error "Cannot detect distribution"
fi

echo "[2/10] Updating package cache..."
$PKG_UPDATE > /dev/null 2>&1

echo "[3/10] Installing Varnish 7..."
if ! command -v varnishd &> /dev/null; then
    case "$PKG_MGR" in
        apt)
            $PKG_INSTALL curl gnupg2 ca-certificates lsb-release
            curl -fsSL https://packagecloud.io/varnishcache/varnish70/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/varnish.gpg
            echo "deb https://packagecloud.io/varnishcache/varnish70/$ID/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/varnishcache_varnish70.list
            apt update
            $PKG_INSTALL varnish
            ;;
        dnf|yum)
            curl -fsSL https://packagecloud.io/varnishcache/varnish70/config_file.repo -o /etc/yum.repos.d/varnishcache_varnish70.repo
            $PKG_INSTALL varnish
            ;;
    esac
else
    log "Varnish already installed"
fi

echo "[4/10] Creating cache directory structure..."
mkdir -p "$CACHE_DIR"
chown varnish:varnish "$CACHE_DIR"
chmod 755 "$CACHE_DIR"

echo "[5/10] Stopping Varnish for configuration..."
VARNISH_STOPPED="true"
systemctl stop varnish

echo "[6/10] Configuring storage backend and memory allocation..."
mkdir -p "${SYSTEMD_DIR}/varnish.service.d"
cat > "${SYSTEMD_DIR}/varnish.service.d/override.conf" << EOF
[Service]
ExecStart=
ExecStart=/usr/sbin/varnishd \\
    -a :${LISTEN_PORT} \\
    -T 127.0.0.1:6082 \\
    -f ${VARNISH_CONFIG} \\
    -s file,${CACHE_DIR}/cache.bin,${STORAGE_SIZE} \\
    -s malloc,${MEMORY_SIZE} \\
    -p workspace_backend=256k \\
    -p workspace_client=256k \\
    -p http_max_hdr=128 \\
    -p thread_pool_min=50 \\
    -p thread_pool_max=5000 \\
    -p thread_pools=4 \\
    -p session_linger=50 \\
    -p feature=+http2
EOF

echo "[7/10] Creating optimized VCL configuration..."
cat > "$VARNISH_CONFIG" << 'EOF'
vcl 4.1;

backend default {
    .host = "127.0.0.1";
    .port = "8080";
    .connect_timeout = 5s;
    .first_byte_timeout = 30s;
    .between_bytes_timeout = 5s;
    .max_connections = 50;
    .probe = {
        .url = "/";
        .timeout = 3s;
        .interval = 10s;
        .window = 5;
        .threshold = 3;
    };
}

sub vcl_recv {
    if (req.method == "PURGE") {
        if (!client.ip ~ purge) {
            return (synth(405, "Method not allowed"));
        }
        return (purge);
    }
    
    if (req.method != "GET" && req.method != "HEAD") {
        return (pass);
    }
    
    if (req.url ~ "\.(css|js|png|gif|jp(e)?g|swf|ico|woff|woff2)$") {
        unset req.http.Cookie;
        set req.ttl = 1h;
    }
    
    if (req.http.Cache-Control ~ "no-cache") {
        return (pass);
    }
    
    return (hash);
}

sub vcl_hash {
    hash_data(req.url);
    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }
    return (lookup);
}

sub vcl_backend_response {
    if (beresp.status == 200 || beresp.status == 301 || beresp.status == 302) {
        if (bereq.url ~ "\.(css|js|png|gif|jp(e)?g|swf|ico|woff|woff2)$") {
            unset beresp.http.Set-Cookie;
            set beresp.ttl = 24h;
            set beresp.http.Cache-Control = "public, max-age=86400";
        } else {
            set beresp.ttl = 2m;
        }
    }
    
    if (beresp.ttl <= 0s) {
        set beresp.ttl = 120s;
        set beresp.uncacheable = true;
        return (deliver);
    }
    
    return (deliver);
}

sub vcl_deliver {
    if (obj.hits > 0) {
        set resp.http.X-Varnish-Cache = "HIT";
    } else {
        set resp.http.X-Varnish-Cache = "MISS";
    }
    
    unset resp.http.Via;
    unset resp.http.X-Varnish;
}

acl purge {
    "localhost";
    "127.0.0.1";
    "::1";
}
EOF

# Update backend in VCL if custom backend provided
if [[ "$BACKEND_HOST" != "127.0.0.1:8080" ]]; then
    IFS=':' read -r BACKEND_IP BACKEND_PORT <<< "$BACKEND_HOST"
    sed -i "s/127.0.0.1/$BACKEND_IP/" "$VARNISH_CONFIG"
    sed -i "s/8080/$BACKEND_PORT/" "$VARNISH_CONFIG"
fi

echo "[8/10] Creating cache management script..."
cat > /usr/local/bin/varnish-cache-manager.sh << 'EOF'
#!/bin/bash
CACHE_DIR="/var/cache/varnish"
STORAGE_FILE="$CACHE_DIR/cache.bin"

case "$1" in
    init)
        mkdir -p "$CACHE_DIR"
        chown varnish:varnish "$CACHE_DIR"
        chmod 755 "$CACHE_DIR"
        ;;
    cleanup)
        find "$CACHE_DIR" -name "*.tmp" -mtime +1 -delete 2>/dev/null || true
        ;;
    status)
        echo "Cache directory: $CACHE_DIR"
        if [[ -f "$STORAGE_FILE" ]]; then
            echo "Storage size: $(du -h "$STORAGE_FILE" 2>/dev/null | cut -f1 || echo 'N/A')"
        fi
        df -h "$CACHE_DIR" 2>/dev/null || echo "Directory not accessible"
        ;;
    *)
        echo "Usage: $0 {init|cleanup|status}"
        exit 1
        ;;
esac
EOF

chmod 755 /usr/local/bin/varnish-cache-manager.sh
/usr/local/bin/varnish-cache-manager.sh init

echo "[9/10] Setting up systemd timer for maintenance..."
cat > /etc/systemd/system/varnish-maintenance.service << EOF
[Unit]
Description=Varnish Cache Maintenance
After=varnish.service

[Service]
Type=oneshot
User=root
ExecStart=/usr/local/bin/varnish-cache-manager.sh cleanup
EOF

cat > /etc/systemd/system/varnish-maintenance.timer << EOF
[Unit]
Description=Run Varnish Cache Maintenance
Requires=varnish-maintenance.service

[Timer]
OnCalendar=daily
Persistent=true

[Install]
WantedBy=timers.target
EOF

systemctl daemon-reload
systemctl enable varnish-maintenance.timer
systemctl start varnish-maintenance.timer

echo "[10/10] Starting and enabling Varnish..."
systemctl daemon-reload
systemctl enable varnish
systemctl start varnish
VARNISH_STOPPED="false"

# Verification
sleep 3
if systemctl is-active --quiet varnish; then
    log "✓ Varnish is running successfully"
    log "✓ Storage backend configured: file,${CACHE_DIR}/cache.bin,${STORAGE_SIZE}"
    log "✓ Memory allocation: malloc,${MEMORY_SIZE}"
    log "✓ Listening on port: ${LISTEN_PORT}"
    log "✓ Backend configured: ${BACKEND_HOST}"
    
    # Show statistics
    if command -v varnishstat &> /dev/null; then
        echo -e "\n${GREEN}Current Varnish Statistics:${NC}"
        varnishstat -1 | grep -E "cache_hit|cache_miss|n_object|g_bytes" || true
    fi
else
    error "Varnish failed to start. Check logs with: journalctl -u varnish"
fi

log "Varnish 7 optimization complete!"
log "Management script: /usr/local/bin/varnish-cache-manager.sh"
log "View status: /usr/local/bin/varnish-cache-manager.sh status"