Install and configure ClickHouse for high-performance analytics with clustering

#!/usr/bin/env bash
set -euo pipefail

# ClickHouse Production Installation Script
# Supports Ubuntu, Debian, AlmaLinux, Rocky Linux, CentOS, RHEL, Fedora

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Configuration
CLICKHOUSE_PASSWORD=""
CLUSTER_NODES=""

usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -p PASSWORD    Set default user password (required)"
    echo "  -c NODES       Comma-separated cluster nodes (optional)"
    echo "  -h            Show this help"
    echo ""
    echo "Example: $0 -p MySecurePass123 -c \"10.0.1.10,10.0.1.11,10.0.1.12\""
    exit 1
}

log() {
    echo -e "${GREEN}[$(date +'%H:%M:%S')] $1${NC}"
}

warn() {
    echo -e "${YELLOW}[WARNING] $1${NC}"
}

error() {
    echo -e "${RED}[ERROR] $1${NC}"
    exit 1
}

cleanup() {
    if [ $? -ne 0 ]; then
        warn "Installation failed. Cleaning up..."
        systemctl stop clickhouse-server 2>/dev/null || true
        systemctl disable clickhouse-server 2>/dev/null || true
    fi
}

trap cleanup ERR

check_prerequisites() {
    log "[1/12] Checking prerequisites..."
    
    if [[ $EUID -ne 0 ]]; then
        error "This script must be run as root or with sudo"
    fi
    
    if [ -z "$CLICKHOUSE_PASSWORD" ]; then
        error "Password is required. Use -p option."
    fi
    
    if [ ${#CLICKHOUSE_PASSWORD} -lt 8 ]; then
        error "Password must be at least 8 characters long"
    fi
}

detect_distro() {
    log "[2/12] Detecting operating system..."
    
    if [ ! -f /etc/os-release ]; then
        error "Cannot detect OS. /etc/os-release not found."
    fi
    
    . /etc/os-release
    
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update && apt upgrade -y"
            FIREWALL_CMD="ufw"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf update -y"
            FIREWALL_CMD="firewall-cmd"
            if ! command -v dnf &> /dev/null; then
                PKG_MGR="yum"
                PKG_INSTALL="yum install -y"
                PKG_UPDATE="yum update -y"
            fi
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum update -y"
            FIREWALL_CMD="firewall-cmd"
            ;;
        *)
            error "Unsupported distribution: $ID"
            ;;
    esac
    
    log "Detected: $PRETTY_NAME using $PKG_MGR"
}

update_system() {
    log "[3/12] Updating system packages..."
    $PKG_UPDATE
    
    case "$PKG_MGR" in
        apt)
            $PKG_INSTALL apt-transport-https ca-certificates dirmngr curl gnupg2
            ;;
        dnf|yum)
            $PKG_INSTALL curl gnupg2 ca-certificates
            ;;
    esac
}

add_repository() {
    log "[4/12] Adding ClickHouse repository..."
    
    case "$PKG_MGR" in
        apt)
            curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' | gpg --dearmor -o /usr/share/keyrings/clickhouse-keyring.gpg
            echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main" > /etc/apt/sources.list.d/clickhouse.list
            chmod 644 /etc/apt/sources.list.d/clickhouse.list
            apt update
            ;;
        dnf|yum)
            cat > /etc/yum.repos.d/clickhouse.repo << 'EOF'
[clickhouse-stable]
name=ClickHouse - Stable Repository
baseurl=https://packages.clickhouse.com/rpm/stable/
gpgcheck=1
enabled=1
gpgkey=https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key
EOF
            chmod 644 /etc/yum.repos.d/clickhouse.repo
            ;;
    esac
}

install_clickhouse() {
    log "[5/12] Installing ClickHouse..."
    export CLICKHOUSE_PASSWORD_SHA256_HEX=$(echo -n "$CLICKHOUSE_PASSWORD" | sha256sum | cut -d' ' -f1)
    $PKG_INSTALL clickhouse-server clickhouse-client
}

configure_clickhouse() {
    log "[6/12] Configuring ClickHouse..."
    
    # Backup original config
    cp /etc/clickhouse-server/config.xml /etc/clickhouse-server/config.xml.backup
    
    # Create users configuration
    mkdir -p /etc/clickhouse-server/users.d
    cat > /etc/clickhouse-server/users.d/default.xml << EOF
<yandex>
    <users>
        <default>
            <password_sha256_hex>$(echo -n "$CLICKHOUSE_PASSWORD" | sha256sum | cut -d' ' -f1)</password_sha256_hex>
            <networks>
                <ip>::/0</ip>
            </networks>
            <profile>default</profile>
            <quota>default</quota>
        </default>
    </users>
</yandex>
EOF
    
    # Set proper permissions
    chown -R clickhouse:clickhouse /etc/clickhouse-server/users.d
    chmod 640 /etc/clickhouse-server/users.d/default.xml
    
    # Configure main settings
    mkdir -p /etc/clickhouse-server/config.d
    cat > /etc/clickhouse-server/config.d/custom.xml << 'EOF'
<yandex>
    <listen_host>0.0.0.0</listen_host>
    <http_port>8123</http_port>
    <tcp_port>9000</tcp_port>
    <interserver_http_port>9009</interserver_http_port>
    
    <max_connections>2048</max_connections>
    <keep_alive_timeout>3</keep_alive_timeout>
    <max_concurrent_queries>200</max_concurrent_queries>
    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
    <mark_cache_size>5368709120</mark_cache_size>
    
    <path>/var/lib/clickhouse/</path>
    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
    <users_config>/etc/clickhouse-server/users.xml</users_config>
    
    <logger>
        <level>information</level>
        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
        <size>1000M</size>
        <count>3</count>
    </logger>
</yandex>
EOF
    
    chown clickhouse:clickhouse /etc/clickhouse-server/config.d/custom.xml
    chmod 640 /etc/clickhouse-server/config.d/custom.xml
}

configure_cluster() {
    if [ -n "$CLUSTER_NODES" ]; then
        log "[7/12] Configuring cluster..."
        
        cat > /etc/clickhouse-server/config.d/cluster.xml << EOF
<yandex>
    <remote_servers>
        <analytics_cluster>
$(
    IFS=','
    shard=1
    for node in $CLUSTER_NODES; do
        echo "            <shard>"
        echo "                <replica>"
        echo "                    <host>$node</host>"
        echo "                    <port>9000</port>"
        echo "                </replica>"
        echo "            </shard>"
        ((shard++))
    done
)
        </analytics_cluster>
    </remote_servers>
    
    <zookeeper incl="zookeeper-servers" optional="true" />
    <macros>
        <cluster>analytics_cluster</cluster>
        <shard>01</shard>
        <replica>$(hostname)</replica>
    </macros>
</yandex>
EOF
        
        chown clickhouse:clickhouse /etc/clickhouse-server/config.d/cluster.xml
        chmod 640 /etc/clickhouse-server/config.d/cluster.xml
    else
        log "[7/12] Skipping cluster configuration (no nodes specified)"
    fi
}

configure_firewall() {
    log "[8/12] Configuring firewall..."
    
    case "$FIREWALL_CMD" in
        ufw)
            if command -v ufw &> /dev/null; then
                ufw --force enable
                ufw allow 8123/tcp comment "ClickHouse HTTP"
                ufw allow 9000/tcp comment "ClickHouse TCP"
                ufw allow 9009/tcp comment "ClickHouse Interserver"
                ufw reload
            fi
            ;;
        firewall-cmd)
            if command -v firewall-cmd &> /dev/null && systemctl is-active --quiet firewalld; then
                firewall-cmd --permanent --add-port=8123/tcp --add-port=9000/tcp --add-port=9009/tcp
                firewall-cmd --reload
            fi
            ;;
    esac
}

configure_selinux() {
    log "[9/12] Configuring SELinux..."
    
    if command -v setsebool &> /dev/null && [ -f /sys/fs/selinux/enforce ]; then
        setsebool -P httpd_can_network_connect 1 2>/dev/null || true
        semanage port -a -t http_port_t -p tcp 8123 2>/dev/null || true
        semanage port -a -t mysqld_port_t -p tcp 9000 2>/dev/null || true
        semanage port -a -t http_port_t -p tcp 9009 2>/dev/null || true
    fi
}

start_services() {
    log "[10/12] Starting ClickHouse service..."
    
    systemctl enable clickhouse-server
    systemctl start clickhouse-server
    
    # Wait for service to be ready
    for i in {1..30}; do
        if systemctl is-active --quiet clickhouse-server; then
            break
        fi
        sleep 2
    done
}

verify_installation() {
    log "[11/12] Verifying installation..."
    
    # Check service status
    if ! systemctl is-active --quiet clickhouse-server; then
        error "ClickHouse service is not running"
    fi
    
    # Test connection
    sleep 5
    if ! echo "SELECT version()" | clickhouse-client --password="$CLICKHOUSE_PASSWORD" &>/dev/null; then
        error "Cannot connect to ClickHouse server"
    fi
    
    # Test HTTP interface
    if ! curl -s "http://localhost:8123/?query=SELECT%201" &>/dev/null; then
        error "HTTP interface is not responding"
    fi
    
    log "ClickHouse is running and accessible"
}

show_summary() {
    log "[12/12] Installation completed successfully!"
    
    echo ""
    echo -e "${GREEN}ClickHouse Installation Summary:${NC}"
    echo "================================"
    echo "HTTP Port: 8123"
    echo "TCP Port: 9000"
    echo "Interserver Port: 9009"
    echo ""
    echo "Connection examples:"
    echo "  clickhouse-client --password='$CLICKHOUSE_PASSWORD'"
    echo "  curl 'http://localhost:8123/?query=SELECT%20version()'"
    echo ""
    echo "Configuration files:"
    echo "  Main config: /etc/clickhouse-server/config.xml"
    echo "  Custom config: /etc/clickhouse-server/config.d/"
    echo "  Users config: /etc/clickhouse-server/users.d/"
    echo ""
    echo "Log files:"
    echo "  Server log: /var/log/clickhouse-server/clickhouse-server.log"
    echo "  Error log: /var/log/clickhouse-server/clickhouse-server.err.log"
    
    if [ -n "$CLUSTER_NODES" ]; then
        echo ""
        echo -e "${YELLOW}Cluster Configuration:${NC}"
        echo "Cluster nodes: $CLUSTER_NODES"
        echo "Configure ZooKeeper for replication if needed"
    fi
}

# Parse arguments
while getopts "p:c:h" opt; do
    case $opt in
        p) CLICKHOUSE_PASSWORD="$OPTARG" ;;
        c) CLUSTER_NODES="$OPTARG" ;;
        h) usage ;;
        *) usage ;;
    esac
done

# Main execution
check_prerequisites
detect_distro
update_system
add_repository
install_clickhouse
configure_clickhouse
configure_cluster
configure_firewall
configure_selinux
start_services
verify_installation
show_summary