Configure Prometheus alerting with AlertManager notifications and webhook integration

Update system packages

Start by updating your package manager to ensure you get the latest versions.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Create AlertManager user

Create a dedicated system user to run AlertManager securely.

sudo useradd --no-create-home --shell /bin/false alertmanager

Download and install AlertManager

Download the latest AlertManager release from the official GitHub repository.

cd /tmp
wget https://github.com/prometheus/alertmanager/releases/download/v0.27.0/alertmanager-0.27.0.linux-amd64.tar.gz
tar xvf alertmanager-0.27.0.linux-amd64.tar.gz
sudo cp alertmanager-0.27.0.linux-amd64/alertmanager /usr/local/bin/
sudo cp alertmanager-0.27.0.linux-amd64/amtool /usr/local/bin/
sudo chown alertmanager:alertmanager /usr/local/bin/alertmanager
sudo chown alertmanager:alertmanager /usr/local/bin/amtool

Create AlertManager directories

Set up the directory structure for AlertManager configuration and data.

sudo mkdir -p /etc/alertmanager
sudo mkdir -p /var/lib/alertmanager
sudo chown alertmanager:alertmanager /etc/alertmanager
sudo chown alertmanager:alertmanager /var/lib/alertmanager

Configure AlertManager

Create the main AlertManager configuration file with routing rules and notification channels.

global:
  smtp_smarthost: 'localhost:587'
  smtp_from: 'alerts@example.com'
  smtp_auth_username: 'alerts@example.com'
  smtp_auth_password: 'your-email-password'
  
route:
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 1h
  receiver: 'default-receiver'
  routes:
  - match:
      severity: critical
    receiver: 'critical-alerts'
  - match:
      alertname: InstanceDown
    receiver: 'instance-down-alerts'
  - match:
      service: database
    receiver: 'database-alerts'

receivers:
name: 'default-receiver'
  email_configs:
  - to: 'admin@example.com'
    subject: 'Alert: {{ .GroupLabels.alertname }}'
    body: |
      {{ range .Alerts }}
      Alert: {{ .Annotations.summary }}
      Description: {{ .Annotations.description }}
      Labels:
      {{ range .Labels.SortedPairs }}
        {{ .Name }}: {{ .Value }}
      {{ end }}
      {{ end }}

name: 'critical-alerts'
  email_configs:
  - to: 'oncall@example.com'
    subject: 'CRITICAL: {{ .GroupLabels.alertname }}'
    body: |
      Critical alert fired!
      {{ range .Alerts }}
      Alert: {{ .Annotations.summary }}
      Description: {{ .Annotations.description }}
      Severity: {{ .Labels.severity }}
      {{ end }}
  slack_configs:
  - api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
    channel: '#alerts'
    title: 'Critical Alert'
    text: |
      {{ range .Alerts }}
      {{ .Annotations.summary }}
      {{ .Annotations.description }}
      {{ end }}

name: 'instance-down-alerts'
  webhook_configs:
  - url: 'http://localhost:8080/webhook/instance-down'
    send_resolved: true
    http_config:
      bearer_token: 'your-webhook-token'
  email_configs:
  - to: 'infrastructure@example.com'
    subject: 'Server Down: {{ .GroupLabels.instance }}'

name: 'database-alerts'
  email_configs:
  - to: 'dba@example.com'
    subject: 'Database Alert: {{ .GroupLabels.alertname }}'
  webhook_configs:
  - url: 'http://localhost:8080/webhook/database'
    send_resolved: true

Set configuration permissions

Ensure AlertManager can read its configuration file with proper ownership.

sudo chown alertmanager:alertmanager /etc/alertmanager/alertmanager.yml
sudo chmod 644 /etc/alertmanager/alertmanager.yml

Create systemd service

Set up AlertManager as a systemd service for automatic startup and management.

[Unit]
Description=Prometheus AlertManager
Wants=network-online.target
After=network-online.target

[Service]
User=alertmanager
Group=alertmanager
Type=simple
WorkingDirectory=/var/lib/alertmanager
ExecStart=/usr/local/bin/alertmanager \
  --config.file=/etc/alertmanager/alertmanager.yml \
  --storage.path=/var/lib/alertmanager \
  --web.listen-address=0.0.0.0:9093 \
  --web.external-url=http://localhost:9093 \
  --cluster.listen-address=0.0.0.0:9094
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target

Start AlertManager service

Enable and start the AlertManager service to begin processing alerts.

sudo systemctl daemon-reload
sudo systemctl enable --now alertmanager
sudo systemctl status alertmanager

Configure Prometheus to use AlertManager

Update your Prometheus configuration to send alerts to AlertManager. If you don't have Prometheus installed yet, check our Prometheus and Grafana setup guide.

global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "alert_rules.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - localhost:9093

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
  
  - job_name: 'node-exporter'
    static_configs:
      - targets: ['localhost:9100']
  
  - job_name: 'alertmanager'
    static_configs:
      - targets: ['localhost:9093']

Create Prometheus alerting rules

Define specific alert conditions that will trigger notifications through AlertManager.

groups:
name: system_alerts
  rules:
  - alert: InstanceDown
    expr: up == 0
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "Instance {{ $labels.instance }} is down"
      description: "{{ $labels.instance }} has been down for more than 5 minutes."
  
  - alert: HighCPUUsage
    expr: 100 - (avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "High CPU usage on {{ $labels.instance }}"
      description: "CPU usage is above 80% for more than 5 minutes."
  
  - alert: HighMemoryUsage
    expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "High memory usage on {{ $labels.instance }}"
      description: "Memory usage is above 90% for more than 5 minutes."
  
  - alert: DiskSpaceLow
    expr: (1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"})) * 100 > 85
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Low disk space on {{ $labels.instance }}"
      description: "Disk usage is above 85% on {{ $labels.mountpoint }}."

name: database_alerts
  rules:
  - alert: DatabaseConnectionsHigh
    expr: mysql_global_status_threads_connected / mysql_global_variables_max_connections * 100 > 80
    for: 2m
    labels:
      severity: warning
      service: database
    annotations:
      summary: "High database connections on {{ $labels.instance }}"
      description: "Database connections are above 80% of maximum."
  
  - alert: DatabaseDown
    expr: mysql_up == 0
    for: 1m
    labels:
      severity: critical
      service: database
    annotations:
      summary: "Database is down on {{ $labels.instance }}"
      description: "MySQL database is not responding."

Set alerting rules permissions

Ensure Prometheus can read the alerting rules file with proper ownership.

sudo chown prometheus:prometheus /etc/prometheus/alert_rules.yml
sudo chmod 644 /etc/prometheus/alert_rules.yml

Restart Prometheus to load alerts

Reload Prometheus configuration to activate the new alerting rules and AlertManager integration.

sudo systemctl restart prometheus
sudo systemctl status prometheus

Configure firewall for AlertManager

Open the necessary ports for AlertManager web interface and cluster communication.

sudo ufw allow 9093/tcp comment 'AlertManager web interface'
sudo ufw allow 9094/tcp comment 'AlertManager cluster'

sudo firewall-cmd --permanent --add-port=9093/tcp
sudo firewall-cmd --permanent --add-port=9094/tcp
sudo firewall-cmd --reload

Create webhook test endpoint

Set up a simple webhook endpoint to test AlertManager webhook integration.

#!/usr/bin/env python3
import json
import logging
from http.server import HTTPServer, BaseHTTPRequestHandler

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class WebhookHandler(BaseHTTPRequestHandler):
    def do_POST(self):
        content_length = int(self.headers['Content-Length'])
        post_data = self.rfile.read(content_length)
        
        try:
            alert_data = json.loads(post_data.decode('utf-8'))
            logger.info(f"Received webhook: {self.path}")
            logger.info(f"Alert data: {json.dumps(alert_data, indent=2)}")
            
            self.send_response(200)
            self.send_header('Content-type', 'application/json')
            self.end_headers()
            self.wfile.write(b'{"status": "received"}')
            
        except Exception as e:
            logger.error(f"Error processing webhook: {e}")
            self.send_response(400)
            self.end_headers()
    
    def log_message(self, format, *args):
        pass  # Suppress default logging

if __name__ == '__main__':
    server = HTTPServer(('localhost', 8080), WebhookHandler)
    logger.info("Webhook test server starting on http://localhost:8080")
    server.serve_forever()

chmod +x /tmp/webhook-test.py
python3 /tmp/webhook-test.py &

Verify AlertManager is running

Check that AlertManager is accessible and processing configuration correctly.

curl http://localhost:9093/-/healthy
curl http://localhost:9093/api/v1/status

Check Prometheus alerts status

Verify that Prometheus is loading alerting rules and can reach AlertManager.

curl http://localhost:9090/api/v1/rules
curl http://localhost:9090/api/v1/alerts

Test alert firing manually

Create a test alert to verify the complete notification pipeline works.

# Send a test alert directly to AlertManager
curl -XPOST http://localhost:9093/api/v1/alerts -H "Content-Type: application/json" -d '[
  {
    "labels": {
      "alertname": "TestAlert",
      "severity": "warning",
      "instance": "test-server"
    },
    "annotations": {
      "summary": "This is a test alert",
      "description": "Testing AlertManager notification pipeline"
    },
    "startsAt": "'$(date -Iseconds)'",
    "generatorURL": "http://localhost:9090/"
  }
]'

View AlertManager web interface

Access the AlertManager web UI to see alerts and silences.

# Open in browser: http://your-server-ip:9093
Or check via curl
curl http://localhost:9093/api/v1/alerts

Create production webhook handler

Set up a more robust webhook endpoint for production use with authentication and logging.

#!/usr/bin/env python3
import json
import logging
import hashlib
import hmac
from datetime import datetime
from http.server import HTTPServer, BaseHTTPRequestHandler

Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler('/var/log/alertmanager-webhook.log'),
        logging.StreamHandler()
    ]
)
logger = logging.getLogger(__name__)

WEBHOOK_SECRET = 'your-webhook-secret-key'

class ProductionWebhookHandler(BaseHTTPRequestHandler):
    def do_POST(self):
        # Verify authentication
        auth_header = self.headers.get('Authorization', '')
        if not auth_header.startswith('Bearer '):
            self.send_error(401, 'Missing or invalid authorization header')
            return
        
        token = auth_header.split(' ')[1]
        if token != 'your-webhook-token':
            self.send_error(403, 'Invalid token')
            return
        
        content_length = int(self.headers['Content-Length'])
        post_data = self.rfile.read(content_length)
        
        try:
            alert_data = json.loads(post_data.decode('utf-8'))
            
            # Log webhook details
            logger.info(f"Webhook received: {self.path}")
            logger.info(f"Number of alerts: {len(alert_data.get('alerts', []))}")
            
            # Process each alert
            for alert in alert_data.get('alerts', []):
                self.process_alert(alert)
            
            # Send success response
            self.send_response(200)
            self.send_header('Content-type', 'application/json')
            self.end_headers()
            response = {
                'status': 'success',
                'processed': len(alert_data.get('alerts', [])),
                'timestamp': datetime.now().isoformat()
            }
            self.wfile.write(json.dumps(response).encode())
            
        except json.JSONDecodeError:
            logger.error("Invalid JSON in webhook payload")
            self.send_error(400, 'Invalid JSON')
        except Exception as e:
            logger.error(f"Error processing webhook: {e}")
            self.send_error(500, 'Internal server error')
    
    def process_alert(self, alert):
        """Process individual alert - customize this for your needs"""
        alertname = alert.get('labels', {}).get('alertname')
        severity = alert.get('labels', {}).get('severity')
        status = alert.get('status')
        
        logger.info(f"Processing alert: {alertname} (severity: {severity}, status: {status})")
        
        # Example: Send to external system, update database, etc.
        if severity == 'critical':
            logger.warning(f"CRITICAL alert received: {alertname}")
            # Add your critical alert handling logic here
        
        if alertname == 'InstanceDown':
            instance = alert.get('labels', {}).get('instance')
            logger.error(f"Instance down alert: {instance}")
            # Add your instance down handling logic here
    
    def log_message(self, format, *args):
        pass  # Use our custom logging instead

if __name__ == '__main__':
    server = HTTPServer(('localhost', 8080), ProductionWebhookHandler)
    logger.info("Production webhook server starting on http://localhost:8080")
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        logger.info("Webhook server stopped")
        server.shutdown()

Create webhook systemd service

Run the webhook handler as a systemd service for production reliability.

[Unit]
Description=AlertManager Webhook Handler
After=network.target

[Service]
Type=simple
User=alertmanager
Group=alertmanager
WorkingDirectory=/opt
ExecStart=/usr/bin/python3 /opt/alertmanager-webhook.py
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target

sudo chown alertmanager:alertmanager /opt/alertmanager-webhook.py
sudo chmod +x /opt/alertmanager-webhook.py
sudo systemctl daemon-reload
sudo systemctl enable --now alertmanager-webhook
sudo systemctl status alertmanager-webhook