Set up OpenTelemetry metrics collection with Prometheus integration for distributed system monitoring

Install OpenTelemetry Collector

Download and install the OpenTelemetry Collector binary from the official GitHub releases.

sudo apt update
wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.91.0/otelcol_0.91.0_linux_amd64.deb
sudo dpkg -i otelcol_0.91.0_linux_amd64.deb

sudo dnf update -y
wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.91.0/otelcol-0.91.0-1.x86_64.rpm
sudo rpm -ivh otelcol-0.91.0-1.x86_64.rpm

Create OpenTelemetry Collector configuration

Configure the collector to receive metrics from various sources and export them to Prometheus. This configuration includes OTLP receivers, processors, and Prometheus exporters.

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318
  prometheus:
    config:
      scrape_configs:
        - job_name: 'otel-collector'
          static_configs:
            - targets: ['localhost:8888']
  hostmetrics:
    collection_interval: 30s
    scrapers:
      cpu:
      disk:
      filesystem:
      memory:
      network:
      process:

processors:
  batch:
    timeout: 1s
    send_batch_size: 1024
  memory_limiter:
    limit_mib: 256
    spike_limit_mib: 64
  resourcedetection:
    detectors: [env, system, docker]
    timeout: 5s
    override: false

exporters:
  prometheus:
    endpoint: "0.0.0.0:8889"
    namespace: "otel"
    const_labels:
      environment: "production"
    enable_open_metrics: true
  logging:
    loglevel: info

service:
  extensions: [health_check, pprof, zpages]
  pipelines:
    metrics:
      receivers: [otlp, prometheus, hostmetrics]
      processors: [memory_limiter, resourcedetection, batch]
      exporters: [prometheus, logging]
  telemetry:
    logs:
      level: "info"
    metrics:
      address: 0.0.0.0:8888

Configure Prometheus to scrape OpenTelemetry metrics

Add the OpenTelemetry Collector as a scrape target in your Prometheus configuration to collect the exported metrics.

global:
  scrape_interval: 30s
  evaluation_interval: 30s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
  
  - job_name: 'otel-collector'
    static_configs:
      - targets: ['localhost:8889']
    scrape_interval: 15s
    metrics_path: '/metrics'
    
  - job_name: 'otel-collector-internal'
    static_configs:
      - targets: ['localhost:8888']
    scrape_interval: 30s
    metrics_path: '/metrics'

Create systemd service for OpenTelemetry Collector

Set up a systemd service to manage the OpenTelemetry Collector process with automatic restarts and proper logging.

[Unit]
Description=OpenTelemetry Collector
Documentation=https://opentelemetry.io/docs/collector/
After=network-online.target
Wants=network-online.target

[Service]
Type=exec
User=otel
Group=otel
ExecStart=/usr/bin/otelcol --config=/etc/otelcol/config.yaml
Restart=on-failure
RestartSec=5
StandardOutput=journal
StandardError=journal
SyslogIdentifier=otel-collector
KillMode=mixed
KillSignal=SIGTERM

[Install]
WantedBy=multi-user.target

Create dedicated user for OpenTelemetry Collector

Create a system user with minimal privileges to run the OpenTelemetry Collector service securely.

sudo useradd --system --no-create-home --shell /bin/false otel
sudo chown -R otel:otel /etc/otelcol
sudo chmod 644 /etc/otelcol/config.yaml

Configure log rotation

Set up log rotation to prevent log files from consuming excessive disk space over time.

/var/log/otel-collector/*.log {
    daily
    missingok
    rotate 30
    compress
    delaycompress
    notifempty
    create 644 otel otel
    postrotate
        systemctl reload otel-collector > /dev/null 2>&1 || true
    endscript
}

Start and enable services

Enable both OpenTelemetry Collector and Prometheus services to start automatically on boot and start them immediately.

sudo systemctl daemon-reload
sudo systemctl enable --now otel-collector
sudo systemctl restart prometheus
sudo systemctl enable prometheus

Configure firewall rules

Open the necessary ports for OpenTelemetry Collector receivers and Prometheus metrics endpoint.

sudo ufw allow 4317/tcp comment 'OpenTelemetry GRPC'
sudo ufw allow 4318/tcp comment 'OpenTelemetry HTTP'
sudo ufw allow 8889/tcp comment 'OpenTelemetry Prometheus metrics'
sudo ufw reload

sudo firewall-cmd --permanent --add-port=4317/tcp --permanent
sudo firewall-cmd --permanent --add-port=4318/tcp --permanent
sudo firewall-cmd --permanent --add-port=8889/tcp --permanent
sudo firewall-cmd --reload

Set up Node.js application metrics

Configure a Node.js application to send metrics to the OpenTelemetry Collector using automatic instrumentation.

npm install @opentelemetry/api @opentelemetry/sdk-node @opentelemetry/auto-instrumentations-node

const { NodeSDK } = require('@opentelemetry/sdk-node');
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node');
const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics');
const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-http');

const metricExporter = new OTLPMetricExporter({
  url: 'http://localhost:4318/v1/metrics',
});

const sdk = new NodeSDK({
  instrumentations: [getNodeAutoInstrumentations()],
  metricReader: new PeriodicExportingMetricReader({
    exporter: metricExporter,
    exportIntervalMillis: 5000,
  }),
  serviceName: 'example-service',
  serviceVersion: '1.0.0',
});

sdk.start();

Configure custom metrics collection

Add custom business metrics to track application-specific KPIs alongside infrastructure metrics.

const { metrics } = require('@opentelemetry/api');

const meter = metrics.getMeter('example-service', '1.0.0');

// Counter for request count
const requestCounter = meter.createCounter('http_requests_total', {
  description: 'Total number of HTTP requests',
});

// Histogram for request duration
const requestDuration = meter.createHistogram('http_request_duration_ms', {
  description: 'Duration of HTTP requests in milliseconds',
});

// Gauge for active connections
const activeConnections = meter.createUpDownCounter('active_connections', {
  description: 'Number of active connections',
});

// Example usage in Express middleware
function metricsMiddleware(req, res, next) {
  const startTime = Date.now();
  
  requestCounter.add(1, {
    method: req.method,
    route: req.route?.path || 'unknown',
  });
  
  activeConnections.add(1);
  
  res.on('finish', () => {
    const duration = Date.now() - startTime;
    requestDuration.record(duration, {
      method: req.method,
      status_code: res.statusCode.toString(),
    });
    
    activeConnections.add(-1);
  });
  
  next();
}

module.exports = { metricsMiddleware };

Configure metrics sampling and filtering

Implement sampling and filtering to reduce metric cardinality and improve performance in high-traffic environments.

processors:
  filter:
    metrics:
      exclude:
        match_type: regexp
        metric_names:
          - ".debug."
          - ".test."
  transform:
    metric_statements:
      - context: metric
        statements:
          - set(description, "Processed by OpenTelemetry") where name == "http_requests_total"
      - context: datapoint
        statements:
          - limit(attributes, 10, [])
          - delete_key(attributes, "sensitive_data")
  groupbyattrs:
    keys:
      - service.name
      - service.version
      - deployment.environment

Set up high availability configuration

Configure multiple OpenTelemetry Collector instances with load balancing for production resilience.

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318
        
processors:
  loadbalancing:
    routing_key: "service"
    resolver:
      static:
        hostnames:
          - otel-collector-1.example.com:4317
          - otel-collector-2.example.com:4317
          - otel-collector-3.example.com:4317
  batch:
    timeout: 1s
    send_batch_size: 1024
    
exporters:
  prometheus:
    endpoint: "0.0.0.0:8889"
    namespace: "otel"
    enable_open_metrics: true
  otlp:
    endpoint: http://prometheus-remote-write.example.com:9090/api/v1/write
    headers:
      X-Prometheus-Remote-Write-Version: "0.1.0"
      
service:
  pipelines:
    metrics:
      receivers: [otlp]
      processors: [loadbalancing, batch]
      exporters: [prometheus, otlp]