Configure Loki and Promtail for centralized Docker log aggregation and analysis

Update system packages

Start by updating your package manager to ensure you have the latest security patches.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Install Docker and Docker Compose

Install Docker and Docker Compose to run Loki, Promtail, and test containers.

curl -fsSL https://get.docker.com | sudo sh
sudo usermod -aG docker $USER
newgrp docker

sudo dnf install -y dnf-plugins-core
sudo dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
sudo dnf install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
sudo systemctl enable --now docker
sudo usermod -aG docker $USER
newgrp docker

Create project directory structure

Create a dedicated directory structure for Loki configuration files and data storage.

mkdir -p ~/loki-stack/{config,data/loki,data/grafana}
cd ~/loki-stack

Configure Loki server

Create the main Loki configuration file with retention policies and storage settings.

auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096
  log_level: info

common:
  instance_addr: 127.0.0.1
  path_prefix: /tmp/loki
  storage:
    filesystem:
      chunks_directory: /tmp/loki/chunks
      rules_directory: /tmp/loki/rules
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

query_range:
  results_cache:
    cache:
      embedded_cache:
        enabled: true
        max_size_mb: 100

schema_config:
  configs:
    - from: 2020-10-24
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093

Retention configuration
limits_config:
  retention_period: 168h  # 7 days
  max_query_length: 12000h
  max_query_parallelism: 16
  max_streams_per_user: 10000
  max_line_size: 256000

Compactor for log retention
compactor:
  working_directory: /tmp/loki/retention
  retention_enabled: true
  retention_delete_delay: 2h
  delete_request_store: filesystem

Configure Promtail for Docker log collection

Create Promtail configuration to automatically discover and collect Docker container logs.

server:
  http_listen_port: 9080
  grpc_listen_port: 0
  log_level: info

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push
    tenant_id: docker

scrape_configs:
  # Docker container logs
  - job_name: docker
    docker_sd_configs:
      - host: unix:///var/run/docker.sock
        refresh_interval: 5s
        filters:
          - name: label
            values: ["logging=promtail"]
    relabel_configs:
      - source_labels: ['__meta_docker_container_name']
        regex: '/(.*)
        target_label: container
      - source_labels: ['__meta_docker_container_log_stream']
        target_label: stream
      - source_labels: ['__meta_docker_container_label_com_docker_compose_service']
        target_label: service
      - source_labels: ['__meta_docker_container_label_com_docker_compose_project']
        target_label: project
    pipeline_stages:
      - docker: {}
      - timestamp:
          source: time
          format: RFC3339Nano
      - output:
          source: output

  # System logs
  - job_name: system
    static_configs:
      - targets:
          - localhost
        labels:
          job: varlogs
          __path__: /var/log/*.log
    pipeline_stages:
      - match:
          selector: '{job="varlogs"}'
          stages:
            - timestamp:
                source: timestamp
                format: Jan 02 15:04:05
            - regex:
                expression: '^(?P\w+ \d+ \d+:\d+:\d+) (?P\w+) (?P\w+)(\[(?P\d+)\])?: (?P.*)'
            - labels:
                hostname:
                service:
                pid:

  # Application logs
  - job_name: app_logs
    static_configs:
      - targets:
          - localhost
        labels:
          job: app
          __path__: /var/log/app/*.log
    pipeline_stages:
      - json:
          expressions:
            level: level
            timestamp: timestamp
            message: message
      - timestamp:
          source: timestamp
          format: 2006-01-02T15:04:05.000Z
      - labels:
          level:

Create Docker Compose configuration

Set up the complete monitoring stack with Loki, Promtail, and Grafana in a single compose file.

version: '3.8'

networks:
  loki-net:
    driver: bridge

volumes:
  grafana-data:
    driver: local
  loki-data:
    driver: local

services:
  loki:
    image: grafana/loki:2.9.2
    container_name: loki
    restart: unless-stopped
    networks:
      - loki-net
    ports:
      - "3100:3100"
    volumes:
      - ./config/loki.yml:/etc/loki/local-config.yaml:ro
      - loki-data:/tmp/loki
    command: -config.file=/etc/loki/local-config.yaml
    healthcheck:
      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
    labels:
      logging: "promtail"

  promtail:
    image: grafana/promtail:2.9.2
    container_name: promtail
    restart: unless-stopped
    networks:
      - loki-net
    ports:
      - "9080:9080"
    volumes:
      - ./config/promtail.yml:/etc/promtail/config.yml:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /var/log:/var/log:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
    command: -config.file=/etc/promtail/config.yml
    depends_on:
      loki:
        condition: service_healthy
    labels:
      logging: "promtail"

  grafana:
    image: grafana/grafana:10.2.0
    container_name: grafana
    restart: unless-stopped
    networks:
      - loki-net
    ports:
      - "3000:3000"
    volumes:
      - grafana-data:/var/lib/grafana
      - ./config/grafana-datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml:ro
      - ./config/grafana-dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml:ro
      - ./dashboards:/var/lib/grafana/dashboards:ro
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin123
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_LOG_LEVEL=info
      - GF_INSTALL_PLUGINS=grafana-clock-panel
    labels:
      logging: "promtail"

  # Test application for log generation
  test-app:
    image: nginx:alpine
    container_name: test-nginx
    restart: unless-stopped
    networks:
      - loki-net
    ports:
      - "8080:80"
    labels:
      logging: "promtail"
      service: "nginx"
      environment: "test"

Configure Grafana data source

Create Grafana configuration to automatically add Loki as a data source.

apiVersion: 1

datasources:
  - name: Loki
    type: loki
    access: proxy
    url: http://loki:3100
    isDefault: true
    editable: true
    jsonData:
      maxLines: 1000
      derivedFields:
        - datasourceUid: prometheus
          matcherRegex: "traceID=(\\w+)"
          name: TraceID
          url: "http://localhost:3000/explore?left=[\"now-1h\",\"now\",\"Jaeger\",{\"query\":\"$${__value.raw}\"}]"

  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: false
    editable: true

Set up dashboard provisioning

Configure Grafana to automatically load dashboards for log analysis.

apiVersion: 1

providers:
  - name: 'Loki Dashboards'
    orgId: 1
    folder: 'Loki'
    type: file
    disableDeletion: false
    updateIntervalSeconds: 30
    allowUiUpdates: true
    options:
      path: /var/lib/grafana/dashboards

Create dashboard directory and sample dashboard

Create a basic Loki dashboard for container log visualization.

mkdir -p ~/loki-stack/dashboards

{
  "dashboard": {
    "id": null,
    "title": "Docker Container Logs",
    "tags": ["loki", "docker", "logs"],
    "timezone": "browser",
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "panels": [
      {
        "id": 1,
        "title": "Container Logs Stream",
        "type": "logs",
        "targets": [
          {
            "expr": "{container=~\".+\"}",
            "refId": "A"
          }
        ],
        "gridPos": {
          "h": 12,
          "w": 24,
          "x": 0,
          "y": 0
        },
        "options": {
          "showTime": true,
          "showLabels": true,
          "showCommonLabels": false,
          "wrapLogMessage": true,
          "sortOrder": "Descending"
        }
      },
      {
        "id": 2,
        "title": "Log Volume by Container",
        "type": "stat",
        "targets": [
          {
            "expr": "sum by (container) (count_over_time({container=~\".+\"}[5m]))",
            "refId": "A"
          }
        ],
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 12
        }
      },
      {
        "id": 3,
        "title": "Error Rate",
        "type": "stat",
        "targets": [
          {
            "expr": "sum(rate({container=~\".+\"} |~ \"(?i)error\")[5m])) * 100",
            "refId": "A"
          }
        ],
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 12,
          "y": 12
        },
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "thresholds"
            },
            "thresholds": {
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "yellow",
                  "value": 5
                },
                {
                  "color": "red",
                  "value": 10
                }
              ]
            },
            "unit": "percent"
          }
        }
      }
    ],
    "refresh": "5s",
    "version": 1
  }
}

Set correct file permissions

Ensure proper ownership and permissions for configuration files and data directories.

sudo chown -R $USER:$USER ~/loki-stack
chmod -R 755 ~/loki-stack/config
chmod 644 ~/loki-stack/config/*.yml
chmod 644 ~/loki-stack/docker-compose.yml

Start the Loki stack

Launch all services using Docker Compose with proper startup ordering.

cd ~/loki-stack
docker-compose up -d

Configure log retention cleanup job

Set up a systemd timer to periodically clean up old logs and maintain disk space.

sudo mkdir -p /etc/systemd/system

[Unit]
Description=Loki Log Cleanup Service
After=docker.service
Requires=docker.service

[Service]
Type=oneshot
User=root
ExecStart=/bin/bash -c 'docker exec loki /usr/bin/loki -config.file=/etc/loki/local-config.yaml -target=compactor -dry-run=false'
RemainAfterExit=no

[Install]
WantedBy=multi-user.target

[Unit]
Description=Run Loki cleanup daily
Requires=loki-cleanup.service

[Timer]
OnCalendar=daily
RandomizedDelaySec=1h
Persistent=true

[Install]
WantedBy=timers.target

sudo systemctl daemon-reload
sudo systemctl enable --now loki-cleanup.timer

Set up log parsing for structured logs

Configure Promtail to parse JSON logs and extract structured data for better querying.

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
  - job_name: docker-json
    docker_sd_configs:
      - host: unix:///var/run/docker.sock
        refresh_interval: 5s
        filters:
          - name: label
            values: ["logging=promtail"]
    relabel_configs:
      - source_labels: ['__meta_docker_container_name']
        regex: '/(.*)
        target_label: container
      - source_labels: ['__meta_docker_container_label_com_docker_compose_service']
        target_label: service
    pipeline_stages:
      - docker: {}
      - json:
          expressions:
            level: level
            timestamp: timestamp
            message: message
            service: service
            trace_id: trace_id
      - timestamp:
          source: timestamp
          format: RFC3339Nano
      - labels:
          level:
          service:
          trace_id:
      - output:
          source: message

Configure alerting rules

Create alerting rules for log-based monitoring and incident response.

groups:
  - name: loki_alerts
    rules:
      - alert: HighErrorRate
        expr: |
          sum(rate({container=~".+"} |~ "(?i)error" [5m])) by (container) > 0.1
        for: 2m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "High error rate detected in container {{ $labels.container }}"
          description: "Container {{ $labels.container }} is generating errors at {{ $value }} errors per second for more than 2 minutes."

      - alert: ContainerLogVolumeHigh
        expr: |
          sum(rate({container=~".+"}[5m])) by (container) > 100
        for: 5m
        labels:
          severity: info
          team: platform
        annotations:
          summary: "High log volume from container {{ $labels.container }}"
          description: "Container {{ $labels.container }} is generating {{ $value }} log lines per second."

      - alert: ServiceDown
        expr: |
          absent_over_time({container=~".+"}[10m])
        for: 1m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "No logs received from container {{ $labels.container }}"
          description: "Container {{ $labels.container }} has not generated any logs for over 10 minutes."

Configure Grafana alerting

Set up Grafana to create alerts based on Loki queries and send notifications.

[alerting]
enabled = true
execute_alerts = true

[unified_alerting]
enabled = true

[smtp]
enabled = false
host = smtp.example.com:587
user = alerts@example.com
password = your_smtp_password
skip_verify = false
from_address = alerts@example.com
from_name = Grafana Alerts

Update Docker Compose with Grafana config

Mount the custom Grafana configuration to enable alerting features.

docker-compose down

Add this volume mount to grafana service in docker-compose.yml
Under volumes section for grafana:
- ./config/grafana.ini:/etc/grafana/grafana.ini:ro

sed -i '/grafana-data:\/var\/lib\/grafana/a\      - ./config/grafana.ini:/etc/grafana/grafana.ini:ro' docker-compose.yml

docker-compose up -d

Check service status

Verify all components are running and healthy.

docker-compose ps
docker-compose logs loki
docker-compose logs promtail
docker-compose logs grafana

Test log collection

Generate test logs and verify they appear in Loki.

# Generate some test logs
docker exec test-nginx sh -c 'echo "Test log entry $(date)" >> /var/log/nginx/access.log'
curl http://localhost:8080

Query Loki API directly
curl -G -s "http://localhost:3100/loki/api/v1/query" \
  --data-urlencode 'query={container="test-nginx"}' \
  --data-urlencode 'limit=10' | jq '.data.result[].values[]'

Check Promtail metrics
curl -s http://localhost:9080/metrics | grep promtail_targets_active_total

Verify Grafana integration

Access Grafana and confirm Loki data source is working.

# Access Grafana at http://localhost:3000
Login: admin / admin123

Test Loki queries in Explore view:
echo "Access Grafana at: http://localhost:3000"
echo "Username: admin"
echo "Password: admin123"
echo "Navigate to Explore -> Loki and try query: {container=\"test-nginx\"}"

Check health endpoints
curl -s http://localhost:3100/ready
curl -s http://localhost:3000/api/health