Monitoring

Work in progress - not finished yet!

Als Monitoring verwenden wir den Prometheus Stack ( Prometheus + Alertmanager + Pushgateway) mit Grafana zur Visualisierung.

Zum Erfassen der Sensordaten verwenden wir neben node_exporter (generelle Hoststatistiken), blackbox_exporter (ICMP & HTTP Tests) und cAdvisor (für Docker) auch Anwendungsspezifische Prometheus Exporter (nginx, mysql, postgresql, ssh, gitlab, grafana, ...). Viele von diesen sind in dieser Liste zu finden.

Möchte man bereitgestellte Sensordaten einer Anwendung (die sich nicht im Prometheus-Format befinden) verarbeiten, so kann man auch einen eigenen Exporter schreiben.

version: '3.9'

services:
  grafana:
    image: grafana/grafana
    restart: always
    volumes:
      - "/srv/monitoring/grafana/lib:/var/lib/grafana"
      - "/srv/monitoring/grafana/etc:/etc/grafana"
    ports:
      - "[::1]:8000:3000"

  prometheus:
    image: quay.io/prometheus/prometheus
    restart: always
    volumes:
      - "/srv/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml"
      - "/srv/monitoring/prometheus/data:/prometheus"
    ports:
      - "[::1]:9090:9090"

  alertmanager:
    image: quay.io/prometheus/alertmanager
    restart: always
    ports:
      - "[::1]:9093:9093"

  pushgateway:
    image: quay.io/prometheus/pushgateway
    restart: always
    ports:
      - "[::1]:9091:9091"

  node_exporter:
    image: quay.io/prometheus/node-exporter
    restart: always
    volumes:
      - "/proc:/host/proc:ro"
      - "/sys:/host/sys:ro"
      - "/:/rootfs:ro"
    command:
      - "--path.procfs=/host/proc"
      - "--path.sysfs=/host/sys"
      - "--path.rootfs=/rootfs"
      - "--collector.filesystem.ignored-mount-points='^(/rootfs|/host|)/(sys|proc|dev|host|etc)($$|/)'"
      - "--collector.filesystem.ignored-fs-types='^(sys|proc|auto|cgroup|devpts|ns|au|fuse\.lxc|mqueue)(fs|)$$'"

  blackbox_exporter:
    image: quay.io/prometheus/blackbox-exporter
    restart: always
    command: "--config.file=/config/config.yaml"
    volumes:
      - "/srv/monitoring/blackbox_exporter/:/config/"

  cadvisor:
    image: gcr.io/cadvisor/cadvisor
    restart: always
    #privileged: true
    #devices:
    #  - "/dev/kmsg:/dev/kmsg"
    volumes:
      - "/:/rootfs:ro"
      - "/var/run:/var/run:ro"
      - "/sys:/sys:ro"
      - "/var/lib/docker:/var/lib/docker:ro"
      - "/cgroup:/cgroup:ro"

# /srv/monitoring/prometheus/prometheus.yaml
global:
  scrape_interval: 30s
  evaluation_interval: 30s

alerting:
  alertmanagers:
  - scheme: http
    static_configs:
    - targets: 
      - 'alertmanager:9093'

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
    - targets: ['localhost:9090']

  - job_name: 'blackbox_exporter_http'
    metrics_path: '/probe'
    params: 
      module: [http_2xx]
    static_configs:
      - targets:
        - https://www.google.de
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

  - job_name: 'blackbox_exporter_icmp'
    metrics_path: '/probe'
    params: 
      module: [icmp]
    static_configs:
      - targets:
        - google.com
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

  - job_name: 'blackbox_exporter'
    static_configs:
      - targets:
        - blackbox-exporter:9115

  - job_name: 'node_exporter'
    static_configs:
      - targets:
        - node_exporter:9100

  - job_name: 'cadvisor'
    static_configs:
      - targets:
        - "cadvisor:8080"

  - job_name: pushgateway
    honor_labels: true
    static_configs:
      - targets: ['pushgateway:9091']

# /srv/monitoring/blackbox_exporter/config.yaml
modules:
  http_2xx:
    prober: http
    http:
      preferred_ip_protocol: "ip4"  # defaults to "ip6"
  icmp:
    prober: icmp
    icmp:
      preferred_ip_protocol: "ip4"  # defaults to "ip6"