add infrastructure monitoring

Add Clickstack (HyperDX) as the aggregation platform. Configure Otel Collector to collect host metrics.
2026-01-07 15:08:22 +01:00
parent 4eeaf483bc
commit 0496728700
13 changed files with 706 additions and 14 deletions
@@ -0,0 +1,38 @@
+version: '3.9'
+
+services:
+  otel-agent:
+    image: otel/opentelemetry-collector-contrib:0.143.0
+    user: "0:0" # Root für Hardware-Zugriff
+    command: ["--config=/etc/otel-agent-config.yaml"]
+    security_opt:
+      - apparmor:unconfined
+    volumes:
+      - {{ data_dir }}/otel-agent-config.yaml:/etc/otel-agent-config.yaml
+      - /:/hostfs:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /sys:/hostfs/sys:ro
+      - /proc:/hostfs/proc:ro
+    environment:
+      - GOMEMLIMIT=180MiB
+      - HOST_PROC=/hostfs/proc
+      - HOST_SYS=/hostfs/sys
+      - HOST_ETC=/hostfs/etc
+      - HOST_VAR=/hostfs/var
+      - HOST_RUN=/hostfs/run
+      - HOST_DEV=/hostfs/dev
+    deploy:
+      mode: global
+      update_config:
+        parallelism: 1
+        delay: 10s
+      resources:
+        limits:
+          memory: 200M
+    networks:
+      - host
+
+networks:
+  host:
+    name: host
+    external: true
@@ -0,0 +1,117 @@
+extensions:
+  # Beobachtet Docker Container
+  docker_observer:
+    endpoint: "unix:///var/run/docker.sock"
+    cache_sync_interval: 30s
+
+receivers:
+  hostmetrics:
+    root_path: /hostfs
+    collection_interval: 15s
+    scrapers:
+      cpu:
+        metrics:
+          system.cpu.time:
+            enabled: true
+          system.cpu.utilization:
+            enabled: true
+      memory:
+        metrics:
+          system.memory.usage:
+            enabled: true
+          system.memory.utilization:
+            enabled: true
+      filesystem:
+        metrics:
+          system.filesystem.usage:
+            enabled: true
+          system.filesystem.utilization:
+            enabled: true
+      paging:
+        metrics:
+          system.paging.usage:
+            enabled: true
+          system.paging.utilization:
+            enabled: true
+          system.paging.faults:
+            enabled: true
+      load:
+      disk:
+      network:
+
+  docker_stats:
+    endpoint: unix:///var/run/docker.sock
+    collection_interval: 30s
+    timeout: 20s
+
+  # receiver_creator:
+  #   watch_observers: [docker_observer]
+  #   receivers:
+  #     filelog:
+  #       rule: type == "container" # Nur für Container
+  #       config:
+  #         include:
+  #           - /hostfs/var/lib/docker/containers/*/*.log
+  #         operators:
+  #           - type: container
+  #             format: docker
+  #             add_metadata_from_filepath: true
+            # - type: json_parser
+            #   timestamp:
+            #     parse_from: time
+            #     layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+            #   severity:
+            #     parse_from: stream
+            #     mapping:
+            #       info: stdout
+            #       error: stderr
+
+  # Ceph Scraping (Funktioniert nur auf Nodes, wo Ceph Mgr läuft)
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: 'ceph-local'
+          scrape_interval: 30s
+          scrape_timeout: 10s
+          static_configs:
+            - targets: ['127.0.0.1:9283']
+          metric_relabel_configs:
+            - source_labels: [__name__]
+              regex: 'ceph_cluster_total_.*|ceph_health_status|ceph_osd_.*|ceph_pool_.*'
+              action: keep
+
+processors:
+  batch:
+    timeout: 5s
+  
+  resourcedetection:
+    detectors: [env, system]
+  
+  resourcedetection/docker:
+    detectors: [env, docker]
+    timeout: 2s
+    override: false
+
+exporters:
+  debug:
+    verbosity: detailed
+  otlp:
+    endpoint: "127.0.0.1:4317"
+    headers:
+      authorization: {{ hyperdx_api_ingestion_key }}
+    compression: gzip
+    tls:
+      insecure: true
+
+service:
+  extensions: [docker_observer]
+  pipelines:
+    metrics:
+      receivers: [hostmetrics, docker_stats, prometheus]
+      # receivers: [hostmetrics]
+      processors: [resourcedetection, batch]
+      exporters: [otlp]
+    # logs:
+    #   receivers: [receiver_creator]
+    #   processors: [resourcedetection/docker, batch]
+    #   exporters: [otlp, debug]