Alert rules

deploy/monitoring/mnema-alerts.yml from mnema-talk v3.3.13. Copy it next to prometheus.yml as mnema-alerts.yml.

# Example Prometheus alert rules for Mnema Talk. Documented in doc/operations.md
# (Monitoring and alerts). Checked in CI with promtool, and a test fails if an
# expression names a metric that /api/metrics does not export.
groups:
  - name: mnema-talk
    rules:
      - alert: MnemaDown
        expr: up{job="mnema-talk"} == 0
        for: 2m
        labels: {severity: page}
        annotations: {summary: "Mnema Talk does not answer the scrape (or the token is wrong)"}

      # /api/health is public and the metrics endpoint is up while the
      # database or storage is not, so check those through their metrics too.
      - alert: MnemaStorageDown
        expr: mnema_storage_up == 0
        for: 5m
        labels: {severity: page}
        annotations: {summary: "Object storage is unreachable: uploads fail and /api/health answers 503"}

      - alert: MnemaHealthFailing
        # needs blackbox_exporter probing https://chat.example.com/api/health
        expr: probe_success{instance="https://chat.example.com/api/health"} == 0
        for: 3m
        labels: {severity: page}
        annotations: {summary: "/api/health fails (database or storage)"}

      - alert: MnemaDatabasePoolExhausted
        expr: mnema_db_connections_acquired >= mnema_db_connections_max
        for: 5m
        labels: {severity: warn}
        annotations: {summary: "All database connections are busy: requests queue"}

      - alert: MnemaMemoryHigh
        # process_resident_memory_bytes is reported on Linux; the compose
        # limit is APP_MEM_LIMIT (see Container resources).
        expr: process_resident_memory_bytes > 800e6
        for: 15m
        labels: {severity: warn}
        annotations: {summary: "Resident memory above 800 MB (about 75% of the default 1g container limit) for 15 minutes"}

      - alert: MnemaGoroutineLeak
        expr: go_goroutines > 5000
        for: 15m
        labels: {severity: warn}
        annotations: {summary: "Unusually many goroutines: possible leak"}

      - alert: MnemaServerErrors
        expr: sum(rate(mnema_http_requests_total{class="5xx"}[10m])) > 0.1
        for: 10m
        labels: {severity: warn}
        annotations: {summary: "More than one 5xx every 10 seconds"}

      - alert: MnemaSlowRequests
        expr: histogram_quantile(0.95, sum by (le) (rate(mnema_http_request_duration_seconds_bucket{route!~"/api/channels/.*/upload|/api/media/.*"}[10m]))) > 1
        for: 15m
        labels: {severity: warn}
        annotations: {summary: "95th percentile API latency above one second"}

      - alert: MnemaIceFailures
        # failed ICE transitions relative to successful ones
        expr: |
          sum(increase(mnema_sfu_ice_transitions_total{state="failed"}[30m])) > 3
          and
          sum(increase(mnema_sfu_ice_transitions_total{state="failed"}[30m]))
            / clamp_min(sum(increase(mnema_sfu_ice_transitions_total{state="connected"}[30m])), 1) > 0.2
        for: 10m
        labels: {severity: warn}
        annotations: {summary: "Many voice connections fail: check UDP port range, WEBRTC_ANNOUNCE_IP and TURN"}

      - alert: MnemaStorageErrors
        expr: sum(increase(mnema_s3_operations_total{result="error"}[15m])) > 5
        labels: {severity: warn}
        annotations: {summary: "Object storage calls fail"}

      - alert: MnemaGoroutinePanic
        # a recovered panic: one connection or call participant was dropped
        # for it. A panic that ends the process shows up as a restart instead.
        expr: sum by (name) (increase(mnema_goroutine_panics_total[15m])) > 0
        labels: {severity: warn}
        annotations: {summary: "The server recovered from a panic in {{ $labels.name }}: a bug, see the log line 'goroutine panicked'"}

      - alert: MnemaBackupTooOld
        # see "Backup age" below: the app cannot know when you last backed up
        expr: time() - mnema_backup_last_success_timestamp_seconds > 36 * 3600
        labels: {severity: warn}
        annotations: {summary: "No successful backup for 36 hours"}

Something wrong on this page? Tell us.