Alert rules
deploy/monitoring/mnema-alerts.yml from mnema-talk v3.3.13. Copy it next to prometheus.yml as mnema-alerts.yml.
# Example Prometheus alert rules for Mnema Talk. Documented in doc/operations.md
# (Monitoring and alerts). Checked in CI with promtool, and a test fails if an
# expression names a metric that /api/metrics does not export.
groups:
- name: mnema-talk
rules:
- alert: MnemaDown
expr: up{job="mnema-talk"} == 0
for: 2m
labels: {severity: page}
annotations: {summary: "Mnema Talk does not answer the scrape (or the token is wrong)"}
# /api/health is public and the metrics endpoint is up while the
# database or storage is not, so check those through their metrics too.
- alert: MnemaStorageDown
expr: mnema_storage_up == 0
for: 5m
labels: {severity: page}
annotations: {summary: "Object storage is unreachable: uploads fail and /api/health answers 503"}
- alert: MnemaHealthFailing
# needs blackbox_exporter probing https://chat.example.com/api/health
expr: probe_success{instance="https://chat.example.com/api/health"} == 0
for: 3m
labels: {severity: page}
annotations: {summary: "/api/health fails (database or storage)"}
- alert: MnemaDatabasePoolExhausted
expr: mnema_db_connections_acquired >= mnema_db_connections_max
for: 5m
labels: {severity: warn}
annotations: {summary: "All database connections are busy: requests queue"}
- alert: MnemaMemoryHigh
# process_resident_memory_bytes is reported on Linux; the compose
# limit is APP_MEM_LIMIT (see Container resources).
expr: process_resident_memory_bytes > 800e6
for: 15m
labels: {severity: warn}
annotations: {summary: "Resident memory above 800 MB (about 75% of the default 1g container limit) for 15 minutes"}
- alert: MnemaGoroutineLeak
expr: go_goroutines > 5000
for: 15m
labels: {severity: warn}
annotations: {summary: "Unusually many goroutines: possible leak"}
- alert: MnemaServerErrors
expr: sum(rate(mnema_http_requests_total{class="5xx"}[10m])) > 0.1
for: 10m
labels: {severity: warn}
annotations: {summary: "More than one 5xx every 10 seconds"}
- alert: MnemaSlowRequests
expr: histogram_quantile(0.95, sum by (le) (rate(mnema_http_request_duration_seconds_bucket{route!~"/api/channels/.*/upload|/api/media/.*"}[10m]))) > 1
for: 15m
labels: {severity: warn}
annotations: {summary: "95th percentile API latency above one second"}
- alert: MnemaIceFailures
# failed ICE transitions relative to successful ones
expr: |
sum(increase(mnema_sfu_ice_transitions_total{state="failed"}[30m])) > 3
and
sum(increase(mnema_sfu_ice_transitions_total{state="failed"}[30m]))
/ clamp_min(sum(increase(mnema_sfu_ice_transitions_total{state="connected"}[30m])), 1) > 0.2
for: 10m
labels: {severity: warn}
annotations: {summary: "Many voice connections fail: check UDP port range, WEBRTC_ANNOUNCE_IP and TURN"}
- alert: MnemaStorageErrors
expr: sum(increase(mnema_s3_operations_total{result="error"}[15m])) > 5
labels: {severity: warn}
annotations: {summary: "Object storage calls fail"}
- alert: MnemaGoroutinePanic
# a recovered panic: one connection or call participant was dropped
# for it. A panic that ends the process shows up as a restart instead.
expr: sum by (name) (increase(mnema_goroutine_panics_total[15m])) > 0
labels: {severity: warn}
annotations: {summary: "The server recovered from a panic in {{ $labels.name }}: a bug, see the log line 'goroutine panicked'"}
- alert: MnemaBackupTooOld
# see "Backup age" below: the app cannot know when you last backed up
expr: time() - mnema_backup_last_success_timestamp_seconds > 36 * 3600
labels: {severity: warn}
annotations: {summary: "No successful backup for 36 hours"}