diff --git a/hosts/aws/tyo0/services/prometheus.nix b/hosts/aws/tyo0/services/prometheus.nix index 3218634..1ce8445 100644 --- a/hosts/aws/tyo0/services/prometheus.nix +++ b/hosts/aws/tyo0/services/prometheus.nix @@ -100,24 +100,44 @@ in alert = "NodeDown"; expr = ''up{job="node_exporter"} == 0''; for = "5m"; + annotations = { + summary = "Node exporter down on {{ $labels.instance }}"; + description = "Node exporter on {{ $labels.instance }} has been down for more than 5 minutes."; + }; } { alert = "HTTPDown"; expr = ''up{job="blackbox_probe"} == 0 or probe_success{job="blackbox_probe"} == 0''; for = "5m"; + annotations = { + summary = "HTTP probe failure on {{ $labels.instance }}"; + description = "The HTTP blackbox probe on {{ $labels.instance }} has failed for more than 5 minutes."; + }; } { alert = "MemoryFull"; expr = ''node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1''; for = "5m"; + annotations = { + summary = "Low available memory on {{ $labels.instance }}"; + description = "{{ $labels.instance }} has less than 10% available memory for more than 5 minutes."; + }; } { alert = "DiskFull"; expr = ''node_filesystem_avail_bytes{mountpoint=~"/|/mnt"} / node_filesystem_size_bytes < 0.1''; + annotations = { + summary = "Low disk space on {{ $labels.instance }}"; + description = "The disk {{ $labels.device }} mounted at {{ $labels.mountpoint }} on {{ $labels.instance }} has less than 10% of empty space available."; + }; } { alert = "UnitFailed"; expr = ''node_systemd_unit_state{state="failed"} == 1''; + annotations = { + summary = "Systemd unit {{ $labels.name }} failure on {{ $labels.instance }}"; + description = "The systemd unit {{ $labels.name }} on {{ $labels.instance }} has entered a {{ $labels.state }} state."; + }; } ]; }; @@ -136,7 +156,26 @@ in configuration = { receivers = lib.singleton { name = "ntfy"; - webhook_configs = lib.singleton { url = "https://ntfy.ny4.dev/alert"; }; + webhook_configs = lib.singleton { + # https://docs.ntfy.sh/publish/#message-templating + url = + let + tmpl = lib.escapeURL '' + {{ range .alerts }}- Status: {{ .status }} + + Labels: + {{ range $k, $v := .labels }} - {{ $k }} = {{ $v }} + {{ end }} + Annotations: + {{ range $k, $v := .annotations }} - {{ $k }} = {{ $v }} + {{ end }} + Source: {{ .generatorURL }} + + {{ end }} + ''; + in + "https://ntfy.ny4.dev/alert?tpl=yes&md=yes&m=${tmpl}"; + }; }; route = { receiver = "ntfy";