nyancat/flake
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tyo0/prometheus: send formatted message to ntfy

Browse source
This commit is contained in:
Guanran Wang 2024-10-04 18:42:27 +08:00
parent 8dabc94210
commit 7d738d5336
Signed by: nyancat
GPG key ID: 91F97D9ED12639CF
1 changed files with 40 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
hosts/aws/tyo0/services/prometheus.nix
View file

@ -100,24 +100,44 @@ in
alert = "NodeDown";
expr = ''up{job="node_exporter"} == 0'';
for = "5m";
annotations = {
summary = "Node exporter down on {{ $labels.instance }}";
description = "Node exporter on {{ $labels.instance }} has been down for more than 5 minutes.";
};
}
{
alert = "HTTPDown";
expr = ''up{job="blackbox_probe"} == 0 or probe_success{job="blackbox_probe"} == 0'';
for = "5m";
annotations = {
summary = "HTTP probe failure on {{ $labels.instance }}";
description = "The HTTP blackbox probe on {{ $labels.instance }} has failed for more than 5 minutes.";
};
}
{
alert = "MemoryFull";
expr = ''node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1'';
for = "5m";
annotations = {
summary = "Low available memory on {{ $labels.instance }}";
description = "{{ $labels.instance }} has less than 10% available memory for more than 5 minutes.";
};
}
{
alert = "DiskFull";
expr = ''node_filesystem_avail_bytes{mountpoint=~"/|/mnt"} / node_filesystem_size_bytes < 0.1'';
annotations = {
summary = "Low disk space on {{ $labels.instance }}";
description = "The disk {{ $labels.device }} mounted at {{ $labels.mountpoint }} on {{ $labels.instance }} has less than 10% of empty space available.";
};
}
{
alert = "UnitFailed";
expr = ''node_systemd_unit_state{state="failed"} == 1'';
annotations = {
summary = "Systemd unit {{ $labels.name }} failure on {{ $labels.instance }}";
description = "The systemd unit {{ $labels.name }} on {{ $labels.instance }} has entered a {{ $labels.state }} state.";
};
}
];
};
@ -136,7 +156,26 @@ in
configuration = {
receivers = lib.singleton {
name = "ntfy";
webhook_configs = lib.singleton { url = "https://ntfy.ny4.dev/alert"; };
webhook_configs = lib.singleton {
# https://docs.ntfy.sh/publish/#message-templating
url =
let
tmpl = lib.escapeURL ''
{{ range .alerts }}- Status: {{ .status }}
Labels:
{{ range $k, $v := .labels }} - {{ $k }} = {{ $v }}
{{ end }}
Annotations:
{{ range $k, $v := .annotations }} - {{ $k }} = {{ $v }}
{{ end }}
Source: {{ .generatorURL }}
{{ end }}
'';
in
"https://ntfy.ny4.dev/alert?tpl=yes&md=yes&m=${tmpl}";
};
};
route = {
receiver = "ntfy";