flake/hosts/tyo0/services/prometheus.nix

142 lines
3.3 KiB
Nix

{
lib,
pkgs,
config,
...
}:
{
services.prometheus = {
enable = true;
listenAddress = "127.0.0.1";
port = 9090;
webExternalUrl = "https://prom.ny4.dev";
exporters.blackbox = {
enable = true;
listenAddress = "127.0.0.1";
port = 9093;
configFile = (pkgs.formats.yaml { }).generate "config.yaml" {
modules = {
http_2xx = {
prober = "http";
};
};
};
};
scrapeConfigs = [
{
job_name = "metrics";
scheme = "https";
basic_auth = {
username = "prometheus";
password_file = config.sops.secrets."prometheus/auth".path;
};
static_configs = lib.singleton {
targets = [
"tyo0.ny4.dev"
"pek0.ny4.dev"
];
};
}
{
job_name = "http";
scheme = "http";
metrics_path = "/probe";
params = {
module = [ "http_2xx" ];
};
static_configs = lib.singleton {
targets = [
"https://blog.ny4.dev"
"https://cinny.ny4.dev"
"https://element.ny4.dev"
"https://git.ny4.dev"
"https://id.ny4.dev"
"https://mastodon.ny4.dev"
"https://matrix.ny4.dev"
"https://ntfy.ny4.dev"
"https://pb.ny4.dev"
"https://reddit.ny4.dev"
"https://rss.ny4.dev"
"https://vault.ny4.dev"
];
};
relabel_configs = [
{
source_labels = [ "__address__" ];
target_label = "__param_target";
}
{
source_labels = [ "__param_target" ];
target_label = "instance";
}
{
target_label = "__address__";
replacement = "127.0.0.1:9093";
}
];
}
];
rules = lib.singleton (
builtins.toJSON {
groups = lib.singleton {
name = "metrics";
rules = [
{
alert = "NodeDown";
expr = ''up == 0'';
for = "5m";
}
{
alert = "HTTPDown";
expr = ''probe_http_status_code < 200 or probe_http_status_code > 299'';
for = "5m";
}
{
alert = "MemoryFull";
expr = ''node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1'';
for = "5m";
}
{
alert = "DiskFull";
expr = ''node_filesystem_avail_bytes{mountpoint=~"/|/mnt"} / node_filesystem_size_bytes < 0.1'';
}
{
alert = "UnitFailed";
expr = ''node_systemd_unit_state{state="failed"} == 1'';
}
];
};
}
);
alertmanagers = lib.singleton {
static_configs = lib.singleton {
targets = [
"127.0.0.1:9092"
];
};
};
alertmanager = {
enable = true;
listenAddress = "127.0.0.1";
port = 9092;
configuration = {
receivers = lib.singleton {
name = "ntfy";
webhook_configs = lib.singleton {
url = "https://ntfy.ny4.dev/alert";
};
};
route = {
receiver = "ntfy";
};
};
};
};
}