flake/hosts/tyo0/services/prometheus.nix

{
  lib,
  pkgs,
  config,
  ...
}:
{
  services.prometheus = {
    enable = true;
    listenAddress = "127.0.0.1";
    port = 9090;
    webExternalUrl = "https://prom.ny4.dev";

    exporters.blackbox = {
      enable = true;
      listenAddress = "127.0.0.1";
      port = 9093;
      configFile = (pkgs.formats.yaml { }).generate "config.yaml" {
        modules = {
          http_2xx = {
            prober = "http";
          };
        };
      };
    };

    scrapeConfigs = [
      {
        job_name = "metrics";
        scheme = "https";
        basic_auth = {
          username = "prometheus";
          password_file = config.sops.secrets."prometheus/auth".path;
        };
        static_configs = lib.singleton {
          targets = [
            "tyo0.ny4.dev"
            "pek0.ny4.dev"
          ];
        };
      }
      {
        job_name = "http";
        scheme = "http";
        metrics_path = "/probe";
        params = {
          module = [ "http_2xx" ];
        };
        static_configs = lib.singleton {
          targets = [
            "https://blog.ny4.dev"
            "https://cinny.ny4.dev"
            "https://element.ny4.dev"
            "https://git.ny4.dev"
            "https://id.ny4.dev"
            "https://mastodon.ny4.dev"
            "https://matrix.ny4.dev"
            "https://ntfy.ny4.dev"
            "https://pb.ny4.dev"
            "https://reddit.ny4.dev"
            "https://rss.ny4.dev"
            "https://vault.ny4.dev"
          ];
        };
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "127.0.0.1:9093";
          }
        ];
      }

    ];

    rules = lib.singleton (
      builtins.toJSON {
        groups = lib.singleton {
          name = "metrics";
          rules = [
            {
              alert = "NodeDown";
              expr = ''up == 0'';
              for = "5m";
            }
            {
              alert = "HTTPDown";
              expr = ''probe_http_status_code < 200 or probe_http_status_code > 299'';
              for = "5m";
            }
            {
              alert = "MemoryFull";
              expr = ''node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1'';
              for = "5m";
            }
            {
              alert = "DiskFull";
              expr = ''node_filesystem_avail_bytes{mountpoint=~"/|/mnt"} / node_filesystem_size_bytes < 0.1'';
            }
            {
              alert = "UnitFailed";
              expr = ''node_systemd_unit_state{state="failed"} == 1'';
            }
          ];
        };
      }
    );

    alertmanagers = lib.singleton {
      static_configs = lib.singleton {
        targets = [
          "127.0.0.1:9092"
        ];
      };
    };

    alertmanager = {
      enable = true;
      listenAddress = "127.0.0.1";
      port = 9092;

      configuration = {
        receivers = lib.singleton {
          name = "ntfy";
          webhook_configs = lib.singleton {
            url = "https://ntfy.ny4.dev/alert";
          };
        };
        route = {
          receiver = "ntfy";
        };
      };
    };
  };
}