From 5f8f3eb171428aede379af4478d8444ab5b571a2 Mon Sep 17 00:00:00 2001
From: Guanran Wang <guanran928@outlook.com>
Date: Sat, 7 Sep 2024 20:38:10 +0800
Subject: [PATCH] nixos/caddy: export metrics to prometheus

---
 hosts/tyo0/services/prometheus.nix    | 40 +++++++++++++-----
 nixos/profiles/prometheus/default.nix | 59 ++++++++++++++++++---------
 2 files changed, 70 insertions(+), 29 deletions(-)

diff --git a/hosts/tyo0/services/prometheus.nix b/hosts/tyo0/services/prometheus.nix
index 07d3fc5..b9a1a42 100644
--- a/hosts/tyo0/services/prometheus.nix
+++ b/hosts/tyo0/services/prometheus.nix
@@ -19,18 +19,17 @@ in
       listenAddress = "127.0.0.1";
       port = ports.blackbox;
       configFile = (pkgs.formats.yaml { }).generate "config.yaml" {
-        modules = {
-          http_2xx = {
-            prober = "http";
-          };
+        modules.http_2xx = {
+          prober = "http";
+          http.fail_if_not_ssl = true;
         };
       };
     };
 
     scrapeConfigs = [
       {
-        job_name = "metrics";
-        scheme = "https";
+        job_name = "node_exporter";
+        metrics_path = "/metrics";
         basic_auth = {
           username = "prometheus";
           password_file = config.sops.secrets."prometheus/auth".path;
@@ -43,8 +42,29 @@ in
         };
       }
       {
-        job_name = "http";
-        scheme = "http";
+        job_name = "caddy";
+        metrics_path = "/caddy";
+        basic_auth = {
+          username = "prometheus";
+          password_file = config.sops.secrets."prometheus/auth".path;
+        };
+        static_configs = lib.singleton {
+          targets = [
+            "tyo0.ny4.dev"
+            "pek0.ny4.dev"
+          ];
+        };
+      }
+      {
+        job_name = "blackbox_exporter";
+        static_configs = lib.singleton {
+          targets = [
+            "127.0.0.1:${toString ports.blackbox}"
+          ];
+        };
+      }
+      {
+        job_name = "blackbox_probe";
         metrics_path = "/probe";
         params = {
           module = [ "http_2xx" ];
@@ -90,12 +110,12 @@ in
           rules = [
             {
               alert = "NodeDown";
-              expr = ''up == 0'';
+              expr = ''up{job="node_exporter"} == 0'';
               for = "5m";
             }
             {
               alert = "HTTPDown";
-              expr = ''probe_http_status_code < 200 or probe_http_status_code > 299'';
+              expr = ''up{job="blackbox_probe"} == 0 or probe_success{job="blackbox_probe"} == 0'';
               for = "5m";
             }
             {
diff --git a/nixos/profiles/prometheus/default.nix b/nixos/profiles/prometheus/default.nix
index 0fe4b4b..b5dd188 100644
--- a/nixos/profiles/prometheus/default.nix
+++ b/nixos/profiles/prometheus/default.nix
@@ -7,23 +7,44 @@
     enabledCollectors = [ "systemd" ];
   };
 
-  services.caddy.settings.apps.http.servers.srv0.routes = lib.singleton {
-    match = lib.singleton {
-      host = [ config.networking.fqdn ];
-      path = [ "/metrics" ];
-    };
-    handle = [
-      {
-        handler = "authentication";
-        providers.http_basic.accounts = lib.singleton {
-          username = "prometheus";
-          password = "$2a$14$2Phk4tobM04H4XiGegB3TuEXkyORCKMKW8TptYPTPXUWmZgtGBj/.";
-        };
-      }
-      {
-        handler = "reverse_proxy";
-        upstreams = lib.singleton { dial = "127.0.0.1:9091"; };
-      }
-    ];
-  };
+  services.caddy.settings.apps.http.servers.srv0.metrics = { };
+  services.caddy.settings.apps.http.servers.srv0.routes = [
+    {
+      match = lib.singleton {
+        host = [ config.networking.fqdn ];
+        path = [ "/metrics" ];
+      };
+      handle = [
+        {
+          handler = "authentication";
+          providers.http_basic.accounts = lib.singleton {
+            username = "prometheus";
+            password = "$2a$14$2Phk4tobM04H4XiGegB3TuEXkyORCKMKW8TptYPTPXUWmZgtGBj/.";
+          };
+        }
+        {
+          handler = "reverse_proxy";
+          upstreams = lib.singleton { dial = "127.0.0.1:9091"; };
+        }
+      ];
+    }
+    {
+      match = lib.singleton {
+        host = [ config.networking.fqdn ];
+        path = [ "/caddy" ];
+      };
+      handle = [
+        {
+          handler = "authentication";
+          providers.http_basic.accounts = lib.singleton {
+            username = "prometheus";
+            password = "$2a$14$2Phk4tobM04H4XiGegB3TuEXkyORCKMKW8TptYPTPXUWmZgtGBj/.";
+          };
+        }
+        {
+          handler = "metrics";
+        }
+      ];
+    }
+  ];
 }