tyo0: add grafana

also enables ntfy authentication for alertmanager

hosts/aws/tyo0/default.nix

@@ -11,6 +11,7 @@
     ./anti-feature.nix
 
     ./services/forgejo.nix
+    ./services/grafana.nix
     ./services/keycloak.nix
     ./services/miniflux.nix
     ./services/murmur.nix
@@ -44,8 +45,27 @@
     "vaultwarden/environment" = {
       restartUnits = [ "vaultwarden.service" ];
     };
+    "grafana/environment" = {
+      restartUnits = [ "grafana.service" ];
+    };
+    "alertmanager/webhook" = {
+      restartUnits = [ "alertmanager.service" ];
+    };
   };
 
+  sops.templates."alertmanager/environment".content =
+    let
+      tmpl = lib.escapeURL ''
+        {{ range .alerts }}- Status: {{ .status }}
+          Summary: {{ .annotations.summary }}
+          Description: {{ .annotations.description }}
+          Source: {{ .generatorURL }}
+        {{ end }}
+      '';
+      token = config.sops.placeholder."alertmanager/webhook";
+    in
+    "ALERTMANAGER_WEBHOOK_URL=https://ntfy.ny4.dev/alert?tpl=yes&md=yes&m=${tmpl}&auth=${token}";
+
   ### Services
   networking.firewall.allowedUDPPorts = [ 443 ];
   networking.firewall.allowedTCPPorts = [
@@ -91,6 +111,7 @@
   systemd.services."caddy".serviceConfig.SupplementaryGroups = [
     "forgejo"
     "ntfy-sh"
+    "grafana"
   ];
 
   services.caddy.settings.apps.http.servers.srv0.routes = [
@@ -243,6 +264,13 @@
       min_wal_size = "1GB";
       max_wal_size = "4GB";
     };
+    initialScript = pkgs.writeText "grafana-init.sql" ''
+      CREATE ROLE "grafana" with LOGIN;
+      CREATE DATABASE "grafana" WITH OWNER "grafana"
+        TEMPLATE template0
+        LC_COLLATE = "C"
+        LC_CTYPE = "C";
+    '';
   };
 
   ### Prevents me from bankrupt

hosts/aws/tyo0/secrets.yaml

@@ -4,6 +4,10 @@ vaultwarden:
     environment: ENC[AES256_GCM,data:+pcUVL7yVXKVp57/feHHWmSuH/2B0hLtADxZWCQOOMG+M3UQh+4dHA5debiv,iv:Zy6xn4Z4VwVXfWWjVeCYY/gRnDp//7yUPLbtLuABFPY=,tag:LxEc31YhgyjEhDrqoJxCJw==,type:str]
 prometheus:
     auth: ENC[AES256_GCM,data:sQ7oEL2gGz2nnn+QGcmmI3IwNEWbZ13s2/3QLj0O0BZp,iv:r7F70DzMNrcuxq2LISwm4tXjiR8m9eyt8GQyiuWxvhM=,tag:LfpxK3wcuMFCmFQn/iPZsw==,type:str]
+grafana:
+    environment: ENC[AES256_GCM,data:eKPAXKZ6FcG6c0AffntCxyB7/1zT1rbThjIGaQKYPoWj2hJp7WbeXjhl3GmhOfeh4a6C3zAlue9myyN3PWCe8C7LxcdK,iv:ARYxxB2XJyYmQgbBPY749jcb+Q+Qv5OBxDPNQLMUqSM=,tag:fKeKQKWPLmrTlMoauCuXQg==,type:str]
+alertmanager:
+    webhook: ENC[AES256_GCM,data:P1gnjw4rSmpNMoC+XM2Lk7UddZgswtB1XzyVNssvr3hs6dQPJAWfScA1Em6REDQ3VYj65g==,iv:UWFG6mgX6K9zeQvvH5/uMtWj2B7ZTkZP66oraf7hD8M=,tag:Z4sLXU0XLZp6J7iw+HFUyQ==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -28,8 +32,8 @@ sops:
             UkYrb3JpZDBzOUgzWXFQbUZnWjNUUjAKKuJmaJ6kV5ITsCMXEOzv9ym3L9VQKoB4
             n/SE4eCXeaoE/1UCdw4VlpyuUuouHh2pgLWJF49dHhY/zhv84sURtA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-09-27T10:33:46Z"
+    lastmodified: "2024-11-10T08:21:14Z"
-    mac: ENC[AES256_GCM,data:7O6cvnkVrD5H9Cwv8LO2E0X4jhuLzjPfsJkKjc+1a/4s07XgyYyYfH4ha2jSkWEmudCcZFLO4iSBZgf128Favw4je6tF58oFKWUjCxIgI4n3b7wSdywh8dZae4Dy+kBEKVgeoR4vXm2VZCKNcTF9SIQFWRXz0SP2uKiTcA5grFA=,iv:J6W8+yfXr+L2/x9KBA+FyiYHXzrdts2rNLtgI2JB7qs=,tag:GXU8Ckl+9vVgBNZGcGzvng==,type:str]
+    mac: ENC[AES256_GCM,data:xQfDh2qH3opNHQ8t5Yy0sktqS1ioeS51UoJlD1vTfQNDSGRngPySwzv1jI7P4Bp9WnEGXw7ILHSU3bCGfTtk18T56cA4Atn3BFSiyc22H6omVLRQ8/0oBhwukxplOgAD9e1UWBlGwPEuDEFtFlqVP9M+83Vf+KGpHoMKCOv7eTU=,iv:5iEYnzYE1w40cmvEWWA4FjFpkTltA/mGNIKiYt8G0DY=,tag:W0rROnEnTBcc8lseEjDRgQ==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
-    version: 3.9.0
+    version: 3.9.1

hosts/aws/tyo0/services/forgejo.nix

@@ -17,6 +17,10 @@
         UNIX_SOCKET_PERMISSION = "660";
       };
 
+      metrics = {
+        ENABLED = true;
+      };
+
       service = {
         ALLOW_ONLY_EXTERNAL_REGISTRATION = true;
       };

hosts/aws/tyo0/services/grafana.nix

@@ -0,0 +1,53 @@
+{ lib, config, ... }:
+{
+  services.grafana = {
+    enable = true;
+    settings = {
+      "auth.generic_oauth" = {
+        enabled = "true";
+        name = "keycloak";
+        allow_sign_up = "true";
+        client_id = "grafana";
+        # client_secret = "YOUR_APP_CLIENT_SECRET";
+        scopes = "openid email profile offline_access roles";
+        email_attribute_path = "email";
+        login_attribute_path = "username";
+        name_attribute_path = "full_name";
+        auth_url = "https://id.ny4.dev/realms/ny4/protocol/openid-connect/auth";
+        token_url = "https://id.ny4.dev/realms/ny4/protocol/openid-connect/token";
+        api_url = "https://id.ny4.dev/realms/ny4/protocol/openid-connect/userinfo";
+        role_attribute_path = "contains(resource_access.grafana.roles[*], 'grafanaadmin') && 'GrafanaAdmin' || contains(resource_access.grafana.roles[*], 'admin') && 'Admin' || contains(resource_access.grafana.roles[*], 'editor') && 'Editor' || contains(resource_access.grafana.roles[*], 'viewer') && 'Viewer'";
+        allow_assign_grafana_admin = true;
+        role_attribute_strict = true;
+      };
+      analytics = {
+        reporting_enabled = false;
+        feedback_links_enabled = false;
+      };
+      auth = {
+        disable_login_form = true;
+      };
+      database = {
+        type = "postgres";
+        name = "grafana";
+        user = "grafana";
+        host = "/run/postgresql";
+      };
+      server = {
+        protocol = "socket";
+        root_url = "https://grafana.ny4.dev/";
+      };
+    };
+  };
+
+  systemd.services."grafana".serviceConfig.EnvironmentFile =
+    config.sops.secrets."grafana/environment".path;
+
+  services.caddy.settings.apps.http.servers.srv0.routes = lib.singleton {
+    match = lib.singleton { host = [ "grafana.ny4.dev" ]; };
+    handle = lib.singleton {
+      handler = "reverse_proxy";
+      upstreams = [ { dial = "unix/${config.services.grafana.settings.server.socket}"; } ];
+    };
+  };
+}

hosts/aws/tyo0/services/miniflux.nix

@@ -20,6 +20,7 @@
     config = {
       CREATE_ADMIN = 0;
       BASE_URL = "https://rss.ny4.dev";
+      METRICS_COLLECTOR = 1;
 
       OAUTH2_PROVIDER = "oidc";
       OAUTH2_CLIENT_ID = "miniflux";

hosts/aws/tyo0/services/ntfy.nix

@@ -8,6 +8,8 @@
       listen-unix = "/run/ntfy-sh/ntfy.sock";
       listen-unix-mode = 432; # 0660
       behind-proxy = true;
+      # TODO: basic auth this
+      enable-metrics = true;
     };
   };
 

hosts/aws/tyo0/services/prometheus.nix

@@ -49,6 +49,24 @@ in
         };
         static_configs = lib.singleton { inherit targets; };
       }
+      {
+        job_name = "ntfy";
+        scheme = "https";
+        metrics_path = "/metrics";
+        static_configs = lib.singleton { targets = [ "ntfy.ny4.dev" ]; };
+      }
+      {
+        job_name = "forgejo";
+        scheme = "https";
+        metrics_path = "/metrics";
+        static_configs = lib.singleton { targets = [ "git.ny4.dev" ]; };
+      }
+      {
+        job_name = "miniflux";
+        scheme = "https";
+        metrics_path = "/metrics";
+        static_configs = lib.singleton { targets = [ "rss.ny4.dev" ]; };
+      }
       {
         job_name = "blackbox_exporter";
         static_configs = lib.singleton { targets = [ "127.0.0.1:${toString ports.blackbox}" ]; };
@@ -153,6 +171,7 @@ in
 
     alertmanager = {
       enable = true;
+      checkConfig = false;
       listenAddress = "127.0.0.1";
       port = ports.alertmanager;
 
@@ -161,17 +180,7 @@ in
           name = "ntfy";
           webhook_configs = lib.singleton {
             # https://docs.ntfy.sh/publish/#message-templating
-            url =
+            url = "$ALERTMANAGER_WEBHOOK_URL";
-              let
-                tmpl = lib.escapeURL ''
-                  {{ range .alerts }}- Status: {{ .status }}
-                    Summary: {{ .annotations.summary }}
-                    Description: {{ .annotations.description }}
-                    Source: {{ .generatorURL }}
-                  {{ end }}
-                '';
-              in
-              "https://ntfy.ny4.dev/alert?tpl=yes&md=yes&m=${tmpl}";
           };
         };
         route = {
@@ -181,6 +190,9 @@ in
     };
   };
 
+  systemd.services."alertmanager".serviceConfig.EnvironmentFile =
+    config.sops.templates."alertmanager/environment".path;
+
   services.caddy.settings.apps.http.servers.srv0.routes = lib.singleton {
     match = lib.singleton { host = [ "prom.ny4.dev" ]; };
     handle = lib.singleton {

infra/cloudflare.tf

@@ -214,6 +214,15 @@ resource "cloudflare_record" "terraform_managed_resource_e2500de6c975c90729b8f35
   zone_id = local.cloudflare_zone_id
 }
 
+resource "cloudflare_record" "grafana" {
+  content = "tyo0.ny4.dev"
+  name    = "grafana"
+  proxied = true
+  ttl     = 1
+  type    = "CNAME"
+  zone_id = local.cloudflare_zone_id
+}
+
 resource "cloudflare_record" "terraform_managed_resource_856ec5e567960bf847db2e814f18168b" {
   content = "google-site-verification=wBL5EFnbnt9lt2j_BtcwlXTaBFlFT563mC1MkCscnR8"
   name    = "ny4.dev"