service/alertmanager: init

machines/newton/services.nix

@@ -112,6 +112,9 @@ in
     prometheus = {
       enable = true;
     };
+    alertmanager = {
+      enable = true;
+    };
     grafana = {
       enable = true;
       passwordFile = secrets."grafana/password".path;

modules/services/alertmanager/config.nix

@@ -0,0 +1,24 @@
+{
+  global = {
+    smtp_smarthost = "localhost:25";
+    smtp_from = "server@buehler.rocks";
+  };
+  # templates = [ ];
+  route = {
+    receiver = "default";
+    group_wait = "30s";
+    group_interval = "5m";
+    repeat_interval = "4h";
+    routes = [ ];
+  };
+  receivers = [
+    {
+      name = "default";
+      email_configs = [
+        {
+          to = "server@buehler.rocks";
+        }
+      ];
+    }
+  ];
+}

modules/services/alertmanager/default.nix

@@ -0,0 +1,91 @@
+# monitoring system services
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.my.services.alertmanager;
+  domain = config.networking.domain;
+in
+{
+  options.my.services.alertmanager = with lib; {
+    enable = mkEnableOption "Prometheus alertmanager for monitoring";
+    port = mkOption {
+      type = types.port;
+      default = 9093;
+      example = 3002;
+      description = "Internal alertmanager port";
+
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = config.services.prometheus.enable;
+        message = ''
+          Enable alertmanager without prometheus does not work. Please enable prometheus as well.
+        '';
+      }
+    ];
+
+    services.prometheus = {
+      alertmanager = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        port = cfg.port;
+        configuration = import ./config.nix;
+        webExternalUrl = "https://alerts.${domain}";
+        # fix issue: https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4556
+        extraFlags = [ "--cluster.advertise-address 127.0.0.1:${toString cfg.port}" ];
+      };
+
+      alertmanagers = [
+        {
+          static_configs = [
+            {
+              targets = [ "localhost:${toString cfg.port}" ];
+            }
+          ];
+        }
+      ];
+      scrapeConfigs = [
+        {
+          job_name = "alertmanager";
+          static_configs = [{
+            targets = [ "127.0.0.1:${toString cfg.port}" ];
+            labels = {
+              instance = config.networking.hostName;
+            };
+          }];
+        }
+      ];
+    };
+    services.grafana.provision = {
+      datasources.settings.datasources = [
+        {
+          name = "Alertmanager";
+          type = "alertmanager";
+          url = "http://127.0.0.1:${toString cfg.port}";
+          jsonData = {
+            implementation = "prometheus";
+            handleGrafanaManagedAlerts = config.services.prometheus.enable;
+          };
+        }
+      ];
+    };
+    my.services.nginx.virtualHosts = [
+      {
+        subdomain = "alerts";
+        inherit (cfg) port;
+      }
+    ];
+
+    webapps.apps = {
+      alertmanager.dashboard = {
+        name = "Alerting";
+        category = "infra";
+        icon = "bell";
+        link = "https://alerts.${domain}";
+        method = "get";
+      };
+    };
+  };
+}

modules/services/default.nix

@@ -1,6 +1,7 @@
 { ... }:
 {
   imports = [
+    ./alertmanager
     ./backup
     ./bazarr
     ./blocky