diff --git a/machines/newton/services.nix b/machines/newton/services.nix index 43f2ef0..5073b82 100644 --- a/machines/newton/services.nix +++ b/machines/newton/services.nix @@ -112,6 +112,9 @@ in prometheus = { enable = true; }; + alertmanager = { + enable = true; + }; grafana = { enable = true; passwordFile = secrets."grafana/password".path; diff --git a/modules/services/alertmanager/config.nix b/modules/services/alertmanager/config.nix new file mode 100644 index 0000000..fcf2581 --- /dev/null +++ b/modules/services/alertmanager/config.nix @@ -0,0 +1,24 @@ +{ + global = { + smtp_smarthost = "localhost:25"; + smtp_from = "server@buehler.rocks"; + }; + # templates = [ ]; + route = { + receiver = "default"; + group_wait = "30s"; + group_interval = "5m"; + repeat_interval = "4h"; + routes = [ ]; + }; + receivers = [ + { + name = "default"; + email_configs = [ + { + to = "server@buehler.rocks"; + } + ]; + } + ]; +} diff --git a/modules/services/alertmanager/default.nix b/modules/services/alertmanager/default.nix new file mode 100644 index 0000000..95c9f29 --- /dev/null +++ b/modules/services/alertmanager/default.nix @@ -0,0 +1,91 @@ +# monitoring system services +{ config, lib, pkgs, ... }: +let + cfg = config.my.services.alertmanager; + domain = config.networking.domain; +in +{ + options.my.services.alertmanager = with lib; { + enable = mkEnableOption "Prometheus alertmanager for monitoring"; + port = mkOption { + type = types.port; + default = 9093; + example = 3002; + description = "Internal alertmanager port"; + + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = config.services.prometheus.enable; + message = '' + Enable alertmanager without prometheus does not work. Please enable prometheus as well. + ''; + } + ]; + + services.prometheus = { + alertmanager = { + enable = true; + listenAddress = "127.0.0.1"; + port = cfg.port; + configuration = import ./config.nix; + webExternalUrl = "https://alerts.${domain}"; + # fix issue: https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4556 + extraFlags = [ "--cluster.advertise-address 127.0.0.1:${toString cfg.port}" ]; + }; + + alertmanagers = [ + { + static_configs = [ + { + targets = [ "localhost:${toString cfg.port}" ]; + } + ]; + } + ]; + scrapeConfigs = [ + { + job_name = "alertmanager"; + static_configs = [{ + targets = [ "127.0.0.1:${toString cfg.port}" ]; + labels = { + instance = config.networking.hostName; + }; + }]; + } + ]; + }; + services.grafana.provision = { + datasources.settings.datasources = [ + { + name = "Alertmanager"; + type = "alertmanager"; + url = "http://127.0.0.1:${toString cfg.port}"; + jsonData = { + implementation = "prometheus"; + handleGrafanaManagedAlerts = config.services.prometheus.enable; + }; + } + ]; + }; + my.services.nginx.virtualHosts = [ + { + subdomain = "alerts"; + inherit (cfg) port; + } + ]; + + webapps.apps = { + alertmanager.dashboard = { + name = "Alerting"; + category = "infra"; + icon = "bell"; + link = "https://alerts.${domain}"; + method = "get"; + }; + }; + }; +} diff --git a/modules/services/default.nix b/modules/services/default.nix index 9f6f718..902ff03 100644 --- a/modules/services/default.nix +++ b/modules/services/default.nix @@ -1,6 +1,7 @@ { ... }: { imports = [ + ./alertmanager ./backup ./bazarr ./blocky