From 32c4c30bdd6d52f00b02dc22b52cf30bcb507599 Mon Sep 17 00:00:00 2001
From: Felix Buehler <account@buehler.rocks>
Date: Mon, 19 Jun 2023 22:20:36 +0200
Subject: [PATCH] service/backup: add alerts when systemd fails

---
 modules/services/backup/default.nix | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/modules/services/backup/default.nix b/modules/services/backup/default.nix
index 09c2d7b..e36cf22 100644
--- a/modules/services/backup/default.nix
+++ b/modules/services/backup/default.nix
@@ -161,5 +161,16 @@ in
         yearly = 2;
       };
     };
+
+    my.services.prometheus.rules = {
+      borgbackup_execution_missing = {
+        condition = ''time() - systemd_timer_last_trigger_seconds{name="borgbackup-job-hetzner.timer"} >= (60 * 60 * (24 + 1))'';
+        description = "{{$labels.instance}}: last backup was 25 hours ago please check.";
+      };
+      borgbackup_last_execution = {
+        condition = ''systemd_unit_state{state="failed", name="borgbackup-job-hetzner.timer"} >= 1'';
+        description = "{{$labels.instance}}: last backup was not successfull please check.";
+      };
+    };
   };
 }