From d4fa5996aecbe71b92f142d4caf6e294ddf0100b Mon Sep 17 00:00:00 2001 From: Felix Buehler Date: Sun, 25 Jun 2023 21:25:55 +0200 Subject: [PATCH] service/blackbox: init --- machines/newton/services.nix | 3 + modules/services/blackbox/default.nix | 92 +++ modules/services/default.nix | 1 + modules/services/homer/config.nix | 1 + pkgs/grafana-dashboards/blackbox.json | 927 ++++++++++++++++++++++++++ pkgs/grafana-dashboards/default.nix | 10 + 6 files changed, 1034 insertions(+) create mode 100644 modules/services/blackbox/default.nix create mode 100644 pkgs/grafana-dashboards/blackbox.json diff --git a/machines/newton/services.nix b/machines/newton/services.nix index 9624a4d..a33c844 100644 --- a/machines/newton/services.nix +++ b/machines/newton/services.nix @@ -132,6 +132,9 @@ in promtail = { enable = true; }; + blackbox = { + enable = true; + }; # Webserver nginx = { enable = true; diff --git a/modules/services/blackbox/default.nix b/modules/services/blackbox/default.nix new file mode 100644 index 0000000..bd480ea --- /dev/null +++ b/modules/services/blackbox/default.nix @@ -0,0 +1,92 @@ +# monitor urls +{ config, lib, pkgs, ... }: +let + cfg = config.my.services.blackbox; + domain = config.networking.domain; + blackBoxConfig = { + modules = { + http_2xx = { + prober = "http"; + http.preferred_ip_protocol = "ip4"; + }; + ssh_banner = { + prober = "tcp"; + tcp.query_response = [ + { + send = "SSH-2.0-blackbox-ssh-check"; + } + { + expect = "^SSH-2.0-"; + } + ]; + }; + }; + }; +in +{ + options.my.services.blackbox = with lib; { + enable = mkEnableOption "Blackbox prometheus exporter"; + + http_endpoints = mkOption { + type = types.listOf types.str; + default = [ ]; + example = literalExpression '' + [ + "https://domain.com" + "https://another-domain.com" + ] + ''; + description = '' + List of domains to test. + ''; + }; + }; + + config = lib.mkIf cfg.enable { + services.prometheus.exporters.blackbox = { + enable = true; + configFile = pkgs.writeText "blackbox-config.yml" (builtins.toJSON blackBoxConfig); + }; + + # relabels as in https://github.com/prometheus/blackbox_exporter#prometheus-configuration + services.prometheus = { + scrapeConfigs = [ + { + job_name = "blackbox"; + metrics_path = "/probe"; + params.module = [ "http_2xx" ]; + static_configs = [ + { + targets = cfg.http_endpoints; + labels = { + instance = config.networking.hostName; + }; + } + ]; + relabel_configs = [ + { + source_labels = [ "__address__" ]; + target_label = "__param_target"; + } + { + source_labels = [ "__param_target" ]; + target_label = "instance"; + } + { + target_label = "__address__"; + replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"; + } + ]; + } + ]; + }; + + services.grafana.provision.dashboards.settings.providers = [ + { + name = "Blackbox"; + options.path = pkgs.grafana-dashboards.blackbox; + disableDeletion = true; + } + ]; + }; +} diff --git a/modules/services/default.nix b/modules/services/default.nix index c707a3e..05d11e2 100644 --- a/modules/services/default.nix +++ b/modules/services/default.nix @@ -5,6 +5,7 @@ ./aria2 ./backup ./bazarr + ./blackbox ./blocky ./dyndns ./freshrss diff --git a/modules/services/homer/config.nix b/modules/services/homer/config.nix index 5504b77..c456ddd 100644 --- a/modules/services/homer/config.nix +++ b/modules/services/homer/config.nix @@ -124,5 +124,6 @@ }); } ); + my.services.blackbox.http_endpoints = lib.mapAttrsToList (key: value: value.dashboard.link) config.webapps.apps; }; } diff --git a/pkgs/grafana-dashboards/blackbox.json b/pkgs/grafana-dashboards/blackbox.json new file mode 100644 index 0000000..b9e3501 --- /dev/null +++ b/pkgs/grafana-dashboards/blackbox.json @@ -0,0 +1,927 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Blackbox exporter HTTP prober dashboard", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 13659, + "graphTooltip": 0, + "id": 58, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "SSL Cert Expiry (days)" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(0, 0, 0, 0)", + "value": null + }, + { + "color": "red", + "value": 0 + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "yellow", + "value": 7 + }, + { + "color": "green", + "value": 24 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "gauge" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 365 + }, + { + "id": "custom.filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "text": "DOWN" + }, + "1": { + "text": "UP" + } + }, + "type": "value" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "custom.width", + "value": 76 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Code" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(0, 0, 0, 0)", + "value": null + }, + { + "color": "green", + "value": 200 + }, + { + "color": "yellow", + "value": 300 + }, + { + "color": "red", + "value": 500 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "text": "" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SSL" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "text": "NO" + }, + "1": { + "text": "OK" + } + }, + "type": "value" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(3, 3, 3, 0)", + "value": null + }, + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "custom.width", + "value": 77 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Probe Duration (s)" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "gauge" + } + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "max", + "value": 3 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "DNS Lookup Duration (s)" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 0.2 + } + ] + } + }, + { + "id": "max", + "value": 0.3 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "gauge" + } + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "decimals", + "value": 3 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Instance" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "${__data.fields.Instance}", + "url": "${__data.fields.Instance}" + } + ] + }, + { + "id": "custom.width", + "value": 276 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "TLS Version" + }, + "properties": [ + { + "id": "custom.width", + "value": 117 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "SSL Cert Expiry (days)" + } + ] + }, + "pluginVersion": "9.5.3", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "expr": "probe_success{job=~\"$job\", instance=~\"$instance\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "expr": "probe_http_ssl{job=~\"$job\", instance=~\"$instance\"} > 0", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "expr": "(probe_ssl_earliest_cert_expiry{job=~\"$job\", instance=~\"$instance\"} - time()) / 3600 / 24", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "expr": "probe_http_status_code{job=~\"$job\", instance=~\"$instance\"} > 0", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "expr": "avg_over_time(probe_duration_seconds{job=~\"$job\", instance=~\"$instance\"}[1m])", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "expr": "probe_tls_version_info{job=~\"$job\", instance=~\"$instance\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "expr": "avg_over_time(probe_dns_lookup_time_seconds{job=~\"$job\", instance=~\"$instance\"}[1m])", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "G" + } + ], + "title": "HTTP Probe Overview", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "instance" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Time 8": true, + "Value": false, + "Value #A": false, + "Value #B": false, + "Value #F": true, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "__name__ 3": true, + "__name__ 4": true, + "__name__ 5": true, + "__name__ 6": true, + "__name__ 7": true, + "job": true, + "job 1": true, + "job 2": true, + "job 3": true, + "job 4": true, + "job 5": true, + "job 6": true, + "job 7": true, + "job 8": true, + "phase": true, + "type": true, + "type 1": true, + "type 2": true, + "type 3": true, + "type 4": true, + "type 5": true, + "type 6": true, + "type 7": true, + "type 8": true, + "version": false + }, + "indexByName": { + "Time 1": 9, + "Time 2": 13, + "Time 3": 17, + "Time 4": 20, + "Time 5": 24, + "Time 6": 28, + "Time 7": 32, + "Value #A": 1, + "Value #B": 3, + "Value #C": 5, + "Value #D": 2, + "Value #E": 6, + "Value #F": 8, + "Value #G": 7, + "__name__ 1": 10, + "__name__ 2": 14, + "__name__ 3": 21, + "__name__ 4": 25, + "__name__ 5": 29, + "instance": 0, + "job 1": 11, + "job 2": 15, + "job 3": 18, + "job 4": 22, + "job 5": 26, + "job 6": 30, + "type 1": 12, + "type 2": 16, + "type 3": 19, + "type 4": 23, + "type 5": 27, + "type 6": 31, + "version": 4 + }, + "renameByName": { + "Value": "Up", + "Value #A": "Status", + "Value #B": "SSL", + "Value #C": "SSL Cert Expiry (days)", + "Value #D": "Code", + "Value #E": "Probe Duration (s)", + "Value #F": "", + "Value #G": "DNS Lookup Duration (s)", + "Value #H": "Probe IP", + "instance": "Instance", + "type 6": "", + "version": "TLS Version" + } + } + } + ], + "transparent": true, + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "description": "Returns how long the probe took to complete in seconds", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "expr": "sum(probe_http_duration_seconds{job=~\"$job\", instance=~\"$instance\"}) by (instance)", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "HTTP Probe Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 8, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "description": "Duration of http request by phase, summed over all redirects", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "expr": "probe_http_duration_seconds{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ phase }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "HTTP Probe Phases Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "repeat": "instance", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "refId": "A" + } + ], + "title": "$instance", + "type": "row" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "blackbox", + "prometheus" + ], + "templating": { + "list": [ + { + "allValue": ".+", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "definition": "label_values(probe_success, job)", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": "label_values(probe_success, job)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "KNkR71YVk" + }, + "definition": "label_values(probe_success{job=~\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(probe_success{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Blackbox Exporter (HTTP prober)", + "uid": "NEzutrbMk", + "version": 1, + "weekStart": "" +} diff --git a/pkgs/grafana-dashboards/default.nix b/pkgs/grafana-dashboards/default.nix index febd8de..423924d 100644 --- a/pkgs/grafana-dashboards/default.nix +++ b/pkgs/grafana-dashboards/default.nix @@ -124,6 +124,7 @@ in }).overrideAttrs (self: super: { src = ./prometheus.json; # sadly only imported dashboards work }); + grafana = (buildGrafanaDashboard { id = 3590; pname = "grafana"; @@ -131,4 +132,13 @@ in }).overrideAttrs (self: super: { src = ./grafana.json; # sadly only imported dashboards work }); + + blackbox = (buildGrafanaDashboard { + id = 13659; + pname = "blackbox"; + version = "1"; + hash = "sha256-nnBFWFDAqKUqTOYxOrkRPlVla4ioQZ6rqEqakdzUj1Q="; + }).overrideAttrs (self: super: { + src = ./blackbox.json; # sadly only imported dashboards work + }); })