service/blackbox: init

This commit is contained in:
Felix Buehler 2023-06-25 21:25:55 +02:00
parent 1b5e41e7ad
commit d4fa5996ae
6 changed files with 1034 additions and 0 deletions

View file

@ -132,6 +132,9 @@ in
promtail = { promtail = {
enable = true; enable = true;
}; };
blackbox = {
enable = true;
};
# Webserver # Webserver
nginx = { nginx = {
enable = true; enable = true;

View file

@ -0,0 +1,92 @@
# monitor urls
{ config, lib, pkgs, ... }:
let
cfg = config.my.services.blackbox;
domain = config.networking.domain;
blackBoxConfig = {
modules = {
http_2xx = {
prober = "http";
http.preferred_ip_protocol = "ip4";
};
ssh_banner = {
prober = "tcp";
tcp.query_response = [
{
send = "SSH-2.0-blackbox-ssh-check";
}
{
expect = "^SSH-2.0-";
}
];
};
};
};
in
{
options.my.services.blackbox = with lib; {
enable = mkEnableOption "Blackbox prometheus exporter";
http_endpoints = mkOption {
type = types.listOf types.str;
default = [ ];
example = literalExpression ''
[
"https://domain.com"
"https://another-domain.com"
]
'';
description = ''
List of domains to test.
'';
};
};
config = lib.mkIf cfg.enable {
services.prometheus.exporters.blackbox = {
enable = true;
configFile = pkgs.writeText "blackbox-config.yml" (builtins.toJSON blackBoxConfig);
};
# relabels as in https://github.com/prometheus/blackbox_exporter#prometheus-configuration
services.prometheus = {
scrapeConfigs = [
{
job_name = "blackbox";
metrics_path = "/probe";
params.module = [ "http_2xx" ];
static_configs = [
{
targets = cfg.http_endpoints;
labels = {
instance = config.networking.hostName;
};
}
];
relabel_configs = [
{
source_labels = [ "__address__" ];
target_label = "__param_target";
}
{
source_labels = [ "__param_target" ];
target_label = "instance";
}
{
target_label = "__address__";
replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
}
];
}
];
};
services.grafana.provision.dashboards.settings.providers = [
{
name = "Blackbox";
options.path = pkgs.grafana-dashboards.blackbox;
disableDeletion = true;
}
];
};
}

View file

@ -5,6 +5,7 @@
./aria2 ./aria2
./backup ./backup
./bazarr ./bazarr
./blackbox
./blocky ./blocky
./dyndns ./dyndns
./freshrss ./freshrss

View file

@ -124,5 +124,6 @@
}); });
} }
); );
my.services.blackbox.http_endpoints = lib.mapAttrsToList (key: value: value.dashboard.link) config.webapps.apps;
}; };
} }

View file

@ -0,0 +1,927 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "Blackbox exporter HTTP prober dashboard",
"editable": true,
"fiscalYearStartMonth": 0,
"gnetId": 13659,
"graphTooltip": 0,
"id": 58,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"cellOptions": {
"type": "auto"
},
"filterable": true,
"inspect": false
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "SSL Cert Expiry (days)"
},
"properties": [
{
"id": "decimals",
"value": 0
},
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "rgba(0, 0, 0, 0)",
"value": null
},
{
"color": "red",
"value": 0
},
{
"color": "orange",
"value": 1
},
{
"color": "yellow",
"value": 7
},
{
"color": "green",
"value": 24
}
]
}
},
{
"id": "custom.cellOptions",
"value": {
"mode": "basic",
"type": "gauge"
}
},
{
"id": "min",
"value": 0
},
{
"id": "max",
"value": 365
},
{
"id": "custom.filterable",
"value": false
}
]
},
{
"matcher": {
"id": "byName",
"options": "Status"
},
"properties": [
{
"id": "mappings",
"value": [
{
"options": {
"0": {
"text": "DOWN"
},
"1": {
"text": "UP"
}
},
"type": "value"
}
]
},
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
{
"id": "custom.cellOptions",
"value": {
"mode": "gradient",
"type": "color-background"
}
},
{
"id": "custom.width",
"value": 76
}
]
},
{
"matcher": {
"id": "byName",
"options": "Code"
},
"properties": [
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "rgba(0, 0, 0, 0)",
"value": null
},
{
"color": "green",
"value": 200
},
{
"color": "yellow",
"value": 300
},
{
"color": "red",
"value": 500
}
]
}
},
{
"id": "custom.cellOptions",
"value": {
"mode": "gradient",
"type": "color-background"
}
},
{
"id": "mappings",
"value": [
{
"options": {
"0": {
"text": ""
}
},
"type": "value"
}
]
},
{
"id": "custom.width",
"value": 78
}
]
},
{
"matcher": {
"id": "byName",
"options": "SSL"
},
"properties": [
{
"id": "mappings",
"value": [
{
"options": {
"0": {
"text": "NO"
},
"1": {
"text": "OK"
}
},
"type": "value"
}
]
},
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "rgba(3, 3, 3, 0)",
"value": null
},
{
"color": "red",
"value": 0
},
{
"color": "green",
"value": 1
}
]
}
},
{
"id": "custom.cellOptions",
"value": {
"mode": "gradient",
"type": "color-background"
}
},
{
"id": "custom.width",
"value": 77
}
]
},
{
"matcher": {
"id": "byName",
"options": "Probe Duration (s)"
},
"properties": [
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.8
},
{
"color": "red",
"value": 2
}
]
}
},
{
"id": "custom.cellOptions",
"value": {
"mode": "basic",
"type": "gauge"
}
},
{
"id": "custom.filterable",
"value": false
},
{
"id": "decimals",
"value": 2
},
{
"id": "max",
"value": 3
}
]
},
{
"matcher": {
"id": "byName",
"options": "DNS Lookup Duration (s)"
},
"properties": [
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.1
},
{
"color": "red",
"value": 0.2
}
]
}
},
{
"id": "max",
"value": 0.3
},
{
"id": "custom.cellOptions",
"value": {
"mode": "basic",
"type": "gauge"
}
},
{
"id": "custom.filterable",
"value": false
},
{
"id": "decimals",
"value": 3
}
]
},
{
"matcher": {
"id": "byName",
"options": "Instance"
},
"properties": [
{
"id": "links",
"value": [
{
"targetBlank": true,
"title": "${__data.fields.Instance}",
"url": "${__data.fields.Instance}"
}
]
},
{
"id": "custom.width",
"value": 276
}
]
},
{
"matcher": {
"id": "byName",
"options": "TLS Version"
},
"properties": [
{
"id": "custom.width",
"value": 117
}
]
}
]
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": false,
"displayName": "SSL Cert Expiry (days)"
}
]
},
"pluginVersion": "9.5.3",
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"expr": "probe_success{job=~\"$job\", instance=~\"$instance\"}",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"expr": "probe_http_ssl{job=~\"$job\", instance=~\"$instance\"} > 0",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"expr": "(probe_ssl_earliest_cert_expiry{job=~\"$job\", instance=~\"$instance\"} - time()) / 3600 / 24",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"expr": "probe_http_status_code{job=~\"$job\", instance=~\"$instance\"} > 0",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "D"
},
{
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"expr": "avg_over_time(probe_duration_seconds{job=~\"$job\", instance=~\"$instance\"}[1m])",
"format": "table",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
"refId": "E"
},
{
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"expr": "probe_tls_version_info{job=~\"$job\", instance=~\"$instance\"}",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "F"
},
{
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"expr": "avg_over_time(probe_dns_lookup_time_seconds{job=~\"$job\", instance=~\"$instance\"}[1m])",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "G"
}
],
"title": "HTTP Probe Overview",
"transformations": [
{
"id": "seriesToColumns",
"options": {
"byField": "instance"
}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Time 1": true,
"Time 2": true,
"Time 3": true,
"Time 4": true,
"Time 5": true,
"Time 6": true,
"Time 7": true,
"Time 8": true,
"Value": false,
"Value #A": false,
"Value #B": false,
"Value #F": true,
"__name__": true,
"__name__ 1": true,
"__name__ 2": true,
"__name__ 3": true,
"__name__ 4": true,
"__name__ 5": true,
"__name__ 6": true,
"__name__ 7": true,
"job": true,
"job 1": true,
"job 2": true,
"job 3": true,
"job 4": true,
"job 5": true,
"job 6": true,
"job 7": true,
"job 8": true,
"phase": true,
"type": true,
"type 1": true,
"type 2": true,
"type 3": true,
"type 4": true,
"type 5": true,
"type 6": true,
"type 7": true,
"type 8": true,
"version": false
},
"indexByName": {
"Time 1": 9,
"Time 2": 13,
"Time 3": 17,
"Time 4": 20,
"Time 5": 24,
"Time 6": 28,
"Time 7": 32,
"Value #A": 1,
"Value #B": 3,
"Value #C": 5,
"Value #D": 2,
"Value #E": 6,
"Value #F": 8,
"Value #G": 7,
"__name__ 1": 10,
"__name__ 2": 14,
"__name__ 3": 21,
"__name__ 4": 25,
"__name__ 5": 29,
"instance": 0,
"job 1": 11,
"job 2": 15,
"job 3": 18,
"job 4": 22,
"job 5": 26,
"job 6": 30,
"type 1": 12,
"type 2": 16,
"type 3": 19,
"type 4": 23,
"type 5": 27,
"type 6": 31,
"version": 4
},
"renameByName": {
"Value": "Up",
"Value #A": "Status",
"Value #B": "SSL",
"Value #C": "SSL Cert Expiry (days)",
"Value #D": "Code",
"Value #E": "Probe Duration (s)",
"Value #F": "",
"Value #G": "DNS Lookup Duration (s)",
"Value #H": "Probe IP",
"instance": "Instance",
"type 6": "",
"version": "TLS Version"
}
}
}
],
"transparent": true,
"type": "table"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"description": "Returns how long the probe took to complete in seconds",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 9
},
"hiddenSeries": false,
"id": 4,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": true,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.5.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"expr": "sum(probe_http_duration_seconds{job=~\"$job\", instance=~\"$instance\"}) by (instance)",
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{ instance }}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "HTTP Probe Duration",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"logBase": 1,
"show": true
},
{
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": 0
}
},
{
"collapsed": true,
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 18
},
"id": 8,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"description": "Duration of http request by phase, summed over all redirects",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 10,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 27
},
"hiddenSeries": false,
"id": 6,
"legend": {
"alignAsTable": false,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 0,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.3.5",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"expr": "probe_http_duration_seconds{job=~\"$job\", instance=~\"$instance\"}",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{ phase }}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "HTTP Probe Phases Duration",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transformations": [],
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"logBase": 1,
"show": true
},
{
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
}
],
"repeat": "instance",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"refId": "A"
}
],
"title": "$instance",
"type": "row"
}
],
"refresh": "1m",
"schemaVersion": 38,
"style": "dark",
"tags": [
"blackbox",
"prometheus"
],
"templating": {
"list": [
{
"allValue": ".+",
"current": {
"selected": false,
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"definition": "label_values(probe_success, job)",
"hide": 0,
"includeAll": true,
"label": "Job",
"multi": false,
"name": "job",
"options": [],
"query": "label_values(probe_success, job)",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".+",
"current": {
"selected": false,
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "KNkR71YVk"
},
"definition": "label_values(probe_success{job=~\"$job\"}, instance)",
"hide": 0,
"includeAll": true,
"label": "Instance",
"multi": false,
"name": "instance",
"options": [],
"query": "label_values(probe_success{job=~\"$job\"}, instance)",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Blackbox Exporter (HTTP prober)",
"uid": "NEzutrbMk",
"version": 1,
"weekStart": ""
}

View file

@ -124,6 +124,7 @@ in
}).overrideAttrs (self: super: { }).overrideAttrs (self: super: {
src = ./prometheus.json; # sadly only imported dashboards work src = ./prometheus.json; # sadly only imported dashboards work
}); });
grafana = (buildGrafanaDashboard { grafana = (buildGrafanaDashboard {
id = 3590; id = 3590;
pname = "grafana"; pname = "grafana";
@ -131,4 +132,13 @@ in
}).overrideAttrs (self: super: { }).overrideAttrs (self: super: {
src = ./grafana.json; # sadly only imported dashboards work src = ./grafana.json; # sadly only imported dashboards work
}); });
blackbox = (buildGrafanaDashboard {
id = 13659;
pname = "blackbox";
version = "1";
hash = "sha256-nnBFWFDAqKUqTOYxOrkRPlVla4ioQZ6rqEqakdzUj1Q=";
}).overrideAttrs (self: super: {
src = ./blackbox.json; # sadly only imported dashboards work
});
}) })