nixos/modules/services/loki/default.nix

199 lines
5.5 KiB
Nix
Raw Normal View History

2023-04-09 23:07:29 +02:00
# log monitoring
2024-07-28 21:08:02 +02:00
{
config,
lib,
pkgs,
...
}:
2023-04-09 23:07:29 +02:00
let
cfg = config.my.services.loki;
in
{
options.my.services.loki = with lib; {
enable = mkEnableOption "loki log monitoring";
port = mkOption {
type = types.port;
default = 3100;
example = 3002;
description = "Internal port";
};
2023-05-21 19:52:45 +02:00
rules = mkOption {
2024-07-28 21:08:02 +02:00
type = types.attrsOf (
types.submodule {
2023-05-21 19:52:45 +02:00
options = {
condition = mkOption {
type = types.str;
description = ''
Loki alert expression.
'';
example = ''count_over_time({job=~"secure"} |="sshd[" |~": Failed|: Invalid|: Connection closed by authenticating user" | __error__="" [15m]) > 15'';
default = null;
};
description = mkOption {
type = types.str;
description = ''
Loki alert message.
'';
example = "Prometheus encountered value {{ $value }} with {{ $labels }}";
default = null;
};
labels = mkOption {
type = types.nullOr (types.attrsOf types.str);
description = ''
Additional alert labels.
'';
example = literalExpression ''
{ severity = "page" };
'';
default = { };
};
time = lib.mkOption {
type = lib.types.str;
description = ''
Time until the alert is fired.
'';
example = "5m";
default = "2m";
};
};
2024-07-28 21:08:02 +02:00
}
);
2023-05-21 19:52:45 +02:00
description = ''
Defines the loki rules.
'';
default = { };
};
2023-04-09 23:07:29 +02:00
};
2024-06-01 18:47:54 +02:00
config =
let
rulerConfig = {
groups = [
{
name = "alerting-rules";
2024-07-28 21:08:02 +02:00
rules = lib.mapAttrsToList (name: opts: {
alert = name;
inherit (opts) condition labels;
for = opts.time;
annotations.description = opts.description;
}) cfg.rules;
2024-06-01 18:47:54 +02:00
}
];
};
rulerFile = pkgs.writeText "ruler.yml" (builtins.toJSON rulerConfig);
in
lib.mkIf cfg.enable {
services = {
loki = {
enable = true;
configuration = {
server = {
http_listen_address = "localhost";
2024-06-01 18:47:54 +02:00
http_listen_port = cfg.port;
};
auth_enabled = false;
2023-04-09 23:07:29 +02:00
2024-06-01 18:47:54 +02:00
common = {
instance_addr = "localhost";
2024-06-01 18:47:54 +02:00
ring.kvstore.store = "inmemory";
replication_factor = 1;
2023-04-09 23:07:29 +02:00
2024-06-01 18:47:54 +02:00
path_prefix = config.services.loki.dataDir;
storage.filesystem = {
chunks_directory = "${config.services.loki.dataDir}/chunks";
rules_directory = "${config.services.loki.dataDir}/rules";
};
2023-05-21 19:52:45 +02:00
};
2024-06-01 18:47:54 +02:00
ruler = lib.mkIf config.my.services.alertmanager.enable {
storage = {
type = "local";
local.directory = "${config.services.loki.dataDir}/ruler";
};
2024-06-01 18:47:54 +02:00
rule_path = "${config.services.loki.dataDir}/rules";
alertmanager_url = "http://localhost:${toString config.my.services.alertmanager.port}";
2024-06-01 18:47:54 +02:00
enable_alertmanager_v2 = true;
};
schema_config = {
2024-07-28 21:08:02 +02:00
configs = [
{
from = "2020-11-08";
store = "tsdb";
object_store = "filesystem";
schema = "v13";
index = {
prefix = "index_";
period = "24h";
};
}
];
2024-06-01 18:47:54 +02:00
};
limits_config = {
max_query_lookback = "672h"; # 28 days
retention_period = "672h"; # 28 days
};
compactor = {
working_directory = "${config.services.loki.dataDir}/compactor";
retention_enabled = true;
delete_request_store = "filesystem";
};
};
2023-05-21 19:52:45 +02:00
};
2024-06-01 18:47:54 +02:00
grafana.provision = {
datasources.settings.datasources = [
{
name = "Loki";
type = "loki";
access = "proxy";
url = "http://localhost:${toString cfg.port}";
2024-06-01 18:47:54 +02:00
}
];
dashboards.settings.providers = [
{
name = "Loki";
options.path = pkgs.grafana-dashboards.loki;
disableDeletion = true;
}
];
};
2023-04-09 23:07:29 +02:00
2024-06-01 18:47:54 +02:00
prometheus = {
scrapeConfigs = [
{
job_name = "loki";
static_configs = [
{
targets = [ "localhost:${toString cfg.port}" ];
2024-06-01 18:47:54 +02:00
labels = {
instance = config.networking.hostName;
};
}
];
}
];
};
};
2023-05-21 19:52:45 +02:00
2024-06-01 18:47:54 +02:00
systemd.tmpfiles.rules = [
"d /var/lib/loki 0700 loki loki - -"
"d /var/lib/loki/ruler 0700 loki loki - -"
"d /var/lib/loki/rules 0700 loki loki - -"
"L /var/lib/loki/ruler/ruler.yml - - - - ${rulerFile}"
];
systemd.services.loki.reloadTriggers = [ rulerFile ];
my.services.loki.rules = {
loki_highLogRate = {
condition = ''sum by (host) (rate({unit="loki.service"}[1m])) > 60'';
description = "Loki has a high logging rate";
};
2023-05-21 19:52:45 +02:00
};
};
2023-04-09 23:07:29 +02:00
}