{ "annotations": { "list": [ { "$$hashKey": "object:7", "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "Loki metrics via 2.0", "editable": true, "fiscalYearStartMonth": 0, "gnetId": 13407, "graphTooltip": 0, "id": 31, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "gridPos": { "h": 2, "w": 24, "x": 0, "y": 0 }, "id": 21, "interval": "", "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "
\n \n Loki Global Metrics\n
", "mode": "html" }, "pluginVersion": "9.4.7", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "loki_build_info", "format": "table", "instant": false, "interval": "", "legendFormat": "", "refId": "A" } ], "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "decimals": 4, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 0, "y": 2 }, "id": 59, "interval": "", "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "/^version$/", "values": false }, "textMode": "auto" }, "pluginVersion": "9.4.7", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "loki_build_info", "format": "table", "instant": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Loki Version", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "decimals": 0, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 3, "y": 2 }, "id": 10, "interval": "", "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.4.7", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(log_messages_total)", "instant": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Message Total", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "decimals": 0, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 6, "y": 2 }, "id": 44, "interval": "", "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "value" }, "pluginVersion": "9.4.7", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "cortex_prometheus_notifications_alertmanagers_discovered", "format": "time_series", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Current Alerts", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "decimals": 0, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 9, "y": 2 }, "id": 48, "interval": "", "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "value" }, "pluginVersion": "9.4.7", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "cortex_prometheus_notifications_sent_total", "format": "time_series", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Alerts Event Total", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "decimals": 4, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 12, "y": 2 }, "id": 23, "interval": "", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.4.7", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(loki_store_series_total)", "instant": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Store Series Total", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "decimals": 4, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 15, "y": 2 }, "id": 41, "interval": "", "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.4.7", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(loki_ingester_chunk_stored_bytes_total)", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Store Chunks Total", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "decimals": 0, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "green", "value": 0 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 18, "y": 2 }, "id": 49, "interval": "", "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "value" }, "pluginVersion": "9.4.7", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(cortex_prometheus_rule_group_rules)", "format": "time_series", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Rules Total", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "decimals": 4, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 21, "y": 2 }, "id": 24, "interval": "", "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.4.7", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "loki_panic_total", "instant": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Panic ", "type": "stat" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 5, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 5 }, "hiddenSeries": false, "id": 8, "interval": "", "legend": { "avg": false, "current": true, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(irate(log_messages_total[1m])) by (level)", "interval": "", "legendFormat": "{{ level }}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Messages Input", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 5 }, "hiddenSeries": false, "id": 53, "interval": "", "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(cortex_s3_request_duration_seconds_bucket[5m])) by (le,operation))", "interval": "", "legendFormat": "{{operation}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "S3 Request Durations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 11 }, "hiddenSeries": false, "id": 11, "interval": "", "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(loki_request_duration_seconds_bucket[5m])) by (le,route))", "interval": "", "legendFormat": "{{ route }}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "API Request Durations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 11 }, "hiddenSeries": false, "id": 12, "interval": "", "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_latency_seconds_bucket[5m])) by (le,type))", "interval": "", "legendFormat": "{{ type }}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "LogQL Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 17 }, "hiddenSeries": false, "id": 13, "interval": "", "legend": { "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_bytes_processed_per_seconds_bucket[5m])) by (le,type))", "interval": "", "legendFormat": "{{ type }}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "LogQL Processed bytes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "Bps", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "id": 43, "panels": [], "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "refId": "A" } ], "title": "Ruler", "type": "row" }, { "aliasColors": {}, "breakPoint": "50%", "combine": { "label": "Others", "threshold": 0 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fontSize": "80%", "format": "short", "gridPos": { "h": 6, "w": 7, "x": 0, "y": 24 }, "id": 46, "interval": "", "legend": { "show": true, "values": true }, "legendType": "Right side", "links": [], "maxDataPoints": 3, "nullPointMode": "connected", "pieType": "pie", "pluginVersion": "7.3.1", "strokeWidth": 1, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "cortex_prometheus_rule_evaluations_total", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "Evaluations", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "cortex_prometheus_notifications_sent_total", "instant": true, "interval": "1m", "legendFormat": "Notifications", "refId": "B" } ], "title": "Rules Status ", "type": "grafana-piechart-panel", "valueName": "current" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 7, "y": 24 }, "hiddenSeries": false, "id": 28, "interval": "", "legend": { "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(rate(cortex_prometheus_notifications_latency_seconds[5m])) by (quantile)", "interval": "", "legendFormat": "TP {{quantile}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Notifications Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 9, "x": 15, "y": 24 }, "hiddenSeries": false, "id": 47, "interval": "", "legend": { "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(rate(cortex_prometheus_rule_evaluation_duration_seconds[5m])) by (quantile)", "interval": "", "legendFormat": "TP {{quantile}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Rule Evaluations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, "id": 34, "panels": [], "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "refId": "A" } ], "title": "Cache ", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 31 }, "hiddenSeries": false, "id": 45, "interval": "", "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(cortex_cache_request_duration_seconds_bucket[5m])) by (le,name,method))", "interval": "", "legendFormat": "{{method}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Cache Request Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 31 }, "hiddenSeries": false, "id": 29, "interval": "", "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(cortex_cache_value_size_bytes_bucket[5m])) by (le,name,method))", "interval": "", "legendFormat": "{{name}} / {{method}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Cache Value Size bytes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "decbytes", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 10, "x": 0, "y": 37 }, "hiddenSeries": false, "id": 27, "legend": { "avg": false, "current": true, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "rate(cortex_cache_hits[5m])", "interval": "", "legendFormat": "{{name}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Hits Keys", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1092", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:1093", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 7, "x": 10, "y": 37 }, "hiddenSeries": false, "id": 30, "legend": { "avg": false, "current": true, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "cortex_cache_background_queue_length", "interval": "", "legendFormat": "{{name}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Background Queue Length", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1092", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:1093", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 7, "x": 17, "y": 37 }, "hiddenSeries": false, "id": 26, "legend": { "avg": false, "current": true, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "cortex_cache_fetched_keys", "interval": "", "legendFormat": "{{name}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Fetched Keys", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1092", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:1093", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "id": 36, "panels": [], "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "refId": "A" } ], "title": "Index", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 44 }, "hiddenSeries": false, "id": 38, "interval": "", "legend": { "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(cortex_cassandra_request_duration_seconds_bucket[5m])) by (le,operation))", "interval": "", "legendFormat": "{{operation}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Cassandra Request Durations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 44 }, "hiddenSeries": false, "id": 39, "interval": "", "legend": { "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(cortex_chunk_store_chunks_per_query_bucket[5m])) by (le))", "interval": "", "legendFormat": "chunks", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Chunk Store Chunks", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "none", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 44 }, "hiddenSeries": false, "id": 54, "interval": "", "legend": { "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(cortex_table_manager_sync_duration_seconds_bucket[5m])) by (le))", "interval": "", "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Table-Manager Sync Durations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "decimals": 2, "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, "id": 19, "panels": [], "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "refId": "A" } ], "title": "Distributor", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 51 }, "hiddenSeries": false, "id": 20, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum by (tenant) (rate(loki_distributor_lines_received_total[5m]))", "interval": "", "legendFormat": "distributor / {{ tenant}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Received Lines / sec", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1092", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:1093", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 51 }, "hiddenSeries": false, "id": 15, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(rate(loki_distributor_bytes_received_total[5m])) by (tenant)", "interval": "", "legendFormat": "distributor / {{tenant}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Distributor Received bytes / sec", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:837", "format": "Bps", "logBase": 1, "show": true }, { "$$hashKey": "object:838", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 51 }, "hiddenSeries": false, "id": 17, "legend": { "avg": false, "current": true, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "rate(loki_distributor_ingester_appends_total[5m])", "interval": "", "legendFormat": "succeeded", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "loki_distributor_ingester_append_failures_total", "interval": "", "legendFormat": "failed", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "batch appends sent to ingesters / sec", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1092", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:1093", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "unit": "short" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 51 }, "hiddenSeries": false, "id": 22, "interval": "", "legend": { "avg": false, "current": true, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum by (status) (rate(loki_store_series_total[5m]))", "instant": false, "interval": "", "legendFormat": "{{status}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Store Series / sec", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:88", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:89", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 57 }, "id": 66, "panels": [], "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "refId": "A" } ], "title": "Ingster", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 58 }, "hiddenSeries": false, "id": 71, "interval": "", "legend": { "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_blocks_per_chunk_bucket[5m])) by (le))", "interval": "", "legendFormat": "blocks", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Blocks / Chunk", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "none", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 58 }, "hiddenSeries": false, "id": 68, "interval": "", "legend": { "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_size_bytes_bucket[5m])) by (le))", "interval": "", "legendFormat": "chunk size", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Chunk Size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "decbytes", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 58 }, "hiddenSeries": false, "id": 72, "interval": "", "legend": { "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.4.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_age_seconds_bucket[5m])) by (le))", "interval": "", "legendFormat": "ages", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Chunk Age", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 64 }, "hiddenSeries": false, "id": 70, "interval": "", "legend": { "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_compression_ratio_bucket[5m])) by (le))", "interval": "", "legendFormat": "ratio", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Chunk Compression Ratios", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "percent", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 64 }, "hiddenSeries": false, "id": 69, "interval": "", "legend": { "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_encode_time_seconds_bucket[5m])) by (le))", "interval": "", "legendFormat": "durations.", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Chunk Encode Time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 64 }, "hiddenSeries": false, "id": 73, "interval": "", "legend": { "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_entries_bucket[5m])) by (le))", "interval": "", "legendFormat": "lines", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Lines / Chunk", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:155", "format": "none", "logBase": 1, "show": true }, { "$$hashKey": "object:156", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 70 }, "hiddenSeries": false, "id": 60, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "rate(loki_ingester_chunks_created_total[5m])", "interval": "", "legendFormat": "create", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total[5m]))", "interval": "", "legendFormat": "flush / {{ reason }}", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum by (fake) (rate(loki_ingester_chunks_stored_total[5m]))", "interval": "", "legendFormat": "stored", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "", "interval": "", "legendFormat": "", "refId": "D" } ], "thresholds": [], "timeRegions": [], "title": "Chunk Status", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1092", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:1093", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 70 }, "hiddenSeries": false, "id": 61, "legend": { "avg": false, "current": true, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "loki_ingester_memory_chunks", "interval": "", "legendFormat": "chunks", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Chunks in Memory", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1092", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:1093", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 70 }, "hiddenSeries": false, "id": 62, "legend": { "avg": false, "current": true, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum by (tenan) (loki_ingester_memory_streams)", "interval": "", "legendFormat": "streams", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Chunks in Streams", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1092", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:1093", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 76 }, "hiddenSeries": false, "id": 67, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "rate(loki_ingester_streams_created_total[5m])", "interval": "", "legendFormat": "create", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "rate(loki_ingester_streams_removed_total[5m])", "interval": "", "legendFormat": "delete", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Streams Stats", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1092", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:1093", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "refresh": "", "revision": 1, "schemaVersion": 38, "style": "dark", "tags": [ "loki" ], "templating": { "list": [] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] }, "timezone": "", "title": "Loki2.0 Global Metrics", "uid": "MQHVDmtWk", "version": 1, "weekStart": "" }