saga: Add Loki and Promtail

machines/saga/configuration.nix

@@ -12,7 +12,9 @@
     ../../users/erwin
 
     ./grafana
+    ./loki
     ./prometheus
+    ./promtail
   ];
 
   eboskma = {

machines/saga/loki/default.nix

@@ -0,0 +1,73 @@
+let
+  dataDir = "/var/lib/loki";
+in
+{
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+
+      server.http_listen_port = 3100;
+
+      ingester = {
+        lifecycler = {
+          address = "0.0.0.0";
+          ring = {
+            kvstore.store = "inmemory";
+            replication_factor = 1;
+          };
+          final_sleep = "0s";
+        };
+        chunk_idle_period = "1h";
+        max_chunk_age = "1h";
+        chunk_target_size = 1024 * 1024; # 1 MiB
+        chunk_retain_period = "30s";
+        max_transfer_retries = 0;
+      };
+
+      schema_config = {
+        configs = [
+          {
+            from = "2024-01-01";
+            store = "tsdb";
+            object_store = "filesystem";
+            schema = "v13";
+            index = {
+              prefix = "_index";
+              period = "24h";
+            };
+          }
+        ];
+      };
+
+      storage_config = {
+        filesystem.directory = "${dataDir}/chunks";
+
+        tsdb_shipper = {
+          active_index_directory = "${dataDir}/tsdb-shipper-active";
+          cache_location = "${dataDir}/tsdb-shipper-cache";
+          cache_ttl = "24h";
+          shared_store = "filesystem";
+        };
+      };
+
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h"; # 1 week
+      };
+
+      chunk_store_config.max_look_back_period = "0s";
+
+      table_manager = {
+        retention_deletes_enabled = false;
+        retention_period = "0s";
+      };
+
+      compactor = {
+        working_directory = dataDir;
+        shared_store = "filesystem";
+        compactor_ring.kvstore.store = "inmemory";
+      };
+    };
+  };
+}

machines/saga/promtail/default.nix

@@ -0,0 +1,33 @@
+{
+  services.promtail = {
+    enable = true;
+    configuration = {
+      clients = [ { url = "http://127.0.0.1:3100/loki/api/v1/push"; } ];
+      positions = {
+        filename = "/tmp/positions.yaml";
+      };
+      scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            labels = {
+              host = "saga";
+              job = "systemd-journal";
+            };
+            max_age = "12h";
+          };
+          relabel_configs = [
+            {
+              source_labels = [ "__journal__systemd_unit" ];
+              target_label = "unit";
+            }
+          ];
+        }
+      ];
+      server = {
+        grpc_listen_port = 0;
+        http_listen_port = 28183;
+      };
+    };
+  };
+}