loki: Create k3s cluster with microvm

.sops.yaml

@@ -8,6 +8,9 @@ keys:
   - &heimdall age1z94c897pvq4tx0xwsj6wr8emnlpmk6u0xks75rydga6r33dlapjqyqqacc
   - &minio age1p5hu2l0ys8z2j9rhf0xp5et2wd4222utyn3tk562ksrxmckye9dqu25f49
   - &proxy age1dg4euuwvqyyuwpjm08psvehgxr5p6q76ht8k4je6z2xc2pv55vksw9ap7m
+  - &k3s-1 age1gsjy4em8u668tnx77jr7kk345m4hzmmt3seclzvsd25ldgwd45pq6zu7cv
+  - &k3s-2 age1ghda0mj5wc2vpksjuvaf3t0xklpcgnykvepzu9k5csf482ngpans9h05pp
+  - &k3s-3 age1mpyg2qcrehfcpksygk9hduz79l93gy2crpwn0vu70mtcmzapeyvqrrjw5r
 creation_rules:
   - path_regex: machines/loki/[^/]+\.yaml$
     key_groups:
@@ -15,6 +18,9 @@ creation_rules:
           - *erwin
           - *erwin_horus
           - *loki
+          - *k3s-1
+          - *k3s-2
+          - *k3s-3
   - path_regex: machines/drone/[^/]+\.yaml$
     key_groups:
       - age:

machines/loki/configuration.nix

@@ -1,4 +1,4 @@
-{ nixos-hardware, ... }:
+{ nixos-hardware, microvm, ... }:
 { pkgs, config, ... }:
 {
   imports = [
@@ -7,6 +7,10 @@
     nixos-hardware.nixosModules.common-gpu-amd
     nixos-hardware.nixosModules.common-pc-ssd
 
+    microvm.nixosModules.host
+
+    ./vm.nix
+
     ../../users/erwin
     ../../users/root
     ../../users/builder
@@ -20,7 +24,6 @@
       };
       builder.enable = true;
     };
-    # backscrub.enable = true;
     base = {
       plymouth.enable = true;
       work = false;

machines/loki/vm.nix

@@ -0,0 +1,156 @@
+{ pkgs, config, inputs, ... }:
+let
+  mkK3sNode = name: isServer: ipv4Address: ipv6Address: macAddress: {
+    config = {
+      imports = [ inputs.sops.nixosModules.sops ];
+
+      microvm = {
+        mem = 1024;
+        balloonMem = 1024;
+        vcpu = 2;
+
+        shares = [
+          {
+            source = "/nix/store";
+            mountPoint = "/nix/.ro-store";
+            tag = "ro-store";
+            proto = "virtiofs";
+          }
+
+          {
+            source = "/var/lib/microvms/${name}/etc";
+            mountPoint = "/etc";
+            proto = "virtiofs";
+            tag = "etc";
+          }
+        ];
+
+        volumes = [
+          {
+            image = "var.img";
+            mountPoint = "/var";
+            size = 5 * 1024;
+          }
+        ];
+
+        interfaces = [{
+          type = "tap";
+          id = "vm-${name}";
+          mac = macAddress;
+        }];
+        hypervisor = "cloud-hypervisor";
+      };
+
+      users.users.root = {
+        initialHashedPassword = "";
+        openssh.authorizedKeys.keyFiles = [
+          (pkgs.fetchurl {
+            url = "https://github.com/eboskma.keys";
+            sha256 = "uwK4FSLSHiwCJU9U7RBFHIoCmr7uUQLM0JM1u0bi4xo=";
+          })
+        ];
+      };
+
+      services = {
+        k3s = {
+          enable = true;
+          package = pkgs.k3s.override {
+            buildGoModule = pkgs.buildGo120Module;
+          };
+          clusterInit = isServer;
+          serverAddr = if !isServer then "https://10.128.0.2:6443" else "";
+          role = "server";
+          tokenFile = config.sops.secrets.k3s-token.path;
+          extraFlags = "--tls-san=${builtins.elemAt (builtins.split "/" ipv4Address) 0}";
+        };
+
+        openssh = {
+          enable = true;
+          hostKeys = [{
+            type = "ed25519";
+            path = "/etc/ssh/ssh_host_ed25519_key";
+          }];
+        };
+      };
+
+      networking = {
+        hostName = name;
+        useNetworkd = true;
+        firewall.allowedTCPPorts = [
+          2379 # etcd
+          2380 # etcd
+          6443 # k3s supervisor / k8s API
+        ];
+      };
+
+      systemd.network = {
+        networks."10-vm-${name}" = {
+          matchConfig.MACAddress = macAddress;
+          addresses = [
+            { addressConfig.Address = ipv4Address; }
+            { addressConfig.Address = ipv6Address; }
+          ];
+          networkConfig = {
+            Gateway = "10.128.0.1";
+            DNS = [ "10.0.0.206" ];
+          };
+        };
+      };
+
+      sops = {
+        defaultSopsFile = ./secrets.yaml;
+        secrets.k3s-token = { };
+      };
+
+      system.stateVersion = "23.11";
+    };
+  };
+in
+{
+  microvm.vms = {
+    k3s-1 = mkK3sNode "k3s-1" true "10.128.0.2/24" "fd12:3456:789a::2/64" "0e:00:00:00:00:01";
+    k3s-2 = mkK3sNode "k3s-2" false "10.128.0.3/24" "fd12:3456:789a::3/64" "0e:00:00:00:00:02";
+    k3s-3 = mkK3sNode "k3s-3" false "10.128.0.4/24" "fd12:3456:789a::4/64" "0e:00:00:00:00:03";
+  };
+
+  systemd.network = {
+    netdevs."10-microvm".netdevConfig = {
+      Kind = "bridge";
+      Name = "microvm";
+    };
+
+    networks."10-microvm" = {
+      matchConfig.Name = "microvm";
+      addresses = [
+        { addressConfig.Address = "10.128.0.1/24"; }
+        { addressConfig.Address = "fd12:3456:789a::1/64"; }
+      ];
+      ipv6Prefixes = [
+        { ipv6PrefixConfig.Prefix = "fd12:3456:789a::/64"; }
+      ];
+    };
+
+    networks."11-microvm" = {
+      matchConfig.Name = "vm-*";
+      networkConfig.Bridge = "microvm";
+    };
+  };
+
+  networking.nat = {
+    enable = true;
+    enableIPv6 = true;
+    externalInterface = "enp4s0";
+    internalInterfaces = [ "microvm" ];
+
+    forwardPorts = [
+      { proto = "tcp"; sourcePort = 6443; destination = "10.128.0.2:6443"; }
+      { proto = "tcp"; sourcePort = 6443; destination = "10.128.0.3:6443"; }
+      { proto = "tcp"; sourcePort = 6443; destination = "10.128.0.4:6443"; }
+      { proto = "tcp"; sourcePort = 6443; destination = "fd12:3456:789a::2:6443"; }
+      { proto = "tcp"; sourcePort = 6443; destination = "fd12:3456:789a::3:6443"; }
+      { proto = "tcp"; sourcePort = 6443; destination = "fd12:3456:789a::4:6443"; }
+    ];
+  };
+
+  environment.systemPackages = with pkgs; [ kubectl kubernetes-helm ];
+}