nixos-config/machines/loki/vm.nix

{
  pkgs,
  config,
  inputs,
  ...
}:
let
  mkK3sNode = name: isServer: ipv4Address: ipv6Address: macAddress: {
    config = {
      imports = [
        inputs.sops.nixosModules.sops
        ../../users/root
      ];

      microvm = {
        mem = 1024;
        balloonMem = 1024;
        vcpu = 2;

        shares = [
          {
            source = "/nix/store";
            mountPoint = "/nix/.ro-store";
            tag = "ro-store";
            proto = "virtiofs";
          }

          {
            source = "/var/lib/microvms/${name}/etc";
            mountPoint = "/etc";
            proto = "virtiofs";
            tag = "etc";
          }
        ];

        volumes = [
          {
            image = "var.img";
            mountPoint = "/var";
            size = 25 * 1024;
          }
        ];

        interfaces = [
          {
            type = "tap";
            id = "vm-${name}";
            mac = macAddress;
          }
        ];
        hypervisor = "cloud-hypervisor";
      };

      services = {
        k3s = {
          enable = true;
          clusterInit = isServer;
          serverAddr = if !isServer then "https://10.128.0.2:6443" else "";
          role = "server";
          tokenFile = config.sops.secrets.k3s-token.path;
          extraFlags = "--tls-san=${builtins.elemAt (builtins.split "/" ipv4Address) 0}";
        };

        openssh = {
          enable = true;
          hostKeys = [
            {
              type = "ed25519";
              path = "/etc/ssh/ssh_host_ed25519_key";
            }
          ];
        };
      };

      networking = {
        hostName = name;
        useNetworkd = true;
        firewall.allowedTCPPorts = [
          2379 # etcd
          2380 # etcd
          6443 # k3s supervisor / k8s API
        ];
      };

      systemd.network = {
        networks."10-vm-${name}" = {
          matchConfig.MACAddress = macAddress;
          addresses = [
            { addressConfig.Address = ipv4Address; }
            { addressConfig.Address = ipv6Address; }
          ];
          networkConfig = {
            Gateway = "10.128.0.1";
            DNS = [ "10.0.0.206" ];
          };
        };
      };

      sops = {
        defaultSopsFile = ./secrets.yaml;
        secrets.k3s-token = { };
      };

      system.stateVersion = "23.11";
    };
  };
in
{
  microvm.vms = {
    k3s-1 = mkK3sNode "k3s-1" true "10.128.0.2/24" "fd12:3456:789a::2/64" "0e:00:00:00:00:01";
    k3s-2 = mkK3sNode "k3s-2" false "10.128.0.3/24" "fd12:3456:789a::3/64" "0e:00:00:00:00:02";
    k3s-3 = mkK3sNode "k3s-3" false "10.128.0.4/24" "fd12:3456:789a::4/64" "0e:00:00:00:00:03";
  };

  systemd.network = {
    netdevs."10-microvm".netdevConfig = {
      Kind = "bridge";
      Name = "microvm";
    };

    networks."10-microvm" = {
      matchConfig.Name = "microvm";
      addresses = [
        { addressConfig.Address = "10.128.0.1/24"; }
        { addressConfig.Address = "fd12:3456:789a::1/64"; }
      ];
      ipv6Prefixes = [ { ipv6PrefixConfig.Prefix = "fd12:3456:789a::/64"; } ];
    };

    networks."11-microvm" = {
      matchConfig.Name = "vm-*";
      networkConfig.Bridge = "microvm";
    };
  };

  networking.nat = {
    enable = true;
    enableIPv6 = true;
    externalInterface = "enp4s0";
    internalInterfaces = [ "microvm" ];

    forwardPorts = [
      {
        proto = "tcp";
        sourcePort = 6443;
        destination = "10.128.0.2:6443";
      }
      {
        proto = "tcp";
        sourcePort = 6443;
        destination = "10.128.0.3:6443";
      }
      {
        proto = "tcp";
        sourcePort = 6443;
        destination = "10.128.0.4:6443";
      }
      {
        proto = "tcp";
        sourcePort = 6443;
        destination = "fd12:3456:789a::2:6443";
      }
      {
        proto = "tcp";
        sourcePort = 6443;
        destination = "fd12:3456:789a::3:6443";
      }
      {
        proto = "tcp";
        sourcePort = 6443;
        destination = "fd12:3456:789a::4:6443";
      }
    ];
  };

  environment.systemPackages = with pkgs; [
    kubectl
    kubernetes-helm
  ];
}