{ pkgs, config, inputs, ... }: let mkK3sNode = name: isServer: ipv4Address: ipv6Address: macAddress: { config = { imports = [ inputs.sops.nixosModules.sops ../../users/root ]; microvm = { mem = 1024; balloonMem = 1024; vcpu = 2; shares = [ { source = "/nix/store"; mountPoint = "/nix/.ro-store"; tag = "ro-store"; proto = "virtiofs"; } { source = "/var/lib/microvms/${name}/etc"; mountPoint = "/etc"; proto = "virtiofs"; tag = "etc"; } ]; volumes = [ { image = "var.img"; mountPoint = "/var"; size = 25 * 1024; } ]; interfaces = [{ type = "tap"; id = "vm-${name}"; mac = macAddress; }]; hypervisor = "cloud-hypervisor"; }; services = { k3s = { enable = true; clusterInit = isServer; serverAddr = if !isServer then "https://10.128.0.2:6443" else ""; role = "server"; tokenFile = config.sops.secrets.k3s-token.path; extraFlags = "--tls-san=${builtins.elemAt (builtins.split "/" ipv4Address) 0}"; }; openssh = { enable = true; hostKeys = [{ type = "ed25519"; path = "/etc/ssh/ssh_host_ed25519_key"; }]; }; }; networking = { hostName = name; useNetworkd = true; firewall.allowedTCPPorts = [ 2379 # etcd 2380 # etcd 6443 # k3s supervisor / k8s API ]; }; systemd.network = { networks."10-vm-${name}" = { matchConfig.MACAddress = macAddress; addresses = [ { addressConfig.Address = ipv4Address; } { addressConfig.Address = ipv6Address; } ]; networkConfig = { Gateway = "10.128.0.1"; DNS = [ "10.0.0.206" ]; }; }; }; sops = { defaultSopsFile = ./secrets.yaml; secrets.k3s-token = { }; }; system.stateVersion = "23.11"; }; }; in { microvm.vms = { k3s-1 = mkK3sNode "k3s-1" true "10.128.0.2/24" "fd12:3456:789a::2/64" "0e:00:00:00:00:01"; k3s-2 = mkK3sNode "k3s-2" false "10.128.0.3/24" "fd12:3456:789a::3/64" "0e:00:00:00:00:02"; k3s-3 = mkK3sNode "k3s-3" false "10.128.0.4/24" "fd12:3456:789a::4/64" "0e:00:00:00:00:03"; }; systemd.network = { netdevs."10-microvm".netdevConfig = { Kind = "bridge"; Name = "microvm"; }; networks."10-microvm" = { matchConfig.Name = "microvm"; addresses = [ { addressConfig.Address = "10.128.0.1/24"; } { addressConfig.Address = "fd12:3456:789a::1/64"; } ]; ipv6Prefixes = [ { ipv6PrefixConfig.Prefix = "fd12:3456:789a::/64"; } ]; }; networks."11-microvm" = { matchConfig.Name = "vm-*"; networkConfig.Bridge = "microvm"; }; }; networking.nat = { enable = true; enableIPv6 = true; externalInterface = "enp4s0"; internalInterfaces = [ "microvm" ]; forwardPorts = [ { proto = "tcp"; sourcePort = 6443; destination = "10.128.0.2:6443"; } { proto = "tcp"; sourcePort = 6443; destination = "10.128.0.3:6443"; } { proto = "tcp"; sourcePort = 6443; destination = "10.128.0.4:6443"; } { proto = "tcp"; sourcePort = 6443; destination = "fd12:3456:789a::2:6443"; } { proto = "tcp"; sourcePort = 6443; destination = "fd12:3456:789a::3:6443"; } { proto = "tcp"; sourcePort = 6443; destination = "fd12:3456:789a::4:6443"; } ]; }; environment.systemPackages = with pkgs; [ kubectl kubernetes-helm ]; }