diff --git a/.sops.yaml b/.sops.yaml index 6b2596f..93efebf 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -8,6 +8,9 @@ keys: - &heimdall age1z94c897pvq4tx0xwsj6wr8emnlpmk6u0xks75rydga6r33dlapjqyqqacc - &minio age1p5hu2l0ys8z2j9rhf0xp5et2wd4222utyn3tk562ksrxmckye9dqu25f49 - &proxy age1dg4euuwvqyyuwpjm08psvehgxr5p6q76ht8k4je6z2xc2pv55vksw9ap7m + - &k3s-1 age1gsjy4em8u668tnx77jr7kk345m4hzmmt3seclzvsd25ldgwd45pq6zu7cv + - &k3s-2 age1ghda0mj5wc2vpksjuvaf3t0xklpcgnykvepzu9k5csf482ngpans9h05pp + - &k3s-3 age1mpyg2qcrehfcpksygk9hduz79l93gy2crpwn0vu70mtcmzapeyvqrrjw5r creation_rules: - path_regex: machines/loki/[^/]+\.yaml$ key_groups: @@ -15,6 +18,9 @@ creation_rules: - *erwin - *erwin_horus - *loki + - *k3s-1 + - *k3s-2 + - *k3s-3 - path_regex: machines/drone/[^/]+\.yaml$ key_groups: - age: diff --git a/machines/loki/configuration.nix b/machines/loki/configuration.nix index fe56e17..7110fae 100644 --- a/machines/loki/configuration.nix +++ b/machines/loki/configuration.nix @@ -1,4 +1,4 @@ -{ nixos-hardware, ... }: +{ nixos-hardware, microvm, ... }: { pkgs, config, ... }: { imports = [ @@ -7,6 +7,10 @@ nixos-hardware.nixosModules.common-gpu-amd nixos-hardware.nixosModules.common-pc-ssd + microvm.nixosModules.host + + ./vm.nix + ../../users/erwin ../../users/root ../../users/builder @@ -20,7 +24,6 @@ }; builder.enable = true; }; - # backscrub.enable = true; base = { plymouth.enable = true; work = false; diff --git a/machines/loki/vm.nix b/machines/loki/vm.nix new file mode 100644 index 0000000..b68f83d --- /dev/null +++ b/machines/loki/vm.nix @@ -0,0 +1,156 @@ +{ pkgs, config, inputs, ... }: +let + mkK3sNode = name: isServer: ipv4Address: ipv6Address: macAddress: { + config = { + imports = [ inputs.sops.nixosModules.sops ]; + + microvm = { + mem = 1024; + balloonMem = 1024; + vcpu = 2; + + shares = [ + { + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + tag = "ro-store"; + proto = "virtiofs"; + } + + { + source = "/var/lib/microvms/${name}/etc"; + mountPoint = "/etc"; + proto = "virtiofs"; + tag = "etc"; + } + ]; + + volumes = [ + { + image = "var.img"; + mountPoint = "/var"; + size = 5 * 1024; + } + ]; + + interfaces = [{ + type = "tap"; + id = "vm-${name}"; + mac = macAddress; + }]; + hypervisor = "cloud-hypervisor"; + }; + + users.users.root = { + initialHashedPassword = ""; + openssh.authorizedKeys.keyFiles = [ + (pkgs.fetchurl { + url = "https://github.com/eboskma.keys"; + sha256 = "uwK4FSLSHiwCJU9U7RBFHIoCmr7uUQLM0JM1u0bi4xo="; + }) + ]; + }; + + services = { + k3s = { + enable = true; + package = pkgs.k3s.override { + buildGoModule = pkgs.buildGo120Module; + }; + clusterInit = isServer; + serverAddr = if !isServer then "https://10.128.0.2:6443" else ""; + role = "server"; + tokenFile = config.sops.secrets.k3s-token.path; + extraFlags = "--tls-san=${builtins.elemAt (builtins.split "/" ipv4Address) 0}"; + }; + + openssh = { + enable = true; + hostKeys = [{ + type = "ed25519"; + path = "/etc/ssh/ssh_host_ed25519_key"; + }]; + }; + }; + + networking = { + hostName = name; + useNetworkd = true; + firewall.allowedTCPPorts = [ + 2379 # etcd + 2380 # etcd + 6443 # k3s supervisor / k8s API + ]; + }; + + systemd.network = { + networks."10-vm-${name}" = { + matchConfig.MACAddress = macAddress; + addresses = [ + { addressConfig.Address = ipv4Address; } + { addressConfig.Address = ipv6Address; } + ]; + networkConfig = { + Gateway = "10.128.0.1"; + DNS = [ "10.0.0.206" ]; + }; + }; + }; + + sops = { + defaultSopsFile = ./secrets.yaml; + secrets.k3s-token = { }; + }; + + system.stateVersion = "23.11"; + }; + }; +in +{ + microvm.vms = { + k3s-1 = mkK3sNode "k3s-1" true "10.128.0.2/24" "fd12:3456:789a::2/64" "0e:00:00:00:00:01"; + k3s-2 = mkK3sNode "k3s-2" false "10.128.0.3/24" "fd12:3456:789a::3/64" "0e:00:00:00:00:02"; + k3s-3 = mkK3sNode "k3s-3" false "10.128.0.4/24" "fd12:3456:789a::4/64" "0e:00:00:00:00:03"; + }; + + systemd.network = { + netdevs."10-microvm".netdevConfig = { + Kind = "bridge"; + Name = "microvm"; + }; + + networks."10-microvm" = { + matchConfig.Name = "microvm"; + addresses = [ + { addressConfig.Address = "10.128.0.1/24"; } + { addressConfig.Address = "fd12:3456:789a::1/64"; } + ]; + ipv6Prefixes = [ + { ipv6PrefixConfig.Prefix = "fd12:3456:789a::/64"; } + ]; + }; + + networks."11-microvm" = { + matchConfig.Name = "vm-*"; + networkConfig.Bridge = "microvm"; + }; + }; + + networking.nat = { + enable = true; + enableIPv6 = true; + externalInterface = "enp4s0"; + internalInterfaces = [ "microvm" ]; + + forwardPorts = [ + { proto = "tcp"; sourcePort = 6443; destination = "10.128.0.2:6443"; } + { proto = "tcp"; sourcePort = 6443; destination = "10.128.0.3:6443"; } + { proto = "tcp"; sourcePort = 6443; destination = "10.128.0.4:6443"; } + { proto = "tcp"; sourcePort = 6443; destination = "fd12:3456:789a::2:6443"; } + { proto = "tcp"; sourcePort = 6443; destination = "fd12:3456:789a::3:6443"; } + { proto = "tcp"; sourcePort = 6443; destination = "fd12:3456:789a::4:6443"; } + ]; + }; + + environment.systemPackages = with pkgs; [ kubectl kubernetes-helm ]; +}