From 6e757d821e2baf245377302714e9aae7da0067d9 Mon Sep 17 00:00:00 2001 From: mehbark Date: Thu, 23 Oct 2025 01:36:03 -0400 Subject: [PATCH 01/22] vesuvius: netboot prospit --- hosts/common/k3s-ports.nix | 9 ++++++++ hosts/common/k3s.nix | 12 +++++++++++ hosts/prospit/configuration.nix | 15 +++++++++++++ hosts/vesuvius/configuration.nix | 1 + hosts/vesuvius/k3s.nix | 12 +++++++++++ hosts/vesuvius/netboot.nix | 37 +++++++++++++------------------- 6 files changed, 64 insertions(+), 22 deletions(-) create mode 100644 hosts/common/k3s-ports.nix create mode 100644 hosts/common/k3s.nix create mode 100644 hosts/prospit/configuration.nix create mode 100644 hosts/vesuvius/k3s.nix diff --git a/hosts/common/k3s-ports.nix b/hosts/common/k3s-ports.nix new file mode 100644 index 0000000..23353b5 --- /dev/null +++ b/hosts/common/k3s-ports.nix @@ -0,0 +1,9 @@ +{ + networking.firewall.allowedTCPPorts = [ + 6443 + ]; + + networking.firewall.allowedUDPPorts = [ + 8472 + ]; +} diff --git a/hosts/common/k3s.nix b/hosts/common/k3s.nix new file mode 100644 index 0000000..84bfce7 --- /dev/null +++ b/hosts/common/k3s.nix @@ -0,0 +1,12 @@ +{ ... }: { + imports = [ + ./k3s-ports.nix + ]; + + services.k3s = { + enable = true; + role = "agent"; + token = "garbage secret"; + serverAddr = "https://10.98.1.147:6443"; + }; +} diff --git a/hosts/prospit/configuration.nix b/hosts/prospit/configuration.nix new file mode 100644 index 0000000..53e1e0c --- /dev/null +++ b/hosts/prospit/configuration.nix @@ -0,0 +1,15 @@ +{ modulesPath, pkgs, ... }: { + imports = [ + ../common/k3s.nix + ../common/nix.nix + ../common/sshd.nix + ../common/users-local.nix + (modulesPath + "/installer/netboot/netboot-minimal.nix") + ]; + + environment.systemPackages = [ + pkgs.fastfetch + ]; + + system.stateVersion = "25.11"; +} diff --git a/hosts/vesuvius/configuration.nix b/hosts/vesuvius/configuration.nix index 482f77b..d060719 100644 --- a/hosts/vesuvius/configuration.nix +++ b/hosts/vesuvius/configuration.nix @@ -2,6 +2,7 @@ { imports = [ ./hardware-configuration.nix + ./k3s.nix ./nix.nix ./zfs.nix ./netboot.nix diff --git a/hosts/vesuvius/k3s.nix b/hosts/vesuvius/k3s.nix new file mode 100644 index 0000000..f28a2cc --- /dev/null +++ b/hosts/vesuvius/k3s.nix @@ -0,0 +1,12 @@ +{ ... }: { + imports = [ + ../common/k3s-ports.nix + ]; + + services.k3s = { + enable = true; + role = "server"; + token = "garbage secret"; + clusterInit = true; + }; +} diff --git a/hosts/vesuvius/netboot.nix b/hosts/vesuvius/netboot.nix index 9222a1c..c2918b2 100644 --- a/hosts/vesuvius/netboot.nix +++ b/hosts/vesuvius/netboot.nix @@ -1,38 +1,31 @@ -{ config, pkgs, ... }: +{ config, lib, pkgs, ... }: let dom_ip = "10.98.2.1"; dhcp_iface = "enp1s0f1"; client_range = "10.98.2.2,10.98.2.100"; - sub_image = pkgs.nixos { - imports = [ "${pkgs.path}/nixos/modules/installer/netboot/netboot-minimal.nix" ]; - system.stateVersion = "25.05"; - services.openssh = { - enable = true; - settings.PasswordAuthentication = true; - settings.KbdInteractiveAuthentication = false; - }; + sub_image = lib.nixosSystem { + system = "x86_64-linux"; - users.users.papatux = { - isNormalUser = true; - description = "papatux"; - extraGroups = [ "networkmanager" "wheel" ]; - hashedPassword = "$6$6GnvJWpo8oOWM1tb$GhuldW5iIdS6OuRyq5u1hSSu0VotQCLac7emA.Kui2hWLozR7EIO4Su6PCo5hTRG8iWnAOlGemQVyejIA9l4j/"; - openssh.authorizedKeys.keys = import ../../papatux-keys.nix; - }; + modules = [ + ../prospit/configuration.nix + ]; }; - + + prospit = sub_image.config.system.build; + ipxe_config = pkgs.writeText "boot.ipxe" '' #!ipxe - kernel http://${dom_ip}:8080/netboot-nixtest/kernel init=/init boot.shell_on_fail - initrd http://${dom_ip}:8080/netboot-nixtest/initrd + kernel http://${dom_ip}:8080/netboot-kernel/bzImage init=${prospit.toplevel}/init boot.shell_on_fail + initrd http://${dom_ip}:8080/netboot-initrd/initrd boot ''; webroot = pkgs.linkFarm "netboot" [ - { name = "netboot-nixtest"; path = sub_image.config.system.build.toplevel; } + { name = "netboot-kernel"; path = prospit.kernel; } + { name = "netboot-initrd"; path = prospit.netbootRamdisk; } { name = "boot.ipxe"; path = ipxe_config; } ]; @@ -61,7 +54,7 @@ in settings.dhcp-userclass = [ "set:ipxe,iPXE" ]; settings.dhcp-boot = [ "tag:!ipxe,ipxe.efi" - "http://${dom_ip}:8080/boot.ipxe" + "http://${dom_ip}:8080/boot.ipxe" ]; }; @@ -77,4 +70,4 @@ in allowedTCPPorts = [ 8080 ]; allowedUDPPorts = [ 67 69 ]; }; -} \ No newline at end of file +} From 3fcba7841fc897ef423097490c4852bd456a53b9 Mon Sep 17 00:00:00 2001 From: Ralakus Date: Thu, 6 Nov 2025 15:52:29 -0500 Subject: [PATCH 02/22] corrected netboot configuration to avoid interface leakage and corrected addresses for dhcp clients on cluster vlan --- hosts/vesuvius/netboot.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/hosts/vesuvius/netboot.nix b/hosts/vesuvius/netboot.nix index c2918b2..d2bb802 100644 --- a/hosts/vesuvius/netboot.nix +++ b/hosts/vesuvius/netboot.nix @@ -1,8 +1,10 @@ { config, lib, pkgs, ... }: let - dom_ip = "10.98.2.1"; + dom_ip = "10.98.3.2"; + vlan_router_ip = "10.98.3.1"; + dns_server_ip = "10.98.0.1"; dhcp_iface = "enp1s0f1"; - client_range = "10.98.2.2,10.98.2.100"; + client_range = "10.98.3.3,10.98.3.100"; sub_image = lib.nixosSystem { @@ -47,10 +49,14 @@ in services.dnsmasq = { enable = true; + settings.domain = "hephaestus.vtluug.org"; + settings.interface = "enp1s0f1"; + settings.bind-interfaces = true; + settings.server = [ "${dns_server_ip}" ]; settings.enable-tftp = true; settings.tftp-root = "${tftproot}"; settings.dhcp-range = "${client_range},12h"; - settings.dhcp-option = [ "option:router,${dom_ip}" ]; + settings.dhcp-option = [ "option:router,${vlan_router_ip}" ]; settings.dhcp-userclass = [ "set:ipxe,iPXE" ]; settings.dhcp-boot = [ "tag:!ipxe,ipxe.efi" From 10ebb8fb6a152b6ae916a07ab2f7af91df8de861 Mon Sep 17 00:00:00 2001 From: Ralakus Date: Thu, 6 Nov 2025 15:59:58 -0500 Subject: [PATCH 03/22] corrected dhcp interface --- hosts/vesuvius/netboot.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/vesuvius/netboot.nix b/hosts/vesuvius/netboot.nix index d2bb802..c7effc0 100644 --- a/hosts/vesuvius/netboot.nix +++ b/hosts/vesuvius/netboot.nix @@ -50,7 +50,7 @@ in services.dnsmasq = { enable = true; settings.domain = "hephaestus.vtluug.org"; - settings.interface = "enp1s0f1"; + settings.interface = "${dhcp_iface}"; settings.bind-interfaces = true; settings.server = [ "${dns_server_ip}" ]; settings.enable-tftp = true; From eb25e2262b3cff1599b7e3a6bb13e9b78c023399 Mon Sep 17 00:00:00 2001 From: mehbark Date: Mon, 10 Nov 2025 20:17:38 -0500 Subject: [PATCH 04/22] hosts/vesuvius: change DNS domain from hephaestus to bastille this is hugely important and a major blocker --- hosts/vesuvius/netboot.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/vesuvius/netboot.nix b/hosts/vesuvius/netboot.nix index c7effc0..e6ba985 100644 --- a/hosts/vesuvius/netboot.nix +++ b/hosts/vesuvius/netboot.nix @@ -49,7 +49,7 @@ in services.dnsmasq = { enable = true; - settings.domain = "hephaestus.vtluug.org"; + settings.domain = "bastille.vtluug.org"; settings.interface = "${dhcp_iface}"; settings.bind-interfaces = true; settings.server = [ "${dns_server_ip}" ]; From 56bf0e8964505919f8d189b2200e17817d7f8ada Mon Sep 17 00:00:00 2001 From: mehbark Date: Mon, 10 Nov 2025 20:37:57 -0500 Subject: [PATCH 05/22] hosts/common: refactor to remove k3s-ports.nix --- hosts/common/k3s-ports.nix | 9 --------- hosts/common/k3s.nix | 9 ++++++--- 2 files changed, 6 insertions(+), 12 deletions(-) delete mode 100644 hosts/common/k3s-ports.nix diff --git a/hosts/common/k3s-ports.nix b/hosts/common/k3s-ports.nix deleted file mode 100644 index 23353b5..0000000 --- a/hosts/common/k3s-ports.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ - networking.firewall.allowedTCPPorts = [ - 6443 - ]; - - networking.firewall.allowedUDPPorts = [ - 8472 - ]; -} diff --git a/hosts/common/k3s.nix b/hosts/common/k3s.nix index 84bfce7..a4f9a7c 100644 --- a/hosts/common/k3s.nix +++ b/hosts/common/k3s.nix @@ -1,11 +1,14 @@ { ... }: { - imports = [ - ./k3s-ports.nix + networking.firewall.allowedTCPPorts = [ + 6443 + ]; + + networking.firewall.allowedUDPPorts = [ + 8472 ]; services.k3s = { enable = true; - role = "agent"; token = "garbage secret"; serverAddr = "https://10.98.1.147:6443"; }; From 7128fff2df415aa36fdf6eda3079c0e0b64a9d82 Mon Sep 17 00:00:00 2001 From: mehbark Date: Mon, 10 Nov 2025 20:39:42 -0500 Subject: [PATCH 06/22] hosts/prospit: set k3s role --- hosts/prospit/configuration.nix | 2 +- hosts/prospit/k3s.nix | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 hosts/prospit/k3s.nix diff --git a/hosts/prospit/configuration.nix b/hosts/prospit/configuration.nix index 53e1e0c..fb9b108 100644 --- a/hosts/prospit/configuration.nix +++ b/hosts/prospit/configuration.nix @@ -1,6 +1,6 @@ { modulesPath, pkgs, ... }: { imports = [ - ../common/k3s.nix + ../k3s.nix ../common/nix.nix ../common/sshd.nix ../common/users-local.nix diff --git a/hosts/prospit/k3s.nix b/hosts/prospit/k3s.nix new file mode 100644 index 0000000..d9480e9 --- /dev/null +++ b/hosts/prospit/k3s.nix @@ -0,0 +1,9 @@ +{ ... }: { + imports = [ + ../common/k3s.nix + ]; + + services.k3s = { + role = "agent"; + }; +} From 4dc169a571a77e053f5f1d2abd999cd8e8040999 Mon Sep 17 00:00:00 2001 From: mehbark Date: Mon, 10 Nov 2025 20:40:01 -0500 Subject: [PATCH 07/22] hosts/vesuvius: become the k3s cluster init --- hosts/vesuvius/k3s.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hosts/vesuvius/k3s.nix b/hosts/vesuvius/k3s.nix index f28a2cc..25e8a73 100644 --- a/hosts/vesuvius/k3s.nix +++ b/hosts/vesuvius/k3s.nix @@ -1,12 +1,10 @@ { ... }: { imports = [ - ../common/k3s-ports.nix + ../common/k3s.nix ]; services.k3s = { - enable = true; role = "server"; - token = "garbage secret"; clusterInit = true; }; } From 194b9cb610cba72c6b13cfdc3deb0b735e3b7686 Mon Sep 17 00:00:00 2001 From: mehbark Date: Mon, 10 Nov 2025 20:51:07 -0500 Subject: [PATCH 08/22] hosts/common/k3s: refactor to take role and clusterInit args --- hosts/common/k3s.nix | 4 +++- hosts/prospit/configuration.nix | 2 +- hosts/prospit/k3s.nix | 9 --------- hosts/vesuvius/configuration.nix | 2 +- hosts/vesuvius/k3s.nix | 10 ---------- 5 files changed, 5 insertions(+), 22 deletions(-) delete mode 100644 hosts/prospit/k3s.nix delete mode 100644 hosts/vesuvius/k3s.nix diff --git a/hosts/common/k3s.nix b/hosts/common/k3s.nix index a4f9a7c..46327b6 100644 --- a/hosts/common/k3s.nix +++ b/hosts/common/k3s.nix @@ -1,4 +1,4 @@ -{ ... }: { +{ role ? "agent", clusterInit ? false }: { networking.firewall.allowedTCPPorts = [ 6443 ]; @@ -8,6 +8,8 @@ ]; services.k3s = { + inherit role clusterInit; + enable = true; token = "garbage secret"; serverAddr = "https://10.98.1.147:6443"; diff --git a/hosts/prospit/configuration.nix b/hosts/prospit/configuration.nix index fb9b108..69b3c1b 100644 --- a/hosts/prospit/configuration.nix +++ b/hosts/prospit/configuration.nix @@ -1,6 +1,6 @@ { modulesPath, pkgs, ... }: { imports = [ - ../k3s.nix + (import ../common/k3s.nix {}) ../common/nix.nix ../common/sshd.nix ../common/users-local.nix diff --git a/hosts/prospit/k3s.nix b/hosts/prospit/k3s.nix deleted file mode 100644 index d9480e9..0000000 --- a/hosts/prospit/k3s.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ ... }: { - imports = [ - ../common/k3s.nix - ]; - - services.k3s = { - role = "agent"; - }; -} diff --git a/hosts/vesuvius/configuration.nix b/hosts/vesuvius/configuration.nix index d060719..3e9e43a 100644 --- a/hosts/vesuvius/configuration.nix +++ b/hosts/vesuvius/configuration.nix @@ -2,7 +2,7 @@ { imports = [ ./hardware-configuration.nix - ./k3s.nix + (import ../common/k3s.nix { role = "server"; clusterInit = true; }) ./nix.nix ./zfs.nix ./netboot.nix diff --git a/hosts/vesuvius/k3s.nix b/hosts/vesuvius/k3s.nix deleted file mode 100644 index 25e8a73..0000000 --- a/hosts/vesuvius/k3s.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ ... }: { - imports = [ - ../common/k3s.nix - ]; - - services.k3s = { - role = "server"; - clusterInit = true; - }; -} From 3845c0b8077bc227de8247b4591fce8ca8c7ec58 Mon Sep 17 00:00:00 2001 From: mehbark Date: Mon, 10 Nov 2025 20:54:10 -0500 Subject: [PATCH 09/22] hosts/prospit: be adopted by bastille --- .../{prospit/configuration.nix => bastille/blade.nix} | 0 hosts/vesuvius/netboot.nix | 10 +++++----- 2 files changed, 5 insertions(+), 5 deletions(-) rename hosts/{prospit/configuration.nix => bastille/blade.nix} (100%) diff --git a/hosts/prospit/configuration.nix b/hosts/bastille/blade.nix similarity index 100% rename from hosts/prospit/configuration.nix rename to hosts/bastille/blade.nix diff --git a/hosts/vesuvius/netboot.nix b/hosts/vesuvius/netboot.nix index e6ba985..e0c090c 100644 --- a/hosts/vesuvius/netboot.nix +++ b/hosts/vesuvius/netboot.nix @@ -11,23 +11,23 @@ let system = "x86_64-linux"; modules = [ - ../prospit/configuration.nix + ../bastille/blade.nix ]; }; - prospit = sub_image.config.system.build; + blade = sub_image.config.system.build; ipxe_config = pkgs.writeText "boot.ipxe" '' #!ipxe - kernel http://${dom_ip}:8080/netboot-kernel/bzImage init=${prospit.toplevel}/init boot.shell_on_fail + kernel http://${dom_ip}:8080/netboot-kernel/bzImage init=${blade.toplevel}/init boot.shell_on_fail initrd http://${dom_ip}:8080/netboot-initrd/initrd boot ''; webroot = pkgs.linkFarm "netboot" [ - { name = "netboot-kernel"; path = prospit.kernel; } - { name = "netboot-initrd"; path = prospit.netbootRamdisk; } + { name = "netboot-kernel"; path = blade.kernel; } + { name = "netboot-initrd"; path = blade.netbootRamdisk; } { name = "boot.ipxe"; path = ipxe_config; } ]; From 3a8573ff327cfc34bddc740fadd886a64c1eb7a9 Mon Sep 17 00:00:00 2001 From: mehbark Date: Mon, 10 Nov 2025 21:08:47 -0500 Subject: [PATCH 10/22] hosts/bastille: fix root password warning --- hosts/bastille/blade.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hosts/bastille/blade.nix b/hosts/bastille/blade.nix index 69b3c1b..0154d94 100644 --- a/hosts/bastille/blade.nix +++ b/hosts/bastille/blade.nix @@ -1,4 +1,4 @@ -{ modulesPath, pkgs, ... }: { +{ modulesPath, pkgs, lib, ... }: { imports = [ (import ../common/k3s.nix {}) ../common/nix.nix @@ -7,6 +7,10 @@ (modulesPath + "/installer/netboot/netboot-minimal.nix") ]; + # when making the ISO, the initialHashedPassword is set to "" for some reason + # we already set a hashed password, so null this + users.users.root.initialHashedPassword = lib.mkForce null; + environment.systemPackages = [ pkgs.fastfetch ]; From 6041f784dfef9fc0834bf623ccd79f8967917914 Mon Sep 17 00:00:00 2001 From: mehbark Date: Mon, 10 Nov 2025 23:46:49 -0500 Subject: [PATCH 11/22] hosts/bastille: automatically set hostname based on mac address --- hosts/bastille/auto-hostname.nix | 39 ++++++++++++++++++++++++++++++++ hosts/bastille/blade.nix | 1 + 2 files changed, 40 insertions(+) create mode 100644 hosts/bastille/auto-hostname.nix diff --git a/hosts/bastille/auto-hostname.nix b/hosts/bastille/auto-hostname.nix new file mode 100644 index 0000000..7b5cd78 --- /dev/null +++ b/hosts/bastille/auto-hostname.nix @@ -0,0 +1,39 @@ +{ pkgs, lib, ... }: +let + # TODO: make this like a python script with a list of interfaces in order of preference + auto-hostname = pkgs.writeShellApplication { + name = "auto-hostname"; + + runtimeInputs = [ + pkgs.hostname + ]; + + text = '' + if [[ -e "/sys/class/net/eno2/address" ]]; then + mac_file="/sys/class/net/eno2/address" + else + mac_file=/sys/class/net/enp0s25/address + fi + + mac=$(cat $mac_file | tr -d '\r\n ' | tr ':' '-') + + hostname "blade-$mac" + ''; + }; +in { + networking.hostName = ""; + + systemd.services."auto-hostname" = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + unitConfig = { + Description = "Automatically set the hostname "; + }; + + serviceConfig = { + Type = "oneshot"; + ExecStart = "${lib.getExe auto-hostname}"; + }; + }; +} diff --git a/hosts/bastille/blade.nix b/hosts/bastille/blade.nix index 0154d94..208bfb9 100644 --- a/hosts/bastille/blade.nix +++ b/hosts/bastille/blade.nix @@ -1,5 +1,6 @@ { modulesPath, pkgs, lib, ... }: { imports = [ + ./auto-hostname.nix (import ../common/k3s.nix {}) ../common/nix.nix ../common/sshd.nix From 95545c0a1672361a1f2228f0bfba97f34d72ed2b Mon Sep 17 00:00:00 2001 From: mehbark Date: Tue, 11 Nov 2025 02:41:30 -0500 Subject: [PATCH 12/22] hosts/bastille: add some actual names to auto-hostname there's still some possible issues with the script (hence the paranoid `echo` after `hostname`) --- hosts/bastille/auto-hostname.nix | 22 ++++++++++++++++++--- hosts/bastille/blade-names.nix | 34 +++++++++++++++++--------------- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/hosts/bastille/auto-hostname.nix b/hosts/bastille/auto-hostname.nix index 7b5cd78..8ec374d 100644 --- a/hosts/bastille/auto-hostname.nix +++ b/hosts/bastille/auto-hostname.nix @@ -1,6 +1,9 @@ { pkgs, lib, ... }: let - # TODO: make this like a python script with a list of interfaces in order of preference + names = import ./blade-names.nix; + + bash-sets = lib.mapAttrsToList (mac: name: "names['${mac}']='${name}'") names; + auto-hostname = pkgs.writeShellApplication { name = "auto-hostname"; @@ -15,9 +18,22 @@ let mac_file=/sys/class/net/enp0s25/address fi - mac=$(cat $mac_file | tr -d '\r\n ' | tr ':' '-') + mac=$(cat $mac_file | tr -d '\r\n ') + + declare -A names + ${lib.concatLines bash-sets} + + if [[ -v names[$mac] ]]; then + name=''${names[$mac]} + else + name="node-(echo $mac | tr ':' '-')" + fi + + echo "mac: '$mac'" + echo "name: '$name'" - hostname "blade-$mac" + hostname "$name" + echo "hostname set to '$(hostname)'" ''; }; in { diff --git a/hosts/bastille/blade-names.nix b/hosts/bastille/blade-names.nix index f656f2d..083cad3 100644 --- a/hosts/bastille/blade-names.nix +++ b/hosts/bastille/blade-names.nix @@ -1,16 +1,18 @@ -# keep-sorted start -[ - "backbiter" - "damocles" - "durendal" - "eyelander" - "excalibur" - "gram" - "gryffindor" - "kusanagi" - "narsil" - "oathbringer" - "riptide" - "sting" -] -# keep-sorted end +{ + # TODO: prospit's a special case and won't remain here forever + "d8:9e:f3:3e:f9:41" = "prospit"; + + "40:f2:e9:c6:65:5f" = "backbiter"; + "40:f2:e9:c6:69:43" = "damocles"; + "40:f2:e9:c6:69:67" = "durendal"; + "40:f2:e9:c6:74:59" = "eyelander"; + "40:f2:e9:c6:75:f1" = "excalibur"; + "40:f2:e9:c6:76:21" = "gram"; + + "unassigned-0" = "gryffindor"; + "unassigned-1" = "kusanagi"; + "unassigned-2" = "narsil"; + "unassigned-3" = "oathbringer"; + "unassigned-4" = "riptide"; + "unassigned-5" = "sting"; +} From 5364e331fb4771809dabd015233af45c8f3f2f34 Mon Sep 17 00:00:00 2001 From: rsk Date: Wed, 26 Nov 2025 04:02:35 -0500 Subject: [PATCH 13/22] fix(common/k3s): use cluster routing IP for vesuvius --- hosts/common/k3s.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/common/k3s.nix b/hosts/common/k3s.nix index 46327b6..1ad0ded 100644 --- a/hosts/common/k3s.nix +++ b/hosts/common/k3s.nix @@ -12,6 +12,6 @@ enable = true; token = "garbage secret"; - serverAddr = "https://10.98.1.147:6443"; + serverAddr = "https://10.98.3.2:6443"; }; } From 4e48b2ba71c5a9eb0764bdf2d2e183574ae0dcf2 Mon Sep 17 00:00:00 2001 From: rsk Date: Sun, 11 Jan 2026 05:49:13 -0500 Subject: [PATCH 14/22] !!breaking!! vesuvius/zfs: bind mount dataset /forge/nix to /nix, keeping the store off the root ssd note that this is live on the machine, which *no longer has a native /nix*. this means that deploying any config which does not include the bind mount (e.g. master) will **break everything**. be duly warned. --- hosts/vesuvius/README.md | 10 ++++++++++ hosts/vesuvius/zfs.nix | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/hosts/vesuvius/README.md b/hosts/vesuvius/README.md index 66f9fe3..2ab93b2 100644 --- a/hosts/vesuvius/README.md +++ b/hosts/vesuvius/README.md @@ -10,3 +10,13 @@ Giant storage server + future LHCPISCSIPXEIDK thing maybe? ## Storage We currently have one (manually created) RAID-Z2 pool mounted at `/forge` with `8` drives of `12 Tb` each. We have capacity for `48`(!) drives, but still only paper (and tape) caddies. + +``` +# for the nix store +zfs create -o mountpoint=legacy \ + -o compression=zstd \ + -o xattr=sa \ + -o acltype=posixacl \ + -o atime=off \ + forge/nix +``` \ No newline at end of file diff --git a/hosts/vesuvius/zfs.nix b/hosts/vesuvius/zfs.nix index df075c4..dfc4afe 100644 --- a/hosts/vesuvius/zfs.nix +++ b/hosts/vesuvius/zfs.nix @@ -13,5 +13,13 @@ fileSystems."/forge" = { device = "forge"; fsType = "zfs"; + neededForBoot = true; + }; + + fileSystems."/nix" = { + device = "/forge/nix"; + fsType = "none"; + options = [ "bind" ]; + depends = [ "/forge" ]; }; } From ba8cc98d9d15fcd0f446d8701b298efd11cfcd04 Mon Sep 17 00:00:00 2001 From: mkl Date: Wed, 21 Jan 2026 21:28:54 -0500 Subject: [PATCH 15/22] hosts/vesuvius: opened port 53 for dns --- hosts/vesuvius/netboot.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/vesuvius/netboot.nix b/hosts/vesuvius/netboot.nix index e0c090c..60d4351 100644 --- a/hosts/vesuvius/netboot.nix +++ b/hosts/vesuvius/netboot.nix @@ -74,6 +74,6 @@ in networking.firewall = { allowedTCPPorts = [ 8080 ]; - allowedUDPPorts = [ 67 69 ]; + allowedUDPPorts = [ 53 67 69 ]; }; } From 77c1bbc0251b181653c25701a926d97c2af29faa Mon Sep 17 00:00:00 2001 From: rsk Date: Fri, 23 Jan 2026 03:01:19 -0500 Subject: [PATCH 16/22] flake: bump --- flake.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index 56947c9..4e2ccba 100644 --- a/flake.lock +++ b/flake.lock @@ -8,11 +8,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1754433428, - "narHash": "sha256-NA/FT2hVhKDftbHSwVnoRTFhes62+7dxZbxj5Gxvghs=", + "lastModified": 1762618334, + "narHash": "sha256-wyT7Pl6tMFbFrs8Lk/TlEs81N6L+VSybPfiIgzU8lbQ=", "owner": "ryantm", "repo": "agenix", - "rev": "9edb1787864c4f59ae5074ad498b6272b3ec308d", + "rev": "fcdea223397448d35d9b31f798479227e80183f6", "type": "github" }, "original": { @@ -48,11 +48,11 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1756770412, - "narHash": "sha256-+uWLQZccFHwqpGqr2Yt5VsW/PbeJVTn9Dk6SHWhNRPw=", + "lastModified": 1768135262, + "narHash": "sha256-PVvu7OqHBGWN16zSi6tEmPwwHQ4rLPU9Plvs8/1TUBY=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "4524271976b625a4a605beefd893f270620fd751", + "rev": "80daad04eddbbf5a4d883996a73f3f542fa437ac", "type": "github" }, "original": { @@ -100,11 +100,11 @@ }, "nixpkgs-lib": { "locked": { - "lastModified": 1754788789, - "narHash": "sha256-x2rJ+Ovzq0sCMpgfgGaaqgBSwY+LST+WbZ6TytnT9Rk=", + "lastModified": 1765674936, + "narHash": "sha256-k00uTP4JNfmejrCLJOwdObYC9jHRrr/5M/a/8L2EIdo=", "owner": "nix-community", "repo": "nixpkgs.lib", - "rev": "a73b9c743612e4244d865a2fdee11865283c04e6", + "rev": "2075416fcb47225d9b68ac469a5c4801a9c4dd85", "type": "github" }, "original": { @@ -115,11 +115,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1757068644, - "narHash": "sha256-NOrUtIhTkIIumj1E/Rsv1J37Yi3xGStISEo8tZm3KW4=", + "lastModified": 1769018530, + "narHash": "sha256-MJ27Cy2NtBEV5tsK+YraYr2g851f3Fl1LpNHDzDX15c=", "owner": "nixos", "repo": "nixpkgs", - "rev": "8eb28adfa3dc4de28e792e3bf49fcf9007ca8ac9", + "rev": "88d3861acdd3d2f0e361767018218e51810df8a1", "type": "github" }, "original": { From a3db521cdd625aaf352cf753be98af6c02a2039c Mon Sep 17 00:00:00 2001 From: rsk Date: Fri, 23 Jan 2026 03:02:08 -0500 Subject: [PATCH 17/22] feat(hosts/vesuvius): working freeIPA client --- hosts/vesuvius/configuration.nix | 1 + hosts/vesuvius/ipa.nix | 22 ++++++++++++++++++++-- secrets/keytabs/vesuvius.keytab.age | 5 +++++ secrets/secrets.nix | 2 +- 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 secrets/keytabs/vesuvius.keytab.age diff --git a/hosts/vesuvius/configuration.nix b/hosts/vesuvius/configuration.nix index 3e9e43a..32e9b26 100644 --- a/hosts/vesuvius/configuration.nix +++ b/hosts/vesuvius/configuration.nix @@ -5,6 +5,7 @@ (import ../common/k3s.nix { role = "server"; clusterInit = true; }) ./nix.nix ./zfs.nix + ./ipa.nix ./netboot.nix ]; diff --git a/hosts/vesuvius/ipa.nix b/hosts/vesuvius/ipa.nix index fc011e1..04a698a 100644 --- a/hosts/vesuvius/ipa.nix +++ b/hosts/vesuvius/ipa.nix @@ -1,10 +1,17 @@ # TODO: /etc/krb5.keytab missing, maybe agenix { config, pkgs, ... }: { - age.secrets."krb5.keytab".file = ../../secrets/krb5.keytab.age; - + age.secrets."krb5.keytab" = { + file = ../../secrets/keytabs/vesuvius.keytab.age; + path = "/etc/krb5.keytab"; + owner = "root"; + group = "root"; + mode = "0600"; + }; environment.variables.KRB5_KTNAME = config.age.secrets."krb5.keytab".path; + networking.domain = "vtluug.org"; + security.ipa = { enable = true; @@ -19,4 +26,15 @@ sha256 = "16wv6kfvnm0hcyzr0wjrgmymw3asm84m8r1wbfq09qvqrjycfc6s"; }; }; + security.sudo.extraRules = [ + { + groups = [ "sudoers" ]; + commands = [ + { + command = "ALL"; + options = [ "SETENV" ]; + } + ]; + } + ]; } diff --git a/secrets/keytabs/vesuvius.keytab.age b/secrets/keytabs/vesuvius.keytab.age new file mode 100644 index 0000000..b20d783 --- /dev/null +++ b/secrets/keytabs/vesuvius.keytab.age @@ -0,0 +1,5 @@ +age-encryption.org/v1 +-> ssh-ed25519 aQG9zQ AwGnHtis2RoMcZHMea4Jbhzm5yYq9uos+1h0WtIVDCM +IAFYorEgHmeGXXfakwa3Fk9UDC5XPJGSkx5PbPRumoo +--- x+eNJlai124x5jAWKQkq2hhDegsZ+GxPlsnoWrd4zv0 +D,|]=V,Aֳ#luy}(bi OFۓwrH_> (zr/)dJ1g3RӂFqx.܃3g*J(oUA dRa߂3@"L-S8_9*V~9yqr_AJЎDR8 4ߩu Ā??Eq \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 177078e..840d696 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -2,6 +2,6 @@ let vesuvius = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOHI7ziwxkEbJzvpaZulPFpDW7l0vbGJ+ifHcHJ2fHex"; zerocool = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+60yHIqES3Dr1Upp23QGwzvqELQEeH6e4lTKTV9iUY root@zerocool"; in { - "krb5.keytab.age".publicKeys = [ vesuvius ]; + "keytabs/vesuvius.keytab.age".publicKeys = [ vesuvius ]; "zerocool/wg.priv.age".publicKeys = [ zerocool ]; } From 790c86e812d3d5f311de0f44e0b96600c182fc9b Mon Sep 17 00:00:00 2001 From: rsk Date: Fri, 23 Jan 2026 03:16:07 -0500 Subject: [PATCH 18/22] refactor(hosts/vesuvius): use roles/common stuff --- hosts/vesuvius/configuration.nix | 37 ++++---------------------------- 1 file changed, 4 insertions(+), 33 deletions(-) diff --git a/hosts/vesuvius/configuration.nix b/hosts/vesuvius/configuration.nix index 32e9b26..f7aac74 100644 --- a/hosts/vesuvius/configuration.nix +++ b/hosts/vesuvius/configuration.nix @@ -7,6 +7,10 @@ ./zfs.nix ./ipa.nix ./netboot.nix + + ../common/tz-locale.nix + ../common/users-local.nix + ../common/sshd.nix ]; boot.loader.systemd-boot.enable = true; @@ -17,44 +21,11 @@ networking.networkmanager.enable = true; networking.networkmanager.unmanaged = [ "interface-name:enp1s0f1" ]; - time.timeZone = "America/New_York"; - - i18n.defaultLocale = "en_US.UTF-8"; - - i18n.extraLocaleSettings = { - LC_ADDRESS = "en_US.UTF-8"; - LC_IDENTIFICATION = "en_US.UTF-8"; - LC_MEASUREMENT = "en_US.UTF-8"; - LC_MONETARY = "en_US.UTF-8"; - LC_NAME = "en_US.UTF-8"; - LC_NUMERIC = "en_US.UTF-8"; - LC_PAPER = "en_US.UTF-8"; - LC_TELEPHONE = "en_US.UTF-8"; - LC_TIME = "en_US.UTF-8"; - }; - - users.users.papatux = { - isNormalUser = true; - description = "papatux"; - extraGroups = [ "networkmanager" "wheel" ]; - openssh.authorizedKeys.keys = import ../../papatux-keys.nix; - }; - - security.sudo.wheelNeedsPassword = false; - nixpkgs.config.allowUnfree = true; environment.systemPackages = with pkgs; [ neovim ]; - services.openssh.enable = true; - - # This value determines the NixOS release from which the default - # settings for stateful data, like file locations and database versions - # on your system were taken. It‘s perfectly fine and recommended to leave - # this value at the release version of the first install of this system. - # Before changing this value read the documentation for this option - # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html). system.stateVersion = "25.05"; # Did you read the comment? } From d2201598d1ce003b3c4f16d9374ebd6f8090274d Mon Sep 17 00:00:00 2001 From: rsk Date: Fri, 23 Jan 2026 03:31:47 -0500 Subject: [PATCH 19/22] feat(common/nfs): mount dirtycow stuff --- hosts/common/nfs.nix | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 hosts/common/nfs.nix diff --git a/hosts/common/nfs.nix b/hosts/common/nfs.nix new file mode 100644 index 0000000..98647c2 --- /dev/null +++ b/hosts/common/nfs.nix @@ -0,0 +1,20 @@ +{ config, pkgs, ... }: +let + mkNfs = {path, options ? [ "vers=4.0" "soft" "nodev" "nosuid" ]}: { + device = "${path}"; + fsType = "nfs"; + inherit options; + }; +in +{ + environment.systemPackages = [ pkgs.nfs-utils ]; + + fileSystems."/nfs/cistern/share" = mkNfs {path = "10.98.0.7:/cistern/nfs/share";}; + fileSystems."/nfs/cistern/files" = mkNfs {path = "10.98.0.7:/cistern/nfs/files";}; + fileSystems."/nfs/cistern/home" = mkNfs { + path = "10.98.0.7:/cistern/nfs/home"; + options = [ "vers=4.0" "soft" "nodev" "nosuid" ]; + }; + fileSystems."/nfs/cistern/libvirt" = mkNfs {path = "10.98.0.7:/cistern/nfs/libvirt";}; + fileSystems."/nfs/cistern/docker/data" = mkNfs {path = "10.98.0.7:/cistern/nfs/docker/data";}; +} \ No newline at end of file From a0c8787a8bc2bbc010e595d889156e14e1607932 Mon Sep 17 00:00:00 2001 From: rsk Date: Fri, 23 Jan 2026 03:32:15 -0500 Subject: [PATCH 20/22] feat(hosts/vesuvius): include dirtycow mounts --- hosts/vesuvius/configuration.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/hosts/vesuvius/configuration.nix b/hosts/vesuvius/configuration.nix index f7aac74..9684500 100644 --- a/hosts/vesuvius/configuration.nix +++ b/hosts/vesuvius/configuration.nix @@ -8,6 +8,7 @@ ./ipa.nix ./netboot.nix + ../common/nfs.nix ../common/tz-locale.nix ../common/users-local.nix ../common/sshd.nix From 946dbc4ba27766dee9b1e9931128715f1c1e1a6d Mon Sep 17 00:00:00 2001 From: mkl Date: Fri, 23 Jan 2026 14:49:54 -0500 Subject: [PATCH 21/22] hosts/vesuvius: added dhcp-host entries for netboot nodes --- hosts/vesuvius/netboot.nix | 44 ++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/hosts/vesuvius/netboot.nix b/hosts/vesuvius/netboot.nix index 60d4351..aa2e0b9 100644 --- a/hosts/vesuvius/netboot.nix +++ b/hosts/vesuvius/netboot.nix @@ -1,4 +1,9 @@ -{ config, lib, pkgs, ... }: +{ + config, + lib, + pkgs, + ... +}: let dom_ip = "10.98.3.2"; vlan_router_ip = "10.98.3.1"; @@ -6,6 +11,7 @@ let dhcp_iface = "enp1s0f1"; client_range = "10.98.3.3,10.98.3.100"; + netboot-hostnames = import ../bastille/blade-names.nix; sub_image = lib.nixosSystem { system = "x86_64-linux"; @@ -26,14 +32,23 @@ let ''; webroot = pkgs.linkFarm "netboot" [ - { name = "netboot-kernel"; path = blade.kernel; } - { name = "netboot-initrd"; path = blade.netbootRamdisk; } - { name = "boot.ipxe"; path = ipxe_config; } + { + name = "netboot-kernel"; + path = blade.kernel; + } + { + name = "netboot-initrd"; + path = blade.netbootRamdisk; + } + { + name = "boot.ipxe"; + path = ipxe_config; + } ]; # fyi this is cause tftpd in dnsmasq chroots and wouldn't follow external symlinks # like the ones in a linkfarm - tftproot = pkgs.runCommand "tftproot-real" {} '' + tftproot = pkgs.runCommand "tftproot-real" { } '' mkdir -p $out cp ${ipxe_config} $out/boot.ipxe cp ${pkgs.ipxe}/ipxe.efi $out/ipxe.efi @@ -62,18 +77,33 @@ in "tag:!ipxe,ipxe.efi" "http://${dom_ip}:8080/boot.ipxe" ]; + # Set hostnames via DHCP + settings.dhcp-host = builtins.map (host: "${host.fst},${host.snd}") ( + lib.lists.filter (host: !lib.strings.hasInfix "unassigned" host.fst) ( + lib.lists.zipLists (builtins.attrNames netboot-hostnames) (builtins.attrValues netboot-hostnames) + ) + ); }; services.nginx = { enable = true; virtualHosts."netboot" = { - listen = [{ port = 8080; addr = "0.0.0.0"; }]; + listen = [ + { + port = 8080; + addr = "0.0.0.0"; + } + ]; locations."/".root = "${webroot}"; }; }; networking.firewall = { allowedTCPPorts = [ 8080 ]; - allowedUDPPorts = [ 53 67 69 ]; + allowedUDPPorts = [ + 53 + 67 + 69 + ]; }; } From 59c983337d5b37530cd3338d15f9e3a770152767 Mon Sep 17 00:00:00 2001 From: mkl Date: Fri, 23 Jan 2026 14:50:51 -0500 Subject: [PATCH 22/22] hosts/bastille: disabled eno1 on lenovo nodes and removed auto-hostname script in favor of dhcp hostnames --- hosts/bastille/auto-hostname.nix | 55 ----------------------------- hosts/bastille/blade.nix | 5 ++- hosts/bastille/eno1-imm-disable.nix | 30 ++++++++++++++++ 3 files changed, 34 insertions(+), 56 deletions(-) delete mode 100644 hosts/bastille/auto-hostname.nix create mode 100644 hosts/bastille/eno1-imm-disable.nix diff --git a/hosts/bastille/auto-hostname.nix b/hosts/bastille/auto-hostname.nix deleted file mode 100644 index 8ec374d..0000000 --- a/hosts/bastille/auto-hostname.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ pkgs, lib, ... }: -let - names = import ./blade-names.nix; - - bash-sets = lib.mapAttrsToList (mac: name: "names['${mac}']='${name}'") names; - - auto-hostname = pkgs.writeShellApplication { - name = "auto-hostname"; - - runtimeInputs = [ - pkgs.hostname - ]; - - text = '' - if [[ -e "/sys/class/net/eno2/address" ]]; then - mac_file="/sys/class/net/eno2/address" - else - mac_file=/sys/class/net/enp0s25/address - fi - - mac=$(cat $mac_file | tr -d '\r\n ') - - declare -A names - ${lib.concatLines bash-sets} - - if [[ -v names[$mac] ]]; then - name=''${names[$mac]} - else - name="node-(echo $mac | tr ':' '-')" - fi - - echo "mac: '$mac'" - echo "name: '$name'" - - hostname "$name" - echo "hostname set to '$(hostname)'" - ''; - }; -in { - networking.hostName = ""; - - systemd.services."auto-hostname" = { - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - - unitConfig = { - Description = "Automatically set the hostname "; - }; - - serviceConfig = { - Type = "oneshot"; - ExecStart = "${lib.getExe auto-hostname}"; - }; - }; -} diff --git a/hosts/bastille/blade.nix b/hosts/bastille/blade.nix index 208bfb9..89bdcc8 100644 --- a/hosts/bastille/blade.nix +++ b/hosts/bastille/blade.nix @@ -1,6 +1,6 @@ { modulesPath, pkgs, lib, ... }: { imports = [ - ./auto-hostname.nix + ./eno1-imm-disable.nix (import ../common/k3s.nix {}) ../common/nix.nix ../common/sshd.nix @@ -8,6 +8,9 @@ (modulesPath + "/installer/netboot/netboot-minimal.nix") ]; + # Get hostname from DHCP request + networking.hostName = ""; + # when making the ISO, the initialHashedPassword is set to "" for some reason # we already set a hashed password, so null this users.users.root.initialHashedPassword = lib.mkForce null; diff --git a/hosts/bastille/eno1-imm-disable.nix b/hosts/bastille/eno1-imm-disable.nix new file mode 100644 index 0000000..87c8d6c --- /dev/null +++ b/hosts/bastille/eno1-imm-disable.nix @@ -0,0 +1,30 @@ +{ pkgs, lib, ... }: +let + eno1-imm-disable = pkgs.writeShellApplication { + name = "eno1-imm-disable"; + + runtimeInputs = [ + pkgs.iproute2 + ]; + + text = '' + if grep "Lenovo NeXtScale nx360 M5" /sys/devices/virtual/dmi/id/product_name; then + ip link set down eno1 + fi + ''; + }; +in { + systemd.services."eno1-imm-disable" = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + unitConfig = { + Description = "Disable eno1 on Lenovo NeXtScale nodes to avoid issues with using the imm interface"; + }; + + serviceConfig = { + Type = "oneshot"; + ExecStart = "${lib.getExe eno1-imm-disable}"; + }; + }; +}