diff --git a/flake.lock b/flake.lock index 56947c9..4e2ccba 100644 --- a/flake.lock +++ b/flake.lock @@ -8,11 +8,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1754433428, - "narHash": "sha256-NA/FT2hVhKDftbHSwVnoRTFhes62+7dxZbxj5Gxvghs=", + "lastModified": 1762618334, + "narHash": "sha256-wyT7Pl6tMFbFrs8Lk/TlEs81N6L+VSybPfiIgzU8lbQ=", "owner": "ryantm", "repo": "agenix", - "rev": "9edb1787864c4f59ae5074ad498b6272b3ec308d", + "rev": "fcdea223397448d35d9b31f798479227e80183f6", "type": "github" }, "original": { @@ -48,11 +48,11 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1756770412, - "narHash": "sha256-+uWLQZccFHwqpGqr2Yt5VsW/PbeJVTn9Dk6SHWhNRPw=", + "lastModified": 1768135262, + "narHash": "sha256-PVvu7OqHBGWN16zSi6tEmPwwHQ4rLPU9Plvs8/1TUBY=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "4524271976b625a4a605beefd893f270620fd751", + "rev": "80daad04eddbbf5a4d883996a73f3f542fa437ac", "type": "github" }, "original": { @@ -100,11 +100,11 @@ }, "nixpkgs-lib": { "locked": { - "lastModified": 1754788789, - "narHash": "sha256-x2rJ+Ovzq0sCMpgfgGaaqgBSwY+LST+WbZ6TytnT9Rk=", + "lastModified": 1765674936, + "narHash": "sha256-k00uTP4JNfmejrCLJOwdObYC9jHRrr/5M/a/8L2EIdo=", "owner": "nix-community", "repo": "nixpkgs.lib", - "rev": "a73b9c743612e4244d865a2fdee11865283c04e6", + "rev": "2075416fcb47225d9b68ac469a5c4801a9c4dd85", "type": "github" }, "original": { @@ -115,11 +115,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1757068644, - "narHash": "sha256-NOrUtIhTkIIumj1E/Rsv1J37Yi3xGStISEo8tZm3KW4=", + "lastModified": 1769018530, + "narHash": "sha256-MJ27Cy2NtBEV5tsK+YraYr2g851f3Fl1LpNHDzDX15c=", "owner": "nixos", "repo": "nixpkgs", - "rev": "8eb28adfa3dc4de28e792e3bf49fcf9007ca8ac9", + "rev": "88d3861acdd3d2f0e361767018218e51810df8a1", "type": "github" }, "original": { diff --git a/hosts/bastille/blade-names.nix b/hosts/bastille/blade-names.nix index f656f2d..083cad3 100644 --- a/hosts/bastille/blade-names.nix +++ b/hosts/bastille/blade-names.nix @@ -1,16 +1,18 @@ -# keep-sorted start -[ - "backbiter" - "damocles" - "durendal" - "eyelander" - "excalibur" - "gram" - "gryffindor" - "kusanagi" - "narsil" - "oathbringer" - "riptide" - "sting" -] -# keep-sorted end +{ + # TODO: prospit's a special case and won't remain here forever + "d8:9e:f3:3e:f9:41" = "prospit"; + + "40:f2:e9:c6:65:5f" = "backbiter"; + "40:f2:e9:c6:69:43" = "damocles"; + "40:f2:e9:c6:69:67" = "durendal"; + "40:f2:e9:c6:74:59" = "eyelander"; + "40:f2:e9:c6:75:f1" = "excalibur"; + "40:f2:e9:c6:76:21" = "gram"; + + "unassigned-0" = "gryffindor"; + "unassigned-1" = "kusanagi"; + "unassigned-2" = "narsil"; + "unassigned-3" = "oathbringer"; + "unassigned-4" = "riptide"; + "unassigned-5" = "sting"; +} diff --git a/hosts/bastille/blade.nix b/hosts/bastille/blade.nix new file mode 100644 index 0000000..89bdcc8 --- /dev/null +++ b/hosts/bastille/blade.nix @@ -0,0 +1,23 @@ +{ modulesPath, pkgs, lib, ... }: { + imports = [ + ./eno1-imm-disable.nix + (import ../common/k3s.nix {}) + ../common/nix.nix + ../common/sshd.nix + ../common/users-local.nix + (modulesPath + "/installer/netboot/netboot-minimal.nix") + ]; + + # Get hostname from DHCP request + networking.hostName = ""; + + # when making the ISO, the initialHashedPassword is set to "" for some reason + # we already set a hashed password, so null this + users.users.root.initialHashedPassword = lib.mkForce null; + + environment.systemPackages = [ + pkgs.fastfetch + ]; + + system.stateVersion = "25.11"; +} diff --git a/hosts/bastille/eno1-imm-disable.nix b/hosts/bastille/eno1-imm-disable.nix new file mode 100644 index 0000000..87c8d6c --- /dev/null +++ b/hosts/bastille/eno1-imm-disable.nix @@ -0,0 +1,30 @@ +{ pkgs, lib, ... }: +let + eno1-imm-disable = pkgs.writeShellApplication { + name = "eno1-imm-disable"; + + runtimeInputs = [ + pkgs.iproute2 + ]; + + text = '' + if grep "Lenovo NeXtScale nx360 M5" /sys/devices/virtual/dmi/id/product_name; then + ip link set down eno1 + fi + ''; + }; +in { + systemd.services."eno1-imm-disable" = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + unitConfig = { + Description = "Disable eno1 on Lenovo NeXtScale nodes to avoid issues with using the imm interface"; + }; + + serviceConfig = { + Type = "oneshot"; + ExecStart = "${lib.getExe eno1-imm-disable}"; + }; + }; +} diff --git a/hosts/common/k3s.nix b/hosts/common/k3s.nix new file mode 100644 index 0000000..1ad0ded --- /dev/null +++ b/hosts/common/k3s.nix @@ -0,0 +1,17 @@ +{ role ? "agent", clusterInit ? false }: { + networking.firewall.allowedTCPPorts = [ + 6443 + ]; + + networking.firewall.allowedUDPPorts = [ + 8472 + ]; + + services.k3s = { + inherit role clusterInit; + + enable = true; + token = "garbage secret"; + serverAddr = "https://10.98.3.2:6443"; + }; +} diff --git a/hosts/common/nfs.nix b/hosts/common/nfs.nix new file mode 100644 index 0000000..98647c2 --- /dev/null +++ b/hosts/common/nfs.nix @@ -0,0 +1,20 @@ +{ config, pkgs, ... }: +let + mkNfs = {path, options ? [ "vers=4.0" "soft" "nodev" "nosuid" ]}: { + device = "${path}"; + fsType = "nfs"; + inherit options; + }; +in +{ + environment.systemPackages = [ pkgs.nfs-utils ]; + + fileSystems."/nfs/cistern/share" = mkNfs {path = "10.98.0.7:/cistern/nfs/share";}; + fileSystems."/nfs/cistern/files" = mkNfs {path = "10.98.0.7:/cistern/nfs/files";}; + fileSystems."/nfs/cistern/home" = mkNfs { + path = "10.98.0.7:/cistern/nfs/home"; + options = [ "vers=4.0" "soft" "nodev" "nosuid" ]; + }; + fileSystems."/nfs/cistern/libvirt" = mkNfs {path = "10.98.0.7:/cistern/nfs/libvirt";}; + fileSystems."/nfs/cistern/docker/data" = mkNfs {path = "10.98.0.7:/cistern/nfs/docker/data";}; +} \ No newline at end of file diff --git a/hosts/vesuvius/README.md b/hosts/vesuvius/README.md index 66f9fe3..2ab93b2 100644 --- a/hosts/vesuvius/README.md +++ b/hosts/vesuvius/README.md @@ -10,3 +10,13 @@ Giant storage server + future LHCPISCSIPXEIDK thing maybe? ## Storage We currently have one (manually created) RAID-Z2 pool mounted at `/forge` with `8` drives of `12 Tb` each. We have capacity for `48`(!) drives, but still only paper (and tape) caddies. + +``` +# for the nix store +zfs create -o mountpoint=legacy \ + -o compression=zstd \ + -o xattr=sa \ + -o acltype=posixacl \ + -o atime=off \ + forge/nix +``` \ No newline at end of file diff --git a/hosts/vesuvius/configuration.nix b/hosts/vesuvius/configuration.nix index 482f77b..9684500 100644 --- a/hosts/vesuvius/configuration.nix +++ b/hosts/vesuvius/configuration.nix @@ -2,9 +2,16 @@ { imports = [ ./hardware-configuration.nix + (import ../common/k3s.nix { role = "server"; clusterInit = true; }) ./nix.nix ./zfs.nix + ./ipa.nix ./netboot.nix + + ../common/nfs.nix + ../common/tz-locale.nix + ../common/users-local.nix + ../common/sshd.nix ]; boot.loader.systemd-boot.enable = true; @@ -15,44 +22,11 @@ networking.networkmanager.enable = true; networking.networkmanager.unmanaged = [ "interface-name:enp1s0f1" ]; - time.timeZone = "America/New_York"; - - i18n.defaultLocale = "en_US.UTF-8"; - - i18n.extraLocaleSettings = { - LC_ADDRESS = "en_US.UTF-8"; - LC_IDENTIFICATION = "en_US.UTF-8"; - LC_MEASUREMENT = "en_US.UTF-8"; - LC_MONETARY = "en_US.UTF-8"; - LC_NAME = "en_US.UTF-8"; - LC_NUMERIC = "en_US.UTF-8"; - LC_PAPER = "en_US.UTF-8"; - LC_TELEPHONE = "en_US.UTF-8"; - LC_TIME = "en_US.UTF-8"; - }; - - users.users.papatux = { - isNormalUser = true; - description = "papatux"; - extraGroups = [ "networkmanager" "wheel" ]; - openssh.authorizedKeys.keys = import ../../papatux-keys.nix; - }; - - security.sudo.wheelNeedsPassword = false; - nixpkgs.config.allowUnfree = true; environment.systemPackages = with pkgs; [ neovim ]; - services.openssh.enable = true; - - # This value determines the NixOS release from which the default - # settings for stateful data, like file locations and database versions - # on your system were taken. It‘s perfectly fine and recommended to leave - # this value at the release version of the first install of this system. - # Before changing this value read the documentation for this option - # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html). system.stateVersion = "25.05"; # Did you read the comment? } diff --git a/hosts/vesuvius/ipa.nix b/hosts/vesuvius/ipa.nix index fc011e1..04a698a 100644 --- a/hosts/vesuvius/ipa.nix +++ b/hosts/vesuvius/ipa.nix @@ -1,10 +1,17 @@ # TODO: /etc/krb5.keytab missing, maybe agenix { config, pkgs, ... }: { - age.secrets."krb5.keytab".file = ../../secrets/krb5.keytab.age; - + age.secrets."krb5.keytab" = { + file = ../../secrets/keytabs/vesuvius.keytab.age; + path = "/etc/krb5.keytab"; + owner = "root"; + group = "root"; + mode = "0600"; + }; environment.variables.KRB5_KTNAME = config.age.secrets."krb5.keytab".path; + networking.domain = "vtluug.org"; + security.ipa = { enable = true; @@ -19,4 +26,15 @@ sha256 = "16wv6kfvnm0hcyzr0wjrgmymw3asm84m8r1wbfq09qvqrjycfc6s"; }; }; + security.sudo.extraRules = [ + { + groups = [ "sudoers" ]; + commands = [ + { + command = "ALL"; + options = [ "SETENV" ]; + } + ]; + } + ]; } diff --git a/hosts/vesuvius/netboot.nix b/hosts/vesuvius/netboot.nix index 9222a1c..aa2e0b9 100644 --- a/hosts/vesuvius/netboot.nix +++ b/hosts/vesuvius/netboot.nix @@ -1,44 +1,54 @@ -{ config, pkgs, ... }: +{ + config, + lib, + pkgs, + ... +}: let - dom_ip = "10.98.2.1"; + dom_ip = "10.98.3.2"; + vlan_router_ip = "10.98.3.1"; + dns_server_ip = "10.98.0.1"; dhcp_iface = "enp1s0f1"; - client_range = "10.98.2.2,10.98.2.100"; + client_range = "10.98.3.3,10.98.3.100"; - sub_image = pkgs.nixos { - imports = [ "${pkgs.path}/nixos/modules/installer/netboot/netboot-minimal.nix" ]; + netboot-hostnames = import ../bastille/blade-names.nix; - system.stateVersion = "25.05"; - services.openssh = { - enable = true; - settings.PasswordAuthentication = true; - settings.KbdInteractiveAuthentication = false; - }; + sub_image = lib.nixosSystem { + system = "x86_64-linux"; - users.users.papatux = { - isNormalUser = true; - description = "papatux"; - extraGroups = [ "networkmanager" "wheel" ]; - hashedPassword = "$6$6GnvJWpo8oOWM1tb$GhuldW5iIdS6OuRyq5u1hSSu0VotQCLac7emA.Kui2hWLozR7EIO4Su6PCo5hTRG8iWnAOlGemQVyejIA9l4j/"; - openssh.authorizedKeys.keys = import ../../papatux-keys.nix; - }; + modules = [ + ../bastille/blade.nix + ]; }; - + + blade = sub_image.config.system.build; + ipxe_config = pkgs.writeText "boot.ipxe" '' #!ipxe - kernel http://${dom_ip}:8080/netboot-nixtest/kernel init=/init boot.shell_on_fail - initrd http://${dom_ip}:8080/netboot-nixtest/initrd + kernel http://${dom_ip}:8080/netboot-kernel/bzImage init=${blade.toplevel}/init boot.shell_on_fail + initrd http://${dom_ip}:8080/netboot-initrd/initrd boot ''; webroot = pkgs.linkFarm "netboot" [ - { name = "netboot-nixtest"; path = sub_image.config.system.build.toplevel; } - { name = "boot.ipxe"; path = ipxe_config; } + { + name = "netboot-kernel"; + path = blade.kernel; + } + { + name = "netboot-initrd"; + path = blade.netbootRamdisk; + } + { + name = "boot.ipxe"; + path = ipxe_config; + } ]; # fyi this is cause tftpd in dnsmasq chroots and wouldn't follow external symlinks # like the ones in a linkfarm - tftproot = pkgs.runCommand "tftproot-real" {} '' + tftproot = pkgs.runCommand "tftproot-real" { } '' mkdir -p $out cp ${ipxe_config} $out/boot.ipxe cp ${pkgs.ipxe}/ipxe.efi $out/ipxe.efi @@ -54,27 +64,46 @@ in services.dnsmasq = { enable = true; + settings.domain = "bastille.vtluug.org"; + settings.interface = "${dhcp_iface}"; + settings.bind-interfaces = true; + settings.server = [ "${dns_server_ip}" ]; settings.enable-tftp = true; settings.tftp-root = "${tftproot}"; settings.dhcp-range = "${client_range},12h"; - settings.dhcp-option = [ "option:router,${dom_ip}" ]; + settings.dhcp-option = [ "option:router,${vlan_router_ip}" ]; settings.dhcp-userclass = [ "set:ipxe,iPXE" ]; settings.dhcp-boot = [ "tag:!ipxe,ipxe.efi" - "http://${dom_ip}:8080/boot.ipxe" + "http://${dom_ip}:8080/boot.ipxe" ]; + # Set hostnames via DHCP + settings.dhcp-host = builtins.map (host: "${host.fst},${host.snd}") ( + lib.lists.filter (host: !lib.strings.hasInfix "unassigned" host.fst) ( + lib.lists.zipLists (builtins.attrNames netboot-hostnames) (builtins.attrValues netboot-hostnames) + ) + ); }; services.nginx = { enable = true; virtualHosts."netboot" = { - listen = [{ port = 8080; addr = "0.0.0.0"; }]; + listen = [ + { + port = 8080; + addr = "0.0.0.0"; + } + ]; locations."/".root = "${webroot}"; }; }; networking.firewall = { allowedTCPPorts = [ 8080 ]; - allowedUDPPorts = [ 67 69 ]; + allowedUDPPorts = [ + 53 + 67 + 69 + ]; }; -} \ No newline at end of file +} diff --git a/hosts/vesuvius/zfs.nix b/hosts/vesuvius/zfs.nix index df075c4..dfc4afe 100644 --- a/hosts/vesuvius/zfs.nix +++ b/hosts/vesuvius/zfs.nix @@ -13,5 +13,13 @@ fileSystems."/forge" = { device = "forge"; fsType = "zfs"; + neededForBoot = true; + }; + + fileSystems."/nix" = { + device = "/forge/nix"; + fsType = "none"; + options = [ "bind" ]; + depends = [ "/forge" ]; }; } diff --git a/secrets/keytabs/vesuvius.keytab.age b/secrets/keytabs/vesuvius.keytab.age new file mode 100644 index 0000000..b20d783 --- /dev/null +++ b/secrets/keytabs/vesuvius.keytab.age @@ -0,0 +1,5 @@ +age-encryption.org/v1 +-> ssh-ed25519 aQG9zQ AwGnHtis2RoMcZHMea4Jbhzm5yYq9uos+1h0WtIVDCM +IAFYorEgHmeGXXfakwa3Fk9UDC5XPJGSkx5PbPRumoo +--- x+eNJlai124x5jAWKQkq2hhDegsZ+GxPlsnoWrd4zv0 +D,|]=V,Aֳ#luy}(bi OFۓwrH_> (zr/)dJ1g3RӂFqx.܃3g*J(oUA dRa߂3@"L-S8_9*V~9yqr_AJЎDR8 4ߩu Ā??Eq \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 177078e..840d696 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -2,6 +2,6 @@ let vesuvius = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOHI7ziwxkEbJzvpaZulPFpDW7l0vbGJ+ifHcHJ2fHex"; zerocool = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+60yHIqES3Dr1Upp23QGwzvqELQEeH6e4lTKTV9iUY root@zerocool"; in { - "krb5.keytab.age".publicKeys = [ vesuvius ]; + "keytabs/vesuvius.keytab.age".publicKeys = [ vesuvius ]; "zerocool/wg.priv.age".publicKeys = [ zerocool ]; }