diff --git a/bin/kt b/bin/kt new file mode 100755 index 0000000..68c5b57 --- /dev/null +++ b/bin/kt @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import logging + +import click + +from kt.commands.checkout.command import checkout +from kt.commands.git_push.command import git_push +from kt.commands.list_kernels.command import list_kernels +from kt.commands.setup.command import setup +from kt.commands.vm.command import vm + +epilog = """ +Base of all tooling used for kernel development. + +All new tooling will be introduced as commands to kt. +""" + + +@click.group(epilog=epilog) +def cli(): + pass + + +def main(): + logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO) + + cli.add_command(list_kernels) + cli.add_command(setup) + cli.add_command(checkout) + cli.add_command(git_push) + cli.add_command(vm) + cli() + + +if __name__ == "__main__": + main() diff --git a/kernel_install_dep.sh b/kernel_install_dep.sh new file mode 100755 index 0000000..e6832b0 --- /dev/null +++ b/kernel_install_dep.sh @@ -0,0 +1,215 @@ +#!/bin/sh +set -e + +# So we can detect what version of Rocky we are running on +. /etc/os-release + +install_kselftest_deps_8() { + echo + echo "Installing kselftest deps for Rocky 8" + echo + sudo dnf -y groupinstall 'Development Tools' + sudo dnf -y install epel-release + sudo dnf -y install --enablerepo=devel \ + VirtualGL \ + alsa-lib-devel \ + bc \ + clang \ + conntrack-tools \ + curl \ + dropwatch \ + dwarves \ + e2fsprogs \ + ethtool \ + fuse \ + glibc \ + iperf3 \ + iptables \ + iputils \ + ipvsadm \ + jq \ + kernel-devel \ + kernel-tools \ + libasan \ + libcap-devel \ + libcap-ng-devel \ + libmnl-devel \ + libreswan \ + libubsan \ + llvm \ + ncurses-devel \ + net-tools \ + netsniff-ng \ + nftables \ + nmap-ncat \ + numactl-devel \ + openssl-devel \ + perf \ + popt-devel \ + python3-pip \ + rsync \ + socat \ + tcpdump \ + teamd \ + traceroute \ + wget + + # Doesn't work for 8.6? + sudo dnf -y install --enablerepo=devel \ + fuse-devel \ + gcc-toolset-13-libasan-devel \ + glibc-static \ + kernel-selftests-internal + + pip3 install --user \ + netaddr \ + packaging \ + pyftpdlib \ + pyparsing \ + pytest \ + scapy \ + tftpy +} + +install_kselftest_deps_9() { + echo + echo "Installing kselftest deps for Rocky 9" + echo + sudo dnf -y groupinstall 'Development Tools' + sudo dnf -y install epel-release + sudo dnf -y install --enablerepo=crb,devel \ + VirtualGL \ + alsa-lib-devel \ + bc \ + clang \ + conntrack-tools \ + curl \ + dropwatch \ + dwarves \ + e2fsprogs \ + ethtool \ + fuse \ + fuse-devel \ + gcc-toolset-13-libasan-devel \ + glibc \ + glibc-static \ + iperf3 \ + iptables \ + iputils \ + ipvsadm \ + jq \ + kernel-devel \ + kernel-selftests-internal \ + kernel-tools \ + libasan \ + libcap-devel \ + libcap-ng-devel \ + libmnl-devel \ + libreswan \ + libubsan \ + llvm \ + ncurses-devel \ + net-tools \ + netsniff-ng \ + nftables \ + nmap-ncat \ + numactl-devel \ + openssl-devel \ + packetdrill \ + perf \ + popt-devel \ + python3-pip \ + rsync \ + socat \ + tcpdump \ + teamd \ + traceroute \ + virtme-ng \ + wget + + pip3 install --user \ + netaddr \ + packaging \ + pyftpdlib \ + pyparsing \ + pytest \ + scapy \ + tftpy \ + wheel +} + +install_kselftest_deps_10() { + echo + echo "Installing kselftest deps for Rocky 10" + echo + sudo dnf -y groupinstall 'Development Tools' + sudo dnf -y install epel-release + sudo dnf -y install --enablerepo=crb,devel \ + alsa-lib-devel \ + bc \ + clang \ + conntrack-tools \ + curl \ + dropwatch \ + dwarves \ + e2fsprogs \ + ethtool \ + fuse \ + fuse-devel \ + glibc \ + glibc-static \ + iperf3 \ + iptables \ + iputils \ + ipvsadm \ + kernel-devel \ + kernel-selftests-internal \ + kernel-tools \ + libasan \ + libasan-static \ + libcap-devel \ + libcap-ng-devel \ + libmnl-devel \ + libreswan \ + libubsan \ + llvm \ + ncurses-devel \ + net-tools \ + nftables \ + nmap-ncat \ + numactl-devel \ + openssl-devel \ + packetdrill \ + perf \ + popt-devel \ + python3-pip \ + rsync \ + socat \ + tcpdump \ + teamd \ + traceroute \ + virtme-ng \ + wget + + pip3 install --user \ + netaddr \ + packaging \ + pyftpdlib \ + pyparsing \ + pytest \ + scapy \ + tftpy \ + wheel +} + +case "$ROCKY_SUPPORT_PRODUCT" in + Rocky-Linux-10) + install_kselftest_deps_10 + ;; + Rocky-Linux-9) + install_kselftest_deps_9 + ;; + Rocky-Linux-8) + install_kselftest_deps_8 + ;; +esac diff --git a/kernel_kselftest.sh b/kernel_kselftest.sh index 7946fed..38b1d59 100755 --- a/kernel_kselftest.sh +++ b/kernel_kselftest.sh @@ -17,203 +17,6 @@ else runs=1 fi -install_kselftest_deps_8() { - echo - echo "Installing kselftest deps for Rocky 8" - echo - sudo dnf -y groupinstall 'Development Tools' - sudo dnf -y install epel-release - sudo dnf -y install --enablerepo=devel \ - VirtualGL \ - alsa-lib-devel \ - bc \ - clang \ - conntrack-tools \ - curl \ - dropwatch \ - dwarves \ - e2fsprogs \ - ethtool \ - fuse \ - glibc \ - iperf3 \ - iptables \ - iputils \ - ipvsadm \ - jq \ - kernel-devel \ - kernel-tools \ - libasan \ - libcap-devel \ - libcap-ng-devel \ - libmnl-devel \ - libreswan \ - libubsan \ - llvm \ - ncurses-devel \ - net-tools \ - netsniff-ng \ - nftables \ - nmap-ncat \ - numactl-devel \ - openssl-devel \ - perf \ - popt-devel \ - python3-pip \ - rsync \ - socat \ - tcpdump \ - teamd \ - traceroute \ - wget - - # Doesn't work for 8.6? - sudo dnf -y install --enablerepo=devel \ - fuse-devel \ - gcc-toolset-13-libasan-devel \ - glibc-static \ - kernel-selftests-internal - - pip3 install --user \ - netaddr \ - packaging \ - pyftpdlib \ - pyparsing \ - pytest \ - scapy \ - tftpy -} - -install_kselftest_deps_9() { - echo - echo "Installing kselftest deps for Rocky 9" - echo - sudo dnf -y groupinstall 'Development Tools' - sudo dnf -y install epel-release - sudo dnf -y install --enablerepo=crb,devel \ - VirtualGL \ - alsa-lib-devel \ - bc \ - clang \ - conntrack-tools \ - curl \ - dropwatch \ - dwarves \ - e2fsprogs \ - ethtool \ - fuse \ - fuse-devel \ - gcc-toolset-13-libasan-devel \ - glibc \ - glibc-static \ - iperf3 \ - iptables \ - iputils \ - ipvsadm \ - jq \ - kernel-devel \ - kernel-selftests-internal \ - kernel-tools \ - libasan \ - libcap-devel \ - libcap-ng-devel \ - libmnl-devel \ - libreswan \ - libubsan \ - llvm \ - ncurses-devel \ - net-tools \ - netsniff-ng \ - nftables \ - nmap-ncat \ - numactl-devel \ - openssl-devel \ - packetdrill \ - perf \ - popt-devel \ - python3-pip \ - rsync \ - socat \ - tcpdump \ - teamd \ - traceroute \ - virtme-ng \ - wget - - pip3 install --user \ - netaddr \ - packaging \ - pyftpdlib \ - pyparsing \ - pytest \ - scapy \ - tftpy \ - wheel -} - -install_kselftest_deps_10() { - echo - echo "Installing kselftest deps for Rocky 10" - echo - sudo dnf -y groupinstall 'Development Tools' - sudo dnf -y install epel-release - sudo dnf -y install --enablerepo=crb,devel \ - alsa-lib-devel \ - bc \ - clang \ - conntrack-tools \ - curl \ - dropwatch \ - dwarves \ - e2fsprogs \ - ethtool \ - fuse \ - fuse-devel \ - glibc \ - glibc-static \ - iperf3 \ - iptables \ - iputils \ - ipvsadm \ - kernel-devel \ - kernel-selftests-internal \ - kernel-tools \ - libasan \ - libasan-static \ - libcap-devel \ - libcap-ng-devel \ - libmnl-devel \ - libreswan \ - libubsan \ - llvm \ - ncurses-devel \ - net-tools \ - nftables \ - nmap-ncat \ - numactl-devel \ - openssl-devel \ - packetdrill \ - perf \ - popt-devel \ - python3-pip \ - rsync \ - socat \ - tcpdump \ - teamd \ - traceroute \ - virtme-ng \ - wget - - pip3 install --user \ - netaddr \ - packaging \ - pyftpdlib \ - pyparsing \ - pytest \ - scapy \ - tftpy \ - wheel -} run_kselftest() { SUDO_TARGETS=$1 @@ -227,17 +30,8 @@ run_kselftest() { done } -case "$ROCKY_SUPPORT_PRODUCT" in - Rocky-Linux-10) - install_kselftest_deps_10 - ;; - Rocky-Linux-9) - install_kselftest_deps_9 - ;; - Rocky-Linux-8) - install_kselftest_deps_8 - ;; -esac +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +"$SCRIPT_DIR/kernel_install_dep.sh" case $(uname -r) in *3.10.0*) @@ -276,4 +70,3 @@ case $(uname -r) in esac run_kselftest "$SUDO_TARGETS" "$SKIP_TARGETS" - diff --git a/kt/KT.md b/kt/KT.md new file mode 100644 index 0000000..c57abf5 --- /dev/null +++ b/kt/KT.md @@ -0,0 +1,295 @@ +# ktools + +Introduction of kt CLI. + +This command is supposed to be the base of all commands used for kernel +development at CIQ. + +To keep things clear, it is introduced as a separate module in +kernel-src-tree-tools and it does not interfere with the current tooling. +By keeping this under the same repo, it will be easier to refactor things. + +## Setup: + +1. Install dependencies globally (you can also create a venv) : +``` +$ python -m pip install -e ".[dev]" +``` +2. The command above will install pre-commit. To setup the pre-commit tool +before you commit something, run this: +``` +$ pre-commit install +``` + +## Implementation details: + +kt/ktlib is the place for common helpers that would be used for kt commands. + +kt/ktlib.config.py is where a Config dataclass is implemented. This is crucial +for future commands and for doing the setup of the kernel developer. At the +moment it contains absolute paths to local directories: +- The working dir (the root directory for the setup) +- The directory parent for each kernel directory +- The parent directory where default images are downloaded +- The parent directory where running vm images for each kernel are stored. + +Each developer has to provide their own configuration in json file and keep +the path in KTOOLS_CONFIG_FILE. Otherwise a default one will be used. + +Example content of the config file: +``` +{ + "base_path": "~/ciq", + "kernels_dir": "~/ciq/kernels", + "images_source_dir": "~/ciq/default_test_images", + "images_dir": "~/ciq/tmp/virt-images", + "ssh_key": "~/.ssh/id_ed25519_generic.pub", +} +``` + +kt/ktlib/kernels.py is the python representation of the kernels.yaml +in kt/data folder. This should be the only source of truth for the kernels +we currently maintain. Ideally, this should be in its own repo, but to keep +things simple, it is part of the kt tool for the time being. + + +The information we store for each kernel is: +- the kernel source tree (at the moment is the same for all) +- the corresponding branch in the kernel source tree +- the rocky staging rpm repo ( it can be lts, fips or cbr) +- the corresponding branch in the rocky staging rpm repo as we support +multiple lts and fips kernels + +For example +``` +kernels: + cbr-7.9: + src_tree_root: kernel-src-tree + src_tree_branch: ciqcbr7_9 + dist_git_root: dist-git-tree-cbr + dist_git_branch: cbr79-7 +``` + +src_tree_root and dist_git_root are references to: + +``` +common_repos: + dist-git-tree-cbr: git@gitlab.com:ctrl-iq/linux/centos/cbr/src/kernel.git + dist-git-tree-lts: git@gitlab.com:ctrl-iq/linux/rocky/lts/src/kernel.git + dist-git-tree-fips: git@gitlab.com:ctrl-iq/linux/rocky/fips/src/kernel.git + kernel-src-tree: https://github.com/ctrliq/kernel-src-tree.git +``` + +A python dataclass KernelInfo that matches every kernel configuration is +introduced in kt/ktlib/kernels.py. The dataclass contains the absolute +path to the local clone of the repos, to make future work easier. And we +keep track of all kernels in KernelsInfo. + +For example, based on the default configuration, the KernelInfo object for the above kernel +will contain the following: +``` +- name: cbr-7.9 +- src_tree_root: RepoInfo(~/ciq/kernel-src-tree, git@gitlab.com:ctrl-iq/linux/centos/cbr/src/kernel.git) +- src_tree_branch: ciqcbr7_9 +- dist_git_root: RepoInfo(~/ciq/dist-git-tree-cbr, git@gitlab.com:ctrl-iq/linux/rocky/fips/src/kernel.git) +- dist_git_branch: cbr79-7 +``` +if the base_path is ~/ciq. + +The KernelInfo dataclass will be used later when we set up each kernel working +environment. + +## Commands + +Make sure kt is reachable from anywhere by adding it's location to PATH. +Example +``` +export PATH=$HOME/ciq/kernel-src-tree-tools/bin:$PATH +``` +If you are unsure how to use kt, just run it with --help. +Example: +``` +$ kt --help +``` + +Run --help for subcommands as well. + +Autocompletion works relatively well. Make sure it's enabled for your shell. +Check the official doc for [click](https://click.palletsprojects.com/en/stable/shell-completion/#enabling-completion) +Example for zsh: +``` +eval "$(_KT_COMPLETE=zsh_source kt)" +``` + +A command implementation is under ```kt/commands/``` folder. +To keep things cleaner, the actual logic is done in impl.py, +while command.py is used for the click interface, like argument and helper logic. + +### kt list-kernels +It shows the kernels we currently maintain. The data is taken from +KernelsInfo object which represents the kernels.yaml file in kt/data. + +Example: + +``` +$ kt list-kernels +cbr-7.9 +fips-8.10 +fips-8.6 +fips-9.2 +fipslegacy-8.6 +lts-8.6 +lts-8.8 +lts-9.2 +lts-9.4 +``` + +### kt setup + +``` +$ kt setup --help +``` + +It prepares the working directory for later commands: + +It clones the common_repos from kernels.yaml file in the config.base_path +directory. +If config.base_path = ~/ciq, these will be created: + +~/ciq/kernel-src-tree + +~/ciq/dist-git-tree-fips + +~/ciq/dist-git-tree-cbr + +~/ciq/dit-git-tree-lts + +~/ciq/kernel-src-tree-tools + +~/ciq/kernel-tools + +If there's a repo that needs to be cloned relevant for any future command, +this is when it should be cloned. + +### kt checkout +Prepares the working directory for a kernel. +It uses the KernelInfo dataclass created based on the kernels.yaml file. + +The working directory location is based on configuration: +`//` + +2 git worktrees are created for a kernel: + +- kernel-dist-git + +- kernel-src-tree + +They will point out to their root sources. Check kt setup for more info. +They should be located in . +The worktrees reference the remote from kernels.yaml. +The local branch is `{}/`. + +If `--change-dir` or `-c` option is used, it will also go to the working +directory of the kernel. + +If `--cleanup` option is used, it will delete the worktree and the local branch +before creating it from scratch again. + +#### Example: +``` +$ kt checkout lts9_4 +``` + +For this configuration +``` +{ + "base_path": "~/ciq", + "kernels_dir": "~/ciq/kernels", + "images_source_dir": "~/ciq/default_test_images", + "images_dir": "~/ciq/tmp/virt-images", + "ssh_key": "~/.ssh/id_ed25519_generic.pub", +} +``` + +This is the working directory for this kernel: +`~/ciq/kernels/lts9_4`. + +2 git worktrees are created: + +1. kernel-dist-git + + This representes branch `{}/lts9_4:origin/lts9_4`. + The source repo is ~/ciq/dist-git-tree-lts + +2. kernel-src-tree + + This representes branch `{}/ciqlts9_4:origin/ciqlts9_4` + The source repo is ~/ciq/kernel-src-tree + +### kt vm + +It spins up a virtual machine for the corresponding kernel. +If the virtual machine does not exist, it gets created. + +First, the vm image base (source) is downloaded if it does not exist +in . +To spin up the machine, a copy of this qcow2 image is put in +. Even if kernels may share the same image base, +they will have their own configuration and image. +cloud-init.yaml configuration is taken from kt/data and modified accordingly +for each user and then put in the same folder. + +Make sure your user is part of the libvirt group, otherwise you would need +to type your root password multiple times when getting access to the vm: +``` +$ sudo usermod -a -G libvirt $(whoami) +``` + +#### Example: +``` +$ kt vm lts9_4 +``` + +For this configuration +``` +{ + "base_path": "~/ciq", + "kernels_dir": "~/ciq/kernels", + "images_source_dir": "~/ciq/default_test_images", + "images_dir": "~/ciq/tmp/virt-images", + "ssh_key": "~/.ssh/id_ed25519_generic.pub", +} +``` + +Here is the qcow2 vm image used as source for other vms as well: + +``` +~/ciq/default_test_images/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2 +``` + +And here are the actual vm configuration and image files: +``` +ciq/tmp/virt-images/lts-9.4/cloud-init.yaml +ciq/tmp/virt-images/lts-9.4/lts-9.4.qcow2 +``` + +The cloud-init.yaml file is adapted from kt/data/cloud-init.yaml base file. + +`virt-install` command is then used to create the vm. + +If `--console` option is used, then `virsh --connect qemu://system console lts9-4` +is run (indirectly). + + +If `--test` option is used, then we connect to the vm via ssh and run + +``` +/kernel-src-tree-tools/kernel-build.sh -n +``` + +reboot +and then + +``` +/kernel-src-tree-tools/kernel-kselftest.sh +``` diff --git a/kt/commands/checkout/command.py b/kt/commands/checkout/command.py new file mode 100644 index 0000000..c7caa13 --- /dev/null +++ b/kt/commands/checkout/command.py @@ -0,0 +1,72 @@ +import click + +from kt.commands.checkout.impl import main +from kt.ktlib.shell_completion import ShellCompletion + +epilog = """ +Prepares the working directory for a kernel. +It uses the KernelInfo dataclass created based on the kernels.yaml file. + +The working directory location is based on configuration: +// + +2 git worktrees are created for a kernel: + +- dist-git-tree + +- kernel-src-tree + +They will point out to their root sources. Check kt setup for more info. +They should be located in . + +The worktrees reference the remote from kernels.yaml. +The local branch is {}/. + +Examples: + +\b +$ kt checkout lts9_4 +\b +$ kt checkout lts9_4 --cleanup +\b +$ kt checkout lts9_4 --cleanup -c +\b +$ kt checkout lts9_4 --cleanup -change-dir +\b +$ kt checkout lts9_4 -e CVE-2022-49909 +Will create folder lts9_4_CVE-2022-49909 instead of lts9_4. +""" + + +@click.command(epilog=epilog) +@click.option( + "-c", + "--change-dir", + is_flag=True, + help="Change directory to the kernel directory", +) +@click.option( + "--override", + is_flag=True, + help="Delete existing worktree for the kernel and start again", +) +@click.option( + "--cleanup", + is_flag=True, + help="Delete existing worktree for the kernel", +) +@click.option( + "-e", + "--extra", + type=str, + help="Feature you'll be working on", +) +@click.argument("kernel", required=True, type=str, shell_complete=ShellCompletion.show_kernels) +def checkout(kernel, change_dir, override, cleanup, extra): + main( + name=kernel, + change_dir=change_dir, + override=override, + cleanup=cleanup, + extra=extra, + ) diff --git a/kt/commands/checkout/impl.py b/kt/commands/checkout/impl.py new file mode 100644 index 0000000..f855c64 --- /dev/null +++ b/kt/commands/checkout/impl.py @@ -0,0 +1,33 @@ +import logging +import os +import shutil + +from kt.ktlib.config import Config +from kt.ktlib.kernel_workspace import KernelWorkspace +from kt.ktlib.kernels import KernelsInfo + + +def main(name: str, change_dir: bool, cleanup: bool, override: bool, extra: str): + config = Config.load() + kernels = KernelsInfo.from_yaml(config=config).kernels + if name not in kernels: + raise ValueError(f"Invalid param: {name} does not exist") + + kernel_info = kernels[name] + kernel_workspace = KernelWorkspace.load(name=name, config=config, kernel_info=kernel_info, extra=extra) + if cleanup: + kernel_workspace.cleanup() + return + + if override: + kernel_workspace.cleanup() + + kernel_workspace.setup() + if change_dir: + shell_exec = os.environ["SHELL"] + if not shutil.which(shell_exec): + raise Exception(f"No executable {shell_exec}; cannot start a new one") + + logging.info(f"Changing directory to {kernel_workspace.folder}") + os.chdir(kernel_workspace.folder) + os.execve(shell_exec, [shell_exec], os.environ) diff --git a/kt/commands/git_push/command.py b/kt/commands/git_push/command.py new file mode 100644 index 0000000..a5c3107 --- /dev/null +++ b/kt/commands/git_push/command.py @@ -0,0 +1,60 @@ +import click + +from kt.commands.git_push.impl import main +from kt.ktlib.shell_completion import ShellCompletion + +epilog = """ +It pushes the branch to remote for the kernelworkspace selected. +Since for every kernel we have two repos, one for the kernel source +and one for dist-git, an extra param is needed to select the repo. +The kernel workspace is created beforehead with kt checkout command. +Therefore, for every kernel we would have the branch +{USER}__, where feature is optional. +This command will push this branch to the origin. + +Examples: + +\b +$ kt git-push lts9_4 -k +Will push the branch {USER}_ciqlts9_4 from kernel-src-tree from lts9_4 kernel +workspace. + +\b +$ kt git-push lts9_4 -k -f +Same as above but it will force push + +\b +$ kt git-push lts9_4 -d +Will push the branch {USER}_lts94-9 from kernel-dist-tree from lts9_4 kernel +workspace. + +""" + + +@click.command(epilog=epilog) +@click.option( + "-k", + "--kernel-source", + is_flag=True, + help="It selects the kernel source repo", +) +@click.option( + "-d", + "--dist-git", + is_flag=True, + help="It selects the dist-git repo", +) +@click.option( + "-f", + "--force", + is_flag=True, + help="It force pushes the branch", +) +@click.argument("kernel_workspace", required=False, shell_complete=ShellCompletion.show_kernel_workspaces) +def git_push(kernel_workspace, kernel_source, dist_git, force): + main( + name=kernel_workspace, + kernel_source=kernel_source, + dist_git=dist_git, + force=force, + ) diff --git a/kt/commands/git_push/impl.py b/kt/commands/git_push/impl.py new file mode 100644 index 0000000..c0bc388 --- /dev/null +++ b/kt/commands/git_push/impl.py @@ -0,0 +1,17 @@ +import logging + +from kt.ktlib.config import Config +from kt.ktlib.kernel_workspace import KernelWorkspace + + +def main(name: str, kernel_source: bool, dist_git: bool, force: bool): + config = Config.load() + kernel_workpath = config.kernels_dir / name + kernel_workspace = KernelWorkspace.load_from_filepath(folder=kernel_workpath) + + if kernel_source: + kernel_workspace.src_worktree.push(force=force) + elif dist_git: + kernel_workspace.dist_worktree.push(force) + else: + logging.error("You need to specify the repo you want to push") diff --git a/kt/commands/list_kernels/command.py b/kt/commands/list_kernels/command.py new file mode 100644 index 0000000..6802462 --- /dev/null +++ b/kt/commands/list_kernels/command.py @@ -0,0 +1,18 @@ +import click + +from kt.commands.list_kernels.impl import main + +epilog = """ +It list all the kernels we currently maintain. + +Example: + +\b +$ kt list-kernels + +""" + + +@click.command(epilog=epilog) +def list_kernels(): + main() diff --git a/kt/commands/list_kernels/impl.py b/kt/commands/list_kernels/impl.py new file mode 100644 index 0000000..b6803c7 --- /dev/null +++ b/kt/commands/list_kernels/impl.py @@ -0,0 +1,9 @@ +from kt.ktlib.config import Config +from kt.ktlib.kernels import KernelsInfo + + +def main(): + config = Config.load() + kernels = KernelsInfo.from_yaml(config=config).kernels + for k in sorted(kernels): + print(k) diff --git a/kt/commands/setup/command.py b/kt/commands/setup/command.py new file mode 100644 index 0000000..c0790b9 --- /dev/null +++ b/kt/commands/setup/command.py @@ -0,0 +1,35 @@ +import click + +from kt.commands.setup.impl import main + +epilog = """ +Prepares the working directory for later commands: + +It clones the common_repos from kernels.yaml file in the config.base_path +directory. +If the repos are cloned already, they will be updated. + +If config.base_path = ~/ciq, these will be created: + +~/ciq/kernel-src-tree + +~/ciq/dist-git-tree-fips + +~/ciq/dist-git-tree-cbr + +~/ciq/dit-git-tree-lts + +~/ciq/kernel-src-tree-tools + +~/ciq/kernel-tools + +Examples: + +\b +$ kt setup +""" + + +@click.command(epilog=epilog) +def setup(): + main() diff --git a/kt/commands/setup/impl.py b/kt/commands/setup/impl.py new file mode 100644 index 0000000..d5cff67 --- /dev/null +++ b/kt/commands/setup/impl.py @@ -0,0 +1,13 @@ +from kt.ktlib.config import Config +from kt.ktlib.kernels import KernelsInfo + + +def main(): + config = Config.load() + + # create working dir if it does not exist + config.base_path.mkdir(parents=True, exist_ok=True) + + repos = KernelsInfo.from_yaml(config=config).repos + for repo in repos.values(): + repo.setup_repo() diff --git a/kt/commands/vm/command.py b/kt/commands/vm/command.py new file mode 100644 index 0000000..ba966fd --- /dev/null +++ b/kt/commands/vm/command.py @@ -0,0 +1,62 @@ +import click + +from kt.commands.vm.impl import main +from kt.ktlib.shell_completion import ShellCompletion + +epilog = """ +It spins up a virtual machine for the corresponding kernel. + +If the virtual machine does not exist, it gets created. + +First, the vm image base (source) is downloaded if it does not exist +in . + +To spin up the machine, a copy of this qcow2 image is put in +. Even if kernels may share the same image base, +they will have their own configuration and image. +cloud-init.yaml configuration is taken from kt/data and modified accordingly +for each user and then put in the same folder. + +Examples: + +\b +$ kt vm lts9_4 +\b +$ kt vm lts9_4 --console +\b +$ kt vm lts9_4 -c +\b +$ kt vm lts9_4 --destroy +\b +$ kt vm lts9_4 -c --override + +""" + + +@click.command(epilog=epilog) +@click.option( + "-c", + "--console", + is_flag=True, + help="It connects to the console of the vm", +) +@click.option( + "-d", + "--destroy", + is_flag=True, + help="It destroys the vm", +) +@click.option( + "--override", + is_flag=True, + help="It destroys the vm if it exists and creates a new one", +) +@click.option( + "--list-all", + is_flag=True, + help="Lists existings vms", +) +@click.option("--test", is_flag=True, help="Build the kernel and run kselftests") +@click.argument("kernel_workspace", required=False, shell_complete=ShellCompletion.show_kernel_workspaces) +def vm(kernel_workspace, console, destroy, override, list_all, test): + main(name=kernel_workspace, console=console, destroy=destroy, override=override, list_all=list_all, test=test) diff --git a/kt/commands/vm/impl.py b/kt/commands/vm/impl.py new file mode 100644 index 0000000..e48038f --- /dev/null +++ b/kt/commands/vm/impl.py @@ -0,0 +1,293 @@ +from __future__ import annotations + +import logging +import os +import time +from dataclasses import dataclass + +import oyaml as yaml +import wget +from git import Repo +from pathlib3x import Path + +from kt.ktlib.config import Config +from kt.ktlib.kernel_workspace import KernelWorkspace +from kt.ktlib.ssh import SshCommand +from kt.ktlib.util import Constants +from kt.ktlib.virt import VirtHelper, VmCommand + +# TODO move this to a separate repo +CLOUD_INIT_BASE_PATH = Path(__file__).parent.parent.parent.joinpath("data/cloud_init.yaml") + + +@dataclass +class Vm: + """ + Class that represents a virtual machine. + + Attributes: + name: name of the vm + + qcow2_source_path: qcow2 path to the vm image used as source + vm_major_version: major vm version (9 for Rocky 9) + qcow2_path: the qcow2 path of the vm image copied from qcow2_source_path + cloud_init_path: cloud_init.yaml config, adapted from data/cloud_init.yaml + """ + + qcow2_source_path: Path + vm_major_version: str + qcow2_path: Path + cloud_init_path: Path + name: str + kernel_workspace: KernelWorkspace + + @classmethod + def load(cls, config: Config, kernel_workspace: KernelWorkspace): + kernel_workspace_str = kernel_workspace.folder.name + kernel_name = cls._extract_kernel_name(kernel_workspace_str) + vm_major_version = cls._extract_major(kernel_name) + + # Image source paths construction + qcow2_source_path = config.images_source_dir / Path(cls._qcow2_name(vm_major_version=vm_major_version)) + + # Actual current image paths construction + work_dir = config.images_dir / Path(kernel_workspace_str) + qcow2_path = work_dir / Path(f"{kernel_workspace_str}.qcow2") + cloud_init_path = work_dir / Path(Constants.CLOUD_INIT) + + return cls( + qcow2_source_path=qcow2_source_path, + vm_major_version=vm_major_version, + qcow2_path=qcow2_path, + cloud_init_path=cloud_init_path, + name=kernel_workspace_str, + kernel_workspace=kernel_workspace, + ) + + @classmethod + def _extract_kernel_name(cls, kernel_workspace): + # _ --> where kernel does not contain any '_' + return kernel_workspace.split("_")[0] + + @classmethod + def _extract_major(cls, full_version): + # lts-9.4 --> return 9 + return full_version.split("-")[-1].split(".")[0] + + @classmethod + def _qcow2_name(cls, vm_major_version: str): + return f"{Constants.DEFAULT_VM_BASE}-{vm_major_version}-{Constants.QCOW2_TRAIL}" + + def _get_vm_url(self): + return f"{Constants.BASE_URL}/{self.vm_major_version}/images/x86_64/{self.qcow2_source_path.name}" + + def _download_source_image(self): + if self.qcow2_source_path.exists(): + logging.info(f"Image {self.qcow2_source_path} already exists, nothing to do") + return + + # Make sure the folder exists + self.qcow2_source_path.parent.mkdir(parents=True, exist_ok=True) + + logging.info("Downloading image") + wget.download(self._get_vm_url(), out=str(self.qcow2_source_path.parent)) + + def _setup_cloud_init(self, config: Config): + data = None + with open(CLOUD_INIT_BASE_PATH) as f: + data = yaml.safe_load(f) + + # replace placeholders with user data + data["users"][0]["name"] = os.environ["USER"] + + # password remains the default for now + data["chpasswd"]["list"][0] = f"{os.environ['USER']}:test" + + # ssh key + with open(config.ssh_key) as f: + ssh_key_content = f.read().strip() + data["users"][0]["ssh_authorized_keys"][0] = ssh_key_content + + data["mounts"][0][1] = str(config.base_path.absolute()) + data["mounts"][1][0] = str(config.base_path.absolute()) + + # Go to the working directory of the kernel + working_dir = config.kernels_dir / Path(self.name) + data["write_files"][0]["content"] = f"cd {str(working_dir)}" + + # Because $HOME is the same as the host, during boot, cloud-init + # sees the home dir already exists and root remains the owner + # change it to $USER + data["runcmd"][0][1] = f"{os.environ['USER']}:{os.environ['USER']}" + data["runcmd"][0][2] = os.environ["HOME"] + + # Install packages needed later + data["runcmd"].append([str(config.base_path / Path("kernel-src-tree-tools") / Path("kernel_install_dep.sh"))]) + # Write this to image cloud_init + with open(self.cloud_init_path, "w") as f: + f.write("#cloud-config\n") + yaml.dump(data, f) + + def _create_image(self, config: Config): + # Make sure the dir exists + self.qcow2_path.parent.mkdir(parents=True, exist_ok=True) + + self._setup_cloud_init(config=config) + # Copy qcow2 image to work dir + self.qcow2_source_path.copy(self.qcow2_path) + + self._virt_install(config=config) + time.sleep(Constants.VM_STARTUP_WAIT_SECONDS) + + def _virt_install(self, config: Config): + return VmCommand.install( + name=self.name, + qcow2_path=self.qcow2_path, + vm_major_version=self.vm_major_version, + cloud_init_path=self.cloud_init_path, + common_dir=config.base_path, + ) + + def setup(self, config: Config): + self._download_source_image() + + def spin_up(self, config: Config) -> VmInstance: + if not VirtHelper.exists(vm_name=self.name): + logging.info(f"VM {self.name} does not exist, creating from scratch...") + + self._create_image(config=config) + return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace) + + logging.info(f"Vm {self.name} already exists") + + if VirtHelper.is_running(vm_name=self.name): + logging.info(f"Vm {self.name} is running, nothing to do") + return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace) + + logging.info(f"Vm {self.name} is not running, starting it") + VmCommand.start(vm_name=self.name) + time.sleep(Constants.VM_STARTUP_WAIT_SECONDS) + + return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace) + + def destroy(self): + if VirtHelper.is_running(vm_name=self.name): + VmCommand.destroy(vm_name=self.name) + + if VirtHelper.exists(vm_name=self.name): + VmCommand.undefine(vm_name=self.name) + + # remove its folder that contains the qcow2 image and cloud-init config + self.qcow2_path.parent.rmtree(ignore_errors=True) + + +class VmInstance: + name: str + ssh_domain: str + kernel_workspace: KernelWorkspace + + def __init__(self, name: str, kernel_workspace: KernelWorkspace): + self.name = name + ip_addr = VirtHelper.ip_addr(vm_name=self.name) + username = os.environ["USER"] + self.domain = f"{username}@{ip_addr}" + self.kernel_workspace = kernel_workspace + + def reboot(self): + logging.debug("Rebooting vm") + + command = ["sudo", "reboot"] + try: + SshCommand.run(domain=self.domain, command=command) + except RuntimeError as e: + if "closed by remote host" in str(e): + pass + + time.sleep(Constants.VM_REBOOT_WAIT_SECONDS) + VmCommand.start(vm_name=self.name) + time.sleep(Constants.VM_STARTUP_WAIT_SECONDS) + + def current_head_sha_long(self): + repo = Repo(self.kernel_workspace.src_worktree.folder) + return repo.head.commit.hexsha + + def current_head_sha_short(self): + return self.current_head_sha_long()[:7] + + def kselftests(self, config): + logging.debug("Running kselftests") + script = str(config.base_path / Path("kernel-src-tree-tools") / Path("kernel_kselftest.sh")) + output_file = self.kernel_workspace.folder.absolute() / Path(f"kselftest-{self.current_head_sha_short()}.log") + ssh_cmd = f"cd {self.kernel_workspace.src_worktree.folder.absolute()} && {script}" + + SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd]) + + def expected_kernel_version(self): + kernel_version = SshCommand.running_kernel_version(domain=self.domain) + subversions = kernel_version.split("-") + if len(subversions) < 2: + return False + + # TODO some proper matching versioning here + install_hash = subversions[-1].split("+")[0] + + head_hash = self.current_head_sha_long() + if not head_hash.startswith(install_hash): + return False + + return True + + def build_kernel(self, config): + logging.debug("Building kernel") + build_script = str(config.base_path / Path("kernel-src-tree-tools") / Path("kernel_build.sh")) + output_file = self.kernel_workspace.folder.absolute() / Path( + f"kernel-build-{self.current_head_sha_short()}.log" + ) + ssh_cmd = f"cd {self.kernel_workspace.src_worktree.folder.absolute()} && {build_script} -n" + + SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd]) + + def test(self, config): + if self.expected_kernel_version(): + logging.info("Expected running kernel version, no need to build the kernel") + else: + self.build_kernel(config=config) + self.reboot() + + if not self.expected_kernel_version(): + raise RuntimeError("Kernel version is not what we expect") + + self.kselftests(config=config) + + def console(self): + VmCommand.console(vm_name=self.name) + + +def main(name: str, console: bool, destroy: bool, override: bool, list_all: bool, test: bool = False): + if list_all: + VmCommand.list_all() + return + + config = Config.load() + kernel_workpath = config.kernels_dir / name + kernel_workspace = KernelWorkspace.load_from_filepath(folder=kernel_workpath) + + vm = Vm.load(config=config, kernel_workspace=kernel_workspace) + if destroy: + vm.destroy() + return + + if override: + vm.destroy() + + vm.setup(config=config) + vm_instance = vm.spin_up(config=config) + + if test: + # Wait for the dependencies to be installed + logging.info("Waiting for the deps to be installed") + time.sleep(Constants.VM_DEPS_INSTALL_WAIT_SECONDS) + vm_instance.test(config=config) + + if console: + vm_instance.console() diff --git a/kt/data/cloud_init.yaml b/kt/data/cloud_init.yaml new file mode 100644 index 0000000..77101c4 --- /dev/null +++ b/kt/data/cloud_init.yaml @@ -0,0 +1,35 @@ +#cloud-config +users: + - name: USER_PLACEHOLDER + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - SSH_KEY_PLACEHOLDER +chpasswd: + expire: false + list: + - USER_PLACEHOLDER:test + +ssh_pwauth: true + +# Ensure the system does not update on boot. +package_upgrade: False + +mounts: + - ["mount_tag_mock_scratch", "SHARED_DIR_PLACEHOLDER", "virtiofs", "rw,relatime,context=unconfined_u:object_r:mock_var_lib_t:s0"] + - ["SHARED_DIR_PLACEHOLDER", "/var/lib/mock", "bind", "defaults,bind"] + +# Change working directory after boot +# Setting up homedir does not work without tricks, because +# the actual working directory is mounted after +# TODO this does not work if the vm is already running... +write_files: + - path: /etc/profile.d/change_dir.sh + content: | + cd WORKING_DIR_PLACEHOLDER + permissions: '0644' + +# if homedir already exists, cloud-init does nothing about it +# and then root is the owner +# workaround to change the owner to user +runcmd: + - [chown, USER_PLACEHOLDER:USER_PLACEHOLDER, HOMEDIR_PLACEHOLDER] diff --git a/kt/data/kernels.yaml b/kt/data/kernels.yaml new file mode 100644 index 0000000..32d82b4 --- /dev/null +++ b/kt/data/kernels.yaml @@ -0,0 +1,46 @@ +common_repos: + dist-git-tree-fips: git@gitlab.com:ctrl-iq-public/fips/src/kernel.git + kernel-src-tree: https://github.com/ctrliq/kernel-src-tree.git + +kernels: + cbr-7.9: + src_tree_root: kernel-src-tree + src_tree_branch: ciqcbr7_9 + dist_git_root: dist-git-tree-cbr + dist_git_branch: cbr79-7 + + lts-8.6: + src_tree_root: kernel-src-tree + src_tree_branch: ciqlts8_6 + dist_git_root: dist-git-tree-lts + dist_git_branch: lts86-8 + + lts-9.2: + src_tree_root: kernel-src-tree + src_tree_branch: ciqlts9_2 + dist_git_root: dist-git-tree-lts + dist_git_branch: lts92-9 + + lts-9.4: + src_tree_root: kernel-src-tree + src_tree_branch: ciqlts9_4 + dist_git_root: dist-git-tree-lts + dist_git_branch: lts94-9 + + fipslegacy-8.6: + src_tree_root: kernel-src-tree + src_tree_branch: fips-legacy-8-compliant/4.18.0-425.13.1 + dist_git_root: dist-git-tree-fips + dist_git_branch: fips-compliant8 + + fips-9.2: + src_tree_root: kernel-src-tree + src_tree_branch: fips-9-compliant/5.14.0-284.30.1 + dist_git_root: dist-git-tree-fips + dist_git_branch: el92-fips-compliant-9 + + lts-9.6: + src_tree_root: kernel-src-tree + src_tree_branch: ciqlts9_6 + dist_git_root: dist-git-tree-lts + dist_git_branch: lts96-9 diff --git a/kt/ktlib/command_runner.py b/kt/ktlib/command_runner.py new file mode 100644 index 0000000..730a6f3 --- /dev/null +++ b/kt/ktlib/command_runner.py @@ -0,0 +1,60 @@ +import logging +import subprocess +import sys + +import pexpect + + +class CommandRunner: + """ + Base class for bash command execution + """ + + @classmethod + def _build_command(cls, **kwargs) -> list[str]: + """Build the full command. Override in subclasses to add prefixes/options.""" + raise NotImplementedError + + @classmethod + def run(cls, **kwargs) -> str: + full_command = cls._build_command(**kwargs) + logging.info(f"Running command {full_command}") + + result = subprocess.run( + full_command, + text=True, + capture_output=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr) + + return result.stdout + + @classmethod + def run_with_output(cls, output_file: str, **kwargs): + full_command = cls._build_command(**kwargs) + logging.info(f"Running command {full_command}") + + # Run the command and stream output + process = subprocess.Popen( + full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, bufsize=1 + ) + + # Read and display output line by line + with open(output_file, "w") as f: + for line in process.stdout: + print(line, end="") + f.write(line) # Print to console + sys.stdout.flush() # Force immediate display + + # Wait for the process to complete + return_code = process.wait() + + if return_code != 0: + raise RuntimeError(f"Command failed {return_code}") + + @classmethod + def spawn(cls, **kwargs) -> pexpect.pty_spawn.spawn: + full_command = cls._build_command(**kwargs) + return pexpect.spawn(full_command[0], full_command[1:]) diff --git a/kt/ktlib/config.py b/kt/ktlib/config.py new file mode 100644 index 0000000..6ff5963 --- /dev/null +++ b/kt/ktlib/config.py @@ -0,0 +1,96 @@ +import json +import os +import warnings +from dataclasses import dataclass +from typing import ClassVar, Optional + +from pathlib3x import Path + +CONFIG_FILE_ENV_VAR = "KTOOLS_CONFIG_FILE" + + +@dataclass +class Config: + """ + Config dataclass that contains the basic paths for each kernel + developer setup. + + base_path The working directory for developers + kernels_dir The directory where each kernel working dir will be created + images_source_dir The directory where the default images will be + downloaded + images_dir The directory where the vm images for each kernel will be + stored + ssh_key: Path to the ssh key (public) shared between host and each vms + + All paths should be absolute, to avoid issues later. + """ + + base_path: Path + kernels_dir: Path + images_source_dir: Path + images_dir: Path + + ssh_key: Path + + DEFAULT: ClassVar = { + "base_path": "~/ciq", + "kernels_dir": "~/ciq/kernels", + "images_source_dir": "~/ciq/default_test_images", + "images_dir": "~/ciq/tmp/virt-images", + "ssh_key": "~/.ssh/id_ed25519_generic.pub", + } + + @classmethod + def from_str_dict(cls, data: dict[str, str]): + # Transform the str values to Path + new_data = {k: Path(v).expanduser() for k, v in data.items()} + + if not all(v.is_absolute() for v in new_data.values()): + raise ValueError("all paths should be absolute; check your config") + + return cls(**new_data) + + @classmethod + def load(cls): + """Load the default configuration. + + The configuration is loaded in this order from: + + 1. The filename provided in KTOOLS_CONFIG_FILE env var; + filename type is a json file + 2. The default configuration + + """ + + filename = os.getenv(CONFIG_FILE_ENV_VAR, None) + return cls.from_filename(filename) + + @classmethod + def from_filename(cls, filename: Optional[str]): + """Load config from filename""" + + if filename is None: + return cls.from_str_dict(cls.DEFAULT) + + if not os.path.exists(filename): + warnings.warn(f"{filename} does not exist, using default config.") + return cls.from_str_dict(cls.DEFAULT) + + with open(filename) as jfd: + json_data = jfd.read() + + return cls.from_json(json_data) + + @classmethod + def from_json(cls, json_data: Optional[str]): + if json_data is None: + return cls.from_str_dict(cls.DEFAULT) + + try: + data = json.loads(json_data) + except ValueError: + warnings.warn("Invalid configuration, using default config.") + data = dict(cls.DEFAULT) + + return cls.from_str_dict(data) diff --git a/kt/ktlib/kernel_workspace.py b/kt/ktlib/kernel_workspace.py new file mode 100644 index 0000000..3257712 --- /dev/null +++ b/kt/ktlib/kernel_workspace.py @@ -0,0 +1,181 @@ +import logging +import os +from dataclasses import dataclass + +from git import GitCommandError, Repo +from pathlib3x import Path + +from kt.ktlib.config import Config +from kt.ktlib.kernels import KernelInfo +from kt.ktlib.util import Constants + + +@dataclass +class RepoWorktree: + source_root: Repo + folder: Path + remote: str + remote_branch: str + local_branch: str + + @classmethod + def load_from_filepath(cls, folder: Path): + if not folder.exists(): + raise RuntimeError(f"{folder} does not exist") + + # check if folder is a git repo + repo = Repo(folder) + + source_root = Path(repo.git.rev_parse("--path-format=absolute", "--git-common-dir")).parent + remote = repo.remotes.origin + + remote_branch = repo.active_branch.tracking_branch() + + local_branch = repo.active_branch.name + + return cls( + source_root=source_root, + folder=folder, + remote=remote, + remote_branch=remote_branch, + local_branch=local_branch, + ) + + def setup(self): + """ + First run: It will create the worktree + Second run: It will update the worktree + """ + + try: + remote_ref = f"{self.remote}/{self.remote_branch}" + self.source_root.git.worktree( + "add", + "--track", + "-b", + self.local_branch, + self.folder, + remote_ref, + ) + + except GitCommandError as e: + if "already exists" in e.stderr: + self.update() + else: + # Make sure the worktree is properly cleaned up + self.cleanup() + raise e + + def update(self): + """ + It will make sure the worktree is up-to-date with remote. + It assumes the worktree is already created and initialized. + """ + logging.info("update") + repo = Repo(self.folder) + + repo.remotes.origin.pull(rebase=True) + + def cleanup(self): + # remove worktree, only if it exists + try: + self.source_root.git.worktree("remove", self.folder, "-f") + except GitCommandError as e: + if f"'{self.folder}' is not a working tree" not in e.stderr: + raise e + + # remove local branch, only if it exists + try: + self.source_root.delete_head(self.local_branch, force=True) + except GitCommandError as e: + if f"branch '{self.local_branch}' not found" not in e.stderr: + raise e + + def push(self, force: bool = False): + repo = Repo(self.folder) + origin = repo.remote(name=self.remote) + args = ["--set-upstream", origin.name, self.local_branch] + if force: + args.append("--force-with-lease") + + repo.git.push(*args) + + +@dataclass +class KernelWorkspace: + folder: Path + dist_worktree: RepoWorktree + src_worktree: RepoWorktree + + @classmethod + def load_from_filepath(cls, folder: Path): + if not folder.exists(): + raise RuntimeError(f"{folder} does not exists") + + ## Get the dist-git-tree path + dist_worktree_path = folder / Constants.DIST_TREE + dist_worktree = RepoWorktree.load_from_filepath(folder=dist_worktree_path) + + src_worktree_path = folder / Constants.SRC_TREE + src_worktree = RepoWorktree.load_from_filepath(folder=src_worktree_path) + + return cls( + folder=folder, + dist_worktree=dist_worktree, + src_worktree=src_worktree, + ) + + @classmethod + def load(cls, name: str, config: Config, kernel_info: KernelInfo, extra: str): + if extra: + name = name + "_" + extra + + folder = config.kernels_dir / Path(name) + user = os.environ["USER"] + default_remote = "origin" + + dist_folder = folder / Path(Constants.DIST_TREE) + dist_local_branch = f"{{{user}}}_{kernel_info.dist_git_branch}" + if extra: + dist_local_branch += f"_{extra}" + + dist_worktree = RepoWorktree( + source_root=Repo(kernel_info.dist_git_root.folder), + folder=dist_folder, + remote=default_remote, + remote_branch=kernel_info.dist_git_branch, + local_branch=dist_local_branch, + ) + + src_folder = folder / Path(Constants.SRC_TREE) + src_local_branch = f"{{{user}}}_{kernel_info.src_tree_branch}" + if extra: + src_local_branch += f"_{extra}" + + src_worktree = RepoWorktree( + source_root=Repo(kernel_info.src_tree_root.folder), + folder=src_folder, + remote=default_remote, + remote_branch=kernel_info.src_tree_branch, + local_branch=src_local_branch, + ) + + return cls( + folder=folder, + dist_worktree=dist_worktree, + src_worktree=src_worktree, + ) + + def setup(self): + # Make sure the folder is created + self.folder.mkdir(parents=True, exist_ok=True) + + self.dist_worktree.setup() + self.src_worktree.setup() + + def cleanup(self): + self.dist_worktree.cleanup() + self.src_worktree.cleanup() + + # Remove working directory that includes the above git worktrees + self.folder.rmtree(ignore_errors=True) diff --git a/kt/ktlib/kernels.py b/kt/ktlib/kernels.py new file mode 100644 index 0000000..a881d44 --- /dev/null +++ b/kt/ktlib/kernels.py @@ -0,0 +1,133 @@ +import logging +from dataclasses import dataclass + +import yaml +from pathlib3x import Path + +from kt.ktlib.config import Config +from kt.ktlib.repo import RepoInfo +from kt.ktlib.util import Constants + +# TODO move this to a separate repo +KERNEL_INFO_YAML_PATH = Path(__file__).parent.parent.joinpath("data/kernels.yaml") + + +@dataclass +class KernelInfo: + """ + Dataclass that represents each kernel entry in the kernels.yaml file + name: name of the kernel + src_tree_root: kernel source tree + src_tree_branch: the corresponding branch in the source tree + dist_git_root: rocky staging rpm repo + dist_git_branch: corresponding branch in the rocky staging rpm repo + + The src_tree_root and dist_git_root contain absolute paths to the local + clone of these repos and their corresponding remote url. + """ + + name: str + + src_tree_root: RepoInfo + src_tree_branch: str + + dist_git_root: RepoInfo + dist_git_branch: str + + +@dataclass +class KernelsInfo: + kernels: dict[str, KernelInfo] + repos: dict[str, RepoInfo] + + @classmethod + def _load_private_repos(cls, config: Config) -> dict[str, str]: + """Load private repository URLs from local config""" + private_repos_path = config.base_path / Constants.PRIVATE_REPOS_CONFIG_FILE + + if not private_repos_path.exists(): + logging.info(f"{private_repos_path} does not exist") + return {} + + with open(private_repos_path) as f: + data = yaml.safe_load(f) + return data.get("private_repos", {}) + + @classmethod + def _get_repos(cls, data: dict, private_data: dict, config: Config): + repos = {} + + try: + items = data[Constants.COMMON_REPOS].items() + except KeyError as e: + raise ValueError(f"Error: {e}; Failed to process {data}") + + for name, url in items: + name_path = config.base_path / Path(name) + repo = RepoInfo(folder=name_path, url=url) + repos[name] = repo + + for name, url in private_data.items(): + name_path = config.base_path / Path(name) + repo = RepoInfo(folder=name_path, url=url) + repos[name] = repo + + return repos + + @classmethod + def _get_kernels_info(cls, data: dict, repos: dict[str, RepoInfo]): + kernels_info = {} + + try: + items = data[Constants.KERNELS].items() + except KeyError as e: + raise ValueError(f"Failed to process {data}: {e}") + + for kernel, info in items: + k_info_dict = {"name": kernel, **info} + + # Make the src_tree_root and dist_git_root absolute paths to the + # local clone of these repos (transformation from src to Path) + # repos dictionary contains the proper absolute paths + if k_info_dict[Constants.DIST_GIT_ROOT] not in repos: + raise ValueError( + ( + f"{k_info_dict[Constants.DIST_GIT_ROOT]} not valid; " + f"it must be a reference to one of {list(repos.keys())}" + ) + ) + + k_info_dict[Constants.DIST_GIT_ROOT] = repos[k_info_dict[Constants.DIST_GIT_ROOT]] + + if k_info_dict[Constants.SRC_TREE_ROOT] not in repos: + raise ValueError( + ( + f"{k_info_dict[Constants.SRC_TREE_ROOT]} not valid; " + f"it must be a reference to one of {list(repos.keys())}" + ) + ) + + k_info_dict[Constants.SRC_TREE_ROOT] = repos[k_info_dict[Constants.SRC_TREE_ROOT]] + + kernel_info = KernelInfo(**k_info_dict) + kernels_info[kernel] = kernel_info + + return kernels_info + + @classmethod + def from_yaml(cls, config: Config): + data = None + + with open(KERNEL_INFO_YAML_PATH) as f: + data = yaml.safe_load(f) + + private_data = cls._load_private_repos(config=config) + + return cls.from_dict(data=data, private_data=private_data, config=config) + + @classmethod + def from_dict(cls, data: dict, private_data: dict, config: Config): + repos = cls._get_repos(data=data, private_data=private_data, config=config) + kernels_info = cls._get_kernels_info(data=data, repos=repos) + + return cls(kernels=kernels_info, repos=repos) diff --git a/kt/ktlib/repo.py b/kt/ktlib/repo.py new file mode 100644 index 0000000..74e1c95 --- /dev/null +++ b/kt/ktlib/repo.py @@ -0,0 +1,44 @@ +import logging +from dataclasses import dataclass + +import git +from git import Repo +from pathlib3x import Path + + +@dataclass +class RepoInfo: + """ + Dataclass that represents a local clone of a git repository. + folder: absolute path to the local clone + url: remote origin + """ + + folder: Path + url: str + + def _clone_repo(self): + """ + It clones the repo into destination folder + """ + + # TODO show progress + logging.info(f"Cloning {self.url} to {self.folder}") + git.Repo.clone_from(self.url, self.folder) + + def _update(self): + repo = Repo(self.folder) + repo.remotes.origin.pull(rebase=True) + + def setup_repo(self): + """ + Set up a git repository at the destination. + If destination already exists and override == True, + nothing is done + """ + if not self.folder.exists(): + self._clone_repo() + return + + logging.info(f"{self.folder} already exists, updating it") + self._update() diff --git a/kt/ktlib/shell_completion.py b/kt/ktlib/shell_completion.py new file mode 100644 index 0000000..cf8aebd --- /dev/null +++ b/kt/ktlib/shell_completion.py @@ -0,0 +1,27 @@ +from kt.ktlib.config import Config +from kt.ktlib.kernels import KernelsInfo + + +class ShellCompletion: + @classmethod + def show_kernels(cls, ctx, param, incomplete): + config = Config.load() + kernels = KernelsInfo.from_yaml(config=config).kernels + + return [kernel for kernel in kernels if kernel.startswith(incomplete)] + + @classmethod + def show_kernel_workspaces(cls, ctx, param, incomplete): + config = Config.load() + + # Since the current tooling uses a bunch of relative paths, we may have other dirs in the kernels directory. + # Therefore, an extra check is required to make sure the kernel workspaces are the one recommended + kernels = KernelsInfo.from_yaml(config=config).kernels.keys() + + kernel_workspaces = [ + kernel_workspace.name + for kernel_workspace in config.kernels_dir.iterdir() + if kernel_workspace.is_dir() and kernel_workspace.name.split("_")[0] in kernels + ] + + return [kernel_workspace for kernel_workspace in kernel_workspaces if kernel_workspace.startswith(incomplete)] diff --git a/kt/ktlib/ssh.py b/kt/ktlib/ssh.py new file mode 100644 index 0000000..300020c --- /dev/null +++ b/kt/ktlib/ssh.py @@ -0,0 +1,14 @@ +from kt.ktlib.command_runner import CommandRunner + + +class SshCommand(CommandRunner): + COMMAND = "ssh" + EXTRA = "-o StrictHostKeyChecking=no" + + @classmethod + def _build_command(cls, domain: str, command: list[str]) -> list[str]: + return [cls.COMMAND, cls.EXTRA, domain] + command + + @classmethod + def running_kernel_version(cls, domain): + return cls.run(domain=domain, command=["uname", "-r"]) diff --git a/kt/ktlib/util.py b/kt/ktlib/util.py new file mode 100644 index 0000000..3a15254 --- /dev/null +++ b/kt/ktlib/util.py @@ -0,0 +1,18 @@ +class Constants: + PRIVATE_REPOS_CONFIG_FILE = ".private_repos.yaml" + SRC_TREE = "kernel-src-tree" + DIST_TREE = "kernel-dist-git" + DIST_GIT_ROOT = "dist_git_root" + SRC_TREE_ROOT = "src_tree_root" + COMMON_REPOS = "common_repos" + KERNELS = "kernels" + + BASE_URL = "https://download.rockylinux.org/pub/rocky" + QCOW2_TRAIL = "GenericCloud-Base.latest.x86_64.qcow2" + DEFAULT_VM_BASE = "Rocky" + + CLOUD_INIT = "cloud_init.yaml" + + VM_DEPS_INSTALL_WAIT_SECONDS = 300 + VM_STARTUP_WAIT_SECONDS = 60 + VM_REBOOT_WAIT_SECONDS = 120 diff --git a/kt/ktlib/virt.py b/kt/ktlib/virt.py new file mode 100644 index 0000000..579f053 --- /dev/null +++ b/kt/ktlib/virt.py @@ -0,0 +1,118 @@ +from enum import Enum + +from pathlib3x import Path + +from kt.ktlib.command_runner import CommandRunner + + +class VmCommandType(Enum): + VIRSH = 1 + VIRT_INSTALL = 2 + + +class VmCommand(CommandRunner): + CONNECT = "--connect" + DOMAIN = "qemu:///system" + COMMAND_MAP = { + VmCommandType.VIRSH: "virsh", + VmCommandType.VIRT_INSTALL: "virt-install", + } + + @classmethod + def _build_command(cls, command_type: VmCommandType, command: list[str]) -> list[str]: + return [ + cls.COMMAND_MAP[command_type], + cls.CONNECT, + cls.DOMAIN, + ] + command + + @classmethod + def dominfo(cls, vm_name: str) -> dict[str, str]: + result = cls.run(command_type=VmCommandType.VIRSH, command=["dominfo", vm_name]).strip().split("\n") + + # TODO Security label has multiple : and it breaks the logic + return dict([tuple("".join(x.split()).split(":")[:2]) for x in result]) + + @classmethod + def install( + cls, + name: str, + qcow2_path: Path, + vm_major_version: str, + cloud_init_path: Path, + common_dir: Path, + ): + command = [ + "--name", + name, + "--disk", + f"{qcow2_path},device=disk,bus=virtio", + f"--os-variant=rocky{vm_major_version}", + "--virt-type", + "kvm", + "--vcpus", + "12,vcpu.cpuset=0-11,vcpu.placement=static", + "--memory", + str(32768), + "--vnc", + "--cloud-init", + f"user-data={cloud_init_path}", + "--filesystem", + f"source={common_dir},target=mount_tag_mock_scratch,accessmode=passthrough,driver.type=virtiofs,driver.queue=1024,binary.path=/usr/libexec/virtiofsd,binary.xattr=on", + "--memorybacking", + "source.type=memfd,access.mode=shared", + "--noautoconsole", + ] + + cls.run(command_type=VmCommandType.VIRT_INSTALL, command=command) + + @classmethod + def start(cls, vm_name: str) -> str: + cls.run(command_type=VmCommandType.VIRSH, command=["start", vm_name]) + + @classmethod + def console(cls, vm_name: str): + child = cls.spawn(command_type=VmCommandType.VIRSH, command=["console", vm_name]) + child.interact() + + @classmethod + def destroy(cls, vm_name: str): + cls.run(command_type=VmCommandType.VIRSH, command=["destroy", vm_name]) + + @classmethod + def undefine(cls, vm_name: str): + cls.run(command_type=VmCommandType.VIRSH, command=["undefine", vm_name]) + + @classmethod + def list_all(cls): + print(cls.run(command_type=VmCommandType.VIRSH, command=["list", "--all"])) + + @classmethod + def domifaddr(cls, vm_name: str) -> list[str]: + return cls.run(command_type=VmCommandType.VIRSH, command=["domifaddr", vm_name]).strip().split("\n")[-1].split() + + +class VirtHelper: + @classmethod + def ip_addr(cls, vm_name: str) -> str: + rc = VmCommand.domifaddr(vm_name=vm_name) + + return rc[-1].split("/")[0] + + @classmethod + def is_running(cls, vm_name: str) -> bool: + try: + result = VmCommand.dominfo(vm_name=vm_name) + except Exception: + return False + + return result["State"] == "running" + + @classmethod + def exists(cls, vm_name: str) -> bool: + try: + result = VmCommand.dominfo(vm_name=vm_name) + except Exception: + return False + + return result["Name"] == vm_name diff --git a/pyproject.toml b/pyproject.toml index 4a36b3b..a076dba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,12 +6,19 @@ requires-python = ">=3.10" readme = "README.md" version = "0.0.1" dependencies = [ + "click", + "gitpython", + "pathlib3x", + "python3-wget", + "oyaml", + "pexpect", ] [project.optional-dependencies] dev = [ "pre-commit", "ruff", + "pytest", ] [tool.ruff] @@ -24,3 +31,7 @@ ignore = ["E501"] [tool.setuptools] # suppress error: Multiple top-level packages py-modules = [] + +[tool.setuptools.packages.find] +where = ["."] +include = ["kt*"] diff --git a/tests/kt/ktlib/test_config.py b/tests/kt/ktlib/test_config.py new file mode 100644 index 0000000..419a6d4 --- /dev/null +++ b/tests/kt/ktlib/test_config.py @@ -0,0 +1,76 @@ +import pytest +from pathlib3x import Path + +from kt.ktlib.config import Config + +DEFAULT_CONFIG = {"base_path": Path("~/ciq").expanduser()} +CONFIG_STR = ( + "{" + '"base_path": "~/ciq",' + '"kernels_dir": "~/ciq/kernels",' + '"images_source_dir": "~/ciq/default_test_images",' + '"images_dir": "~/ciq/tmp/virt-images",' + '"ssh_key": "~/ciq/id_ed25519_generic.pub"' + "}" +) + + +def test_config_load_default_fallback(): + config = Config.load() + assert config.base_path == DEFAULT_CONFIG["base_path"] + + +def test_config_load_from_filename_None(): + config = Config.from_filename(None) + assert config.base_path == DEFAULT_CONFIG["base_path"] + + +@pytest.mark.filterwarnings("ignore", message=r"*using default config") +def test_config_load_from_filename_not_exists(): + config = Config.from_filename("test") + assert config.base_path == DEFAULT_CONFIG["base_path"] + + +def test_config_load_from_json_None(): + config = Config.from_json(None) + assert config.base_path == DEFAULT_CONFIG["base_path"] + + +@pytest.mark.filterwarnings("ignore", message=r"*using default config") +def test_config_load_from_json_data_None(): + json_data = "" + + config = Config.from_json(json_data) + assert config.base_path == DEFAULT_CONFIG["base_path"] + + +def test_config_load_from_json_data_empty(): + json_data = "{}" + + with pytest.raises(TypeError, match="missing 5 required positional arguments:"): + config = Config.from_json(json_data) # noqa F841 + + +def test_config_load_from_json_proper_base_path(): + config = Config.from_json(CONFIG_STR) + assert config.base_path == Path("~/ciq").expanduser() + + +def test_config_load_from_json_proper_kernels_dir(): + config = Config.from_json(CONFIG_STR) + assert config.kernels_dir == Path("~/ciq/kernels").expanduser() + + +def test_config_load_from_json_proper_images_source_dir(): + config = Config.from_json(CONFIG_STR) + assert config.images_source_dir == Path("~/ciq/default_test_images").expanduser() + + +def test_config_load_from_json_proper_images_dir(): + config = Config.from_json(CONFIG_STR) + assert config.images_dir == Path("~/ciq/tmp/virt-images").expanduser() + + +def test_config_load_from_json_proper_ssh_key(): + config = Config.from_json(CONFIG_STR) + assert config.ssh_key == Path("~/ciq/id_ed25519_generic.pub").expanduser() diff --git a/tests/kt/ktlib/test_kernels.py b/tests/kt/ktlib/test_kernels.py new file mode 100644 index 0000000..f502e12 --- /dev/null +++ b/tests/kt/ktlib/test_kernels.py @@ -0,0 +1,100 @@ +import pytest +from pathlib3x import Path + +from kt.ktlib.config import Config +from kt.ktlib.kernels import KernelsInfo + +common_repos = {"dist-git-tree-cbr": "dist-url", "kernel-src-tree": "src-url"} + +kernels = { + "kernel1": { + "src_tree_root": "kernel-src-tree", + "src_tree_branch": "src-branch", + "dist_git_root": "dist-git-tree-cbr", + "dist_git_branch": "dist-branch", + } +} + + +data = {"common_repos": common_repos, "kernels": kernels} + + +def test_kernels_from_empty_dict(): + data = {} + config = Config.from_str_dict(Config.DEFAULT) + + with pytest.raises(ValueError, match="Failed to process"): + kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config) # noqa F841 + + +def test_kernels_no_kernels_key(): + data = {"common_repos": {}} + config = Config.from_str_dict(Config.DEFAULT) + + with pytest.raises(ValueError, match="Failed to process"): + kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config) # noqa F841 + + +def test_kernels_invalid_kernel(): + data = { + "common_repos": {}, + "kernels": { + "kernel1": { + "src_tree_root": "src-tree-url", + "src_tree_branch": "src-branch", + "dist_git_root": "dist-git-url", + "dist_git_branch": "dist-branch", + } + }, + } + config = Config.from_str_dict(Config.DEFAULT) + + with pytest.raises(ValueError, match="not valid; it must be a reference to"): + kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config) # noqa F841 + + +def test_kernels_from_dict_check_len(): + config = Config.from_str_dict(Config.DEFAULT) + + kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config) + assert len(kernels_info.kernels) == 1 + + +def test_kernels_from_dict_check_name(): + config = Config.from_str_dict(Config.DEFAULT) + + kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config) + name = list(kernels_info.kernels.keys())[0] + assert name == "kernel1" + + +def test_kernels_from_dict_check_dist_branch(): + config = Config.from_str_dict(Config.DEFAULT) + + kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config) + kernel_info = list(kernels_info.kernels.values())[0] + assert kernel_info.dist_git_branch == "dist-branch" + + +def test_kernels_from_dict_check_src_branch(): + config = Config.from_str_dict(Config.DEFAULT) + + kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config) + kernel_info = list(kernels_info.kernels.values())[0] + assert kernel_info.src_tree_branch == "src-branch" + + +def test_kernels_from_dict_check_dist_root(): + config = Config.from_str_dict(Config.DEFAULT) + + kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config) + kernel_info = list(kernels_info.kernels.values())[0] + assert kernel_info.dist_git_root.folder == config.base_path / Path("dist-git-tree-cbr") + + +def test_kernels_from_dict_check_src_root(): + config = Config.from_str_dict(Config.DEFAULT) + + kernels_info = KernelsInfo.from_dict(data=data, private_data={}, config=config) + kernel_info = list(kernels_info.kernels.values())[0] + assert kernel_info.src_tree_root.folder == config.base_path / Path("kernel-src-tree")