build.sh

#!/bin/bash
# shellcheck disable=SC1117

# This script builds the kernel, kernel modules, device trees and boot scripts for the A20 linux system that we use.

set -eu

# Some helper log functions to improve visibility:

info_h1()
{
    echo -e "\n\033[1;36m${1}\033[0m"
}

info_h2()
{
    echo -e "\n\033[1;33m${1}\033[0m"
}

info_h3()
{
    echo -e "\033[1;37m${1}\033[0m"
}

info_err()
{
    echo -e "\n\033[1;31m${1}\033[0m"
}

export CROSS_COMPILE="${CROSS_COMPILE:-aarch64-linux-gnu-}"
cpu_cnt="$(nproc)"
export MAKEFLAGS="-j ${cpu_cnt}"
echo "Compiling with ${cpu_cnt} CPUs..."

ARCH="${ARCH:-arm64}"
UM_ARCH="imx8mm" # Empty string, or sun7i for R1, or imx6dl for R2, or imx8mm for Colorado

# common directory variablesS
SYSCONFDIR="${SYSCONFDIR:-/etc}"
SRC_DIR="$(pwd)"
BUILD_DIR_TEMPLATE="_build"
BUILD_DIR="${BUILD_DIR:-${SRC_DIR}/${BUILD_DIR_TEMPLATE}}"

# Debian package information
PACKAGE_NAME="${PACKAGE_NAME:-um-kernel}"
RELEASE_VERSION="${RELEASE_VERSION:-999.999.999}"

# Which kernel to build
LINUX_SRC_DIR="${SRC_DIR}/linux"

# Setup internal variables
KCONFIG="${SRC_DIR}/configs/sx8m_defconfig"
KERNEL_BUILD_DIR="${SRC_DIR}/_build/sx8m-linux"
KERNEL_IMAGE="uImage-sx8m"
DEBIAN_DIR="${BUILD_DIR}/debian"
BOOT_FILE_OUTPUT_DIR="${DEBIAN_DIR}/boot"

# Init RAM FS definitions
INITRAMFS_MODULES_REQUIRED="loop.ko leds-pca963x.ko"
INITRAMFS_SRC_DIR="${SRC_DIR}/initramfs"
INITRAMFS_DST_DIR="${KERNEL_BUILD_DIR}/initramfs"
INITRAMFS_MODULES_DIR="${KERNEL_BUILD_DIR}/initramfs/lib/modules"
INITRAMFS_SOURCE="${INITRAMFS_SOURCE:-initramfs/initramfs.lst}"
INITRAMFS_DEST="${INITRAMFS_DST_DIR}/$(basename "${INITRAMFS_SOURCE}")"

# We need freescale proprietary DMA drivers for the UART they are binary blobs in the Linux mainline
# They can be found in git://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git
PROPRIETARY_FIRMWARE_DIR="${SRC_DIR}/proprietary_firmware"
PROPRIETARY_FIRMWARE_OUTPUT_DIR="${DEBIAN_DIR}/lib/firmware"
PROPRIETARY_FIRMWARE_INITRAMFS_DIR="${INITRAMFS_DST_DIR}/lib/firmware"


BB_VERSION="1.36.1"
BB_URL="https://busybox.net/downloads/busybox-${BB_VERSION}.tar.bz2"
BB_BIN="busybox"
BB_PKG="${BUILD_DIR}/busybox-${BB_VERSION}.tar.bz2"
BB_DIR="${BUILD_DIR}/busybox-${BB_VERSION}"

DEPMOD="${DEPMOD:-/sbin/depmod}"

if [ ! -x "${DEPMOD}" ]; then
    DEPMOD="busybox depmod"

    if [ ! -x "${DEPMOD}" ]; then
        echo "No depmod binary available. Cannot continue."
        exit 1
    fi
fi

# Add the UM_ARCH to release version keeping a possible -dev on the most right side
if [[ ${RELEASE_VERSION} == *'-dev' ]]; then
    RELEASE_VERSION="${RELEASE_VERSION/-dev/-${UM_ARCH}-dev}"
else
    RELEASE_VERSION="${RELEASE_VERSION}-${UM_ARCH}"
fi;

##
# busybox_get() - Download and build the Busybox package
# param1:	Writable path where to store the Busybox binary
#
# Busybox is downloaded from the global variable ${BB_PKG}.
busybox_get()
{
    info_h2 "### Preparing Busybox... ###"
        
    DEST_DIR="${1}"

    if [ ! -d "${DEST_DIR}" ]; then
        info_err "No initramfs dir set to download busybox into."
        exit 1
    fi

    if [ ! -f "${BB_PKG}" ]; then
        info_h3 "\nDownloading Busybox tarbal from ${BB_URL} ...\n"
        mkdir -p "${BUILD_DIR}"
        if ! wget "${BB_URL}" -P "${BUILD_DIR}"; then
            info_err "Unable to download the busybox package '${BB_URL}'. Update the download URL."
            exit 1
        fi
    fi

    if [ ! -d "${BB_DIR}" ]; then
        info_h3 "Unpacking Busybox tarbal ${BB_PKG} ..."
        if ! tar xvjf "${BB_PKG}" -C "${BUILD_DIR}" > /dev/null 2>&1; then
            info_err "Unable to extract Busybox package '${BB_PKG}'."
            exit 1
        fi
    fi

    cp "${SRC_DIR}/configs/busybox_config" "${BB_DIR}/.config"

    info_h3 "\nCompiling Busybox...\n"
    ARCH="${ARCH}" CROSS_COMPILE="${CROSS_COMPILE}" make -C "${BB_DIR}"

    mv "${BB_DIR}/${BB_BIN}" "${DEST_DIR}/${BB_BIN}"
    cd "${SRC_DIR}"

    if [ ! -x "${DEST_DIR}/${BB_BIN}" ]; then
        info_err "Failed to get busybox."
        exit 1
    fi

    info_h3 "Finished preparing Busybox"
}

##
# initramfs_add_modules()
#
# In initramfs we can make drivers available by adding them to
# the 'INITRAMFS_MODULES_REQUIRED' variable. This function
# will check for dependencies in the installed modules 'modules.dep' file
# and add the requested modules and its dependencies to initramfs.
initramfs_add_modules()
{
    info_h2 "### Adding initramfs modules. ###"

    KERNEL_RELEASE=$(cat "${KERNEL_BUILD_DIR}/include/config/kernel.release")

    if [ ! -d "${DEBIAN_DIR}/lib" ] || [ -z "${KERNEL_RELEASE}" ] || \
        [ ! -f "${DEBIAN_DIR}/lib/modules/${KERNEL_RELEASE}/modules.dep" ]; then
        info_err "Error, no modules installed, cannot continue."
        exit 1
    fi

    if [ -d "${INITRAMFS_MODULES_DIR}" ] && [ -z "${INITRAMFS_MODULES_DIR##*/initramfs/lib/modules*}" ]; then
        rm -rf "${INITRAMFS_MODULES_DIR}"
    fi

    if [ -n "${INITRAMFS_MODULES_REQUIRED}" ]; then
        mkdir -p "${INITRAMFS_MODULES_DIR}/${KERNEL_RELEASE}"
        {
            echo -e "\n# kernel modules"
            echo "dir /lib/modules/ 0755 0 0"
            echo "dir /lib/modules/${KERNEL_RELEASE}/ 0755 0 0"
        } >> "${INITRAMFS_DEST}"

        MODULES_DIR="${DEBIAN_DIR}/lib/modules/${KERNEL_RELEASE}"
        INITRAMFS_MODULES="${INITRAMFS_MODULES_REQUIRED}"

        for module in ${INITRAMFS_MODULES_REQUIRED}; do
            dependencies="$(grep "${module}:" "${MODULES_DIR}/modules.dep" | sed -e "s|^.*:\s*||")"
            for dependency in ${dependencies}; do
                dep_module="$(basename "${dependency}")"
                info_h3 "Adding dependency: '${dep_module}' for module: '${module}'"
                if [ -n "${INITRAMFS_MODULES##*"${dep_module}"*}" ]; then
                    INITRAMFS_MODULES="${INITRAMFS_MODULES} ${dep_module}"
                fi
            done
        done
    fi

    for module in ${INITRAMFS_MODULES}; do
        if [ -z "$(find "${MODULES_DIR}" -name "${module}" -print -exec cp "{}" "${INITRAMFS_MODULES_DIR}/${KERNEL_RELEASE}" \;)" ]; then
            info_err "Error: kernel module: '${module}' not available."
            exit 1
        fi

        info_h3 "Adding kernel module: '${module}' to initrd."
        echo "file /lib/modules/${KERNEL_RELEASE}/${module} ${INITRAMFS_MODULES_DIR}/${KERNEL_RELEASE}/${module} 0755 0 0" >> "${INITRAMFS_DEST}"
    done

    if [ -n "${INITRAMFS_MODULES}" ] && ! ${DEPMOD} -ab "${INITRAMFS_DST_DIR}" "${KERNEL_RELEASE}"; then
        info_err "Failed to generate module dependencies."
        exit 1
    fi

    for moddep in "${INITRAMFS_MODULES_DIR}/${KERNEL_RELEASE}/modules."*; do
        if [ -f "${moddep}" ]; then
            moddep="$(basename "${moddep}")"
            echo "file /lib/modules/${KERNEL_RELEASE}/${moddep} ${INITRAMFS_MODULES_DIR}/${KERNEL_RELEASE}/${moddep} 0755 0 0" >> "${INITRAMFS_DEST}"
        fi
    done

    info_h3 "Finished adding initramfs modules."
}

##
# initramfs_prepare() - Prepare the initramfs tree
#
# To be able to create an initramfs in the temporary build directory, where
# the kernel expects these files due to 'INITRAMFS_SOURCE' being set, we need
# to copy the source initramfs files and put some expected binaries in place.
initramfs_prepare()
{
    info_h2 "### Preparing initramfs. ###"

    if [ -d "${INITRAMFS_DST_DIR}" ]; then
        rm -rf "${INITRAMFS_DST_DIR}"
    fi

    mkdir -p "${INITRAMFS_DST_DIR}"

    cp -a "${INITRAMFS_SRC_DIR}/"* "${INITRAMFS_DST_DIR}"

    busybox_get "${INITRAMFS_DST_DIR}"

    info_h3 "Finished preparing initramfs."
}

build_menuconfig()
{
    info_h1 "\nRunning Menuconfig for ${KCONFIG}\n"

    mkdir -p "${KERNEL_BUILD_DIR}"
    
    cp "${KCONFIG}" "${KERNEL_BUILD_DIR}"/.config
    ARCH="${ARCH}" make O="${KERNEL_BUILD_DIR}" -C "${LINUX_SRC_DIR}" menuconfig
    ARCH="${ARCH}" make O="${KERNEL_BUILD_DIR}" -C "${LINUX_SRC_DIR}" savedefconfig
    cp "${KERNEL_BUILD_DIR}"/defconfig "${KCONFIG}"
}

##
# kernel_build_command() - Wrapper function for Kernel build commands
#
# Wrap the argument into a Linux Kernel cross-compile command.
kernel_build_command()
{
    info_h2 "### Compiling the Kernel ${*} ###"
    if [ ! -d "${KERNEL_BUILD_DIR}" ]; then
        mkdir -p "${KERNEL_BUILD_DIR}"
    fi

    cp "${KCONFIG}" "${KERNEL_BUILD_DIR}"/.config
    ARCH="${ARCH}" CROSS_COMPILE="${CROSS_COMPILE}" make -C "${LINUX_SRC_DIR}" O="${KERNEL_BUILD_DIR}" olddefconfig
    ARCH="${ARCH}" CROSS_COMPILE="${CROSS_COMPILE}" make -C "${LINUX_SRC_DIR}" O="${KERNEL_BUILD_DIR}" "${@}"
    cd "${SRC_DIR}"
    info_h3 "Finished compiling the Kernel"
}

##
# kernel_build() - Build the Linux Kernel image
#
# Builds the Kernel
# Installs the modules in the build output lib directory
# Creates the 'initramfs'
# Creates a uImage binary in the build output boot directory.
kernel_build()
{
    info_h1 "##### Building Kernel. #####"

    # Prepare the initramfs 1st time
    initramfs_prepare

    # Copy the proprietary dma firmware before building the kernel so it endup in the initramfs
    copy_dma_firmware

    # Configure the kernel
    # The "kernel_build_command" can be used without arguments for test (the "all" target for
    # the make file), but for production it is better to split the Image and modules compilation,
    # so the compilation log is more organized about what is being compiled
    kernel_build_command Image
    kernel_build_command modules

    # Build the Kernel modules and generate dependency list
    kernel_modules_install

    # New that all modules have been build and the dependency file is properly generated,
    # we can add the required Kernel modules to initramfs
    initramfs_add_modules

    # Here we need to rebuild the kernel Image to include the updated initramfs with kernel modules
    kernel_build_command LOADADDR=0x40480000 Image

    # Install Kernel image

    if [ -d "${BOOT_FILE_OUTPUT_DIR}" ] && [ -z "${BOOT_FILE_OUTPUT_DIR##*_build*}" ]; then
        rm -r "${BOOT_FILE_OUTPUT_DIR}"
    fi
    mkdir -p "${BOOT_FILE_OUTPUT_DIR}"

    cp "${KERNEL_BUILD_DIR}/arch/arm64/boot/Image" "${BOOT_FILE_OUTPUT_DIR}/${KERNEL_IMAGE}"
    info_h3 "Finished building Kernel."
}

##
# kernel_modules_install() - Install the Kernel modules
#
# Generates a 'lib/modules/[Kernel version]' directory structure
# in the build output directory.
kernel_modules_install()
{
    info_h2 "##### Install Kernel modules. #####"

    KERNEL_RELEASE=$(cat "${KERNEL_BUILD_DIR}/include/config/kernel.release")

    if [ -z "${KERNEL_RELEASE}" ]; then
        info_err "Error, unable to get kernel release version, cannot continue."
        exit 1
    fi

    if [ -d "${DEBIAN_DIR}/lib/modules" ] && [ -z "${BOOT_FILE_OUTPUT_DIR##*_build*}" ]; then
        rm -r "${DEBIAN_DIR}/lib/modules"
    fi

    kernel_build_command INSTALL_MOD_PATH="${DEBIAN_DIR}" modules_install

    if ! "${DEPMOD}" -ab "${DEBIAN_DIR}" "${KERNEL_RELEASE}"; then
        info_err "Error, failed to generate module dependencies."
        exit 1
    fi

    info_h3 "Finished installing Kernel modules."
}

##
# dtb_build() - Compile product specific device-tree binary files
#
# In the U-Boot stage the boot script is executed, that in turn reads
# the machine article number from the I2C EEPROM. This article number
# is in Hexadecimal format. The boot-script will then load a device-tree
# with corresponding article number into memory.
#
# This function will parse a file called article.links and compile the device-tree
# binaries described above.
dtb_build()
{
    info_h1 "##### Building Device-trees. #####"

    if [ -d "${KERNEL_BUILD_DIR}/dtb" ]; then
        rm -rf "${KERNEL_BUILD_DIR}/dtb"
    fi

    mkdir -p "${KERNEL_BUILD_DIR}/dtb"

    if [ ! -d "${BOOT_FILE_OUTPUT_DIR}" ]; then
        mkdir -p "${BOOT_FILE_OUTPUT_DIR}"
    fi

    rm -rf "${BOOT_FILE_OUTPUT_DIR}/"*".dtb"

    # Build the device trees that we need
    for dts in "dts/"*".dts"; do
        dts="$(basename "${dts}")"
        dt="${dts%.dts}"
        info_h2 "\n### Building devicetree blob '${dt}' ###"
        info_h3 "Using version of DTC: $(dtc --version)"
        cpp -nostdinc -undef -D__DTS__ -x assembler-with-cpp \
            -I "${LINUX_SRC_DIR}/include" -I "${LINUX_SRC_DIR}/arch/${ARCH}/boot/dts" \
            -o "${KERNEL_BUILD_DIR}/dtb/.${dt}.dtb.tmp" "dts/${dts}"
        dtc -@ -I dts -o "${BOOT_FILE_OUTPUT_DIR}/${dt}.dtb" -O dtb "${KERNEL_BUILD_DIR}/dtb/.${dt}.dtb.tmp"
    done


    info_h3 "\nFinished building Device-trees.\n"
}

# We need this because the imx8m uart uses this.
# We can choose not to use it and configure it differently in the device-tree.
copy_dma_firmware()
{
    info_h2 "### Copying proprietary firmware. ###"

    # Ensure the output directory exists
    mkdir -p "${PROPRIETARY_FIRMWARE_OUTPUT_DIR}"
    mkdir -p "${PROPRIETARY_FIRMWARE_INITRAMFS_DIR}/imx/sdma"

    # Copy all directories and files under proprietary_firmware to the output directory
    rsync -av --exclude 'Readme.md' "${PROPRIETARY_FIRMWARE_DIR}/" "${PROPRIETARY_FIRMWARE_OUTPUT_DIR}/"

    # Copy specific proprietary firmware to the initramfs dir too
    cp "${PROPRIETARY_FIRMWARE_DIR}/imx/sdma/sdma-imx7d.bin" "${INITRAMFS_DST_DIR}"
    cp "${PROPRIETARY_FIRMWARE_DIR}/imx/sdma/sdma-imx6q.bin" "${INITRAMFS_DST_DIR}"

    info_h3 "Finished copying proprietary firmware."
}

create_debian_package()
{
    info_h1 "##### Building Debian package. #####"

    if [ ! -d "${BOOT_FILE_OUTPUT_DIR}" ]; then
        info_err "Error, boot directory not created, no boot files to package."
        exit 1
    fi

    if ! ls "${BOOT_FILE_OUTPUT_DIR}/uImage"* 1> /dev/null 2>&1; then
        info_err "Error, no Kernel binary installed, run 'kernel' build first."
        exit 1
    fi

    if [ ! -d "${DEBIAN_DIR}/lib" ] || [ ! -f "${DEBIAN_DIR}/lib/modules/${KERNEL_RELEASE}/modules.dep" ]; then
        info_err "Error, no modules installed, run 'kernel' build first."
        exit 1
    fi

    if ! ls "${BOOT_FILE_OUTPUT_DIR}/"*".dtb" 1> /dev/null 2>&1; then
        info_err "Error, no Kernel device-tree files installed, run 'dtbs' build first."
        exit 1
    fi

    if ! ls "${PROPRIETARY_FIRMWARE_OUTPUT_DIR}/imx/sdma/"*".bin" 1> /dev/null 2>&1; then
        info_err "Error, linux-firmware proprietary directory not found for DMA drivers."
        exit 1
    fi

    mkdir -p "${DEBIAN_DIR}/DEBIAN"
    cp "${SRC_DIR}"/debian/* "${DEBIAN_DIR}/DEBIAN/"
    sed -i -e 's|@ARCH@|'"${ARCH}"'|g' \
        -e 's|@PACKAGE_NAME@|'"${PACKAGE_NAME}"'|g' \
        -e 's|@RELEASE_VERSION@|'"${RELEASE_VERSION}"'|g' \
        "${DEBIAN_DIR}/DEBIAN/control"

    DEB_PACKAGE="${PACKAGE_NAME}_${RELEASE_VERSION}_${ARCH}.deb"

    # Build the Debian package
    fakeroot dpkg-deb --build "${DEBIAN_DIR}" "${BUILD_DIR}/${DEB_PACKAGE}"

    cp "${BUILD_DIR}/${DEB_PACKAGE}" "${SRC_DIR}"

    info_h3 "Finished building Debian package."
    info_h3 "To check the contents of the Debian package run 'dpkg-deb -c um-kernel*.deb'"
}

usage()
{
    echo ""
    echo "This is the build script for Linux Kernel related build artifacts and configure the Kernel."
    echo ""
    echo "  Usage: ${0} [kernel|dtbs|menuconfig|oldconfig|deb|clean]"
    echo "  For Kernel config modification use: ${0} menuconfig"
    echo ""
    echo "  -h Print this help text and exit"
    echo ""
    echo "  By default the script can be executed with 'no' arguments, all required artifacts"
    echo "  will be build resulting in a 'um-kernel-[RELEASE_VERSION].deb' package."
    echo "  Either one of the above optional arguments can be passed to the script to build that"
    echo "  specific artifact."
    echo ""
    echo "  The package release version can be passed by passing 'RELEASE_VERSION' through the run environment."
}

while getopts ":h" options; do
    case "${options}" in
    h)
        usage
        exit 0
        ;;
    :)
        echo "Option -${OPTARG} requires an argument."
        exit 1
        ;;
    ?)
        echo "Invalid option: -${OPTARG}"
        exit 1
        ;;
    esac
done
shift "$((OPTIND - 1))"


if [ "${#}" -gt 1 ]; then
    echo "Too many arguments."
    usage
    exit 1
fi

if [ "${#}" -eq 0 ]; then
    kernel_build
    dtb_build
    create_debian_package
    exit 0
fi

case "${1-}" in
    deb)
        kernel_build
        dtb_build
        create_debian_package
        ;;
    dtbs)
        dtb_build
        ;;
    kernel)
        kernel_build
        ;;
    menuconfig)
        build_menuconfig
        ;;
    clean)
        if [ -d "${BUILD_DIR}" ] && [ -z "${BUILD_DIR##*_build*}" ]; then
            rm -rf "${BUILD_DIR}"
        fi
        echo "Cleaned up '${BUILD_DIR}'."
        if [ -d "${BB_DIR}" ] && [ -z "${BB_DIR##*busybox-*}" ]; then
            rm -rf "${BB_DIR}"
        fi
        echo "Cleaned up '${BB_DIR}'."
        if [ -f "${BB_PKG}" ]; then
            unlink "${BB_PKG}"
        fi
        echo "Removed '${BB_PKG}'."
        exit 0
        ;;
    *)
        echo "Error, unknown build option given"
        usage
        exit 1
        ;;
esac

exit 0