Merge branch 'devel' of github.com:steviehs/CustomPiOS into devel

Author: steviehs

.github/workflows/docker-build.yml

@@ -1,48 +1,55 @@
 name: ci
-
 on:
   push:
     branches:
       - "master"
       - "devel"
       - "docker-github-actions"
-
+      - "release/v1"
+      - "beta"
+    tags:
+      - "*"
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
       -
-        name: Checkout
-        uses: actions/checkout@v2
+        name: 📥 Checkout
+        uses: actions/checkout@v4
       -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        name: 🖥️ Set up QEMU
+        uses: docker/setup-qemu-action@v3
       -
-        name: Docker meta tag
-        id: docker_meta
-        uses: docker/metadata-action@v3.5.0
+        name: 🏷️ Docker meta tag
+        id: meta
+        uses: docker/metadata-action@v5
         with:
-          images: guysoft/custompios
-          tag-sha: true
+          images: |
+            ghcr.io/${{ github.repository_owner }}/custompios
+          tags: |
+            type=ref,event=branch
+            type=ref,event=tag
+            type=sha
       -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        name: 🛠️ Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
       -
-        name: Login to DockerHub
-        uses: docker/login-action@v1
+        name: 🔑 Login to GitHub Container Registry
+        uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       -
-        name: Build and push
+        name: 🏗️ Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: src
           file: src/Dockerfile
           platforms: linux/amd64,linux/arm64,linux/arm/v7
           push: true
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
           secrets: |
-            GIT_AUTH_TOKEN=${{ secrets.MY_GITHUB_TOKEN }}
+            github_token=${{ secrets.GITHUB_TOKEN }}

.github/workflows/tests.yaml

@@ -0,0 +1,37 @@
+name: Run Tests via Makefile
+
+on:
+  push:
+    branches:
+      - "master"
+      - "devel"
+      - "docker-github-actions"
+      - "release/v1"
+      - "feature/unit-testing"
+      - "beta"
+    tags:
+      - "*"
+
+jobs:
+  test:
+    name: Run Makefile Tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # - name: Install dependencies
+      #   run: |
+      #     sudo apt-get update
+      #     sudo apt-get install -y qemu-user-static binfmt-support
+
+      - name: Run Makefile
+        run: make test
+
+      - name: Archive test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results
+          path: tests/
+          retention-days: 60

.gitignore

@@ -1,3 +1,4 @@
 src/config.local
 src/image/*.zip
+src/image/*.xz
 **/key.json

Makefile

@@ -0,0 +1,57 @@
+# Makefile for running shell script tests
+
+# Shell to use
+SHELL := /bin/bash
+
+# Find all test files
+TEST_DIR := tests
+TEST_FILES := $(wildcard $(TEST_DIR)/test_*.sh)
+
+# Colors for output
+BLUE := \033[1;34m
+GREEN := \033[1;32m
+RED := \033[1;31m
+NC := \033[0m # No Color
+
+.PHONY: all test clean help
+
+# Default target
+all: test
+
+# Help message
+help:
+	@echo "Available targets:"
+	@echo "  make test  - Run all tests"
+	@echo "  make clean - Clean up temporary files"
+	@echo "  make help  - Show this help message"
+
+# Run all tests
+test:
+	@echo -e "$(BLUE)Running tests...$(NC)"
+	@echo "═══════════════════════════════════════"
+	@success=true; \
+	for test in $(TEST_FILES); do \
+		echo -e "$(BLUE)Running $$test...$(NC)"; \
+		if chmod +x $$test && ./$$test; then \
+			echo -e "$(GREEN)✓ $$test passed$(NC)"; \
+		else \
+			echo -e "$(RED)✗ $$test failed$(NC)"; \
+			success=false; \
+		fi; \
+		echo "───────────────────────────────────────"; \
+	done; \
+	echo ""; \
+	if $$success; then \
+		echo -e "$(GREEN)All tests passed successfully!$(NC)"; \
+		exit 0; \
+	else \
+		echo -e "$(RED)Some tests failed!$(NC)"; \
+		exit 1; \
+	fi
+
+# Clean up any temporary files (if needed)
+clean:
+	@echo -e "$(BLUE)Cleaning up...$(NC)"
+	@find $(TEST_DIR) -type f -name "*.tmp" -delete
+	@find $(TEST_DIR) -type f -name "*.log" -delete
+	@echo -e "$(GREEN)Cleanup complete$(NC)"

README.rst

@@ -5,6 +5,8 @@ CustomPiOS
 .. :scale: 50 %
 .. :alt: CustomPiOS logo
 
+.. class:: center
+
 A `Raspberry Pi <http://www.raspberrypi.org/>`_ and other ARM devices distribution builder. CustomPiOS opens an already existing image, modifies it and repackages the image ready to ship.
 
 This repository contains the source script to generate a distribution out of an existing `Raspbian <http://www.raspbian.org/>`_ distro image, or Armbian devices.
@@ -50,20 +52,23 @@ Developing
 Requirements
 ~~~~~~~~~~~~
 
-#. `qemu-arm-static <http://packages.debian.org/sid/qemu-user-static>`_
+#. `qemu-arm-static <http://packages.debian.org/sid/qemu-user-static>`_ or gentoo qemu with static USE
 #. Downloaded `Raspbian <http://www.raspbian.org/>`_ image.
 #. root privileges for chroot
 #. Bash
+#. jq
 #. git
 #. realpath
+#. file
 #. sudo (the script itself calls it, running as root without sudo won't work)
 #. p7zip-full
 #. Python 3.2+
+#. GitPython
 
 Known to work building configurations
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 1. Using the `CustomPiOS docker image <https://hub.docker.com/r/guysoft/custompios>`_
-2. Linux (Ubuntu / Debian etc)
+2. Linux (Ubuntu / Debian / Gentoo etc)
 3. OS X -  `See this thread for information <https://github.com/guysoft/OctoPi/issues/388#issuecomment-316327106>`_
 
 
@@ -151,12 +156,27 @@ Usage
 #. Run ``src/build`` as root.
 #. The final image will be created at the ``src/workspace``
 
+Testing
+-------
+
+You can run unit tests to test CustomPiOS if you have changed it by running::
+
+    make test
+
+You can add more tests to the ``tests`` folder.
+
+Community
+--------
+|discord|
+
+.. |discord| image:: https://img.shields.io/discord/1119337877734699018?label=discord&logo=discord&logoColor=white
+   :target: https://discord.gg/rK72VZVt
 
 List of Distributions using CustomPiOS
 --------------------------------------
 
 * `OctoPi <https://octopi.octoprint.org/>`_ - The ready-to-go Raspberry Pi image with OctoPrint
-* `FarmPi <https://farmpi.kevenaar.name/>`_ - An Ubuntu ARM 64bit Raspbery Pi image running `OctoFarm <https://octofarm.net/>`_
+* `FarmPi <https://farmpi.kevenaar.name/>`_ - An Ubuntu ARM 64bit Raspberry Pi image running `OctoFarm <https://octofarm.net/>`_
 * `FullPageOS <https://github.com/guysoft/FullPageOS>`_ - A Raspberry Pi distro to display a full page browser on boot
 * `Zynthian <http://zynthian.org/>`_ - Open Synth Platform
 * `ElectricSheepPi <https://github.com/guysoft/ElectricSheepPi>`_ - A Raspberry Pi distribution to run Electric Sheep digital art
@@ -167,9 +187,13 @@ List of Distributions using CustomPiOS
 * `HotSpotOS <https://github.com/guysoft/HostSpotOS>`_ - Makes a Raspberry Pi start a hotspot, if no wifi was found to connect to
 * `MtigOS <https://github.com/guysoft/MtigOS>`_ - Distro that lets you receive, store and graph sensor information from ESP8266 chips. It uses and MTIG stack: Mosquitto, Telegraf, InfluxDB and Grafana which are all pre-configured to work together. They automatically update using Docker.
 * `Tilti-Pi <https://github.com/myoung34/tilty-pi>`_ - Distro that lets you submit BLE data for the  `tilt hydrometer <https://tilthydrometer.com/>`_ via the `tilty <https://github.com/myoung34/tilty>`_ package and a built in `dashboard <https://github.com/myoung34/tilty-dashboard>`_
-* `MainsailOS <https://github.com/raymondh2/MainsailOS>`_ - Distro that packages the `Mainsail <https://github.com/meteyou/mainsail>`_ web UI, the `Moonraker <https://github.com/Arksine/moonraker>`_ API, and the `Klipper <https://github.com/KevinOConnor/klipper>`_ 3D printer firmware in an easy to package.
+* `MainsailOS <https://github.com/mainsail-crew/mainsailos>`_ - Distro that packages the `Mainsail <https://github.com/mainsail-crew/mainsail>`_ web UI, the `Moonraker <https://github.com/Arksine/moonraker>`_ API, and the `Klipper <https://github.com/klipper3d/klipper>`_ 3D printer firmware in an easy to package.
 * `UbuntuDockerPi <https://github.com/guysoft/UbuntuDockerPi>`_ - Distro ships with Ubuntu ARM 64bit Docker and docker-compose ready to build stuff for arm64v8/aarch64 or host whatever you like.
 * `FluiddPi <https://github.com/cadriel/fluiddpi>`_ - Distro that packages `Fluidd <https://github.com/cadriel/fluidd>`_, `Moonraker <https://github.com/Arksine/moonraker>`_, and `Klipper <https://github.com/KevinOConnor/klipper>`_ into the ultimate 3D printer firmware package.
 * `My Naturewatch Camera <https://github.com/interactionresearchstudio/NaturewatchCameraServer>`_ - A Python / OpenCV camera server to stream Pi camera content to a remote client through a website.
+* `PiFireOS <https://github.com/calonmerc/PiFireOS>`_ - Distro for pellet grill/smoker control, running `PiFire <https://nebhead.github.io/PiFire>`_.
+* `MonsterPi <https://docs.fdm-monster.net/guides/monsterpi>`_ - An Ubuntu ARM 64bit Raspberry Pi image running `FDM Monster <https://fdm-monster.net/>`_. This 3D Print server will help you connect 200+ OctoPrints together while providing a strong, professional workflow.
+* `AllStarLink <https://allstarlink.org>`_ - AllStarLink is a network of Amateur Radio repeaters, remote base stations and hot spots accessible to each other via Voice over Internet Protocol. The ASL3 Pi Appliance uses CustomPiOS
+* `LEDPotato <https://github.com/guysoft/LEDPotato>`_ - A Le Potato AML-S905X-CC distrubtion that controls ws2812 LEDs over uart right on boot 
 
 Code contribution would be appreciated!

pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "custompios"
+version = "2.0.0"
+description = "A Raspberry Pi and other ARM devices distribution builder. CustomPiOS opens an already existing image, modifies it and repackages the image ready to ship."
+authors = ["Guy Sheffer <guysoft@gmail.com>"]
+license = "GPLv3"
+readme = "README.rst"
+packages = [
+#     { include = "src/*" },
+    { include = "custompios_core", from = "src" }
+    ]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+GitPython = "^3.1.41"
+
+[tool.poetry.group.dev.dependencies]
+types-PyYAML = "^6.0.12.12"
+
+[tool.poetry.scripts]
+custompios_build = 'custompios_core.multi_build:main'
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

src/Dockerfile

@@ -1,22 +1,37 @@
-FROM debian:buster
+FROM debian:bookworm-backports
 
 MAINTAINER Guy Sheffer <guysoft at gmail dot com>
 
-RUN set -x \
-    && apt-get update && apt-get install -y \
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        sudo \
         build-essential \
+        sudo \
         curl \
         git \
         wget \
         p7zip-full \
         python3 \
+        python3-distutils \
+        python3-dev \
+        python3-git \
+        python3-yaml \
         binfmt-support \
-        qemu \
+        qemu-system \
+        qemu-user \
         qemu-user-static \
         sudo \
+        file \
+        fdisk \
+        btrfs-progs \
+        jq \
         zip \
+        xz-utils \
         lsof \
-    && rm -rf /var/lib/apt/lists/*
+        f2fs-tools \
+  && rm -rf /var/lib/apt/lists/* \
+  && apt -qyy clean
 
 RUN ln -s /CustomPiOS/nightly_build_scripts/custompios_nightly_build /usr/bin/build
 RUN ln -s /usr/bin/python3 /usr/bin/python

src/base_image_downloader_wrapper.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set +x
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "$DIR/argparse.bash" || exit 1
+argparse "$@" <<EOF || exit 1
+
+parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder")
+parser.add_argument('-s', '--sha256', action='store_true', help='Create a sha256 hash for the .img file in .sha256')
+EOF
+
+if [ -z "${CUSTOM_PI_OS_PATH}" ];then
+    echo "Error: you must have \${CUSTOM_PI_OS_PATH} set"
+    exit 1
+fi
+
+# source "${DIST_PATH}/config"
+source "${CUSTOM_PI_OS_PATH}/config" "${WORKSPACE_SUFFIX}"
+
+python3 ${CUSTOM_PI_OS_PATH}/custompios_core/base_image_downloader.py "${WORKSPACE_SUFFIX}"
+

src/build

@@ -10,15 +10,19 @@ define(){ IFS='\n' read -r -d '' ${1} || true; }
 
 define SCRIPT <<'EOF'
 BUILD_SCRIPT_PATH=$(dirname $(realpath -s $BASH_SOURCE))
+export EXTRA_BOARD_CONFIG=$(mktemp)
+${BUILD_SCRIPT_PATH}/custompios_core/generate_board_config.py "${EXTRA_BOARD_CONFIG}"
+echo "Temp source file: ${EXTRA_BOARD_CONFIG}"
+
 source ${BUILD_SCRIPT_PATH}/common.sh
 install_cleanup_trap
 
 CUSTOM_OS_PATH=$(dirname $(realpath -s $0))
 
-source ${CUSTOM_PI_OS_PATH}/config ${@}
+source ${CUSTOM_PI_OS_PATH}/config "${1}" "${EXTRA_BOARD_CONFIG}" ${@}
 ${CUSTOM_PI_OS_PATH}/config_sanity
 
-[ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios
+[ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios ${@}
 EOF
 
 if [ "$LOG" != "no" ]; then