From f2c80ad6c4cfe237b4fa6e50b11d87d42cd420b9 Mon Sep 17 00:00:00 2001 From: Peter Dragun Date: Tue, 1 Jul 2025 14:57:07 +0200 Subject: [PATCH 1/4] ci: Setup pre-commit hooks: ruff, mypy, codespell, yamlfix --- .github/workflows/dangerjs.yml | 26 +++++++++++++++ .github/workflows/issue_comment.yml | 2 +- .github/workflows/new_issues.yml | 2 +- .github/workflows/new_prs.yml | 4 +-- .pre-commit-config.yaml | 49 +++++++++++++++++------------ README.md | 2 +- pyproject.toml | 22 +++++++++++++ 7 files changed, 82 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/dangerjs.yml create mode 100644 pyproject.toml diff --git a/.github/workflows/dangerjs.yml b/.github/workflows/dangerjs.yml new file mode 100644 index 0000000..6b03259 --- /dev/null +++ b/.github/workflows/dangerjs.yml @@ -0,0 +1,26 @@ +name: DangerJS Pull Request linter + +on: + pull_request_target: + types: [opened, edited, reopened, synchronize] + +permissions: + pull-requests: write + contents: write + +jobs: + pull-request-style-linter: + runs-on: ubuntu-latest + steps: + - name: Check out PR head + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: DangerJS pull request linter + uses: espressif/shared-github-dangerjs@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + instructions-contributions-file: CONTRIBUTING.rst + instructions-gitlab-mirror: 'true' diff --git a/.github/workflows/issue_comment.yml b/.github/workflows/issue_comment.yml index 840cff5..c1388ce 100644 --- a/.github/workflows/issue_comment.yml +++ b/.github/workflows/issue_comment.yml @@ -16,6 +16,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} JIRA_PASS: ${{ secrets.JIRA_PASS }} JIRA_PROJECT: IDFGH - JIRA_COMPONENT: GitHub + JIRA_COMPONENT: tools JIRA_URL: ${{ secrets.JIRA_URL }} JIRA_USER: ${{ secrets.JIRA_USER }} diff --git a/.github/workflows/new_issues.yml b/.github/workflows/new_issues.yml index b110599..6507b58 100644 --- a/.github/workflows/new_issues.yml +++ b/.github/workflows/new_issues.yml @@ -16,6 +16,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} JIRA_PASS: ${{ secrets.JIRA_PASS }} JIRA_PROJECT: IDFGH - JIRA_COMPONENT: GitHub + JIRA_COMPONENT: tools JIRA_URL: ${{ secrets.JIRA_URL }} JIRA_USER: ${{ secrets.JIRA_USER }} diff --git a/.github/workflows/new_prs.yml b/.github/workflows/new_prs.yml index 4c80fde..cd328e3 100644 --- a/.github/workflows/new_prs.yml +++ b/.github/workflows/new_prs.yml @@ -4,7 +4,7 @@ name: Sync remain PRs to Jira # Note that, PRs can also get synced when new PR comment is created on: schedule: - - cron: "0 * * * *" + - cron: 0 * * * * jobs: sync_prs_to_jira: @@ -21,6 +21,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} JIRA_PASS: ${{ secrets.JIRA_PASS }} JIRA_PROJECT: IDFGH - JIRA_COMPONENT: GitHub + JIRA_COMPONENT: tools JIRA_URL: ${{ secrets.JIRA_URL }} JIRA_USER: ${{ secrets.JIRA_USER }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 263280b..ea2ce71 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,40 +1,49 @@ -# See https://pre-commit.com for more information -# See https://pre-commit.com/hooks.html for more hooks ---- minimum_pre_commit_version: 3.3.0 -default_install_hook_types: [pre-commit,commit-msg] +default_install_hook_types: [pre-commit, commit-msg] +default_stages: [pre-commit] + +ci: + autofix_prs: false + autoupdate_commit_msg: 'ci: Bump pre-commit hooks' + autoupdate_schedule: quarterly repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-executables-have-shebangs - id: mixed-line-ending - args: ['-f=lf'] - - id: double-quote-string-fixer - - - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - args: ['--config=.flake8'] + args: [-f=lf] - - repo: https://github.com/pycqa/isort - rev: 5.11.5 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.2 hooks: - - id: isort - name: isort (python) + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.1.1' # Use the sha / tag you want to point at + rev: v1.16.1 hooks: - - id: mypy + - id: mypy additional_dependencies: [] - repo: https://github.com/espressif/conventional-precommit-linter - rev: v1.2.0 + rev: v1.10.0 hooks: - id: conventional-precommit-linter stages: [commit-msg] + + - repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + additional_dependencies: + - tomli + + - repo: https://github.com/lyz-code/yamlfix/ + rev: 1.17.0 + hooks: + - id: yamlfix diff --git a/README.md b/README.md index 07699d3..d43c795 100644 --- a/README.md +++ b/README.md @@ -1 +1 @@ -# python-binary-action \ No newline at end of file +# python-binary-action diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0612870 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[tool.mypy] + disallow_incomplete_defs = true + disallow_untyped_defs = true + ignore_missing_imports = true + python_version = "3.13" + warn_no_return = true + warn_return_any = true + +[tool.ruff] + line-length = 120 + target-version = "py313" + +[tool.ruff.format] + quote-style = "double" + indent-style = "space" + docstring-code-format = true + +[tool.yamlfix] + whitelines = 1 + section_whitelines = 1 + sequence_style = "keep_style" + explicit_start = false From d26460617e80b5b36569831991c0a7e5d75e0eb3 Mon Sep 17 00:00:00 2001 From: Peter Dragun Date: Tue, 1 Jul 2025 15:11:58 +0200 Subject: [PATCH 2/4] feat: Add a initial version of the action --- README.md | 279 ++++++++++++++++++++++++++++++++++- action.yml | 315 ++++++++++++++++++++++++++++++++++++++++ process_include_dirs.py | 99 +++++++++++++ 3 files changed, 692 insertions(+), 1 deletion(-) create mode 100644 action.yml create mode 100644 process_include_dirs.py diff --git a/README.md b/README.md index d43c795..1904fd6 100644 --- a/README.md +++ b/README.md @@ -1 +1,278 @@ -# python-binary-action +[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/espressif/python-binary-action/master.svg)](https://results.pre-commit.ci/latest/github/espressif/python-binary-action/master) + +# Python Binary Build Action + +A GitHub Action for building Python applications into standalone executables using `PyInstaller` across multiple architectures and platforms. + +## Overview + +This action automates the process of creating standalone executables from Python applications using `PyInstaller`. It supports multiple target platforms including Windows, macOS, and Linux (both x86_64 and ARM architectures). The action handles platform-specific configurations, dependency installation, and binary verification automatically. + +### Motivation + +This action provides several key benefits: + +**Centralized Build Logic**: Eliminates code duplication across repositories by providing a single, centrally maintained build solution. This ensures consistent build processes and easier maintenance. + +**Reduced Antivirus False Positives**: Antivirus software often flags PyInstaller-generated executables as suspicious. By centrally controlling `PyInstaller` versions and configurations, this action helps minimize false positive reports through tested and optimized settings. + +**User Accessibility**: Many users prefer standalone executables over Python scripts, as they eliminate the need to install Python and manage dependencies. This is particularly valuable for: + +- Users unfamiliar with Python installation and package management +- Legacy systems with unsupported Python versions +- Deployment scenarios requiring minimal dependencies + +## Supported Platforms + +- **Windows**: `windows-amd64` +- **Linux**: `linux-amd64`, `linux-armv7`, `linux-aarch64` +- **macOS**: `macos-amd64`, `macos-arm64` + +## Features + +- **Multi-architecture support**: Builds for x86_64 and ARM architectures +- **Cross-platform compatibility**: Supports Windows, macOS, and Linux +- **Automatic dependency handling**: Installs Python dependencies and system packages +- **Flexible data file inclusion**: Supports per-script configuration with wildcard support +- **Executable verification**: Tests built executables to ensure they run correctly +- **Windows icon support**: Allows custom icons for Windows executables +- **Flexible `PyInstaller` configuration**: Supports additional `PyInstaller` arguments +- **ARM cross-compilation**: Uses Public Preview runners for aarch64 and Docker containers with Qemu for ARMv7 architecture builds +- **GLIBC 2.31+ support**: Uses older Linux images in Docker for best compatibility with Linux targets + +## Usage Examples + +For more detailed explanation of input variables and advanced use cases, please see the [Inputs](#inputs) section below. + +### Basic Usage + +```yaml +- name: Build Python executable + uses: espressif/python-binary-action@master + with: + scripts: 'main.py' + output-dir: './dist' + target-platform: 'linux-amd64' +``` + +### Multi-File Build with Data Files + +Sometimes your Python script might require additional non-Python files, like assets (images), JSON or YAML files. These can be included for either all scripts or you can define files per script. + +For example with the following project structure: + +```txt +my_script/ +src/ +├── assets/ +| └── image.svg +├── config/ +| └── config.json +├── app.py +├── main.py +├── icon.ico +├── pyproject.toml +└── README.md +``` + +There is a config file that is used in both `app.py` and `main.py`, but only `app.py` requires images. The action can look something like this: + +```yaml +- name: Build multiple executables + uses: espressif/python-binary-action@master + with: + # Mandatory args + scripts: 'main.py app.py' + output-dir: './binaries' + target-platform: 'windows-amd64' + # Optional args; non-python files to include + include-data-dirs: | + { + "app.py": ["./assets"], + "*": ["./config"] + } + icon-file: './icon.ico' # Icon for Windows executable +``` + +There are two options how to define data files to be included for all scripts: + +1. With wildcard `*` + +```json +{ + "*": ["./config"] +} +``` + +2. Simple list + +```json +["./config"] +``` + +Both options are equivalent, but the wildcard allows you to define additional data files that are specific for one script. As can be seen in the above example for `app.py`. + +### Custom Executable Names + +Sometimes it might be useful to have a control over the name of build binary (executable). For example if we have a following structure of the project: + +```txt +my_script/ +├── src/ +│ ├── __init__.py +│ └── __main__.py +├── pyproject.toml +└── README.md +``` + +Building the project with default configuration will result in script name `__main__.py`, which is probably not desirable. To solve this issue, we can pass optional argument `script-name` that will be used as basename for build binaries (executables). + +```yaml +- name: Build with custom names + uses: espressif/python-binary-action@master + with: + scripts: 'src/__main__.py' + script-name: 'my_script' + output-dir: './dist' + target-platform: 'linux-amd64' +``` + +### ARM Architecture Build + +```yaml +- name: Build for ARMv7 + uses: espressif/python-binary-action@master + with: + scripts: 'main.py' + output-dir: './arm-binaries' + target-platform: 'linux-armv7' + additional-arm-packages: 'openssl libffi-dev libffi7 libssl-dev' + python-version: '3.11' +``` + +### Custom PyInstaller Configuration + +```yaml +- name: Build with custom options + uses: espressif/python-binary-action@master + with: + scripts: 'app.py' + output-dir: './dist' + target-platform: 'macos-arm64' + additional-args: '--hidden-import=requests --hidden-import=urllib3 --strip' + pyinstaller-version: '6.3.0' + test-command-args: '--version' +``` + +### Complete Workflow + +Here you can see a simplified version of workflow used in [esptool](https://github.com/espressif/esptool/) repository: + +```yaml +name: Build Executables + +on: [push, pull_request] + +jobs: + build: + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false # Avoid failure of all action in case of one of them fails + # Define platform matrix to build on multiple platforms at the same time + matrix: + platform: [windows-amd64, linux-amd64, macos-arm64, linux-aarch64] + include: + - platform: windows-amd64 + runner: windows-latest + - platform: linux-amd64 + runner: ubuntu-latest + - platform: macos-arm64 + runner: macos-latest + - platform: linux-aarch64 + runner: ubuntu-24.04-arm + + env: + # Used for additional data to be included in executables + # Env variable is only for action simplification + STUBS_DIR: ./esptool/targets/stub_flasher/ + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Build executable + uses: espressif/python-binary-action@master + with: + # Required options + scripts: 'esptool.py' # Building from 'esptool.py' file + output-dir: './${{ matrix.platform }}' + target-platform: ${{ matrix.platform }} + # Optional args; non-python files that will be added to build executable + # We want to include the content of subdirectories `1` and `2` of the directory stored in environment variable `STUBS_DIR`. + include-data-dirs: | + { + "esptool.py": [ + "${{ env.STUBS_DIR }}1", + "${{ env.STUBS_DIR }}2", + ] + } + + - name: Add license and readme + shell: bash + run: cp LICENSE README.md ./${{ matrix.platform }} + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: executable-${{ matrix.platform }} + path: ./${{ matrix.platform }} +``` + +## Inputs + +### Required Inputs + +| Input | Description | Example | +|-------------------|-------------------------------------------------|----------------------------| +| `scripts` | Space-separated list of Python scripts to build | `"esptool.py espefuse.py"` | +| `output-dir` | Output directory for built executables | `"./dist-linux-amd64"` | +| `target-platform` | Target platform for the build | `"linux-amd64"` | + +### Optional Inputs + +| Input | Description | Default | Example | +|---------------------------|-------------------------------------------|---------------------------------------------|----------------------------------------------| +| `script-name` | Custom names for the output executables. Must provide exactly one name per script in the same order. (On Windows `.exe` suffix will be added to each name) | `""` | `"foo bar"` | +| `include-data-dirs` | Mapping script names to data directories to include. Supports wildcards (*). | `[]` | `{"main.py": ["./data"], "*": ["./common"]}` | +| `icon-file` | Path to icon file (Windows only) | `""` | `"./icon.ico"` | +| `python-version` | Python version to use for building | `"3.13"` | `"3.12"` | +| `pyinstaller-version` | PyInstaller version to install | `6.11.1` | `""` (use latest) | +| `additional-args` | Additional PyInstaller arguments | `""` | `"--hidden-import=module"` | +| `pip-extra-index-url` | Extra pip index URL | `https://dl.espressif.com/pypi` | `""` | +| `install-deps-command` | Command to install project dependencies | `"pip install --user --prefer-binary -e ."` | `"pip install -r requirements.txt"` | +| `additional-arm-packages` | ARMv7 ONLY: Additional system packages | `""` | `"openssl libffi-dev"` | +| `test-command-args` | Command arguments to test executables | `"--help"` | `"--version"` | + +> [!IMPORTANT] +> Be careful when changing `pyinstaller-version` as it might lead to increased false positives with anti-virus software. It is recommended to check your executables with antivirus software such as [Virustotal](https://www.virustotal.com/gui/home/upload). + +## Outputs + +| Output | Description | +|------------------------|----------------------------------------------------------------| +| `executable-extension` | File extension of built executables (e.g., `.exe` for Windows) | +| `build-success` | Whether the build was successful (`true`/`false`) | + +## Notes + +- For 32-bit ARM architecture (linux-armv7), the action uses Docker containers to provide the necessary build environment +- For 64-bit ARM architecture please use the GitHub provided runners, e.g. `ubuntu-24.04-arm`. Please note that this is still in public preview so there might be some changes to images. For more details see [available runners](https://docs.github.com/en/actions/how-tos/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources). +- Windows builds automatically include `.exe` extensions +- The action automatically tests built executables using the specified test command arguments +- System packages for ARMv7 builds can be customized using the `additional-arm-packages` input. For other systems, this can be done before running this action. +- It is recommended to add `fail-fast: false` to your matrix strategy to prevent one platform failure from stopping all builds diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..ccd1b4a --- /dev/null +++ b/action.yml @@ -0,0 +1,315 @@ +name: Python Binary Build Action +description: Build Python applications into standalone executables for multiple architectures + +inputs: + scripts: + description: Space-separated list of Python scripts to build + required: true + output-dir: + description: Output directory for built executables + required: true + include-data-dirs: + description: 'JSON object mapping script names to data directories to include. + Supports wildcards (*) and exact matches. Format: {"script.py": ["./data"], + "*": ["./common"]}' + required: false + default: '[]' + script-name: + description: Space-separated list of output names for executables. Must match + the length of `scripts` variable. If not provided, the script name will be used. + required: false + default: '' + icon-file: + description: Path to icon file (Windows only) + required: false + default: '' + target-platform: + description: 'Target platform: windows-amd64, linux-amd64, macos-amd64, macos-arm64, + linux-armv7, linux-aarch64' + required: true + python-version: + description: Python version to use for building. Make sure that the version is + available on all runners. + required: false + default: '3.13' + pyinstaller-version: + description: PyInstaller version to install. Default is 6.11.1 because it has + the lowest false positive rate with antivirus. For latest version, use empty string. + required: false + default: 6.11.1 + additional-args: + description: Additional PyInstaller arguments + required: false + default: '' + pip-extra-index-url: + description: Extra pip index URL + required: false + default: https://dl.espressif.com/pypi + install-deps-command: + description: Command to install project dependencies. Command will be executed + like `python -m {install-deps-command}` + required: false + default: pip install --user --prefer-binary -e . + additional-arm-packages: + description: 'ARMv7 ONLY: Additional system packages to install (space-separated). + e.g. for cryptography: openssl libffi-dev libffi7 libssl-dev' + required: false + default: '' + test-command-args: + description: Command arguments to test binaries (e.g. "--help") + required: false + default: --help + +outputs: + executable-extension: + description: File extension of built executables + value: ${{ steps.setup-platform.outputs.exe-extension }} + build-success: + description: Whether the build was successful + value: ${{ steps.build.outputs.success }} + +runs: + using: composite + steps: + - name: Setup platform variables + id: setup-platform + shell: bash + run: | + case "${{ inputs.target-platform }}" in + windows-amd64) + echo "exe-extension=.exe" >> $GITHUB_OUTPUT + echo "data-separator=;" >> $GITHUB_OUTPUT + ;; + linux-amd64) + echo "exe-extension=" >> $GITHUB_OUTPUT + echo "data-separator=:" >> $GITHUB_OUTPUT + ;; + macos-amd64|macos-arm64) + echo "exe-extension=" >> $GITHUB_OUTPUT + echo "data-separator=:" >> $GITHUB_OUTPUT + ;; + linux-armv7) + echo "exe-extension=" >> $GITHUB_OUTPUT + echo "arm-arch=armv7" >> $GITHUB_OUTPUT + echo "data-separator=:" >> $GITHUB_OUTPUT + ;; + linux-aarch64) + echo "exe-extension=" >> $GITHUB_OUTPUT + echo "data-separator=:" >> $GITHUB_OUTPUT + ;; + esac + + - name: Build for ARMv7 architecture + if: inputs.target-platform == 'linux-armv7' + uses: uraimo/run-on-arch-action@v3 + with: + arch: ${{ steps.setup-platform.outputs.arm-arch }} + # This cannot be updated because of missing libffi7 in 24.04 (cryptography requires it) + distro: ubuntu22.04 + shell: /bin/bash + githubToken: ${{ github.token }} + setup: mkdir -p "${PWD}/${{ inputs.output-dir }}" + dockerRunArgs: | + --volume "${PWD}/${{ inputs.output-dir }}:/${{ inputs.output-dir }}" + --volume "${GITHUB_ACTION_PATH}/process_include_dirs.py:${PWD}/process_include_dirs.py" + install: | + apt-get update -y + apt-get install -y software-properties-common + add-apt-repository -y ppa:deadsnakes/ppa + apt-get update -y + apt-get install --ignore-missing -y curl python${{ inputs.python-version }} python${{ inputs.python-version }}-dev pkg-config gcc g++ patchelf binutils zlib1g-dev ${{ inputs.additional-arm-packages }} + # Install pip for requested Python version using get-pip.py as ensurepip does not work here + curl -sS https://bootstrap.pypa.io/get-pip.py | python${{ inputs.python-version }} + python${{ inputs.python-version }} -m pip install --upgrade pip setuptools wheel + run: | + export PIP_BREAK_SYSTEM_PACKAGES=1 + adduser --disabled-password --gecos "" builder + chmod -R a+rwx /home/runner/work + su builder <<'EOF' + export PATH=$PATH:/home/builder/.local/bin + + # Set pip extra index if provided + if [ -n "${{ inputs.pip-extra-index-url }}" ]; then + export PIP_EXTRA_INDEX_URL="${{ inputs.pip-extra-index-url }}" + fi + + # Install PyInstaller + if [ "${{ inputs.pyinstaller-version }}" = "latest" ]; then + python${{ inputs.python-version }} -m pip install pyinstaller + else + python${{ inputs.python-version }} -m pip install pyinstaller==${{ inputs.pyinstaller-version }} + fi + + # Install dependencies + python${{ inputs.python-version }} -m ${{ inputs.install-deps-command }} + + # Build each file + IFS=' ' read -ra PYTHON_FILES <<< "${{ inputs.scripts }}" + IFS=' ' read -ra SCRIPT_NAMES <<< "${{ inputs.script-name }}" + + for i in "${!PYTHON_FILES[@]}"; do + file="${PYTHON_FILES[$i]}" + echo "Building $file for ${{ inputs.target-platform }}..." + + cmd="python${{ inputs.python-version }} -m PyInstaller --onefile --distpath=${{ inputs.output-dir }}" + + # Add custom name if provided + if [ -n "${{ inputs.script-name }}" ] && [ $i -lt ${#SCRIPT_NAMES[@]} ]; then + custom_name="${SCRIPT_NAMES[$i]}" + cmd="$cmd --name=$custom_name" + echo "Using custom name: $custom_name" + fi + + # Add include-data-dirs using Python script + echo "Processing include-data-dirs for $file..." + include_flags=$(python${{ inputs.python-version }} process_include_dirs.py '${{ inputs.include-data-dirs }}' '${{ steps.setup-platform.outputs.data-separator }}' $file) + echo "Include flags result: '$include_flags'" + if [ -n "$include_flags" ]; then + cmd="$cmd $include_flags" + echo "Added include flags to command" + else + echo "No include flags generated" + fi + + # Add additional arguments + if [ -n "${{ inputs.additional-args }}" ]; then + cmd="$cmd ${{ inputs.additional-args }}" + fi + + cmd="$cmd $file" + echo "Executing: $cmd" + eval "$cmd" + done + + # Test binaries + for i in "${!PYTHON_FILES[@]}"; do + file="${PYTHON_FILES[$i]}" + + # Determine executable name + if [ -n "${{ inputs.script-name }}" ] && [ $i -lt ${#SCRIPT_NAMES[@]} ]; then + custom_name="${SCRIPT_NAMES[$i]}" + executable="${{ inputs.output-dir }}/${custom_name}${{ steps.setup-platform.outputs.exe-extension }}" + else + base_name=$(basename "$file" .py) + executable="${{ inputs.output-dir }}/${base_name}${{ steps.setup-platform.outputs.exe-extension }}" + fi + + echo "Testing $executable..." + if [ -f "$executable" ]; then + echo "✓ $executable exists ($(du -h "$executable" | cut -f1))" + "$executable" ${{ inputs.test-command-args }} || exit 1 + else + echo "✗ $executable not found" + exit 1 + fi + done + EOF + + - name: Install PyInstaller (non-ARMv7) + if: inputs.target-platform != 'linux-armv7' + shell: bash + run: | + if [ -n "${{ inputs.pip-extra-index-url }}" ]; then + echo "PIP_EXTRA_INDEX_URL=${{ inputs.pip-extra-index-url }}" >> $GITHUB_ENV + fi + + if [ "${{ inputs.pyinstaller-version }}" = "latest" ]; then + python -m pip install pyinstaller + else + python -m pip install pyinstaller==${{ inputs.pyinstaller-version }} + fi + + - name: Install project dependencies (non-ARMv7) + if: inputs.target-platform != 'linux-armv7' + shell: bash + run: ${{ inputs.install-deps-command }} + + - name: Build with PyInstaller (non-ARMv7) + if: inputs.target-platform != 'linux-armv7' + id: build + shell: bash + run: | + # Build each Python file + IFS=' ' read -ra PYTHON_FILES <<< "${{ inputs.scripts }}" + IFS=' ' read -ra SCRIPT_NAMES <<< "${{ inputs.script-name }}" + + for i in "${!PYTHON_FILES[@]}"; do + file="${PYTHON_FILES[$i]}" + echo "Building $file for ${{ inputs.target-platform }}..." + + # Start building the command + cmd="python -m PyInstaller --onefile --distpath=${{ inputs.output-dir }}" + + # Add custom name if provided + if [ -n "${{ inputs.script-name }}" ] && [ $i -lt ${#SCRIPT_NAMES[@]} ]; then + custom_name="${SCRIPT_NAMES[$i]}" + cmd="$cmd --name=$custom_name" + echo "Using custom name: $custom_name" + fi + + # Windows-specific options + if [ "${{ inputs.target-platform }}" = "windows-amd64" ]; then + if [ -n "${{ inputs.icon-file }}" ]; then + cmd="$cmd --icon=${{ inputs.icon-file }}" + fi + fi + + # Add include-data-dirs using Python script + echo "Processing include-data-dirs for $file..." + include_flags=$(python $GITHUB_ACTION_PATH/process_include_dirs.py '${{ inputs.include-data-dirs }}' '${{ steps.setup-platform.outputs.data-separator }}' $file) + echo "Include flags result: '$include_flags'" + if [ -n "$include_flags" ]; then + cmd="$cmd $include_flags" + echo "Added include flags to command" + else + echo "No include flags generated" + fi + + # Add additional arguments + if [ -n "${{ inputs.additional-args }}" ]; then + cmd="$cmd ${{ inputs.additional-args }}" + fi + + # Add the file to build + cmd="$cmd $file" + + echo "Executing: $cmd" + eval "$cmd" + done + + echo "success=true" >> $GITHUB_OUTPUT + + - name: Verify builds (non-ARMv7) + if: inputs.target-platform != 'linux-armv7' + shell: bash + run: |- + echo "Verifying built executables..." + IFS=' ' read -ra PYTHON_FILES <<< "${{ inputs.scripts }}" + IFS=' ' read -ra SCRIPT_NAMES <<< "${{ inputs.script-name }}" + + for i in "${!PYTHON_FILES[@]}"; do + file="${PYTHON_FILES[$i]}" + + # Determine executable name + if [ -n "${{ inputs.script-name }}" ] && [ $i -lt ${#SCRIPT_NAMES[@]} ]; then + custom_name="${SCRIPT_NAMES[$i]}" + executable="${{ inputs.output-dir }}/${custom_name}${{ steps.setup-platform.outputs.exe-extension }}" + else + base_name=$(basename "$file" .py) + executable="${{ inputs.output-dir }}/${base_name}${{ steps.setup-platform.outputs.exe-extension }}" + fi + + if [ -f "$executable" ]; then + echo "✓ $executable exists ($(du -h "$executable" | cut -f1))" + # Test that it runs + if "$executable" ${{ inputs.test-command-args }}; then + echo "✓ $executable runs successfully" + else + echo "⚠ $executable may have issues" + exit 1 + fi + else + echo "✗ $executable not found" + exit 1 + fi + done diff --git a/process_include_dirs.py b/process_include_dirs.py new file mode 100644 index 0000000..dd4af92 --- /dev/null +++ b/process_include_dirs.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +""" +Process include-data-dirs JSON and generate PyInstaller arguments +Supports per-script configuration with wildcard support +""" + +import json +import argparse +import fnmatch +import sys + + +def fix_windows_paths(include_dirs_json: str) -> str: + """Fix Windows paths in the include directories""" + if sys.platform == "win32": + include_dirs_json = include_dirs_json.replace("\\", "/") + return include_dirs_json + + +def process_include_dirs(include_dirs_json: str, data_separator: str, target_script: str) -> str: + """Convert include directories JSON to PyInstaller --add-data flags + for a specific script""" + try: + include_dirs_config = json.loads(fix_windows_paths(include_dirs_json)) + if not include_dirs_config: + return "" + + # For simplicity, we also support simple list format + # This is equivalent to the dict format with wildcard + if isinstance(include_dirs_config, list): + # Simple format: apply to all scripts + include_dirs = include_dirs_config + elif isinstance(include_dirs_config, dict): + # Dict format: per-script configuration + include_dirs = [] + processed_patterns = set() + + # Check for wildcard entries first + for pattern, dirs in include_dirs_config.items(): + if pattern == "*" or fnmatch.fnmatch(target_script, pattern): + if isinstance(dirs, list): + include_dirs.extend(dirs) + else: + include_dirs.append(dirs) + processed_patterns.add(pattern) + + # Check for exact script name (only if not already processed by wildcard) + if target_script in include_dirs_config and target_script not in processed_patterns: + dirs = include_dirs_config[target_script] + if isinstance(dirs, list): + include_dirs.extend(dirs) + else: + include_dirs.append(dirs) + else: + return "" + + if not include_dirs: + return "" + + flags = [] + for item in include_dirs: + flags.append(f"--add-data='{item}{data_separator}{item}'") + + return " ".join(flags) + except (json.JSONDecodeError, KeyError, TypeError) as e: + print( + f"Warning: Error processing include dirs for {target_script}: {e}", + file=sys.stderr, + ) + return "" + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Process include directories for PyInstaller", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Simple list format (applies to all scripts) + python process_include_dirs.py '["./data"]' ':' 'main.py' + + # Dict format (per-script configuration with wildcard support) + python process_include_dirs.py '{"main.py": ["./assets"], "*": ["./common"]}' ':' 'main.py' + """, # noqa: E501 + ) + parser.add_argument("include_dirs", help="JSON string of include directories") + parser.add_argument( + "data_separator", + help="Data separator for PyInstaller --add-data (; for Windows, : for Unix)", + ) + parser.add_argument("target_script", help="Target script name for filtering") + + args = parser.parse_args() + result = process_include_dirs(args.include_dirs, args.data_separator, args.target_script) + print(result) + + +if __name__ == "__main__": + main() From 69f703ffc35e6e9657b36d08ede35af37780dcc4 Mon Sep 17 00:00:00 2001 From: Peter Dragun Date: Thu, 3 Jul 2025 16:23:59 +0200 Subject: [PATCH 3/4] feat(windows): Add signing of binaries --- README.md | 20 +++++++++++++ Sign-File.ps1 | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ action.yml | 27 ++++++++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 Sign-File.ps1 diff --git a/README.md b/README.md index 1904fd6..058e001 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,24 @@ Building the project with default configuration will result in script name `__ma test-command-args: '--version' ``` +### Signing Windows Binaries + +If you would like to sign Windows binaries, you can set `certificate` and the action will also take care of signing all binaries. +It is also recommended to use `certificate-password`. + +The `certificate` should be a PFX (Personal Information Exchange) certificate file encoded in base64 format. + +```yaml +- name: Build Python executable + uses: espressif/python-binary-action@master + with: + scripts: 'app.py' + output-dir: './dist' + target-platform: 'windows-amd64' + certificate: ${{ secrets.CERTIFICATE }} + certificate-password: ${{ secrets.CERTIFICATE_PASSWORD }} +``` + ### Complete Workflow Here you can see a simplified version of workflow used in [esptool](https://github.com/espressif/esptool/) repository: @@ -257,6 +275,8 @@ jobs: | `install-deps-command` | Command to install project dependencies | `"pip install --user --prefer-binary -e ."` | `"pip install -r requirements.txt"` | | `additional-arm-packages` | ARMv7 ONLY: Additional system packages | `""` | `"openssl libffi-dev"` | | `test-command-args` | Command arguments to test executables | `"--help"` | `"--version"` | +| `certificate` | Certificate to use for signing binaries | `""` | `${{ secrets.CERTIFICATE }}` | +| `certificate-password` | Password for the certificate | `""` | `${{ secrets.CERTIFICATE_PASSWORD }}` | > [!IMPORTANT] > Be careful when changing `pyinstaller-version` as it might lead to increased false positives with anti-virus software. It is recommended to check your executables with antivirus software such as [Virustotal](https://www.virustotal.com/gui/home/upload). diff --git a/Sign-File.ps1 b/Sign-File.ps1 new file mode 100644 index 0000000..1fb542e --- /dev/null +++ b/Sign-File.ps1 @@ -0,0 +1,79 @@ +[CmdletBinding()] +param ( + [Parameter()] + [String] + $Path +) + + +function FindSignTool { + $SignTool = "signtool.exe" + if (Get-Command $SignTool -ErrorAction SilentlyContinue) { + return $SignTool + } + $SignTool = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\x64\signtool.exe" + if (Test-Path -Path $SignTool -PathType Leaf) { + return $SignTool + } + $SignTool = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\x86\signtool.exe" + if (Test-Path -Path $SignTool -PathType Leaf) { + return $SignTool + } + $sdkVers = "10.0.22000.0", "10.0.20348.0", "10.0.19041.0", "10.0.17763.0" + Foreach ($ver in $sdkVers) + { + $SignTool = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\${ver}\x64\signtool.exe" + if (Test-Path -Path $SignTool -PathType Leaf) { + return $SignTool + } + } + "signtool.exe not found" + Exit 1 +} + +function SignFile { + param( + [Parameter()] + [String] + $Path + ) + + $SignTool = FindSignTool + "Using: $SignTool" + "Signing file: $Path" + $CertificateFile = [system.io.path]::GetTempPath() + "certificate.pfx" + + if ($null -eq $env:CERTIFICATE) { + "CERTIFICATE variable not set, unable to sign the file" + Exit 1 + } + + if ("" -eq $env:CERTIFICATE) { + "CERTIFICATE variable is empty, unable to sign the file" + Exit 1 + } + + $SignParameters = @("sign", "/tr", 'http://timestamp.digicert.com', "/td", "SHA256", "/f", $CertificateFile, "/fd", "SHA256") + if ($env:CERTIFICATE_PASSWORD) { + "CERTIFICATE_PASSWORD detected, using the password" + $SignParameters += "/p" + $SignParameters += $env:CERTIFICATE_PASSWORD + } + $SignParameters += $Path + + [byte[]]$CertificateBytes = [convert]::FromBase64String($env:CERTIFICATE) + [IO.File]::WriteAllBytes($CertificateFile, $CertificateBytes) + + &$SignTool $SignParameters + + if (0 -eq $LASTEXITCODE) { + Remove-Item $CertificateFile + } else { + Remove-Item $CertificateFile + "Signing failed" + Exit 1 + } + +} + +SignFile ${Path} diff --git a/action.yml b/action.yml index ccd1b4a..3d716a5 100644 --- a/action.yml +++ b/action.yml @@ -59,6 +59,14 @@ inputs: description: Command arguments to test binaries (e.g. "--help") required: false default: --help + certificate: + description: Certificate to use for signing binaries + required: false + default: '' + certificate-password: + description: Password for the certificate + required: false + default: '' outputs: executable-extension: @@ -313,3 +321,22 @@ runs: exit 1 fi done + + - name: Sign binaries + if: inputs.target-platform == 'windows-amd64' + env: + CERTIFICATE: ${{ inputs.certificate }} + CERTIFICATE_PASSWORD: ${{ inputs.certificate-password }} + shell: pwsh + run: |- + if ([string]::IsNullOrEmpty($env:CERTIFICATE)) { + Write-Host "::warning title=Signing::Certificate is not set, skipping signing" + exit 0 + } + + $pythonFiles = "${{ inputs.scripts }}".Split(' ') + foreach ($file in $pythonFiles) { + $baseName = [System.IO.Path]::GetFileNameWithoutExtension($file) + $executable = "./${{ inputs.output-dir }}/${baseName}${{ steps.setup-platform.outputs.exe-extension }}" + & (Join-Path $env:GITHUB_ACTION_PATH "Sign-File.ps1") -Path $executable + } From 188759e19af1d2731d875fe7ec11dd8e1c0182cd Mon Sep 17 00:00:00 2001 From: Peter Dragun Date: Mon, 14 Jul 2025 13:36:29 +0200 Subject: [PATCH 4/4] feat: Use docker for linux-amd64 and linux-aarch64 Docker was used to maximize compatibility with olders systems, mainly because of issues with GLIBC. For better maintainability, the action was split into multiple files. --- action.yml | 289 ++++++++++++++------------------------ build_with_pyinstaller.sh | 76 ++++++++++ process_include_dirs.py | 16 ++- setup_environment.sh | 34 +++++ test_executables.sh | 46 ++++++ 5 files changed, 275 insertions(+), 186 deletions(-) create mode 100755 build_with_pyinstaller.sh create mode 100755 setup_environment.sh create mode 100755 test_executables.sh diff --git a/action.yml b/action.yml index 3d716a5..28b4fc9 100644 --- a/action.yml +++ b/action.yml @@ -34,7 +34,8 @@ inputs: default: '3.13' pyinstaller-version: description: PyInstaller version to install. Default is 6.11.1 because it has - the lowest false positive rate with antivirus. For latest version, use empty string. + the lowest false positive rate with antivirus. For latest version, use empty + string. required: false default: 6.11.1 additional-args: @@ -87,39 +88,51 @@ runs: windows-amd64) echo "exe-extension=.exe" >> $GITHUB_OUTPUT echo "data-separator=;" >> $GITHUB_OUTPUT + echo "needs-docker=false" >> $GITHUB_OUTPUT + echo "needs-arm-emulation=false" >> $GITHUB_OUTPUT ;; - linux-amd64) + linux-amd64|linux-aarch64) echo "exe-extension=" >> $GITHUB_OUTPUT echo "data-separator=:" >> $GITHUB_OUTPUT + echo "needs-docker=true" >> $GITHUB_OUTPUT + echo "needs-arm-emulation=false" >> $GITHUB_OUTPUT ;; macos-amd64|macos-arm64) echo "exe-extension=" >> $GITHUB_OUTPUT echo "data-separator=:" >> $GITHUB_OUTPUT + echo "needs-docker=false" >> $GITHUB_OUTPUT + echo "needs-arm-emulation=false" >> $GITHUB_OUTPUT ;; linux-armv7) - echo "exe-extension=" >> $GITHUB_OUTPUT - echo "arm-arch=armv7" >> $GITHUB_OUTPUT - echo "data-separator=:" >> $GITHUB_OUTPUT - ;; - linux-aarch64) echo "exe-extension=" >> $GITHUB_OUTPUT echo "data-separator=:" >> $GITHUB_OUTPUT + echo "needs-docker=false" >> $GITHUB_OUTPUT + echo "needs-arm-emulation=true" >> $GITHUB_OUTPUT ;; esac + - name: Prepare build environment + shell: bash + run: | + # Create output directory + mkdir -p "${{ inputs.output-dir }}" + + # Write include-data-dirs to a temporary file to avoid shell escaping issues + echo '${{ inputs.include-data-dirs }}' > ${GITHUB_ACTION_PATH}/include_data_dirs.json + - name: Build for ARMv7 architecture if: inputs.target-platform == 'linux-armv7' uses: uraimo/run-on-arch-action@v3 with: - arch: ${{ steps.setup-platform.outputs.arm-arch }} - # This cannot be updated because of missing libffi7 in 24.04 (cryptography requires it) - distro: ubuntu22.04 + arch: armv7 + # Use Ubuntu 20.04 because of missing libffi7 in 24.04 (cryptography requires it) and better compatibility with GLIBC + distro: ubuntu20.04 shell: /bin/bash githubToken: ${{ github.token }} setup: mkdir -p "${PWD}/${{ inputs.output-dir }}" dockerRunArgs: | --volume "${PWD}/${{ inputs.output-dir }}:/${{ inputs.output-dir }}" - --volume "${GITHUB_ACTION_PATH}/process_include_dirs.py:${PWD}/process_include_dirs.py" + --volume "${GITHUB_ACTION_PATH}:/github/action" install: | apt-get update -y apt-get install -y software-properties-common @@ -130,197 +143,105 @@ runs: curl -sS https://bootstrap.pypa.io/get-pip.py | python${{ inputs.python-version }} python${{ inputs.python-version }} -m pip install --upgrade pip setuptools wheel run: | - export PIP_BREAK_SYSTEM_PACKAGES=1 adduser --disabled-password --gecos "" builder chmod -R a+rwx /home/runner/work su builder <<'EOF' export PATH=$PATH:/home/builder/.local/bin - # Set pip extra index if provided - if [ -n "${{ inputs.pip-extra-index-url }}" ]; then - export PIP_EXTRA_INDEX_URL="${{ inputs.pip-extra-index-url }}" - fi - - # Install PyInstaller - if [ "${{ inputs.pyinstaller-version }}" = "latest" ]; then - python${{ inputs.python-version }} -m pip install pyinstaller - else - python${{ inputs.python-version }} -m pip install pyinstaller==${{ inputs.pyinstaller-version }} - fi - - # Install dependencies - python${{ inputs.python-version }} -m ${{ inputs.install-deps-command }} - - # Build each file - IFS=' ' read -ra PYTHON_FILES <<< "${{ inputs.scripts }}" - IFS=' ' read -ra SCRIPT_NAMES <<< "${{ inputs.script-name }}" - - for i in "${!PYTHON_FILES[@]}"; do - file="${PYTHON_FILES[$i]}" - echo "Building $file for ${{ inputs.target-platform }}..." - - cmd="python${{ inputs.python-version }} -m PyInstaller --onefile --distpath=${{ inputs.output-dir }}" - - # Add custom name if provided - if [ -n "${{ inputs.script-name }}" ] && [ $i -lt ${#SCRIPT_NAMES[@]} ]; then - custom_name="${SCRIPT_NAMES[$i]}" - cmd="$cmd --name=$custom_name" - echo "Using custom name: $custom_name" - fi - - # Add include-data-dirs using Python script - echo "Processing include-data-dirs for $file..." - include_flags=$(python${{ inputs.python-version }} process_include_dirs.py '${{ inputs.include-data-dirs }}' '${{ steps.setup-platform.outputs.data-separator }}' $file) - echo "Include flags result: '$include_flags'" - if [ -n "$include_flags" ]; then - cmd="$cmd $include_flags" - echo "Added include flags to command" - else - echo "No include flags generated" - fi - - # Add additional arguments - if [ -n "${{ inputs.additional-args }}" ]; then - cmd="$cmd ${{ inputs.additional-args }}" - fi - - cmd="$cmd $file" - echo "Executing: $cmd" - eval "$cmd" - done - - # Test binaries - for i in "${!PYTHON_FILES[@]}"; do - file="${PYTHON_FILES[$i]}" - - # Determine executable name - if [ -n "${{ inputs.script-name }}" ] && [ $i -lt ${#SCRIPT_NAMES[@]} ]; then - custom_name="${SCRIPT_NAMES[$i]}" - executable="${{ inputs.output-dir }}/${custom_name}${{ steps.setup-platform.outputs.exe-extension }}" - else - base_name=$(basename "$file" .py) - executable="${{ inputs.output-dir }}/${base_name}${{ steps.setup-platform.outputs.exe-extension }}" - fi - - echo "Testing $executable..." - if [ -f "$executable" ]; then - echo "✓ $executable exists ($(du -h "$executable" | cut -f1))" - "$executable" ${{ inputs.test-command-args }} || exit 1 - else - echo "✗ $executable not found" - exit 1 - fi - done + # Setup environment using helper script + /github/action/setup_environment.sh \ + "python${{ inputs.python-version }}" \ + "${{ inputs.pyinstaller-version }}" \ + "${{ inputs.pip-extra-index-url }}" \ + "${{ inputs.install-deps-command }}" + + # Write include-data-dirs to a temporary file to avoid shell escaping issues + echo '${{ inputs.include-data-dirs }}' > /tmp/include_data_dirs.json + + # Build each file using the script + export GITHUB_ACTION_PATH=/github/action + /github/action/build_with_pyinstaller.sh \ + "python${{ inputs.python-version }}" \ + "${{ inputs.target-platform }}" \ + "${{ inputs.output-dir }}" \ + "${{ inputs.scripts }}" \ + "${{ inputs.script-name }}" \ + "" \ + "/tmp/include_data_dirs.json" \ + "${{ steps.setup-platform.outputs.data-separator }}" \ + "${{ inputs.additional-args }}" + + # Test binaries using helper script + /github/action/test_executables.sh \ + "${{ inputs.scripts }}" \ + "${{ inputs.script-name }}" \ + "${{ inputs.output-dir }}" \ + "${{ steps.setup-platform.outputs.exe-extension }}" \ + "${{ inputs.test-command-args }}" EOF - - name: Install PyInstaller (non-ARMv7) - if: inputs.target-platform != 'linux-armv7' + - name: Build for Linux (Docker) + # Running in Docker to link older versions of GLIBC, because the GitHub runners don't offer ubuntu 20.04 anymore + if: steps.setup-platform.outputs.needs-docker == 'true' shell: bash run: | - if [ -n "${{ inputs.pip-extra-index-url }}" ]; then - echo "PIP_EXTRA_INDEX_URL=${{ inputs.pip-extra-index-url }}" >> $GITHUB_ENV - fi - - if [ "${{ inputs.pyinstaller-version }}" = "latest" ]; then - python -m pip install pyinstaller - else - python -m pip install pyinstaller==${{ inputs.pyinstaller-version }} - fi - - - name: Install project dependencies (non-ARMv7) - if: inputs.target-platform != 'linux-armv7' + docker run --rm \ + -v "${PWD}/${{ inputs.output-dir }}:/${{ inputs.output-dir }}" \ + -v "${GITHUB_ACTION_PATH}:/github/action" \ + -v "${GITHUB_WORKSPACE}:/${GITHUB_WORKSPACE}" \ + -w ${GITHUB_WORKSPACE} \ + ubuntu:20.04 \ + bash -c " + apt-get update -y && + apt-get install -y software-properties-common curl && + add-apt-repository -y ppa:deadsnakes/ppa && + apt-get update -y && + apt-get install -y python${{ inputs.python-version }} python${{ inputs.python-version }}-dev pkg-config gcc g++ patchelf binutils zlib1g-dev && + curl -sS https://bootstrap.pypa.io/get-pip.py | python${{ inputs.python-version }} && + python${{ inputs.python-version }} -m pip install --upgrade pip setuptools wheel && + + export GITHUB_ACTION_PATH=/github/action && + /github/action/setup_environment.sh \ + 'python${{ inputs.python-version }}' \ + '${{ inputs.pyinstaller-version }}' \ + '${{ inputs.pip-extra-index-url }}' \ + '${{ inputs.install-deps-command }}' && + + # Execute the build script + /github/action/build_with_pyinstaller.sh 'python${{ inputs.python-version }}' '${{ inputs.target-platform }}' '${{ inputs.output-dir }}' '${{ inputs.scripts }}' '${{ inputs.script-name }}' '${{ inputs.icon-file }}' '/github/action/include_data_dirs.json' '${{ steps.setup-platform.outputs.data-separator }}' '${{ inputs.additional-args }}' + " + + - name: Setup environment (Native platforms) + if: | + steps.setup-platform.outputs.needs-docker == 'false' && steps.setup-platform.outputs.needs-arm-emulation == 'false' shell: bash - run: ${{ inputs.install-deps-command }} - - - name: Build with PyInstaller (non-ARMv7) - if: inputs.target-platform != 'linux-armv7' + run: | + $GITHUB_ACTION_PATH/setup_environment.sh \ + 'python' \ + '${{ inputs.pyinstaller-version }}' \ + '${{ inputs.pip-extra-index-url }}' \ + '${{ inputs.install-deps-command }}' + + - name: Build with PyInstaller (Native platforms) + if: | + steps.setup-platform.outputs.needs-docker == 'false' && steps.setup-platform.outputs.needs-arm-emulation == 'false' id: build shell: bash run: | - # Build each Python file - IFS=' ' read -ra PYTHON_FILES <<< "${{ inputs.scripts }}" - IFS=' ' read -ra SCRIPT_NAMES <<< "${{ inputs.script-name }}" - - for i in "${!PYTHON_FILES[@]}"; do - file="${PYTHON_FILES[$i]}" - echo "Building $file for ${{ inputs.target-platform }}..." - - # Start building the command - cmd="python -m PyInstaller --onefile --distpath=${{ inputs.output-dir }}" - - # Add custom name if provided - if [ -n "${{ inputs.script-name }}" ] && [ $i -lt ${#SCRIPT_NAMES[@]} ]; then - custom_name="${SCRIPT_NAMES[$i]}" - cmd="$cmd --name=$custom_name" - echo "Using custom name: $custom_name" - fi - - # Windows-specific options - if [ "${{ inputs.target-platform }}" = "windows-amd64" ]; then - if [ -n "${{ inputs.icon-file }}" ]; then - cmd="$cmd --icon=${{ inputs.icon-file }}" - fi - fi - - # Add include-data-dirs using Python script - echo "Processing include-data-dirs for $file..." - include_flags=$(python $GITHUB_ACTION_PATH/process_include_dirs.py '${{ inputs.include-data-dirs }}' '${{ steps.setup-platform.outputs.data-separator }}' $file) - echo "Include flags result: '$include_flags'" - if [ -n "$include_flags" ]; then - cmd="$cmd $include_flags" - echo "Added include flags to command" - else - echo "No include flags generated" - fi - - # Add additional arguments - if [ -n "${{ inputs.additional-args }}" ]; then - cmd="$cmd ${{ inputs.additional-args }}" - fi - - # Add the file to build - cmd="$cmd $file" - - echo "Executing: $cmd" - eval "$cmd" - done + $GITHUB_ACTION_PATH/build_with_pyinstaller.sh 'python' '${{ inputs.target-platform }}' '${{ inputs.output-dir }}' '${{ inputs.scripts }}' '${{ inputs.script-name }}' '${{ inputs.icon-file }}' '${{ inputs.include-data-dirs }}' '${{ steps.setup-platform.outputs.data-separator }}' '${{ inputs.additional-args }}' echo "success=true" >> $GITHUB_OUTPUT - - name: Verify builds (non-ARMv7) - if: inputs.target-platform != 'linux-armv7' + - name: Verify builds + if: steps.setup-platform.outputs.needs-arm-emulation == 'false' shell: bash run: |- - echo "Verifying built executables..." - IFS=' ' read -ra PYTHON_FILES <<< "${{ inputs.scripts }}" - IFS=' ' read -ra SCRIPT_NAMES <<< "${{ inputs.script-name }}" - - for i in "${!PYTHON_FILES[@]}"; do - file="${PYTHON_FILES[$i]}" - - # Determine executable name - if [ -n "${{ inputs.script-name }}" ] && [ $i -lt ${#SCRIPT_NAMES[@]} ]; then - custom_name="${SCRIPT_NAMES[$i]}" - executable="${{ inputs.output-dir }}/${custom_name}${{ steps.setup-platform.outputs.exe-extension }}" - else - base_name=$(basename "$file" .py) - executable="${{ inputs.output-dir }}/${base_name}${{ steps.setup-platform.outputs.exe-extension }}" - fi - - if [ -f "$executable" ]; then - echo "✓ $executable exists ($(du -h "$executable" | cut -f1))" - # Test that it runs - if "$executable" ${{ inputs.test-command-args }}; then - echo "✓ $executable runs successfully" - else - echo "⚠ $executable may have issues" - exit 1 - fi - else - echo "✗ $executable not found" - exit 1 - fi - done + $GITHUB_ACTION_PATH/test_executables.sh \ + "${{ inputs.scripts }}" \ + "${{ inputs.script-name }}" \ + "${{ inputs.output-dir }}" \ + "${{ steps.setup-platform.outputs.exe-extension }}" \ + "${{ inputs.test-command-args }}" - name: Sign binaries if: inputs.target-platform == 'windows-amd64' diff --git a/build_with_pyinstaller.sh b/build_with_pyinstaller.sh new file mode 100755 index 0000000..71f26cb --- /dev/null +++ b/build_with_pyinstaller.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Build Python scripts with PyInstaller +# Usage: ./build_with_pyinstaller.sh [python_version] [target_platform] [output_dir] [scripts] [script_names] [icon_file] [include_data_dirs] [data_separator] [additional_args] + +set -e + +# Parse arguments +PYTHON_VERSION="${1:-python}" +TARGET_PLATFORM="${2}" +OUTPUT_DIR="${3}" +SCRIPTS="${4}" +SCRIPT_NAMES="${5}" +ICON_FILE="${6}" +INCLUDE_DATA_DIRS="${7}" +DATA_SEPARATOR="${8}" +ADDITIONAL_ARGS="${9}" + +echo "Building with PyInstaller..." +echo "Python version: $PYTHON_VERSION" +echo "Target platform: $TARGET_PLATFORM" +echo "Output directory: $OUTPUT_DIR" +echo "Scripts: $SCRIPTS" +echo "Script names: $SCRIPT_NAMES" + +# Build each Python file +IFS=' ' read -ra PYTHON_FILES <<< "$SCRIPTS" +IFS=' ' read -ra SCRIPT_NAMES_ARRAY <<< "$SCRIPT_NAMES" + +for i in "${!PYTHON_FILES[@]}"; do + file="${PYTHON_FILES[$i]}" + echo "Building $file for $TARGET_PLATFORM..." + + # Start building the command + cmd="$PYTHON_VERSION -m PyInstaller --onefile --distpath=$OUTPUT_DIR" + + # Add custom name if provided + if [ -n "$SCRIPT_NAMES" ] && [ $i -lt ${#SCRIPT_NAMES_ARRAY[@]} ]; then + custom_name="${SCRIPT_NAMES_ARRAY[$i]}" + cmd="$cmd --name=$custom_name" + echo "Using custom name: $custom_name" + fi + + # Windows-specific options + if [ "$TARGET_PLATFORM" = "windows-amd64" ]; then + if [ -n "$ICON_FILE" ]; then + cmd="$cmd --icon=$ICON_FILE" + fi + fi + + # Add include-data-dirs using Python script + if [ -n "$INCLUDE_DATA_DIRS" ]; then + echo "Processing include-data-dirs for $file..." + include_flags=$($PYTHON_VERSION $GITHUB_ACTION_PATH/process_include_dirs.py "$INCLUDE_DATA_DIRS" "$DATA_SEPARATOR" "$file") + echo "Include flags result: '$include_flags'" + if [ -n "$include_flags" ]; then + cmd="$cmd $include_flags" + echo "Added include flags to command" + else + echo "No include flags generated" + fi + fi + + # Add additional arguments + if [ -n "$ADDITIONAL_ARGS" ]; then + cmd="$cmd $ADDITIONAL_ARGS" + fi + + # Add the file to build + cmd="$cmd $file" + + echo "Executing: $cmd" + eval "$cmd" +done + +echo "Build completed successfully" diff --git a/process_include_dirs.py b/process_include_dirs.py index dd4af92..e6fae89 100644 --- a/process_include_dirs.py +++ b/process_include_dirs.py @@ -8,6 +8,7 @@ import argparse import fnmatch import sys +import os def fix_windows_paths(include_dirs_json: str) -> str: @@ -81,9 +82,12 @@ def main() -> None: # Dict format (per-script configuration with wildcard support) python process_include_dirs.py '{"main.py": ["./assets"], "*": ["./common"]}' ':' 'main.py' + + # File input (to avoid shell escaping issues) + python process_include_dirs.py '/tmp/include_data_dirs.json' ':' 'main.py' """, # noqa: E501 ) - parser.add_argument("include_dirs", help="JSON string of include directories") + parser.add_argument("include_dirs", help="JSON string or file path of include directories") parser.add_argument( "data_separator", help="Data separator for PyInstaller --add-data (; for Windows, : for Unix)", @@ -91,7 +95,15 @@ def main() -> None: parser.add_argument("target_script", help="Target script name for filtering") args = parser.parse_args() - result = process_include_dirs(args.include_dirs, args.data_separator, args.target_script) + + # Check if include_dirs is a file path + if os.path.isfile(args.include_dirs): + with open(args.include_dirs, "r") as f: + include_dirs_content = f.read() + else: + include_dirs_content = args.include_dirs + + result = process_include_dirs(include_dirs_content, args.data_separator, args.target_script) print(result) diff --git a/setup_environment.sh b/setup_environment.sh new file mode 100755 index 0000000..5c82fb6 --- /dev/null +++ b/setup_environment.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Setup Python environment for PyInstaller builds +# Usage: ./setup_environment.sh [python_version] [pyinstaller_version] [pip_extra_index_url] [install_deps_command] + +set -e + +PYTHON_VERSION="${1:-python}" +PYINSTALLER_VERSION="${2:-6.11.1}" +PIP_EXTRA_INDEX_URL="${3:-}" +INSTALL_DEPS_COMMAND="${4:-pip install --user --prefer-binary -e .}" + +echo "Setting up Python environment..." +echo "Python version: $PYTHON_VERSION" +echo "PyInstaller version: $PYINSTALLER_VERSION" + +# Set pip extra index if provided +if [ -n "$PIP_EXTRA_INDEX_URL" ]; then + export PIP_EXTRA_INDEX_URL="$PIP_EXTRA_INDEX_URL" + echo "Using extra pip index: $PIP_EXTRA_INDEX_URL" +fi + +# Install PyInstaller +if [ -z "$PYINSTALLER_VERSION" ]; then + $PYTHON_VERSION -m pip install pyinstaller +else + $PYTHON_VERSION -m pip install pyinstaller==$PYINSTALLER_VERSION +fi + +# Install dependencies +echo "Installing project dependencies..." +$PYTHON_VERSION -m $INSTALL_DEPS_COMMAND + +echo "Environment setup completed successfully" diff --git a/test_executables.sh b/test_executables.sh new file mode 100755 index 0000000..1276482 --- /dev/null +++ b/test_executables.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Test built executables +# Usage: ./test_executables.sh [scripts] [script_names] [output_dir] [exe_extension] [test_args] + +set -e + +SCRIPTS="${1}" +SCRIPT_NAMES="${2}" +OUTPUT_DIR="${3}" +EXE_EXTENSION="${4}" +TEST_ARGS="${5:---help}" + +echo "Testing built executables..." + +IFS=' ' read -ra PYTHON_FILES <<< "$SCRIPTS" +IFS=' ' read -ra SCRIPT_NAMES_ARRAY <<< "$SCRIPT_NAMES" + +for i in "${!PYTHON_FILES[@]}"; do + file="${PYTHON_FILES[$i]}" + + # Determine executable name + if [ -n "$SCRIPT_NAMES" ] && [ $i -lt ${#SCRIPT_NAMES_ARRAY[@]} ]; then + custom_name="${SCRIPT_NAMES_ARRAY[$i]}" + executable="$OUTPUT_DIR/${custom_name}${EXE_EXTENSION}" + else + base_name=$(basename "$file" .py) + executable="$OUTPUT_DIR/${base_name}${EXE_EXTENSION}" + fi + + echo "Testing $executable..." + if [ -f "$executable" ]; then + echo "✓ $executable exists ($(du -h "$executable" | cut -f1))" + if "$executable" $TEST_ARGS; then + echo "✓ $executable runs successfully" + else + echo "⚠ $executable may have issues" + exit 1 + fi + else + echo "✗ $executable not found" + exit 1 + fi +done + +echo "All executables tested successfully"