diff --git a/.github/workflows/dangerjs.yml b/.github/workflows/dangerjs.yml new file mode 100644 index 0000000..6b03259 --- /dev/null +++ b/.github/workflows/dangerjs.yml @@ -0,0 +1,26 @@ +name: DangerJS Pull Request linter + +on: + pull_request_target: + types: [opened, edited, reopened, synchronize] + +permissions: + pull-requests: write + contents: write + +jobs: + pull-request-style-linter: + runs-on: ubuntu-latest + steps: + - name: Check out PR head + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: DangerJS pull request linter + uses: espressif/shared-github-dangerjs@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + instructions-contributions-file: CONTRIBUTING.rst + instructions-gitlab-mirror: 'true' diff --git a/.github/workflows/issue_comment.yml b/.github/workflows/issue_comment.yml index 840cff5..c1388ce 100644 --- a/.github/workflows/issue_comment.yml +++ b/.github/workflows/issue_comment.yml @@ -16,6 +16,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} JIRA_PASS: ${{ secrets.JIRA_PASS }} JIRA_PROJECT: IDFGH - JIRA_COMPONENT: GitHub + JIRA_COMPONENT: tools JIRA_URL: ${{ secrets.JIRA_URL }} JIRA_USER: ${{ secrets.JIRA_USER }} diff --git a/.github/workflows/new_issues.yml b/.github/workflows/new_issues.yml index b110599..6507b58 100644 --- a/.github/workflows/new_issues.yml +++ b/.github/workflows/new_issues.yml @@ -16,6 +16,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} JIRA_PASS: ${{ secrets.JIRA_PASS }} JIRA_PROJECT: IDFGH - JIRA_COMPONENT: GitHub + JIRA_COMPONENT: tools JIRA_URL: ${{ secrets.JIRA_URL }} JIRA_USER: ${{ secrets.JIRA_USER }} diff --git a/.github/workflows/new_prs.yml b/.github/workflows/new_prs.yml index 4c80fde..cd328e3 100644 --- a/.github/workflows/new_prs.yml +++ b/.github/workflows/new_prs.yml @@ -4,7 +4,7 @@ name: Sync remain PRs to Jira # Note that, PRs can also get synced when new PR comment is created on: schedule: - - cron: "0 * * * *" + - cron: 0 * * * * jobs: sync_prs_to_jira: @@ -21,6 +21,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} JIRA_PASS: ${{ secrets.JIRA_PASS }} JIRA_PROJECT: IDFGH - JIRA_COMPONENT: GitHub + JIRA_COMPONENT: tools JIRA_URL: ${{ secrets.JIRA_URL }} JIRA_USER: ${{ secrets.JIRA_USER }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 263280b..ea2ce71 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,40 +1,49 @@ -# See https://pre-commit.com for more information -# See https://pre-commit.com/hooks.html for more hooks ---- minimum_pre_commit_version: 3.3.0 -default_install_hook_types: [pre-commit,commit-msg] +default_install_hook_types: [pre-commit, commit-msg] +default_stages: [pre-commit] + +ci: + autofix_prs: false + autoupdate_commit_msg: 'ci: Bump pre-commit hooks' + autoupdate_schedule: quarterly repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-executables-have-shebangs - id: mixed-line-ending - args: ['-f=lf'] - - id: double-quote-string-fixer - - - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - args: ['--config=.flake8'] + args: [-f=lf] - - repo: https://github.com/pycqa/isort - rev: 5.11.5 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.2 hooks: - - id: isort - name: isort (python) + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.1.1' # Use the sha / tag you want to point at + rev: v1.16.1 hooks: - - id: mypy + - id: mypy additional_dependencies: [] - repo: https://github.com/espressif/conventional-precommit-linter - rev: v1.2.0 + rev: v1.10.0 hooks: - id: conventional-precommit-linter stages: [commit-msg] + + - repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + additional_dependencies: + - tomli + + - repo: https://github.com/lyz-code/yamlfix/ + rev: 1.17.0 + hooks: + - id: yamlfix diff --git a/README.md b/README.md index 07699d3..058e001 100644 --- a/README.md +++ b/README.md @@ -1 +1,298 @@ -# python-binary-action \ No newline at end of file +[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/espressif/python-binary-action/master.svg)](https://results.pre-commit.ci/latest/github/espressif/python-binary-action/master) + +# Python Binary Build Action + +A GitHub Action for building Python applications into standalone executables using `PyInstaller` across multiple architectures and platforms. + +## Overview + +This action automates the process of creating standalone executables from Python applications using `PyInstaller`. It supports multiple target platforms including Windows, macOS, and Linux (both x86_64 and ARM architectures). The action handles platform-specific configurations, dependency installation, and binary verification automatically. + +### Motivation + +This action provides several key benefits: + +**Centralized Build Logic**: Eliminates code duplication across repositories by providing a single, centrally maintained build solution. This ensures consistent build processes and easier maintenance. + +**Reduced Antivirus False Positives**: Antivirus software often flags PyInstaller-generated executables as suspicious. By centrally controlling `PyInstaller` versions and configurations, this action helps minimize false positive reports through tested and optimized settings. + +**User Accessibility**: Many users prefer standalone executables over Python scripts, as they eliminate the need to install Python and manage dependencies. This is particularly valuable for: + +- Users unfamiliar with Python installation and package management +- Legacy systems with unsupported Python versions +- Deployment scenarios requiring minimal dependencies + +## Supported Platforms + +- **Windows**: `windows-amd64` +- **Linux**: `linux-amd64`, `linux-armv7`, `linux-aarch64` +- **macOS**: `macos-amd64`, `macos-arm64` + +## Features + +- **Multi-architecture support**: Builds for x86_64 and ARM architectures +- **Cross-platform compatibility**: Supports Windows, macOS, and Linux +- **Automatic dependency handling**: Installs Python dependencies and system packages +- **Flexible data file inclusion**: Supports per-script configuration with wildcard support +- **Executable verification**: Tests built executables to ensure they run correctly +- **Windows icon support**: Allows custom icons for Windows executables +- **Flexible `PyInstaller` configuration**: Supports additional `PyInstaller` arguments +- **ARM cross-compilation**: Uses Public Preview runners for aarch64 and Docker containers with Qemu for ARMv7 architecture builds +- **GLIBC 2.31+ support**: Uses older Linux images in Docker for best compatibility with Linux targets + +## Usage Examples + +For more detailed explanation of input variables and advanced use cases, please see the [Inputs](#inputs) section below. + +### Basic Usage + +```yaml +- name: Build Python executable + uses: espressif/python-binary-action@master + with: + scripts: 'main.py' + output-dir: './dist' + target-platform: 'linux-amd64' +``` + +### Multi-File Build with Data Files + +Sometimes your Python script might require additional non-Python files, like assets (images), JSON or YAML files. These can be included for either all scripts or you can define files per script. + +For example with the following project structure: + +```txt +my_script/ +src/ +├── assets/ +| └── image.svg +├── config/ +| └── config.json +├── app.py +├── main.py +├── icon.ico +├── pyproject.toml +└── README.md +``` + +There is a config file that is used in both `app.py` and `main.py`, but only `app.py` requires images. The action can look something like this: + +```yaml +- name: Build multiple executables + uses: espressif/python-binary-action@master + with: + # Mandatory args + scripts: 'main.py app.py' + output-dir: './binaries' + target-platform: 'windows-amd64' + # Optional args; non-python files to include + include-data-dirs: | + { + "app.py": ["./assets"], + "*": ["./config"] + } + icon-file: './icon.ico' # Icon for Windows executable +``` + +There are two options how to define data files to be included for all scripts: + +1. With wildcard `*` + +```json +{ + "*": ["./config"] +} +``` + +2. Simple list + +```json +["./config"] +``` + +Both options are equivalent, but the wildcard allows you to define additional data files that are specific for one script. As can be seen in the above example for `app.py`. + +### Custom Executable Names + +Sometimes it might be useful to have a control over the name of build binary (executable). For example if we have a following structure of the project: + +```txt +my_script/ +├── src/ +│ ├── __init__.py +│ └── __main__.py +├── pyproject.toml +└── README.md +``` + +Building the project with default configuration will result in script name `__main__.py`, which is probably not desirable. To solve this issue, we can pass optional argument `script-name` that will be used as basename for build binaries (executables). + +```yaml +- name: Build with custom names + uses: espressif/python-binary-action@master + with: + scripts: 'src/__main__.py' + script-name: 'my_script' + output-dir: './dist' + target-platform: 'linux-amd64' +``` + +### ARM Architecture Build + +```yaml +- name: Build for ARMv7 + uses: espressif/python-binary-action@master + with: + scripts: 'main.py' + output-dir: './arm-binaries' + target-platform: 'linux-armv7' + additional-arm-packages: 'openssl libffi-dev libffi7 libssl-dev' + python-version: '3.11' +``` + +### Custom PyInstaller Configuration + +```yaml +- name: Build with custom options + uses: espressif/python-binary-action@master + with: + scripts: 'app.py' + output-dir: './dist' + target-platform: 'macos-arm64' + additional-args: '--hidden-import=requests --hidden-import=urllib3 --strip' + pyinstaller-version: '6.3.0' + test-command-args: '--version' +``` + +### Signing Windows Binaries + +If you would like to sign Windows binaries, you can set `certificate` and the action will also take care of signing all binaries. +It is also recommended to use `certificate-password`. + +The `certificate` should be a PFX (Personal Information Exchange) certificate file encoded in base64 format. + +```yaml +- name: Build Python executable + uses: espressif/python-binary-action@master + with: + scripts: 'app.py' + output-dir: './dist' + target-platform: 'windows-amd64' + certificate: ${{ secrets.CERTIFICATE }} + certificate-password: ${{ secrets.CERTIFICATE_PASSWORD }} +``` + +### Complete Workflow + +Here you can see a simplified version of workflow used in [esptool](https://github.com/espressif/esptool/) repository: + +```yaml +name: Build Executables + +on: [push, pull_request] + +jobs: + build: + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false # Avoid failure of all action in case of one of them fails + # Define platform matrix to build on multiple platforms at the same time + matrix: + platform: [windows-amd64, linux-amd64, macos-arm64, linux-aarch64] + include: + - platform: windows-amd64 + runner: windows-latest + - platform: linux-amd64 + runner: ubuntu-latest + - platform: macos-arm64 + runner: macos-latest + - platform: linux-aarch64 + runner: ubuntu-24.04-arm + + env: + # Used for additional data to be included in executables + # Env variable is only for action simplification + STUBS_DIR: ./esptool/targets/stub_flasher/ + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Build executable + uses: espressif/python-binary-action@master + with: + # Required options + scripts: 'esptool.py' # Building from 'esptool.py' file + output-dir: './${{ matrix.platform }}' + target-platform: ${{ matrix.platform }} + # Optional args; non-python files that will be added to build executable + # We want to include the content of subdirectories `1` and `2` of the directory stored in environment variable `STUBS_DIR`. + include-data-dirs: | + { + "esptool.py": [ + "${{ env.STUBS_DIR }}1", + "${{ env.STUBS_DIR }}2", + ] + } + + - name: Add license and readme + shell: bash + run: cp LICENSE README.md ./${{ matrix.platform }} + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: executable-${{ matrix.platform }} + path: ./${{ matrix.platform }} +``` + +## Inputs + +### Required Inputs + +| Input | Description | Example | +|-------------------|-------------------------------------------------|----------------------------| +| `scripts` | Space-separated list of Python scripts to build | `"esptool.py espefuse.py"` | +| `output-dir` | Output directory for built executables | `"./dist-linux-amd64"` | +| `target-platform` | Target platform for the build | `"linux-amd64"` | + +### Optional Inputs + +| Input | Description | Default | Example | +|---------------------------|-------------------------------------------|---------------------------------------------|----------------------------------------------| +| `script-name` | Custom names for the output executables. Must provide exactly one name per script in the same order. (On Windows `.exe` suffix will be added to each name) | `""` | `"foo bar"` | +| `include-data-dirs` | Mapping script names to data directories to include. Supports wildcards (*). | `[]` | `{"main.py": ["./data"], "*": ["./common"]}` | +| `icon-file` | Path to icon file (Windows only) | `""` | `"./icon.ico"` | +| `python-version` | Python version to use for building | `"3.13"` | `"3.12"` | +| `pyinstaller-version` | PyInstaller version to install | `6.11.1` | `""` (use latest) | +| `additional-args` | Additional PyInstaller arguments | `""` | `"--hidden-import=module"` | +| `pip-extra-index-url` | Extra pip index URL | `https://dl.espressif.com/pypi` | `""` | +| `install-deps-command` | Command to install project dependencies | `"pip install --user --prefer-binary -e ."` | `"pip install -r requirements.txt"` | +| `additional-arm-packages` | ARMv7 ONLY: Additional system packages | `""` | `"openssl libffi-dev"` | +| `test-command-args` | Command arguments to test executables | `"--help"` | `"--version"` | +| `certificate` | Certificate to use for signing binaries | `""` | `${{ secrets.CERTIFICATE }}` | +| `certificate-password` | Password for the certificate | `""` | `${{ secrets.CERTIFICATE_PASSWORD }}` | + +> [!IMPORTANT] +> Be careful when changing `pyinstaller-version` as it might lead to increased false positives with anti-virus software. It is recommended to check your executables with antivirus software such as [Virustotal](https://www.virustotal.com/gui/home/upload). + +## Outputs + +| Output | Description | +|------------------------|----------------------------------------------------------------| +| `executable-extension` | File extension of built executables (e.g., `.exe` for Windows) | +| `build-success` | Whether the build was successful (`true`/`false`) | + +## Notes + +- For 32-bit ARM architecture (linux-armv7), the action uses Docker containers to provide the necessary build environment +- For 64-bit ARM architecture please use the GitHub provided runners, e.g. `ubuntu-24.04-arm`. Please note that this is still in public preview so there might be some changes to images. For more details see [available runners](https://docs.github.com/en/actions/how-tos/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources). +- Windows builds automatically include `.exe` extensions +- The action automatically tests built executables using the specified test command arguments +- System packages for ARMv7 builds can be customized using the `additional-arm-packages` input. For other systems, this can be done before running this action. +- It is recommended to add `fail-fast: false` to your matrix strategy to prevent one platform failure from stopping all builds diff --git a/Sign-File.ps1 b/Sign-File.ps1 new file mode 100644 index 0000000..1fb542e --- /dev/null +++ b/Sign-File.ps1 @@ -0,0 +1,79 @@ +[CmdletBinding()] +param ( + [Parameter()] + [String] + $Path +) + + +function FindSignTool { + $SignTool = "signtool.exe" + if (Get-Command $SignTool -ErrorAction SilentlyContinue) { + return $SignTool + } + $SignTool = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\x64\signtool.exe" + if (Test-Path -Path $SignTool -PathType Leaf) { + return $SignTool + } + $SignTool = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\x86\signtool.exe" + if (Test-Path -Path $SignTool -PathType Leaf) { + return $SignTool + } + $sdkVers = "10.0.22000.0", "10.0.20348.0", "10.0.19041.0", "10.0.17763.0" + Foreach ($ver in $sdkVers) + { + $SignTool = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\${ver}\x64\signtool.exe" + if (Test-Path -Path $SignTool -PathType Leaf) { + return $SignTool + } + } + "signtool.exe not found" + Exit 1 +} + +function SignFile { + param( + [Parameter()] + [String] + $Path + ) + + $SignTool = FindSignTool + "Using: $SignTool" + "Signing file: $Path" + $CertificateFile = [system.io.path]::GetTempPath() + "certificate.pfx" + + if ($null -eq $env:CERTIFICATE) { + "CERTIFICATE variable not set, unable to sign the file" + Exit 1 + } + + if ("" -eq $env:CERTIFICATE) { + "CERTIFICATE variable is empty, unable to sign the file" + Exit 1 + } + + $SignParameters = @("sign", "/tr", 'http://timestamp.digicert.com', "/td", "SHA256", "/f", $CertificateFile, "/fd", "SHA256") + if ($env:CERTIFICATE_PASSWORD) { + "CERTIFICATE_PASSWORD detected, using the password" + $SignParameters += "/p" + $SignParameters += $env:CERTIFICATE_PASSWORD + } + $SignParameters += $Path + + [byte[]]$CertificateBytes = [convert]::FromBase64String($env:CERTIFICATE) + [IO.File]::WriteAllBytes($CertificateFile, $CertificateBytes) + + &$SignTool $SignParameters + + if (0 -eq $LASTEXITCODE) { + Remove-Item $CertificateFile + } else { + Remove-Item $CertificateFile + "Signing failed" + Exit 1 + } + +} + +SignFile ${Path} diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..28b4fc9 --- /dev/null +++ b/action.yml @@ -0,0 +1,263 @@ +name: Python Binary Build Action +description: Build Python applications into standalone executables for multiple architectures + +inputs: + scripts: + description: Space-separated list of Python scripts to build + required: true + output-dir: + description: Output directory for built executables + required: true + include-data-dirs: + description: 'JSON object mapping script names to data directories to include. + Supports wildcards (*) and exact matches. Format: {"script.py": ["./data"], + "*": ["./common"]}' + required: false + default: '[]' + script-name: + description: Space-separated list of output names for executables. Must match + the length of `scripts` variable. If not provided, the script name will be used. + required: false + default: '' + icon-file: + description: Path to icon file (Windows only) + required: false + default: '' + target-platform: + description: 'Target platform: windows-amd64, linux-amd64, macos-amd64, macos-arm64, + linux-armv7, linux-aarch64' + required: true + python-version: + description: Python version to use for building. Make sure that the version is + available on all runners. + required: false + default: '3.13' + pyinstaller-version: + description: PyInstaller version to install. Default is 6.11.1 because it has + the lowest false positive rate with antivirus. For latest version, use empty + string. + required: false + default: 6.11.1 + additional-args: + description: Additional PyInstaller arguments + required: false + default: '' + pip-extra-index-url: + description: Extra pip index URL + required: false + default: https://dl.espressif.com/pypi + install-deps-command: + description: Command to install project dependencies. Command will be executed + like `python -m {install-deps-command}` + required: false + default: pip install --user --prefer-binary -e . + additional-arm-packages: + description: 'ARMv7 ONLY: Additional system packages to install (space-separated). + e.g. for cryptography: openssl libffi-dev libffi7 libssl-dev' + required: false + default: '' + test-command-args: + description: Command arguments to test binaries (e.g. "--help") + required: false + default: --help + certificate: + description: Certificate to use for signing binaries + required: false + default: '' + certificate-password: + description: Password for the certificate + required: false + default: '' + +outputs: + executable-extension: + description: File extension of built executables + value: ${{ steps.setup-platform.outputs.exe-extension }} + build-success: + description: Whether the build was successful + value: ${{ steps.build.outputs.success }} + +runs: + using: composite + steps: + - name: Setup platform variables + id: setup-platform + shell: bash + run: | + case "${{ inputs.target-platform }}" in + windows-amd64) + echo "exe-extension=.exe" >> $GITHUB_OUTPUT + echo "data-separator=;" >> $GITHUB_OUTPUT + echo "needs-docker=false" >> $GITHUB_OUTPUT + echo "needs-arm-emulation=false" >> $GITHUB_OUTPUT + ;; + linux-amd64|linux-aarch64) + echo "exe-extension=" >> $GITHUB_OUTPUT + echo "data-separator=:" >> $GITHUB_OUTPUT + echo "needs-docker=true" >> $GITHUB_OUTPUT + echo "needs-arm-emulation=false" >> $GITHUB_OUTPUT + ;; + macos-amd64|macos-arm64) + echo "exe-extension=" >> $GITHUB_OUTPUT + echo "data-separator=:" >> $GITHUB_OUTPUT + echo "needs-docker=false" >> $GITHUB_OUTPUT + echo "needs-arm-emulation=false" >> $GITHUB_OUTPUT + ;; + linux-armv7) + echo "exe-extension=" >> $GITHUB_OUTPUT + echo "data-separator=:" >> $GITHUB_OUTPUT + echo "needs-docker=false" >> $GITHUB_OUTPUT + echo "needs-arm-emulation=true" >> $GITHUB_OUTPUT + ;; + esac + + - name: Prepare build environment + shell: bash + run: | + # Create output directory + mkdir -p "${{ inputs.output-dir }}" + + # Write include-data-dirs to a temporary file to avoid shell escaping issues + echo '${{ inputs.include-data-dirs }}' > ${GITHUB_ACTION_PATH}/include_data_dirs.json + + - name: Build for ARMv7 architecture + if: inputs.target-platform == 'linux-armv7' + uses: uraimo/run-on-arch-action@v3 + with: + arch: armv7 + # Use Ubuntu 20.04 because of missing libffi7 in 24.04 (cryptography requires it) and better compatibility with GLIBC + distro: ubuntu20.04 + shell: /bin/bash + githubToken: ${{ github.token }} + setup: mkdir -p "${PWD}/${{ inputs.output-dir }}" + dockerRunArgs: | + --volume "${PWD}/${{ inputs.output-dir }}:/${{ inputs.output-dir }}" + --volume "${GITHUB_ACTION_PATH}:/github/action" + install: | + apt-get update -y + apt-get install -y software-properties-common + add-apt-repository -y ppa:deadsnakes/ppa + apt-get update -y + apt-get install --ignore-missing -y curl python${{ inputs.python-version }} python${{ inputs.python-version }}-dev pkg-config gcc g++ patchelf binutils zlib1g-dev ${{ inputs.additional-arm-packages }} + # Install pip for requested Python version using get-pip.py as ensurepip does not work here + curl -sS https://bootstrap.pypa.io/get-pip.py | python${{ inputs.python-version }} + python${{ inputs.python-version }} -m pip install --upgrade pip setuptools wheel + run: | + adduser --disabled-password --gecos "" builder + chmod -R a+rwx /home/runner/work + su builder <<'EOF' + export PATH=$PATH:/home/builder/.local/bin + + # Setup environment using helper script + /github/action/setup_environment.sh \ + "python${{ inputs.python-version }}" \ + "${{ inputs.pyinstaller-version }}" \ + "${{ inputs.pip-extra-index-url }}" \ + "${{ inputs.install-deps-command }}" + + # Write include-data-dirs to a temporary file to avoid shell escaping issues + echo '${{ inputs.include-data-dirs }}' > /tmp/include_data_dirs.json + + # Build each file using the script + export GITHUB_ACTION_PATH=/github/action + /github/action/build_with_pyinstaller.sh \ + "python${{ inputs.python-version }}" \ + "${{ inputs.target-platform }}" \ + "${{ inputs.output-dir }}" \ + "${{ inputs.scripts }}" \ + "${{ inputs.script-name }}" \ + "" \ + "/tmp/include_data_dirs.json" \ + "${{ steps.setup-platform.outputs.data-separator }}" \ + "${{ inputs.additional-args }}" + + # Test binaries using helper script + /github/action/test_executables.sh \ + "${{ inputs.scripts }}" \ + "${{ inputs.script-name }}" \ + "${{ inputs.output-dir }}" \ + "${{ steps.setup-platform.outputs.exe-extension }}" \ + "${{ inputs.test-command-args }}" + EOF + + - name: Build for Linux (Docker) + # Running in Docker to link older versions of GLIBC, because the GitHub runners don't offer ubuntu 20.04 anymore + if: steps.setup-platform.outputs.needs-docker == 'true' + shell: bash + run: | + docker run --rm \ + -v "${PWD}/${{ inputs.output-dir }}:/${{ inputs.output-dir }}" \ + -v "${GITHUB_ACTION_PATH}:/github/action" \ + -v "${GITHUB_WORKSPACE}:/${GITHUB_WORKSPACE}" \ + -w ${GITHUB_WORKSPACE} \ + ubuntu:20.04 \ + bash -c " + apt-get update -y && + apt-get install -y software-properties-common curl && + add-apt-repository -y ppa:deadsnakes/ppa && + apt-get update -y && + apt-get install -y python${{ inputs.python-version }} python${{ inputs.python-version }}-dev pkg-config gcc g++ patchelf binutils zlib1g-dev && + curl -sS https://bootstrap.pypa.io/get-pip.py | python${{ inputs.python-version }} && + python${{ inputs.python-version }} -m pip install --upgrade pip setuptools wheel && + + export GITHUB_ACTION_PATH=/github/action && + /github/action/setup_environment.sh \ + 'python${{ inputs.python-version }}' \ + '${{ inputs.pyinstaller-version }}' \ + '${{ inputs.pip-extra-index-url }}' \ + '${{ inputs.install-deps-command }}' && + + # Execute the build script + /github/action/build_with_pyinstaller.sh 'python${{ inputs.python-version }}' '${{ inputs.target-platform }}' '${{ inputs.output-dir }}' '${{ inputs.scripts }}' '${{ inputs.script-name }}' '${{ inputs.icon-file }}' '/github/action/include_data_dirs.json' '${{ steps.setup-platform.outputs.data-separator }}' '${{ inputs.additional-args }}' + " + + - name: Setup environment (Native platforms) + if: | + steps.setup-platform.outputs.needs-docker == 'false' && steps.setup-platform.outputs.needs-arm-emulation == 'false' + shell: bash + run: | + $GITHUB_ACTION_PATH/setup_environment.sh \ + 'python' \ + '${{ inputs.pyinstaller-version }}' \ + '${{ inputs.pip-extra-index-url }}' \ + '${{ inputs.install-deps-command }}' + + - name: Build with PyInstaller (Native platforms) + if: | + steps.setup-platform.outputs.needs-docker == 'false' && steps.setup-platform.outputs.needs-arm-emulation == 'false' + id: build + shell: bash + run: | + $GITHUB_ACTION_PATH/build_with_pyinstaller.sh 'python' '${{ inputs.target-platform }}' '${{ inputs.output-dir }}' '${{ inputs.scripts }}' '${{ inputs.script-name }}' '${{ inputs.icon-file }}' '${{ inputs.include-data-dirs }}' '${{ steps.setup-platform.outputs.data-separator }}' '${{ inputs.additional-args }}' + + echo "success=true" >> $GITHUB_OUTPUT + + - name: Verify builds + if: steps.setup-platform.outputs.needs-arm-emulation == 'false' + shell: bash + run: |- + $GITHUB_ACTION_PATH/test_executables.sh \ + "${{ inputs.scripts }}" \ + "${{ inputs.script-name }}" \ + "${{ inputs.output-dir }}" \ + "${{ steps.setup-platform.outputs.exe-extension }}" \ + "${{ inputs.test-command-args }}" + + - name: Sign binaries + if: inputs.target-platform == 'windows-amd64' + env: + CERTIFICATE: ${{ inputs.certificate }} + CERTIFICATE_PASSWORD: ${{ inputs.certificate-password }} + shell: pwsh + run: |- + if ([string]::IsNullOrEmpty($env:CERTIFICATE)) { + Write-Host "::warning title=Signing::Certificate is not set, skipping signing" + exit 0 + } + + $pythonFiles = "${{ inputs.scripts }}".Split(' ') + foreach ($file in $pythonFiles) { + $baseName = [System.IO.Path]::GetFileNameWithoutExtension($file) + $executable = "./${{ inputs.output-dir }}/${baseName}${{ steps.setup-platform.outputs.exe-extension }}" + & (Join-Path $env:GITHUB_ACTION_PATH "Sign-File.ps1") -Path $executable + } diff --git a/build_with_pyinstaller.sh b/build_with_pyinstaller.sh new file mode 100755 index 0000000..71f26cb --- /dev/null +++ b/build_with_pyinstaller.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Build Python scripts with PyInstaller +# Usage: ./build_with_pyinstaller.sh [python_version] [target_platform] [output_dir] [scripts] [script_names] [icon_file] [include_data_dirs] [data_separator] [additional_args] + +set -e + +# Parse arguments +PYTHON_VERSION="${1:-python}" +TARGET_PLATFORM="${2}" +OUTPUT_DIR="${3}" +SCRIPTS="${4}" +SCRIPT_NAMES="${5}" +ICON_FILE="${6}" +INCLUDE_DATA_DIRS="${7}" +DATA_SEPARATOR="${8}" +ADDITIONAL_ARGS="${9}" + +echo "Building with PyInstaller..." +echo "Python version: $PYTHON_VERSION" +echo "Target platform: $TARGET_PLATFORM" +echo "Output directory: $OUTPUT_DIR" +echo "Scripts: $SCRIPTS" +echo "Script names: $SCRIPT_NAMES" + +# Build each Python file +IFS=' ' read -ra PYTHON_FILES <<< "$SCRIPTS" +IFS=' ' read -ra SCRIPT_NAMES_ARRAY <<< "$SCRIPT_NAMES" + +for i in "${!PYTHON_FILES[@]}"; do + file="${PYTHON_FILES[$i]}" + echo "Building $file for $TARGET_PLATFORM..." + + # Start building the command + cmd="$PYTHON_VERSION -m PyInstaller --onefile --distpath=$OUTPUT_DIR" + + # Add custom name if provided + if [ -n "$SCRIPT_NAMES" ] && [ $i -lt ${#SCRIPT_NAMES_ARRAY[@]} ]; then + custom_name="${SCRIPT_NAMES_ARRAY[$i]}" + cmd="$cmd --name=$custom_name" + echo "Using custom name: $custom_name" + fi + + # Windows-specific options + if [ "$TARGET_PLATFORM" = "windows-amd64" ]; then + if [ -n "$ICON_FILE" ]; then + cmd="$cmd --icon=$ICON_FILE" + fi + fi + + # Add include-data-dirs using Python script + if [ -n "$INCLUDE_DATA_DIRS" ]; then + echo "Processing include-data-dirs for $file..." + include_flags=$($PYTHON_VERSION $GITHUB_ACTION_PATH/process_include_dirs.py "$INCLUDE_DATA_DIRS" "$DATA_SEPARATOR" "$file") + echo "Include flags result: '$include_flags'" + if [ -n "$include_flags" ]; then + cmd="$cmd $include_flags" + echo "Added include flags to command" + else + echo "No include flags generated" + fi + fi + + # Add additional arguments + if [ -n "$ADDITIONAL_ARGS" ]; then + cmd="$cmd $ADDITIONAL_ARGS" + fi + + # Add the file to build + cmd="$cmd $file" + + echo "Executing: $cmd" + eval "$cmd" +done + +echo "Build completed successfully" diff --git a/process_include_dirs.py b/process_include_dirs.py new file mode 100644 index 0000000..e6fae89 --- /dev/null +++ b/process_include_dirs.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +""" +Process include-data-dirs JSON and generate PyInstaller arguments +Supports per-script configuration with wildcard support +""" + +import json +import argparse +import fnmatch +import sys +import os + + +def fix_windows_paths(include_dirs_json: str) -> str: + """Fix Windows paths in the include directories""" + if sys.platform == "win32": + include_dirs_json = include_dirs_json.replace("\\", "/") + return include_dirs_json + + +def process_include_dirs(include_dirs_json: str, data_separator: str, target_script: str) -> str: + """Convert include directories JSON to PyInstaller --add-data flags + for a specific script""" + try: + include_dirs_config = json.loads(fix_windows_paths(include_dirs_json)) + if not include_dirs_config: + return "" + + # For simplicity, we also support simple list format + # This is equivalent to the dict format with wildcard + if isinstance(include_dirs_config, list): + # Simple format: apply to all scripts + include_dirs = include_dirs_config + elif isinstance(include_dirs_config, dict): + # Dict format: per-script configuration + include_dirs = [] + processed_patterns = set() + + # Check for wildcard entries first + for pattern, dirs in include_dirs_config.items(): + if pattern == "*" or fnmatch.fnmatch(target_script, pattern): + if isinstance(dirs, list): + include_dirs.extend(dirs) + else: + include_dirs.append(dirs) + processed_patterns.add(pattern) + + # Check for exact script name (only if not already processed by wildcard) + if target_script in include_dirs_config and target_script not in processed_patterns: + dirs = include_dirs_config[target_script] + if isinstance(dirs, list): + include_dirs.extend(dirs) + else: + include_dirs.append(dirs) + else: + return "" + + if not include_dirs: + return "" + + flags = [] + for item in include_dirs: + flags.append(f"--add-data='{item}{data_separator}{item}'") + + return " ".join(flags) + except (json.JSONDecodeError, KeyError, TypeError) as e: + print( + f"Warning: Error processing include dirs for {target_script}: {e}", + file=sys.stderr, + ) + return "" + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Process include directories for PyInstaller", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Simple list format (applies to all scripts) + python process_include_dirs.py '["./data"]' ':' 'main.py' + + # Dict format (per-script configuration with wildcard support) + python process_include_dirs.py '{"main.py": ["./assets"], "*": ["./common"]}' ':' 'main.py' + + # File input (to avoid shell escaping issues) + python process_include_dirs.py '/tmp/include_data_dirs.json' ':' 'main.py' + """, # noqa: E501 + ) + parser.add_argument("include_dirs", help="JSON string or file path of include directories") + parser.add_argument( + "data_separator", + help="Data separator for PyInstaller --add-data (; for Windows, : for Unix)", + ) + parser.add_argument("target_script", help="Target script name for filtering") + + args = parser.parse_args() + + # Check if include_dirs is a file path + if os.path.isfile(args.include_dirs): + with open(args.include_dirs, "r") as f: + include_dirs_content = f.read() + else: + include_dirs_content = args.include_dirs + + result = process_include_dirs(include_dirs_content, args.data_separator, args.target_script) + print(result) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0612870 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[tool.mypy] + disallow_incomplete_defs = true + disallow_untyped_defs = true + ignore_missing_imports = true + python_version = "3.13" + warn_no_return = true + warn_return_any = true + +[tool.ruff] + line-length = 120 + target-version = "py313" + +[tool.ruff.format] + quote-style = "double" + indent-style = "space" + docstring-code-format = true + +[tool.yamlfix] + whitelines = 1 + section_whitelines = 1 + sequence_style = "keep_style" + explicit_start = false diff --git a/setup_environment.sh b/setup_environment.sh new file mode 100755 index 0000000..5c82fb6 --- /dev/null +++ b/setup_environment.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Setup Python environment for PyInstaller builds +# Usage: ./setup_environment.sh [python_version] [pyinstaller_version] [pip_extra_index_url] [install_deps_command] + +set -e + +PYTHON_VERSION="${1:-python}" +PYINSTALLER_VERSION="${2:-6.11.1}" +PIP_EXTRA_INDEX_URL="${3:-}" +INSTALL_DEPS_COMMAND="${4:-pip install --user --prefer-binary -e .}" + +echo "Setting up Python environment..." +echo "Python version: $PYTHON_VERSION" +echo "PyInstaller version: $PYINSTALLER_VERSION" + +# Set pip extra index if provided +if [ -n "$PIP_EXTRA_INDEX_URL" ]; then + export PIP_EXTRA_INDEX_URL="$PIP_EXTRA_INDEX_URL" + echo "Using extra pip index: $PIP_EXTRA_INDEX_URL" +fi + +# Install PyInstaller +if [ -z "$PYINSTALLER_VERSION" ]; then + $PYTHON_VERSION -m pip install pyinstaller +else + $PYTHON_VERSION -m pip install pyinstaller==$PYINSTALLER_VERSION +fi + +# Install dependencies +echo "Installing project dependencies..." +$PYTHON_VERSION -m $INSTALL_DEPS_COMMAND + +echo "Environment setup completed successfully" diff --git a/test_executables.sh b/test_executables.sh new file mode 100755 index 0000000..1276482 --- /dev/null +++ b/test_executables.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Test built executables +# Usage: ./test_executables.sh [scripts] [script_names] [output_dir] [exe_extension] [test_args] + +set -e + +SCRIPTS="${1}" +SCRIPT_NAMES="${2}" +OUTPUT_DIR="${3}" +EXE_EXTENSION="${4}" +TEST_ARGS="${5:---help}" + +echo "Testing built executables..." + +IFS=' ' read -ra PYTHON_FILES <<< "$SCRIPTS" +IFS=' ' read -ra SCRIPT_NAMES_ARRAY <<< "$SCRIPT_NAMES" + +for i in "${!PYTHON_FILES[@]}"; do + file="${PYTHON_FILES[$i]}" + + # Determine executable name + if [ -n "$SCRIPT_NAMES" ] && [ $i -lt ${#SCRIPT_NAMES_ARRAY[@]} ]; then + custom_name="${SCRIPT_NAMES_ARRAY[$i]}" + executable="$OUTPUT_DIR/${custom_name}${EXE_EXTENSION}" + else + base_name=$(basename "$file" .py) + executable="$OUTPUT_DIR/${base_name}${EXE_EXTENSION}" + fi + + echo "Testing $executable..." + if [ -f "$executable" ]; then + echo "✓ $executable exists ($(du -h "$executable" | cut -f1))" + if "$executable" $TEST_ARGS; then + echo "✓ $executable runs successfully" + else + echo "⚠ $executable may have issues" + exit 1 + fi + else + echo "✗ $executable not found" + exit 1 + fi +done + +echo "All executables tested successfully"