diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index d4c1d05a1..a649c8af1 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -178,6 +178,7 @@ jobs: --platform linux \ --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" \ --max-shards 2 \ + --event ${{ github.event_name }} \ ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} \ --free-runners \ > matrix.json diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index ab97c1cd0..7bfef5801 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -92,7 +92,11 @@ jobs: - name: Generate build matrix id: set-matrix run: | +<<<<<<< HEAD uv run ci-matrix.py --platform darwin --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} --free-runners > matrix.json +======= + uv run ci-matrix.py --platform darwin --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" --event ${{ github.event_name }} ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json +>>>>>>> refs/tags/20260610 # Extract python-build matrix echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index fbf633419..9199b50d7 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -92,7 +92,11 @@ jobs: - name: Generate build matrix id: set-matrix run: | +<<<<<<< HEAD uv run ci-matrix.py --platform windows --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} --free-runners > matrix.json +======= + uv run ci-matrix.py --platform windows --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" --event ${{ github.event_name }} ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json +>>>>>>> refs/tags/20260610 # Extract python-build matrix echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c7e0193b5..4a9716cb3 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -8,28 +8,41 @@ Building distributions See the [documentation](https://gregoryszorc.com/docs/python-build-standalone/main/building.html) for instructions on building distributions locally. -CI labels -========= -By default, submitting a pull request triggers a complete build of all -distributions in CI, which can be time-consuming. +Pull request labels +=================== +By default, pull requests build a small subset of targets defined in +``ci-defaults.yaml`` under ``pull_request``. Pushes to ``main`` build the full +matrix from ``ci-targets.yaml``. -To conserve CI resources and reduce build times, you can limit the matrix of -distributions built by applying specific labels to your pull request. Only -distributions matching the specified labels will be built. +Pull request labels can be used to change what CI builds: -The following label prefixes can be used to customize the build matrix: +* ``platform:`` filters the selected targets by platform. +* ``arch:`` filters the selected targets by architecture. +* ``libc:`` filters the selected targets by libc. +* ``python:`` filters the selected Python versions. +* ``build:`` filters the selected build options by component. -* `platform` -* `python` -* `build` -* `arch` -* `libc` +The ``:all`` labels expand only their own dimension: -To bypass CI entirely for changes that do not affect the build (such as -documentation updates), use the `ci:skip` label. +* ``platform:all`` expands the selected platforms. +* ``arch:all`` expands the selected architectures. +* ``libc:all`` expands the selected libc variants. +* ``python:all`` expands the selected Python versions. +* ``build:all`` expands the selected build options. -Please utilize these tags when appropriate for your changes to minimize CI -resource consumption. +Use ``ci:all-targets`` to build the full matrix from ``ci-targets.yaml``. + +Examples: + +* ``platform:linux`` builds only the Linux targets from ``ci-defaults.yaml``. +* ``python:3.13`` builds the default targets with Python 3.13. +* ``build:pgo`` builds the selected targets whose build options include ``pgo``. +* ``platform:linux,arch:all,libc:all,python:all,build:all`` builds the full + Linux matrix. + +To bypass CI entirely for changes that do not affect the build, use the +``ci:skip`` label. The ``documentation`` label is treated the same way. To run +a dry-run build matrix, use ``ci:dry-run``. Releases ======== diff --git a/ci-defaults.yaml b/ci-defaults.yaml new file mode 100644 index 000000000..56f612675 --- /dev/null +++ b/ci-defaults.yaml @@ -0,0 +1,28 @@ +# Describes the default targets that CI will build for different events. + +pull_request: + python_version: "3.14" + + targets: + x86_64-pc-windows-msvc: + build_options: + - pgo + + aarch64-apple-darwin: + build_options: + - pgo+lto + + x86_64-unknown-linux-gnu: + build_options: + - pgo+lto + - freethreaded+pgo+lto + + x86_64-unknown-linux-musl: + build_options: + - lto + - lto+static + - freethreaded+lto + + armv7-unknown-linux-gnueabihf: + build_options: + - lto diff --git a/ci-matrix.py b/ci-matrix.py index e4b1f3ba8..ff2d28185 100644 --- a/ci-matrix.py +++ b/ci-matrix.py @@ -16,6 +16,7 @@ CI_TARGETS_YAML = "ci-targets.yaml" CI_RUNNERS_YAML = "ci-runners.yaml" +CI_DEFAULTS_YAML = "ci-defaults.yaml" CI_EXTRA_SKIP_LABELS = ["documentation"] CI_MATRIX_SIZE_LIMIT = 256 # The maximum size of a matrix in GitHub Actions @@ -40,7 +41,7 @@ def meets_conditional_version(version: str, min_version: str) -> bool: def parse_labels(labels: str | None) -> dict[str, set[str]]: - """Parse labels into a dict of category filters.""" + """Parse labels into a dict of category -> set of values.""" if not labels: return {} @@ -75,30 +76,157 @@ def parse_labels(labels: str | None) -> dict[str, set[str]]: return result -def should_include_entry(entry: dict[str, str], filters: dict[str, set[str]]) -> bool: - """Check if an entry satisfies the label filters.""" - if filters.get("directives") and "skip" in filters["directives"]: - return False +def get_all_build_options(ci_config: dict[str, Any], target_triple: str) -> list[str]: + """Get all build options (including conditional) for a target from ci-targets.yaml.""" + for platform_config in ci_config.values(): + if target_triple in platform_config: + config = platform_config[target_triple] + options = list(config["build_options"]) + for conditional in config.get("build_options_conditional", []): + options.extend(conditional["options"]) + return options + raise KeyError(f"Target triple {target_triple!r} not found in ci-targets.yaml") + + +def find_target_platform(ci_config: dict[str, Any], target_triple: str) -> str: + """Find which platform a target triple belongs to in ci-targets.yaml.""" + for platform, platform_config in ci_config.items(): + if target_triple in platform_config: + return platform + raise KeyError(f"Target triple {target_triple!r} not found in ci-targets.yaml") + + +def expand_default_triples( + ci_config: dict[str, Any], + pull_request_defaults: dict[str, Any], + labels: dict[str, set[str]], +) -> set[str]: + """Compute the set of allowed target triples for a pull request. + + Starts from the explicit defaults in ci-defaults.yaml. When a target + dimension has an :all label, that dimension is relaxed and additional + triples from ci-targets.yaml that match on the non-expanded dimensions + are included. + """ + default_triples = set(pull_request_defaults["targets"]) + + platform_labels = labels.get("platform", set()) + arch_labels = labels.get("arch", set()) + libc_labels = labels.get("libc", set()) + + platform_filters = platform_labels - {"all"} + arch_filters = arch_labels - {"all"} + libc_filters = libc_labels - {"all"} + + expand_platform = "all" in platform_labels or bool(platform_filters) + expand_arch = "all" in arch_labels or bool(arch_filters) + expand_libc = "all" in libc_labels or bool(libc_filters) + + if not (expand_platform or expand_arch or expand_libc): + return default_triples + + # Build reference tuples from the default triples. + default_attrs = [] + for triple in default_triples: + platform = find_target_platform(ci_config, triple) + config = ci_config[platform][triple] + default_attrs.append( + ( + platform, + config["arch"], + config.get("arch_variant"), + config.get("libc"), + ) + ) - if filters.get("platform") and entry["platform"] not in filters["platform"]: + # Include any triple whose non-expanded dimensions match a default. + allowed = set(default_triples) + for platform, platform_config in ci_config.items(): + for triple, config in platform_config.items(): + for d_platform, d_arch, d_arch_variant, d_libc in default_attrs: + if platform_filters: + if platform not in platform_filters: + continue + elif "all" not in platform_labels and platform != d_platform: + continue + + if arch_filters: + if config["arch"] not in arch_filters: + continue + if config.get("arch_variant") != d_arch_variant: + continue + elif "all" not in arch_labels and ( + config["arch"] != d_arch + or config.get("arch_variant") != d_arch_variant + ): + continue + + if libc_filters: + if config.get("libc") not in libc_filters: + continue + elif "all" not in libc_labels and config.get("libc") != d_libc: + continue + allowed.add(triple) + break + + return allowed + + +def should_include_entry( + entry: dict[str, str], + labels: dict[str, set[str]], + pull_request_defaults: dict[str, Any] | None = None, + allowed_triples: set[str] | None = None, +) -> bool: + """Check if a matrix entry should be included. + + For pull requests, entries are restricted to the allowed target set + (computed by expand_default_triples), the default python version, and + the curated build options — unless overridden by labels. For pushes + (pull_request_defaults is None), only label filters apply. + """ + if pull_request_defaults is not None: + triple = entry["target_triple"] + default_targets = pull_request_defaults["targets"] + + # Target must be in the allowed set. + if allowed_triples is not None and triple not in allowed_triples: + return False + + # Python: restrict to default version unless python labels override. + if not labels.get("python"): + if entry["python"] != pull_request_defaults["python_version"]: + return False + + # Build options: restrict to curated defaults for default triples + # unless build labels override. Non-default triples (brought in by + # :all expansion) are unrestricted. + if not labels.get("build") and triple in default_targets: + if entry["build_options"] not in default_targets[triple]["build_options"]: + return False + + # Label filters + platform_filters = labels.get("platform", set()) - {"all"} + if platform_filters and entry["platform"] not in platform_filters: return False - if filters.get("python") and entry["python"] not in filters["python"]: + python_filters = labels.get("python", set()) - {"all"} + if python_filters and entry["python"] not in python_filters: return False - if filters.get("arch") and entry["arch"] not in filters["arch"]: + arch_filters = labels.get("arch", set()) - {"all"} + if arch_filters and entry["arch"] not in arch_filters: return False - if ( - filters.get("libc") - and entry.get("libc") - and entry["libc"] not in filters["libc"] - ): + libc_filters = labels.get("libc", set()) - {"all"} + if libc_filters and entry.get("libc") and entry["libc"] not in libc_filters: return False - if filters.get("build"): - build_options = set(entry.get("build_options", "").split("+")) - if not all(f in build_options for f in filters["build"]): + build_filters = labels.get("build", set()) - {"all"} + if build_filters: + build_components = set(entry.get("build_options", "").split("+")) + required = {c for f in build_filters for c in f.split("+")} + if not required.issubset(build_components): return False return True @@ -106,17 +234,32 @@ def should_include_entry(entry: dict[str, str], filters: dict[str, set[str]]) -> def generate_docker_matrix_entries( runners: dict[str, Any], + python_entries: list[dict[str, str]], platform_filter: str | None = None, free_runners: bool = False, ) -> list[dict[str, str]]: - """Generate matrix entries for docker image builds.""" + """Generate matrix entries for Docker image builds.""" if platform_filter and platform_filter != "linux": return [] + needed_archs = { + runners[entry["runner"]]["arch"] + for entry in python_entries + if entry.get("platform") == "linux" + } + matrix_entries = [] for image in DOCKER_BUILD_IMAGES: +<<<<<<< HEAD # Find appropriate runner for Linux platform with the specified architecture runner = find_runner(runners, "linux", image["arch"], free_runners) +======= + if image["arch"] not in needed_archs: + continue + + # Find appropriate runner for Linux platform with the specified architecture. + runner = find_runner(runners, "linux", image["arch"], False) +>>>>>>> refs/tags/20260610 entry = { "name": image["name"], @@ -203,14 +346,6 @@ def generate_python_build_matrix_entries( free_runners, ) - # Apply label filters if present - if label_filters: - matrix_entries = [ - entry - for entry in matrix_entries - if should_include_entry(entry, label_filters) - ] - return matrix_entries @@ -327,6 +462,50 @@ def add_python_build_entries_for_config( matrix_entries.append(entry) +def validate_pull_request_defaults( + ci_config: dict[str, Any], pull_request_defaults: dict[str, Any] +) -> None: + """Validate the pull_request defaults in ci-defaults.yaml.""" + all_triples = set() + for platform_config in ci_config.values(): + all_triples.update(platform_config.keys()) + + for triple in pull_request_defaults["targets"]: + if triple not in all_triples: + print( + f"error: target triple {triple!r} in {CI_DEFAULTS_YAML}:pull_request " + f"not found in {CI_TARGETS_YAML}", + file=sys.stderr, + ) + sys.exit(1) + + # Validate that each build option listed is valid for the target. + all_options = set(get_all_build_options(ci_config, triple)) + for option in pull_request_defaults["targets"][triple]["build_options"]: + if option not in all_options: + print( + f"error: build option {option!r} for {triple} in " + f"{CI_DEFAULTS_YAML}:pull_request not found in {CI_TARGETS_YAML} " + f"(valid: {sorted(all_options)})", + file=sys.stderr, + ) + sys.exit(1) + + # Validate that the default python version exists in ci-targets.yaml. + default_version = pull_request_defaults["python_version"] + for triple in pull_request_defaults["targets"]: + platform = find_target_platform(ci_config, triple) + ci_versions = ci_config[platform][triple]["python_versions"] + if default_version not in ci_versions: + print( + f"error: python version {default_version!r} in " + f"{CI_DEFAULTS_YAML}:pull_request not available for {triple} in " + f"{CI_TARGETS_YAML} (valid: {ci_versions})", + file=sys.stderr, + ) + sys.exit(1) + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Generate a JSON matrix for building distributions in CI" @@ -344,7 +523,12 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--labels", - help="Comma-separated list of labels to filter by (e.g., 'platform:darwin,python:3.13,build:debug'), all must match.", + help="Comma-separated list of labels to filter by (e.g., 'platform:linux,python:3.13')", + ) + parser.add_argument( + "--event", + choices=["pull_request", "push"], + help="The GitHub event type. When 'pull_request', uses ci-defaults.yaml for the default subset.", ) parser.add_argument( "--free-runners", @@ -370,11 +554,14 @@ def main() -> None: labels = parse_labels(args.labels) with open(CI_TARGETS_YAML) as f: - config = yaml.safe_load(f) + ci_config = yaml.safe_load(f) with open(CI_RUNNERS_YAML) as f: runners = yaml.safe_load(f) + with open(CI_DEFAULTS_YAML) as f: + ci_defaults = yaml.safe_load(f) or {} + # If only free runners are allowed, reduce to a subset if args.free_runners: runners = { @@ -383,16 +570,48 @@ def main() -> None: if runner_config.get("free") } + # Check for skip directive + if labels.get("directives") and "skip" in labels["directives"]: + # Emit empty matrices + result = {} + if args.matrix_type in ["python-build", "all"]: + if args.max_shards: + result["python-build"] = { + str(i): {"include": []} for i in range(args.max_shards) + } + else: + result["python-build"] = {"include": []} + if args.matrix_type in ["docker-build", "all"]: + result["docker-build"] = {"include": []} + if args.matrix_type in ["crate-build", "all"]: + result["crate-build"] = {"include": []} + print(json.dumps(result)) + return + + event_defaults = ci_defaults.get(args.event) if args.event else None + if "all-targets" in labels.get("directives", set()): + event_defaults = None + + allowed_triples = None + if event_defaults is not None: + validate_pull_request_defaults(ci_config, event_defaults) + allowed_triples = expand_default_triples(ci_config, event_defaults, labels) + result = {} - # Generate python build entries + # Generate all python build entries, then filter python_entries = generate_python_build_matrix_entries( - config, + ci_config, runners, args.platform, labels, args.free_runners, ) + python_entries = [ + entry + for entry in python_entries + if should_include_entry(entry, labels, event_defaults, allowed_triples) + ] # Output python-build matrix if requested if args.matrix_type in ["python-build", "all"]: @@ -420,18 +639,19 @@ def main() -> None: result["python-build"] = {"include": python_entries} # Generate docker-build matrix if requested - # Only include docker builds if there are Linux python builds + # Only include docker builds if there are Linux python builds. if args.matrix_type in ["docker-build", "all"]: - # Check if we have any Linux python builds + # Check if we have any Linux python builds. has_linux_builds = any( entry.get("platform") == "linux" for entry in python_entries ) - # If no platform filter or explicitly requesting docker-build only, include docker builds - # Otherwise, only include if there are Linux python builds + # If no platform filter or explicitly requesting docker-build only, include docker builds. + # Otherwise, only include if there are Linux python builds. if args.matrix_type == "docker-build" or has_linux_builds: docker_entries = generate_docker_matrix_entries( runners, + python_entries, args.platform, args.free_runners, ) @@ -442,7 +662,7 @@ def main() -> None: crate_entries = generate_crate_build_matrix_entries( python_entries, runners, - config, + ci_config, # Use the full target config so --force-crate-build adds all native crate builds. args.force_crate_build, args.platform, args.free_runners, diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index fdb49c588..3385c2fda 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -154,12 +154,15 @@ else patch -p1 -i "${ROOT}/patch-macos-link-extension-modules.patch" fi +<<<<<<< HEAD # Also on macOS, the `python` executable is linked against libraries defined by statically # linked modules. But those libraries should only get linked into libpython, not the # executable. This behavior is kinda suspect on all platforms, as it could be adding # library dependencies that shouldn't need to be there. # PYSTANDALONE: skip this patch. +======= +>>>>>>> refs/tags/20260610 # The macOS code for sniffing for _dyld_shared_cache_contains_path falls back on a # possibly inappropriate code path if a configure time check fails. This is not # appropriate for certain cross-compiling scenarios. See discussion at @@ -438,19 +441,15 @@ CONFIGURE_FLAGS=" # Build a libpython3.x.so, but statically link the interpreter against # libpython. -# -# For now skip this on macos, because it causes some linker failures. Note that -# this patch mildly conflicts with the macos-only patch-python-link-modules -# applied above, so you will need to resolve that conflict if you re-enable -# this for macos. -if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then +# Merged upstream in Python 3.15, https://github.com/python/cpython/pull/133313 +if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]; then if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then patch -p1 -i "${ROOT}/patch-python-configure-add-enable-static-libpython-for-interpreter.patch" else patch -p1 -i "${ROOT}/patch-python-configure-add-enable-static-libpython-for-interpreter-${PYTHON_MAJMIN_VERSION}.patch" fi - CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --enable-static-libpython-for-interpreter" fi +CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --enable-static-libpython-for-interpreter" if [ "${CC}" = "musl-clang" ]; then # In order to build the _blake2 extension module with SSE3+ instructions, we need @@ -631,12 +630,6 @@ if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" && -n "${CROSS_COMPILING}" && "$ PROFILE_TASK="${PROFILE_TASK} --ignore test_strftime_y2k" fi -# On 3.14+ `test_json.test_recursion.TestCRecursion.test_highly_nested_objects_decoding` fails during -# PGO due to RecursionError not being raised as expected. See https://github.com/python/cpython/issues/140125 -if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then - PROFILE_TASK="${PROFILE_TASK} --ignore test_json" -fi - # PGO optimized / BOLT instrumented binaries segfault in a test_bytes test. Skip it. if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" && "${TARGET_TRIPLE}" == x86_64* ]]; then PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.BytesTest.test_from_format" @@ -753,6 +746,14 @@ elif [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_11}" ]; then patch -p1 -i "${ROOT}/patch-getpath-use-base_executable-for-executable_dir.patch" fi +# Cherry-pick python/cpython#100373 from 3.12 onto 3.10/3.11 to fix DER parsing with OpenSSL 3.5.7. +# The old implementation does heuristics on the error codes returned from the ASN.1 parser, which +# have changed slightly in openssl/openssl#30986 (commit 738688d762 in 3.5.7). The new +# implementation does a better job of parsing and avoids the heuristics in the first place. +if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_11}" ]; then + patch -p1 -i "${ROOT}/patch-python-der-eof-parsing.patch" +fi + # We patched configure.ac above. Reflect those changes. autoconf @@ -1376,6 +1377,13 @@ if [ -d "${TOOLS_PATH}/deps/lib/tcl9" ]; then ) fi +# Prune the tk demos which are > 1 MB and not used +rm -rf "${ROOT}/out/python/install/lib/tk9.0/demos" + +# Prune the tcl/tk translations +rm -rf "${ROOT}/out/python/install/lib/tcl9.0/msgs" +rm -rf "${ROOT}/out/python/install/lib/tk9.0/msgs" + # Copy the terminfo database if present. if [ -d "${TOOLS_PATH}/deps/usr/share/terminfo" ]; then cp -av "${TOOLS_PATH}/deps/usr/share/terminfo" "${ROOT}/out/python/install/share/" diff --git a/cpython-unix/build-patchelf.sh b/cpython-unix/build-patchelf.sh index c21663303..1acf6c559 100755 --- a/cpython-unix/build-patchelf.sh +++ b/cpython-unix/build-patchelf.sh @@ -13,6 +13,37 @@ tar -xf "patchelf-${PATCHELF_VERSION}.tar.bz2" pushd patchelf-0.13.1.20211127.72b6d44 +# TODO: Drop this patch once patchelf is updated to 0.14.0 or newer, +# which includes native LoongArch64 support. +# See: https://github.com/astral-sh/python-build-standalone/issues/1106 +if [[ "${TARGET_TRIPLE}" = loongarch64* ]]; then + patch -p1 << 'EOF' +diff --git a/src/patchelf.cc b/src/patchelf.cc +index 2b7ec8b9..06f41c6f 100644 +--- a/src/patchelf.cc ++++ b/src/patchelf.cc +@@ -57,6 +57,10 @@ static int forcedPageSize = DEFAULT_PAGESIZE; + static int forcedPageSize = -1; + #endif + ++#ifndef EM_LOONGARCH ++#define EM_LOONGARCH 258 ++#endif ++ + using FileContents = std::shared_ptr>; + + #define ElfFileParams class Elf_Ehdr, class Elf_Phdr, class Elf_Shdr, class Elf_Addr, class Elf_Off, class Elf_Dyn, class Elf_Sym, class Elf_Verneed, class Elf_Versym +@@ -460,6 +464,7 @@ unsigned int ElfFile::getPageSize() const + case EM_PPC64: + case EM_AARCH64: + case EM_TILEGX: ++ case EM_LOONGARCH: + return 0x10000; + default: + return 0x1000; +EOF +fi + CC="${HOST_CC}" CXX="${HOST_CXX}" CFLAGS="${EXTRA_HOST_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_HOST_CFLAGS} -fPIC" \ ./configure \ --build="${BUILD_TRIPLE}" \ diff --git a/cpython-unix/build.Dockerfile b/cpython-unix/build.Dockerfile index 5a5dcad8a..e10c0f02d 100644 --- a/cpython-unix/build.Dockerfile +++ b/cpython-unix/build.Dockerfile @@ -7,12 +7,14 @@ # Compression packages are needed to extract archives. # # Various other build tools are needed for various building. +# Note linux-headers is installed to source a missing UAPI header, see below RUN ulimit -n 10000 && apt-get install \ bzip2 \ ca-certificates \ curl \ file \ libc6-dev \ + linux-headers-3.16.0-6-common \ libffi-dev \ make \ patch \ @@ -23,3 +25,8 @@ RUN ulimit -n 10000 && apt-get install \ unzip \ zip \ zlib1g-dev + +# Debian Jessie's linux-libc-dev is missing the vm_sockets header due to a typo +# see https://lists.openwall.net/netdev/2014/12/01/2 +RUN install -m 0644 /usr/src/linux-headers-3.16.0-6-common/include/uapi/linux/vm_sockets.h \ + /usr/include/linux/vm_sockets.h \ No newline at end of file diff --git a/cpython-unix/extension-modules.yml b/cpython-unix/extension-modules.yml index 524c59b80..e49a115c6 100644 --- a/cpython-unix/extension-modules.yml +++ b/cpython-unix/extension-modules.yml @@ -442,6 +442,8 @@ _remote_debugging: - _remote_debugging/code_objects.c - _remote_debugging/frame_cache.c - _remote_debugging/frames.c + - _remote_debugging/gc_stats.c + - _remote_debugging/interpreters.c - _remote_debugging/module.c - _remote_debugging/object_reading.c - _remote_debugging/subprocess.c @@ -720,6 +722,9 @@ _testcapi: maximum-python-version: "3.12" - source: _testcapi/watchers.c minimum-python-version: "3.12" + - source: _testcapi/weakref.c + minimum-python-version: "3.13" + maximum-python-version: "3.14" _testexternalinspection: minimum-python-version: '3.13' @@ -786,10 +791,15 @@ _testlimitedcapi: sources-conditional: - source: _testlimitedcapi/codec.c minimum-python-version: "3.14" + - source: _testlimitedcapi/slots.c + minimum-python-version: "3.15" - source: _testlimitedcapi/threadstate.c minimum-python-version: "3.15" - source: _testlimitedcapi/version.c minimum-python-version: "3.14" + - source: _testlimitedcapi/weakref.c + minimum-python-version: "3.13" + maximum-python-version: "3.14" _testmultiphase: minimum-python-version: '3.10' diff --git a/cpython-unix/patch-jit-llvm-version-3.13.patch b/cpython-unix/patch-jit-llvm-version-3.13.patch index f8f6fd0a7..f980855d7 100644 --- a/cpython-unix/patch-jit-llvm-version-3.13.patch +++ b/cpython-unix/patch-jit-llvm-version-3.13.patch @@ -1,4 +1,5 @@ diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py +index b68da1b6be1..cf04f83539e 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -8,7 +8,7 @@ @@ -7,6 +8,6 @@ diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py -_LLVM_VERSION = 18 +_LLVM_VERSION = 22 - _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") - - _P = typing.ParamSpec("_P") + _LLVM_VERSION_PATTERN = re.compile( + rf"(? +Date: Fri, 24 Mar 2023 09:04:30 -0400 +Subject: [PATCH 1/1] gh-100372: Use BIO_eof to detect EOF for + SSL_FILETYPE_ASN1 (GH-100373) + +In PEM, we need to parse until error and then suppress `PEM_R_NO_START_LINE`, because PEM allows arbitrary leading and trailing data. DER, however, does not. Parsing until error and suppressing `ASN1_R_HEADER_TOO_LONG` doesn't quite work because that error also covers some cases that should be rejected. + +Instead, check `BIO_eof` early and stop the loop that way. + +Automerge-Triggered-By: GH:Yhg1s +--- + Lib/test/test_ssl.py | 2 ++ + .../2022-12-20-10-55-14.gh-issue-100372.utfP65.rst | 2 ++ + Modules/_ssl.c | 10 ++++++---- + 3 files changed, 10 insertions(+), 4 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst + +diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py +index 1317efb33c2..abf024fb89d 100644 +--- a/Lib/test/test_ssl.py ++++ b/Lib/test/test_ssl.py +@@ -1289,6 +1289,8 @@ def test_load_verify_cadata(self): + "not enough data: cadata does not contain a certificate" + ): + ctx.load_verify_locations(cadata=b"broken") ++ with self.assertRaises(ssl.SSLError): ++ ctx.load_verify_locations(cadata=cacert_der + b"A") + + @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows") + def test_load_dh_params(self): +diff --git a/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst b/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst +new file mode 100644 +index 00000000000..ec37aff5092 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst +@@ -0,0 +1,2 @@ ++:meth:`ssl.SSLContext.load_verify_locations` no longer incorrectly accepts ++some cases of trailing data when parsing DER. +diff --git a/Modules/_ssl.c b/Modules/_ssl.c +index 36b66cdb531..3fbb37332f6 100644 +--- a/Modules/_ssl.c ++++ b/Modules/_ssl.c +@@ -3930,7 +3930,7 @@ _add_ca_certs(PySSLContext *self, const void *data, Py_ssize_t len, + { + BIO *biobuf = NULL; + X509_STORE *store; +- int retval = -1, err, loaded = 0; ++ int retval = -1, err, loaded = 0, was_bio_eof = 0; + + assert(filetype == SSL_FILETYPE_ASN1 || filetype == SSL_FILETYPE_PEM); + +@@ -3958,6 +3958,10 @@ _add_ca_certs(PySSLContext *self, const void *data, Py_ssize_t len, + int r; + + if (filetype == SSL_FILETYPE_ASN1) { ++ if (BIO_eof(biobuf)) { ++ was_bio_eof = 1; ++ break; ++ } + cert = d2i_X509_bio(biobuf, NULL); + } else { + cert = PEM_read_bio_X509(biobuf, NULL, +@@ -3993,9 +3997,7 @@ _add_ca_certs(PySSLContext *self, const void *data, Py_ssize_t len, + } + _setSSLError(get_state_ctx(self), msg, 0, __FILE__, __LINE__); + retval = -1; +- } else if ((filetype == SSL_FILETYPE_ASN1) && +- (ERR_GET_LIB(err) == ERR_LIB_ASN1) && +- (ERR_GET_REASON(err) == ASN1_R_HEADER_TOO_LONG)) { ++ } else if ((filetype == SSL_FILETYPE_ASN1) && was_bio_eof) { + /* EOF ASN1 file, not an error */ + ERR_clear_error(); + retval = 0; +-- +2.50.1 (Apple Git-155) + diff --git a/cpython-unix/patch-python-getpath-backport-3.13.patch b/cpython-unix/patch-python-getpath-backport-3.13.patch index d6c853deb..048529ce1 100644 --- a/cpython-unix/patch-python-getpath-backport-3.13.patch +++ b/cpython-unix/patch-python-getpath-backport-3.13.patch @@ -1,8 +1,7 @@ -From 3342daa091b0a8e6cf15fdaaa2c6fc2f9dcc8a60 Mon Sep 17 00:00:00 2001 +From 74a45cb4adeb18e5e071d9495244b1f2ad23e874 Mon Sep 17 00:00:00 2001 From: Geoffrey Thomas Date: Tue, 16 Dec 2025 09:29:55 -0500 -Subject: [PATCH 1/1] Backport relevant parts of 3.14 getpath.c to 3.13 -Forwarded: not-needed +Subject: [PATCH] Backport relevant parts of 3.14 getpath.c to 3.13 --- Modules/getpath.c | 38 +++++++++++++++----------------------- @@ -78,15 +77,15 @@ index d0128b20fae..50612432027 100644 } diff --git a/configure.ac b/configure.ac -index 94776540d1b..fbc6cfd0de4 100644 +index a016a439c6c..071f8905ab0 100644 --- a/configure.ac +++ b/configure.ac -@@ -5217,7 +5217,7 @@ fi +@@ -5242,7 +5242,7 @@ fi # checks for library functions AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ -- copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ -+ copy_file_range ctermid dladdr dup dup3 execv explicit_bzero explicit_memset \ +- copy_file_range ctermid dup execv explicit_bzero explicit_memset \ ++ copy_file_range ctermid dladdr dup execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ @@ -105,5 +104,5 @@ index e18a6426b06..10b0cc1dafd 100644 #undef HAVE_DLFCN_H -- -2.50.1 (Apple Git-155) +2.54.0 diff --git a/cpython-unix/patch-python-link-modules-3.10.patch b/cpython-unix/patch-python-link-modules-3.10.patch deleted file mode 100644 index 58247aebd..000000000 --- a/cpython-unix/patch-python-link-modules-3.10.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/Makefile.pre.in b/Makefile.pre.in ---- a/Makefile.pre.in -+++ b/Makefile.pre.in -@@ -563,7 +563,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c - - # Build the interpreter - $(BUILDPYTHON): Programs/python.o $(LIBRARY_DEPS) -- $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) -+ $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(SYSLIBS) - - platform: $(BUILDPYTHON) pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform diff --git a/cpython-unix/patch-python-link-modules-3.11.patch b/cpython-unix/patch-python-link-modules-3.11.patch deleted file mode 100644 index 8bc7aee63..000000000 --- a/cpython-unix/patch-python-link-modules-3.11.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/Makefile.pre.in b/Makefile.pre.in -index b356f6293e..89fddd4d4e 100644 ---- a/Makefile.pre.in -+++ b/Makefile.pre.in -@@ -702,7 +702,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c - - # Build the interpreter - $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) -- $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) -+ $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(SYSLIBS) - - platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform diff --git a/cpython-unix/patch-python-link-modules-3.15.patch b/cpython-unix/patch-python-link-modules-3.15.patch deleted file mode 100644 index 5225bb6f9..000000000 --- a/cpython-unix/patch-python-link-modules-3.15.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/Makefile.pre.in b/Makefile.pre.in -index 120a6add385..4d8abc5256a 100644 ---- a/Makefile.pre.in -+++ b/Makefile.pre.in -@@ -990,7 +990,7 @@ clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c - - # Build the interpreter - $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) -- $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) -+ $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(SYSLIBS) - - platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform diff --git a/cpython-unix/targets.yml b/cpython-unix/targets.yml index 588c2db79..d0db3e39e 100644 --- a/cpython-unix/targets.yml +++ b/cpython-unix/targets.yml @@ -267,6 +267,7 @@ loongarch64-unknown-linux-gnu: - '3.12' - '3.13' - '3.14' + - '3.15' docker_image_suffix: .cross-loongarch64 host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -275,6 +276,7 @@ loongarch64-unknown-linux-gnu: target_ldflags: # Hardening - '-Wl,-z,noexecstack' + - '-Wl,-z,max-page-size=0x10000' needs: - autoconf - bdb diff --git a/cpython-windows/build.py b/cpython-windows/build.py index ef5097601..5a6fc9051 100644 --- a/cpython-windows/build.py +++ b/cpython-windows/build.py @@ -296,6 +296,58 @@ def find_vcvarsall_path(msvc_version): ) +<<<<<<< HEAD +======= +class NoSearchStringError(Exception): + """Represents a missing search string when replacing content in a file.""" + + +def static_replace_in_file(p: pathlib.Path, search, replace): + """Replace occurrences of a string in a file. + + The updated file contents are written out in place. + """ + + with p.open("rb") as fh: + data = fh.read() + + # Build should be as deterministic as possible. Assert that wanted changes + # actually occur. + if search not in data: + raise NoSearchStringError("search string (%s) not in %s" % (search, p)) + + log("replacing `%s` with `%s` in %s" % (search, replace, p)) + data = data.replace(search, replace) + + with p.open("wb") as fh: + fh.write(data) + + +def apply_source_patch(cpython_source_path: pathlib.Path, patch_path: pathlib.Path): + with patch_path.open("rb") as fh: + patch = fh.read().replace(b"\r\n", b"\n") + + with tempfile.NamedTemporaryFile("wb", delete=False) as fh: + fh.write(patch) + normalized_patch = pathlib.Path(fh.name) + + try: + subprocess.run( + [ + "git.exe", + "-C", + str(cpython_source_path), + "apply", + "--whitespace=nowarn", + str(normalized_patch), + ], + check=True, + ) + finally: + normalized_patch.unlink() + + +>>>>>>> refs/tags/20260610 OPENSSL_PROPS_REMOVE_RULES_LEGACY = b""" <_SSLDLL Include="$(opensslOutDir)\libcrypto$(_DLLSuffix).dll" /> @@ -357,8 +409,10 @@ def hack_props( mpdecimal_version = DOWNLOADS["mpdecimal"]["version"] - if meets_python_minimum_version(python_version, "3.14") or arch == "arm64": - tcltk_commit = DOWNLOADS["tk-windows-bin"]["git_commit"] + if meets_python_minimum_version(python_version, "3.15"): + tcltk_commit = DOWNLOADS["tk-windows-bin-903"]["git_commit"] + elif meets_python_minimum_version(python_version, "3.14") or arch == "arm64": + tcltk_commit = DOWNLOADS["tk-windows-bin-8614"]["git_commit"] else: tcltk_commit = DOWNLOADS["tk-windows-bin-8612"]["git_commit"] @@ -660,17 +714,17 @@ def hack_project_files( pass # Our custom OpenSSL build has applink.c in a different location from the - # binary OpenSSL distribution. This is no longer relevant for 3.12+ per - # https://github.com/python/cpython/pull/131839, so we allow it to fail.swe - try: + # binary OpenSSL distribution. + # Starting with 3.12 CPython on Windows is built without uplink support + # so this modification is not needed. + # https://github.com/python/cpython/pull/131839 + if meets_python_maximum_version(python_version, "3.11"): ssl_proj = pcbuild_path / "_ssl.vcxproj" static_replace_in_file( ssl_proj, rb'', rb'', ) - except NoSearchStringError: - pass # Python 3.12+ uses the the pre-built tk-windows-bin 8.6.12 which doesn't # have a standalone zlib DLL, so we remove references to it. For Python @@ -975,6 +1029,7 @@ def build_openssl_for_arch( build_root: pathlib.Path, *, jom_archive, + with_uplink: bool = False, ): nasm_version = DOWNLOADS["nasm-windows-bin"]["version"] @@ -994,14 +1049,15 @@ def build_openssl_for_arch( source_root = build_root / ("openssl-%s" % openssl_version) - # uplink.c tries to find the OPENSSL_Applink function exported from the current - # executable. However, it is exported from _ssl[_d].pyd in shared builds. So - # update its sounce to look for it from there. - static_replace_in_file( - source_root / "ms" / "uplink.c", - b"((h = GetModuleHandle(NULL)) == NULL)", - b'((h = GetModuleHandleA("_ssl.pyd")) == NULL) if ((h = GetModuleHandleA("_ssl_d.pyd")) == NULL) if ((h = GetModuleHandle(NULL)) == NULL)', - ) + if with_uplink: + # uplink.c tries to find the OPENSSL_Applink function exported from the + # current executable. However, it is exported from _ssl[_d].pyd in shared + # builds. So update its source to look for it from there. + static_replace_in_file( + source_root / "ms" / "uplink.c", + b"((h = GetModuleHandle(NULL)) == NULL)", + b'((h = GetModuleHandleA("_ssl.pyd")) == NULL) if ((h = GetModuleHandleA("_ssl_d.pyd")) == NULL) if ((h = GetModuleHandle(NULL)) == NULL)', + ) if arch == "x86": configure = "VC-WIN32" @@ -1015,26 +1071,27 @@ def build_openssl_for_arch( else: raise Exception("unhandled architecture: %s" % arch) - # The official CPython OpenSSL builds hack ms/uplink.c to change the - # ``GetModuleHandle(NULL)`` invocation to load things from _ssl.pyd - # instead. But since we statically link the _ssl extension, this hackery - # is not required. - # Set DESTDIR to affect install location. dest_dir = build_root / "install" env["DESTDIR"] = str(dest_dir) install_root = dest_dir / prefix + configure_args = [ + str(perl_path), + "Configure", + configure, + "no-idea", + "no-mdc2", + "no-tests", + "--prefix=/%s" % prefix, + ] + if with_uplink: + log("building OpenSSL with uplink support for Python <3.12") + else: + configure_args.append("no-uplink") + exec_and_log( - [ - str(perl_path), - "Configure", - configure, - "no-idea", - "no-mdc2", - "no-tests", - "--prefix=/%s" % prefix, - ], + configure_args, source_root, { **env, @@ -1076,6 +1133,7 @@ def build_openssl( perl_path: pathlib.Path, arch: str, dest_archive: pathlib.Path, + with_uplink: bool = False, ): """Build OpenSSL from sources using the Perl executable specified.""" @@ -1103,6 +1161,7 @@ def build_openssl( nasm_archive, root_32, jom_archive=jom_archive, + with_uplink=with_uplink, ) elif arch == "amd64": root_64.mkdir() @@ -1114,6 +1173,7 @@ def build_openssl( nasm_archive, root_64, jom_archive=jom_archive, + with_uplink=with_uplink, ) elif arch == "arm64": root_arm64.mkdir() @@ -1125,6 +1185,7 @@ def build_openssl( nasm_archive, root_arm64, jom_archive=jom_archive, + with_uplink=with_uplink, ) else: raise Exception("unhandled architecture: %s" % arch) @@ -1274,20 +1335,23 @@ def collect_python_build_artifacts( pcbuild_path: pathlib.Path, out_dir: pathlib.Path, python_majmin: str, - arch: str, + pcbuild_directory: str, config: str, openssl_entry: str, zlib_entry: str, freethreaded: bool, ): """Collect build artifacts from Python. - Copies them into an output directory and returns a data structure describing the files. """ - outputs_path = pcbuild_path / arch + arch = pcbuild_directory + # Python 3.15 suffixes the directory with 't' for free-threading + if arch.endswith("t"): + arch = arch.removesuffix("t") + outputs_path = pcbuild_path / pcbuild_directory intermediates_path = ( - pcbuild_path / "obj" / ("%s%s_%s" % (python_majmin, arch, config)) + pcbuild_path / "obj" / ("%s%s_%s" % (python_majmin, pcbuild_directory, config)) ) if not outputs_path.exists(): @@ -1339,6 +1403,11 @@ def collect_python_build_artifacts( } other_projects = {"pythoncore"} +<<<<<<< HEAD +======= + other_projects.add("python3dll") + other_projects.add("python3tdll") +>>>>>>> refs/tags/20260610 # Projects providing dependencies. depends_projects = set() @@ -1590,6 +1659,7 @@ def build_cpython( xz_archive = download_entry("xz", BUILD) zlib_archive = download_entry(zlib_entry, BUILD) +<<<<<<< HEAD # On CPython 3.14+, we use the latest tcl/tk version which has additional # runtime dependencies, so we are conservative and use the old version # elsewhere. The old version isn't built for arm64, so we use the new @@ -1599,6 +1669,22 @@ def build_cpython( if meets_python_minimum_version(python_version, "3.14") or arch == "arm64" else "tk-windows-bin-8612" ) +======= + setuptools_wheel = download_entry("setuptools", BUILD) + pip_wheel = download_entry("pip", BUILD) + + # We use a prebuild tcl/tk from the upstream CPython project. + # Tcl/tk 8.6.14+ has an additional runtime dependency. We are conservative and + # use an old version prior to CPython 3.14. The older tck/tk release + # is not available for arm64 so we use a newer release there as well. + # On CPython 3.14+ we match the version included in the Python.org release. + if meets_python_minimum_version(python_version, "3.15"): + tk_bin_entry = "tk-windows-bin-903" + elif meets_python_minimum_version(python_version, "3.14") or arch == "arm64": + tk_bin_entry = "tk-windows-bin-8614" + else: + tk_bin_entry = "tk-windows-bin-8612" +>>>>>>> refs/tags/20260610 tk_bin_archive = download_entry( tk_bin_entry, BUILD, local_name="tk-windows-bin.tar.gz" ) @@ -1625,6 +1711,11 @@ def build_cpython( python_exe = "python.exe" pythonw_exe = "pythonw.exe" + # Python 3.15 uses the default name for the executable in a suffixed directory + instrumented_python_exe = python_exe + if meets_python_minimum_version(python_version, "3.15") and freethreaded: + instrumented_python_exe = "python.exe" + if arch == "amd64": build_platform = "x64" build_directory = "amd64" @@ -1637,6 +1728,12 @@ def build_cpython( else: raise Exception("unhandled architecture: %s" % arch) + pcbuild_directory = build_directory + # Starting with 3.15, free-threaded CPython outputs use a `t` suffix. + # The third-party dependency archives still use the base architecture name. + if freethreaded and meets_python_minimum_version(python_version, "3.15"): + pcbuild_directory = f"{build_directory}t" + tempdir_opts = ( {"ignore_cleanup_errors": True} if sys.version_info >= (3, 12) else {} ) @@ -1677,7 +1774,7 @@ def build_cpython( extract_tar_to_directory(libffi_archive, td) # Delete the tk nmake helper, it's not needed and links msvc - if tk_bin_entry == "tk-windows-bin": + if tk_bin_entry in ("tk-windows-bin-8614", "tk-windows-bin-903"): tcltk_commit: str = DOWNLOADS[tk_bin_entry]["git_commit"] tcltk_path = td / ("cpython-bin-deps-%s" % tcltk_commit) ( @@ -1691,6 +1788,18 @@ def build_cpython( cpython_source_path = td / ("Python-%s" % python_version) pcbuild_path = cpython_source_path / "PCbuild" + # Cherry-pick python/cpython#100373 from 3.12 onto 3.10/3.11 to fix DER + # parsing with OpenSSL 3.5.7. The old implementation does heuristics + # on the error codes returned from the ASN.1 parser, which have changed + # slightly in openssl/openssl#30986 (commit 738688d762 in 3.5.7). The + # new implementation does a better job of parsing and avoids the + # heuristics in the first place. + if meets_python_maximum_version(python_version, "3.11"): + apply_source_patch( + cpython_source_path, + SUPPORT / "patch-python-der-eof-parsing.patch", + ) + out_dir = td / "out" build_dir = out_dir / "python" / "build" @@ -1761,7 +1870,10 @@ def build_cpython( # test execution. We work around this by invoking the test harness # separately for each test. instrumented_python = ( - pcbuild_path / build_directory / "instrumented" / python_exe + pcbuild_path + / pcbuild_directory + / "instrumented" + / instrumented_python_exe ) tests = subprocess.run( @@ -1870,7 +1982,7 @@ def build_cpython( "--source", str(cpython_source_path), "--build", - str(pcbuild_path / build_directory), + str(pcbuild_path / pcbuild_directory), "--copy", str(install_dir), "--temp", @@ -1914,7 +2026,7 @@ def build_cpython( pcbuild_path, out_dir / "python", "".join(entry["version"].split(".")[0:2]), - build_directory, + pcbuild_directory, artifact_config, openssl_entry=openssl_entry, zlib_entry=zlib_entry, @@ -2160,8 +2272,14 @@ def main() -> None: else: openssl_entry = "openssl-3.5" + openssl_with_uplink = args.python in ["cpython-3.10", "cpython-3.11"] + if openssl_with_uplink: + openssl_build_options = f"{build_options}-uplink" + else: + openssl_build_options = f"{build_options}-no-uplink" + openssl_archive = BUILD / ( - "%s-%s-%s.tar" % (openssl_entry, target_triple, build_options) + "%s-%s-%s.tar" % (openssl_entry, target_triple, openssl_build_options) ) if not openssl_archive.exists(): perl_path = fetch_strawberry_perl() / "perl" / "bin" / "perl.exe" @@ -2171,6 +2289,7 @@ def main() -> None: perl_path, arch, dest_archive=openssl_archive, + with_uplink=openssl_with_uplink, ) libffi_archive = BUILD / ("libffi-%s-%s.tar" % (target_triple, build_options)) diff --git a/cpython-windows/patch-python-der-eof-parsing.patch b/cpython-windows/patch-python-der-eof-parsing.patch new file mode 100644 index 000000000..e48a39b30 --- /dev/null +++ b/cpython-windows/patch-python-der-eof-parsing.patch @@ -0,0 +1,77 @@ +From acfe02f3b05436658d92add6b168538b30f357f0 Mon Sep 17 00:00:00 2001 +From: David Benjamin +Date: Fri, 24 Mar 2023 09:04:30 -0400 +Subject: [PATCH 1/1] gh-100372: Use BIO_eof to detect EOF for + SSL_FILETYPE_ASN1 (GH-100373) + +In PEM, we need to parse until error and then suppress `PEM_R_NO_START_LINE`, because PEM allows arbitrary leading and trailing data. DER, however, does not. Parsing until error and suppressing `ASN1_R_HEADER_TOO_LONG` doesn't quite work because that error also covers some cases that should be rejected. + +Instead, check `BIO_eof` early and stop the loop that way. + +Automerge-Triggered-By: GH:Yhg1s +--- + Lib/test/test_ssl.py | 2 ++ + .../2022-12-20-10-55-14.gh-issue-100372.utfP65.rst | 2 ++ + Modules/_ssl.c | 10 ++++++---- + 3 files changed, 10 insertions(+), 4 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst + +diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py +index 1317efb33c2..abf024fb89d 100644 +--- a/Lib/test/test_ssl.py ++++ b/Lib/test/test_ssl.py +@@ -1289,6 +1289,8 @@ def test_load_verify_cadata(self): + "not enough data: cadata does not contain a certificate" + ): + ctx.load_verify_locations(cadata=b"broken") ++ with self.assertRaises(ssl.SSLError): ++ ctx.load_verify_locations(cadata=cacert_der + b"A") + + @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows") + def test_load_dh_params(self): +diff --git a/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst b/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst +new file mode 100644 +index 00000000000..ec37aff5092 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst +@@ -0,0 +1,2 @@ ++:meth:`ssl.SSLContext.load_verify_locations` no longer incorrectly accepts ++some cases of trailing data when parsing DER. +diff --git a/Modules/_ssl.c b/Modules/_ssl.c +index 36b66cdb531..3fbb37332f6 100644 +--- a/Modules/_ssl.c ++++ b/Modules/_ssl.c +@@ -3930,7 +3930,7 @@ _add_ca_certs(PySSLContext *self, const void *data, Py_ssize_t len, + { + BIO *biobuf = NULL; + X509_STORE *store; +- int retval = -1, err, loaded = 0; ++ int retval = -1, err, loaded = 0, was_bio_eof = 0; + + assert(filetype == SSL_FILETYPE_ASN1 || filetype == SSL_FILETYPE_PEM); + +@@ -3958,6 +3958,10 @@ _add_ca_certs(PySSLContext *self, const void *data, Py_ssize_t len, + int r; + + if (filetype == SSL_FILETYPE_ASN1) { ++ if (BIO_eof(biobuf)) { ++ was_bio_eof = 1; ++ break; ++ } + cert = d2i_X509_bio(biobuf, NULL); + } else { + cert = PEM_read_bio_X509(biobuf, NULL, +@@ -3993,9 +3997,7 @@ _add_ca_certs(PySSLContext *self, const void *data, Py_ssize_t len, + } + _setSSLError(get_state_ctx(self), msg, 0, __FILE__, __LINE__); + retval = -1; +- } else if ((filetype == SSL_FILETYPE_ASN1) && +- (ERR_GET_LIB(err) == ERR_LIB_ASN1) && +- (ERR_GET_REASON(err) == ASN1_R_HEADER_TOO_LONG)) { ++ } else if ((filetype == SSL_FILETYPE_ASN1) && was_bio_eof) { + /* EOF ASN1 file, not an error */ + ERR_clear_error(); + retval = 0; +-- +2.50.1 (Apple Git-155) + diff --git a/pythonbuild/disttests/__init__.py b/pythonbuild/disttests/__init__.py index 7b92f43f2..1c41ff322 100644 --- a/pythonbuild/disttests/__init__.py +++ b/pythonbuild/disttests/__init__.py @@ -239,7 +239,7 @@ def test_testcapi(self): def test_sqlite(self): import sqlite3 - self.assertEqual(sqlite3.sqlite_version_info, (3, 50, 4)) + self.assertEqual(sqlite3.sqlite_version_info, (3, 53, 1)) # Optional SQLite3 features are enabled. conn = sqlite3.connect(":memory:") @@ -314,12 +314,32 @@ def test_ssl(self): if os.name == "nt" and sys.version_info[0:2] < (3, 11): wanted_version = (1, 1, 1, 23, 15) else: - wanted_version = (3, 5, 0, 6, 0) + wanted_version = (3, 5, 0, 7, 0) self.assertEqual(ssl.OPENSSL_VERSION_INFO, wanted_version) ssl.create_default_context() + @unittest.skipIf(os.name != "nt", "Windows-specific OpenSSL uplink regression") + def test_ssl_with_keylogfile(self): + # Validate that a SSLContext can be created when SSLKEYLOGFILE is set + # https://github.com/astral-sh/python-build-standalone/issues/640 + import ssl + + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as td: + keylog_path = str(Path(td) / "sslkeylog.log") + original_keylog_path = os.environ.get("SSLKEYLOGFILE") + os.environ["SSLKEYLOGFILE"] = keylog_path + try: + context = ssl.create_default_context() + self.assertEqual(context.keylog_filename, keylog_path) + del context + finally: + if original_keylog_path is None: + os.environ.pop("SSLKEYLOGFILE", None) + else: + os.environ["SSLKEYLOGFILE"] = original_keylog_path + @unittest.skipIf( sys.version_info[:2] < (3, 13), "Free-threaded builds are only available in 3.13+", @@ -424,6 +444,54 @@ def assertPythonWorks(path: Path, argv0: Optional[str] = None): with self.subTest(msg="weird argv[0]"): assertPythonWorks(sys.executable, argv0="/dev/null") + @unittest.skipUnless(sys.platform == "linux", "Linux-specific socket constant") + # TODO(jjh) remove when musl builds use a sysroot + @unittest.skipIf( + os.environ["TARGET_TRIPLE"].endswith("-musl"), + "kernel headers not available in musl", + ) + def test_socket_af_vsock(self): + import socket + + self.assertTrue(hasattr(socket, "AF_VSOCK")) + self.assertEqual(socket.AF_VSOCK, 40) + + @unittest.skipUnless(sys.platform == "linux", "Linux-specific prctl") + @unittest.skipIf( + "static" in os.environ["BUILD_OPTIONS"], + "cannot import libc on static builds", + ) + def test_nx_thread_creation(self): + "Test that thread creation works under e.g. systemd's MemoryDenyWriteExecute." + # Note that NX cannot be unset so this pollutes the current process, + # but if something else breaks under NX we probably want to know! + import ctypes + import threading + + libc = ctypes.CDLL(None, use_errno=True) + # + PR_SET_MDWE = 65 + PR_GET_MDWE = 66 + PR_MDWE_REFUSE_EXEC_GAIN = 1 << 0 + PR_MDWE_NO_INHERIT = 1 << 1 + mdwe = libc.prctl(PR_GET_MDWE, 0, 0, 0, 0) + if mdwe < 0: + self.skipTest("prctl(PR_SET_MDWE) unsupported") + elif not (mdwe & PR_MDWE_REFUSE_EXEC_GAIN): + if ( + libc.prctl( + PR_SET_MDWE, PR_MDWE_REFUSE_EXEC_GAIN | PR_MDWE_NO_INHERIT, 0, 0, 0 + ) + != 0 + ): + self.fail("prctl(PR_SET_MDWE): " + os.strerror(ctypes.get_errno())) + + a = [] + t = threading.Thread(target=a.append, args=("Thread was here",)) + t.start() + t.join() + self.assertEqual(a, ["Thread was here"]) + if __name__ == "__main__": unittest.main() diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index 8a28180e8..4c8896888 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -75,37 +75,37 @@ "python_tag": "cp312", }, "cpython-3.13": { - "url": "https://www.python.org/ftp/python/3.13.13/Python-3.13.13.tar.xz", - "size": 22957612, - "sha256": "2ab91ff401783ccca64f75d10c882e957bdfd60e2bf5a72f8421793729b78a71", - "version": "3.13.13", + "url": "https://www.python.org/ftp/python/3.13.14/Python-3.13.14.tar.xz", + "size": 23021880, + "sha256": "639e43243c620a308f968213df9e00f2f8f62332f7adbaa7a7eeb9783057c690", + "version": "3.13.14", "licenses": ["Python-2.0", "CNRI-Python"], "license_file": "LICENSE.cpython.txt", "python_tag": "cp313", }, "cpython-3.14": { - "url": "https://www.python.org/ftp/python/3.14.4/Python-3.14.4.tar.xz", - "size": 23855332, - "sha256": "d923c51303e38e249136fc1bdf3568d56ecb03214efdef48516176d3d7faaef8", - "version": "3.14.4", + "url": "https://www.python.org/ftp/python/3.14.6/Python-3.14.6.tar.xz", + "size": 23921184, + "sha256": "143b1dddefaec3bd2e21e3b839b34a2b7fb9842272883c576420d605e9f30c63", + "version": "3.14.6", "licenses": ["Python-2.0", "CNRI-Python"], "license_file": "LICENSE.cpython.txt", "python_tag": "cp314", }, "cpython-3.15": { - "url": "https://www.python.org/ftp/python/3.15.0/Python-3.15.0a8.tar.xz", - "size": 35130268, - "sha256": "28f1b6358609042ebcc81488ec24569519f50804bb07dc23cc707b281b031c69", - "version": "3.15.0a8", + "url": "https://www.python.org/ftp/python/3.15.0/Python-3.15.0b2.tar.xz", + "size": 35381676, + "sha256": "d14f474ab679e90bc734b02ff58447b6ec99a821af61d6ff0c1da0f86e341a71", + "version": "3.15.0b2", "licenses": ["Python-2.0", "CNRI-Python"], "license_file": "LICENSE.cpython.txt", "python_tag": "cp315", }, "expat": { - "url": "https://github.com/libexpat/libexpat/releases/download/R_2_6_3/expat-2.6.3.tar.xz", - "size": 485600, - "sha256": "274db254a6979bde5aad404763a704956940e465843f2a9bd9ed7af22e2c0efc", - "version": "2.6.3", + "url": "https://github.com/libexpat/libexpat/releases/download/R_2_8_1/expat-2.8.1.tar.xz", + "size": 512224, + "sha256": "10b195ee78160a908388180a8fe3603d4e9a12f4755fbf5f3816b23a9d750da0", + "version": "2.8.1", "library_names": ["expat"], "licenses": ["MIT"], "license_file": "LICENSE.expat.txt", @@ -257,10 +257,10 @@ }, # Remember to update OPENSSL_VERSION_INFO in pythonbuild/disttests/ whenever upgrading. "openssl-3.5": { - "url": "https://github.com/openssl/openssl/releases/download/openssl-3.5.6/openssl-3.5.6.tar.gz", - "size": 53121812, - "sha256": "deae7c80cba99c4b4f940ecadb3c3338b13cb77418409238e57d7f31f2a3b736", - "version": "3.5.6", + "url": "https://github.com/openssl/openssl/releases/download/openssl-3.5.7/openssl-3.5.7.tar.gz", + "size": 53153930, + "sha256": "a8c0d28a529ca480f9f36cf5792e2cd21984552a3c8e4aa11a24aa31aeac98e8", + "version": "3.5.7", "library_names": ["crypto", "ssl"], "licenses": ["Apache-2.0"], "license_file": "LICENSE.openssl-3.txt", @@ -279,10 +279,10 @@ "version": "0.13.1", }, "pip": { - "url": "https://files.pythonhosted.org/packages/de/f0/c81e05b613866b76d2d1066490adf1a3dbc4ee9d9c839961c3fc8a6997af/pip-26.0.1-py3-none-any.whl", - "size": 1787723, - "sha256": "bdb1b08f4274833d62c1aa29e20907365a2ceb950410df15fc9521bad440122b", - "version": "26.0.1", + "url": "https://files.pythonhosted.org/packages/5d/95/6b5cb3461ea5673ba0995989746db58eb18b91b54dbf331e72f569540946/pip-26.1.2-py3-none-any.whl", + "size": 1813144, + "sha256": "382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab", + "version": "26.1.2", }, "readline": { # Mirrored from https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz @@ -302,11 +302,11 @@ }, # Remember to update pythonbuild/disttests/ when version changed. "sqlite": { - "url": "https://www.sqlite.org/2025/sqlite-autoconf-3500400.tar.gz", - "size": 3173050, - "sha256": "a3db587a1b92ee5ddac2f66b3edb41b26f9c867275782d46c3a088977d6a5b18", - "version": "3500400", - "actual_version": "3.50.4.0", + "url": "https://www.sqlite.org/2026/sqlite-autoconf-3530100.tar.gz", + "size": 3275272, + "sha256": "83e6b2020a034e9a7ad4a72feea59e1ad52f162e09cbd26735a3ffb98359fc4f", + "version": "3530100", + "actual_version": "3.53.1.0", "library_names": ["sqlite3"], "licenses": [], "license_file": "LICENSE.sqlite.txt", @@ -344,7 +344,14 @@ "licenses": ["TCL"], "license_file": "LICENSE.tcl.txt", }, - "tk-windows-bin": { + "tk-windows-bin-903": { + "url": "https://github.com/python/cpython-bin-deps/archive/2f788ebecac8d4bc3c7fa982b55a6c6923aa55fb.tar.gz", + "size": 18527780, + "sha256": "ac7e489d1fdabb0dbb69896aa8d191b5a87d053ce306fdffa51bbd77b94dbafc", + "version": "9.0.3", + "git_commit": "2f788ebecac8d4bc3c7fa982b55a6c6923aa55fb", + }, + "tk-windows-bin-8614": { "url": "https://github.com/python/cpython-bin-deps/archive/c624cc881bd0e5071dec9de4b120cbe9985d8c14.tar.gz", "size": 9497943, "sha256": "9b8e77d55f40ceaedd140ccca0daa804f0d43346d5abfcead9b547b5590f82f8", diff --git a/src/validation.rs b/src/validation.rs index c72804cad..a6d54787c 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -110,6 +110,7 @@ const PE_ALLOWED_LIBRARIES: &[&str] = &[ "WINMM.dll", "WS2_32.dll", // Our libraries. +<<<<<<< HEAD // "libcrypto-1_1.dll", // "libcrypto-1_1-x64.dll", // "libcrypto-3.dll", @@ -135,6 +136,34 @@ const PE_ALLOWED_LIBRARIES: &[&str] = &[ // "sqlite3.dll", // "tcl86t.dll", // "tk86t.dll", +======= + "libcrypto-1_1.dll", + "libcrypto-1_1-x64.dll", + "libcrypto-3.dll", + "libcrypto-3-arm64.dll", + "libcrypto-3-x64.dll", + "libffi-8.dll", + "libssl-1_1.dll", + "libssl-1_1-x64.dll", + "libssl-3.dll", + "libssl-3-arm64.dll", + "libssl-3-x64.dll", + "python3.dll", + "python3t.dll", + "python39.dll", + "python310.dll", + "python311.dll", + "python312.dll", + "python313.dll", + "python313t.dll", + "python314.dll", + "python314t.dll", + "python315.dll", + "python315t.dll", + "sqlite3.dll", + "tcl86t.dll", + "tk86t.dll", +>>>>>>> refs/tags/20260610 ]; // CPython 3.14 and ARM64 use a newer version of tcl/tk (8.6.14+) which includes a bundled zlib that @@ -255,6 +284,7 @@ static ELF_ALLOWED_LIBRARIES_BY_TRIPLE: Lazy