Skip to content

Remove venv creation for python package install stage #3366

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
T-Dynamos wants to merge 6 commits into
base: develop
Choose a base branch
from
+65 −195
Draft

Remove venv creation for python package install stage #3366

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
84c4448
remove legacy venv creation
T-Dynamos Aug 15, 2026
6fe9751
Fix flake8
T-Dynamos Aug 15, 2026
87ae095
Add --no-deps
T-Dynamos Aug 16, 2026
765956d
Fix bug with version pinning
T-Dynamos Aug 16, 2026
346fb9f
Correct the wording slightly
T-Dynamos Aug 16, 2026
35f3a2f
Remove has_package check
T-Dynamos Aug 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
256 changes: 63 additions & 193 deletions pythonforandroid/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
from contextlib import suppress
import copy
import glob
import os
import json
Expand All @@ -22,13 +20,13 @@
from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64
from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint, Out_Style, Out_Fore)
from pythonforandroid.pythonpackage import get_package_name
from pythonforandroid.recipe import CythonRecipe, Recipe, PyProjectRecipe
from pythonforandroid.recipe import Recipe, PyProjectRecipe
from pythonforandroid.recommendations import (
check_ndk_version, check_target_api, check_ndk_api,
RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API)
from pythonforandroid.util import (
current_directory, ensure_dir,
BuildInterruptingException, rmdir
BuildInterruptingException
)


Expand Down Expand Up @@ -564,117 +562,6 @@ def project_has_setup_py(project_dir):
))


def run_setuppy_install(ctx, project_dir, env=None, arch=None):
env = env or {}

with current_directory(project_dir):
info('got setup.py or similar, running project install. ' +
'(disable this behavior with --ignore-setup-py)')

# Compute & output the constraints we will use:
info('Contents that will be used for constraints.txt:')
constraints = subprocess.check_output([
join(
ctx.build_dir, "venv", "bin", "pip"
),
"freeze"
], env=copy.copy(env))
with suppress(AttributeError):
constraints = constraints.decode("utf-8", "replace")
info(constraints)

# Make sure all packages found are fixed in version
# by writing a constraint file, to avoid recipes being
# upgraded & reinstalled:
with open('._tmp_p4a_recipe_constraints.txt', 'wb') as fileh:
fileh.write(constraints.encode("utf-8", "replace"))
try:

info('Populating venv\'s site-packages with '
'ctx.get_site_packages_dir()...')

# Copy dist contents into site-packages for discovery.
# Why this is needed:
# --target is somewhat evil and messes with discovery of
# packages in PYTHONPATH if that also includes the target
# folder. So we need to use the regular virtualenv
# site-packages folder instead.
# Reference:
# https://github.com/pypa/pip/issues/6223
ctx_site_packages_dir = os.path.normpath(
os.path.abspath(ctx.get_site_packages_dir(arch))
)
venv_site_packages_dir = os.path.normpath(os.path.join(
ctx.build_dir, "venv", "lib", [
f for f in os.listdir(os.path.join(
ctx.build_dir, "venv", "lib"
)) if f.startswith("python")
][0], "site-packages"
))
copied_over_contents = []
for f in os.listdir(ctx_site_packages_dir):
full_path = os.path.join(ctx_site_packages_dir, f)
if not os.path.exists(os.path.join(
venv_site_packages_dir, f
)):
if os.path.isdir(full_path):
shutil.copytree(full_path, os.path.join(
venv_site_packages_dir, f
))
else:
shutil.copy2(full_path, os.path.join(
venv_site_packages_dir, f
))
copied_over_contents.append(f)

# Get listing of virtualenv's site-packages, to see the
# newly added things afterwards & copy them back into
# the distribution folder / build context site-packages:
previous_venv_contents = os.listdir(
venv_site_packages_dir
)

# Actually run setup.py:
info('Launching package install...')
shprint(sh.bash, '-c', (
"'" + join(
ctx.build_dir, "venv", "bin", "pip"
).replace("'", "'\"'\"'") + "' " +
"install -c ._tmp_p4a_recipe_constraints.txt -v ."
).format(ctx.get_site_packages_dir(arch).
replace("'", "'\"'\"'")),
_env=copy.copy(env))

# Go over all new additions and copy them back:
info('Copying additions resulting from setup.py back '
'into ctx.get_site_packages_dir()...')
new_venv_additions = []
for f in (set(os.listdir(venv_site_packages_dir)) -
set(previous_venv_contents)):
new_venv_additions.append(f)
full_path = os.path.join(venv_site_packages_dir, f)
if os.path.isdir(full_path):
shutil.copytree(full_path, os.path.join(
ctx_site_packages_dir, f
))
else:
shutil.copy2(full_path, os.path.join(
ctx_site_packages_dir, f
))

# Undo all the changes we did to the venv-site packages:
info('Reverting additions to '
'virtualenv\'s site-packages...')
for f in set(copied_over_contents + new_venv_additions):
full_path = os.path.join(venv_site_packages_dir, f)
if os.path.isdir(full_path):
rmdir(full_path)
else:
os.remove(full_path)
finally:
os.remove("._tmp_p4a_recipe_constraints.txt")


def is_wheel_platform_independent(whl_name):
name, version, build, tags = parse_wheel_filename(whl_name)
return all(tag.platform == "any" for tag in tags)
Expand Down Expand Up @@ -750,6 +637,9 @@ def process_python_modules(ctx, modules, arch):
# add extra index urls
for index in ctx.extra_index_urls:
indices.extend(["--extra-index-url", index])

state = [pip, platforms, indices, env]

try:
shprint(
pip, 'install', *modules,
Expand All @@ -759,7 +649,7 @@ def process_python_modules(ctx, modules, arch):
)
except Exception as e:
warning(f"Auto module resolution failed: {e}")
return processed_modules
return processed_modules, state

with open(path, "r") as f:
try:
Expand All @@ -772,9 +662,9 @@ def process_python_modules(ctx, modules, arch):
if "install" not in report.keys():
# pip changed json reporting format?
warning("Auto module resolution failed: invalid json!")
return processed_modules
return processed_modules, state

info('Extra resolved pure python dependencies :')
info('Extra resolved python platform dependencies :')

ignored_str = " (ignored)"
# did we find any non pure python package?
Expand All @@ -795,8 +685,7 @@ def process_python_modules(ctx, modules, arch):
any_not_pure_python = True
pure_python = False

# does this module matches any recipe name?
if mname.lower().replace("-", "_") in _requirement_names:
if mname.lower().replace("-", "_") in _requirement_names or mname.lower() in _requirement_names:
continue

color = Out_Fore.GREEN if pure_python else Out_Fore.RED
Expand All @@ -809,14 +698,15 @@ def process_python_modules(ctx, modules, arch):
)

if pure_python:
# Direct whl file to avoid resolving again
processed_modules.append(module["download_info"]["url"])
info(" ")

if any_not_pure_python:
warning("Some packages were ignored because they are not pure Python.")
warning("To install the ignored packages, explicitly list them in your requirements file.")

return processed_modules
return processed_modules, state


def run_pymodules_install(ctx, arch, modules, project_dir=None,
Expand All @@ -833,9 +723,17 @@ def run_pymodules_install(ctx, arch, modules, project_dir=None,

info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE FOR ARCH: {} ***'.format(arch))

modules = process_python_modules(ctx, modules, arch)
# Restore version strings from environment
for index, module in enumerate(modules):
if (m_version := os.environ.get(f'VERSION_{module}', None)) is not None:
modules[index] = f'{module}=={m_version}'

modules, state = process_python_modules(ctx, modules, arch)
# Reuse the state constructed
pip, platforms, indices, env = state

modules = [m for m in modules if ctx.not_has_package(m, arch)]
# It always runs with --upgrade so this is not required as it skips if module already exists
# modules = [m for m in modules if ctx.not_has_package(m, arch)]

# We change current working directory later, so this has to be an absolute
# path or `None` in case that we didn't supply the `project_dir` via kwargs
Expand All @@ -859,84 +757,56 @@ def run_pymodules_install(ctx, arch, modules, project_dir=None,
"If this fails, it may mean that the module has compiled "
"components and needs a recipe."
)

if project_has_setup_py(project_dir) and not ignore_setup_py:
info(
"Will process project install, if it fails then the "
"project may not be compatible for Android install."
)

# Use our hostpython to create the virtualenv
host_python = sh.Command(ctx.hostpython)
with current_directory(join(ctx.build_dir)):
shprint(host_python, '-m', 'venv', '--clear', 'venv')

# Prepare base environment:
base_env = dict(copy.copy(os.environ))
base_env["PYTHONPATH"] = ctx.get_site_packages_dir(arch)

# Install Cython in case modules need it to build:
info('Install Cython in case one of the modules needs it to build')
shprint(sh.bash, '-c', (
"venv/bin/pip install Cython"
), _env=copy.copy(base_env))

# Get environment variables for build (with CC/compiler set):
standard_recipe = CythonRecipe()
standard_recipe.ctx = ctx
# (note: following line enables explicit -lpython... linker options)
standard_recipe.call_hostpython_via_targetpython = False
recipe_env = standard_recipe.get_recipe_env(ctx.archs[0])
env = copy.copy(base_env)
env.update(recipe_env)

# Make sure our build package dir is available, and the virtualenv
# site packages come FIRST (so the proper pip version is used):
env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir(arch)
env["PYTHONPATH"] = os.path.abspath(join(
ctx.build_dir, "venv", "lib",
"python" + ctx.python_recipe.major_minor_version_string,
"site-packages")) + ":" + env["PYTHONPATH"]

# Install the manually specified requirements first:
if not modules:
info('There are no Python modules to install, skipping')
else:
info('Creating a requirements.txt file for the Python modules')
with open('requirements.txt', 'w') as fileh:
for module in modules:
key = 'VERSION_' + module
if key in environ:
line = '{}=={}\n'.format(module, environ[key])
else:
line = '{}\n'.format(module)
fileh.write(line)

info('Installing Python modules with pip')
info(
"IF THIS FAILS, THE MODULES MAY NEED A RECIPE. "
"A reason for this is often modules compiling "
"native code that is unaware of Android cross-compilation "
"and does not work without additional "
"changes / workarounds."
)
if not modules:
info('There are no Python modules to install, skipping')
else:

info('Installing Python modules with pip')
info(
"IF THIS FAILS, THE MODULES MAY NEED A RECIPE. "
"A reason for this is often modules compiling "
"native code that is unaware of Android cross-compilation "
"and does not work without additional "
"changes / workarounds."
)
# --no-deps is required here as auto resolution is already done above
shprint(
pip, 'install', *modules,
'--target', ctx.get_site_packages_dir(arch),
'--upgrade', '--ignore-installed', '--no-deps',
'--disable-pip-version-check', '--only-binary=:all:',
*platforms, *indices, _env=env
)

shprint(sh.bash, '-c', (
"venv/bin/pip " +
"install -v --target '{0}' --no-deps -r requirements.txt"
).format(ctx.get_site_packages_dir(arch).replace("'", "'\"'\"'")),
_env=copy.copy(env))

# Afterwards, run setup.py if present:
if project_has_setup_py(project_dir) and not ignore_setup_py:
run_setuppy_install(ctx, project_dir, env, arch)
elif not ignore_setup_py:
info("No setup.py found in project directory: " + str(project_dir))

# Strip object files after potential Cython or native code builds:
if not ctx.with_debug_symbols:
standard_recipe.strip_object_files(
arch, env, build_dir=ctx.build_dir
# Afterwards, run setup.py if present:
if project_has_setup_py(project_dir) and not ignore_setup_py:
with current_directory(project_dir):
# TODO: It will only work for basic python projects with no compiled components
shprint(
pip, 'install', ".",
'--target', ctx.get_site_packages_dir(arch),
'--disable-pip-version-check', '--upgrade',
*platforms, *indices, _env=env
)
elif not ignore_setup_py:
info("No setup.py found in project directory: " + str(project_dir))

# Strip object files after potential Cython or native code builds:
if not ctx.with_debug_symbols and env.get("STRIP", None) is not None:
info('Stripping object files')
shprint(
sh.find, '.', '-iname', '*.so',
'-exec', env['STRIP'].split(' ')[0],
'--strip-unneeded', '{}', ';',
_env=env
)


def biglink(ctx, arch):
Expand Down
4 changes: 2 additions & 2 deletions tests/test_build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ def test_python_module_parser(self):
arch = ctx.archs[0]

# should not alter original module name (like with adding version number)
assert "kivy_garden.frostedglass" in process_python_modules(ctx, ["kivy_garden.frostedglass"], arch)
assert "kivy_garden.frostedglass" in process_python_modules(ctx, ["kivy_garden.frostedglass"], arch)[0]

# should skip urls and other unsupported format
modules = ["https://example.com/some.zip", "git+https://github.com/kivy/python-for-android@develop"]
result = process_python_modules(ctx, modules, arch)
result = process_python_modules(ctx, modules, arch)[0]
assert modules == result

def test_is_wheel_compatible(self):
Expand Down
Loading