diff --git a/pulp_rust/app/serializers.py b/pulp_rust/app/serializers.py index 9bc6c64..8bae301 100755 --- a/pulp_rust/app/serializers.py +++ b/pulp_rust/app/serializers.py @@ -1,12 +1,33 @@ -from gettext import gettext as _ +import logging +from gettext import gettext as _ +from django.db.utils import IntegrityError +from pydantic import TypeAdapter, ValidationError from rest_framework import serializers from pulpcore.plugin import models as core_models from pulpcore.plugin import serializers as core_serializers +from pulpcore.plugin.util import get_domain from . import models +log = logging.getLogger(__name__) + + +class IndexRootSerializer(serializers.Serializer): + """ + A Serializer for summary information of an index. + """ + + dl = serializers.CharField(help_text=_("URL of the index root"), read_only=True) + api = serializers.CharField(help_text=_("URL of the API root"), read_only=True) + auth_required = serializers.BooleanField( + help_text=_( + "Indicates whether this is a private registry that requires all operations to be authenticated" + ), + read_only=True, + ) + class RustDependencySerializer(serializers.ModelSerializer): """ @@ -178,31 +199,187 @@ class Meta: model = models.RustContent -class RustRemoteSerializer(core_serializers.RemoteSerializer): +class RustDependencySerializer(serializers.ModelSerializer): """ - A Serializer for RustRemote. + Serializer for RustDependency. - Add any new fields if defined on RustRemote. - Similar to the example above, in RustContentSerializer. - Additional validators can be added to the parent validators list + Represents a single dependency entry from the Cargo package index. + """ - For example:: + name = serializers.CharField( + help_text=_("Dependency name as used in code (may be renamed via 'package' field)") + ) + + req = serializers.CharField( + help_text=_("Version requirement string (e.g., '^1.0', '>=0.2.3,<0.3')") + ) + + features = serializers.ListField( + child=serializers.CharField(), + default=list, + required=False, + help_text=_("List of feature flags to enable for this dependency"), + ) + + optional = serializers.BooleanField( + default=False, required=False, help_text=_("Whether this is an optional dependency") + ) + + default_features = serializers.BooleanField( + default=True, + required=False, + help_text=_("Whether to enable the dependency's default features"), + ) + + target = serializers.CharField( + allow_null=True, + required=False, + help_text=_("Platform-specific target (e.g., 'cfg(unix)', 'cfg(windows)')"), + ) + + kind = serializers.ChoiceField( + choices=[("normal", "Normal"), ("dev", "Development"), ("build", "Build")], + default="normal", + required=False, + help_text=_( + "Dependency type: 'normal' (runtime), 'dev' (development), or 'build' (build script)" + ), + ) + + registry = serializers.CharField( + allow_null=True, + required=False, + help_text=_("Alternative registry URL if dependency is from a different registry"), + ) + + package = serializers.CharField( + allow_null=True, + required=False, + help_text=_("Original crate name if the dependency was renamed"), + ) class Meta: - validators = core_serializers.RemoteSerializer.Meta.validators - + [myValidator1, myValidator2] + model = models.RustDependency + fields = ( + "name", + "req", + "features", + "optional", + "default_features", + "target", + "kind", + "registry", + "package", + ) + + +class RustContentSerializer(core_serializers.SingleArtifactContentSerializer): + """ + Serializer for RustContent (Cargo package version). + + Represents a single version of a Rust crate as defined in the Cargo registry + index specification. Includes package metadata, dependencies, and features. + """ + + name = serializers.CharField(help_text=_("Package name (crate name)")) + + vers = serializers.CharField(help_text=_("Semantic version string (SemVer 2.0.0)")) - By default the 'policy' field in core_serializers.RemoteSerializer only validates the choice - 'immediate'. To add on-demand support for more 'policy' options, e.g. 'streamed' or - 'on_demand', re-define the 'policy' option as follows:: + dependencies = RustDependencySerializer( + many=True, required=False, help_text=_("List of dependencies for this package version") + ) + + cksum = serializers.CharField(help_text=_("SHA256 checksum of the .crate file (tarball)")) + + features = serializers.JSONField( + default=dict, + required=False, + help_text=_( + "Feature flags mapping - maps feature names to lists of features/dependencies " + "they enable" + ), + ) + + features2 = serializers.JSONField( + default=dict, + required=False, + allow_null=True, + help_text=_("Extended feature syntax support (newer registry format)"), + ) + + yanked = serializers.BooleanField( + default=False, + required=False, + help_text=_("Whether this version has been yanked (removed from normal use)"), + ) + + links = serializers.CharField( + allow_null=True, + required=False, + help_text=_("Name of native library this package links to (from Cargo.toml 'links' field)"), + ) + + v = serializers.IntegerField( + default=1, required=False, help_text=_("Schema version of the index entry format") + ) + rust_version = serializers.CharField( + allow_null=True, + required=False, + help_text=_("Minimum Rust compiler version required (MSRV)"), + ) + + def create(self, validated_data): + """Create RustContent and related dependencies.""" + dependencies_data = validated_data.pop("dependencies", []) + content = super().create(validated_data) + + # Create dependency records + for dep_data in dependencies_data: + models.RustDependency.objects.create(content=content, **dep_data) + + return content + + def update(self, instance, validated_data): + """Update RustContent and related dependencies.""" + dependencies_data = validated_data.pop("dependencies", None) + + instance = super().update(instance, validated_data) + + if dependencies_data is not None: + # Replace all dependencies + instance.dependencies.all().delete() + for dep_data in dependencies_data: + models.RustDependency.objects.create(content=instance, **dep_data) + + return instance + + class Meta: + fields = core_serializers.SingleArtifactContentSerializer.Meta.fields + ( + "name", + "vers", + "dependencies", + "cksum", + "features", + "features2", + "yanked", + "links", + "v", + "rust_version", + ) + model = models.RustContent + + +class RustRemoteSerializer(core_serializers.RemoteSerializer): + """ + A Serializer for RustRemote. + """ policy = serializers.ChoiceField( help_text="The policy to use when downloading content. The possible values include: " - "'immediate', 'on_demand', and 'streamed'. 'immediate' is the default.", + "'immediate', 'on_demand', and 'streamed'. 'streamed' is the default.", choices=models.Remote.POLICY_CHOICES, - default=models.Remote.IMMEDIATE + default=models.Remote.STREAMED, ) - """ class Meta: fields = core_serializers.RemoteSerializer.Meta.fields @@ -258,3 +435,31 @@ class Meta: class Meta: fields = core_serializers.DistributionSerializer.Meta.fields + ("allow_uploads", "remote") model = models.RustDistribution + + +class RepositoryAddCachedContentSerializer( + core_serializers.ValidateFieldsMixin, serializers.Serializer +): + remote = core_serializers.DetailRelatedField( + required=False, + view_name_pattern=r"remotes(-.*/.*)-detail", + queryset=models.Remote.objects.all(), + help_text=_( + "A remote to use to identify content that was cached. This will override a " + "remote set on repository." + ), + ) + + def validate(self, data): + data = super().validate(data) + repository = None + if "repository_pk" in self.context: + repository = models.Repository.objects.get(pk=self.context["repository_pk"]) + remote = data.get("remote", None) or getattr(repository, "remote", None) + + if not remote: + raise serializers.ValidationError( + {"remote": _("This field is required since a remote is not set on the repository.")} + ) + self.check_cross_domains({"repository": repository, "remote": remote}) + return data diff --git a/pulp_rust/app/settings.py b/pulp_rust/app/settings.py index c67a45c..664dcab 100755 --- a/pulp_rust/app/settings.py +++ b/pulp_rust/app/settings.py @@ -1,6 +1,3 @@ -""" -Check `Plugin Writer's Guide`_ for more details. +import socket -.. _Plugin Writer's Guide: - https://pulpproject.org/pulpcore/docs/dev/ -""" +CRATES_IO_API_HOSTNAME = "https://" + socket.getfqdn() diff --git a/pulp_rust/app/tasks/__init__.py b/pulp_rust/app/tasks/__init__.py index 72ae454..9b19390 100755 --- a/pulp_rust/app/tasks/__init__.py +++ b/pulp_rust/app/tasks/__init__.py @@ -1 +1,2 @@ from .synchronizing import synchronize # noqa +from .streaming import add_cached_content_to_repository # noqa diff --git a/pulp_rust/app/tasks/streaming.py b/pulp_rust/app/tasks/streaming.py new file mode 100644 index 0000000..e5787a0 --- /dev/null +++ b/pulp_rust/app/tasks/streaming.py @@ -0,0 +1,40 @@ +import datetime + +from asgiref.sync import sync_to_async + +from pulpcore.plugin.models import Content, ContentArtifact, RemoteArtifact +from pulpcore.plugin.tasking import add_and_remove + +from pulp_rust.app.models import RustRemote, RustRepository + + +async def aadd_and_remove(*args, **kwargs): + return await sync_to_async(add_and_remove)(*args, **kwargs) + + +def add_cached_content_to_repository(repository_pk=None, remote_pk=None): + """ + Create a new repository version by adding content that was cached by pulpcore-content when + streaming it from a remote. + + Args: + repository_pk (uuid): The primary key for a Repository for which a new Repository Version + should be created. + remote_pk (uuid): The primary key for a Remote which will be used to identify Content + created by pulpcore-content when it streamed it to clients. + """ + repository = RustRepository.objects.get(pk=repository_pk) + remote = RustRemote.objects.get(pk=remote_pk) + + latest_version = repository.latest_version() + + if latest_version.number == 0: + date_min = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + else: + date_min = latest_version.pulp_created + with repository.new_version(base_version=None) as new_version: + ca_id_list = RemoteArtifact.objects.filter( + remote=remote, pulp_created__gte=date_min + ).values_list("content_artifact") + content_list = ContentArtifact.objects.filter(pk__in=ca_id_list).values_list("content") + new_version.add_content(Content.objects.filter(pk__in=content_list)) diff --git a/pulp_rust/app/tasks/synchronizing.py b/pulp_rust/app/tasks/synchronizing.py index 8f2064b..bed6c1f 100755 --- a/pulp_rust/app/tasks/synchronizing.py +++ b/pulp_rust/app/tasks/synchronizing.py @@ -73,6 +73,8 @@ async def run(self): downloader = self.remote.get_downloader(url=self.remote.url) result = await downloader.run() # Use ProgressReport to report progress + raise NotImplemented("Not implemented") + for entry in self.read_my_metadata_file_somehow(result.path): unit = RustContent(entry) # make the content unit in memory-only artifact = Artifact(entry) # make Artifact in memory-only diff --git a/pulp_rust/app/urls.py b/pulp_rust/app/urls.py new file mode 100644 index 0000000..86c8123 --- /dev/null +++ b/pulp_rust/app/urls.py @@ -0,0 +1,26 @@ +from django.conf import settings +from django.urls import path, re_path + +from pulp_rust.app.views import IndexRoot, CargoApiViewSet, CargoDownloadApiView + +if settings.DOMAIN_ENABLED: + path_re = r"(?P[-a-zA-Z0-9_]+)/(?P[\w-]+)/(?P.*)" + CRATES_IO_URL = "pulp/cargo///" +else: + path_re = r"(?P[\w-]+)/(?P.*)" + CRATES_IO_URL = "pulp/cargo//" + + +urlpatterns = [ + path(CRATES_IO_URL + "config.json", IndexRoot.as_view({"get": "retrieve"}), name="index-root"), + path( + CRATES_IO_URL + "api/v1/crates///", + CargoDownloadApiView.as_view(), + name="cargo-download-api", + ), + path(CRATES_IO_URL + "/", CargoApiViewSet.as_view({"get": "retrieve"}), name="cargo-api"), + + + + # re_path(rf"^pulp/cargo/{path_re}$", CargoApiViewSet.as_view({"get": "retrieve"})), +] diff --git a/pulp_rust/app/views.py b/pulp_rust/app/views.py new file mode 100644 index 0000000..ea85361 --- /dev/null +++ b/pulp_rust/app/views.py @@ -0,0 +1,299 @@ +import json +import logging + +from rest_framework.views import APIView +from rest_framework.viewsets import ViewSet +from rest_framework.renderers import BrowsableAPIRenderer, JSONRenderer, TemplateHTMLRenderer +from rest_framework.response import Response +from rest_framework.decorators import api_view +from rest_framework.exceptions import NotAcceptable, Throttled +from django.core.exceptions import ObjectDoesNotExist +from django.shortcuts import redirect +from datetime import datetime, timezone, timedelta + +from django.contrib.sessions.models import Session +from django.db import transaction +from django.db.utils import DatabaseError +from django.http.response import ( + Http404, + HttpResponseNotFound, + HttpResponseForbidden, + HttpResponseBadRequest, + StreamingHttpResponse, + HttpResponse, +) +from drf_spectacular.utils import extend_schema +from dynaconf import settings +from itertools import chain +from packaging.utils import canonicalize_name +from urllib.parse import urljoin, urlparse, urlunsplit +from pathlib import PurePath + +from pulpcore.plugin.viewsets import OperationPostponedResponse +from pulpcore.plugin.tasking import dispatch +from pulpcore.plugin.util import get_domain, get_url + +from pulp_rust.app.models import RustDistribution, RustRepository, RustContent +from pulp_rust.app.serializers import ( + IndexRootSerializer, + RustContentSerializer, +) +from pulp_rust.app import tasks + +log = logging.getLogger(__name__) + +ORIGIN_HOST = settings.CONTENT_ORIGIN if settings.CONTENT_ORIGIN else settings.PYPI_API_HOSTNAME +BASE_CONTENT_URL = urljoin(ORIGIN_HOST, settings.CONTENT_PATH_PREFIX) +BASE_API_URL = settings.CRATES_IO_API_HOSTNAME +CRATES_IO_API = "/api/v1/crates/" + + +class ApiMixin: + """Mixin to get index specific info.""" + + _distro = None + + @property + def distribution(self): + if not self._distro: + self._distro = self.get_distribution(self.kwargs["repo"]) + return self._distro + + @staticmethod + def get_distribution(repo): + """Finds the distribution associated with this base_path.""" + distro_qs = RustDistribution.objects.select_related( + "repository", "repository_version", "remote" + ) + try: + return distro_qs.get(base_path=repo, pulp_domain=get_domain()) + except ObjectDoesNotExist: + raise Http404(f"No RustDistribution found for base_path {path}") + + @staticmethod + def get_repository_version(distribution): + """Finds the repository version this distribution is serving.""" + rep = distribution.repository + rep_version = distribution.repository_version + if rep: + return rep.latest_version() + elif rep_version: + return rep_version + else: + raise Http404("No repository associated with this index") + + @staticmethod + def get_content(repository_version): + """Returns queryset of the content in this repository version.""" + return RustContent.objects.filter(pk__in=repository_version.content) + + def get_rvc(self): + """Takes the base_path and returns the repository_version and content.""" + if self.distribution.remote: + if not self.distribution.repository: + return None, None + repo_ver = self.get_repository_version(self.distribution) + content = self.get_content(repo_ver) + return repo_ver, content + + def initial(self, request, *args, **kwargs): + """Perform common initialization tasks for API endpoints.""" + super().initial(request, *args, **kwargs) + domain_name = get_domain().name + log.warning(self.kwargs) + repo = self.kwargs["repo"] + if settings.DOMAIN_ENABLED: + self.base_content_url = urljoin(BASE_CONTENT_URL, f"pulp/cargo/{domain_name}/{repo}/") + self.base_api_url = urljoin(BASE_API_URL, f"pulp/cargo/{domain_name}/{repo}/") + self.base_download_url = urljoin( + BASE_API_URL, f"pulp/cargo/{domain_name}/{repo}{CRATES_IO_API}" + ) + else: + self.base_content_url = urljoin(BASE_CONTENT_URL, f"pulp/cargo/{repo}/") + self.base_api_url = urljoin(BASE_API_URL, f"pulp/cargo/{repo}/") + self.base_download_url = urljoin(BASE_API_URL, f"pulp/cargo/{repo}{CRATES_IO_API}") + + @classmethod + def urlpattern(cls): + """Mocking NamedModelViewSet behavior to get Cargo APIs to support RBAC access polices.""" + return f"pulp/cargo/{cls.endpoint_name}" + + +class CargoApiViewSet(ApiMixin, ViewSet): + """View for the Cargo JSON metadata endpoint.""" + + endpoint_name = "api" + DEFAULT_ACCESS_POLICY = { + "statements": [ + { + "action": ["retrieve"], + "principal": "*", + "effect": "allow", + }, + ], + } + + @extend_schema( + tags=["Cargo: Metadata"], + responses={200: RustContentSerializer}, + summary="Get package metadata", + ) + def retrieve(self, request, path): + """ + Retrieve crate metadata for the sparse protocol. + + The sparse protocol uses a directory structure based on crate name length: + - 1 char: 1/{crate} + - 2 chars: 2/{crate} + - 3 chars: 3/{first-char}/{crate} + - 4+ chars: {first-two}/{second-two}/{crate} + + Returns newline-delimited JSON, one version per line. + """ + repo_ver, content = self.get_rvc() + + if content is None: + return HttpResponseNotFound("No content available") + + # Extract crate name from the path + meta_path = PurePath(path) + crate_name = meta_path.name.lower() + + # Query for all versions of this crate + crate_versions = content.filter(name=crate_name).order_by("vers") + + if not crate_versions.exists(): + return HttpResponseNotFound(f"Crate '{crate_name}' not found") + + # Build newline-delimited JSON response + lines = [] + for crate_version in crate_versions: + # Fetch dependencies for this version + deps = [] + for dep in crate_version.dependencies.all(): + dep_obj = { + "name": dep.name, + "req": dep.req, + "features": dep.features, + "optional": dep.optional, + "default_features": dep.default_features, + "target": dep.target, + "kind": dep.kind, + "registry": dep.registry, + "package": dep.package, + } + deps.append(dep_obj) + + # Build the version object according to sparse protocol + version_obj = { + "name": crate_version.name, + "vers": crate_version.vers, + "deps": deps, + "cksum": crate_version.cksum, + "features": crate_version.features, + "yanked": crate_version.yanked, + "links": crate_version.links, + "v": crate_version.v, + } + + # Add optional fields only if present + if crate_version.features2: + version_obj["features2"] = crate_version.features2 + if crate_version.rust_version: + version_obj["rust_version"] = crate_version.rust_version + + # Serialize to JSON and add to lines + lines.append(json.dumps(version_obj)) + + # Join with newlines and return as plain text + response_text = "\n".join(lines) + return HttpResponse(response_text, content_type="text/plain") + + +class IndexRoot(ApiMixin, ViewSet): + """View for base_url of distribution.""" + + endpoint_name = "root" + DEFAULT_ACCESS_POLICY = { + "statements": [ + { + "action": ["retrieve"], + "principal": "*", + "effect": "allow", + }, + ], + } + + @extend_schema(responses={200: IndexRootSerializer}, summary="Get index info") + def retrieve(self, request, path): + """Gets index route.""" + return Response( + data={ + "dl": self.base_download_url, + "api": self.base_api_url, + "auth-required": False, + } + ) + + +class CargoDownloadApiView(APIView): + """ + ViewSet for interacting with maven deploy API + """ + + model = RustRepository + queryset = RustRepository.objects.all() + + lookup_field = "name" + + # Authentication disabled for now + authentication_classes = [] + permission_classes = [] + + def redirect_to_content_app(self, distribution, relative_path, request): + scheme = request.META.get("HTTP_X_FORWARDED_PROTO", request.scheme) + hostname = request.META.get("HTTP_X_FORWARDED_HOST", request.get_host()) + content_origin = f"{scheme}://{hostname}" + return redirect( + f"{content_origin}{settings.CONTENT_PATH_PREFIX}" + f"{get_full_path(distribution.base_path)}/{relative_path}" + ) + + def get_repository_and_distributions(self, name): + repository = get_object_or_404(RustRepository, name=name, pulp_domain=get_domain()) + distribution = get_object_or_404( + RustDistribution, repository=repository, pulp_domain=get_domain() + ) + return repository, distribution + + def get(self, request, name, version): + """ + Responds to GET requests about packages by reference + """ + repo, distro = self.get_repository_and_distributions(name) + content = get_object_or_404(RustPackage, name=name, vers=version, pk__in=repo.latest_version().content) + relative_path = content.contentartifact_set.get().relative_path + return self.redirect_to_content_app(distro, relative_path, request) + + +def has_task_completed(task): + """ + Verify whether an immediate task ran properly. + + Returns: + bool: True if the task ended successfully. + + Raises: + Exception: If an error occured during the task's runtime. + Throttled: If the task did not run due to resource constraints. + + """ + if task.state == "completed": + task.delete() + return True + elif task.state == "canceled": + raise Throttled() + else: + error = task.error + task.delete() + raise Exception(str(error)) diff --git a/pulp_rust/app/viewsets.py b/pulp_rust/app/viewsets.py index c9a1adf..d3bbda3 100755 --- a/pulp_rust/app/viewsets.py +++ b/pulp_rust/app/viewsets.py @@ -167,6 +167,42 @@ def sync(self, request, pk): ) return core.OperationPostponedResponse(result, request) + @extend_schema( + description="Trigger an asynchronous task to add cached content to a repository.", + summary="Add cached content", + responses={202: AsyncOperationResponseSerializer}, + ) + @action( + detail=True, + methods=["post"], + serializer_class=serializers.RepositoryAddCachedContentSerializer, + ) + def add_cached_content(self, request, pk): + """ + Add to the repository any new content that was cached using the remote since the last + repository version was created. + + The ``repository`` field has to be provided. + """ + serializer = serializers.RepositoryAddCachedContentSerializer( + data=request.data, context={"request": request, "repository_pk": pk} + ) + serializer.is_valid(raise_exception=True) + + repository = self.get_object() + remote = serializer.validated_data.get("remote", repository.remote) + + result = dispatch( + add_cached_content_to_repository, + shared_resources=[remote], + exclusive_resources=[repository], + kwargs={ + "remote_pk": str(remote.pk), + "repository_pk": str(repository.pk), + }, + ) + return OperationPostponedResponse(result, request) + class RustRepositoryVersionViewSet(core.RepositoryVersionViewSet): """ diff --git a/pulp_rust/tests/functional/api/test_crud_remotes.py b/pulp_rust/tests/functional/api/test_crud_remotes.py new file mode 100644 index 0000000..34b0c1a --- /dev/null +++ b/pulp_rust/tests/functional/api/test_crud_remotes.py @@ -0,0 +1,71 @@ +import json +import uuid + +import pytest + +import django + +from pulpcore.client.pulp_rust.exceptions import ApiException + +django.setup() +from pulp_rust.app.serializers import RustRemoteSerializer # noqa + + +@pytest.mark.parallel +def test_remote_crud_workflow(rust_remote_api_client, gen_object_with_cleanup, monitor_task): + remote_data = {"name": str(uuid.uuid4()), "url": "http://example.com"} + remote = gen_object_with_cleanup(rust_remote_api_client, remote_data) + assert remote.url == remote_data["url"] + assert remote.name == remote_data["name"] + + with pytest.raises(ApiException) as exc: + gen_object_with_cleanup(rust_remote_api_client, remote_data) + assert exc.value.status == 400 + assert json.loads(exc.value.body) == {"name": ["This field must be unique."]} + + update_response = rust_remote_api_client.partial_update( + remote.pulp_href, {"url": "https://example.com"} + ) + task = monitor_task(update_response.task) + assert task.created_resources == [] + + remote = rust_remote_api_client.read(remote.pulp_href) + assert remote.url == "https://example.com" + + all_new_remote_data = {"name": str(uuid.uuid4()), "url": "http://example.com"} + update_response = rust_remote_api_client.update(remote.pulp_href, all_new_remote_data) + task = monitor_task(update_response.task) + assert task.created_resources == [] + + remote = rust_remote_api_client.read(remote.pulp_href) + assert remote.name == all_new_remote_data["name"] + assert remote.url == all_new_remote_data["url"] + + +@pytest.mark.parallel +def test_create_rust_remote_with_invalid_parameter(): + unexpected_field_remote_data = { + "name": str(uuid.uuid4()), + "url": "http://example.com", + "foo": "bar", + } + + rust_remote_serializer = RustRemoteSerializer(data=unexpected_field_remote_data) + + assert rust_remote_serializer.is_valid() is False + assert rust_remote_serializer.errors["foo"][0].title() == "Unexpected Field" + + +@pytest.mark.parallel +def test_create_rust_remote_without_url(rust_remote_api_client, gen_object_with_cleanup): + rust_remote_serializer = RustRemoteSerializer(data={"name": str(uuid.uuid4())}) + + assert rust_remote_serializer.is_valid() is False + assert rust_remote_serializer.errors["url"][0].title() == "This Field Is Required." + + +@pytest.mark.parallel +def test_default_remote_policy_immediate(rust_remote_api_client, gen_object_with_cleanup): + remote_data = {"name": str(uuid.uuid4()), "url": "http://example.com"} + remote = gen_object_with_cleanup(rust_remote_api_client, remote_data) + assert remote.policy == "immediate" diff --git a/pulp_rust/tests/functional/api/test_download_content.py b/pulp_rust/tests/functional/api/test_download_content.py new file mode 100644 index 0000000..1442d80 --- /dev/null +++ b/pulp_rust/tests/functional/api/test_download_content.py @@ -0,0 +1,80 @@ +"""Tests that verify download of content served by Pulp.""" + +from aiohttp.client_exceptions import ClientResponseError +import hashlib +from urllib.parse import urljoin + +from pulp_rust.tests.functional.utils import download_file + + +def test_download_content( + rust_distribution_factory, + rust_remote_factory, + rust_repo_factory, + rust_artifact_api_client, + rust_distro_api_client, + rust_repo_api_client, + monitor_task, + distribution_base_url, +): + """Verify whether content served by pulp can be downloaded. + + The process of creating a Cargo mirror is: + + 1. Create a Rust Remote with a URL pointing to the root of a Cargo repository. + 2. Create a distribution with the remote set HREF from 1. + + Do the following: + + 1. Create a Rust Remote and a Distribution. + 2. Select a random content unit in the distribution. Download that + content unit from Pulp, and verify that the content unit has the + same checksum when fetched directly from Cargo. + """ + remote = rust_remote_factory(url="sparse+https://index.crates.io/") + repository = rust_repo_factory(remote=remote.pulp_href) + distribution = rust_distribution_factory( + remote=remote.pulp_href, repository=repository.pulp_href + ) + + # Pick a content unit, and download it from the remote repository + unit_path = "api/v1/crates/ripgrep/15.1.0/download" + remote_unit_url = urljoin(remote.url, unit_path) + downloaded_file = download_file(remote_unit_url) + remote_unit_checksum = hashlib.sha256(downloaded_file.body).hexdigest() + + # And from Pulp + pulp_unit_url = urljoin(distribution_base_url(distribution.base_url), unit_path) + downloaded_file = download_file(pulp_unit_url) + pulp_unit_checksum = hashlib.sha256(downloaded_file.body).hexdigest() + + assert remote_unit_checksum == pulp_unit_checksum + + # Check that Pulp created a Rust artifact + content_response = rust_artifact_api_client.list() # todo: filter for better idempotence + assert content_response.count == 1 + + # Remove the remote from the distribution + monitor_task( + rust_distro_api_client.partial_update(distribution.pulp_href, {"remote": None}).task + ) + + # Assert that the rust artifact is no longer available + try: + download_file(pulp_unit_url) + except ClientResponseError as e: + assert e.status == 404 + + # Assert that the repository version is 0 + assert repository.latest_version_href.endswith("/versions/0/") + + # Add cached content to the repository + monitor_task(rust_repo_api_client.add_cached_content(repository.pulp_href, {}).task) + + # Assert that the repository is at version 1 + repository = rust_repo_api_client.read(repository.pulp_href) + assert repository.latest_version_href.endswith("/versions/1/") + + # Assert that it is now once again available from the same distribution + downloaded_file = download_file(pulp_unit_url) + assert downloaded_file.response_obj.status == 200 diff --git a/pulp_rust/tests/functional/conftest.py b/pulp_rust/tests/functional/conftest.py new file mode 100644 index 0000000..ea3af8d --- /dev/null +++ b/pulp_rust/tests/functional/conftest.py @@ -0,0 +1,71 @@ +import uuid + +import pytest + +from pulpcore.client.pulp_rust import ( + ApiClient, + ContentArtifactApi, + DistributionsRustApi, + RemotesRustApi, + RepositoriesRustApi, +) + + +@pytest.fixture(scope="session") +def rust_client(_api_client_set, bindings_cfg): + api_client = ApiClient(bindings_cfg) + _api_client_set.add(api_client) + yield api_client + _api_client_set.remove(api_client) + + +@pytest.fixture(scope="session") +def rust_artifact_api_client(rust_client): + return ContentArtifactApi(rust_client) + + +@pytest.fixture(scope="session") +def rust_distro_api_client(rust_client): + return DistributionsRustApi(rust_client) + + +@pytest.fixture(scope="session") +def rust_repo_api_client(rust_client): + return RepositoriesRustApi(rust_client) + + +@pytest.fixture(scope="session") +def rust_remote_api_client(rust_client): + return RemotesRustApi(rust_client) + + +@pytest.fixture +def rust_distribution_factory(rust_distro_api_client, gen_object_with_cleanup): + def _rust_distribution_factory(**kwargs): + data = {"base_path": str(uuid.uuid4()), "name": str(uuid.uuid4())} + data.update(kwargs) + return gen_object_with_cleanup(rust_distro_api_client, data) + + return _rust_distribution_factory + + +@pytest.fixture +def rust_repo_factory(rust_repo_api_client, gen_object_with_cleanup): + """A factory to generate a Rust Repository with auto-deletion after the test run.""" + + def _rust_repo_factory(**kwargs): + kwargs.setdefault("name", str(uuid.uuid4())) + return gen_object_with_cleanup(rust_repo_api_client, kwargs) + + yield _rust_repo_factory + + +@pytest.fixture +def rust_remote_factory(rust_remote_api_client, gen_object_with_cleanup): + """A factory to generate a Rust Remote with auto-deletion after the test run.""" + + def _rust_remote_factory(**kwargs): + kwargs.setdefault("name", str(uuid.uuid4())) + return gen_object_with_cleanup(rust_remote_api_client, kwargs) + + yield _rust_remote_factory diff --git a/pulp_rust/tests/functional/utils.py b/pulp_rust/tests/functional/utils.py new file mode 100644 index 0000000..1130a04 --- /dev/null +++ b/pulp_rust/tests/functional/utils.py @@ -0,0 +1,34 @@ +# coding=utf-8 +"""Utilities for tests for the maven plugin.""" +import aiohttp +import asyncio +from dataclasses import dataclass + + +@dataclass +class Download: + """Class for representing a downloaded file.""" + + body: bytes + response_obj: aiohttp.ClientResponse + + def __init__(self, body, response_obj): + self.body = body + self.response_obj = response_obj + + +def download_file(url, auth=None, headers=None): + """Download a file. + + :param url: str URL to the file to download + :param auth: `aiohttp.BasicAuth` containing basic auth credentials + :param headers: dict of headers to send with the GET request + :return: Download + """ + return asyncio.run(_download_file(url, auth=auth, headers=headers)) + + +async def _download_file(url, auth=None, headers=None): + async with aiohttp.ClientSession(auth=auth, raise_for_status=True) as session: + async with session.get(url, verify_ssl=False, headers=headers) as response: + return Download(body=await response.read(), response_obj=response) diff --git a/pyproject.toml b/pyproject.toml index 060d231..8a4b41e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ ] requires-python = ">=3.11" dependencies = [ - "pulpcore>=3.73.0,<3.100", + "pulpcore>=3.100.0,<3.115", ] [project.urls]