diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 025f9e4ccd..a4530bcc3d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -47,7 +47,7 @@ "**/.pytest_cache/**": true, "**/build/**": true, "**/dist/**": true, - "**/pyrit/auxiliary_attacks/gcg/attack/**": true, + "**/pyrit/executor/promptgen/gcg/attack/**": true, "**/doc/**": true, "**/.ty_cache/**": true, "**/frontend/node_modules/**": true, diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 219f504b80..461321309a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -91,5 +91,5 @@ repos: entry: uv run --link-mode=copy ty check language: system files: ^pyrit/ - exclude: ^pyrit/auxiliary_attacks/ + exclude: ^pyrit/executor/promptgen/gcg/ types: [python] diff --git a/MANIFEST.in b/MANIFEST.in index 5d5311b48e..9ceb63cd55 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,5 +8,5 @@ recursive-include pyrit *.png recursive-include pyrit *.wav recursive-include pyrit *.mp4 recursive-include pyrit *.md -include pyrit/auxiliary_attacks/gcg/src/Dockerfile +include pyrit/executor/promptgen/gcg/src/Dockerfile recursive-include pyrit/backend/frontend * diff --git a/doc/code/auxiliary_attacks/0_auxiliary_attacks.ipynb b/doc/code/executor/gcg/0_gcg.ipynb similarity index 93% rename from doc/code/auxiliary_attacks/0_auxiliary_attacks.ipynb rename to doc/code/executor/gcg/0_gcg.ipynb index e396a6b671..bd31eeba96 100644 --- a/doc/code/auxiliary_attacks/0_auxiliary_attacks.ipynb +++ b/doc/code/executor/gcg/0_gcg.ipynb @@ -7,7 +7,7 @@ "lines_to_next_cell": 0 }, "source": [ - "# Auxiliary Attacks" + "# Using GCG Adversarial Suffixes" ] }, { @@ -17,17 +17,7 @@ "lines_to_next_cell": 0 }, "source": [ - "> ⚠️ **Experimental module.** `pyrit.auxiliary_attacks` is experimental: its\n", - "> APIs may change in any release without a deprecation cycle. Importing the\n", - "> package (or any submodule) emits a `pyrit.exceptions.ExperimentalWarning`.\n", - "> Pin pyrit to a specific version if you depend on it. To silence the\n", - "> warning:\n", - ">\n", - "> ```python\n", - "> import warnings\n", - "> from pyrit.exceptions import ExperimentalWarning\n", - "> warnings.filterwarnings(\"ignore\", category=ExperimentalWarning)\n", - "> ```" + "The greedy coordinate gradient (GCG) [@zou2023gcg] algorithm generates adversarial suffixes that can be appended to a base prompt to jailbreak a language model. PyRIT provides an Azure Machine Learning (AML) pipeline for generating these suffixes." ] }, { @@ -36,25 +26,13 @@ "metadata": { "lines_to_next_cell": 0 }, - "source": [ - "Auxiliary attacks cover a variety of techniques that do not fit into the core PyRIT functionality.\n", - "\n", - "These attack pipelines may be useful to run before orchestrating other attacks. For example, we provide an Azure Machine Learning (AML) pipeline for generating suffixes using the greedy coordinate gradient (GCG) [@zou2023gcg] algorithm." - ] - }, - { - "cell_type": "markdown", - "id": "3", - "metadata": { - "lines_to_next_cell": 0 - }, "source": [ "## GCG Suffixes" ] }, { "cell_type": "markdown", - "id": "4", + "id": "3", "metadata": { "lines_to_next_cell": 0 }, @@ -67,7 +45,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "4", "metadata": {}, "outputs": [ { @@ -167,7 +145,7 @@ }, { "cell_type": "markdown", - "id": "6", + "id": "5", "metadata": {}, "source": [ "Next, let's apply a GCG suffix trained on Phi-3-mini to the base prompt using the `SuffixAppendConverter`." @@ -176,7 +154,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "6", "metadata": {}, "outputs": [ { @@ -264,7 +242,8 @@ ], "metadata": { "jupytext": { - "cell_metadata_filter": "-all" + "cell_metadata_filter": "-all", + "main_language": "python" }, "language_info": { "codemirror_mode": { diff --git a/doc/code/auxiliary_attacks/0_auxiliary_attacks.py b/doc/code/executor/gcg/0_gcg.py similarity index 75% rename from doc/code/auxiliary_attacks/0_auxiliary_attacks.py rename to doc/code/executor/gcg/0_gcg.py index 3f486c3f63..b90c880f9a 100644 --- a/doc/code/auxiliary_attacks/0_auxiliary_attacks.py +++ b/doc/code/executor/gcg/0_gcg.py @@ -9,23 +9,9 @@ # jupytext_version: 1.17.3 # --- # %% [markdown] -# # Auxiliary Attacks +# # Using GCG Adversarial Suffixes # %% [markdown] -# > ⚠️ **Experimental module.** `pyrit.auxiliary_attacks` is experimental: its -# > APIs may change in any release without a deprecation cycle. Importing the -# > package (or any submodule) emits a `pyrit.exceptions.ExperimentalWarning`. -# > Pin pyrit to a specific version if you depend on it. To silence the -# > warning: -# > -# > ```python -# > import warnings -# > from pyrit.exceptions import ExperimentalWarning -# > warnings.filterwarnings("ignore", category=ExperimentalWarning) -# > ``` -# %% [markdown] -# Auxiliary attacks cover a variety of techniques that do not fit into the core PyRIT functionality. -# -# These attack pipelines may be useful to run before orchestrating other attacks. For example, we provide an Azure Machine Learning (AML) pipeline for generating suffixes using the greedy coordinate gradient (GCG) [@zou2023gcg] algorithm. +# The greedy coordinate gradient (GCG) [@zou2023gcg] algorithm generates adversarial suffixes that can be appended to a base prompt to jailbreak a language model. PyRIT provides an Azure Machine Learning (AML) pipeline for generating these suffixes. # %% [markdown] # ## GCG Suffixes # %% [markdown] diff --git a/doc/code/auxiliary_attacks/1_gcg_azure_ml.ipynb b/doc/code/executor/gcg/1_gcg_azure_ml.ipynb similarity index 88% rename from doc/code/auxiliary_attacks/1_gcg_azure_ml.ipynb rename to doc/code/executor/gcg/1_gcg_azure_ml.ipynb index eb30d7e990..bb0f021272 100644 --- a/doc/code/auxiliary_attacks/1_gcg_azure_ml.ipynb +++ b/doc/code/executor/gcg/1_gcg_azure_ml.ipynb @@ -12,23 +12,6 @@ "cell_type": "markdown", "id": "1", "metadata": {}, - "source": [ - "> ⚠️ **Experimental module.** `pyrit.auxiliary_attacks` is experimental: its\n", - "> APIs may change in any release without a deprecation cycle. Importing the\n", - "> package below emits a `pyrit.exceptions.ExperimentalWarning`. Pin pyrit to\n", - "> a specific version if you depend on it. To silence the warning:\n", - ">\n", - "> ```python\n", - "> import warnings\n", - "> from pyrit.exceptions import ExperimentalWarning\n", - "> warnings.filterwarnings(\"ignore\", category=ExperimentalWarning)\n", - "> ```" - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, "source": [ "This notebook shows how to generate GCG [@zou2023gcg] suffixes using Azure Machine Learning (AML), which consists of three main steps:\n", "1. Connect to an Azure Machine Learning (AML) workspace.\n", @@ -38,7 +21,7 @@ }, { "cell_type": "markdown", - "id": "3", + "id": "2", "metadata": {}, "source": [ "## Connect to Azure Machine Learning Workspace" @@ -46,7 +29,7 @@ }, { "cell_type": "markdown", - "id": "4", + "id": "3", "metadata": {}, "source": [ "The [workspace](https://docs.microsoft.com/en-us/azure/machine-learning/concept-workspace) is the top-level resource for Azure Machine Learning (AML), providing a centralized place to work with all the artifacts you create when using AML. In this section, we will connect to the workspace in which the job will be run.\n", @@ -57,7 +40,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "4", "metadata": {}, "outputs": [ { @@ -86,7 +69,7 @@ }, { "cell_type": "markdown", - "id": "6", + "id": "5", "metadata": {}, "source": [ "The Azure ML SDK emits a fair amount of telemetry to stderr that looks\n", @@ -101,7 +84,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "6", "metadata": {}, "outputs": [], "source": [ @@ -115,7 +98,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "7", "metadata": {}, "outputs": [ { @@ -135,7 +118,7 @@ }, { "cell_type": "markdown", - "id": "9", + "id": "8", "metadata": {}, "source": [ "## Create AML Environment" @@ -143,24 +126,24 @@ }, { "cell_type": "markdown", - "id": "10", + "id": "9", "metadata": {}, "source": [ "To install the dependencies needed to run GCG, we create an AML environment from a\n", - "[Dockerfile](../../../pyrit/auxiliary_attacks/gcg/src/Dockerfile). The Dockerfile uses\n", + "[Dockerfile](https://github.com/microsoft/PyRIT/blob/main/pyrit/executor/promptgen/gcg/src/Dockerfile). The Dockerfile uses\n", "an NVIDIA CUDA base image with Python 3.11 and installs PyRIT with the `gcg` extra." ] }, { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "10", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Environment({'arm_type': 'environment_version', 'latest_version': None, 'image': None, 'intellectual_property': None, 'is_anonymous': False, 'auto_increment_version': False, 'auto_delete_setting': None, 'name': 'pyrit-gcg', 'description': 'PyRIT GCG environment: CUDA 12.1 + Python 3.11 + pip install -e .[gcg]', 'tags': {'Owner': 'unknown'}, 'properties': {'azureml.labels': 'latest'}, 'print_as_yaml': False, 'id': '/subscriptions/db1ba766-2ca3-42c6-a19a-0f0d43134a8c/resourceGroups/gcg-romanlutz/providers/Microsoft.MachineLearningServices/workspaces/gcg-romanlutz/environments/pyrit-gcg/versions/17', 'Resource__source_path': '', 'base_path': './git/copilot-worktrees/PyRIT/romanlutz-upgraded-barnacle/doc/code/auxiliary_attacks', 'creation_context': , 'serialize': , 'version': '17', 'conda_file': None, 'build': , 'inference_config': None, 'os_type': 'Linux', 'conda_file_path': None, 'path': None, 'datastore': None, 'upload_hash': None, 'translated_conda_file': None})" + "Environment({'arm_type': 'environment_version', 'latest_version': None, 'image': None, 'intellectual_property': None, 'is_anonymous': False, 'auto_increment_version': False, 'auto_delete_setting': None, 'name': 'pyrit-gcg', 'description': 'PyRIT GCG environment: CUDA 12.1 + Python 3.11 + pip install -e .[gcg]', 'tags': {'Owner': 'unknown'}, 'properties': {'azureml.labels': 'latest'}, 'print_as_yaml': False, 'id': '/subscriptions/db1ba766-2ca3-42c6-a19a-0f0d43134a8c/resourceGroups/gcg-romanlutz/providers/Microsoft.MachineLearningServices/workspaces/gcg-romanlutz/environments/pyrit-gcg/versions/17', 'Resource__source_path': '', 'base_path': './git/copilot-worktrees/PyRIT/romanlutz-upgraded-barnacle/doc/code/executor/gcg', 'creation_context': , 'serialize': , 'version': '17', 'conda_file': None, 'build': , 'inference_config': None, 'os_type': 'Linux', 'conda_file_path': None, 'path': None, 'datastore': None, 'upload_hash': None, 'translated_conda_file': None})" ] }, "execution_count": null, @@ -180,7 +163,7 @@ "env_docker_context = Environment(\n", " build=BuildContext(\n", " path=Path(HOME_PATH),\n", - " dockerfile_path=\"pyrit/auxiliary_attacks/gcg/src/Dockerfile\",\n", + " dockerfile_path=\"pyrit/executor/promptgen/gcg/src/Dockerfile\",\n", " ),\n", " name=\"pyrit-gcg\",\n", " description=\"PyRIT GCG environment: CUDA 12.1 + Python 3.11 + pip install -e .[gcg]\",\n", @@ -192,7 +175,7 @@ }, { "cell_type": "markdown", - "id": "12", + "id": "11", "metadata": {}, "source": [ "## Submit Training Job to AML" @@ -200,11 +183,11 @@ }, { "cell_type": "markdown", - "id": "13", + "id": "12", "metadata": {}, "source": [ "Finally, we configure the command to run the GCG algorithm. The entry point is\n", - "[`pyrit.auxiliary_attacks.gcg.experiments.run`](../../../pyrit/auxiliary_attacks/gcg/experiments/run.py),\n", + "[`pyrit.executor.promptgen.gcg.experiments.run`](https://github.com/microsoft/PyRIT/blob/main/pyrit/executor/promptgen/gcg/experiments/run.py),\n", "invoked as a module so the uploaded code snapshot takes priority over the\n", "Docker-installed package (Python's `-m` flag puts the cwd at the front of `sys.path`).\n", "\n", @@ -212,7 +195,7 @@ "``GCGDataConfig`` (CSV paths/counts). We build both locally with whatever\n", "overrides we want, serialize each into a JSON file the AML job can read as\n", "an input, and ship those paths through the job command. Defaults come from\n", - "the dataclasses in ``pyrit.auxiliary_attacks.gcg.config``; goals and targets\n", + "the dataclasses in ``pyrit.executor.promptgen.gcg.config``; goals and targets\n", "flow into ``GCGGenerator.execute_async`` at runtime, not through the config.\n", "\n", "We also have to specify a GPU compute target. In our experience, a GPU instance with\n", @@ -226,22 +209,13 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "13", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "./AppData/Local/Temp/ipykernel_95196/4292719205.py:3: ExperimentalWarning: pyrit.auxiliary_attacks is experimental: APIs may change in any release without a deprecation cycle. Pin pyrit to a specific version if you depend on this module. To silence: warnings.filterwarnings('ignore', category=pyrit.exceptions.ExperimentalWarning).\n", - " from pyrit.auxiliary_attacks.gcg import (\n" - ] - } - ], + "outputs": [], "source": [ "import tempfile\n", "\n", - "from pyrit.auxiliary_attacks.gcg import (\n", + "from pyrit.executor.promptgen.gcg import (\n", " GCGAlgorithmConfig,\n", " GCGConfig,\n", " GCGDataConfig,\n", @@ -271,7 +245,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -280,7 +254,7 @@ "job = command(\n", " code=Path(HOME_PATH),\n", " command=(\n", - " \"python -m pyrit.auxiliary_attacks.gcg.experiments.run\"\n", + " \"python -m pyrit.executor.promptgen.gcg.experiments.run\"\n", " \" --config ${{inputs.config}}\"\n", " \" --data ${{inputs.data}}\"\n", " \" --output-dir ${{outputs.results}}\"\n", @@ -302,7 +276,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "15", "metadata": {}, "outputs": [ { @@ -373,7 +347,7 @@ }, { "cell_type": "markdown", - "id": "17", + "id": "16", "metadata": {}, "source": [ "## Wait for the Job to Complete and Inspect the Generated Suffix\n", @@ -394,7 +368,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "17", "metadata": {}, "outputs": [ { @@ -490,7 +464,8 @@ ], "metadata": { "jupytext": { - "cell_metadata_filter": "-all" + "cell_metadata_filter": "-all", + "main_language": "python" }, "language_info": { "codemirror_mode": { diff --git a/doc/code/auxiliary_attacks/1_gcg_azure_ml.py b/doc/code/executor/gcg/1_gcg_azure_ml.py similarity index 89% rename from doc/code/auxiliary_attacks/1_gcg_azure_ml.py rename to doc/code/executor/gcg/1_gcg_azure_ml.py index b6620f65b3..985ba6399c 100644 --- a/doc/code/auxiliary_attacks/1_gcg_azure_ml.py +++ b/doc/code/executor/gcg/1_gcg_azure_ml.py @@ -12,18 +12,6 @@ # %% [markdown] # # 1. Generating GCG Suffixes Using Azure Machine Learning -# %% [markdown] -# > ⚠️ **Experimental module.** `pyrit.auxiliary_attacks` is experimental: its -# > APIs may change in any release without a deprecation cycle. Importing the -# > package below emits a `pyrit.exceptions.ExperimentalWarning`. Pin pyrit to -# > a specific version if you depend on it. To silence the warning: -# > -# > ```python -# > import warnings -# > from pyrit.exceptions import ExperimentalWarning -# > warnings.filterwarnings("ignore", category=ExperimentalWarning) -# > ``` - # %% [markdown] # This notebook shows how to generate GCG [@zou2023gcg] suffixes using Azure Machine Learning (AML), which consists of three main steps: # 1. Connect to an Azure Machine Learning (AML) workspace. @@ -77,7 +65,7 @@ # %% [markdown] # To install the dependencies needed to run GCG, we create an AML environment from a -# [Dockerfile](../../../pyrit/auxiliary_attacks/gcg/src/Dockerfile). The Dockerfile uses +# [Dockerfile](https://github.com/microsoft/PyRIT/blob/main/pyrit/executor/promptgen/gcg/src/Dockerfile). The Dockerfile uses # an NVIDIA CUDA base image with Python 3.11 and installs PyRIT with the `gcg` extra. # %% @@ -92,7 +80,7 @@ env_docker_context = Environment( build=BuildContext( path=Path(HOME_PATH), - dockerfile_path="pyrit/auxiliary_attacks/gcg/src/Dockerfile", + dockerfile_path="pyrit/executor/promptgen/gcg/src/Dockerfile", ), name="pyrit-gcg", description="PyRIT GCG environment: CUDA 12.1 + Python 3.11 + pip install -e .[gcg]", @@ -106,7 +94,7 @@ # %% [markdown] # Finally, we configure the command to run the GCG algorithm. The entry point is -# [`pyrit.auxiliary_attacks.gcg.experiments.run`](../../../pyrit/auxiliary_attacks/gcg/experiments/run.py), +# [`pyrit.executor.promptgen.gcg.experiments.run`](https://github.com/microsoft/PyRIT/blob/main/pyrit/executor/promptgen/gcg/experiments/run.py), # invoked as a module so the uploaded code snapshot takes priority over the # Docker-installed package (Python's `-m` flag puts the cwd at the front of `sys.path`). # @@ -114,7 +102,7 @@ # ``GCGDataConfig`` (CSV paths/counts). We build both locally with whatever # overrides we want, serialize each into a JSON file the AML job can read as # an input, and ship those paths through the job command. Defaults come from -# the dataclasses in ``pyrit.auxiliary_attacks.gcg.config``; goals and targets +# the dataclasses in ``pyrit.executor.promptgen.gcg.config``; goals and targets # flow into ``GCGGenerator.execute_async`` at runtime, not through the config. # # We also have to specify a GPU compute target. In our experience, a GPU instance with @@ -127,7 +115,7 @@ # %% import tempfile -from pyrit.auxiliary_attacks.gcg import ( +from pyrit.executor.promptgen.gcg import ( GCGAlgorithmConfig, GCGConfig, GCGDataConfig, @@ -159,7 +147,7 @@ job = command( code=Path(HOME_PATH), command=( - "python -m pyrit.auxiliary_attacks.gcg.experiments.run" + "python -m pyrit.executor.promptgen.gcg.experiments.run" " --config ${{inputs.config}}" " --data ${{inputs.data}}" " --output-dir ${{outputs.results}}" diff --git a/doc/generate_docs/pct_to_ipynb.py b/doc/generate_docs/pct_to_ipynb.py index 68270cfe95..e3ef4cacc0 100644 --- a/doc/generate_docs/pct_to_ipynb.py +++ b/doc/generate_docs/pct_to_ipynb.py @@ -8,7 +8,7 @@ skip_files = { "conf.py", - # auxiliary_attacks + # executor / gcg "1_gcg_azure_ml.py", # missing required env variables # converters "2_audio_converters.py", # requires Azure Speech API key diff --git a/doc/myst.yml b/doc/myst.yml index c0f22d68ff..c302b62f5a 100644 --- a/doc/myst.yml +++ b/doc/myst.yml @@ -108,6 +108,9 @@ project: - file: code/executor/6_benchmark.ipynb - file: code/executor/7_promptgen.ipynb - file: code/executor/8_modality_feedback.ipynb + - file: code/executor/gcg/0_gcg.ipynb + children: + - file: code/executor/gcg/1_gcg_azure_ml.ipynb - file: code/targets/0_prompt_targets.md children: - file: code/targets/1_openai_chat_target.ipynb @@ -162,9 +165,6 @@ project: - file: code/setup/2_resiliency.ipynb - file: code/setup/default_values.md - file: code/setup/pyrit_initializer.ipynb - - file: code/auxiliary_attacks/0_auxiliary_attacks.ipynb - children: - - file: code/auxiliary_attacks/1_gcg_azure_ml.ipynb - file: code/scenarios/0_scenarios.ipynb children: - file: code/scenarios/0_attack_techniques.ipynb @@ -180,7 +180,6 @@ project: children: - file: api/pyrit_analytics.md - file: api/pyrit_auth.md - - file: api/pyrit_auxiliary_attacks.md - file: api/pyrit_cli_api_client.md - file: api/pyrit_cli_pyrit_scan.md - file: api/pyrit_cli_pyrit_shell.md diff --git a/pyproject.toml b/pyproject.toml index a4a88c776f..5561171afe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -211,7 +211,7 @@ replace-imports-with-any = [ ] [tool.ty.src] -exclude = ["doc/code/", "pyrit/auxiliary_attacks/"] +exclude = ["doc/code/", "pyrit/executor/promptgen/gcg/"] # Relax type checking for HuggingFace module [[tool.ty.overrides]] @@ -426,7 +426,7 @@ notice-rgx = "Copyright \\(c\\) Microsoft Corporation\\.\\s*\\n.*Licensed under # Backend API routes raise HTTPException handled by FastAPI, not true exceptions "pyrit/backend/**/*.py" = ["DOC501", "B008"] # Vendored GCG attack code — external project with its own style -"pyrit/auxiliary_attacks/**/*.py" = ["A", "B905", "D", "DOC", "N", "PERF", "SIM101", "SIM108"] +"pyrit/executor/promptgen/gcg/**/*.py" = ["A", "B905", "D", "DOC", "N", "PERF", "SIM101", "SIM108"] "pyrit/__init__.py" = ["D104"] # Allow broad pytest.raises(Exception) in tests "tests/**/*.py" = ["ANN", "B017"] diff --git a/pyrit/auxiliary_attacks/__init__.py b/pyrit/auxiliary_attacks/__init__.py deleted file mode 100644 index aadde75182..0000000000 --- a/pyrit/auxiliary_attacks/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -""" -Experimental auxiliary attack implementations (e.g., GCG). - -This subpackage is **experimental**: APIs may change in any release without a -deprecation cycle. Pin pyrit to a specific version if you depend on it. To -silence the warning emitted on import:: - - import warnings - from pyrit.exceptions import ExperimentalWarning - warnings.filterwarnings("ignore", category=ExperimentalWarning) -""" - -import warnings - -from pyrit.exceptions import ExperimentalWarning - -warnings.warn( - "pyrit.auxiliary_attacks is experimental: APIs may change in any release " - "without a deprecation cycle. Pin pyrit to a specific version if you depend " - "on this module. To silence: " - "warnings.filterwarnings('ignore', category=pyrit.exceptions.ExperimentalWarning).", - ExperimentalWarning, - stacklevel=2, -) diff --git a/pyrit/converter/suffix_append_converter.py b/pyrit/converter/suffix_append_converter.py index effb23890f..14120e6a41 100644 --- a/pyrit/converter/suffix_append_converter.py +++ b/pyrit/converter/suffix_append_converter.py @@ -11,7 +11,7 @@ class SuffixAppendConverter(Converter): Appends a specified suffix to the prompt. E.g. with a suffix `!!!`, it converts a prompt of `test` to `test !!!`. - See https://github.com/microsoft/PyRIT/tree/main/pyrit/auxiliary_attacks/gcg for adversarial suffix generation. + See https://github.com/microsoft/PyRIT/tree/main/pyrit/executor/promptgen/gcg for adversarial suffix generation. """ SUPPORTED_INPUT_TYPES = ("text",) diff --git a/pyrit/auxiliary_attacks/gcg/__init__.py b/pyrit/executor/promptgen/gcg/__init__.py similarity index 50% rename from pyrit/auxiliary_attacks/gcg/__init__.py rename to pyrit/executor/promptgen/gcg/__init__.py index 160b2f313a..6e58eb52d1 100644 --- a/pyrit/auxiliary_attacks/gcg/__init__.py +++ b/pyrit/executor/promptgen/gcg/__init__.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Public API for the Greedy Coordinate Gradient (GCG) auxiliary attack. +"""Public API for the Greedy Coordinate Gradient (GCG) adversarial-suffix generator. The primary entry point is ``GCG`` (alias for ``GCGGenerator``), a ``pyrit.executor.promptgen.core.PromptGeneratorStrategy`` that produces @@ -9,7 +9,7 @@ Example: - from pyrit.auxiliary_attacks.gcg import ( + from pyrit.executor.promptgen.gcg import ( GCG, GCGAlgorithmConfig, GCGModelConfig, @@ -23,11 +23,21 @@ goals=["how do I ..."], targets=["Sure, here is ..."], ) + +This subpackage is **experimental**: APIs may change in any release without a +deprecation cycle. Pin pyrit to a specific version if you depend on it. To +silence the warning emitted on import:: + + import warnings + from pyrit.exceptions import ExperimentalWarning + warnings.filterwarnings("ignore", category=ExperimentalWarning) """ +import warnings from typing import TYPE_CHECKING, Any -from pyrit.auxiliary_attacks.gcg.config import ( +from pyrit.exceptions import ExperimentalWarning +from pyrit.executor.promptgen.gcg.config import ( GCGAlgorithmConfig, GCGConfig, GCGDataConfig, @@ -36,8 +46,17 @@ GCGStrategyConfig, ) +warnings.warn( + "pyrit.executor.promptgen.gcg is experimental: APIs may change in any release " + "without a deprecation cycle. Pin pyrit to a specific version if you depend " + "on this module. To silence: " + "warnings.filterwarnings('ignore', category=pyrit.exceptions.ExperimentalWarning).", + ExperimentalWarning, + stacklevel=2, +) + # Torch-dependent symbols are exposed lazily via PEP 562 __getattr__ so that -# `from pyrit.auxiliary_attacks.gcg import GCGConfig` works on installs that +# `from pyrit.executor.promptgen.gcg import GCGConfig` works on installs that # only have the base `dev` extra (no torch). Touching any of these names from # the package root triggers the underlying module import on first access; if # torch is missing the user gets a clear ModuleNotFoundError pointing at torch. @@ -46,36 +65,36 @@ # module imports cleanly without torch) but are routed through the same lazy # mechanism so all GCG public symbols share one re-export pathway. _LAZY_IMPORTS = { - "CandidateFilter": ("pyrit.auxiliary_attacks.gcg.extension_protocols", "CandidateFilter"), - "CrossEntropyLoss": ("pyrit.auxiliary_attacks.gcg.default_implementations", "CrossEntropyLoss"), - "GCG": ("pyrit.auxiliary_attacks.gcg.generator", "GCGGenerator"), - "GCGContext": ("pyrit.auxiliary_attacks.gcg.generator", "GCGContext"), - "GCGGenerator": ("pyrit.auxiliary_attacks.gcg.generator", "GCGGenerator"), - "GCGResult": ("pyrit.auxiliary_attacks.gcg.generator", "GCGResult"), - "LengthPreservingFilter": ("pyrit.auxiliary_attacks.gcg.default_implementations", "LengthPreservingFilter"), - "LiteralStringInit": ("pyrit.auxiliary_attacks.gcg.default_implementations", "LiteralStringInit"), - "LossFunction": ("pyrit.auxiliary_attacks.gcg.extension_protocols", "LossFunction"), - "SamplingStrategy": ("pyrit.auxiliary_attacks.gcg.extension_protocols", "SamplingStrategy"), - "StandardGCGSampling": ("pyrit.auxiliary_attacks.gcg.default_implementations", "StandardGCGSampling"), - "SuffixInitializer": ("pyrit.auxiliary_attacks.gcg.extension_protocols", "SuffixInitializer"), - "load_goals_and_targets": ("pyrit.auxiliary_attacks.gcg.data", "load_goals_and_targets"), + "CandidateFilter": ("pyrit.executor.promptgen.gcg.extension_protocols", "CandidateFilter"), + "CrossEntropyLoss": ("pyrit.executor.promptgen.gcg.default_implementations", "CrossEntropyLoss"), + "GCG": ("pyrit.executor.promptgen.gcg.generator", "GCGGenerator"), + "GCGContext": ("pyrit.executor.promptgen.gcg.generator", "GCGContext"), + "GCGGenerator": ("pyrit.executor.promptgen.gcg.generator", "GCGGenerator"), + "GCGResult": ("pyrit.executor.promptgen.gcg.generator", "GCGResult"), + "LengthPreservingFilter": ("pyrit.executor.promptgen.gcg.default_implementations", "LengthPreservingFilter"), + "LiteralStringInit": ("pyrit.executor.promptgen.gcg.default_implementations", "LiteralStringInit"), + "LossFunction": ("pyrit.executor.promptgen.gcg.extension_protocols", "LossFunction"), + "SamplingStrategy": ("pyrit.executor.promptgen.gcg.extension_protocols", "SamplingStrategy"), + "StandardGCGSampling": ("pyrit.executor.promptgen.gcg.default_implementations", "StandardGCGSampling"), + "SuffixInitializer": ("pyrit.executor.promptgen.gcg.extension_protocols", "SuffixInitializer"), + "load_goals_and_targets": ("pyrit.executor.promptgen.gcg.data", "load_goals_and_targets"), } if TYPE_CHECKING: - from pyrit.auxiliary_attacks.gcg.data import load_goals_and_targets - from pyrit.auxiliary_attacks.gcg.default_implementations import ( + from pyrit.executor.promptgen.gcg.data import load_goals_and_targets + from pyrit.executor.promptgen.gcg.default_implementations import ( CrossEntropyLoss, LengthPreservingFilter, LiteralStringInit, StandardGCGSampling, ) - from pyrit.auxiliary_attacks.gcg.extension_protocols import ( + from pyrit.executor.promptgen.gcg.extension_protocols import ( CandidateFilter, LossFunction, SamplingStrategy, SuffixInitializer, ) - from pyrit.auxiliary_attacks.gcg.generator import ( + from pyrit.executor.promptgen.gcg.generator import ( GCGContext, GCGGenerator, GCGResult, diff --git a/pyrit/auxiliary_attacks/gcg/attack/__init__.py b/pyrit/executor/promptgen/gcg/attack/__init__.py similarity index 100% rename from pyrit/auxiliary_attacks/gcg/attack/__init__.py rename to pyrit/executor/promptgen/gcg/attack/__init__.py diff --git a/pyrit/auxiliary_attacks/gcg/attack/base/__init__.py b/pyrit/executor/promptgen/gcg/attack/base/__init__.py similarity index 100% rename from pyrit/auxiliary_attacks/gcg/attack/base/__init__.py rename to pyrit/executor/promptgen/gcg/attack/base/__init__.py diff --git a/pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py b/pyrit/executor/promptgen/gcg/attack/base/attack_manager.py similarity index 99% rename from pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py rename to pyrit/executor/promptgen/gcg/attack/base/attack_manager.py index ef61ebc4d7..c5c65c07f9 100644 --- a/pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py +++ b/pyrit/executor/promptgen/gcg/attack/base/attack_manager.py @@ -29,7 +29,7 @@ Phi3ForCausalLM, ) -from pyrit.auxiliary_attacks.gcg.experiments.log import ( +from pyrit.executor.promptgen.gcg.experiments.log import ( log_gpu_memory, log_loss, log_table_summary, diff --git a/pyrit/auxiliary_attacks/gcg/attack/gcg/__init__.py b/pyrit/executor/promptgen/gcg/attack/gcg/__init__.py similarity index 100% rename from pyrit/auxiliary_attacks/gcg/attack/gcg/__init__.py rename to pyrit/executor/promptgen/gcg/attack/gcg/__init__.py diff --git a/pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py b/pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py similarity index 98% rename from pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py rename to pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py index ea0677527e..dfe5a11d31 100644 --- a/pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py +++ b/pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py @@ -10,19 +10,19 @@ import torch.nn as nn from tqdm.auto import tqdm -from pyrit.auxiliary_attacks.gcg.attack.base.attack_manager import ( +from pyrit.executor.promptgen.gcg.attack.base.attack_manager import ( AttackPrompt, MultiPromptAttack, PromptManager, get_embedding_matrix, get_embeddings, ) -from pyrit.auxiliary_attacks.gcg.default_implementations import ( +from pyrit.executor.promptgen.gcg.default_implementations import ( CrossEntropyLoss, LengthPreservingFilter, StandardGCGSampling, ) -from pyrit.auxiliary_attacks.gcg.extension_protocols import CandidateFilter, LossFunction, SamplingStrategy +from pyrit.executor.promptgen.gcg.extension_protocols import CandidateFilter, LossFunction, SamplingStrategy logger = logging.getLogger(__name__) diff --git a/pyrit/auxiliary_attacks/gcg/config.py b/pyrit/executor/promptgen/gcg/config.py similarity index 98% rename from pyrit/auxiliary_attacks/gcg/config.py rename to pyrit/executor/promptgen/gcg/config.py index c2debada6e..ad07b1122c 100644 --- a/pyrit/auxiliary_attacks/gcg/config.py +++ b/pyrit/executor/promptgen/gcg/config.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from pathlib import Path - from pyrit.auxiliary_attacks.gcg.extension_protocols import ( + from pyrit.executor.promptgen.gcg.extension_protocols import ( CandidateFilter, LossFunction, SamplingStrategy, @@ -78,7 +78,7 @@ class GCGDataConfig: Used as a typed bundle for AML transport (a job ships its data config as a separate JSON file alongside the strategy ``GCGConfig``). Library callers loading goals/targets from a CSV can construct one and pass it to - ``pyrit.auxiliary_attacks.gcg.data.load_goals_and_targets``. + ``pyrit.executor.promptgen.gcg.data.load_goals_and_targets``. Attributes: train_data (str): URL or filesystem path to the training-data CSV. Empty @@ -209,7 +209,7 @@ def __post_init__(self) -> None: self._validate_extensions() def _validate_extensions(self) -> None: - from pyrit.auxiliary_attacks.gcg.extension_protocols import ( + from pyrit.executor.promptgen.gcg.extension_protocols import ( CandidateFilter, LossFunction, SamplingStrategy, @@ -284,7 +284,7 @@ class GCGOutputConfig: class GCGConfig: """Top-level strategy configuration for one GCG attack run. - Bundles everything ``pyrit.auxiliary_attacks.gcg.GCGGenerator``'s + Bundles everything ``pyrit.executor.promptgen.gcg.GCGGenerator``'s constructor needs. Per-execution data (goals, targets) is **not** here — those flow through ``GCGGenerator.execute_async``, and for AML transport they ride alongside this object as a separate ``GCGDataConfig`` JSON. diff --git a/pyrit/auxiliary_attacks/gcg/data.py b/pyrit/executor/promptgen/gcg/data.py similarity index 92% rename from pyrit/auxiliary_attacks/gcg/data.py rename to pyrit/executor/promptgen/gcg/data.py index 1c8b8efd3a..84ca1e79f1 100644 --- a/pyrit/auxiliary_attacks/gcg/data.py +++ b/pyrit/executor/promptgen/gcg/data.py @@ -13,12 +13,12 @@ from types import SimpleNamespace from typing import TYPE_CHECKING -from pyrit.auxiliary_attacks.gcg.attack.base.attack_manager import ( +from pyrit.executor.promptgen.gcg.attack.base.attack_manager import ( get_goals_and_targets as _legacy_loader, ) if TYPE_CHECKING: - from pyrit.auxiliary_attacks.gcg.config import GCGDataConfig + from pyrit.executor.promptgen.gcg.config import GCGDataConfig def load_goals_and_targets( diff --git a/pyrit/auxiliary_attacks/gcg/default_implementations.py b/pyrit/executor/promptgen/gcg/default_implementations.py similarity index 97% rename from pyrit/auxiliary_attacks/gcg/default_implementations.py rename to pyrit/executor/promptgen/gcg/default_implementations.py index 3967c128c7..502c6f6065 100644 --- a/pyrit/auxiliary_attacks/gcg/default_implementations.py +++ b/pyrit/executor/promptgen/gcg/default_implementations.py @@ -42,7 +42,7 @@ class StandardGCGSampling: sampler, which always samples uniformly within the top-k. Reproduces ``GCGPromptManager.sample_control`` from - ``pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py`` byte-for-byte. + ``pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py`` byte-for-byte. """ def sample_candidates( @@ -118,9 +118,9 @@ class CrossEntropyLoss: The same skip is applied when ``target_weight == 0`` for symmetry. Reproduces ``AttackPrompt.target_loss`` + ``AttackPrompt.control_loss`` - from ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py``, + from ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py``, combined per ``GCGMultiPromptAttack.step`` in - ``pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py``. + ``pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py``. """ def __init__(self, *, target_weight: float = 1.0, control_weight: float = 0.0) -> None: @@ -217,7 +217,7 @@ class LengthPreservingFilter: matching the safety pass at the top of ``get_filtered_cands``. Reproduces ``MultiPromptAttack.get_filtered_cands`` from - ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py``. + ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py``. """ def __init__(self, *, filter: bool = True) -> None: @@ -292,7 +292,7 @@ class LiteralStringInit: Reproduces the literal-string ``control_init`` parameter assignment (``self.control = control_init``) inside ``AttackPrompt.__init__`` in - ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py``. + ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py``. """ def __init__(self, *, suffix: str) -> None: diff --git a/pyrit/auxiliary_attacks/gcg/experiments/README.md b/pyrit/executor/promptgen/gcg/experiments/README.md similarity index 81% rename from pyrit/auxiliary_attacks/gcg/experiments/README.md rename to pyrit/executor/promptgen/gcg/experiments/README.md index bade583b7a..0409d33616 100644 --- a/pyrit/auxiliary_attacks/gcg/experiments/README.md +++ b/pyrit/executor/promptgen/gcg/experiments/README.md @@ -11,7 +11,7 @@ The primary entry point is `GCG.execute_async` (`GCG` is an alias for ```python import asyncio -from pyrit.auxiliary_attacks.gcg import GCG, GCGModelConfig +from pyrit.executor.promptgen.gcg import GCG, GCGModelConfig generator = GCG( models=[GCGModelConfig(name="meta-llama/Llama-2-7b-chat-hf")], @@ -21,7 +21,7 @@ result = asyncio.run(generator.execute_async(goals=[...], targets=[...])) `GCGConfig` is composed of nested sub-configs (`GCGModelConfig`, `GCGDataConfig`, `GCGAlgorithmConfig`, `GCGStrategyConfig`, `GCGOutputConfig`); all are re-exported -from `pyrit.auxiliary_attacks.gcg`. See `pyrit/auxiliary_attacks/gcg/config.py` +from `pyrit.executor.promptgen.gcg`. See `pyrit/executor/promptgen/gcg/config.py` for the full surface and defaults. ### Running on Azure ML @@ -34,10 +34,10 @@ config.to_json_file("inputs/config.json") ``` ``` -python -m pyrit.auxiliary_attacks.gcg.experiments.run --config inputs/config.json +python -m pyrit.executor.promptgen.gcg.experiments.run --config inputs/config.json ``` -The notebook at `doc/code/auxiliary_attacks/1_gcg_azure_ml.py` builds a config +The notebook at `doc/code/executor/gcg/1_gcg_azure_ml.py` builds a config locally, ships it to Azure ML as a job input, and the AML job invokes `run.py` with the path to the deserialized JSON. diff --git a/pyrit/auxiliary_attacks/gcg/experiments/__init__.py b/pyrit/executor/promptgen/gcg/experiments/__init__.py similarity index 100% rename from pyrit/auxiliary_attacks/gcg/experiments/__init__.py rename to pyrit/executor/promptgen/gcg/experiments/__init__.py diff --git a/pyrit/auxiliary_attacks/gcg/experiments/log.py b/pyrit/executor/promptgen/gcg/experiments/log.py similarity index 100% rename from pyrit/auxiliary_attacks/gcg/experiments/log.py rename to pyrit/executor/promptgen/gcg/experiments/log.py diff --git a/pyrit/auxiliary_attacks/gcg/experiments/run.py b/pyrit/executor/promptgen/gcg/experiments/run.py similarity index 93% rename from pyrit/auxiliary_attacks/gcg/experiments/run.py rename to pyrit/executor/promptgen/gcg/experiments/run.py index 396f46131d..3bc76d53c2 100644 --- a/pyrit/auxiliary_attacks/gcg/experiments/run.py +++ b/pyrit/executor/promptgen/gcg/experiments/run.py @@ -8,7 +8,7 @@ ``to_json_file`` methods, ships them to Azure ML as job inputs, and the job's command line is:: - python -m pyrit.auxiliary_attacks.gcg.experiments.run \\ + python -m pyrit.executor.promptgen.gcg.experiments.run \\ --config inputs/config.json \\ --data inputs/data.json \\ --output-dir ${{outputs.results}} @@ -23,9 +23,9 @@ from dataclasses import replace from pathlib import Path -from pyrit.auxiliary_attacks.gcg.config import GCGConfig, GCGDataConfig, GCGOutputConfig -from pyrit.auxiliary_attacks.gcg.data import load_goals_and_targets -from pyrit.auxiliary_attacks.gcg.generator import GCGGenerator +from pyrit.executor.promptgen.gcg.config import GCGConfig, GCGDataConfig, GCGOutputConfig +from pyrit.executor.promptgen.gcg.data import load_goals_and_targets +from pyrit.executor.promptgen.gcg.generator import GCGGenerator from pyrit.setup.initialization import _load_environment_files diff --git a/pyrit/auxiliary_attacks/gcg/extension_protocols.py b/pyrit/executor/promptgen/gcg/extension_protocols.py similarity index 96% rename from pyrit/auxiliary_attacks/gcg/extension_protocols.py rename to pyrit/executor/promptgen/gcg/extension_protocols.py index 973fb22a2b..8385bc98ae 100644 --- a/pyrit/auxiliary_attacks/gcg/extension_protocols.py +++ b/pyrit/executor/promptgen/gcg/extension_protocols.py @@ -24,7 +24,7 @@ Tensor-typed signatures are kept lazy via ``from __future__ import annotations`` plus a ``TYPE_CHECKING`` import for ``torch`` so that -``pyrit.auxiliary_attacks.gcg.extension_protocols`` itself imports cleanly on +``pyrit.executor.promptgen.gcg.extension_protocols`` itself imports cleanly on installs that only have the base ``dev`` extra (no torch). At call time the implementations are still operating on real ``torch.Tensor`` objects — the forward references just keep the runtime import side-effect free. @@ -61,7 +61,7 @@ class SamplingStrategy(Protocol): References: ``GCGPromptManager.sample_control`` in - ``pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py``. + ``pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py``. """ def sample_candidates( @@ -135,9 +135,9 @@ class LossFunction(Protocol): References: ``AttackPrompt.target_loss`` and ``AttackPrompt.control_loss`` in - ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py``, plus + ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py``, plus the weighted-sum aggregation inside ``GCGMultiPromptAttack.step`` in - ``pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py``. + ``pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py``. """ def compute_loss( @@ -195,7 +195,7 @@ class CandidateFilter(Protocol): References: ``MultiPromptAttack.get_filtered_cands`` in - ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py``. + ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py``. """ def filter_candidates( @@ -251,7 +251,7 @@ class SuffixInitializer(Protocol): References: ``AttackPrompt.__init__`` in - ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py`` assigns + ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py`` assigns ``self.control = control_init``. The same ``control_init`` parameter is threaded through the ``PromptManager``, ``MultiPromptAttack``, ``ProgressiveMultiPromptAttack``, ``IndividualPromptAttack``, and diff --git a/pyrit/auxiliary_attacks/gcg/generator.py b/pyrit/executor/promptgen/gcg/generator.py similarity index 98% rename from pyrit/auxiliary_attacks/gcg/generator.py rename to pyrit/executor/promptgen/gcg/generator.py index 12ef46040c..f4456d2da1 100644 --- a/pyrit/auxiliary_attacks/gcg/generator.py +++ b/pyrit/executor/promptgen/gcg/generator.py @@ -45,25 +45,25 @@ import torch.multiprocessing as mp from pydantic import Field -import pyrit.auxiliary_attacks.gcg.attack.gcg.gcg_attack as attack_lib -from pyrit.auxiliary_attacks.gcg.attack.base.attack_manager import ( +import pyrit.executor.promptgen.gcg.attack.gcg.gcg_attack as attack_lib +from pyrit.common.utils import combine_dict +from pyrit.executor.promptgen.core.prompt_generator_strategy import ( + PromptGeneratorStrategy, + PromptGeneratorStrategyContext, + PromptGeneratorStrategyResult, +) +from pyrit.executor.promptgen.gcg.attack.base.attack_manager import ( IndividualPromptAttack, ProgressiveMultiPromptAttack, get_workers, ) -from pyrit.auxiliary_attacks.gcg.config import ( +from pyrit.executor.promptgen.gcg.config import ( GCGAlgorithmConfig, GCGModelConfig, GCGOutputConfig, GCGStrategyConfig, ) -from pyrit.auxiliary_attacks.gcg.experiments.log import log_gpu_memory, log_train_goals -from pyrit.common.utils import combine_dict -from pyrit.executor.promptgen.core.prompt_generator_strategy import ( - PromptGeneratorStrategy, - PromptGeneratorStrategyContext, - PromptGeneratorStrategyResult, -) +from pyrit.executor.promptgen.gcg.experiments.log import log_gpu_memory, log_train_goals from pyrit.models import ComponentIdentifier, Identifiable logger = logging.getLogger(__name__) diff --git a/pyrit/auxiliary_attacks/gcg/src/Dockerfile b/pyrit/executor/promptgen/gcg/src/Dockerfile similarity index 100% rename from pyrit/auxiliary_attacks/gcg/src/Dockerfile rename to pyrit/executor/promptgen/gcg/src/Dockerfile diff --git a/tests/end_to_end/auxiliary_attacks/__init__.py b/tests/end_to_end/auxiliary_attacks/__init__.py deleted file mode 100644 index acd459977f..0000000000 --- a/tests/end_to_end/auxiliary_attacks/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -"""End-to-end tests for PyRIT auxiliary attacks (GCG).""" diff --git a/tests/end_to_end/auxiliary_attacks/test_gcg_aml_e2e.py b/tests/end_to_end/executor/promptgen/gcg/test_gcg_aml_e2e.py similarity index 96% rename from tests/end_to_end/auxiliary_attacks/test_gcg_aml_e2e.py rename to tests/end_to_end/executor/promptgen/gcg/test_gcg_aml_e2e.py index bf761f9ee8..83d5078d80 100644 --- a/tests/end_to_end/auxiliary_attacks/test_gcg_aml_e2e.py +++ b/tests/end_to_end/executor/promptgen/gcg/test_gcg_aml_e2e.py @@ -3,7 +3,7 @@ """End-to-end test for the GCG Azure ML pipeline. -Executes `doc/code/auxiliary_attacks/1_gcg_azure_ml.py` directly as a Python +Executes `doc/code/executor/gcg/1_gcg_azure_ml.py` directly as a Python script (the jupytext percent-format `# %%` markers are plain comments, so the file is valid Python). After the notebook submits the AML job, this test polls until the job reaches a terminal state and asserts success. @@ -55,7 +55,7 @@ "AZURE_ML_WORKSPACE_NAME", "HUGGINGFACE_TOKEN", ) -_NOTEBOOK_PATH = Path(HOME_PATH) / "doc" / "code" / "auxiliary_attacks" / "1_gcg_azure_ml.py" +_NOTEBOOK_PATH = Path(HOME_PATH) / "doc" / "code" / "executor" / "gcg" / "1_gcg_azure_ml.py" _DEFAULT_MAX_WAIT_SECONDS = 5400 # 90 minutes _POLL_INTERVAL_SECONDS = 30 _TERMINAL_STATES = {"Completed", "Failed", "Canceled", "CancelRequested"} diff --git a/tests/integration/auxiliary_attacks/test_gcg_attack_wiring_integration.py b/tests/integration/executor/promptgen/gcg/test_gcg_attack_wiring_integration.py similarity index 97% rename from tests/integration/auxiliary_attacks/test_gcg_attack_wiring_integration.py rename to tests/integration/executor/promptgen/gcg/test_gcg_attack_wiring_integration.py index 1a759d9b0a..9490ac9d56 100644 --- a/tests/integration/auxiliary_attacks/test_gcg_attack_wiring_integration.py +++ b/tests/integration/executor/promptgen/gcg/test_gcg_attack_wiring_integration.py @@ -28,22 +28,22 @@ import pytest attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, mlflow, etc.) not installed", ) torch = pytest.importorskip("torch", reason="torch not installed") gcg_attack_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.gcg.gcg_attack", + "pyrit.executor.promptgen.gcg.attack.gcg.gcg_attack", reason="GCG optional dependencies not installed", ) generator_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.generator", + "pyrit.executor.promptgen.gcg.generator", reason="GCG optional dependencies (torch, transformers, etc.) not installed", ) -from pyrit.auxiliary_attacks.gcg.config import ( # noqa: E402 +from pyrit.executor.promptgen.gcg.config import ( # noqa: E402 GCGAlgorithmConfig, GCGModelConfig, GCGOutputConfig, diff --git a/tests/integration/auxiliary_attacks/test_gcg_integration.py b/tests/integration/executor/promptgen/gcg/test_gcg_integration.py similarity index 98% rename from tests/integration/auxiliary_attacks/test_gcg_integration.py rename to tests/integration/executor/promptgen/gcg/test_gcg_integration.py index f2944bf480..a5ca3cdf78 100644 --- a/tests/integration/auxiliary_attacks/test_gcg_integration.py +++ b/tests/integration/executor/promptgen/gcg/test_gcg_integration.py @@ -25,13 +25,13 @@ from transformers import AutoTokenizer, GPT2LMHeadModel # noqa: E402 -from pyrit.auxiliary_attacks.gcg.attack.base.attack_manager import ( # noqa: E402 +from pyrit.executor.promptgen.gcg.attack.base.attack_manager import ( # noqa: E402 get_embedding_layer, get_embedding_matrix, get_embeddings, get_nonascii_toks, ) -from pyrit.auxiliary_attacks.gcg.attack.gcg.gcg_attack import ( # noqa: E402 +from pyrit.executor.promptgen.gcg.attack.gcg.gcg_attack import ( # noqa: E402 GCGAttackPrompt, GCGPromptManager, token_gradients, diff --git a/tests/integration/auxiliary_attacks/test_notebooks_auxiliary.py b/tests/integration/executor/promptgen/gcg/test_notebooks_gcg.py similarity index 90% rename from tests/integration/auxiliary_attacks/test_notebooks_auxiliary.py rename to tests/integration/executor/promptgen/gcg/test_notebooks_gcg.py index f1755912c4..e59098b802 100644 --- a/tests/integration/auxiliary_attacks/test_notebooks_auxiliary.py +++ b/tests/integration/executor/promptgen/gcg/test_notebooks_gcg.py @@ -10,7 +10,7 @@ from pyrit.common import path -nb_directory_path = pathlib.Path(path.DOCS_CODE_PATH, "auxiliary_attacks").resolve() +nb_directory_path = pathlib.Path(path.DOCS_CODE_PATH, "executor", "gcg").resolve() skipped_files = [ "1_gcg_azure_ml.ipynb", # missing required env variables diff --git a/tests/unit/auxiliary_attacks/__init__.py b/tests/unit/auxiliary_attacks/__init__.py deleted file mode 100644 index 9a0454564d..0000000000 --- a/tests/unit/auxiliary_attacks/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. diff --git a/tests/unit/auxiliary_attacks/gcg/__init__.py b/tests/unit/auxiliary_attacks/gcg/__init__.py deleted file mode 100644 index 9a0454564d..0000000000 --- a/tests/unit/auxiliary_attacks/gcg/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. diff --git a/tests/unit/auxiliary_attacks/gcg/test_attack_manager_helpers.py b/tests/unit/executor/promptgen/gcg/test_attack_manager_helpers.py similarity index 98% rename from tests/unit/auxiliary_attacks/gcg/test_attack_manager_helpers.py rename to tests/unit/executor/promptgen/gcg/test_attack_manager_helpers.py index cf45cb4bdf..f1e2bdfaaa 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_attack_manager_helpers.py +++ b/tests/unit/executor/promptgen/gcg/test_attack_manager_helpers.py @@ -8,7 +8,7 @@ import pytest attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, mlflow, etc.) not installed", ) NpEncoder = attack_manager_mod.NpEncoder diff --git a/tests/unit/auxiliary_attacks/gcg/test_config.py b/tests/unit/executor/promptgen/gcg/test_config.py similarity index 99% rename from tests/unit/auxiliary_attacks/gcg/test_config.py rename to tests/unit/executor/promptgen/gcg/test_config.py index 922b0ffedd..09665f9733 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_config.py +++ b/tests/unit/executor/promptgen/gcg/test_config.py @@ -13,7 +13,7 @@ import pytest -from pyrit.auxiliary_attacks.gcg.config import ( +from pyrit.executor.promptgen.gcg.config import ( GCGAlgorithmConfig, GCGConfig, GCGDataConfig, diff --git a/tests/unit/auxiliary_attacks/gcg/test_data_and_config.py b/tests/unit/executor/promptgen/gcg/test_data_and_config.py similarity index 93% rename from tests/unit/auxiliary_attacks/gcg/test_data_and_config.py rename to tests/unit/executor/promptgen/gcg/test_data_and_config.py index 793cf468ef..d41ffef2db 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_data_and_config.py +++ b/tests/unit/executor/promptgen/gcg/test_data_and_config.py @@ -8,29 +8,29 @@ import pytest -from pyrit.auxiliary_attacks.gcg.config import GCGConfig, GCGDataConfig, GCGModelConfig, GCGOutputConfig +from pyrit.executor.promptgen.gcg.config import GCGConfig, GCGDataConfig, GCGModelConfig, GCGOutputConfig attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, accelerate, etc.) not installed", ) get_goals_and_targets = attack_manager_mod.get_goals_and_targets data_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.data", + "pyrit.executor.promptgen.gcg.data", reason="GCG data module requires torch (transitive via attack_manager)", ) load_goals_and_targets = data_mod.load_goals_and_targets run_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.experiments.run", + "pyrit.executor.promptgen.gcg.experiments.run", reason="GCG run module not available", ) _main_async = run_mod._main_async _resolve_output = run_mod._resolve_output generator_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.generator", + "pyrit.executor.promptgen.gcg.generator", reason="GCG generator module not available", ) GCGGenerator = generator_mod.GCGGenerator @@ -153,7 +153,7 @@ def test_override_falls_back_to_default_basename(self, tmp_path: Path) -> None: class TestMainAsyncCli: """Tests for ``run.py``'s ``--config`` + ``--data`` CLI wrapper around GCGGenerator.execute_async.""" - @patch("pyrit.auxiliary_attacks.gcg.experiments.run._load_environment_files") + @patch("pyrit.executor.promptgen.gcg.experiments.run._load_environment_files") async def test_raises_when_no_token_anywhere(self, mock_load_env: MagicMock, tmp_path: Path) -> None: config = GCGConfig(models=[GCGModelConfig(name="org/model")]) config_path = tmp_path / "config.json" @@ -166,8 +166,8 @@ async def test_raises_when_no_token_anywhere(self, mock_load_env: MagicMock, tmp with pytest.raises(ValueError, match="No HuggingFace token available"): await _main_async(str(config_path), str(data_path)) - @patch("pyrit.auxiliary_attacks.gcg.experiments.run._load_environment_files") - @patch("pyrit.auxiliary_attacks.gcg.experiments.run.load_goals_and_targets") + @patch("pyrit.executor.promptgen.gcg.experiments.run._load_environment_files") + @patch("pyrit.executor.promptgen.gcg.experiments.run.load_goals_and_targets") async def test_passes_loaded_goals_to_generator_and_uses_env_token( self, mock_loader: MagicMock, diff --git a/tests/unit/auxiliary_attacks/gcg/test_default_implementations.py b/tests/unit/executor/promptgen/gcg/test_default_implementations.py similarity index 94% rename from tests/unit/auxiliary_attacks/gcg/test_default_implementations.py rename to tests/unit/executor/promptgen/gcg/test_default_implementations.py index 8b89745052..1c79cdcd55 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_default_implementations.py +++ b/tests/unit/executor/promptgen/gcg/test_default_implementations.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Tests for ``pyrit.auxiliary_attacks.gcg.default_implementations``. +"""Tests for ``pyrit.executor.promptgen.gcg.default_implementations``. These tests verify byte-identical parity between the four default implementations and the legacy GCG attack code paths they reproduce: @@ -15,7 +15,7 @@ inside ``AttackPrompt.__init__`` Mocking patterns follow the conventions established in -``tests/unit/auxiliary_attacks/gcg/test_gcg_core.py`` (``object.__new__`` +``tests/unit/executor/promptgen/gcg/test_gcg_core.py`` (``object.__new__`` to skip the real ``__init__``, ``MagicMock`` tokenizers). """ @@ -26,23 +26,23 @@ torch = pytest.importorskip("torch", reason="GCG default implementations require torch") attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, mlflow, etc.) not installed", ) gcg_attack_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.gcg.gcg_attack", + "pyrit.executor.promptgen.gcg.attack.gcg.gcg_attack", reason="GCG optional dependencies not installed", ) -import pyrit.auxiliary_attacks.gcg as gcg_pkg # noqa: E402 -from pyrit.auxiliary_attacks.gcg import ( # noqa: E402 +import pyrit.executor.promptgen.gcg as gcg_pkg # noqa: E402 +from pyrit.executor.promptgen.gcg import ( # noqa: E402 CrossEntropyLoss, LengthPreservingFilter, LiteralStringInit, StandardGCGSampling, ) -from pyrit.auxiliary_attacks.gcg import default_implementations as defaults_module # noqa: E402 -from pyrit.auxiliary_attacks.gcg.config import GCGAlgorithmConfig # noqa: E402 +from pyrit.executor.promptgen.gcg import default_implementations as defaults_module # noqa: E402 +from pyrit.executor.promptgen.gcg.config import GCGAlgorithmConfig # noqa: E402 AttackPrompt = attack_manager_mod.AttackPrompt MultiPromptAttack = attack_manager_mod.MultiPromptAttack @@ -65,8 +65,8 @@ def test_default_is_reexported_with_identity(self, name: str) -> None: package_attr = getattr(gcg_pkg, name) module_attr = getattr(defaults_module, name) assert package_attr is module_attr, ( - f"{name} re-exported from pyrit.auxiliary_attacks.gcg must be the same " - f"object as pyrit.auxiliary_attacks.gcg.default_implementations.{name}" + f"{name} re-exported from pyrit.executor.promptgen.gcg must be the same " + f"object as pyrit.executor.promptgen.gcg.default_implementations.{name}" ) @pytest.mark.parametrize("name", DEFAULT_NAMES) @@ -95,7 +95,7 @@ def _make_legacy_prompt_manager( def test_sample_candidates_matches_legacy_with_ascii_only(self) -> None: """Legacy reference: ``GCGPromptManager.sample_control(grad, batch_size, topk=top_k, temp=1.0, allow_non_ascii=False)`` in - ``pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py``. + ``pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py``. """ n_control_tokens = 5 vocab_size = 50 @@ -200,7 +200,7 @@ def test_compute_loss_matches_legacy_weighted_sum(self) -> None: ``target_weight * AttackPrompt.target_loss(logits, ids).mean(dim=-1)`` ``+ control_weight * AttackPrompt.control_loss(logits, ids).mean(dim=-1)``, per ``GCGMultiPromptAttack.step`` in - ``pyrit/auxiliary_attacks/gcg/attack/gcg/gcg_attack.py``. + ``pyrit/executor/promptgen/gcg/attack/gcg/gcg_attack.py``. """ batch_size = 4 seq_len = 10 @@ -359,7 +359,7 @@ def test_filter_candidates_matches_legacy_filtered(self) -> None: """Legacy reference: ``MultiPromptAttack.get_filtered_cands(0, control_cand, filter_cand=True, curr_control=...)`` in - ``pyrit/auxiliary_attacks/gcg/attack/base/attack_manager.py``. + ``pyrit/executor/promptgen/gcg/attack/base/attack_manager.py``. With the helper tokenizer: - Row 0 ``[3, 0, 1]`` -> decode ``"xxx"`` (len 3); retok len 3 == diff --git a/tests/unit/auxiliary_attacks/test_experimental_warning.py b/tests/unit/executor/promptgen/gcg/test_experimental_warning.py similarity index 77% rename from tests/unit/auxiliary_attacks/test_experimental_warning.py rename to tests/unit/executor/promptgen/gcg/test_experimental_warning.py index 693e5bf50b..29cff9b408 100644 --- a/tests/unit/auxiliary_attacks/test_experimental_warning.py +++ b/tests/unit/executor/promptgen/gcg/test_experimental_warning.py @@ -4,19 +4,19 @@ import importlib import warnings -import pyrit.auxiliary_attacks +import pyrit.executor.promptgen.gcg from pyrit.exceptions import ExperimentalWarning -def test_importing_auxiliary_attacks_emits_experimental_warning() -> None: +def test_importing_gcg_emits_experimental_warning() -> None: with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - importlib.reload(pyrit.auxiliary_attacks) + importlib.reload(pyrit.executor.promptgen.gcg) experimental = [w for w in caught if issubclass(w.category, ExperimentalWarning)] assert len(experimental) == 1 message = str(experimental[0].message) - assert "pyrit.auxiliary_attacks is experimental" in message + assert "pyrit.executor.promptgen.gcg is experimental" in message assert "ExperimentalWarning" in message @@ -28,7 +28,7 @@ def test_experimental_warning_can_be_silenced() -> None: with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") warnings.filterwarnings("ignore", category=ExperimentalWarning) - importlib.reload(pyrit.auxiliary_attacks) + importlib.reload(pyrit.executor.promptgen.gcg) experimental = [w for w in caught if issubclass(w.category, ExperimentalWarning)] assert experimental == [] diff --git a/tests/unit/auxiliary_attacks/gcg/test_extension_protocols.py b/tests/unit/executor/promptgen/gcg/test_extension_protocols.py similarity index 92% rename from tests/unit/auxiliary_attacks/gcg/test_extension_protocols.py rename to tests/unit/executor/promptgen/gcg/test_extension_protocols.py index 404ef4b541..1aa3c0335d 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_extension_protocols.py +++ b/tests/unit/executor/promptgen/gcg/test_extension_protocols.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Tests for :mod:`pyrit.auxiliary_attacks.gcg.extension_protocols`. +"""Tests for :mod:`pyrit.executor.promptgen.gcg.extension_protocols`. These are typing-surface tests: they verify the four ``Protocol``s are exposed, are ``runtime_checkable``, and accept a minimal in-test concrete @@ -19,14 +19,14 @@ # file is skipped on installs that only have the base ``dev`` extra. torch = pytest.importorskip("torch", reason="GCG extension protocols reference torch.Tensor") -import pyrit.auxiliary_attacks.gcg as gcg_pkg # noqa: E402 -from pyrit.auxiliary_attacks.gcg import ( # noqa: E402 +import pyrit.executor.promptgen.gcg as gcg_pkg # noqa: E402 +from pyrit.executor.promptgen.gcg import ( # noqa: E402 CandidateFilter, LossFunction, SamplingStrategy, SuffixInitializer, ) -from pyrit.auxiliary_attacks.gcg import extension_protocols as protocols_module # noqa: E402 +from pyrit.executor.promptgen.gcg import extension_protocols as protocols_module # noqa: E402 PROTOCOL_NAMES = ( "CandidateFilter", @@ -45,8 +45,8 @@ def test_protocols_are_reexported_from_package_with_identity() -> None: package_attr = getattr(gcg_pkg, name) module_attr = getattr(protocols_module, name) assert package_attr is module_attr, ( - f"{name} re-exported from pyrit.auxiliary_attacks.gcg must be the same " - f"object as pyrit.auxiliary_attacks.gcg.extension_protocols.{name}" + f"{name} re-exported from pyrit.executor.promptgen.gcg must be the same " + f"object as pyrit.executor.promptgen.gcg.extension_protocols.{name}" ) diff --git a/tests/unit/auxiliary_attacks/gcg/test_gcg_core.py b/tests/unit/executor/promptgen/gcg/test_gcg_core.py similarity index 99% rename from tests/unit/auxiliary_attacks/gcg/test_gcg_core.py rename to tests/unit/executor/promptgen/gcg/test_gcg_core.py index a90563ed85..3c25054844 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_gcg_core.py +++ b/tests/unit/executor/promptgen/gcg/test_gcg_core.py @@ -7,7 +7,7 @@ import pytest attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, mlflow, etc.) not installed", ) torch = pytest.importorskip("torch", reason="torch not installed") @@ -23,7 +23,7 @@ get_embeddings = attack_manager_mod.get_embeddings gcg_attack_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.gcg.gcg_attack", + "pyrit.executor.promptgen.gcg.attack.gcg.gcg_attack", reason="GCG optional dependencies not installed", ) GCGMultiPromptAttack = gcg_attack_mod.GCGMultiPromptAttack diff --git a/tests/unit/auxiliary_attacks/gcg/test_generator.py b/tests/unit/executor/promptgen/gcg/test_generator.py similarity index 98% rename from tests/unit/auxiliary_attacks/gcg/test_generator.py rename to tests/unit/executor/promptgen/gcg/test_generator.py index 956dcc4953..f35e90db9a 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_generator.py +++ b/tests/unit/executor/promptgen/gcg/test_generator.py @@ -12,7 +12,7 @@ import pytest -from pyrit.auxiliary_attacks.gcg.config import ( +from pyrit.executor.promptgen.gcg.config import ( GCGAlgorithmConfig, GCGModelConfig, GCGOutputConfig, @@ -23,7 +23,7 @@ from pathlib import Path generator_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.generator", + "pyrit.executor.promptgen.gcg.generator", reason="GCG optional dependencies (torch, transformers, etc.) not installed", ) GCGGenerator = generator_mod.GCGGenerator @@ -357,7 +357,7 @@ def filter_candidates( patch.object(gen, "_create_attack", return_value=fake_attack) as mock_create_attack, patch.object(gen, "_build_logfile_path", return_value=str(tmp_path / "result.json")), patch.object(gen, "_read_result", return_value=GCGResult(final_suffix="x")), - patch("pyrit.auxiliary_attacks.gcg.generator.asyncio.to_thread", new=AsyncMock(return_value=None)), + patch("pyrit.executor.promptgen.gcg.generator.asyncio.to_thread", new=AsyncMock(return_value=None)), ): await gen._perform_async(context=context) diff --git a/tests/unit/auxiliary_attacks/gcg/test_get_goals_and_targets.py b/tests/unit/executor/promptgen/gcg/test_get_goals_and_targets.py similarity index 98% rename from tests/unit/auxiliary_attacks/gcg/test_get_goals_and_targets.py rename to tests/unit/executor/promptgen/gcg/test_get_goals_and_targets.py index 9a6c0c93da..08c78df1a4 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_get_goals_and_targets.py +++ b/tests/unit/executor/promptgen/gcg/test_get_goals_and_targets.py @@ -8,7 +8,7 @@ import pytest attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, mlflow, etc.) not installed", ) get_goals_and_targets = attack_manager_mod.get_goals_and_targets diff --git a/tests/unit/auxiliary_attacks/gcg/test_log.py b/tests/unit/executor/promptgen/gcg/test_log.py similarity index 92% rename from tests/unit/auxiliary_attacks/gcg/test_log.py rename to tests/unit/executor/promptgen/gcg/test_log.py index da50956ed6..400fced0d7 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_log.py +++ b/tests/unit/executor/promptgen/gcg/test_log.py @@ -7,7 +7,7 @@ import pytest log_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.experiments.log", + "pyrit.executor.promptgen.gcg.experiments.log", reason="GCG optional dependencies not installed", ) get_gpu_memory = log_mod.get_gpu_memory @@ -90,28 +90,28 @@ class TestGpuMemoryLogging: uses stdlib imports, so these tests run in any CI environment. """ - @patch("pyrit.auxiliary_attacks.gcg.experiments.log.sp") + @patch("pyrit.executor.promptgen.gcg.experiments.log.sp") def test_get_gpu_memory_parses_nvidia_smi(self, mock_sp: MagicMock) -> None: """Should parse nvidia-smi output into a dict of GPU -> free memory.""" mock_sp.check_output.return_value = b"memory.free [MiB]\n8000 MiB\n16000 MiB\n" result = get_gpu_memory() assert result == {"gpu1_free_memory": 8000, "gpu2_free_memory": 16000} - @patch("pyrit.auxiliary_attacks.gcg.experiments.log.sp") + @patch("pyrit.executor.promptgen.gcg.experiments.log.sp") def test_get_gpu_memory_single_gpu(self, mock_sp: MagicMock) -> None: """Should handle single GPU output.""" mock_sp.check_output.return_value = b"memory.free [MiB]\n24000 MiB\n" result = get_gpu_memory() assert result == {"gpu1_free_memory": 24000} - @patch("pyrit.auxiliary_attacks.gcg.experiments.log.sp") + @patch("pyrit.executor.promptgen.gcg.experiments.log.sp") def test_log_gpu_memory_logs_via_logging(self, mock_sp: MagicMock) -> None: """Should log GPU memory info without error on the success path.""" mock_sp.check_output.return_value = b"memory.free [MiB]\n8000 MiB\n16000 MiB\n" # Should not raise log_gpu_memory(step=5) - @patch("pyrit.auxiliary_attacks.gcg.experiments.log.sp") + @patch("pyrit.executor.promptgen.gcg.experiments.log.sp") def test_log_gpu_memory_swallows_nvidia_smi_failure(self, mock_sp: MagicMock) -> None: """Should swallow exceptions when nvidia-smi is not available. @@ -123,7 +123,7 @@ def test_log_gpu_memory_swallows_nvidia_smi_failure(self, mock_sp: MagicMock) -> # Must not raise log_gpu_memory(step=5) - @patch("pyrit.auxiliary_attacks.gcg.experiments.log.sp") + @patch("pyrit.executor.promptgen.gcg.experiments.log.sp") def test_get_gpu_memory_handles_nvidia_smi_failure(self, mock_sp: MagicMock) -> None: """`get_gpu_memory` itself should propagate the exception (only `log_gpu_memory` is expected to swallow it).""" diff --git a/tests/unit/auxiliary_attacks/gcg/test_multi_prompt_attack.py b/tests/unit/executor/promptgen/gcg/test_multi_prompt_attack.py similarity index 98% rename from tests/unit/auxiliary_attacks/gcg/test_multi_prompt_attack.py rename to tests/unit/executor/promptgen/gcg/test_multi_prompt_attack.py index 0eeef9e859..6100643399 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_multi_prompt_attack.py +++ b/tests/unit/executor/promptgen/gcg/test_multi_prompt_attack.py @@ -6,7 +6,7 @@ import pytest attack_manager_mod = pytest.importorskip( - "pyrit.auxiliary_attacks.gcg.attack.base.attack_manager", + "pyrit.executor.promptgen.gcg.attack.base.attack_manager", reason="GCG optional dependencies (torch, mlflow, etc.) not installed", ) IndividualPromptAttack = attack_manager_mod.IndividualPromptAttack diff --git a/tests/unit/auxiliary_attacks/gcg/test_public_api.py b/tests/unit/executor/promptgen/gcg/test_public_api.py similarity index 86% rename from tests/unit/auxiliary_attacks/gcg/test_public_api.py rename to tests/unit/executor/promptgen/gcg/test_public_api.py index a0b114eb73..9f637fffa4 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_public_api.py +++ b/tests/unit/executor/promptgen/gcg/test_public_api.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Tests for the top-level :mod:`pyrit.auxiliary_attacks.gcg` public API surface.""" +"""Tests for the top-level :mod:`pyrit.executor.promptgen.gcg` public API surface.""" import pytest @@ -10,8 +10,8 @@ # Skip the whole file on installs that only have the base `dev` extra. pytest.importorskip("torch", reason="GCG public API exposes torch-dependent symbols") -import pyrit.auxiliary_attacks.gcg as gcg_pkg -from pyrit.auxiliary_attacks.gcg import ( +import pyrit.executor.promptgen.gcg as gcg_pkg +from pyrit.executor.promptgen.gcg import ( GCG, GCGAlgorithmConfig, GCGConfig, @@ -49,7 +49,7 @@ def test_public_api_symbols_are_exported() -> None: def test_public_api_symbols_are_importable_from_package() -> None: # Smoke-test that the imports at module top resolved to real objects so the - # short import path (e.g. ``from pyrit.auxiliary_attacks.gcg import GCG``) + # short import path (e.g. ``from pyrit.executor.promptgen.gcg import GCG``) # stays stable. symbols = ( GCG,