Apply filterwarnings whilst importing plugins

changelog/12697.improvement.rst

@@ -0,0 +1 @@
+Warnings emitted while importing plugins through :option:`-p`, :envvar:`PYTEST_PLUGINS`, or entry point autoloading are now captured, filtered, and reported like other warnings.

src/_pytest/config/__init__.py

@@ -196,6 +196,10 @@ def main(
         arguments directly from the process command line (:data:`sys.argv`).
     :param plugins: List of plugin objects to be auto-registered during initialization.
 
+    .. warning::
+        pytest's warning filters do not apply whilst importing module
+        names passed via ``plugins``.
+
     :returns: An exit code.
     """
     return _main(args=args, plugins=plugins, prog="pytest.main()")
@@ -1225,6 +1229,25 @@ def _catch_configured_warnings(
             apply_warning_filters(config_filters, cmdline_filters)
             yield log
 
+    @contextlib.contextmanager
+    def _capture_plugin_import_warnings(self) -> Iterator[None]:
+        with self._catch_configured_warnings(record=True) as records:
+            # mypy can't infer that record=True means log is not None; help it.
+            assert records is not None
+
+            try:
+                yield
+            finally:
+                for warning_message in records:
+                    self.hook.pytest_warning_recorded.call_historic(
+                        kwargs=dict(
+                            warning_message=warning_message,
+                            nodeid="",
+                            when="config",
+                            location=None,
+                        )
+                    )
+
     def _do_configure(self) -> None:
         assert not self._configured
         self._configured = True
@@ -1607,17 +1630,31 @@ def parse(self, args: list[str], addopts: bool = True) -> None:
         self._checkversion()
         self._consider_importhook()
         self._configure_python_path()
-        self.pluginmanager.consider_preparse(args, exclude_only=False)
-        if (
-            not os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD")
-            and not self.known_args_namespace.disable_plugin_autoload
+
+        # Apply filterwarnings to whilst importing plugins.
+        warnings_plugin_enabled = self.pluginmanager.hasplugin("warnings")
+        for plugin in self.known_args_namespace.plugins:
+            plugin = plugin.strip()
+            if plugin == "no:warnings":
+                warnings_plugin_enabled = False
+            elif plugin == "warnings":
+                warnings_plugin_enabled = True
+        with (
+            self._capture_plugin_import_warnings()
+            if warnings_plugin_enabled
+            else contextlib.nullcontext()
         ):
-            # Autoloading from distribution package entry point has
-            # not been disabled.
-            self.pluginmanager.load_setuptools_entrypoints("pytest11")
-        # Otherwise only plugins explicitly specified in PYTEST_PLUGINS
-        # are going to be loaded.
-        self.pluginmanager.consider_env()
+            self.pluginmanager.consider_preparse(args, exclude_only=False)
+            if (
+                not os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD")
+                and not self.known_args_namespace.disable_plugin_autoload
+            ):
+                # Autoloading from distribution package entry point has
+                # not been disabled.
+                self.pluginmanager.load_setuptools_entrypoints("pytest11")
+            # Otherwise only plugins explicitly specified in PYTEST_PLUGINS
+            # are going to be loaded.
+            self.pluginmanager.consider_env()
 
         # Parse again, now including options added in pytest_addoption
         # by third-party plugins loaded above. This way they're available

testing/deprecated_test.py

@@ -12,9 +12,17 @@
 def test_external_plugins_integrated(pytester: Pytester, plugin) -> None:
     pytester.syspathinsert()
     pytester.makepyfile(**{plugin: ""})
+    recorded = []
 
-    with pytest.warns(pytest.PytestConfigWarning):
-        pytester.parseconfig("-p", plugin)
+    class Recorder:
+        def pytest_warning_recorded(self, warning_message):
+            recorded.append(warning_message)
+
+    pytester.plugins = [Recorder()]
+    pytester.parseconfig("-p", plugin)
+
+    assert len(recorded) == 1
+    assert recorded[0].category is pytest.PytestConfigWarning
 
 
 def test_hookspec_via_function_attributes_are_deprecated():

testing/plugins_integration/pytest.ini

@@ -6,3 +6,4 @@ filterwarnings =
     error::pytest.PytestWarning
     ignore:usefixtures.* without arguments has no effect:pytest.PytestWarning
     ignore:.*.fspath is deprecated and will be replaced by .*.path.*:pytest.PytestDeprecationWarning
+    ignore:_pytest.python.CallSpec2 has been renamed to CallSpec.:pytest.PytestRemovedIn10Warning

testing/test_warnings.py

@@ -733,6 +733,79 @@ def pytest_configure():
     result.stderr.no_fnmatch_line("*from pytest_configure*")
 
 
+class TestPluginImportWarning:
+    """filterwarnings apply to warnings emitted whilst importing plugins.
+
+    Issue #12697.
+    """
+
+    @staticmethod
+    def _make_plugin_with_import_warning(pytester: Pytester) -> None:
+        pytester.makepyfile(
+            warning_plugin="""
+                import warnings
+                warnings.warn("from plugin import", DeprecationWarning)
+            """,
+            test_it="def test_it(): pass",
+        )
+
+    def test_plugin_import_warning(self, pytester: Pytester) -> None:
+        self._make_plugin_with_import_warning(pytester)
+        pytester.plugins = ["warning_plugin"]
+
+        result = pytester.runpytest_subprocess()
+
+        result.assert_outcomes(passed=1, warnings=1)
+        result.stdout.fnmatch_lines("*DeprecationWarning: from plugin import")
+
+    def test_plugin_import_warning_without_warnings_plugin(
+        self,
+        pytester: Pytester,
+    ) -> None:
+        pytester.makeini(
+            """
+            [pytest]
+            filterwarnings =
+                error::DeprecationWarning
+            """
+        )
+        self._make_plugin_with_import_warning(pytester)
+        pytester.plugins = ["warning_plugin"]
+
+        result = pytester.runpytest_subprocess("-p", "no:warnings")
+
+        result.assert_outcomes(passed=1)
+        result.stdout.no_fnmatch_line("*from plugin import*")
+        result.stderr.no_fnmatch_line("*from plugin import*")
+
+    def test_plugin_import_warning_with_warnings_plugin_reenabled(
+        self,
+        pytester: Pytester,
+    ) -> None:
+        self._make_plugin_with_import_warning(pytester)
+        pytester.syspathinsert()
+
+        result = pytester.runpytest(
+            "-p", "warning_plugin", "-p", "no:warnings", "-p", "warnings"
+        )
+
+        result.assert_outcomes(passed=1)
+        result.stdout.fnmatch_lines("*DeprecationWarning: from plugin import")
+
+    def test_plugin_import_warning_from_pytest_plugins(
+        self,
+        pytester: Pytester,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        self._make_plugin_with_import_warning(pytester)
+        monkeypatch.setenv("PYTEST_PLUGINS", "warning_plugin")
+
+        result = pytester.runpytest_subprocess()
+
+        result.assert_outcomes(passed=1, warnings=1)
+        result.stdout.fnmatch_lines("*DeprecationWarning: from plugin import")
+
+
 class TestStackLevel:
     @pytest.fixture
     def capwarn(self, pytester: Pytester):