Add the Provide: pypi in most cases

Usually the pypi ecosystem will do the right thing and respond with a
name but in cases where it does not (bad network, bad json, message
without a name), still figure out a reasonable Provide to add.

Signed-off-by: William Douglas <william.douglas@intel.com>

Author: William Douglas

autospec/buildreq.py

@@ -709,23 +709,28 @@ def get_data_from_pypi(self, name, config):
         """Use pypi for getting package requires and metadata."""
         # First look for a local override
         pypi_json = ""
+        pypi_name = pypidata.get_pypi_name(name)
         pypi_file = os.path.join(config.download_path, "pypi.json")
         if os.path.isfile(pypi_file):
             with open(pypi_file, "r") as pfile:
                 pypi_json = pfile.read()
         else:
             # Try and grab the pypi details for the package
             if config.alias:
-                name = config.alias
-            pypi_name = pypidata.get_pypi_name(name)
+                pypi_name = config.alias
+            pypi_name = pypidata.get_pypi_name(pypi_name)
             pypi_json = pypidata.get_pypi_metadata(pypi_name)
-        if pypi_json:
+        if not pypi_json:
+            self.pypi_provides = pypi_name
+        else:
             try:
                 package_pypi = json.loads(pypi_json)
             except json.JSONDecodeError:
-                package_pypi = {}
+                package_pypi = {"name": pypi_name}
             if package_pypi.get("name"):
                 self.pypi_provides = package_pypi["name"]
+            else:
+                self.pypi_provides = pypi_name
             if package_pypi.get("requires"):
                 for pkg in package_pypi["requires"]:
                     self.add_requires(f"pypi({pkg})", config.os_packages, override=True, subpkg="python3")

autospec/pypidata.py

@@ -27,6 +27,15 @@ def pkg_search(name):
         return False
 
 
+def fixup_pypi_prefix(name):
+    """Try and chop off the 'pypi-' or 'python-' prefix for names."""
+    name = name.lower().replace('-', '_')
+    for prefix in ["pypi_", "python_"]:
+        if name.startswith(prefix):
+            name = name[len(prefix):]
+    return name
+
+
 def get_pypi_name(name, miss=False):
     """Try and verify the pypi name for a given package name."""
     # normalize the name for matching as pypi is case insensitve for search
@@ -35,11 +44,9 @@ def get_pypi_name(name, miss=False):
     if pkg_search(name):
         return name
     # Maybe we have a prefix
-    for prefix in ["pypi_", "python_"]:
-        if name.startswith(prefix):
-            name = name[len(prefix):]
-            if pkg_search(name):
-                return name
+    name = fixup_pypi_prefix(name)
+    if pkg_search(name):
+        return name
     # Some cases where search fails (Sphinx)
     # Just try the name we were given
     if miss:

tests/test_buildreq.py

@@ -499,6 +499,95 @@ def test_scan_for_configure_pypi(self):
         self.assertEqual(self.reqs.requires['python3'], pypi_requires)
         self.assertEqual(ssummary, summary)
 
+    def test_scan_for_configure_pypi_no_name_in_json(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test connecting to pypi but not getting a name back.
+        """
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        orig_pypi_name = buildreq.pypidata.get_pypi_name
+        orig_pypi_meta = buildreq.pypidata.get_pypi_metadata
+        name = "pypi-name"
+        content = json.dumps({})
+        buildreq.pypidata.pkg_search = MagicMock(return_value=False)
+        buildreq.pypidata.get_pypi_metadata = MagicMock(return_value=content)
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), name, conf)
+
+        post_summary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+        buildreq.pypidata.get_pypi_name = orig_pypi_name
+        buildreq.pypidata.get_pypi_metadata = orig_pypi_meta
+
+        self.assertEqual(self.reqs.pypi_provides, "name")
+        self.assertEqual(post_summary, orig_summary)
+
+    def test_scan_for_configure_pypi_no_json(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test being unable to connect to pypi.
+        """
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        orig_pypi_name = buildreq.pypidata.get_pypi_name
+        orig_pypi_meta = buildreq.pypidata.get_pypi_metadata
+        name = "pypi-name"
+        buildreq.pypidata.pkg_search = MagicMock(return_value=False)
+        buildreq.pypidata.get_pypi_metadata = MagicMock(return_value="")
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), name, conf)
+
+        post_summary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+        buildreq.pypidata.get_pypi_name = orig_pypi_name
+        buildreq.pypidata.get_pypi_metadata = orig_pypi_meta
+
+        self.assertEqual(self.reqs.pypi_provides, "name")
+        self.assertEqual(post_summary, orig_summary)
+
+    def test_scan_for_configure_pypi_bad_json(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test being given bad json data.
+        """
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        orig_pypi_name = buildreq.pypidata.get_pypi_name
+        orig_pypi_meta = buildreq.pypidata.get_pypi_metadata
+        orig_json_loads = buildreq.json.loads
+        name = "pypi-name"
+        content = json.dumps({})
+        buildreq.pypidata.pkg_search = MagicMock(return_value=False)
+        buildreq.pypidata.get_pypi_metadata = MagicMock(return_value=content)
+        buildreq.json.loads = MagicMock(side_effect=json.JSONDecodeError("", "", 0))
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), name, conf)
+
+        post_summary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+        buildreq.pypidata.get_pypi_name = orig_pypi_name
+        buildreq.pypidata.get_pypi_metadata = orig_pypi_meta
+        buildreq.json.loads = orig_json_loads
+
+        self.assertEqual(self.reqs.pypi_provides, "name")
+        self.assertEqual(post_summary, orig_summary)
+
     def test_scan_for_configure_pypi_override(self):
         """
         Test scan_for_configure when distutils is being used for the build