Update is_metapackage check within set_dev_version tooling (#39048)

* add is_metapackage property to ParsedSetup
* add tests for new is_metapackage property
* utilize new property to prevent erroneous filtering of corehttp during set_dev_version calls

tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py

@@ -52,6 +52,7 @@ def __init__(
         keywords: List[str],
         ext_package: str,
         ext_modules: List[Extension],
+        metapackage: bool
     ):
         self.name: str = name
         self.version: str = version
@@ -66,6 +67,7 @@ def __init__(
         self.keywords: List[str] = keywords
         self.ext_package = ext_package
         self.ext_modules = ext_modules
+        self.is_metapackage = metapackage
 
         self.is_pyproject = self.setup_filename.endswith(".toml")
 
@@ -90,6 +92,7 @@ def from_path(cls, parse_directory_or_file: str):
             keywords,
             ext_package,
             ext_modules,
+            metapackage,
         ) = parse_setup(parse_directory_or_file)
 
         return cls(
@@ -106,6 +109,7 @@ def from_path(cls, parse_directory_or_file: str):
             keywords,
             ext_package,
             ext_modules,
+            metapackage
         )
 
     def get_build_config(self) -> Optional[Dict[str, Any]]:
@@ -226,7 +230,7 @@ def read_setup_py_content(setup_filename: str) -> str:
 
 def parse_setup_py(
     setup_filename: str,
-) -> Tuple[str, str, str, List[str], bool, str, str, Dict[str, Any], bool, List[str], List[str], str, List[Extension]]:
+) -> Tuple[str, str, str, List[str], bool, str, str, Dict[str, Any], bool, List[str], List[str], str, List[Extension], bool]:
     """
     Used to evaluate a setup.py (or a directory containing a setup.py) and return a tuple containing:
     (
@@ -242,7 +246,8 @@ def parse_setup_py(
         <classifiers>,
         <keywords>,
         <ext_packages>,
-        <ext_modules>
+        <ext_modules>,
+        <is_metapackage>
     )
     """
 
@@ -296,6 +301,11 @@ def setup(*args, **kwargs):
 
     if packages:
         name_space = packages[0]
+        metapackage = False
+    else:
+        metapackage = True
+
+
 
     requires = kwargs.get("install_requires", [])
     package_data = kwargs.get("package_data", None)
@@ -322,14 +332,15 @@ def setup(*args, **kwargs):
         classifiers,            # List[str],
         keywords,               # List[str] ADJUSTED
         ext_package,            # str
-        ext_modules,            # List[Extension]
+        ext_modules,            # List[Extension],
+        metapackage             # bool
     )
     # fmt: on
 
 
 def parse_pyproject(
     pyproject_filename: str,
-) -> Tuple[str, str, str, List[str], bool, str, str, Dict[str, Any], bool, List[str], List[str], str, List[Extension]]:
+) -> Tuple[str, str, str, List[str], bool, str, str, Dict[str, Any], bool, List[str], List[str], str, List[Extension], bool]:
     """
     Used to evaluate a pyproject (or a directory containing a pyproject.toml) with a [project] configuration within.
     Returns a tuple containing:
@@ -346,7 +357,8 @@ def parse_pyproject(
         <classifiers>,
         <keywords>,
         <ext_packages>,
-        <ext_modules>
+        <ext_modules>,
+        <is_metapackage>
     )
     """
     toml_dict = get_pyproject_dict(pyproject_filename)
@@ -383,6 +395,7 @@ def parse_pyproject(
     include_package_data = get_value_from_dict(toml_dict, "tool.setuptools.include-package-data", True)
     classifiers = project_config.get("classifiers", [])
     keywords = project_config.get("keywords", [])
+    metapackage = False
 
     # as of setuptools 74.1 ext_packages and ext_modules are now present in tool.setuptools config namespace
     ext_package = get_value_from_dict(toml_dict, "tool.setuptools.ext-package", None)
@@ -404,6 +417,7 @@ def parse_pyproject(
         keywords,               # List[str] ADJUSTED
         ext_package,            # str
         ext_modules,            # List[Extension]
+        metapackage             # bool
     )
     # fmt: on
 
@@ -468,7 +482,8 @@ def parse_setup(
         <classifiers>,
         <keywords>,
         <ext_packages>,
-        <ext_modules>
+        <ext_modules>,
+        <is_metapackage>
     )
 
     If a pyproject.toml (containing [project]) or a setup.py is NOT found, a ValueError will be raised.

tools/azure-sdk-tools/ci_tools/versioning/version_shared.py

@@ -32,23 +32,14 @@
 
 from typing import List
 
-
 def path_excluded(path, additional_excludes):
     return (
         any([excl in path for excl in additional_excludes])
         or "tests" in os.path.normpath(path).split(os.sep)
-        or is_metapackage(path)
+        or ParsedSetup.from_path(path).is_metapackage
     )
 
 
-# Metapackages do not have an 'azure' folder within them
-def is_metapackage(package_path):
-    dir_path = package_path if path.isdir(package_path) else path.split(package_path)[0]
-
-    azure_path = path.join(dir_path, "azure")
-    return not path.exists(azure_path)
-
-
 def get_setup_py_paths(glob_string, base_path, additional_excludes):
     setup_paths = discover_targeted_packages(glob_string, base_path)
     filtered_paths = [p for p in setup_paths if not path_excluded(p, additional_excludes)]

tools/azure-sdk-tools/tests/integration/scenarios/setup_py_metapackage/README.md

@@ -0,0 +1 @@
+This folder contains a basic `setup.py` scenario that follows the azure-sdk conventions for a "metapackage." We use this to ensure that our metapackage detection within ParsedSetup is working properly.

tools/azure-sdk-tools/tests/integration/scenarios/setup_py_metapackage/setup.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+#-------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+#--------------------------------------------------------------------------
+
+from setuptools import setup
+from io import open
+setup(
+    name='azure-keyvault',
+    version='4.2.1b1',
+    description='Microsoft Azure Key Vault Client Libraries for Python',
+    long_description_content_type="text/markdown",
+    license='MIT License',
+    author='Microsoft Corporation',
+    author_email="[email protected]",
+    url="https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/keyvault/azure-keyvault",
+    keywords="azure, azure sdk",
+    classifiers=[
+        "Development Status :: 5 - Production/Stable",
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3 :: Only',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
+        'License :: OSI Approved :: MIT License',
+    ],
+    zip_safe=False,
+    install_requires=[
+        'azure-keyvault-certificates~=4.4',
+        'azure-keyvault-secrets~=4.4',
+        'azure-keyvault-keys~=4.5',
+    ],
+)

tools/azure-sdk-tools/tests/test_parse_functionality.py

@@ -10,9 +10,11 @@
     os.path.dirname(__file__),
 )
 scenarios_folder = os.path.join(os.path.dirname(__file__), "integration", "scenarios")
+metapackage_scenario = os.path.join(scenarios_folder, "setup_py_metapackage")
 pyproject_scenario = os.path.join(scenarios_folder, "pyproject_project_def")
 pyproject_extension_scenario = os.path.join(scenarios_folder, "pyproject_project_def_with_extension")
 
+
 def test_parse_require():
     test_scenarios = [
         ("ConfigArgParse>=0.12.0", "configargparse", ">=0.12.0"),
@@ -123,6 +125,7 @@ def test_sdk_sample_setup(test_patch):
     assert result.classifiers[0] == "Development Status :: 5 - Production/Stable"
     assert result.classifiers[5] == "Programming Language :: Python :: 3.8"
     assert result.keywords[0] == "azure sdk"
+    assert result.is_metapackage == False
     assert len(result.keywords) == 2
 
 
@@ -205,23 +208,37 @@ def test_parse_recognizes_extensions(test_patch):
     assert result.ext_package == "azure.storage.extensions"
     assert result.ext_modules is not None
     assert result.is_pyproject == False
+    assert result.is_metapackage == False
     assert len(result.ext_modules) == 1
     assert str(type(result.ext_modules[0])) == "<class 'setuptools.extension.Extension'>"
 
 
+def test_metapackage_detection():
+    parsed_project = ParsedSetup.from_path(metapackage_scenario)
+    assert parsed_project.is_metapackage == True
+    assert parsed_project.name == "azure-keyvault"
+
+
 def test_parse_pyproject():
     # ensure that we can parse from a folder and a specific file
     parsed_project = ParsedSetup.from_path(pyproject_scenario)
 
     assert parsed_project.name == "azure-keyvault-keys"
     assert parsed_project.version == "0.0.1"
-    assert parsed_project.requires == ["azure-common~=1.1", "azure-core<2.0.0,>=1.24.0", "cryptography>=2.1.4", "isodate>=0.6.1", "typing-extensions>=4.0.1"]
+    assert parsed_project.requires == [
+        "azure-common~=1.1",
+        "azure-core<2.0.0,>=1.24.0",
+        "cryptography>=2.1.4",
+        "isodate>=0.6.1",
+        "typing-extensions>=4.0.1",
+    ]
     assert parsed_project.python_requires == ">=3.7"
     assert parsed_project.is_new_sdk == True
     assert parsed_project.is_pyproject == True
-    assert parsed_project.package_data == { "py.typed": ["py.typed"] }
+    assert parsed_project.package_data == {"py.typed": ["py.typed"]}
     assert parsed_project.include_package_data == True
     assert parsed_project.folder == pyproject_scenario
+    assert parsed_project.is_metapackage == False
     assert parsed_project.namespace == "azure.keyvault.keys"
 
 
@@ -231,15 +248,22 @@ def test_parse_pyproject_extensions():
 
     assert parsed_project.name == "azure-keyvault-keys"
     assert parsed_project.version == "0.0.1b1"
-    assert parsed_project.requires == ["azure-common~=1.1", "azure-core<2.0.0,>=1.24.0", "cryptography>=2.1.4", "isodate>=0.6.1", "typing-extensions>=4.0.1"]
+    assert parsed_project.requires == [
+        "azure-common~=1.1",
+        "azure-core<2.0.0,>=1.24.0",
+        "cryptography>=2.1.4",
+        "isodate>=0.6.1",
+        "typing-extensions>=4.0.1",
+    ]
     assert parsed_project.python_requires == ">=3.8"
     assert parsed_project.is_new_sdk == True
     assert parsed_project.is_pyproject == True
-    assert parsed_project.package_data == { "py.typed": ["py.typed"] }
+    assert parsed_project.package_data == {"py.typed": ["py.typed"]}
     assert parsed_project.include_package_data == True
     assert parsed_project.folder == pyproject_extension_scenario
     assert parsed_project.namespace == "azure.keyvault.keys"
     assert parsed_project.ext_package == "azure.keyvault.keys"
     assert parsed_project.ext_modules is not None
+    assert parsed_project.is_metapackage == False
     assert len(parsed_project.ext_modules) == 1
     assert str(type(parsed_project.ext_modules[0])) == "<class 'setuptools.extension.Extension'>"

tools/azure-sdk-tools/tests/test_requirements_parse.py

@@ -62,7 +62,9 @@ def test_replace_dev_reqs_relative(tmp_directory_create):
     expected_output_folder = os.path.join(repo_root, "sdk", "core", "azure-core", ".tmp_whl_dir")
 
     # Get the current relative versions of the packages to properly construct the expected results
-    coretestserver_version = ParsedSetup.from_path(os.path.join(repo_root, "sdk", "core", "azure-core", "tests", "testserver_tests", "coretestserver")).version
+    coretestserver_version = ParsedSetup.from_path(
+        os.path.join(repo_root, "sdk", "core", "azure-core", "tests", "testserver_tests", "coretestserver")
+    ).version
     identity_version = ParsedSetup.from_path(os.path.join(repo_root, "sdk", "identity", "azure-identity")).version
     mgmt_core_version = ParsedSetup.from_path(os.path.join(repo_root, "sdk", "core", "azure-mgmt-core")).version
     sdk_tools_version = ParsedSetup.from_path(os.path.join(repo_root, "tools", "azure-sdk-tools")).version
@@ -80,7 +82,6 @@ def test_replace_dev_reqs_relative(tmp_directory_create):
         os.path.join(expected_output_folder, f"azure_core-{core_version}-py3-none-any.whl"),
         os.path.join(expected_output_folder, f"azure_core-{core_version}-py3-none-any.whl"),
     ]
-
     requirements_before = get_requirements_from_file(requirements_file)
     replace_dev_reqs(requirements_file, core_location, None)
     requirements_after = get_requirements_from_file(requirements_file)