Add plumbing for legacy repository search and refactor repository fixtures

pyproject.toml

@@ -94,7 +94,10 @@ build-backend = "poetry.core.masonry.api"
 extend-exclude = [
     "docs/*",
     # External to the project's coding standards
-    "tests/**/fixtures/*",
+    "tests/fixtures/git/*",
+    "tests/fixtures/project_with_setup*/*",
+    "tests/masonry/builders/fixtures/pep_561_stub_only*/*",
+    "tests/utils/fixtures/setups/*",
 ]
 fix = true
 line-length = 88

src/poetry/repositories/legacy_repository.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from contextlib import suppress
+from functools import cached_property
 from typing import TYPE_CHECKING
 from typing import Any
 
@@ -11,6 +13,7 @@
 from poetry.repositories.exceptions import PackageNotFound
 from poetry.repositories.http_repository import HTTPRepository
 from poetry.repositories.link_sources.html import SimpleRepositoryPage
+from poetry.repositories.link_sources.html import SimpleRepositoryRootPage
 
 
 if TYPE_CHECKING:
@@ -36,18 +39,6 @@ def __init__(
 
         super().__init__(name, url.rstrip("/"), config, disable_cache, pool_size)
 
-    @property
-    def packages(self) -> list[Package]:
-        # LegacyRepository._packages is not populated and other implementations
-        # implicitly rely on this (e.g. Pool.search via
-        # LegacyRepository.search). To avoid special-casing Pool or changing
-        # behavior, we stub and return an empty list.
-        #
-        # TODO: Rethinking search behaviour and design.
-        # Ref: https://github.com/python-poetry/poetry/issues/2446 and
-        # https://github.com/python-poetry/poetry/pull/6669#discussion_r990874908.
-        return []
-
     def package(
         self, name: str, version: Version, extras: list[str] | None = None
     ) -> Package:
@@ -135,7 +126,29 @@ def _get_release_info(
         )
 
     def _get_page(self, name: NormalizedName) -> SimpleRepositoryPage:
-        response = self._get_response(f"/{name}/")
-        if not response:
+        if not (response := self._get_response(f"/{name}/")):
             raise PackageNotFound(f"Package [{name}] not found.")
         return SimpleRepositoryPage(response.url, response.text)
+
+    @cached_property
+    def root_page(self) -> SimpleRepositoryRootPage:
+        if not (response := self._get_response("/")):
+            self._log(
+                f"Unable to retrieve package listing from package source {self.name}",
+                level="error",
+            )
+            return SimpleRepositoryRootPage()
+
+        return SimpleRepositoryRootPage(response.text)
+
+    def search(self, query: str) -> list[Package]:
+        results: list[Package] = []
+
+        for candidate in self.root_page.search(query):
+            with suppress(PackageNotFound):
+                page = self.get_page(candidate)
+
+                for package in page.packages:
+                    results.append(package)
+
+        return results

src/poetry/repositories/link_sources/html.py

@@ -68,6 +68,40 @@ def _link_cache(self) -> LinkCache:
         return links
 
 
+class SimpleRepositoryRootPage:
+    """
+    This class represents the parsed content of a "simple" repository's root page. This follows the
+    specification laid out in PEP 503.
+
+    See: https://peps.python.org/pep-0503/
+    """
+
+    def __init__(self, content: str | None = None) -> None:
+        parser = HTMLPageParser()
+        parser.feed(content or "")
+        self._parsed = parser.anchors
+
+    def search(self, query: str) -> list[str]:
+        results: list[str] = []
+
+        for anchor in self._parsed:
+            href = anchor.get("href")
+            if href and query in href:
+                results.append(href.rstrip("/"))
+
+        return results
+
+    @cached_property
+    def package_names(self) -> list[str]:
+        results: list[str] = []
+
+        for anchor in self._parsed:
+            if href := anchor.get("href"):
+                results.append(href.rstrip("/"))
+
+        return results
+
+
 class SimpleRepositoryPage(HTMLPage):
     def __init__(self, url: str, content: str) -> None:
         if not url.endswith("/"):

tests/conftest.py

@@ -11,6 +11,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import Any
+from typing import Iterator
 
 import httpretty
 import keyring
@@ -62,6 +63,11 @@
     from tests.types import SetProjectContext
 
 
+pytest_plugins = [
+    "tests.repositories.fixtures",
+]
+
+
 def pytest_addoption(parser: Parser) -> None:
     parser.addoption(
         "--integration",
@@ -279,7 +285,6 @@ def download_mock(mocker: MockerFixture) -> None:
     # Patch download to not download anything but to just copy from fixtures
     mocker.patch("poetry.utils.helpers.download_file", new=mock_download)
     mocker.patch("poetry.packages.direct_origin.download_file", new=mock_download)
-    mocker.patch("poetry.repositories.http_repository.download_file", new=mock_download)
 
 
 @pytest.fixture(autouse=True)
@@ -323,7 +328,7 @@ def git_mock(mocker: MockerFixture) -> None:
 @pytest.fixture
 def http() -> Iterator[type[httpretty.httpretty]]:
     httpretty.reset()
-    with httpretty.enabled(allow_net_connect=False):
+    with httpretty.enabled(allow_net_connect=False, verbose=True):
         yield httpretty
 
 
@@ -532,6 +537,12 @@ def httpretty_windows_mock_urllib3_wait_for_socket(mocker: MockerFixture) -> Non
     mocker.patch("urllib3.util.wait.select_wait_for_socket", returns=True)
 
 
+@pytest.fixture
+def disable_http_status_force_list(mocker: MockerFixture) -> Iterator[None]:
+    mocker.patch("poetry.utils.authenticator.STATUS_FORCELIST", [])
+    yield
+
+
 @pytest.fixture(autouse=True)
 def tmp_working_directory(tmp_path: Path) -> Iterator[Path]:
     with switch_working_directory(tmp_path):

tests/console/commands/test_search.py

@@ -19,12 +19,6 @@
 )
 
 
-@pytest.fixture(autouse=True)
-def mock_search_http_response(http: type[httpretty.httpretty]) -> None:
-    with FIXTURES_DIRECTORY.joinpath("search.html").open(encoding="utf-8") as f:
-        http.register_uri("GET", "https://pypi.org/search", f.read())
-
-
 @pytest.fixture
 def tester(command_tester_factory: CommandTesterFactory) -> CommandTester:
     return command_tester_factory("search")

tests/helpers.py

@@ -43,6 +43,11 @@
     from tests.types import HTTPrettyResponse
 
 FIXTURE_PATH = Path(__file__).parent / "fixtures"
+FIXTURE_PATH_INSTALLATION = Path(__file__).parent / "installation" / "fixtures"
+FIXTURE_PATH_DISTRIBUTIONS = FIXTURE_PATH / "distributions"
+FIXTURE_PATH_REPOSITORIES = Path(__file__).parent / "repositories" / "fixtures"
+FIXTURE_PATH_REPOSITORIES_LEGACY = FIXTURE_PATH_REPOSITORIES / "legacy"
+FIXTURE_PATH_REPOSITORIES_PYPI = FIXTURE_PATH_REPOSITORIES / "pypi.org"
 
 # Used as a mock for latest git revision.
 MOCK_DEFAULT_GIT_REVISION = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24"

tests/inspection/test_lazy_wheel.py

@@ -32,6 +32,7 @@
     from tests.types import HTTPPrettyRequestCallbackWrapper
     from tests.types import HTTPrettyRequestCallback
     from tests.types import HTTPrettyResponse
+    from tests.types import PackageDistributionLookup
 
     class RequestCallbackFactory(Protocol):
         def __call__(
@@ -110,7 +111,10 @@ def build_partial_response(
 
 
 @pytest.fixture
-def handle_request_factory(fixture_dir: FixtureDirGetter) -> RequestCallbackFactory:
+def handle_request_factory(
+    fixture_dir: FixtureDirGetter,
+    package_distribution_lookup: PackageDistributionLookup,
+) -> RequestCallbackFactory:
     def _factory(
         *,
         accept_ranges: str | None = "bytes",
@@ -122,17 +126,12 @@ def handle_request(
         ) -> HTTPrettyResponse:
             name = Path(urlparse(uri).path).name
 
-            wheel = Path(__file__).parents[1] / (
-                "repositories/fixtures/pypi.org/dists/" + name
+            wheel = package_distribution_lookup(name) or package_distribution_lookup(
+                "demo-0.1.0-py2.py3-none-any.whl"
             )
 
-            if not wheel.exists():
-                wheel = fixture_dir("distributions") / name
-
-                if not wheel.exists():
-                    wheel = (
-                        fixture_dir("distributions") / "demo-0.1.0-py2.py3-none-any.whl"
-                    )
+            if not wheel:
+                return 404, response_headers, b"Not Found"
 
             wheel_bytes = wheel.read_bytes()
 

tests/installation/conftest.py

@@ -1,45 +0,0 @@
-from __future__ import annotations
-
-import re
-
-from pathlib import Path
-from typing import TYPE_CHECKING
-from typing import Any
-from urllib.parse import urlparse
-
-import pytest
-
-
-if TYPE_CHECKING:
-    from httpretty import httpretty
-    from httpretty.core import HTTPrettyRequest
-
-    from tests.types import FixtureDirGetter
-
-
-@pytest.fixture
-def mock_file_downloads(http: type[httpretty], fixture_dir: FixtureDirGetter) -> None:
-    def callback(
-        request: HTTPrettyRequest, uri: str, headers: dict[str, Any]
-    ) -> list[int | dict[str, Any] | bytes]:
-        name = Path(urlparse(uri).path).name
-
-        fixture = Path(__file__).parent.parent.joinpath(
-            "repositories/fixtures/pypi.org/dists/" + name
-        )
-
-        if not fixture.exists():
-            fixture = fixture_dir("distributions") / name
-
-            if not fixture.exists():
-                fixture = (
-                    fixture_dir("distributions") / "demo-0.1.0-py2.py3-none-any.whl"
-                )
-
-        return [200, headers, fixture.read_bytes()]
-
-    http.register_uri(
-        http.GET,
-        re.compile("^https://files.pythonhosted.org/.*$"),
-        body=callback,
-    )

tests/installation/fixtures/old-lock.test

@@ -79,34 +79,34 @@ content-hash = "123456789"
 
 [metadata.files]
 attrs = [
-    {file = "attrs-17.4.0-py2.py3-none-any.whl", hash = "sha256:a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450"},
-    {file = "attrs-17.4.0.tar.gz", hash = "sha256:1c7960ccfd6a005cd9f7ba884e6316b5e430a3f1a6c37c5f87d8b43f83b54ec9"},
+    {file = "attrs-17.4.0-py2.py3-none-any.whl", hash = "sha256:d38e57f381e891928357c68e300d28d3d4dcddc50486d5f8dfaf743d40477619"},
+    {file = "attrs-17.4.0.tar.gz", hash = "sha256:eb7536a1e6928190b3008c5b350bdf9850d619fff212341cd096f87a27a5e564"},
 ]
 colorama = [
-    {file = "colorama-0.3.9-py2.py3-none-any.whl", hash = "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda"},
-    {file = "colorama-0.3.9.tar.gz", hash = "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1"},
+    {file = "colorama-0.3.9-py2.py3-none-any.whl", hash = "sha256:5b632359f1ed2b7676a869812ba0edaacb99be04679b29eb56c07a5e137ab5a2"},
+    {file = "colorama-0.3.9.tar.gz", hash = "sha256:4c5a15209723ce1330a5c193465fe221098f761e9640d823a2ce7c03f983137f"},
 ]
 funcsigs = [
-    {file = "funcsigs-1.0.2-py2.py3-none-any.whl", hash = "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca"},
-    {file = "funcsigs-1.0.2.tar.gz", hash = "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50"},
+    {file = "funcsigs-1.0.2-py2.py3-none-any.whl", hash = "sha256:510ab97424949e726b4b44294018e90142c9aadf8e737cf3a125b4cffed42e79"},
+    {file = "funcsigs-1.0.2.tar.gz", hash = "sha256:c55716fcd1228645c214b44568d1fb9af2e28668a9c58e72a17a89a75d24ecd6"},
 ]
 more-itertools = [
-    {file = "more-itertools-4.1.0.tar.gz", hash = "sha256:c9ce7eccdcb901a2c75d326ea134e0886abfbea5f93e91cc95de9507c0816c44"},
-    {file = "more_itertools-4.1.0-py2-none-any.whl", hash = "sha256:11a625025954c20145b37ff6309cd54e39ca94f72f6bb9576d1195db6fa2442e"},
-    {file = "more_itertools-4.1.0-py3-none-any.whl", hash = "sha256:0dd8f72eeab0d2c3bd489025bb2f6a1b8342f9b198f6fc37b52d15cfa4531fea"},
+    {file = "more-itertools-4.1.0.tar.gz", hash = "sha256:bab2dc6f4be8f9a4a72177842c5283e2dff57c167439a03e3d8d901e854f0f2e"},
+    {file = "more_itertools-4.1.0-py2-none-any.whl", hash = "sha256:5dd7dfd88d2fdaea446da478ffef8d7151fdf26ee92ac7ed7b14e8d71efe4b62"},
+    {file = "more_itertools-4.1.0-py3-none-any.whl", hash = "sha256:29b1e1661aaa56875ce090fa219fa84dfc13daecb52cd4fae321f6f57b419ec4"},
 ]
 pluggy = [
-    {file = "pluggy-0.6.0.tar.gz", hash = "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff"},
+    {file = "pluggy-0.6.0.tar.gz", hash = "sha256:a982e208d054867661d27c6d2a86b17ba05fbb6b1bdc01f42660732dd107f865"},
 ]
 py = [
-    {file = "py-1.5.3-py2.py3-none-any.whl", hash = "sha256:983f77f3331356039fdd792e9220b7b8ee1aa6bd2b25f567a963ff1de5a64f6a"},
-    {file = "py-1.5.3.tar.gz", hash = "sha256:29c9fab495d7528e80ba1e343b958684f4ace687327e6f789a94bf3d1915f881"},
+    {file = "py-1.5.3-py2.py3-none-any.whl", hash = "sha256:43ee6c7f95e0ec6a906de49906b79d138d89728fff17109d49f086abc2fdd985"},
+    {file = "py-1.5.3.tar.gz", hash = "sha256:2df2c513c3af11de15f58189ba5539ddc4768c6f33816dc5c03950c8bd6180fa"},
 ]
 pytest = [
-    {file = "pytest-3.5.0-py2.py3-none-any.whl", hash = "sha256:6266f87ab64692112e5477eba395cfedda53b1933ccd29478e671e73b420c19c"},
-    {file = "pytest-3.5.0.tar.gz", hash = "sha256:fae491d1874f199537fd5872b5e1f0e74a009b979df9d53d1553fd03da1703e1"},
+    {file = "pytest-3.5.0-py2.py3-none-any.whl", hash = "sha256:28e4d9c2ae3196d74805c2eba24f350ae4c791a5b9b397c79b41506a48dc64ca"},
+    {file = "pytest-3.5.0.tar.gz", hash = "sha256:677b1d6decd29c041fe64276f29f79fbe66e40c59e445eb251366b4a8ab8bf68"},
 ]
 six = [
-    {file = "six-1.11.0-py2.py3-none-any.whl", hash = "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"},
-    {file = "six-1.11.0.tar.gz", hash = "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9"},
+    {file = "six-1.11.0-py2.py3-none-any.whl", hash = "sha256:112f5b46e6aa106db3e4e2494a03694c938f41c4c4535edbdfc816c2e0cb50f2"},
+    {file = "six-1.11.0.tar.gz", hash = "sha256:268a4ccb159c1a2d2c79336b02e75058387b0cdbb4cea2f07846a758f48a356d"},
 ]