From 47808507cee3d2721382d25f48577af61b1b9cdc Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 12 May 2026 10:07:50 +0200 Subject: [PATCH] feat(resolver): add `LocalIndexProvider` `LocalIndexProvider` resolves packages from Fromager's local cache directories by scanning for wheel and sdist files. It supports flat and nested (PyPI-like) directory layouts. Caching is disabled since `os.scandir` and `parse_sdist/wheel_filename` are fast (1-3 microseconds per file). In the future, `fromager.finders`' `find_sdist()` and `find_wheel()` will use `LocalIndexProvider` internally. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Christian Heimes --- src/fromager/finders.py | 137 ++++++++++++++++++++++++++++++++- tests/test_finders.py | 162 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 295 insertions(+), 4 deletions(-) diff --git a/src/fromager/finders.py b/src/fromager/finders.py index 69c31e33..f11e1e88 100644 --- a/src/fromager/finders.py +++ b/src/fromager/finders.py @@ -1,14 +1,22 @@ from __future__ import annotations import logging +import os import pathlib import re import typing from packaging.requirements import Requirement -from packaging.utils import BuildTag, canonicalize_name - -from . import overrides, resolver +from packaging.tags import Tag +from packaging.utils import ( + BuildTag, + canonicalize_name, + parse_sdist_filename, + parse_wheel_filename, +) + +from . import overrides, requirements_file, resolver +from .candidate import Candidate from .constraints import Constraints from .requirements_file import RequirementType @@ -66,6 +74,129 @@ def __init__( ) +class LocalIndexProvider(resolver.BaseProvider): + """Lookup distributions in a local directory + + The local index provider is designed for Fromager's internal cache + directories. It looks up sdist and wheels files in a local directory or + directory tree. A flat provider has all files in one directory. If flat + is false, then the provider expects nested directories like on PyPI: + + - flat: ``{path}/meson_python-0.19.0-2-py3-none-any.whl`` + - nested: ``{path}/meson-python/meson_python-0.19.0-2-py3-none-any.whl`` + + Caching is disabled, so the provider picks up local changes immedately. + Local file lookups and wheel/sdist filename parsing are fast. + """ + + provider_description: typing.ClassVar[str] = ( + "Local provider (path: {self.path}, flat: {self.flat})" + ) + + def __init__( + self, + *, + path: pathlib.Path, + flat: bool, + include_sdists: bool = True, + include_wheels: bool = True, + constraints: Constraints | None = None, + req_type: requirements_file.RequirementType | None = None, + ): + super().__init__( + constraints=constraints, + req_type=req_type, + use_resolver_cache=False, + cooldown=None, + ) + self.path = path + self.flat = flat + self.include_sdists = include_sdists + self.include_wheels = include_wheels + self.supports_upload_time = False + + @property + def cache_key(self) -> str: + """No caching, local lookup is fast""" + raise NotImplementedError() + + def find_candidates(self, identifier: str) -> resolver.Candidates: + """Find matching distribution files for the given package identifier.""" + normalized_identifier = canonicalize_name(identifier) + # directory uses normalized name with dash + path = self.path if self.flat else self.path / normalized_identifier + + try: + entries = os.scandir(path) + except OSError as err: + logger.debug("cannot read from directory %r: %s", path, err) + return [] + + candidates: list[Candidate] = [] + for entry in entries: + if not entry.is_file(): + # not a file, ignore + continue + + if entry.name.endswith((".tar.gz", ".zip")): + if not self.include_sdists: + if resolver.DEBUG_RESOLVER: + logger.debug("skipping %r because it is an sdist", entry.name) + continue + is_sdist = True + elif entry.name.endswith(".whl"): + if not self.include_wheels: + if resolver.DEBUG_RESOLVER: + logger.debug("skipping %r because it is a wheel", entry.name) + continue + is_sdist = False + else: + # ignore other files + continue + + try: + if is_sdist: + name, version = parse_sdist_filename(entry.name) + tags: frozenset[Tag] = frozenset() + build_tag: BuildTag = () + else: + name, version, build_tag, tags = parse_wheel_filename(entry.name) + except Exception as err: + # Ignore files with invalid versions + if resolver.DEBUG_RESOLVER: + logger.debug("invalid file name %r: %s", entry.name, err) + continue + + if name != normalized_identifier: + if resolver.DEBUG_RESOLVER: + logger.debug( + "dist name %r of %r does not match identifier %r", + name, + entry.name, + identifier, + ) + continue + + if not is_sdist and not tags.intersection(resolver.SUPPORTED_TAGS): + # ignore wheels unless they have supported tags + if resolver.DEBUG_RESOLVER: + logger.debug("ignoring %r with tags %r", entry.name, tags) + continue + + c = Candidate( + name=name, + version=version, + url=entry.path, + is_sdist=is_sdist, + build_tag=build_tag, + has_metadata=False, + upload_time=None, + ) + candidates.append(c) + + return candidates + + def _dist_name_to_filename(dist_name: str) -> str: """Transform the dist name into a prefix for a filename. diff --git a/tests/test_finders.py b/tests/test_finders.py index 110ccfa1..c09f2451 100644 --- a/tests/test_finders.py +++ b/tests/test_finders.py @@ -1,9 +1,11 @@ import pathlib import pytest +import resolvelib from packaging.requirements import Requirement +from packaging.version import Version -from fromager import context, finders +from fromager import constraints, context, finders from fromager.requirements_file import RequirementType @@ -146,3 +148,161 @@ def test_pypi_cache_provider() -> None: finders.PyPICacheProvider( cache_server_url=url, include_sdists=False, include_wheels=False ) + + +# -- LocalIndexProvider ------------------------------------------------------ + + +def _create_local_file(directory: pathlib.Path, filename: str) -> pathlib.Path: + """Create an empty file in the given directory.""" + path = directory / filename + path.touch() + return path + + +def test_local_init(tmp_path: pathlib.Path) -> None: + provider = finders.LocalIndexProvider(path=tmp_path, flat=True) + assert provider.path == tmp_path + assert provider.flat is True + assert provider.include_sdists is True + assert provider.include_wheels is True + assert provider.supports_upload_time is False + assert provider.use_cache_candidates is False + assert provider.cooldown is None + + with pytest.raises(NotImplementedError): + _ = provider.cache_key + + desc = provider.get_provider_description() + assert "Local" in desc + assert str(tmp_path) in desc + + +def test_local_find_candidates_flat(tmp_path: pathlib.Path) -> None: + """Flat mode finds wheels, sdists, and build-tagged wheels in one directory.""" + # wheel and sdist with same version + _create_local_file(tmp_path, "example_pkg-1.0.0-py3-none-any.whl") + _create_local_file(tmp_path, "example_pkg-1.0.0.tar.gz") + # sdist filenames can use non-normalized names (dash, underscore, dot) + _create_local_file(tmp_path, "example-pkg-2.0.0.tar.gz") + _create_local_file(tmp_path, "example.pkg-3.0.0.tar.gz") + _create_local_file(tmp_path, "example_pkg-4.0.0-2-py3-none-any.whl") + # noise: other package, directory shaped like a wheel, non-dist file, invalid sdist + _create_local_file(tmp_path, "other_pkg-1.0.0-py3-none-any.whl") + tmp_path.joinpath("example_pkg-9.0.0-py3-none-any.whl").mkdir() + _create_local_file(tmp_path, "README.md") + _create_local_file(tmp_path, "not-a-valid-sdist.tar.gz") + + provider = finders.LocalIndexProvider(path=tmp_path, flat=True) + candidates = list(provider.find_candidates("example-pkg")) + + # find_candidates filters by normalized name; other_pkg is excluded + assert len(candidates) == 5 + assert all(c.name == "example-pkg" for c in candidates) + + # version 1.0.0 has both a wheel and an sdist + v1 = [c for c in candidates if c.version == Version("1.0.0")] + assert len(v1) == 2 + assert {c.is_sdist for c in v1} == {True, False} + whl = next(c for c in v1 if not c.is_sdist) + assert whl.upload_time is None + assert whl.has_metadata is False + + # all sdist name variants normalize to example-pkg + sdists = [c for c in candidates if c.is_sdist] + assert {str(c.version) for c in sdists} == {"1.0.0", "2.0.0", "3.0.0"} + + assert next(c for c in candidates if c.version == Version("4.0.0")).build_tag == ( + 2, + "", + ) + + +def test_local_find_candidates_nested(tmp_path: pathlib.Path) -> None: + """Nested mode looks up a subdirectory using the canonicalized name.""" + pkg_dir = tmp_path / "my-package" + pkg_dir.mkdir() + _create_local_file(pkg_dir, "my_package-1.0.0-py3-none-any.whl") + + provider = finders.LocalIndexProvider(path=tmp_path, flat=False) + + # Various spellings should all resolve via canonicalize_name + for name in ("My_Package", "my.package", "MY-PACKAGE", "my-package"): + candidates = list(provider.find_candidates(name)) + assert len(candidates) == 1, f"failed for {name!r}" + assert candidates[0].name == "my-package" + assert candidates[0].version == Version("1.0.0") + + # missing subdirectory returns no candidates + assert list(provider.find_candidates("no-such-pkg")) == [] + + +def test_local_find_candidates_include_flags(tmp_path: pathlib.Path) -> None: + """include_sdists=False / include_wheels=False filters correctly.""" + _create_local_file(tmp_path, "example_pkg-1.0.0.tar.gz") + _create_local_file(tmp_path, "example_pkg-1.0.0-py3-none-any.whl") + + wheels_only = finders.LocalIndexProvider( + path=tmp_path, flat=True, include_sdists=False + ) + assert all(not c.is_sdist for c in wheels_only.find_candidates("example-pkg")) + + sdists_only = finders.LocalIndexProvider( + path=tmp_path, flat=True, include_wheels=False + ) + assert all(c.is_sdist for c in sdists_only.find_candidates("example-pkg")) + + +def test_local_find_candidates_empty_dir(tmp_path: pathlib.Path) -> None: + provider = finders.LocalIndexProvider(path=tmp_path, flat=True) + assert list(provider.find_candidates("nonexistent-pkg")) == [] + + +def test_local_find_matches(tmp_path: pathlib.Path) -> None: + """find_matches sorts by version descending and respects constraints.""" + _create_local_file(tmp_path, "example_pkg-1.0.0-py3-none-any.whl") + _create_local_file(tmp_path, "example_pkg-1.5.0-py3-none-any.whl") + _create_local_file(tmp_path, "example_pkg-2.0.0-py3-none-any.whl") + _create_local_file(tmp_path, "other_pkg-5.0.0-py3-none-any.whl") + + provider = finders.LocalIndexProvider(path=tmp_path, flat=True) + req = Requirement("example-pkg") + identifier = provider.identify(req) + matches = list( + provider.find_matches( + identifier=identifier, + requirements={identifier: [req]}, + incompatibilities={}, + ) + ) + # other_pkg is excluded by name; results sorted highest first + assert [m.version for m in matches] == [ + Version("2.0.0"), + Version("1.5.0"), + Version("1.0.0"), + ] + + # constraint filters out 2.0.0 + c = constraints.Constraints() + c.add_constraint("example-pkg<2") + provider_c = finders.LocalIndexProvider(path=tmp_path, flat=True, constraints=c) + matches_c = list( + provider_c.find_matches( + identifier=identifier, + requirements={identifier: [req]}, + incompatibilities={}, + ) + ) + assert [m.version for m in matches_c] == [Version("1.5.0"), Version("1.0.0")] + + # no match raises + provider_miss = finders.LocalIndexProvider(path=tmp_path, flat=True) + req_miss = Requirement("example-pkg>=5.0") + with pytest.raises(resolvelib.resolvers.ResolverException): + list( + provider_miss.find_matches( + identifier=identifier, + requirements={identifier: [req_miss]}, + incompatibilities={}, + ) + )