From f012cebf59c5d662318b8e9a5ba89427a116b573 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 9 May 2026 20:48:21 +0200 Subject: [PATCH] feat: replace local index server with uv flat index *incomplete and work-in-progress* `uv` supports flat indexes, which are local directories that contain a bunch of wheel files. Experiment if it is possible to replace our custom wheel server with flat indexes. We would not need to spin up a web server, symlink files, and uv could directly access the files from disk. See: https://docs.astral.sh/uv/concepts/indexes/#flat-indexes Signed-off-by: Christian Heimes --- src/fromager/build_environment.py | 4 +++- src/fromager/context.py | 20 ++++++++++++++++++++ src/fromager/server.py | 2 +- src/fromager/wheels.py | 5 +++-- tests/test_dependencies.py | 2 ++ tests/test_server.py | 1 + 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/fromager/build_environment.py b/src/fromager/build_environment.py index a5480be5..a4fe3373 100644 --- a/src/fromager/build_environment.py +++ b/src/fromager/build_environment.py @@ -195,6 +195,8 @@ def install(self, reqs: typing.Iterable[Requirement]) -> None: # UV does not compile byte code by default cmd = [ "uv", + "--config-file", + str(self._ctx.uv_toml), "pip", "install", "--verbose", @@ -203,7 +205,7 @@ def install(self, reqs: typing.Iterable[Requirement]) -> None: ":all:", ] cmd.extend(self._ctx.pip_constraint_args) - cmd.extend(self._ctx.pip_wheel_server_args) + # cmd.extend(self._ctx.pip_wheel_server_args) cmd.extend(str(req) for req in reqs) self.run( diff --git a/src/fromager/context.py b/src/fromager/context.py index 1bc543ed..72ad20ea 100644 --- a/src/fromager/context.py +++ b/src/fromager/context.py @@ -31,6 +31,23 @@ BuildRequirements = dict[str, list[tuple[str, NormalizedName, Version, Requirement]]] ROOT_BUILD_REQUIREMENT = canonicalize_name("", validate=False) +# custom uv settings to use flat, local indexes +UV_TOML = """ +cache-dir = "{ctx.uv_cache}" +system-certs = true + +[[index]] +name = "fromager-wheels-download" +default = true +url = "{ctx.wheels_downloads}" +format = "flat" + +[[index]] +name = "fromager-wheels-prebuilt" +url = "{ctx.wheels_prebuilt}" +format = "flat" +""" + class WorkContext: def __init__( @@ -76,6 +93,7 @@ def __init__( self.wheel_server_dir = self.wheels_repo / "simple" self.work_dir = pathlib.Path(work_dir).resolve() self.graph_file = self.work_dir / "graph.json" + self.uv_toml = self.work_dir / "uv.toml" self.uv_cache = self.work_dir / "uv-cache" self.wheel_server_url = wheel_server_url self.logs_dir = self.work_dir / "logs" @@ -201,6 +219,8 @@ def setup(self) -> None: if not p.exists(): logger.debug("creating %s", p) p.mkdir(parents=True) + logger.debug("create %s", self.uv_toml) + self.uv_toml.write_text(UV_TOML.format(ctx=self)) def clean_build_dirs( self, diff --git a/src/fromager/server.py b/src/fromager/server.py index 7d282f87..c350179d 100644 --- a/src/fromager/server.py +++ b/src/fromager/server.py @@ -33,7 +33,7 @@ def start_wheel_server(ctx: context.WorkContext) -> None: if ctx.wheel_server_url: logger.debug("using external wheel server at %s", ctx.wheel_server_url) return - run_wheel_server(ctx) + # run_wheel_server(ctx) def run_wheel_server( diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index f82042d2..14b28f65 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -493,8 +493,9 @@ def get_wheel_server_urls( if cache_wheel_server_url: # put cache after local server so we always check local server first wheel_server_urls.append(cache_wheel_server_url) - if not wheel_server_urls: - raise ValueError("no wheel server urls configured") + # XXX + # if not wheel_server_urls: + # raise ValueError("no wheel server urls configured") return wheel_server_urls diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index aad6d73f..ffc65678 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -179,6 +179,7 @@ def test_get_build_system_dependencies_cached( assert results == set([Requirement("foo==1.0")]) +@pytest.mark.skip("Should be a E2E test, accesses PyPI") @patch("fromager.dependencies._write_requirements_file") @_clean_build_artifacts @pytest.mark.network @@ -237,6 +238,7 @@ def test_get_build_backend_dependencies_cached( assert results == set([Requirement("foo==1.0")]) +@pytest.mark.skip("Should be a E2E test, accesses PyPI") @patch("fromager.dependencies._write_requirements_file") @_clean_build_artifacts @pytest.mark.network diff --git a/tests/test_server.py b/tests/test_server.py index d507682e..34e0a7c1 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -181,6 +181,7 @@ def test_start_wheel_server_uses_external_url( assert tmp_context.wheel_server_url == "http://external:8080/simple/" +@pytest.mark.skip("WIP: run_wheel_server disabled") @patch("fromager.server.run_wheel_server") @patch("fromager.server.update_wheel_mirror") def test_start_wheel_server_starts_local(