From 1271ff8e400983b2ab643fb5269a311440b557cb Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 13:57:18 -0700 Subject: [PATCH 01/47] feat(config): add named VOSpace services --- CONTEXT.md | 10 ++++ canfar/models/config.py | 9 ++++ canfar/models/http.py | 38 ++++++++++++++- docs/cli/quick-start.md | 6 +++ pyproject.toml | 9 ++++ tests/test_cli_server.py | 1 + tests/test_models_config.py | 97 ++++++++++++++++++++++++++++++++++++- tests/test_models_http.py | 25 +++++++++- uv.lock | 37 ++++++++++++++ 9 files changed, 229 insertions(+), 3 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 847238d1..89391ed4 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -16,6 +16,14 @@ _Avoid_: Base URL, host, cluster Required, unique, user-facing handle for one **Science Platform Server**, used to reference it in configuration and commands. _Avoid_: Label, alias, key +**VOSpace Service**: +Storage service associated with a **Science Platform Server**, identified by an IVOA registry resource URI and accessed through a base HTTP endpoint. +_Avoid_: Storage backend, filesystem + +**Storage Name**: +Required, globally unique, user-facing handle for one **VOSpace Service**. `local` is reserved for the user's local filesystem. +_Avoid_: Server Name, alias, key + **Identity Provider (IDP)**: Organization that issues user identity for CANFAR authentication. Initial IDPs are `Canadian Astronomy Data Centre (CADC)` and `SKA Regional Centre Network (SRCNet)`. @@ -69,6 +77,8 @@ _Avoid_: Resource profile, quota - A **CANFAR Science Platform** exposes one or more **Science Platform Servers**. - A **Science Platform Server** is identified by its **Server Name**; its IVOA URI is discovery metadata, and two Server Names may point at the same endpoint. +- A **Science Platform Server** can expose multiple **VOSpace Services**. +- A **VOSpace Service** is identified by its **Storage Name** and uses its parent **Science Platform Server**'s **Identity Provider (IDP)**. - An **Identity Provider (IDP)** can support one or more **Science Platform Servers**. - **Authentication** and **Platform** are separate seams with independent ownership. - An **Authentication Record** belongs to one **Identity Provider (IDP)**. diff --git a/canfar/models/config.py b/canfar/models/config.py index f5473217..19734c0b 100644 --- a/canfar/models/config.py +++ b/canfar/models/config.py @@ -182,6 +182,7 @@ def settings_customise_sources( def _inject_server_names(self) -> Configuration: """Inject dict keys into each server record and validate Server Names.""" updated: dict[str, Server] = {} + storage_servers: dict[str, str] = {} for name, server in self.servers.items(): if not _SERVER_NAME_PATTERN.match(name): msg = ( @@ -189,6 +190,14 @@ def _inject_server_names(self) -> Configuration: r"^[A-Za-z][A-Za-z0-9_-]*$" ) raise ValueError(msg) + for storage_name in server.storage: + if previous_server := storage_servers.get(storage_name): + msg = ( + f"Duplicate Storage Name '{storage_name}' in Science Platform " + f"Servers '{previous_server}' and '{name}'." + ) + raise ValueError(msg) + storage_servers[storage_name] = name updated[name] = server.model_copy(update={"name": name}, deep=True) self.servers = updated return self diff --git a/canfar/models/http.py b/canfar/models/http.py index dd09dea3..1d228c64 100644 --- a/canfar/models/http.py +++ b/canfar/models/http.py @@ -2,7 +2,9 @@ from __future__ import annotations -from pydantic import AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, Field +from typing import Any + +from pydantic import AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, Field, field_validator DEFAULT_SERVER_CORES = 2 """Default CPU core limit when context enrichment is unavailable.""" @@ -14,6 +16,15 @@ """Default GPU count when context enrichment is unavailable.""" +class VOSpaceService(BaseModel): + """VOSpace Service discovered through an IVOA registry.""" + + model_config = ConfigDict(extra="forbid") + + uri: AnyUrl + url: AnyHttpUrl + + class Server(BaseModel): """Science Platform Server Details.""" @@ -72,6 +83,12 @@ class Server(BaseModel): min_length=1, max_length=64, ) + storage: dict[str, VOSpaceService] = Field( + default_factory=dict, + title="VOSpace Services", + description="VOSpace Services keyed by globally unique Storage Name.", + ) + cores: int = Field( default=DEFAULT_SERVER_CORES, title="Default CPU Core Limit", @@ -98,3 +115,22 @@ class Server(BaseModel): "when known." ), ) + + @field_validator("storage", mode="before") + @classmethod + def _validate_storage_names(cls, value: Any) -> Any: + """Validate Storage Names before global string constraints run.""" + if isinstance(value, dict): + for name in value: + if not isinstance(name, str) or ( + not name + or name == "local" + or any(character in name for character in ":\x00\n") + or name.startswith("-") + ): + msg = ( + f"Invalid Storage Name {name!r}: must be non-empty, must not " + "be 'local', contain colon, NUL, or newline, or start with '-'." + ) + raise ValueError(msg) + return value diff --git a/docs/cli/quick-start.md b/docs/cli/quick-start.md index dcfb005e..2d28f03d 100644 --- a/docs/cli/quick-start.md +++ b/docs/cli/quick-start.md @@ -8,6 +8,12 @@ Create a notebook Session from a terminal, open it, inspect it, and clean it up. pip install canfar --upgrade ``` +Install the pinned optional storage dependencies when you need data support: + +```bash +pip install "canfar[data]" --upgrade +``` + ## 2. Log in ```bash diff --git a/pyproject.toml b/pyproject.toml index 48b07818..2dbe86b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,9 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel.force-include] "canfar/utils/data/scientists.txt" = "canfar/utils/data/scientists.txt" +[tool.hatch.metadata] +allow-direct-references = true + [project] name = "canfar" version = "1.4.1" @@ -58,6 +61,12 @@ dependencies = [ "typer>=0.16.0", ] +[project.optional-dependencies] +data = [ + "vosfs @ git+https://github.com/shinybrar/vosfs@v0.6.0", + "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.5.0#subdirectory=src/fsspec-cli", +] + [tool.uv] [dependency-groups] dev = [ diff --git a/tests/test_cli_server.py b/tests/test_cli_server.py index 9604bd43..58999efc 100644 --- a/tests/test_cli_server.py +++ b/tests/test_cli_server.py @@ -149,6 +149,7 @@ def test_server_ls_json_output(tmp_path: Path) -> None: "version", "auths", "idp", + "storage", "cores", "ram", "gpus", diff --git a/tests/test_models_config.py b/tests/test_models_config.py index dd010c28..d14e6cb3 100644 --- a/tests/test_models_config.py +++ b/tests/test_models_config.py @@ -20,7 +20,7 @@ X509Credential, ) from canfar.models.config import Configuration, _CanfarEnvSettingsSource -from canfar.models.http import Server +from canfar.models.http import Server, VOSpaceService from canfar.models.registry import ContainerRegistry @@ -137,6 +137,49 @@ def test_invalid_idp_key_rejected(self, idp_key: str) -> None: }, ) + @pytest.mark.parametrize( + "storage_name", + ["", "local", "archive:old", "archive\x00old", "archive\nold", "-archive"], + ) + def test_invalid_storage_name_rejected(self, storage_name: str) -> None: + """Invalid Storage Names fail with the rejected name and constraints.""" + with pytest.raises(ValidationError, match="Invalid Storage Name") as exc_info: + Configuration( + servers={ + "canfar": { + "storage": { + storage_name: { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + } + } + } + } + ) + + assert repr(storage_name) in str(exc_info.value) + + def test_duplicate_storage_name_across_servers_rejected(self) -> None: + """A Storage Name identifies at most one service across all Servers.""" + service = VOSpaceService( + uri="ivo://cadc.nrc.ca/arc", + url="https://ws-cadc.canfar.net/arc", + ) + + with pytest.raises( + ValidationError, + match=( + "Duplicate Storage Name 'shared' in Science Platform Servers " + "'canfar' and 'srcnet'" + ), + ): + Configuration( + servers={ + "canfar": Server(storage={"shared": service}), + "srcnet": Server(storage={"shared": service}), + } + ) + def test_valid_active_references(self) -> None: """Validation passes when active authentication and server exist.""" config = Configuration( @@ -316,6 +359,58 @@ def test_complex_round_trip_serialization(self, tmp_path: Path) -> None: assert loaded_oidc.expiry.access == 1893456000 assert loaded_oidc.expiry.refresh is None + def test_v1_storage_json_and_yaml_round_trip(self, tmp_path: Path) -> None: + """Multiple VOSpace Services retain stable nested Storage Names.""" + config = Configuration.model_validate( + _sample_config( + servers={ + "canfar": { + **_sample_config()["servers"]["canfar"], + "storage": { + "canSRC": { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + }, + "canSRCs3": { + "uri": "ivo://cadc.nrc.ca/arc-s3", + "url": "https://ws-cadc.canfar.net/arc-s3", + }, + }, + } + } + ) + ) + + json_data = config.model_dump(mode="json") + assert list(json_data["servers"]["canfar"]["storage"]) == [ + "canSRC", + "canSRCs3", + ] + assert "name" not in json_data["servers"]["canfar"]["storage"]["canSRC"] + assert Configuration.model_validate_json(config.model_dump_json()) == config + + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config.save() + loaded = Configuration() + + assert list(loaded.servers["canfar"].storage) == ["canSRC", "canSRCs3"] + assert loaded.servers["canfar"].idp == "cadc" + assert loaded == config + + def test_existing_v1_configuration_without_storage_loads_unchanged( + self, tmp_path: Path + ) -> None: + """The optional storage mapping does not require a schema migration.""" + config_path = tmp_path / "config.yaml" + config_path.write_text(yaml.safe_dump(_sample_config()), encoding="utf-8") + + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + + assert config.version == 1 + assert config.servers["canfar"].storage == {} + def test_save_creates_directory(self, tmp_path: Path) -> None: """Save creates parent directories when missing.""" config = Configuration() diff --git a/tests/test_models_http.py b/tests/test_models_http.py index ffb28de4..7d26c347 100644 --- a/tests/test_models_http.py +++ b/tests/test_models_http.py @@ -3,7 +3,29 @@ import pytest from pydantic import ValidationError -from canfar.models.http import Server +from canfar.models.http import Server, VOSpaceService + + +class TestVOSpaceService: + """Test VOSpace Service configuration.""" + + def test_requires_registry_uri_and_http_endpoint(self) -> None: + """A VOSpace Service carries only its registry URI and base URL.""" + service = VOSpaceService( + uri="ivo://cadc.nrc.ca/arc", + url="https://ws-cadc.canfar.net/arc", + ) + + assert service.model_dump(mode="json") == { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + } + + with pytest.raises(ValidationError): + VOSpaceService( + uri="ivo://cadc.nrc.ca/arc", + url="vos://ws-cadc.canfar.net/arc", + ) class TestServer: @@ -33,6 +55,7 @@ def test_default_values(self) -> None: assert server.url is None assert server.version is None assert server.status is None + assert server.storage == {} def test_with_all_values(self) -> None: """Test Server with all custom values.""" diff --git a/uv.lock b/uv.lock index 94ce2ee3..63835be4 100644 --- a/uv.lock +++ b/uv.lock @@ -129,6 +129,12 @@ dependencies = [ { name = "typer" }, ] +[package.optional-dependencies] +data = [ + { name = "fsspec-cli" }, + { name = "vosfs" }, +] + [package.dev-dependencies] dev = [ { name = "ipython", version = "8.39.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -158,6 +164,7 @@ requires-dist = [ { name = "cadcutils", specifier = ">=1.5.4" }, { name = "click", specifier = ">=8.4.1" }, { name = "defusedxml", specifier = ">=0.7.1" }, + { name = "fsspec-cli", marker = "extra == 'data'", git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.5.0" }, { name = "httpx", extras = ["http2"], specifier = ">=0.28.1" }, { name = "humanize", specifier = ">=4.12.3" }, { name = "pydantic", specifier = ">=2.9.2" }, @@ -167,7 +174,9 @@ requires-dist = [ { name = "rich", specifier = ">=13.9.4" }, { name = "segno", specifier = ">=1.6.6" }, { name = "typer", specifier = ">=0.16.0" }, + { name = "vosfs", marker = "extra == 'data'", git = "https://github.com/shinybrar/vosfs?rev=v0.6.0" }, ] +provides-extras = ["data"] [package.metadata.requires-dev] dev = [ @@ -662,6 +671,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/02/be4a57b60c7149b55b9e3b3c13f609cd8eb5307c751f22bd8fb8d262e75b/filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51", size = 46036, upload-time = "2026-07-08T05:46:57.53Z" }, ] +[[package]] +name = "fsspec" +version = "2026.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, +] + +[[package]] +name = "fsspec-cli" +version = "0.5.0" +source = { git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.5.0#ddef461b464d41811b6ddbad8eb1a64e37d2d85e" } +dependencies = [ + { name = "fsspec" }, + { name = "typer" }, +] + [[package]] name = "ghp-import" version = "2.1.0" @@ -2167,6 +2194,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/7c/4e7225d46d634a0d8d534dd8a6ce0c319d09b4d0cf0337eb314ca4789d8c/virtualenv-21.6.1-py3-none-any.whl", hash = "sha256:afe991df855715a2b2f60edfcc0107ef95a79fdfd8cb4cdaa71603d1c12e463b", size = 5506392, upload-time = "2026-07-10T19:33:51.629Z" }, ] +[[package]] +name = "vosfs" +version = "0.6.0" +source = { git = "https://github.com/shinybrar/vosfs?rev=v0.6.0#200eae23fbc5e2bd58693b5c14ce12b2795d0636" } +dependencies = [ + { name = "defusedxml" }, + { name = "fsspec" }, + { name = "httpx" }, +] + [[package]] name = "watchdog" version = "6.0.0" From 9c516037ad675d9598b303172f54b8cc574dcf8f Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 14:01:11 -0700 Subject: [PATCH 02/47] refactor(config): address VOSpace review findings --- CONTEXT.md | 4 ++-- canfar/models/config.py | 4 ++-- tests/test_models_http.py | 18 +++++++++++++----- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 89391ed4..d1608f28 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -17,11 +17,11 @@ Required, unique, user-facing handle for one **Science Platform Server**, used t _Avoid_: Label, alias, key **VOSpace Service**: -Storage service associated with a **Science Platform Server**, identified by an IVOA registry resource URI and accessed through a base HTTP endpoint. +Remote astronomical storage service associated with a **Science Platform Server**. _Avoid_: Storage backend, filesystem **Storage Name**: -Required, globally unique, user-facing handle for one **VOSpace Service**. `local` is reserved for the user's local filesystem. +Required, globally unique, user-facing handle for one **VOSpace Service**. _Avoid_: Server Name, alias, key **Identity Provider (IDP)**: diff --git a/canfar/models/config.py b/canfar/models/config.py index 19734c0b..7c946410 100644 --- a/canfar/models/config.py +++ b/canfar/models/config.py @@ -179,8 +179,8 @@ def settings_customise_sources( ) @model_validator(mode="after") - def _inject_server_names(self) -> Configuration: - """Inject dict keys into each server record and validate Server Names.""" + def _normalize_and_validate_servers(self) -> Configuration: + """Inject Server Names and validate Server and Storage Name keys.""" updated: dict[str, Server] = {} storage_servers: dict[str, str] = {} for name, server in self.servers.items(): diff --git a/tests/test_models_http.py b/tests/test_models_http.py index 7d26c347..cdd4b958 100644 --- a/tests/test_models_http.py +++ b/tests/test_models_http.py @@ -9,7 +9,7 @@ class TestVOSpaceService: """Test VOSpace Service configuration.""" - def test_requires_registry_uri_and_http_endpoint(self) -> None: + def test_serializes_registry_uri_and_http_endpoint(self) -> None: """A VOSpace Service carries only its registry URI and base URL.""" service = VOSpaceService( uri="ivo://cadc.nrc.ca/arc", @@ -21,11 +21,19 @@ def test_requires_registry_uri_and_http_endpoint(self) -> None: "url": "https://ws-cadc.canfar.net/arc", } + @pytest.mark.parametrize( + "payload", + [ + {"uri": "not-a-url", "url": "https://ws-cadc.canfar.net/arc"}, + {"uri": "ivo://cadc.nrc.ca/arc", "url": "vos://example.com/arc"}, + {"url": "https://ws-cadc.canfar.net/arc"}, + {"uri": "ivo://cadc.nrc.ca/arc"}, + ], + ) + def test_requires_valid_registry_uri_and_http_endpoint(self, payload: dict) -> None: + """Both VOSpace Service identifiers are required and validated.""" with pytest.raises(ValidationError): - VOSpaceService( - uri="ivo://cadc.nrc.ca/arc", - url="vos://ws-cadc.canfar.net/arc", - ) + VOSpaceService.model_validate(payload) class TestServer: From 9de7bcc481850e6c106fc0a60be592dda3f17648 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 14:15:02 -0700 Subject: [PATCH 03/47] fix(config): validate normalized storage names --- canfar/models/config.py | 10 ++++--- canfar/models/http.py | 54 ++++++++++++++++++++++++++----------- tests/test_models_config.py | 51 ++++++++++++++++++++++++++--------- tests/test_models_http.py | 12 +++++++++ 4 files changed, 95 insertions(+), 32 deletions(-) diff --git a/canfar/models/config.py b/canfar/models/config.py index 7c946410..6ee857da 100644 --- a/canfar/models/config.py +++ b/canfar/models/config.py @@ -182,7 +182,7 @@ def settings_customise_sources( def _normalize_and_validate_servers(self) -> Configuration: """Inject Server Names and validate Server and Storage Name keys.""" updated: dict[str, Server] = {} - storage_servers: dict[str, str] = {} + server_name_by_storage_name: dict[str, str] = {} for name, server in self.servers.items(): if not _SERVER_NAME_PATTERN.match(name): msg = ( @@ -191,13 +191,15 @@ def _normalize_and_validate_servers(self) -> Configuration: ) raise ValueError(msg) for storage_name in server.storage: - if previous_server := storage_servers.get(storage_name): + if previous_server_name := server_name_by_storage_name.get( + storage_name + ): msg = ( f"Duplicate Storage Name '{storage_name}' in Science Platform " - f"Servers '{previous_server}' and '{name}'." + f"Servers '{previous_server_name}' and '{name}'." ) raise ValueError(msg) - storage_servers[storage_name] = name + server_name_by_storage_name[storage_name] = name updated[name] = server.model_copy(update={"name": name}, deep=True) self.servers = updated return self diff --git a/canfar/models/http.py b/canfar/models/http.py index 1d228c64..c4a1bf81 100644 --- a/canfar/models/http.py +++ b/canfar/models/http.py @@ -24,6 +24,15 @@ class VOSpaceService(BaseModel): uri: AnyUrl url: AnyHttpUrl + @field_validator("url") + @classmethod + def _reject_capabilities_endpoint(cls, url: AnyHttpUrl) -> AnyHttpUrl: + """Require the VOSpace base endpoint rather than its capabilities URL.""" + if (url.path or "").rstrip("/").endswith("/capabilities"): + msg = "VOSpace Service base URL must not end with /capabilities." + raise ValueError(msg) + return url + class Server(BaseModel): """Science Platform Server Details.""" @@ -119,18 +128,33 @@ class Server(BaseModel): @field_validator("storage", mode="before") @classmethod def _validate_storage_names(cls, value: Any) -> Any: - """Validate Storage Names before global string constraints run.""" - if isinstance(value, dict): - for name in value: - if not isinstance(name, str) or ( - not name - or name == "local" - or any(character in name for character in ":\x00\n") - or name.startswith("-") - ): - msg = ( - f"Invalid Storage Name {name!r}: must be non-empty, must not " - "be 'local', contain colon, NUL, or newline, or start with '-'." - ) - raise ValueError(msg) - return value + """Normalize and validate Storage Names before Pydantic transforms keys.""" + if not isinstance(value, dict): + return value + + normalized: dict[str, Any] = {} + original_name_by_normalized_name: dict[str, str] = {} + for original_name, service in value.items(): + name = original_name.strip() if isinstance(original_name, str) else None + if name is None or ( + not name + or name == "local" + or any(character in name for character in ":\x00\n") + or name.startswith("-") + ): + msg = ( + f"Invalid Storage Name {original_name!r}: after whitespace " + "normalization it must be non-empty, differ from reserved 'local', " + "contain no colon, NUL, or newline, and not start with '-'." + ) + raise ValueError(msg) + if name in normalized: + previous_original_name = original_name_by_normalized_name[name] + msg = ( + f"Storage Names {previous_original_name!r} and {original_name!r} " + f"both normalize to {name!r}; use unique names." + ) + raise ValueError(msg) + normalized[name] = service + original_name_by_normalized_name[name] = original_name + return normalized diff --git a/tests/test_models_config.py b/tests/test_models_config.py index d14e6cb3..cecb8f23 100644 --- a/tests/test_models_config.py +++ b/tests/test_models_config.py @@ -139,18 +139,28 @@ def test_invalid_idp_key_rejected(self, idp_key: str) -> None: @pytest.mark.parametrize( "storage_name", - ["", "local", "archive:old", "archive\x00old", "archive\nold", "-archive"], + [ + "", + "local", + " local ", + "archive:old", + "archive\x00old", + "archive\nold", + "-archive", + ], ) def test_invalid_storage_name_rejected(self, storage_name: str) -> None: """Invalid Storage Names fail with the rejected name and constraints.""" with pytest.raises(ValidationError, match="Invalid Storage Name") as exc_info: - Configuration( - servers={ - "canfar": { - "storage": { - storage_name: { - "uri": "ivo://cadc.nrc.ca/arc", - "url": "https://ws-cadc.canfar.net/arc", + Configuration.model_validate( + { + "servers": { + "canfar": { + "storage": { + storage_name: { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + } } } } @@ -160,7 +170,7 @@ def test_invalid_storage_name_rejected(self, storage_name: str) -> None: assert repr(storage_name) in str(exc_info.value) def test_duplicate_storage_name_across_servers_rejected(self) -> None: - """A Storage Name identifies at most one service across all Servers.""" + """A Storage Name identifies one service across Science Platform Servers.""" service = VOSpaceService( uri="ivo://cadc.nrc.ca/arc", url="https://ws-cadc.canfar.net/arc", @@ -173,13 +183,28 @@ def test_duplicate_storage_name_across_servers_rejected(self) -> None: "'canfar' and 'srcnet'" ), ): - Configuration( - servers={ - "canfar": Server(storage={"shared": service}), - "srcnet": Server(storage={"shared": service}), + Configuration.model_validate( + { + "servers": { + "canfar": Server(storage={"shared": service}), + "srcnet": Server(storage={"shared": service}), + } } ) + def test_normalized_storage_name_collision_rejected(self) -> None: + """Whitespace normalization cannot silently replace a VOSpace Service.""" + service = { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + } + + with pytest.raises( + ValidationError, + match="Storage Names 'arc' and ' arc ' both normalize to 'arc'", + ): + Server(storage={"arc": service, " arc ": service}) + def test_valid_active_references(self) -> None: """Validation passes when active authentication and server exist.""" config = Configuration( diff --git a/tests/test_models_http.py b/tests/test_models_http.py index cdd4b958..ff7ed3c1 100644 --- a/tests/test_models_http.py +++ b/tests/test_models_http.py @@ -35,6 +35,18 @@ def test_requires_valid_registry_uri_and_http_endpoint(self, payload: dict) -> N with pytest.raises(ValidationError): VOSpaceService.model_validate(payload) + @pytest.mark.parametrize( + "url", + [ + "https://ws-cadc.canfar.net/arc/capabilities", + "https://ws-cadc.canfar.net/arc/capabilities/", + ], + ) + def test_rejects_capabilities_endpoint_as_base_url(self, url: str) -> None: + """A VOSpace Service URL names the base service, not capabilities.""" + with pytest.raises(ValidationError, match="must not end with /capabilities"): + VOSpaceService(uri="ivo://cadc.nrc.ca/arc", url=url) + class TestServer: """Test Server class.""" From 690f3bd0afd1bc2de2270ba0bb1360a30480ec2e Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 14:18:15 -0700 Subject: [PATCH 04/47] test(config): isolate storage validation tests --- tests/test_models_config.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/test_models_config.py b/tests/test_models_config.py index cecb8f23..8e4de0d4 100644 --- a/tests/test_models_config.py +++ b/tests/test_models_config.py @@ -149,9 +149,14 @@ def test_invalid_idp_key_rejected(self, idp_key: str) -> None: "-archive", ], ) - def test_invalid_storage_name_rejected(self, storage_name: str) -> None: + def test_invalid_storage_name_rejected( + self, storage_name: str, tmp_path: Path + ) -> None: """Invalid Storage Names fail with the rejected name and constraints.""" - with pytest.raises(ValidationError, match="Invalid Storage Name") as exc_info: + with ( + patch("canfar.models.config.CONFIG_PATH", tmp_path / "config.yaml"), + pytest.raises(ValidationError, match="Invalid Storage Name") as exc_info, + ): Configuration.model_validate( { "servers": { @@ -169,18 +174,23 @@ def test_invalid_storage_name_rejected(self, storage_name: str) -> None: assert repr(storage_name) in str(exc_info.value) - def test_duplicate_storage_name_across_servers_rejected(self) -> None: + def test_duplicate_storage_name_across_servers_rejected( + self, tmp_path: Path + ) -> None: """A Storage Name identifies one service across Science Platform Servers.""" service = VOSpaceService( uri="ivo://cadc.nrc.ca/arc", url="https://ws-cadc.canfar.net/arc", ) - with pytest.raises( - ValidationError, - match=( - "Duplicate Storage Name 'shared' in Science Platform Servers " - "'canfar' and 'srcnet'" + with ( + patch("canfar.models.config.CONFIG_PATH", tmp_path / "config.yaml"), + pytest.raises( + ValidationError, + match=( + "Duplicate Storage Name 'shared' in Science Platform Servers " + "'canfar' and 'srcnet'" + ), ), ): Configuration.model_validate( From 5f75b13196ae181d3363d79fa99861a3d42969f1 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 14:34:02 -0700 Subject: [PATCH 05/47] fix(config): validate raw storage names --- canfar/models/http.py | 18 +++++++++--------- tests/test_models_config.py | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/canfar/models/http.py b/canfar/models/http.py index c4a1bf81..28824a98 100644 --- a/canfar/models/http.py +++ b/canfar/models/http.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any +from typing import Annotated, Any from pydantic import AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, Field, field_validator @@ -42,7 +42,6 @@ class Server(BaseModel): extra="forbid", json_schema_mode_override="serialization", str_strip_whitespace=True, - str_max_length=256, str_min_length=1, ) @@ -79,7 +78,7 @@ class Server(BaseModel): min_length=2, max_length=8, ) - auths: list[str] | None = Field( + auths: list[Annotated[str, Field(max_length=256)]] | None = Field( default=None, title="Supported Auth Modes", description="Authentication modes supported by the Server", @@ -123,6 +122,7 @@ class Server(BaseModel): "Persisted compatibility field for discovery reachability status " "when known." ), + max_length=256, ) @field_validator("storage", mode="before") @@ -135,13 +135,13 @@ def _validate_storage_names(cls, value: Any) -> Any: normalized: dict[str, Any] = {} original_name_by_normalized_name: dict[str, str] = {} for original_name, service in value.items(): - name = original_name.strip() if isinstance(original_name, str) else None - if name is None or ( - not name - or name == "local" - or any(character in name for character in ":\x00\n") - or name.startswith("-") + if not isinstance(original_name, str) or any( + character in original_name for character in ":\x00\r\n" ): + name = None + else: + name = original_name.strip() + if name is None or (not name or name == "local" or name.startswith("-")): msg = ( f"Invalid Storage Name {original_name!r}: after whitespace " "normalization it must be non-empty, differ from reserved 'local', " diff --git a/tests/test_models_config.py b/tests/test_models_config.py index 8e4de0d4..33e11380 100644 --- a/tests/test_models_config.py +++ b/tests/test_models_config.py @@ -146,6 +146,9 @@ def test_invalid_idp_key_rejected(self, idp_key: str) -> None: "archive:old", "archive\x00old", "archive\nold", + "archive\n", + "archive\r", + "archive\r\n", "-archive", ], ) @@ -174,6 +177,27 @@ def test_invalid_storage_name_rejected( assert repr(storage_name) in str(exc_info.value) + def test_long_storage_name_allowed(self) -> None: + """Storage Names do not inherit Server field length limits.""" + storage_name = "a" * 257 + service = { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + } + + server = Server(storage={storage_name: service}) + + assert list(server.storage) == [storage_name] + + def test_storage_name_whitespace_is_trimmed(self) -> None: + """Valid surrounding whitespace remains normalized.""" + service = { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + } + + assert list(Server(storage={" arc ": service}).storage) == ["arc"] + def test_duplicate_storage_name_across_servers_rejected( self, tmp_path: Path ) -> None: From 639c245d1ba52d352dc6a787ac79310721e6ba29 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 15:25:29 -0700 Subject: [PATCH 06/47] feat(storage): materialize authenticated vosfs sources --- canfar/_storage.py | 144 +++++++++++++++++++++ canfar/auth/oidc.py | 49 +++++++ canfar/hooks/httpx/auth.py | 51 +------- tests/test_storage.py | 259 +++++++++++++++++++++++++++++++++++++ 4 files changed, 458 insertions(+), 45 deletions(-) create mode 100644 canfar/_storage.py create mode 100644 tests/test_storage.py diff --git a/canfar/_storage.py b/canfar/_storage.py new file mode 100644 index 00000000..07a6c91a --- /dev/null +++ b/canfar/_storage.py @@ -0,0 +1,144 @@ +"""Private adapters for configured VOSpace Services.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, cast + +from canfar.auth import oidc, x509 +from canfar.client import HTTPClient +from canfar.exceptions.context import AuthContextError +from canfar.models.auth import OIDCCredential, X509Credential +from canfar.models.config import Configuration + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + from pathlib import Path + from typing import NoReturn + + from fsspec.spec import AbstractFileSystem + from fsspec_cli import AsyncFilesystemSource + from pydantic import SecretStr + + +def _vospace_source( + storage_name: str, + *, + token: str | SecretStr | None = None, + certificate: Path | None = None, +) -> AsyncFilesystemSource: + """Return a fresh authenticated async filesystem source.""" + + @asynccontextmanager + async def source() -> AsyncIterator[AbstractFileSystem]: + config = Configuration() + endpoint, idp = _resolve_storage(config, storage_name) + credentials = await _materialize_credentials( + config, + idp, + endpoint, + token=token, + certificate=certificate, + ) + + from vosfs import VOSpaceFileSystem # noqa: PLC0415 + + if token_value := credentials.get("token"): + filesystem = VOSpaceFileSystem( + endpoint, + token=token_value, + asynchronous=True, + skip_instance_cache=True, + ) + else: + filesystem = VOSpaceFileSystem( + endpoint, + certfile=credentials["certfile"], + asynchronous=True, + skip_instance_cache=True, + ) + try: + yield filesystem + finally: + await filesystem.aclose() + + return source + + +def _resolve_storage(config: Configuration, storage_name: str) -> tuple[str, str]: + """Resolve one Storage Name to its endpoint and parent server IDP.""" + for server in config.servers.values(): + service = server.storage.get(storage_name) + if service is not None: + if server.idp is None: + msg = ( + f"Storage Name '{storage_name}' belongs to a Science Platform " + "Server without an IDP." + ) + raise ValueError(msg) + return str(service.url), server.idp + msg = f"Storage Name '{storage_name}' is not configured." + raise KeyError(msg) + + +async def _materialize_credentials( + config: Configuration, + idp: str, + endpoint: str, + *, + token: str | SecretStr | None, + certificate: Path | None, +) -> dict[str, str]: + """Return only literal credential material accepted by vosfs.""" + try: + client = HTTPClient( + config=config, + authentication_idp=idp, + url=endpoint, + token=token, + certificate=certificate, + ) + if client.token is not None: + return {"token": client.token.get_secret_value()} + if client.certificate is not None: + return {"certfile": x509.valid(client.certificate)} + + credential = client.authentication_record + if isinstance(credential, X509Credential): + if credential.path is None: + _fail_authentication(idp) + return {"certfile": cast("str", x509.inspect(credential.path)["path"])} + if not isinstance(credential, OIDCCredential): + _fail_authentication(idp) + + if credential.expired: + parameters = _refresh_parameters(credential) + refreshed = await oidc.refresh(*parameters) + credential = oidc._persist_refreshed_credential( # noqa: SLF001 + config, + credential, + refreshed, + ) + if credential.token.access is None: + _fail_authentication(idp) + return {"token": credential.token.access.get_secret_value()} + except (KeyError, OSError, TypeError, ValueError): + _fail_authentication(idp) + + +def _fail_authentication(idp: str) -> NoReturn: + """Raise the fixed secret-safe authentication failure.""" + reason = "Credential cannot be used. Run 'canfar login' for this IDP." + raise AuthContextError(idp, reason) from None + + +def _refresh_parameters(credential: OIDCCredential) -> tuple[str, str, str, str]: + """Return literal refresh inputs for an eligible Authentication Record.""" + if not credential.refreshable: + raise ValueError + return ( + cast("str", credential.endpoints.token), + cast("str", credential.client.identity), + cast("SecretStr", credential.client.secret).get_secret_value(), + cast("SecretStr", credential.token.refresh).get_secret_value(), + ) diff --git a/canfar/auth/oidc.py b/canfar/auth/oidc.py index 3679e46d..e2161c72 100644 --- a/canfar/auth/oidc.py +++ b/canfar/auth/oidc.py @@ -20,6 +20,8 @@ if TYPE_CHECKING: from authlib.integrations.httpx_client import AsyncOAuth2Client + from canfar.models.config import Configuration + log = get_logger(__name__) _BASIC_AUTH_METHOD = "client_secret_basic" @@ -193,6 +195,53 @@ def _validated_refresh(refreshed: Any) -> dict[str, Any]: return dict(refreshed) +def _persist_refreshed_credential( + config: Configuration, + credential: OIDCCredential, + refreshed: dict[str, Any], +) -> OIDCCredential: + """Validate and atomically persist refreshed OIDC state.""" + previous_refresh = credential.token.refresh + previous_refresh_value = ( + previous_refresh.get_secret_value() if previous_refresh is not None else None + ) + returned_refresh = refreshed.get("refresh_token") + refresh_value = ( + returned_refresh + if isinstance(returned_refresh, str) and returned_refresh + else previous_refresh_value + ) + access_value = refreshed.get("access_token") + if not isinstance(access_value, str) or not access_value: + msg = "OIDC token refresh failed: malformed token response" + raise ValueError(msg) + try: + token = Token( + access=SecretStr(access_value), + refresh=SecretStr(refresh_value) if refresh_value is not None else None, + token_type=refreshed.get("token_type") or credential.token.token_type, + scope=refreshed.get("scope") or credential.token.scope, + ) + expiry = Expiry( + access=refreshed.get("expires_at"), + refresh=( + None + if refresh_value != previous_refresh_value + else credential.expiry.refresh + ), + ) + except (KeyError, TypeError, ValidationError): + msg = "OIDC token refresh failed: malformed token response" + raise ValueError(msg) from None + + updated = credential.model_copy(update={"token": token, "expiry": expiry}) + candidate = config.model_copy(deep=True) + candidate.update_credential(updated) + candidate.save() + config.update_credential(updated) + return updated + + async def refresh( url: str, identity: str, diff --git a/canfar/hooks/httpx/auth.py b/canfar/hooks/httpx/auth.py index 391d5079..92d338f7 100644 --- a/canfar/hooks/httpx/auth.py +++ b/canfar/hooks/httpx/auth.py @@ -32,16 +32,15 @@ import asyncio from typing import TYPE_CHECKING, Any, Callable, cast -from pydantic import SecretStr, ValidationError - from canfar import get_logger from canfar.auth import oidc -from canfar.models.auth import Expiry, OIDCCredential, Token +from canfar.models.auth import OIDCCredential if TYPE_CHECKING: from collections.abc import Awaitable, MutableMapping import httpx + from pydantic import SecretStr from canfar.client import HTTPClient @@ -97,51 +96,13 @@ def _apply_refreshed_token( request: httpx.Request, ) -> None: """Atomically persist refreshed OIDC state, then update active headers.""" - previous_refresh = credential.token.refresh - previous_refresh_value = ( - previous_refresh.get_secret_value() if previous_refresh is not None else None - ) - returned_refresh = refreshed.get("refresh_token") - refresh_value = ( - returned_refresh - if isinstance(returned_refresh, str) and returned_refresh - else previous_refresh_value + updated = oidc._persist_refreshed_credential( # noqa: SLF001 + client.config, credential, refreshed ) - rotated = refresh_value != previous_refresh_value - access_value = refreshed.get("access_token") - if not isinstance(access_value, str) or not access_value: - msg = "OIDC token refresh failed: malformed token response" - raise ValueError(msg) - access_token = SecretStr(access_value) - token_type = refreshed.get("token_type") - if token_type is None or token_type == "": - token_type = credential.token.token_type - scope = refreshed.get("scope") - if scope is None or scope == "": - scope = credential.token.scope - try: - token = Token( - access=access_token, - refresh=SecretStr(refresh_value) if refresh_value is not None else None, - token_type=token_type, - scope=scope, - ) - expiry = Expiry( - access=refreshed.get("expires_at"), - refresh=None if rotated else credential.expiry.refresh, - ) - except (KeyError, TypeError, ValidationError): - msg = "OIDC token refresh failed: malformed token response" - raise ValueError(msg) from None - - updated = credential.model_copy(update={"token": token, "expiry": expiry}) - candidate = client.config.model_copy(deep=True) - candidate.update_credential(updated) - candidate.save() - client.config.update_credential(updated) log.debug("Authentication refreshed and configuration saved.") - _apply_access_header(access_token, httpx_client_headers, request) + assert updated.token.access is not None + _apply_access_header(updated.token.access, httpx_client_headers, request) log.debug("HTTP request headers updated with new token.") log.info("OIDC Access Token Refreshed.") diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 00000000..3adc6a3e --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,259 @@ +"""Tests for private authenticated VOSpace source adapters.""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Any +from unittest.mock import AsyncMock, Mock + +import pytest +import vosfs +from pydantic import AnyHttpUrl, AnyUrl + +from canfar._storage import _vospace_source +from canfar.exceptions.context import AuthContextError +from canfar.models.active import ActiveConfig +from canfar.models.config import Configuration +from canfar.models.http import Server, VOSpaceService +from tests.helpers.config import oidc_credential, x509_credential + +if TYPE_CHECKING: + from pathlib import Path + + +class _Filesystem: + """Record construction and cleanup without VOSpace I/O.""" + + async_impl = True + + def __init__(self, endpoint: str, **kwargs: Any) -> None: + self.endpoint = endpoint + self.kwargs = kwargs + self.asynchronous = kwargs["asynchronous"] + self.closed = False + + async def aclose(self) -> None: + self.closed = True + + +def _config( + *, + credential: Any, + endpoint: str = "https://inactive.example/vospace", +) -> Configuration: + server = Server( + idp=credential.idp, + uri=AnyUrl("ivo://inactive.example/skaha"), + url=AnyHttpUrl("https://inactive.example/skaha"), + version="v1", + storage={ + "archive": VOSpaceService( + uri=AnyUrl("ivo://inactive.example/arc"), + url=AnyHttpUrl(endpoint), + ) + }, + ) + return Configuration( + active=ActiveConfig(authentication="active", server=None), + authentication={ + "active": x509_credential("active"), + credential.idp: credential, + }, + servers={"inactive": server}, + ) + + +@pytest.mark.asyncio +async def test_source_reloads_config_and_runtime_token_wins( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Entry reloads endpoint state and keeps token-over-certificate precedence.""" + config_path = tmp_path / "config.yaml" + monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) + config = _config(credential=oidc_credential("inactive")) + config.save() + source = _vospace_source( + "archive", + token="runtime-token", + certificate=tmp_path / "ignored.pem", + ) + + config.servers["inactive"].storage["archive"].url = AnyHttpUrl( + "https://changed.example/vospace" + ) + config.save() + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with source() as filesystem: + assert filesystem.endpoint == "https://changed.example/vospace" + assert filesystem.kwargs == { + "token": "runtime-token", + "asynchronous": True, + "skip_instance_cache": True, + } + assert filesystem.async_impl is True + assert filesystem.asynchronous is True + assert filesystem.closed is False + + assert filesystem.closed is True + + +@pytest.mark.asyncio +async def test_expired_inactive_oidc_refreshes_once_and_persists( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """An inactive Server uses its own IDP and persists one shared refresh.""" + config_path = tmp_path / "config.yaml" + monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) + config = _config( + credential=oidc_credential( + "inactive", + access="old-access-secret", + refresh="refresh-secret", + access_expiry=1.0, + ) + ) + config.save() + refresh = AsyncMock( + return_value={ + "access_token": "new-access-secret", + "expires_at": 9_999_999_999.0, + } + ) + monkeypatch.setattr("canfar._storage.oidc.refresh", refresh) + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with _vospace_source("archive")() as filesystem: + assert filesystem.kwargs["token"] == "new-access-secret" + + refresh.assert_awaited_once_with( + "https://oidc.example.com/token", + "test-client", + "test-secret", + "refresh-secret", + ) + persisted = Configuration() # ty: ignore[missing-argument] + saved = persisted.get_credential("inactive") + assert saved.mode == "oidc" + assert saved.token.access is not None + assert saved.token.access.get_secret_value() == "new-access-secret" + + +@pytest.mark.asyncio +async def test_saved_x509_is_validated_before_construction( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Saved X.509 material becomes only an inspected literal certfile path.""" + config_path = tmp_path / "config.yaml" + certificate = tmp_path / "saved.pem" + monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) + _config(credential=x509_credential("inactive", path=certificate)).save() + inspect = Mock() + + def inspect_certificate(path: Path) -> dict[str, object]: + inspect(path) + return {"path": path.as_posix(), "expiry": 9_999_999_999.0} + + monkeypatch.setattr("canfar._storage.x509.inspect", inspect_certificate) + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with _vospace_source("archive")() as filesystem: + assert filesystem.kwargs["certfile"] == certificate.as_posix() + + inspect.assert_called_once_with(certificate) + + +@pytest.mark.asyncio +async def test_runtime_x509_overrides_saved_oidc( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A validated runtime certificate wins over the saved Authentication Record.""" + config_path = tmp_path / "config.yaml" + certificate = tmp_path / "runtime.pem" + monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) + _config(credential=oidc_credential("inactive")).save() + monkeypatch.setattr( + "canfar._storage.x509.inspect", + lambda path: {"path": path.as_posix(), "expiry": 9_999_999_999.0}, + ) + valid = Mock(return_value=certificate.as_posix()) + monkeypatch.setattr("canfar._storage.x509.valid", valid) + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with _vospace_source("archive", certificate=certificate)() as filesystem: + assert filesystem.kwargs["certfile"] == certificate.as_posix() + + valid.assert_called_once_with(certificate) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("exit_kind", ["error", "cancel"]) +async def test_source_closes_on_failure_and_cancellation( + exit_kind: str, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Context exit always closes a yielded filesystem.""" + config_path = tmp_path / "config.yaml" + monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) + _config(credential=oidc_credential("inactive")).save() + filesystems: list[_Filesystem] = [] + + def build(endpoint: str, **kwargs: Any) -> _Filesystem: + filesystem = _Filesystem(endpoint, **kwargs) + filesystems.append(filesystem) + return filesystem + + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", build) + + async def use_source() -> None: + async with _vospace_source("archive")(): + if exit_kind == "error": + raise RuntimeError + await asyncio.Event().wait() + + task = asyncio.create_task(use_source()) + await asyncio.sleep(0) + if exit_kind == "error": + with pytest.raises(RuntimeError): + await task + else: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert filesystems[0].closed is True + + +@pytest.mark.asyncio +async def test_unrefreshable_oidc_fails_secret_safe_before_vospace( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Bad saved auth reports a login hint without exposing secret values.""" + config_path = tmp_path / "config.yaml" + monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) + _config( + credential=oidc_credential( + "inactive", + access="old-access-secret", + refresh="refresh-secret", + access_expiry=1.0, + refresh_expiry=1.0, + ) + ).save() + constructor = AsyncMock() + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", constructor) + + with pytest.raises(AuthContextError, match="canfar login") as exc_info: + async with _vospace_source("archive")(): + pass + + message = str(exc_info.value) + assert "old-access-secret" not in message + assert "refresh-secret" not in message + constructor.assert_not_called() From 9cb8d6ca01e6c6ebd6bc2ac5c361a98223117b74 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 15:31:56 -0700 Subject: [PATCH 07/47] fix(storage): address authenticated source review --- canfar/_storage.py | 102 ++++++------------------------------- canfar/auth/oidc.py | 16 +++++- canfar/client.py | 37 +++++++++++++- canfar/hooks/httpx/auth.py | 24 ++------- canfar/models/config.py | 15 ++++++ tests/test_storage.py | 77 +++++++++++++++++++++++++--- 6 files changed, 155 insertions(+), 116 deletions(-) diff --git a/canfar/_storage.py b/canfar/_storage.py index 07a6c91a..fd7ded8d 100644 --- a/canfar/_storage.py +++ b/canfar/_storage.py @@ -3,12 +3,10 @@ from __future__ import annotations from contextlib import asynccontextmanager -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING -from canfar.auth import oidc, x509 from canfar.client import HTTPClient from canfar.exceptions.context import AuthContextError -from canfar.models.auth import OIDCCredential, X509Credential from canfar.models.config import Configuration if TYPE_CHECKING: @@ -32,18 +30,22 @@ def _vospace_source( @asynccontextmanager async def source() -> AsyncIterator[AbstractFileSystem]: config = Configuration() - endpoint, idp = _resolve_storage(config, storage_name) - credentials = await _materialize_credentials( - config, - idp, - endpoint, - token=token, - certificate=certificate, - ) + endpoint, idp = config._resolve_storage(storage_name) # noqa: SLF001 + try: + client = HTTPClient( + config=config, + authentication_idp=idp, + url=endpoint, + token=token, + certificate=certificate, + ) + token_value, certfile = await client._materialize_credentials() # noqa: SLF001 + except (KeyError, OSError, TypeError, ValueError): + _fail_authentication(idp) from vosfs import VOSpaceFileSystem # noqa: PLC0415 - if token_value := credentials.get("token"): + if token_value is not None: filesystem = VOSpaceFileSystem( endpoint, token=token_value, @@ -51,9 +53,10 @@ async def source() -> AsyncIterator[AbstractFileSystem]: skip_instance_cache=True, ) else: + assert certfile is not None filesystem = VOSpaceFileSystem( endpoint, - certfile=credentials["certfile"], + certfile=certfile, asynchronous=True, skip_instance_cache=True, ) @@ -65,80 +68,7 @@ async def source() -> AsyncIterator[AbstractFileSystem]: return source -def _resolve_storage(config: Configuration, storage_name: str) -> tuple[str, str]: - """Resolve one Storage Name to its endpoint and parent server IDP.""" - for server in config.servers.values(): - service = server.storage.get(storage_name) - if service is not None: - if server.idp is None: - msg = ( - f"Storage Name '{storage_name}' belongs to a Science Platform " - "Server without an IDP." - ) - raise ValueError(msg) - return str(service.url), server.idp - msg = f"Storage Name '{storage_name}' is not configured." - raise KeyError(msg) - - -async def _materialize_credentials( - config: Configuration, - idp: str, - endpoint: str, - *, - token: str | SecretStr | None, - certificate: Path | None, -) -> dict[str, str]: - """Return only literal credential material accepted by vosfs.""" - try: - client = HTTPClient( - config=config, - authentication_idp=idp, - url=endpoint, - token=token, - certificate=certificate, - ) - if client.token is not None: - return {"token": client.token.get_secret_value()} - if client.certificate is not None: - return {"certfile": x509.valid(client.certificate)} - - credential = client.authentication_record - if isinstance(credential, X509Credential): - if credential.path is None: - _fail_authentication(idp) - return {"certfile": cast("str", x509.inspect(credential.path)["path"])} - if not isinstance(credential, OIDCCredential): - _fail_authentication(idp) - - if credential.expired: - parameters = _refresh_parameters(credential) - refreshed = await oidc.refresh(*parameters) - credential = oidc._persist_refreshed_credential( # noqa: SLF001 - config, - credential, - refreshed, - ) - if credential.token.access is None: - _fail_authentication(idp) - return {"token": credential.token.access.get_secret_value()} - except (KeyError, OSError, TypeError, ValueError): - _fail_authentication(idp) - - def _fail_authentication(idp: str) -> NoReturn: """Raise the fixed secret-safe authentication failure.""" reason = "Credential cannot be used. Run 'canfar login' for this IDP." raise AuthContextError(idp, reason) from None - - -def _refresh_parameters(credential: OIDCCredential) -> tuple[str, str, str, str]: - """Return literal refresh inputs for an eligible Authentication Record.""" - if not credential.refreshable: - raise ValueError - return ( - cast("str", credential.endpoints.token), - cast("str", credential.client.identity), - cast("SecretStr", credential.client.secret).get_secret_value(), - cast("SecretStr", credential.token.refresh).get_secret_value(), - ) diff --git a/canfar/auth/oidc.py b/canfar/auth/oidc.py index e2161c72..a9c36b0c 100644 --- a/canfar/auth/oidc.py +++ b/canfar/auth/oidc.py @@ -7,7 +7,7 @@ import time from collections.abc import Awaitable, Callable, Generator from contextlib import contextmanager -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import httpx from authlib.integrations.base_client.errors import OAuthError @@ -195,6 +195,20 @@ def _validated_refresh(refreshed: Any) -> dict[str, Any]: return dict(refreshed) +def _refresh_parameters( + credential: OIDCCredential, +) -> tuple[str, str, str, str] | None: + """Return complete literal refresh inputs when the record is eligible.""" + if not credential.refreshable: + return None + return ( + cast("str", credential.endpoints.token), + cast("str", credential.client.identity), + cast("SecretStr", credential.client.secret).get_secret_value(), + cast("SecretStr", credential.token.refresh).get_secret_value(), + ) + + def _persist_refreshed_credential( config: Configuration, credential: OIDCCredential, diff --git a/canfar/client.py b/canfar/client.py index a6f63408..bd526abf 100644 --- a/canfar/client.py +++ b/canfar/client.py @@ -20,7 +20,7 @@ from typing_extensions import Self from canfar import __version__, get_logger -from canfar.auth import x509 +from canfar.auth import oidc, x509 from canfar.exceptions.context import AuthContextError from canfar.hooks.httpx import auth, debug, errors, expiry from canfar.models.auth import ( @@ -231,6 +231,41 @@ def _resolved_authentication_record(self) -> AuthenticationCredential | None: ) return credential + async def _materialize_credentials(self) -> tuple[str | None, str | None]: + """Return one literal token or validated certificate path.""" + if self.token is not None: + token = self.token.get_secret_value() + if token: + return token, None + raise ValueError + if self.certificate is not None: + return None, x509.valid(self.certificate) + + credential = self.authentication_record + if isinstance(credential, X509Credential): + if credential.path is None: + raise ValueError + return None, str(x509.inspect(credential.path)["path"]) + if not isinstance(credential, OIDCCredential): + raise TypeError + + if credential.expired: + parameters = oidc._refresh_parameters(credential) # noqa: SLF001 + if parameters is None: + raise ValueError + refreshed = await oidc.refresh(*parameters) + credential = oidc._persist_refreshed_credential( # noqa: SLF001 + self.config, + credential, + refreshed, + ) + if credential.token.access is None: + raise ValueError + token = credential.token.access.get_secret_value() + if not token: + raise ValueError + return token, None + def _get_base_url(self) -> URL: """Get the base URL for the client. diff --git a/canfar/hooks/httpx/auth.py b/canfar/hooks/httpx/auth.py index 92d338f7..0f99b8b6 100644 --- a/canfar/hooks/httpx/auth.py +++ b/canfar/hooks/httpx/auth.py @@ -30,7 +30,7 @@ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING, Any, Callable, cast +from typing import TYPE_CHECKING, Any, Callable from canfar import get_logger from canfar.auth import oidc @@ -70,24 +70,6 @@ def _apply_access_header( request.headers["Authorization"] = header -def _refresh_parameters( - credential: OIDCCredential, -) -> tuple[str, str, str, str] | None: - """Return complete refresh inputs when the record is eligible.""" - if not credential.refreshable: - return None - token_url = cast("str", credential.endpoints.token) - identity = cast("str", credential.client.identity) - client_secret = cast("SecretStr", credential.client.secret) - refresh_token = cast("SecretStr", credential.token.refresh) - return ( - token_url, - identity, - client_secret.get_secret_value(), - refresh_token.get_secret_value(), - ) - - def _apply_refreshed_token( client: HTTPClient, credential: OIDCCredential, @@ -138,7 +120,7 @@ def hook(request: httpx.Request) -> None: log.debug("Skipping auth refresh, access token is not expired.") return - parameters = _refresh_parameters(credential) + parameters = oidc._refresh_parameters(credential) # noqa: SLF001 if parameters is None: log.warning("OIDC Authentication Record cannot be refreshed.") return @@ -200,7 +182,7 @@ async def ahook(request: httpx.Request) -> None: ) return - parameters = _refresh_parameters(credential) + parameters = oidc._refresh_parameters(credential) # noqa: SLF001 if parameters is None: log.warning("OIDC Authentication Record cannot be refreshed.") return diff --git a/canfar/models/config.py b/canfar/models/config.py index 6ee857da..3e16d46d 100644 --- a/canfar/models/config.py +++ b/canfar/models/config.py @@ -340,6 +340,21 @@ def _get_server_by_name(self, name: str) -> Server: raise KeyError(msg) return self.servers[name] + def _resolve_storage(self, storage_name: str) -> tuple[str, str]: + """Resolve a Storage Name to its endpoint and parent server IDP.""" + for server in self.servers.values(): + service = server.storage.get(storage_name) + if service is not None: + if server.idp is None: + msg = ( + f"Storage Name '{storage_name}' belongs to a Science Platform " + "Server without an IDP." + ) + raise ValueError(msg) + return str(service.url), server.idp + msg = f"Storage Name '{storage_name}' is not configured." + raise KeyError(msg) + def upsert_credential(self, credential: AuthenticationCredential) -> None: """Insert or replace a validated Authentication Record. diff --git a/tests/test_storage.py b/tests/test_storage.py index 3adc6a3e..249ab15b 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -104,7 +104,7 @@ async def test_expired_inactive_oidc_refreshes_once_and_persists( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - """An inactive Server uses its own IDP and persists one shared refresh.""" + """An inactive Science Platform Server uses its IDP and persists refresh.""" config_path = tmp_path / "config.yaml" monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) config = _config( @@ -122,7 +122,7 @@ async def test_expired_inactive_oidc_refreshes_once_and_persists( "expires_at": 9_999_999_999.0, } ) - monkeypatch.setattr("canfar._storage.oidc.refresh", refresh) + monkeypatch.setattr("canfar.client.oidc.refresh", refresh) monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) async with _vospace_source("archive")() as filesystem: @@ -141,6 +141,25 @@ async def test_expired_inactive_oidc_refreshes_once_and_persists( assert saved.token.access.get_secret_value() == "new-access-secret" +@pytest.mark.asyncio +async def test_valid_saved_oidc_access_token_is_reused( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A current saved OIDC Authentication Record needs no refresh.""" + config_path = tmp_path / "config.yaml" + monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) + _config(credential=oidc_credential("inactive", access="current-token")).save() + refresh = AsyncMock() + monkeypatch.setattr("canfar.client.oidc.refresh", refresh) + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with _vospace_source("archive")() as filesystem: + assert filesystem.kwargs["token"] == "current-token" + + refresh.assert_not_awaited() + + @pytest.mark.asyncio async def test_saved_x509_is_validated_before_construction( monkeypatch: pytest.MonkeyPatch, @@ -157,7 +176,7 @@ def inspect_certificate(path: Path) -> dict[str, object]: inspect(path) return {"path": path.as_posix(), "expiry": 9_999_999_999.0} - monkeypatch.setattr("canfar._storage.x509.inspect", inspect_certificate) + monkeypatch.setattr("canfar.client.x509.inspect", inspect_certificate) monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) async with _vospace_source("archive")() as filesystem: @@ -167,7 +186,7 @@ def inspect_certificate(path: Path) -> dict[str, object]: @pytest.mark.asyncio -async def test_runtime_x509_overrides_saved_oidc( +async def test_runtime_x509_overrides_saved_authentication_record( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: @@ -177,11 +196,11 @@ async def test_runtime_x509_overrides_saved_oidc( monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) _config(credential=oidc_credential("inactive")).save() monkeypatch.setattr( - "canfar._storage.x509.inspect", + "canfar.client.x509.inspect", lambda path: {"path": path.as_posix(), "expiry": 9_999_999_999.0}, ) valid = Mock(return_value=certificate.as_posix()) - monkeypatch.setattr("canfar._storage.x509.valid", valid) + monkeypatch.setattr("canfar.client.x509.valid", valid) monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) async with _vospace_source("archive", certificate=certificate)() as filesystem: @@ -190,6 +209,31 @@ async def test_runtime_x509_overrides_saved_oidc( valid.assert_called_once_with(certificate) +@pytest.mark.asyncio +async def test_invalid_saved_x509_fails_before_vospace( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """An invalid X.509 Authentication Record fails with a clean login hint.""" + config_path = tmp_path / "config.yaml" + certificate = tmp_path / "invalid.pem" + monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) + _config(credential=x509_credential("inactive", path=certificate)).save() + monkeypatch.setattr( + "canfar.client.x509.inspect", + Mock(side_effect=ValueError("certificate parse detail")), + ) + constructor = Mock() + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", constructor) + + with pytest.raises(AuthContextError, match="canfar login") as exc_info: + async with _vospace_source("archive")(): + pass + + assert "certificate parse detail" not in str(exc_info.value) + constructor.assert_not_called() + + @pytest.mark.asyncio @pytest.mark.parametrize("exit_kind", ["error", "cancel"]) async def test_source_closes_on_failure_and_cancellation( @@ -234,7 +278,7 @@ async def test_unrefreshable_oidc_fails_secret_safe_before_vospace( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - """Bad saved auth reports a login hint without exposing secret values.""" + """Bad saved Authentication reports a hint without exposing secret values.""" config_path = tmp_path / "config.yaml" monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) _config( @@ -257,3 +301,22 @@ async def test_unrefreshable_oidc_fails_secret_safe_before_vospace( assert "old-access-secret" not in message assert "refresh-secret" not in message constructor.assert_not_called() + + +@pytest.mark.asyncio +async def test_empty_saved_oidc_token_fails_cleanly( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """An empty saved token cannot fall through to certificate construction.""" + config_path = tmp_path / "config.yaml" + monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) + _config(credential=oidc_credential("inactive", access="")).save() + constructor = Mock() + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", constructor) + + with pytest.raises(AuthContextError, match="canfar login"): + async with _vospace_source("archive")(): + pass + + constructor.assert_not_called() From 878f2a5e999edf8c245776afa496a0bcfed57b3e Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 15:34:28 -0700 Subject: [PATCH 08/47] test(storage): isolate authenticated source config --- tests/test_storage.py | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/tests/test_storage.py b/tests/test_storage.py index 249ab15b..bf482c52 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -63,14 +63,19 @@ def _config( ) +@pytest.fixture(autouse=True) +def _isolate_configuration(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Isolate every source test from the developer's configuration.""" + config_path = tmp_path / "config.yaml" + monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) + + @pytest.mark.asyncio async def test_source_reloads_config_and_runtime_token_wins( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: """Entry reloads endpoint state and keeps token-over-certificate precedence.""" - config_path = tmp_path / "config.yaml" - monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) config = _config(credential=oidc_credential("inactive")) config.save() source = _vospace_source( @@ -102,11 +107,8 @@ async def test_source_reloads_config_and_runtime_token_wins( @pytest.mark.asyncio async def test_expired_inactive_oidc_refreshes_once_and_persists( monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, ) -> None: """An inactive Science Platform Server uses its IDP and persists refresh.""" - config_path = tmp_path / "config.yaml" - monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) config = _config( credential=oidc_credential( "inactive", @@ -144,11 +146,8 @@ async def test_expired_inactive_oidc_refreshes_once_and_persists( @pytest.mark.asyncio async def test_valid_saved_oidc_access_token_is_reused( monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, ) -> None: """A current saved OIDC Authentication Record needs no refresh.""" - config_path = tmp_path / "config.yaml" - monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) _config(credential=oidc_credential("inactive", access="current-token")).save() refresh = AsyncMock() monkeypatch.setattr("canfar.client.oidc.refresh", refresh) @@ -166,9 +165,7 @@ async def test_saved_x509_is_validated_before_construction( tmp_path: Path, ) -> None: """Saved X.509 material becomes only an inspected literal certfile path.""" - config_path = tmp_path / "config.yaml" certificate = tmp_path / "saved.pem" - monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) _config(credential=x509_credential("inactive", path=certificate)).save() inspect = Mock() @@ -191,9 +188,7 @@ async def test_runtime_x509_overrides_saved_authentication_record( tmp_path: Path, ) -> None: """A validated runtime certificate wins over the saved Authentication Record.""" - config_path = tmp_path / "config.yaml" certificate = tmp_path / "runtime.pem" - monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) _config(credential=oidc_credential("inactive")).save() monkeypatch.setattr( "canfar.client.x509.inspect", @@ -215,9 +210,7 @@ async def test_invalid_saved_x509_fails_before_vospace( tmp_path: Path, ) -> None: """An invalid X.509 Authentication Record fails with a clean login hint.""" - config_path = tmp_path / "config.yaml" certificate = tmp_path / "invalid.pem" - monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) _config(credential=x509_credential("inactive", path=certificate)).save() monkeypatch.setattr( "canfar.client.x509.inspect", @@ -239,11 +232,8 @@ async def test_invalid_saved_x509_fails_before_vospace( async def test_source_closes_on_failure_and_cancellation( exit_kind: str, monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, ) -> None: """Context exit always closes a yielded filesystem.""" - config_path = tmp_path / "config.yaml" - monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) _config(credential=oidc_credential("inactive")).save() filesystems: list[_Filesystem] = [] @@ -276,11 +266,8 @@ async def use_source() -> None: @pytest.mark.asyncio async def test_unrefreshable_oidc_fails_secret_safe_before_vospace( monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, ) -> None: - """Bad saved Authentication reports a hint without exposing secret values.""" - config_path = tmp_path / "config.yaml" - monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) + """Bad saved Authentication Record reports a secret-safe login hint.""" _config( credential=oidc_credential( "inactive", @@ -306,11 +293,8 @@ async def test_unrefreshable_oidc_fails_secret_safe_before_vospace( @pytest.mark.asyncio async def test_empty_saved_oidc_token_fails_cleanly( monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, ) -> None: """An empty saved token cannot fall through to certificate construction.""" - config_path = tmp_path / "config.yaml" - monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) _config(credential=oidc_credential("inactive", access="")).save() constructor = Mock() monkeypatch.setattr(vosfs, "VOSpaceFileSystem", constructor) From d69de914eeeb26ebbce524f6ecedee2284ee92d8 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 15:28:04 -0700 Subject: [PATCH 09/47] feat(discovery): discover primary VOSpace services --- canfar/idp.py | 7 ++ canfar/models/registry.py | 2 + canfar/server.py | 187 +++++++++++++++++++++++++---- canfar/utils/discover.py | 20 +++- canfar/utils/vosi.py | 22 ++++ tests/test_idp.py | 7 ++ tests/test_server.py | 243 +++++++++++++++++++++++++++++++++++++- 7 files changed, 457 insertions(+), 31 deletions(-) diff --git a/canfar/idp.py b/canfar/idp.py index 25f37030..6f715f05 100644 --- a/canfar/idp.py +++ b/canfar/idp.py @@ -19,6 +19,7 @@ class IdpInfo(BaseModel): auth_mode: Default authentication mode for the IDP. registry_url: IVOA registry resource-caps URL for server discovery. registry_name: Registry display name used for server discovery. + preferred_storage_leaf: Preferred primary VOSpace registry URI leaf. dev_registries: Development registry resource-caps URLs keyed by URL. oidc_discovery_url: OIDC discovery URL when ``auth_mode`` is ``oidc``. oidc_issuer: Exact issuer expected from OIDC discovery metadata. @@ -36,6 +37,10 @@ class IdpInfo(BaseModel): default=None, description="Registry display name used for server discovery.", ) + preferred_storage_leaf: str | None = Field( + default=None, + description="Preferred primary VOSpace registry URI leaf.", + ) dev_registries: dict[str, str] = Field( default_factory=dict, description="Development registry resource-caps URLs keyed by URL.", @@ -56,6 +61,7 @@ class IdpInfo(BaseModel): name="Canadian Astronomy Data Centre", auth_mode="x509", registry_name="CADC", + preferred_storage_leaf="arc", registry_url=AnyHttpUrl( "https://ws.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/reg/resource-caps" ), @@ -70,6 +76,7 @@ class IdpInfo(BaseModel): name="SKA Regional Centre Network", auth_mode="oidc", registry_name="SRCNet", + preferred_storage_leaf="cavern", registry_url=AnyHttpUrl("https://spsrc27.iaa.csic.es/reg/resource-caps"), oidc_discovery_url=AnyHttpUrl( "https://ska-iam.stfc.ac.uk/.well-known/openid-configuration" diff --git a/canfar/models/registry.py b/canfar/models/registry.py index e0797480..e40a09a4 100644 --- a/canfar/models/registry.py +++ b/canfar/models/registry.py @@ -25,6 +25,8 @@ class IVOARegistrySearch(BaseModel): } ) + preferred_storage_leaf: str | None = None + names: dict[str, str] = Field( default={ "ivo://canfar.net/src/skaha": "canSRC", diff --git a/canfar/server.py b/canfar/server.py index d788110c..c2c4a687 100644 --- a/canfar/server.py +++ b/canfar/server.py @@ -4,7 +4,7 @@ import asyncio from dataclasses import dataclass -from typing import TYPE_CHECKING, Literal +from typing import Literal from xml.etree.ElementTree import ParseError import httpx @@ -16,22 +16,23 @@ from canfar.errors import ErrorCode, StructuredError from canfar.exceptions.context import AuthContextError, AuthExpiredError from canfar.hooks.httpx.auth import AuthenticationError as HTTPAuthenticationError -from canfar.idp import registry_sources +from canfar.idp import get_idp, registry_sources from canfar.models.config import Configuration from canfar.models.http import ( DEFAULT_SERVER_CORES, DEFAULT_SERVER_GPUS, DEFAULT_SERVER_RAM_GB, Server, + VOSpaceService, ) from canfar.models.registry import IVOARegistrySearch +from canfar.models.registry import Server as RegistryResource from canfar.utils import vosi from canfar.utils.discover import Discover log = get_logger(__name__) -if TYPE_CHECKING: - from canfar.models.registry import Server as DiscoveredServer +_STORAGE_RESOURCE_UNSET = object() class ServerSelectorError(ValueError): @@ -136,6 +137,11 @@ def discover( known = canonical.get(name, known_servers.get(name)) if server.version is None or not server.auths: if known is not None and known.version is not None and known.auths: + if server.storage: + known = known.model_copy( + update={"storage": {**known.storage, **server.storage}}, + deep=True, + ) canonical[name] = known continue merged_server = server @@ -144,6 +150,8 @@ def discover( include={"idp", "name", "uri", "url", "version", "auths"}, exclude_none=True, ) + if server.storage: + updates["storage"] = {**known.storage, **server.storage} merged_server = known.model_copy(update=updates, deep=True) canonical[name] = merged_server @@ -397,7 +405,10 @@ async def _discover_for_idp( ServerDiscoveryError: If registry retrieval fails. """ sources = registry_sources(idp, include_dev=dev) - search = IVOARegistrySearch(registries=sources) + search = IVOARegistrySearch( + registries=sources, + preferred_storage_leaf=get_idp(idp).preferred_storage_leaf, + ) async with Discover(search, timeout=timeout) as discovery: registries = await asyncio.gather( *(discovery.fetch(url, name) for url, name in sources.items()) @@ -412,12 +423,21 @@ async def _discover_for_idp( msg = f"Failed to discover servers for IDP '{idp}': {errors}" raise ServerDiscoveryError(msg) - endpoints = [] + resources = [] for registry in successful_registries: - endpoints.extend(discovery.extract(registry, dev=dev)) + resources.extend(discovery.extract(registry, dev=dev)) + endpoints = [ + resource for resource in resources if resource.uri.endswith("/skaha") + ] if not endpoints: return [] + storage_by_namespace = { + (resource.registry, _registry_namespace(resource.uri)): resource + for resource in resources + if resource.uri.endswith(f"/{search.preferred_storage_leaf}") + } + checked = await asyncio.gather( *(discovery.check(endpoint) for endpoint in endpoints) ) @@ -427,6 +447,9 @@ async def _discover_for_idp( idp, config=config, timeout=timeout, + storage_resource=storage_by_namespace.get( + (endpoint.registry, _registry_namespace(endpoint.uri)) + ), ) for endpoint in checked if endpoint.status == 200 @@ -440,12 +463,18 @@ def _host_slug(uri: AnyUrl) -> str | None: return uri.host.replace(".", "-") +def _registry_namespace(uri: str) -> str: + """Return the IVOA registry URI namespace before its resource leaf.""" + return uri.rpartition("/")[0] + + def _discovered_to_server( - endpoint: DiscoveredServer, + endpoint: RegistryResource, idp: str, *, config: Configuration | None = None, timeout: int = 2, + storage_resource: RegistryResource | None = None, ) -> Server: """Convert a registry discovery record to a persisted HTTP server model. @@ -457,6 +486,7 @@ def _discovered_to_server( idp: Canonical IDP key. config: Configuration whose Authentication Record authorizes enrichment. timeout: HTTP timeout in seconds for VOSI capabilities requests. + storage_resource: Same-namespace preferred VOSpace registry record, if any. Returns: Server: Persisted server model with capabilities metadata when available. @@ -473,6 +503,7 @@ def _discovered_to_server( config=config, strict=False, timeout=timeout, + storage_resource=storage_resource, ) @@ -483,6 +514,7 @@ def enrich( authentication_idp: str | None = None, strict: bool = True, timeout: int = 2, + storage_resource: RegistryResource | None | object = _STORAGE_RESOURCE_UNSET, ) -> Server: """Return a validated Server enriched from its VOSI capabilities. @@ -497,6 +529,9 @@ def enrich( cannot be retrieved or parsed. Discovery uses non-strict mode so one malformed endpoint does not abort listing for an IDP. timeout: HTTP timeout in seconds for VOSI capabilities requests. + storage_resource: Retained same-namespace VOSpace registry record. Passing + ``None`` records that the preferred resource was absent; omitting the + argument leaves storage outside this inspection. Returns: Server: Copy with version and auth modes populated when discoverable. @@ -505,29 +540,33 @@ def enrich( ServerFetchError: If ``strict`` is ``True`` and capabilities cannot be retrieved, parsed, or contain no session capabilities. """ - if server.url is None: - msg = "Server URL is required to inspect capabilities." - raise ServerFetchError(msg) - base_config = config or Configuration() # ty: ignore[missing-argument] active_idp = authentication_idp or server.idp or base_config.active.authentication - try: - from canfar.client import HTTPClient # noqa: PLC0415 - - with HTTPClient( + if storage_resource is not _STORAGE_RESOURCE_UNSET: + server = _enrich_storage( + server, + storage_resource=( + storage_resource + if isinstance(storage_resource, RegistryResource) + else None + ), config=base_config, authentication_idp=active_idp, - url=server.url, + strict=strict, timeout=timeout, - raise_http_errors=False, - ) as client: - request_client = client.client - request_client.headers["Accept"] = "application/xml" - request_client.headers.pop("Content-Type", None) - request_client.headers.pop("X-Skaha-Registry-Auth", None) - response = request_client.get("capabilities") - response.raise_for_status() - capabilities = vosi.capabilities(xml=response.text) + ) + if server.url is None: + msg = "Server URL is required to inspect capabilities." + raise ServerFetchError(msg) + try: + capabilities = vosi.capabilities( + xml=_fetch_capabilities( + server.url, + config=base_config, + authentication_idp=active_idp, + timeout=timeout, + ) + ) except ( httpx.HTTPError, OSError, @@ -589,6 +628,102 @@ def enrich( ) +def _enrich_storage( + server: Server, + *, + storage_resource: RegistryResource | None, + config: Configuration, + authentication_idp: str, + strict: bool, + timeout: int, +) -> Server: + """Validate and attach one retained primary VOSpace registry resource.""" + error: BaseException | None = None + if storage_resource is None: + leaf = get_idp(authentication_idp).preferred_storage_leaf + subject = f"same-namespace '{leaf}' registry record" + error = ValueError( + f"No {subject} found for Science Platform Server '{server.name}'." + ) + else: + subject = storage_resource.uri + try: + xml = _fetch_capabilities( + AnyHttpUrl(storage_resource.url), + config=config, + authentication_idp=authentication_idp, + timeout=timeout, + ) + valid = vosi.is_vospace_service(xml) + except ( + httpx.HTTPError, + OSError, + AuthContextError, + AuthExpiredError, + CertificateError, + HTTPAuthenticationError, + ParseError, + DefusedXmlException, + ValueError, + ) as exc: + error = exc + else: + if not valid: + error = ValueError( + "required VOSpace node capability is missing or malformed" + ) + elif server.name is None: + error = ValueError("Science Platform Server has no Server Name") + + if error is not None: + return _keep_or_raise( + server, + strict=strict, + error=( + f"Failed to inspect VOSpace Service '{subject}' for Science " + f"Platform Server '{server.name}': {error}" + ), + cause=error, + debug="Skipping VOSpace Service %s during discovery: %s", + args=(subject, error), + ) + + assert storage_resource is not None + assert server.name is not None + service = VOSpaceService(uri=storage_resource.uri, url=storage_resource.url) + + return server.model_copy( + update={"storage": {**server.storage, server.name: service}}, + deep=True, + ) + + +def _fetch_capabilities( + url: AnyHttpUrl, + *, + config: Configuration, + authentication_idp: str, + timeout: int, +) -> str: + """Fetch one VOSI capabilities document through the existing HTTP seam.""" + from canfar.client import HTTPClient # noqa: PLC0415 + + with HTTPClient( + config=config, + authentication_idp=authentication_idp, + url=url, + timeout=timeout, + raise_http_errors=False, + ) as client: + request_client = client.client + request_client.headers["Accept"] = "application/xml" + request_client.headers.pop("Content-Type", None) + request_client.headers.pop("X-Skaha-Registry-Auth", None) + response = request_client.get("capabilities") + response.raise_for_status() + return response.text + + def _keep_or_raise( server: Server, *, diff --git a/canfar/utils/discover.py b/canfar/utils/discover.py index baceb574..d8e4ac71 100644 --- a/canfar/utils/discover.py +++ b/canfar/utils/discover.py @@ -3,6 +3,7 @@ from __future__ import annotations import time +from urllib.parse import urlsplit, urlunsplit import httpx from typing_extensions import Self @@ -75,7 +76,7 @@ async def fetch(self, url: str, name: str) -> IVOARegistry: return IVOARegistry(name=name, content="", success=False, error=error_msg) def extract(self, registry: IVOARegistry, dev: bool = False) -> list[Server]: - """Extract capabilities endpoints from registry content.""" + """Extract Science Platform and preferred VOSpace registry records.""" if not registry.success or not registry.content: return [] @@ -89,8 +90,11 @@ def extract(self, registry: IVOARegistry, dev: bool = False) -> list[Server]: uri, url = line.split("=", 1) uri, url = uri.strip(), url.strip() - if url.endswith("/skaha/capabilities") and uri.endswith("/skaha"): - url = url.replace("/capabilities", "") + leaf = uri.rpartition("/")[2] + if leaf in {"skaha", self.config.preferred_storage_leaf}: + url = _without_terminal_capabilities(url) + if url is None: + continue # Apply exclusion filters if not dev and any( word in uri.lower() or word in url.lower() @@ -105,7 +109,7 @@ def extract(self, registry: IVOARegistry, dev: bool = False) -> list[Server]: registry=registry.name, uri=uri, url=url, - name=self.config.names.get(uri), + name=self.config.names.get(uri) if leaf == "skaha" else None, ) endpoints.append(endpoint) @@ -119,3 +123,11 @@ async def check(self, endpoint: Server) -> Server: except httpx.HTTPError: endpoint.status = None return endpoint + + +def _without_terminal_capabilities(url: str) -> str | None: + """Remove only a terminal ``/capabilities`` path component.""" + parsed = urlsplit(url) + if not parsed.path.endswith("/capabilities"): + return None + return urlunsplit(parsed._replace(path=parsed.path.removesuffix("/capabilities"))) diff --git a/canfar/utils/vosi.py b/canfar/utils/vosi.py index fc699972..7d2f1946 100644 --- a/canfar/utils/vosi.py +++ b/canfar/utils/vosi.py @@ -9,6 +9,8 @@ LEGACY_SESSIONS_STDID = "vos://cadc.nrc.ca~vospace/CADC/std/Proc#sessions-1.0" PLATFORM_PREFIX = "http://www.opencadc.org/std/platform#session-" +VOSPACE_NODES_STDID = "ivo://ivoa.net/std/VOSpace/v2.0#nodes" +_XSI_TYPE = "{http://www.w3.org/2001/XMLSchema-instance}type" # Auth mode rename map AUTH_RENAME = { @@ -199,3 +201,23 @@ def capabilities( # noqa: PLR0912 too many branches (clarity) reverse=True, ) return items + + +def is_vospace_service(xml: str) -> bool: + """Return whether VOSI XML exposes the OpenCADC VOSpace node binding.""" + root = ElementTree.fromstring(xml) + for capability in root.findall(".//{*}capability"): + if capability.get("standardID") != VOSPACE_NODES_STDID: + continue + for interface in capability.findall("{*}interface"): + if interface.get("role") != "std": + continue + if interface.get(_XSI_TYPE, "").rsplit(":", 1)[-1] != "ParamHTTP": + continue + if any( + (access.get("use") in {None, "base"}) + and bool((access.text or "").strip()) + for access in interface.findall("{*}accessURL") + ): + return True + return False diff --git a/tests/test_idp.py b/tests/test_idp.py index 938a4c47..d37c5923 100644 --- a/tests/test_idp.py +++ b/tests/test_idp.py @@ -23,6 +23,13 @@ def test_list_idps_returns_idp_info_instances(self) -> None: assert len(idps) == 2 assert all(isinstance(idp, IdpInfo) for idp in idps) + def test_builtin_primary_storage_preferences(self) -> None: + """Each built-in IDP declares its primary VOSpace resource leaf.""" + assert {idp.key: idp.preferred_storage_leaf for idp in list_idps()} == { + "cadc": "arc", + "srcnet": "cavern", + } + class TestGetIdpCadc: """Tests for get_idp() with the cadc entry.""" diff --git a/tests/test_server.py b/tests/test_server.py index c22ee7a1..cd2ef5fe 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -15,7 +15,8 @@ from canfar.models.active import ActiveConfig from canfar.models.auth import OIDCCredential, X509Credential from canfar.models.config import Configuration -from canfar.models.http import Server +from canfar.models.http import Server, VOSpaceService +from canfar.models.registry import IVOARegistry, IVOARegistrySearch from canfar.models.registry import Server as DiscoveredServer from canfar.server import ( ServerDiscoveryError, @@ -25,11 +26,13 @@ _discovered_to_server, activate, discover, + enrich, use, ) from canfar.server import ( list_servers as server_list, ) +from canfar.utils.discover import Discover from tests.helpers.config import assign_servers if TYPE_CHECKING: @@ -38,6 +41,16 @@ _CADC_URI = "ivo://cadc.nrc.ca/skaha" _CADC_URL = "https://ws-uv.canfar.net/skaha" +_VOSPACE_CAPABILITIES = """ + + + + https://storage.example/arc/nodes + + + +""" + def _http_client_factory( transport: httpx.BaseTransport, @@ -345,6 +358,234 @@ def test_discover_merges_servers_through_public_api(self, tmp_path: Path) -> Non "https://cadc.example/skaha" ) + @pytest.mark.asyncio + async def test_registry_retains_only_skaha_and_preferred_storage_records( + self, + ) -> None: + """CADC discovery retains ARC despite Vault and preserves registry URLs.""" + registry = IVOARegistry( + name="CADC", + content=( + "ivo://cadc.nrc.ca/skaha=" + "https://platform.example/skaha/capabilities\n" + "ivo://cadc.nrc.ca/arc=" + "https://storage.example/custom/capabilities\n" + "ivo://cadc.nrc.ca/vault=" + "https://storage.example/vault/capabilities\n" + "ivo://other.example/arc=" + "https://platform.example/skaha-arc/capabilities" + ), + ) + search = IVOARegistrySearch(preferred_storage_leaf="arc") + + async with Discover(search) as discovery: + resources = discovery.extract(registry) + + assert [(resource.uri, resource.url) for resource in resources] == [ + ("ivo://cadc.nrc.ca/skaha", "https://platform.example/skaha"), + ("ivo://cadc.nrc.ca/arc", "https://storage.example/custom"), + ("ivo://other.example/arc", "https://platform.example/skaha-arc"), + ] + + def test_discover_refreshes_primary_storage_and_preserves_manual_entries( + self, + tmp_path: Path, + ) -> None: + """Rediscovery updates only the generated Server Name storage entry.""" + manual = VOSpaceService( + uri="ivo://cadc.nrc.ca/custom", + url="https://manual.example/custom", + ) + old_primary = VOSpaceService( + uri="ivo://cadc.nrc.ca/arc", + url="https://old.example/arc", + ) + known = _cadc_server( + name="canfar", + storage={"canfar": old_primary, "archive": manual}, + ) + registry_body = "\n".join( + ( + f"{_CADC_URI}={_CADC_URL}/capabilities", + "ivo://cadc.nrc.ca/arc=https://storage.example/arc/capabilities", + "ivo://cadc.nrc.ca/vault=https://storage.example/vault/capabilities", + ) + ) + + def registry_response(request: httpx.Request) -> httpx.Response: + if request.method == "GET": + return httpx.Response(200, text=registry_body, request=request) + return httpx.Response(200, request=request) + + session_capabilities = """ + + + + https://ws-uv.canfar.net/skaha/v1 + + + + + """ + + def capabilities_response(request: httpx.Request) -> httpx.Response: + content = ( + _VOSPACE_CAPABILITIES + if str(request.url) == "https://storage.example/arc/capabilities" + else session_capabilities + ) + return httpx.Response(200, text=content, request=request) + + real_async_client = httpx.AsyncClient + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = _anonymous_config(known) + with ( + patch( + "canfar.utils.discover.httpx.AsyncClient", + side_effect=lambda **_kwargs: real_async_client( + transport=httpx.MockTransport(registry_response) + ), + ), + patch( + "canfar.client.Client", + side_effect=_http_client_factory( + httpx.MockTransport(capabilities_response) + ), + ), + ): + [discovered] = discover("cadc", config=config) + + persisted = Configuration().servers["canfar"] + + assert ( + discovered.storage + == persisted.storage + == { + "canfar": VOSpaceService( + uri="ivo://cadc.nrc.ca/arc", + url="https://storage.example/arc", + ), + "archive": manual, + } + ) + + @pytest.mark.parametrize("mode", ["missing", "malformed", "unreachable"]) + def test_storage_inspection_fail_or_keep_outcomes(self, mode: str) -> None: + """Storage failures are kept non-strict and actionable when strict.""" + storage_resource = ( + None + if mode == "missing" + else DiscoveredServer( + registry="CADC", + uri="ivo://cadc.nrc.ca/arc", + url="https://storage.example/arc", + ) + ) + + def response(request: httpx.Request) -> httpx.Response: + if mode == "unreachable": + message = "storage unavailable" + raise httpx.ConnectError(message, request=request) + return httpx.Response(200, text="", request=request) + + server = _cadc_server() + transport = httpx.MockTransport(response) + with patch("canfar.client.Client", side_effect=_http_client_factory(transport)): + assert ( + enrich( + server, + config=_anonymous_config(), + storage_resource=storage_resource, + strict=False, + ) + == server + ) + + with ( + patch("canfar.client.Client", side_effect=_http_client_factory(transport)), + pytest.raises(ServerFetchError, match="VOSpace Service") as exc_info, + ): + enrich( + server, + config=_anonymous_config(), + storage_resource=storage_resource, + strict=True, + ) + + if mode == "malformed": + assert isinstance(exc_info.value.__cause__, ValueError) + + @pytest.mark.asyncio + async def test_srcnet_pairs_each_server_with_same_namespace_cavern(self) -> None: + """SRCNet Cavern records pair by IVOA namespace, not list position.""" + resources = [ + DiscoveredServer( + registry="SRCNet", + uri="ivo://canfar.net/src/skaha", + url="https://one.example/skaha", + status=200, + name="canSRC", + ), + DiscoveredServer( + registry="SRCNet", + uri="ivo://swesrc.chalmers.se/skaha", + url="https://two.example/skaha", + status=200, + name="sweSRC", + ), + DiscoveredServer( + registry="SRCNet", + uri="ivo://swesrc.chalmers.se/cavern", + url="https://storage.example/two", + ), + DiscoveredServer( + registry="SRCNet", + uri="ivo://canfar.net/src/cavern", + url="https://storage.example/one", + ), + ] + mock_discovery = AsyncMock() + mock_discovery.fetch.return_value = MagicMock(success=True, content="line") + mock_discovery.extract = MagicMock(return_value=resources) + mock_discovery.check = AsyncMock(side_effect=lambda item: item) + mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) + mock_discovery.__aexit__ = AsyncMock(return_value=None) + + def enriched( + server: Server, + *, + storage_resource: DiscoveredServer, + **_kwargs: object, + ) -> Server: + return server.model_copy( + update={ + "version": "v1", + "auths": ["oidc"], + "storage": { + server.name: VOSpaceService( + uri=storage_resource.uri, + url=storage_resource.url, + ) + }, + }, + deep=True, + ) + + with ( + patch("canfar.server.Discover", return_value=mock_discovery), + patch("canfar.server.enrich", side_effect=enriched), + ): + servers = await _discover_for_idp("srcnet") + + assert { + server.name: str(server.storage[server.name].uri) for server in servers + } == { + "canSRC": "ivo://canfar.net/src/cavern", + "sweSRC": "ivo://swesrc.chalmers.se/cavern", + } + @pytest.mark.parametrize( "capabilities_case", [ From 9fcfde93f8779f56a5b8ab7ede7eef642ce0333f Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 15:33:24 -0700 Subject: [PATCH 10/47] test(discovery): isolate storage inspection config --- tests/test_server.py | 48 ++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/tests/test_server.py b/tests/test_server.py index cd2ef5fe..0c819657 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -391,7 +391,7 @@ def test_discover_refreshes_primary_storage_and_preserves_manual_entries( self, tmp_path: Path, ) -> None: - """Rediscovery updates only the generated Server Name storage entry.""" + """Rediscovery updates only the generated Storage Name entry.""" manual = VOSpaceService( uri="ivo://cadc.nrc.ca/custom", url="https://manual.example/custom", @@ -472,7 +472,9 @@ def capabilities_response(request: httpx.Request) -> httpx.Response: ) @pytest.mark.parametrize("mode", ["missing", "malformed", "unreachable"]) - def test_storage_inspection_fail_or_keep_outcomes(self, mode: str) -> None: + def test_storage_inspection_fail_or_keep_outcomes( + self, mode: str, tmp_path: Path + ) -> None: """Storage failures are kept non-strict and actionable when strict.""" storage_resource = ( None @@ -492,27 +494,35 @@ def response(request: httpx.Request) -> httpx.Response: server = _cadc_server() transport = httpx.MockTransport(response) - with patch("canfar.client.Client", side_effect=_http_client_factory(transport)): - assert ( + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = _anonymous_config() + with patch( + "canfar.client.Client", side_effect=_http_client_factory(transport) + ): + assert ( + enrich( + server, + config=config, + storage_resource=storage_resource, + strict=False, + ) + == server + ) + + with ( + patch( + "canfar.client.Client", + side_effect=_http_client_factory(transport), + ), + pytest.raises(ServerFetchError, match="VOSpace Service") as exc_info, + ): enrich( server, - config=_anonymous_config(), + config=config, storage_resource=storage_resource, - strict=False, + strict=True, ) - == server - ) - - with ( - patch("canfar.client.Client", side_effect=_http_client_factory(transport)), - pytest.raises(ServerFetchError, match="VOSpace Service") as exc_info, - ): - enrich( - server, - config=_anonymous_config(), - storage_resource=storage_resource, - strict=True, - ) if mode == "malformed": assert isinstance(exc_info.value.__cause__, ValueError) From 29a49dcb7c019047bb5c0ee091a650ae852586ab Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 15:57:21 -0700 Subject: [PATCH 11/47] fix(storage): preserve environment credential precedence --- canfar/_storage.py | 17 ++++++++++------ tests/test_storage.py | 46 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/canfar/_storage.py b/canfar/_storage.py index fd7ded8d..e0cd0149 100644 --- a/canfar/_storage.py +++ b/canfar/_storage.py @@ -3,7 +3,7 @@ from __future__ import annotations from contextlib import asynccontextmanager -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from canfar.client import HTTPClient from canfar.exceptions.context import AuthContextError @@ -32,12 +32,17 @@ async def source() -> AsyncIterator[AbstractFileSystem]: config = Configuration() endpoint, idp = config._resolve_storage(storage_name) # noqa: SLF001 try: + client_kwargs: dict[str, Any] = { + "config": config, + "authentication_idp": idp, + "url": endpoint, + } + if token is not None: + client_kwargs["token"] = token + if certificate is not None: + client_kwargs["certificate"] = certificate client = HTTPClient( - config=config, - authentication_idp=idp, - url=endpoint, - token=token, - certificate=certificate, + **client_kwargs, ) token_value, certfile = await client._materialize_credentials() # noqa: SLF001 except (KeyError, OSError, TypeError, ValueError): diff --git a/tests/test_storage.py b/tests/test_storage.py index bf482c52..60cb5039 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -104,6 +104,52 @@ async def test_source_reloads_config_and_runtime_token_wins( assert filesystem.closed is True +@pytest.mark.asyncio +async def test_environment_token_preserves_runtime_precedence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The source leaves an omitted token open to HTTPClient environment settings.""" + _config(credential=x509_credential("inactive")).save() + monkeypatch.delenv("CANFAR_CERTIFICATE", raising=False) + monkeypatch.setenv("CANFAR_TOKEN", "environment-token") + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with _vospace_source("archive")() as filesystem: + assert filesystem.kwargs == { + "token": "environment-token", + "asynchronous": True, + "skip_instance_cache": True, + } + + +@pytest.mark.asyncio +async def test_environment_certificate_preserves_runtime_precedence( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The source leaves an omitted certificate open to settings sources.""" + certificate = tmp_path / "environment.pem" + _config(credential=oidc_credential("inactive")).save() + monkeypatch.delenv("CANFAR_TOKEN", raising=False) + monkeypatch.setenv("CANFAR_CERTIFICATE", certificate.as_posix()) + monkeypatch.setattr( + "canfar.client.x509.inspect", + lambda path: {"path": path.as_posix(), "expiry": 9_999_999_999.0}, + ) + valid = Mock(return_value=certificate.as_posix()) + monkeypatch.setattr("canfar.client.x509.valid", valid) + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with _vospace_source("archive")() as filesystem: + assert filesystem.kwargs == { + "certfile": certificate.as_posix(), + "asynchronous": True, + "skip_instance_cache": True, + } + + valid.assert_called_once_with(certificate) + + @pytest.mark.asyncio async def test_expired_inactive_oidc_refreshes_once_and_persists( monkeypatch: pytest.MonkeyPatch, From c8f09242f07d7153d9eb22ee79a23012ce3d5412 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 15:57:34 -0700 Subject: [PATCH 12/47] fix(discovery): carry storage failures into activation --- canfar/models/config.py | 3 +++ canfar/server.py | 42 ++++++++++++++++++++++--------- tests/test_server.py | 56 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 14 deletions(-) diff --git a/canfar/models/config.py b/canfar/models/config.py index 3e16d46d..50ab545b 100644 --- a/canfar/models/config.py +++ b/canfar/models/config.py @@ -12,6 +12,7 @@ BaseModel, ConfigDict, Field, + PrivateAttr, model_validator, ) @@ -114,6 +115,8 @@ class Configuration(BaseSettings): str_strip_whitespace=True, ) + _storage_discovery_errors: dict[str, str] = PrivateAttr(default_factory=dict) + version: Literal[1] = Field( default=1, description="Configuration schema version.", diff --git a/canfar/server.py b/canfar/server.py index c2c4a687..ca60023e 100644 --- a/canfar/server.py +++ b/canfar/server.py @@ -433,7 +433,7 @@ async def _discover_for_idp( return [] storage_by_namespace = { - (resource.registry, _registry_namespace(resource.uri)): resource + _registry_namespace(resource.uri): resource for resource in resources if resource.uri.endswith(f"/{search.preferred_storage_leaf}") } @@ -448,7 +448,7 @@ async def _discover_for_idp( config=config, timeout=timeout, storage_resource=storage_by_namespace.get( - (endpoint.registry, _registry_namespace(endpoint.uri)) + _registry_namespace(endpoint.uri) ), ) for endpoint in checked @@ -525,9 +525,11 @@ def enrich( Authentication or Server Selection. authentication_idp: Optional Authentication Record selector. Defaults to the Server IDP, then the active Authentication. - strict: When ``False``, return the original server if capabilities - cannot be retrieved or parsed. Discovery uses non-strict mode so - one malformed endpoint does not abort listing for an IDP. + strict: When ``False``, keep usable registry and existing storage data + when session or storage capabilities cannot be retrieved or parsed. + Other successful enrichment may still be returned, so the result can + be partial. Discovery uses non-strict mode so one malformed endpoint + does not abort listing for an IDP. timeout: HTTP timeout in seconds for VOSI capabilities requests. storage_resource: Retained same-namespace VOSpace registry record. Passing ``None`` records that the preferred resource was absent; omitting the @@ -542,6 +544,14 @@ def enrich( """ base_config = config or Configuration() # ty: ignore[missing-argument] active_idp = authentication_idp or server.idp or base_config.active.authentication + if ( + strict + and storage_resource is _STORAGE_RESOURCE_UNSET + and server.name in base_config._storage_discovery_errors # noqa: SLF001 + ): + raise ServerFetchError( + base_config._storage_discovery_errors[server.name] # noqa: SLF001 + ) if storage_resource is not _STORAGE_RESOURCE_UNSET: server = _enrich_storage( server, @@ -607,7 +617,7 @@ def enrich( ) try: - return Server.model_validate( + enriched = Server.model_validate( { **server.model_dump(mode="python"), "url": primary["baseurl"], @@ -626,6 +636,8 @@ def enrich( ), args=(server.url, exc), ) + else: + return enriched def _enrich_storage( @@ -676,26 +688,32 @@ def _enrich_storage( error = ValueError("Science Platform Server has no Server Name") if error is not None: - return _keep_or_raise( + message = ( + f"Failed to inspect VOSpace Service '{subject}' for Science " + f"Platform Server '{server.name}': {error}" + ) + kept = _keep_or_raise( server, strict=strict, - error=( - f"Failed to inspect VOSpace Service '{subject}' for Science " - f"Platform Server '{server.name}': {error}" - ), + error=message, cause=error, debug="Skipping VOSpace Service %s during discovery: %s", args=(subject, error), ) + if server.name is not None: + config._storage_discovery_errors[server.name] = message # noqa: SLF001 + return kept assert storage_resource is not None assert server.name is not None service = VOSpaceService(uri=storage_resource.uri, url=storage_resource.url) - return server.model_copy( + enriched = server.model_copy( update={"storage": {**server.storage, server.name: service}}, deep=True, ) + config._storage_discovery_errors.pop(server.name, None) # noqa: SLF001 + return enriched def _fetch_capabilities( diff --git a/tests/test_server.py b/tests/test_server.py index 0c819657..5130670f 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -527,6 +527,58 @@ def response(request: httpx.Request) -> httpx.Response: if mode == "malformed": assert isinstance(exc_info.value.__cause__, ValueError) + def test_activation_surfaces_storage_failure_retained_by_discovery(self) -> None: + """Real non-strict discovery carries its storage error into activation.""" + endpoint = DiscoveredServer( + registry="CADC source", + uri=_CADC_URI, + url=_CADC_URL, + status=200, + name="CADC-CANFAR", + ) + mock_discovery = AsyncMock() + mock_discovery.fetch.return_value = MagicMock(success=True, content="line") + mock_discovery.extract = MagicMock(return_value=[endpoint]) + mock_discovery.check = AsyncMock(side_effect=lambda item: item) + mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) + mock_discovery.__aexit__ = AsyncMock(return_value=None) + session_capabilities = """ + + + + https://ws-uv.canfar.net/skaha/v1 + + + + + """ + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, + text=session_capabilities, + request=request, + ) + ) + config = _anonymous_config() + + with ( + patch("canfar.server.Discover", return_value=mock_discovery), + patch( + "canfar.client.Client", + side_effect=_http_client_factory(transport), + ), + ): + [discovered] = discover("cadc", config=config, save=False) + with pytest.raises( + ServerFetchError, + match="same-namespace 'arc' registry record", + ): + activate("cadc", _CADC_URI, config=config) + + assert discovered.version == "v1" + assert discovered.storage == {} + @pytest.mark.asyncio async def test_srcnet_pairs_each_server_with_same_namespace_cavern(self) -> None: """SRCNet Cavern records pair by IVOA namespace, not list position.""" @@ -546,12 +598,12 @@ async def test_srcnet_pairs_each_server_with_same_namespace_cavern(self) -> None name="sweSRC", ), DiscoveredServer( - registry="SRCNet", + registry="SRCNet mirror B", uri="ivo://swesrc.chalmers.se/cavern", url="https://storage.example/two", ), DiscoveredServer( - registry="SRCNet", + registry="SRCNet mirror A", uri="ivo://canfar.net/src/cavern", url="https://storage.example/one", ), From 7193c6958b74c4b9135ee9d28f544f0b841d2cde Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 16:09:30 -0700 Subject: [PATCH 13/47] feat(data): embed upstream fsspec commands --- canfar/cli/data.py | 85 +++++++++++++++ canfar/cli/main.py | 10 +- pyproject.toml | 4 - tests/test_cli_data.py | 181 ++++++++++++++++++++++++++++++++ tests/test_data_dependencies.py | 22 ++++ uv.lock | 13 +-- 6 files changed, 301 insertions(+), 14 deletions(-) create mode 100644 canfar/cli/data.py create mode 100644 tests/test_cli_data.py create mode 100644 tests/test_data_dependencies.py diff --git a/canfar/cli/data.py b/canfar/cli/data.py new file mode 100644 index 00000000..e3324052 --- /dev/null +++ b/canfar/cli/data.py @@ -0,0 +1,85 @@ +"""Mount the upstream data command application.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING + +import typer +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper +from fsspec.implementations.local import LocalFileSystem +from fsspec_cli import App +from typer.core import TyperGroup +from typer.main import get_group + +from canfar._storage import _vospace_source +from canfar.models.config import Configuration + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from fsspec.spec import AbstractFileSystem + from fsspec_cli import AsyncFilesystemSource + from typer._click.core import Command, Context + +_DATA_GROUP_META_KEY = "canfar.data_group" + + +@asynccontextmanager +async def _local_source() -> AsyncIterator[AbstractFileSystem]: + """Yield a fresh asynchronous wrapper around the local filesystem.""" + yield AsyncFileSystemWrapper( + LocalFileSystem(skip_instance_cache=True), + asynchronous=True, + ) + + +def _sources() -> dict[str, AsyncFilesystemSource]: + """Build the mapped sources for one data command invocation.""" + config = Configuration() + sources = { + storage_name: _vospace_source(storage_name) + for server in config.servers.values() + for storage_name in server.storage + } + sources["local"] = _local_source + return sources + + +def _upstream_group() -> TyperGroup: + """Build the released upstream application with CANFAR policy.""" + return get_group( + App( + _sources(), + capabilities={"recursion": {"copy": True, "remove": False}}, + ).typer_app + ) + + +class _DataGroup(TyperGroup): + """Resolve the embedded app lazily so imports perform no configuration I/O.""" + + @staticmethod + def _delegate(ctx: Context) -> TyperGroup: + group = ctx.meta.get(_DATA_GROUP_META_KEY) + if group is None: + group = _upstream_group() + ctx.meta[_DATA_GROUP_META_KEY] = group + assert isinstance(group, TyperGroup) + return group + + def list_commands(self, ctx: Context) -> list[str]: + """List the unchanged upstream commands.""" + return self._delegate(ctx).list_commands(ctx) + + def get_command(self, ctx: Context, cmd_name: str) -> Command | None: + """Resolve an unchanged upstream command.""" + return self._delegate(ctx).get_command(ctx, cmd_name) + + +data = typer.Typer( + cls=_DataGroup, + help="Operate on configured data sources.", + add_completion=False, + no_args_is_help=True, +) diff --git a/canfar/cli/main.py b/canfar/cli/main.py index 7e41834d..a7c6f0c0 100644 --- a/canfar/cli/main.py +++ b/canfar/cli/main.py @@ -11,6 +11,7 @@ from canfar.cli.auth import auth from canfar.cli.config import config from canfar.cli.create import create +from canfar.cli.data import data from canfar.cli.delete import delete from canfar.cli.events import events from canfar.cli.image import image @@ -115,7 +116,8 @@ def warning_writer(error: StructuredError) -> None: if ctx.invoked_subcommand is None: get_console().print(ctx.get_help()) raise typer.Exit(0) - set_before_command(ctx, _emit_banner_for_command) + if ctx.invoked_subcommand != "data": + set_before_command(ctx, _emit_banner_for_command) cli: typer.Typer = typer.Typer( @@ -161,6 +163,12 @@ def warning_writer(error: StructuredError) -> None: rich_help_panel="Auth Management", ) +cli.add_typer( + data, + name="data", + rich_help_panel="Data Management", +) + cli.add_typer( create, no_args_is_help=True, diff --git a/pyproject.toml b/pyproject.toml index 2dbe86b1..0532e437 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,10 +59,6 @@ dependencies = [ "rich>=13.9.4", "segno>=1.6.6", "typer>=0.16.0", -] - -[project.optional-dependencies] -data = [ "vosfs @ git+https://github.com/shinybrar/vosfs@v0.6.0", "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.5.0#subdirectory=src/fsspec-cli", ] diff --git a/tests/test_cli_data.py b/tests/test_cli_data.py new file mode 100644 index 00000000..0440d73b --- /dev/null +++ b/tests/test_cli_data.py @@ -0,0 +1,181 @@ +"""Tests for the embedded upstream data command group.""" + +from __future__ import annotations + +import importlib +import sys +from contextlib import asynccontextmanager +from functools import partial +from types import SimpleNamespace +from typing import TYPE_CHECKING +from unittest.mock import patch + +import pytest +import typer +from typer.testing import CliRunner + +import canfar.cli.data as data_cli +from canfar.cli.main import cli + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Mapping + from pathlib import Path + + from fsspec.spec import AbstractFileSystem + from fsspec_cli import AsyncFilesystemSource + +runner = CliRunner() + + +def _configuration(*storage_names: str) -> SimpleNamespace: + storage = dict.fromkeys(storage_names, object()) + return SimpleNamespace(servers={"server": SimpleNamespace(storage=storage)}) + + +def test_root_help_advertises_data() -> None: + """The standard CANFAR command surface contains the data group.""" + result = runner.invoke(cli, ["--help"]) + + assert result.exit_code == 0 + assert "data" in result.output + + +def test_upstream_app_receives_configured_sources_and_policy(monkeypatch) -> None: + """CANFAR supplies named sources and policy without extending commands.""" + captured: dict[str, object] = {} + factories: dict[str, object] = {} + + def source_factory(name: str) -> object: + factory = object() + factories[name] = factory + return factory + + class FakeApp: + def __init__( + self, + sources: Mapping[str, AsyncFilesystemSource], + *, + capabilities: object, + ) -> None: + captured["sources"] = sources + captured["capabilities"] = capabilities + self.typer_app = typer.Typer() + + monkeypatch.setattr( + data_cli, + "Configuration", + partial(_configuration, "arc", "cavern"), + ) + monkeypatch.setattr(data_cli, "_vospace_source", source_factory) + monkeypatch.setattr(data_cli, "App", FakeApp) + + data_cli._upstream_group() # noqa: SLF001 + + assert captured["sources"] == { + **factories, + "local": data_cli._local_source, # noqa: SLF001 + } + assert captured["capabilities"] == {"recursion": {"copy": True, "remove": False}} + + +@pytest.mark.asyncio +async def test_local_source_returns_fresh_async_wrappers() -> None: + """Each local source entry owns a fresh asynchronous wrapper.""" + async with data_cli._local_source() as first: # noqa: SLF001 + assert first.asynchronous is True + async with data_cli._local_source() as second: # noqa: SLF001 + assert second.asynchronous is True + + assert first is not second + assert first.sync_fs is not second.sync_fs + + +def test_data_cat_delegates_without_active_server_banner( + monkeypatch, + tmp_path: Path, +) -> None: + """Data stdout belongs byte-for-byte to the upstream command.""" + payload = tmp_path / "payload.txt" + payload.write_text("upstream output\n", encoding="utf-8") + monkeypatch.setattr(data_cli, "Configuration", _configuration) + + result = runner.invoke(cli, ["data", "cat", f"local:{payload}"]) + + assert result.exit_code == 0, result.output + assert result.stdout == "upstream output\n" + + +def test_data_recursive_copy_delegates_to_released_contract( + monkeypatch, + tmp_path: Path, +) -> None: + """The embedded app retains its enabled recursive-copy behavior.""" + source = tmp_path / "source" + source.mkdir() + (source / "payload.txt").write_text("copied", encoding="utf-8") + destination = tmp_path / "destination" + monkeypatch.setattr(data_cli, "Configuration", _configuration) + + result = runner.invoke( + cli, + ["data", "cp", "-R", f"local:{source}", f"local:{destination}"], + ) + + assert result.exit_code == 0, result.output + assert (destination / "payload.txt").read_text(encoding="utf-8") == "copied" + + +def test_data_recursive_remove_is_disabled(monkeypatch, tmp_path: Path) -> None: + """CANFAR keeps upstream recursive removal source-free and disabled.""" + monkeypatch.setattr(data_cli, "Configuration", _configuration) + + result = runner.invoke(cli, ["data", "rm", "-R", f"local:{tmp_path}"]) + + assert result.exit_code == 2 + assert result.stdout == "" + assert result.stderr == "rm: recursive removal disabled by application\n" + + +def test_data_cross_source_move_retains_upstream_rejection(monkeypatch) -> None: + """CANFAR does not reinterpret or implement cross-source move.""" + + @asynccontextmanager + async def unused_source() -> AsyncIterator[AbstractFileSystem]: + msg = "source-free rejection must not acquire filesystems" + raise AssertionError(msg) + yield + + monkeypatch.setattr(data_cli, "Configuration", partial(_configuration, "arc")) + monkeypatch.setattr(data_cli, "_vospace_source", lambda _name: unused_source) + + result = runner.invoke(cli, ["data", "mv", "arc:/a", "local:/b"]) + + assert result.exit_code == 2 + assert result.stdout == "" + assert result.stderr == "mv: cross-source move unsupported\n" + + +@pytest.mark.parametrize("operand", [":/path", "/bare/local/path"]) +def test_data_deprecated_operand_grammar_is_unsupported( + monkeypatch, + operand: str, +) -> None: + """Only the upstream explicit ``name:/absolute/path`` grammar is accepted.""" + monkeypatch.setattr(data_cli, "Configuration", _configuration) + + result = runner.invoke(cli, ["data", "ls", operand]) + + assert result.exit_code != 0 + + +def test_importing_data_module_does_not_load_configuration() -> None: + """Registering the command performs no configuration filesystem I/O.""" + original = sys.modules.pop("canfar.cli.data") + try: + with patch( + "canfar.models.config.Configuration", + side_effect=AssertionError("configuration loaded during import"), + ): + importlib.import_module("canfar.cli.data") + finally: + sys.modules["canfar.cli.data"] = original diff --git a/tests/test_data_dependencies.py b/tests/test_data_dependencies.py new file mode 100644 index 00000000..a46001df --- /dev/null +++ b/tests/test_data_dependencies.py @@ -0,0 +1,22 @@ +"""Package metadata tests for the standard data dependencies.""" + +from __future__ import annotations + +from pathlib import Path + +import tomllib + + +def test_tagged_data_dependencies_are_standard_dependencies() -> None: + """A standard CANFAR install includes both immutable upstream releases.""" + metadata = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + + assert ( + "vosfs @ git+https://github.com/shinybrar/vosfs@v0.6.0" + in metadata["project"]["dependencies"] + ) + assert ( + "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.5.0" + "#subdirectory=src/fsspec-cli" in metadata["project"]["dependencies"] + ) + assert "data" not in metadata["project"].get("optional-dependencies", {}) diff --git a/uv.lock b/uv.lock index 63835be4..4ffeb2b5 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14'", @@ -118,6 +118,7 @@ dependencies = [ { name = "cadcutils" }, { name = "click" }, { name = "defusedxml" }, + { name = "fsspec-cli" }, { name = "httpx", extra = ["http2"] }, { name = "humanize" }, { name = "pydantic" }, @@ -127,11 +128,6 @@ dependencies = [ { name = "rich" }, { name = "segno" }, { name = "typer" }, -] - -[package.optional-dependencies] -data = [ - { name = "fsspec-cli" }, { name = "vosfs" }, ] @@ -164,7 +160,7 @@ requires-dist = [ { name = "cadcutils", specifier = ">=1.5.4" }, { name = "click", specifier = ">=8.4.1" }, { name = "defusedxml", specifier = ">=0.7.1" }, - { name = "fsspec-cli", marker = "extra == 'data'", git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.5.0" }, + { name = "fsspec-cli", git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.5.0" }, { name = "httpx", extras = ["http2"], specifier = ">=0.28.1" }, { name = "humanize", specifier = ">=4.12.3" }, { name = "pydantic", specifier = ">=2.9.2" }, @@ -174,9 +170,8 @@ requires-dist = [ { name = "rich", specifier = ">=13.9.4" }, { name = "segno", specifier = ">=1.6.6" }, { name = "typer", specifier = ">=0.16.0" }, - { name = "vosfs", marker = "extra == 'data'", git = "https://github.com/shinybrar/vosfs?rev=v0.6.0" }, + { name = "vosfs", git = "https://github.com/shinybrar/vosfs?rev=v0.6.0" }, ] -provides-extras = ["data"] [package.metadata.requires-dev] dev = [ From 0ef59c98f71f219e6a2c544b83a9e04374b9021b Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 16:12:09 -0700 Subject: [PATCH 14/47] docs(data): remove obsolete optional install --- docs/cli/quick-start.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/cli/quick-start.md b/docs/cli/quick-start.md index 2d28f03d..dcfb005e 100644 --- a/docs/cli/quick-start.md +++ b/docs/cli/quick-start.md @@ -8,12 +8,6 @@ Create a notebook Session from a terminal, open it, inspect it, and clean it up. pip install canfar --upgrade ``` -Install the pinned optional storage dependencies when you need data support: - -```bash -pip install "canfar[data]" --upgrade -``` - ## 2. Log in ```bash From f8ef7adf0af491c7e6dd5c602016fde99068d928 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 16:12:52 -0700 Subject: [PATCH 15/47] test(data): cover mapped source delegation --- tests/test_cli_data.py | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/test_cli_data.py b/tests/test_cli_data.py index 0440d73b..bdceb476 100644 --- a/tests/test_cli_data.py +++ b/tests/test_cli_data.py @@ -105,6 +105,29 @@ def test_data_cat_delegates_without_active_server_banner( assert result.stdout == "upstream output\n" +def test_data_long_listing_delegates_unchanged( + monkeypatch, + tmp_path: Path, +) -> None: + """The documented ``ls -lh name:/path`` form reaches upstream unchanged.""" + (tmp_path / "payload.txt").write_text("listed", encoding="utf-8") + monkeypatch.setattr( + data_cli, + "Configuration", + partial(_configuration, "canSRC"), + ) + monkeypatch.setattr( + data_cli, + "_vospace_source", + lambda _name: data_cli._local_source, # noqa: SLF001 + ) + + result = runner.invoke(cli, ["data", "ls", "-lh", f"canSRC:{tmp_path}"]) + + assert result.exit_code == 0, result.output + assert "payload.txt" in result.stdout + + def test_data_recursive_copy_delegates_to_released_contract( monkeypatch, tmp_path: Path, @@ -114,11 +137,20 @@ def test_data_recursive_copy_delegates_to_released_contract( source.mkdir() (source / "payload.txt").write_text("copied", encoding="utf-8") destination = tmp_path / "destination" - monkeypatch.setattr(data_cli, "Configuration", _configuration) + monkeypatch.setattr( + data_cli, + "Configuration", + partial(_configuration, "canSRC"), + ) + monkeypatch.setattr( + data_cli, + "_vospace_source", + lambda _name: data_cli._local_source, # noqa: SLF001 + ) result = runner.invoke( cli, - ["data", "cp", "-R", f"local:{source}", f"local:{destination}"], + ["data", "cp", "-R", f"local:{source}", f"canSRC:{destination}"], ) assert result.exit_code == 0, result.output @@ -170,6 +202,8 @@ def test_data_deprecated_operand_grammar_is_unsupported( def test_importing_data_module_does_not_load_configuration() -> None: """Registering the command performs no configuration filesystem I/O.""" + package = sys.modules["canfar.cli"] + original_attribute = package.data original = sys.modules.pop("canfar.cli.data") try: with patch( @@ -179,3 +213,4 @@ def test_importing_data_module_does_not_load_configuration() -> None: importlib.import_module("canfar.cli.data") finally: sys.modules["canfar.cli.data"] = original + package.data = original_attribute From d12b197ce110c5b7dc00ea6eace76c51cef3bf98 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 16:17:36 -0700 Subject: [PATCH 16/47] docs(data): explain explicit cross-source copy --- docs/cli/quick-start.md | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/docs/cli/quick-start.md b/docs/cli/quick-start.md index dcfb005e..24d8126d 100644 --- a/docs/cli/quick-start.md +++ b/docs/cli/quick-start.md @@ -27,7 +27,26 @@ canfar auth show canfar server ls ``` -## 3. Create a notebook +## 3. Work with data + +The standard installation includes data commands. Use a configured Storage Name +or the reserved `local` name with an absolute path: + +```bash +canfar data ls -lh canSRC:/ +canfar data cp local:/absolute/path/file.fits canSRC:/folder/file.fits +``` + +Cross-source `mv` is unsupported. Copy the file, verify the destination, and +then remove the source with a separate command: + +```bash +canfar data cp canSRC:/folder/file.fits other:/folder/file.fits +canfar data ls -lh other:/folder/file.fits +canfar data rm canSRC:/folder/file.fits +``` + +## 4. Create a notebook ```bash canfar create notebook skaha/astroml:latest @@ -40,7 +59,7 @@ registry. Use the full image name when you want to be explicit: canfar create notebook images.canfar.net/skaha/astroml:latest ``` -## 4. Check status +## 5. Check status ```bash canfar ps @@ -53,7 +72,7 @@ Use machine output in scripts: canfar ps --json ``` -## 5. Open the notebook +## 6. Open the notebook ```bash canfar open $(canfar ps -q) @@ -61,14 +80,14 @@ canfar open $(canfar ps -q) Notebook Sessions usually take 60-120 seconds to become reachable. -## 6. Inspect startup events +## 7. Inspect startup events ```bash canfar events $(canfar ps -q) canfar logs $(canfar ps -q) ``` -## 7. Clean up +## 8. Clean up ```bash canfar delete $(canfar ps -q) From 3ec51a356d4da5b05eb6176cd319b398dd823dd6 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 16:28:53 -0700 Subject: [PATCH 17/47] docs(data): use default CADC storage name --- docs/cli/quick-start.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/cli/quick-start.md b/docs/cli/quick-start.md index 24d8126d..6ab6aabf 100644 --- a/docs/cli/quick-start.md +++ b/docs/cli/quick-start.md @@ -33,17 +33,19 @@ The standard installation includes data commands. Use a configured Storage Name or the reserved `local` name with an absolute path: ```bash -canfar data ls -lh canSRC:/ -canfar data cp local:/absolute/path/file.fits canSRC:/folder/file.fits +canfar data ls -lh canfar:/ +canfar data cp local:/absolute/path/file.fits canfar:/folder/file.fits ``` Cross-source `mv` is unsupported. Copy the file, verify the destination, and -then remove the source with a separate command: +then remove the source with a separate command. In this example, `archive` is +a placeholder for a second Storage Name that you configured; replace it with +that Storage Name: ```bash -canfar data cp canSRC:/folder/file.fits other:/folder/file.fits -canfar data ls -lh other:/folder/file.fits -canfar data rm canSRC:/folder/file.fits +canfar data cp canfar:/folder/file.fits archive:/folder/file.fits +canfar data ls -lh archive:/folder/file.fits +canfar data rm canfar:/folder/file.fits ``` ## 4. Create a notebook From f919b3c181dbbcfbc013c0ce2184c70bda976cf0 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 16:35:51 -0700 Subject: [PATCH 18/47] docs(data): document supported transfer workflows --- docs/cli/cli-help.md | 12 ++ docs/cli/data.md | 114 +++++++++++++++++++ docs/presentations/srcnet-june-2026-demo.typ | 11 +- mkdocs.yml | 1 + 4 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 docs/cli/data.md diff --git a/docs/cli/cli-help.md b/docs/cli/cli-help.md index dd4e9ca1..2561eb44 100644 --- a/docs/cli/cli-help.md +++ b/docs/cli/cli-help.md @@ -184,6 +184,18 @@ canfar stats Use `canfar image --help` and `canfar stats --help` for command-specific options. +## Data + +```bash +canfar data ls -lh canfar:/ +canfar data cp local:/absolute/path/file.fits canfar:/folder/file.fits +``` + +Data operands use explicit `Storage-Name:/absolute/path` or +`local:/absolute/path` syntax. See [Data commands](data.md) for recursive copy, +cross-source copy and verification, recursive-removal policy, and the exact +supported boundary. + ## Client configuration ```bash diff --git a/docs/cli/data.md b/docs/cli/data.md new file mode 100644 index 00000000..6ff5bea4 --- /dev/null +++ b/docs/cli/data.md @@ -0,0 +1,114 @@ +# Data commands + +Use `canfar data` to work with configured VOSpace Services and the local +filesystem through the embedded `fsspec-cli` command application. + +## Install and authenticate + +Data commands are included in the standard installation: + +```bash +pip install canfar +canfar login cadc +``` + +CANFAR installs the audited upstream releases as normal dependencies through +immutable Git references: + +- `vosfs @ git+https://github.com/shinybrar/vosfs@v0.6.0` +- `fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.5.0#subdirectory=src/fsspec-cli` + +There is no separate data extra. + +## Address mapped sources + +Every remote source uses its configured Storage Name. The reserved `local` +source addresses the machine where the command runs. Operands always use an +explicit mapped name and absolute path: + +```text +Storage-Name:/absolute/path +local:/absolute/path +``` + +For a default CADC login, the discovered primary Storage Name is `canfar`: + +```bash +canfar data ls -lh canfar:/ +canfar data ls -lh canfar:/folder +``` + +Use `ls -lh` (or `ll -h`) for a human-readable long listing. Standalone +`ls -h` is not supported. Empty `:/path`, bare local paths, `active:/path`, and +the retired `canfar storage` command are not aliases. + +## Copy files and directories + +Copy one file between local and remote sources: + +```bash +canfar data cp local:/absolute/path/file.fits canfar:/folder/file.fits +canfar data cp canfar:/folder/file.fits local:/absolute/path/file.fits +``` + +Recursive copy is enabled for admitted local and remote source pairs: + +```bash +canfar data cp -R local:/absolute/path/dataset canfar:/folder/dataset +``` + +The tagged upstream implementation builds a bounded manifest, copies files +through host-local staging when sources differ, and verifies destination +metadata. Recursive copy is not atomic and does not create a snapshot; inspect +the destination before removing any source data. + +## Move data between sources + +Cross-source `mv` is unsupported. Move a file explicitly by copying it, +verifying the destination, and only then issuing a separate source removal. In +this example, `archive` stands for a second Storage Name that you configured: + +```bash +canfar data cp canfar:/folder/file.fits archive:/folder/file.fits +canfar data ls -lh archive:/folder/file.fits +canfar data rm canfar:/folder/file.fits +``` + +Do not use this sequence as a one-command or atomic move. CANFAR does not +advertise same-source `mv` for current VOSpace sources either. + +Recursive removal is disabled by application policy. `rm -R` exits with status +2 and reports: + +```text +rm: recursive removal disabled by application +``` + +Because directory removal is disabled, directory movement is not a supported +workflow in this release. Any future one-command relocation would be a +separately named, opt-in orchestration feature with stronger destination +verification and residual-state semantics—not portable `mv`. + +## Output and accepted omissions + +Data command stdout belongs to the embedded command; CANFAR does not prepend +the active-Server banner or add JSON/YAML envelopes. Diagnostics are written to +stderr. + +This release intentionally provides no public CANFAR storage Python API, FUSE +mount, signed-URL extension, progress display, confirmation prompt, `:/path` or +bare-path shorthand, `active` alias, `canfar storage` alias, recursive removal, +or cross-source/current-VOSpace `mv` workflow. + +## Audited upstream releases + +CANFAR's integration is based on +[`shinybrar/vosfs` PR #294](https://github.com/shinybrar/vosfs/pull/294), +audited at commit +[`9e5314db4706894d31d54d245392f43b9556cfbb`](https://github.com/shinybrar/vosfs/commit/9e5314db4706894d31d54d245392f43b9556cfbb). +The installed pair is the tagged +[`vosfs` v0.6.0](https://github.com/shinybrar/vosfs/releases/tag/v0.6.0) and +[`fsspec-cli-v0.5.0`](https://github.com/shinybrar/vosfs/releases/tag/fsspec-cli-v0.5.0) +releases. CANFAR tests its composition, configuration, authentication, and +output seams; exhaustive filesystem-command and backend matrices remain in the +upstream project. diff --git a/docs/presentations/srcnet-june-2026-demo.typ b/docs/presentations/srcnet-june-2026-demo.typ index a554f59e..945de9a4 100644 --- a/docs/presentations/srcnet-june-2026-demo.typ +++ b/docs/presentations/srcnet-june-2026-demo.typ @@ -90,17 +90,16 @@ for node in ["canSRC", "sweSRC"]: await session.create(image="skaha/base-notebook:latest", cmd="env") ``` -== Next: storage, same front door +== Data, same front door -Coming in #link("https://github.com/opencadc/canfar/pull/83")[#raw("#83")] — file management over VOSpace, no second login: +File management over named VOSpace Services, using the saved identity: ```bash -canfar server use sweSRC -canfar storage ls -canfar storage mv /data/filename @canSRC:/data/ # Syntax TBD +canfar data ls -lh canSRC:/ +canfar data cp local:/absolute/path/filename canSRC:/data/filename ``` -- Familiar verbs: `ls` · `cp` · `mkdir` · `mv` · `rm` +- Cross-source movement is explicit `cp`, verify, then separate `rm` - Reuses your active identity — *no extra auth* #align(center)[_Your files, your compute, your platform — one door, one login._] diff --git a/mkdocs.yml b/mkdocs.yml index c0e93c09..e8391982 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -259,6 +259,7 @@ nav: - Helpers: client/helpers.md - CLI: - 5-Minute Tutorial: cli/quick-start.md + - Data Commands: cli/data.md - Authentication and Servers: cli/authentication-contexts.md - Logging: cli/logging.md - Reference: cli/cli-help.md From 6b84518665e9d5b2a73fe204d806980774ad0d33 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 16:35:58 -0700 Subject: [PATCH 19/47] test(data): close integration validation gaps --- tests/test_cli_data.py | 22 ++++++++++++++ tests/test_data_smoke.py | 64 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 tests/test_data_smoke.py diff --git a/tests/test_cli_data.py b/tests/test_cli_data.py index bdceb476..1f6dbf08 100644 --- a/tests/test_cli_data.py +++ b/tests/test_cli_data.py @@ -200,6 +200,28 @@ def test_data_deprecated_operand_grammar_is_unsupported( assert result.exit_code != 0 +@pytest.mark.parametrize( + ("arguments", "diagnostic"), + [ + (["storage", "ls"], "No such command 'storage'"), + (["data", "ls", "active:/"], "unknown filesystem"), + (["data", "ls", "-h", "local:/"], "-h: unsupported option"), + ], +) +def test_data_retired_aliases_and_standalone_h_are_unsupported( + monkeypatch, + arguments: list[str], + diagnostic: str, +) -> None: + """Retired host aliases do not widen the upstream mapped grammar.""" + monkeypatch.setattr(data_cli, "Configuration", _configuration) + + result = runner.invoke(cli, arguments) + + assert result.exit_code == 2 + assert diagnostic in result.stderr + + def test_importing_data_module_does_not_load_configuration() -> None: """Registering the command performs no configuration filesystem I/O.""" package = sys.modules["canfar.cli"] diff --git a/tests/test_data_smoke.py b/tests/test_data_smoke.py new file mode 100644 index 00000000..302a0ce3 --- /dev/null +++ b/tests/test_data_smoke.py @@ -0,0 +1,64 @@ +"""Optional authenticated smoke test for the configured CADC VOSpace Service.""" + +from __future__ import annotations + +from typing import NoReturn + +import pytest +from typer.testing import CliRunner + +from canfar.auth import x509 +from canfar.cli.main import cli +from canfar.models.config import Configuration + + +def _skip(reason: str) -> NoReturn: + pytest.skip( + "live data smoke requires Storage Name 'canfar' and a valid saved " + f"Authentication Record/certificate: {reason}" + ) + + +def _require_live_credentials() -> None: + """Skip unless persisted configuration can authenticate the live source.""" + try: + config = Configuration() # ty: ignore[missing-argument] + except Exception: # noqa: BLE001 - optional environment preflight. + _skip("configuration is unavailable") + + try: + _endpoint, idp = config._resolve_storage("canfar") # noqa: SLF001 + credential = config.get_credential(idp) + except (KeyError, ValueError): + _skip("the named service or its Authentication Record is unavailable") + + if credential.mode == "x509": + if credential.path is None: + _skip("the saved X.509 certificate path is missing") + try: + x509.valid(credential.path) + x509.expiry(credential.path) + except (OSError, ValueError): + _skip("the saved X.509 certificate is unavailable or invalid") + return + + access = credential.token.access + has_current_access = ( + access is not None + and bool(access.get_secret_value()) + and not credential.expired + ) + if not has_current_access and not credential.refreshable: + _skip("the saved OIDC credential cannot authenticate or refresh") + + +@pytest.mark.integration +@pytest.mark.slow +def test_canfar_data_lists_live_primary_storage() -> None: + """List the configured live CADC primary VOSpace Service when available.""" + _require_live_credentials() + + result = CliRunner().invoke(cli, ["data", "ls", "-lh", "canfar:/"]) + + assert result.exit_code == 0, result.output + assert not result.stdout.startswith("@") From a8880159b55322886fe774b36ceb67ff6ab4599d Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 16:39:20 -0700 Subject: [PATCH 20/47] docs(data): clarify credential and removal policy --- docs/cli/data.md | 6 +++--- docs/presentations/srcnet-june-2026-demo.typ | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/cli/data.md b/docs/cli/data.md index 6ff5bea4..1e6497d1 100644 --- a/docs/cli/data.md +++ b/docs/cli/data.md @@ -13,7 +13,7 @@ canfar login cadc ``` CANFAR installs the audited upstream releases as normal dependencies through -immutable Git references: +tagged Git references: - `vosfs @ git+https://github.com/shinybrar/vosfs@v0.6.0` - `fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.5.0#subdirectory=src/fsspec-cli` @@ -84,8 +84,8 @@ Recursive removal is disabled by application policy. `rm -R` exits with status rm: recursive removal disabled by application ``` -Because directory removal is disabled, directory movement is not a supported -workflow in this release. Any future one-command relocation would be a +Because recursive directory removal is disabled, directory movement is not a +supported workflow in this release. Any future one-command relocation would be a separately named, opt-in orchestration feature with stronger destination verification and residual-state semantics—not portable `mv`. diff --git a/docs/presentations/srcnet-june-2026-demo.typ b/docs/presentations/srcnet-june-2026-demo.typ index 945de9a4..9370c342 100644 --- a/docs/presentations/srcnet-june-2026-demo.typ +++ b/docs/presentations/srcnet-june-2026-demo.typ @@ -92,7 +92,7 @@ for node in ["canSRC", "sweSRC"]: == Data, same front door -File management over named VOSpace Services, using the saved identity: +File management over named VOSpace Services, using the parent server's saved Authentication Record: ```bash canfar data ls -lh canSRC:/ @@ -100,6 +100,6 @@ canfar data cp local:/absolute/path/filename canSRC:/data/filename ``` - Cross-source movement is explicit `cp`, verify, then separate `rm` -- Reuses your active identity — *no extra auth* +- Uses the named Storage Service's parent-server IDP, even when that server is inactive #align(center)[_Your files, your compute, your platform — one door, one login._] From 5db3df9ec9e8c2ec10c5dd4092bb04775deff143 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 16:40:18 -0700 Subject: [PATCH 21/47] docs(data): clarify source mapping lifecycle --- docs/cli/data.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/cli/data.md b/docs/cli/data.md index 1e6497d1..355ab682 100644 --- a/docs/cli/data.md +++ b/docs/cli/data.md @@ -26,6 +26,12 @@ Every remote source uses its configured Storage Name. The reserved `local` source addresses the machine where the command runs. Operands always use an explicit mapped name and absolute path: +On each CLI invocation, CANFAR rebuilds the source mapping from every Storage +Name on every configured Science Platform Server and always adds `local`. +Active Server Selection does not filter data sources. Each mapped source is a +factory that creates a fresh asynchronous filesystem when the command acquires +it. + ```text Storage-Name:/absolute/path local:/absolute/path From 48306d74eb831f0b2c6dfde246e241aa97b2ef0a Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 16:49:14 -0700 Subject: [PATCH 22/47] docs(data): align VOSpace credential ownership --- docs/presentations/srcnet-june-2026-demo.typ | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/presentations/srcnet-june-2026-demo.typ b/docs/presentations/srcnet-june-2026-demo.typ index 9370c342..b7c6b534 100644 --- a/docs/presentations/srcnet-june-2026-demo.typ +++ b/docs/presentations/srcnet-june-2026-demo.typ @@ -92,7 +92,7 @@ for node in ["canSRC", "sweSRC"]: == Data, same front door -File management over named VOSpace Services, using the parent server's saved Authentication Record: +File management over named VOSpace Services, using their configured Authentication Records: ```bash canfar data ls -lh canSRC:/ @@ -100,6 +100,7 @@ canfar data cp local:/absolute/path/filename canSRC:/data/filename ``` - Cross-source movement is explicit `cp`, verify, then separate `rm` -- Uses the named Storage Service's parent-server IDP, even when that server is inactive +- Each VOSpace Service uses its parent Science Platform Server's IDP +- CANFAR uses that IDP's saved Authentication Record, even when the Science Platform Server is inactive #align(center)[_Your files, your compute, your platform — one door, one login._] From 4288f3d1426c49a21b5cfff0bac7d515bbf1dc4e Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 16:49:21 -0700 Subject: [PATCH 23/47] test(data): prove source rebuilding and freshness --- tests/test_cli_data.py | 64 +++++++++++++++++++++++++++++------------- tests/test_storage.py | 28 ++++++++++++++++++ 2 files changed, 73 insertions(+), 19 deletions(-) diff --git a/tests/test_cli_data.py b/tests/test_cli_data.py index 1f6dbf08..29f3fa5a 100644 --- a/tests/test_cli_data.py +++ b/tests/test_cli_data.py @@ -41,14 +41,36 @@ def test_root_help_advertises_data() -> None: def test_upstream_app_receives_configured_sources_and_policy(monkeypatch) -> None: - """CANFAR supplies named sources and policy without extending commands.""" - captured: dict[str, object] = {} - factories: dict[str, object] = {} + """Every app build maps all configured Servers plus local, not only active.""" + captured_sources: list[dict[str, object]] = [] + captured_capabilities: list[object] = [] + + def source_factory(_name: str) -> object: + return object() + + configurations = iter( + [ + SimpleNamespace( + active=SimpleNamespace(server="first"), + servers={ + "first": SimpleNamespace(storage={"arc": object()}), + "second": SimpleNamespace(storage={"cavern": object()}), + }, + ), + SimpleNamespace( + active=SimpleNamespace(server="second"), + servers={ + "first": SimpleNamespace(storage={"arc": object()}), + "second": SimpleNamespace( + storage={"cavern": object(), "vault": object()} + ), + }, + ), + ] + ) - def source_factory(name: str) -> object: - factory = object() - factories[name] = factory - return factory + def configuration() -> SimpleNamespace: + return next(configurations) class FakeApp: def __init__( @@ -57,25 +79,29 @@ def __init__( *, capabilities: object, ) -> None: - captured["sources"] = sources - captured["capabilities"] = capabilities + captured_sources.append(dict(sources)) + captured_capabilities.append(capabilities) self.typer_app = typer.Typer() - monkeypatch.setattr( - data_cli, - "Configuration", - partial(_configuration, "arc", "cavern"), - ) + monkeypatch.setattr(data_cli, "Configuration", configuration) monkeypatch.setattr(data_cli, "_vospace_source", source_factory) monkeypatch.setattr(data_cli, "App", FakeApp) + data_cli._upstream_group() # noqa: SLF001 data_cli._upstream_group() # noqa: SLF001 - assert captured["sources"] == { - **factories, - "local": data_cli._local_source, # noqa: SLF001 - } - assert captured["capabilities"] == {"recursion": {"copy": True, "remove": False}} + assert [set(sources) for sources in captured_sources] == [ + {"arc", "cavern", "local"}, + {"arc", "cavern", "vault", "local"}, + ] + assert all( + sources["local"] is data_cli._local_source # noqa: SLF001 + for sources in captured_sources + ) + assert captured_capabilities == [ + {"recursion": {"copy": True, "remove": False}}, + {"recursion": {"copy": True, "remove": False}}, + ] @pytest.mark.asyncio diff --git a/tests/test_storage.py b/tests/test_storage.py index 60cb5039..601eff55 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -104,6 +104,34 @@ async def test_source_reloads_config_and_runtime_token_wins( assert filesystem.closed is True +@pytest.mark.asyncio +async def test_source_factory_constructs_fresh_filesystem_per_acquisition( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Repeated acquisition constructs and closes distinct VOSpace filesystems.""" + _config(credential=oidc_credential("inactive", access="current-token")).save() + filesystems: list[_Filesystem] = [] + + def build(endpoint: str, **kwargs: Any) -> _Filesystem: + filesystem = _Filesystem(endpoint, **kwargs) + filesystems.append(filesystem) + return filesystem + + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", build) + source = _vospace_source("archive") + + async with source() as first: + assert first.closed is False + assert first.closed is True + + async with source() as second: + assert second.closed is False + assert second.closed is True + + assert first is not second + assert filesystems == [first, second] + + @pytest.mark.asyncio async def test_environment_token_preserves_runtime_precedence( monkeypatch: pytest.MonkeyPatch, From e06c6ebd60047ea476afbebedc82767afc0d8aa4 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 17:09:31 -0700 Subject: [PATCH 24/47] fix(server): make storage inspection stateless --- canfar/models/config.py | 3 - canfar/models/registry.py | 3 + canfar/server.py | 212 +++++++++++++++++++++++++----- canfar/utils/discover.py | 29 +++- tests/test_platform_enrichment.py | 41 +++++- tests/test_server.py | 102 +++++++++++++- 6 files changed, 340 insertions(+), 50 deletions(-) diff --git a/canfar/models/config.py b/canfar/models/config.py index 50ab545b..3e16d46d 100644 --- a/canfar/models/config.py +++ b/canfar/models/config.py @@ -12,7 +12,6 @@ BaseModel, ConfigDict, Field, - PrivateAttr, model_validator, ) @@ -115,8 +114,6 @@ class Configuration(BaseSettings): str_strip_whitespace=True, ) - _storage_discovery_errors: dict[str, str] = PrivateAttr(default_factory=dict) - version: Literal[1] = Field( default=1, description="Configuration schema version.", diff --git a/canfar/models/registry.py b/canfar/models/registry.py index e40a09a4..0e7f8684 100644 --- a/canfar/models/registry.py +++ b/canfar/models/registry.py @@ -69,6 +69,8 @@ class IVOARegistry(BaseModel): name: str content: str + source: str | None = None + development: bool = False success: bool = True error: str | None = None @@ -77,6 +79,7 @@ class Server(BaseModel): """Model to store Canfar Server endpoint information.""" registry: str + development: bool = False uri: str url: str status: int | None = None diff --git a/canfar/server.py b/canfar/server.py index ca60023e..b633e799 100644 --- a/canfar/server.py +++ b/canfar/server.py @@ -254,6 +254,7 @@ def activate( resolved, config=target_config, idp=idp, + dev=dev, timeout=timeout, ) target_config.set_active_selection(idp, validated) @@ -404,14 +405,23 @@ async def _discover_for_idp( Raises: ServerDiscoveryError: If registry retrieval fails. """ + idp_info = get_idp(idp) sources = registry_sources(idp, include_dev=dev) + development_sources = set(idp_info.dev_registries) search = IVOARegistrySearch( registries=sources, - preferred_storage_leaf=get_idp(idp).preferred_storage_leaf, + preferred_storage_leaf=idp_info.preferred_storage_leaf, ) async with Discover(search, timeout=timeout) as discovery: registries = await asyncio.gather( - *(discovery.fetch(url, name) for url, name in sources.items()) + *( + discovery.fetch( + url, + name, + development=url in development_sources, + ) + for url, name in sources.items() + ) ) successful_registries = [ registry for registry in registries if registry.success @@ -432,28 +442,35 @@ async def _discover_for_idp( if not endpoints: return [] - storage_by_namespace = { - _registry_namespace(resource.uri): resource + storage_resources = [ + resource for resource in resources if resource.uri.endswith(f"/{search.preferred_storage_leaf}") - } + ] checked = await asyncio.gather( *(discovery.check(endpoint) for endpoint in endpoints) ) - return [ - _discovered_to_server( - endpoint, - idp, - config=config, - timeout=timeout, - storage_resource=storage_by_namespace.get( - _registry_namespace(endpoint.uri) - ), + reachable = [endpoint for endpoint in checked if endpoint.status == 200] + return list( + await asyncio.gather( + *( + asyncio.to_thread( + _discovered_to_server, + endpoint, + idp, + config=config, + timeout=timeout, + storage_resource=_select_storage_resource( + endpoint, + storage_resources, + strict=False, + ), + ) + for endpoint in reachable + ) ) - for endpoint in checked - if endpoint.status == 200 - ] + ) def _host_slug(uri: AnyUrl) -> str | None: @@ -468,6 +485,132 @@ def _registry_namespace(uri: str) -> str: return uri.rpartition("/")[0] +def _select_storage_resource( + endpoint: RegistryResource, + resources: list[RegistryResource], + *, + strict: bool, +) -> RegistryResource | None: + """Pair an endpoint with one unambiguous same-environment VOSpace record.""" + candidates = [ + resource + for resource in resources + if _registry_namespace(resource.uri) == _registry_namespace(endpoint.uri) + and resource.development == endpoint.development + ] + same_registry = [ + resource for resource in candidates if resource.registry == endpoint.registry + ] + preferred = same_registry or candidates + if len(preferred) == 1: + return preferred[0] + if len(preferred) > 1: + message = ( + "Multiple preferred VOSpace registry records found for Science " + f"Platform Server '{endpoint.name or endpoint.uri}' in namespace " + f"'{_registry_namespace(endpoint.uri)}'." + ) + if strict: + raise ServerFetchError(message) + log.debug("%s Omitting generated storage configuration.", message) + return None + + +async def _discover_storage_resource( + server: Server, + idp: str, + *, + dev: bool, + timeout: int, +) -> RegistryResource | None: + """Return fresh registry evidence for a server's primary VOSpace service.""" + if server.uri is None: + msg = "Server URI is required to inspect its VOSpace Service." + raise ServerFetchError(msg) + + idp_info = get_idp(idp) + sources = registry_sources(idp, include_dev=dev) + development_sources = set(idp_info.dev_registries) + search = IVOARegistrySearch( + registries=sources, + preferred_storage_leaf=idp_info.preferred_storage_leaf, + ) + async with Discover(search, timeout=timeout) as discovery: + registries = await asyncio.gather( + *( + discovery.fetch( + url, + name, + development=url in development_sources, + ) + for url, name in sources.items() + ) + ) + successful = [registry for registry in registries if registry.success] + if not successful: + errors = "; ".join( + f"{registry.name}: {registry.error}" for registry in registries + ) + msg = ( + f"Failed to inspect VOSpace registry records for IDP '{idp}': {errors}" + ) + raise ServerFetchError(msg) + + resources = [ + resource + for registry in successful + for resource in discovery.extract(registry, dev=dev) + ] + + endpoints = [ + resource + for resource in resources + if resource.uri == str(server.uri) and resource.uri.endswith("/skaha") + ] + matching_urls = [ + endpoint + for endpoint in endpoints + if server.url is not None and endpoint.url == str(server.url) + ] + if len(matching_urls) == 1: + endpoint = matching_urls[0] + elif len(endpoints) == 1: + endpoint = endpoints[0] + elif not endpoints: + msg = ( + f"No Science Platform registry record found for Server '{server.name}' " + f"with URI '{server.uri}'." + ) + raise ServerFetchError(msg) + else: + msg = ( + f"Multiple Science Platform registry records found for Server " + f"'{server.name}' with URI '{server.uri}'." + ) + raise ServerFetchError(msg) + + storage_resources = [ + resource + for resource in resources + if resource.uri.endswith(f"/{search.preferred_storage_leaf}") + ] + return _select_storage_resource(endpoint, storage_resources, strict=True) + + +def _configured_storage_resource(server: Server) -> RegistryResource | None: + """Convert the persisted primary VOSpace service to inspection evidence.""" + if server.name is None: + return None + service = server.storage.get(server.name) + if service is None: + return None + return RegistryResource( + registry="configuration", + uri=str(service.uri), + url=str(service.url), + ) + + def _discovered_to_server( endpoint: RegistryResource, idp: str, @@ -544,14 +687,6 @@ def enrich( """ base_config = config or Configuration() # ty: ignore[missing-argument] active_idp = authentication_idp or server.idp or base_config.active.authentication - if ( - strict - and storage_resource is _STORAGE_RESOURCE_UNSET - and server.name in base_config._storage_discovery_errors # noqa: SLF001 - ): - raise ServerFetchError( - base_config._storage_discovery_errors[server.name] # noqa: SLF001 - ) if storage_resource is not _STORAGE_RESOURCE_UNSET: server = _enrich_storage( server, @@ -617,7 +752,7 @@ def enrich( ) try: - enriched = Server.model_validate( + return Server.model_validate( { **server.model_dump(mode="python"), "url": primary["baseurl"], @@ -636,8 +771,6 @@ def enrich( ), args=(server.url, exc), ) - else: - return enriched def _enrich_storage( @@ -692,7 +825,7 @@ def _enrich_storage( f"Failed to inspect VOSpace Service '{subject}' for Science " f"Platform Server '{server.name}': {error}" ) - kept = _keep_or_raise( + return _keep_or_raise( server, strict=strict, error=message, @@ -700,20 +833,14 @@ def _enrich_storage( debug="Skipping VOSpace Service %s during discovery: %s", args=(subject, error), ) - if server.name is not None: - config._storage_discovery_errors[server.name] = message # noqa: SLF001 - return kept - assert storage_resource is not None assert server.name is not None service = VOSpaceService(uri=storage_resource.uri, url=storage_resource.url) - enriched = server.model_copy( + return server.model_copy( update={"storage": {**server.storage, server.name: service}}, deep=True, ) - config._storage_discovery_errors.pop(server.name, None) # noqa: SLF001 - return enriched def _fetch_capabilities( @@ -763,6 +890,7 @@ def _validate_server( *, config: Configuration | None = None, idp: str | None = None, + dev: bool = False, timeout: int = 2, ) -> Server: """Fetch and validate a server before persisting it as active. @@ -771,6 +899,7 @@ def _validate_server( server: Candidate server record. config: Configuration to use while validating the candidate selection. idp: Authentication IDP to pair with the candidate server. + dev: Include development registry evidence during validation. timeout: HTTP timeout in seconds for validation requests. Returns: @@ -781,12 +910,23 @@ def _validate_server( """ base_config = config or Configuration() # ty: ignore[missing-argument] active_idp = idp or server.idp or base_config.active.authentication + storage_resource = _configured_storage_resource(server) + if storage_resource is None: + storage_resource = asyncio.run( + _discover_storage_resource( + server, + active_idp, + dev=dev, + timeout=timeout, + ) + ) enriched = enrich( server, config=base_config, authentication_idp=active_idp, strict=True, timeout=timeout, + storage_resource=storage_resource, ) if enriched.url is None or enriched.version is None: msg = "Server URL and version are required before activation." diff --git a/canfar/utils/discover.py b/canfar/utils/discover.py index d8e4ac71..b45dc032 100644 --- a/canfar/utils/discover.py +++ b/canfar/utils/discover.py @@ -51,12 +51,19 @@ async def __aexit__( """ await self.client.aclose() - async def fetch(self, url: str, name: str) -> IVOARegistry: + async def fetch( + self, + url: str, + name: str, + *, + development: bool = False, + ) -> IVOARegistry: """Fetch registry contents. Args: url (str): Registry URL. name (str): Common name for the registry. + development: Whether this source contains development records. Returns: RegistryInfo: Registry information. @@ -70,10 +77,23 @@ async def fetch(self, url: str, name: str) -> IVOARegistry: f"[dim]Fetched {name} in {elapsed:.2f}s[/dim]" ) - return IVOARegistry(name=name, content=response.text, success=True) + return IVOARegistry( + name=name, + source=url, + development=development, + content=response.text, + success=True, + ) except httpx.HTTPError as error: error_msg = str(error) - return IVOARegistry(name=name, content="", success=False, error=error_msg) + return IVOARegistry( + name=name, + source=url, + development=development, + content="", + success=False, + error=error_msg, + ) def extract(self, registry: IVOARegistry, dev: bool = False) -> list[Server]: """Extract Science Platform and preferred VOSpace registry records.""" @@ -106,7 +126,8 @@ def extract(self, registry: IVOARegistry, dev: bool = False) -> list[Server]: if (registry.name, uri) in self.config.omit: continue endpoint = Server( - registry=registry.name, + registry=registry.source or registry.name, + development=registry.development, uri=uri, url=url, name=self.config.names.get(uri) if leaf == "skaha" else None, diff --git a/tests/test_platform_enrichment.py b/tests/test_platform_enrichment.py index cfc659b2..8d578b6f 100644 --- a/tests/test_platform_enrichment.py +++ b/tests/test_platform_enrichment.py @@ -23,7 +23,7 @@ X509Credential, ) from canfar.models.config import Configuration -from canfar.models.http import Server +from canfar.models.http import Server, VOSpaceService from canfar.models.registry import ContainerRegistry from tests.test_auth_x509 import generate_cert @@ -39,6 +39,25 @@ "memoryGB": {"defaultLimit": 192}, "gpus": {"options": [0, 1, 2, 4]}, } +_VOSPACE_CAPABILITIES = """ + + + + https://storage.example/arc/nodes + + + +""" + + +def _primary_storage(name: str, *, leaf: str = "arc") -> dict[str, VOSpaceService]: + """Return persisted primary storage evidence for activation tests.""" + return { + name: VOSpaceService( + uri=AnyUrl(f"ivo://storage.example/{leaf}"), + url=AnyHttpUrl(f"https://storage.example/{leaf}"), + ) + } def _http_client_factory( @@ -149,6 +168,7 @@ def test_activation_enriches_resources_and_round_trips_server_name( """Activation owns HTTP enrichment while Server remains persisted data.""" known = _server( name="Stable-Name", + storage=_primary_storage("Stable-Name"), cores=8, ram=64, gpus=1, @@ -159,6 +179,8 @@ def test_activation_enriches_resources_and_round_trips_server_name( def response(request: httpx.Request) -> httpx.Response: """Serve capabilities and context for activation enrichment.""" requests.append(request) + if request.url.host == "storage.example": + return httpx.Response(200, text=_VOSPACE_CAPABILITIES, request=request) if request.url.path.endswith("/capabilities"): return httpx.Response( 200, text=_capabilities(_CADC_URL), request=request @@ -189,12 +211,14 @@ def response(request: httpx.Request) -> httpx.Response: assert persisted.active.server == "Stable-Name" assert persisted.servers["Stable-Name"] == activated.server assert [request.url.path for request in requests] == [ + "/arc/capabilities", "/skaha/capabilities", "/skaha/v2/context", ] assert [request.extensions["timeout"] for request in requests] == [ {"connect": 7, "read": 7, "write": 7, "pool": 7}, {"connect": 7, "read": 7, "write": 7, "pool": 7}, + {"connect": 7, "read": 7, "write": 7, "pool": 7}, ] @pytest.mark.parametrize("failure", ["transport", "parse", "validation"]) @@ -204,10 +228,19 @@ def test_activation_uses_resource_defaults_when_context_is_unusable( failure: str, ) -> None: """Expected context failures keep server identity and safe defaults.""" - known = _server(name="Stable-Name", cores=8, ram=64, gpus=1, status="reachable") + known = _server( + name="Stable-Name", + storage=_primary_storage("Stable-Name"), + cores=8, + ram=64, + gpus=1, + status="reachable", + ) def response(request: httpx.Request) -> httpx.Response: """Serve capabilities and a parametrized unusable context reply.""" + if request.url.host == "storage.example": + return httpx.Response(200, text=_VOSPACE_CAPABILITIES, request=request) if request.url.path.endswith("/capabilities"): return httpx.Response( 200, text=_capabilities(_CADC_URL), request=request @@ -445,6 +478,7 @@ def test_refresh_can_persist_without_candidate_platform_state( name="SRCNet", uri=AnyUrl("ivo://srcnet.example/skaha"), url=AnyHttpUrl("https://srcnet.example/skaha"), + storage=_primary_storage("SRCNet", leaf="cavern"), version=None, auths=None, ) @@ -455,6 +489,8 @@ def test_refresh_can_persist_without_candidate_platform_state( def platform_response(request: httpx.Request) -> httpx.Response: """Serve capabilities then a failed context fetch during refresh.""" platform_requests.append(request) + if request.url.host == "storage.example": + return httpx.Response(200, text=_VOSPACE_CAPABILITIES, request=request) if request.url.path.endswith("/capabilities"): return httpx.Response(200, text=capabilities, request=request) if request.url.path.endswith("/v2/context"): @@ -520,6 +556,7 @@ def token_response(request: httpx.Request) -> httpx.Response: assert [request.headers["Authorization"] for request in platform_requests] == [ f"Bearer {_REFRESHED_TOKEN}", f"Bearer {_REFRESHED_TOKEN}", + f"Bearer {_REFRESHED_TOKEN}", ] assert config.active.model_dump(mode="json") == before_active assert { diff --git a/tests/test_server.py b/tests/test_server.py index 5130670f..79be2254 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -3,6 +3,7 @@ from __future__ import annotations from pathlib import Path +from threading import Barrier from typing import TYPE_CHECKING from unittest.mock import AsyncMock, MagicMock, patch @@ -24,6 +25,7 @@ ServerSelectionRequiredError, _discover_for_idp, _discovered_to_server, + _select_storage_resource, activate, discover, enrich, @@ -527,8 +529,8 @@ def response(request: httpx.Request) -> httpx.Response: if mode == "malformed": assert isinstance(exc_info.value.__cause__, ValueError) - def test_activation_surfaces_storage_failure_retained_by_discovery(self) -> None: - """Real non-strict discovery carries its storage error into activation.""" + def test_activation_freshly_inspects_storage_missing_during_discovery(self) -> None: + """Activation obtains fresh evidence without transient Configuration state.""" endpoint = DiscoveredServer( registry="CADC source", uri=_CADC_URI, @@ -570,18 +572,20 @@ def test_activation_surfaces_storage_failure_retained_by_discovery(self) -> None ), ): [discovered] = discover("cadc", config=config, save=False) + reloaded = Configuration.model_validate(config.model_dump(mode="python")) with pytest.raises( ServerFetchError, match="same-namespace 'arc' registry record", ): - activate("cadc", _CADC_URI, config=config) + activate("cadc", _CADC_URI, config=reloaded) assert discovered.version == "v1" assert discovered.storage == {} + assert "_storage_discovery_errors" not in Configuration.__private_attributes__ @pytest.mark.asyncio - async def test_srcnet_pairs_each_server_with_same_namespace_cavern(self) -> None: - """SRCNet Cavern records pair by IVOA namespace, not list position.""" + async def test_cross_registry_singletons_pair_by_namespace(self) -> None: + """A lone same-environment fallback may cross registry provenance.""" resources = [ DiscoveredServer( registry="SRCNet", @@ -648,6 +652,94 @@ def enriched( "sweSRC": "ivo://swesrc.chalmers.se/cavern", } + def test_storage_pairing_never_crosses_prod_and_dev_sources(self) -> None: + """A same-namespace record from another environment is not a fallback.""" + endpoint = DiscoveredServer( + registry="https://registry.example/prod", + development=False, + uri="ivo://cadc.nrc.ca/skaha", + url="https://platform.example/skaha", + name="canfar", + ) + dev_storage = DiscoveredServer( + registry="https://registry.example/dev", + development=True, + uri="ivo://cadc.nrc.ca/arc", + url="https://storage.example/arc", + ) + + assert _select_storage_resource(endpoint, [dev_storage], strict=False) is None + + def test_ambiguous_cross_registry_storage_is_not_last_write_wins(self) -> None: + """Multiple namespace fallbacks are omitted or actionable, never arbitrary.""" + endpoint = DiscoveredServer( + registry="https://registry.example/platform", + uri="ivo://example.org/skaha", + url="https://platform.example/skaha", + name="example", + ) + storage = [ + DiscoveredServer( + registry=f"https://registry.example/storage-{index}", + uri="ivo://example.org/cavern", + url=f"https://storage-{index}.example/cavern", + ) + for index in (1, 2) + ] + + assert _select_storage_resource(endpoint, storage, strict=False) is None + with pytest.raises(ServerFetchError, match="Multiple preferred VOSpace"): + _select_storage_resource(endpoint, storage, strict=True) + + @pytest.mark.asyncio + async def test_capability_enrichment_runs_concurrently_off_event_loop( + self, + ) -> None: + """Blocking authenticated capability clients run in concurrent workers.""" + endpoints = [ + DiscoveredServer( + registry="SRCNet", + uri=f"ivo://site-{index}.example/skaha", + url=f"https://site-{index}.example/skaha", + status=200, + name=f"site-{index}", + ) + for index in (1, 2) + ] + mock_discovery = AsyncMock() + mock_discovery.fetch.return_value = MagicMock( + success=True, + content="line", + ) + mock_discovery.extract = MagicMock(return_value=endpoints) + mock_discovery.check = AsyncMock(side_effect=lambda item: item) + mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) + mock_discovery.__aexit__ = AsyncMock(return_value=None) + concurrent = Barrier(2, timeout=2) + + def convert( + endpoint: DiscoveredServer, + idp: str, + **_kwargs: object, + ) -> Server: + concurrent.wait() + return Server( + idp=idp, + name=endpoint.name, + uri=AnyUrl(endpoint.uri), + url=AnyHttpUrl(endpoint.url), + version="v1", + auths=["oidc"], + ) + + with ( + patch("canfar.server.Discover", return_value=mock_discovery), + patch("canfar.server._discovered_to_server", side_effect=convert), + ): + servers = await _discover_for_idp("srcnet") + + assert [server.name for server in servers] == ["site-1", "site-2"] + @pytest.mark.parametrize( "capabilities_case", [ From 87d7cf67218a0293f8383bf6c5040fae988b00ce Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 17:09:41 -0700 Subject: [PATCH 25/47] refactor(data): simplify source authentication failure --- canfar/_storage.py | 10 ++-------- tests/test_storage.py | 3 --- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/canfar/_storage.py b/canfar/_storage.py index e0cd0149..55627fd7 100644 --- a/canfar/_storage.py +++ b/canfar/_storage.py @@ -12,7 +12,6 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator from pathlib import Path - from typing import NoReturn from fsspec.spec import AbstractFileSystem from fsspec_cli import AsyncFilesystemSource @@ -46,7 +45,8 @@ async def source() -> AsyncIterator[AbstractFileSystem]: ) token_value, certfile = await client._materialize_credentials() # noqa: SLF001 except (KeyError, OSError, TypeError, ValueError): - _fail_authentication(idp) + reason = "Credential cannot be used. Run 'canfar login' for this IDP." + raise AuthContextError(idp, reason) from None from vosfs import VOSpaceFileSystem # noqa: PLC0415 @@ -71,9 +71,3 @@ async def source() -> AsyncIterator[AbstractFileSystem]: await filesystem.aclose() return source - - -def _fail_authentication(idp: str) -> NoReturn: - """Raise the fixed secret-safe authentication failure.""" - reason = "Credential cannot be used. Run 'canfar login' for this IDP." - raise AuthContextError(idp, reason) from None diff --git a/tests/test_storage.py b/tests/test_storage.py index 601eff55..2e0c4b45 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -24,8 +24,6 @@ class _Filesystem: """Record construction and cleanup without VOSpace I/O.""" - async_impl = True - def __init__(self, endpoint: str, **kwargs: Any) -> None: self.endpoint = endpoint self.kwargs = kwargs @@ -97,7 +95,6 @@ async def test_source_reloads_config_and_runtime_token_wins( "asynchronous": True, "skip_instance_cache": True, } - assert filesystem.async_impl is True assert filesystem.asynchronous is True assert filesystem.closed is False From 1f3cfbe295a5cc2111b1008db0eded5ba13de4c8 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 18:39:40 -0700 Subject: [PATCH 26/47] fix(server): isolate registry discovery evidence --- canfar/server.py | 269 ++++++++++++++++++++++++--------------- canfar/utils/discover.py | 9 +- tests/test_server.py | 45 ++++++- 3 files changed, 215 insertions(+), 108 deletions(-) diff --git a/canfar/server.py b/canfar/server.py index b633e799..9c56bc86 100644 --- a/canfar/server.py +++ b/canfar/server.py @@ -88,6 +88,16 @@ class ServerActivation: reason: Literal["active", "remembered", "single", "selected"] +@dataclass(frozen=True) +class _RegistryEvidence: + """Registry resources plus acquisition outcome for one IDP inspection.""" + + preferred_storage_leaf: str | None + resources: tuple[RegistryResource, ...] + errors: tuple[str, ...] + available: bool + + def discover( idp: str, *, @@ -384,6 +394,91 @@ def _resolve_selector( return None +async def _load_registry_evidence( + idp: str, + *, + dev: bool, + timeout: int, + check_platforms: bool, +) -> _RegistryEvidence: + """Acquire and extract registry records through one shared pipeline.""" + idp_info = get_idp(idp) + sources = registry_sources(idp, include_dev=dev) + development_sources = set(idp_info.dev_registries) + search = IVOARegistrySearch( + registries=sources, + preferred_storage_leaf=idp_info.preferred_storage_leaf, + ) + async with Discover(search, timeout=timeout) as discovery: + registries = await asyncio.gather( + *( + discovery.fetch( + url, + name, + development=url in development_sources, + ) + for url, name in sources.items() + ) + ) + successful = [registry for registry in registries if registry.success] + resources = [ + resource + for registry in successful + for resource in discovery.extract(registry, dev=dev) + ] + if check_platforms: + endpoints = [ + resource for resource in resources if resource.uri.endswith("/skaha") + ] + await asyncio.gather(*(discovery.check(endpoint) for endpoint in endpoints)) + + return _RegistryEvidence( + preferred_storage_leaf=idp_info.preferred_storage_leaf, + resources=tuple(resources), + errors=tuple( + f"{registry.name}: {registry.error}" + for registry in registries + if not registry.success + ), + available=bool(successful), + ) + + +async def _enrichment_config_copies( + config: Configuration | None, + idp: str, + *, + endpoint: RegistryResource, + count: int, +) -> list[Configuration] | None: + """Materialize credentials once, then isolate worker configuration state.""" + from canfar.client import HTTPClient # noqa: PLC0415 + + base_config = config or Configuration() + client = HTTPClient( + config=base_config, + authentication_idp=idp, + url=AnyHttpUrl(endpoint.url), + ) + if client.authentication_record is not None: + try: + await client._materialize_credentials() # noqa: SLF001 + except ( + KeyError, + OSError, + AuthContextError, + AuthExpiredError, + CertificateError, + TypeError, + ValueError, + ) as exc: + log.debug("Skipping capability enrichment for IDP %s: %s", idp, exc) + return None + + values = base_config.model_dump(mode="python") + return [Configuration.model_validate(values) for _ in range(count)] + + async def _discover_for_idp( idp: str, *, @@ -405,72 +500,62 @@ async def _discover_for_idp( Raises: ServerDiscoveryError: If registry retrieval fails. """ - idp_info = get_idp(idp) - sources = registry_sources(idp, include_dev=dev) - development_sources = set(idp_info.dev_registries) - search = IVOARegistrySearch( - registries=sources, - preferred_storage_leaf=idp_info.preferred_storage_leaf, + evidence = await _load_registry_evidence( + idp, + dev=dev, + timeout=timeout, + check_platforms=True, ) - async with Discover(search, timeout=timeout) as discovery: - registries = await asyncio.gather( - *( - discovery.fetch( - url, - name, - development=url in development_sources, - ) - for url, name in sources.items() - ) - ) - successful_registries = [ - registry for registry in registries if registry.success - ] - if not successful_registries: - errors = "; ".join( - f"{registry.name}: {registry.error}" for registry in registries - ) - msg = f"Failed to discover servers for IDP '{idp}': {errors}" - raise ServerDiscoveryError(msg) - - resources = [] - for registry in successful_registries: - resources.extend(discovery.extract(registry, dev=dev)) - endpoints = [ - resource for resource in resources if resource.uri.endswith("/skaha") - ] - if not endpoints: - return [] + if not evidence.available: + errors = "; ".join(evidence.errors) + msg = f"Failed to discover servers for IDP '{idp}': {errors}" + raise ServerDiscoveryError(msg) - storage_resources = [ - resource - for resource in resources - if resource.uri.endswith(f"/{search.preferred_storage_leaf}") - ] + endpoints = [ + resource + for resource in evidence.resources + if resource.uri.endswith("/skaha") and resource.status == 200 + ] + if not endpoints: + return [] - checked = await asyncio.gather( - *(discovery.check(endpoint) for endpoint in endpoints) - ) - reachable = [endpoint for endpoint in checked if endpoint.status == 200] - return list( - await asyncio.gather( - *( - asyncio.to_thread( - _discovered_to_server, + storage_resources = [ + resource + for resource in evidence.resources + if resource.uri.endswith(f"/{evidence.preferred_storage_leaf}") + ] + worker_configs = await _enrichment_config_copies( + config, + idp, + endpoint=endpoints[0], + count=len(endpoints), + ) + if worker_configs is None: + return [_registry_resource_to_server(endpoint, idp) for endpoint in endpoints] + + return list( + await asyncio.gather( + *( + asyncio.to_thread( + _discovered_to_server, + endpoint, + idp, + config=worker_config, + timeout=timeout, + storage_resource=_select_storage_resource( endpoint, - idp, - config=config, - timeout=timeout, - storage_resource=_select_storage_resource( - endpoint, - storage_resources, - strict=False, - ), - ) - for endpoint in reachable + storage_resources, + strict=False, + ), + ) + for endpoint, worker_config in zip( + endpoints, + worker_configs, + strict=True, ) ) ) + ) def _host_slug(uri: AnyUrl) -> str | None: @@ -528,43 +613,20 @@ async def _discover_storage_resource( msg = "Server URI is required to inspect its VOSpace Service." raise ServerFetchError(msg) - idp_info = get_idp(idp) - sources = registry_sources(idp, include_dev=dev) - development_sources = set(idp_info.dev_registries) - search = IVOARegistrySearch( - registries=sources, - preferred_storage_leaf=idp_info.preferred_storage_leaf, + evidence = await _load_registry_evidence( + idp, + dev=dev, + timeout=timeout, + check_platforms=False, ) - async with Discover(search, timeout=timeout) as discovery: - registries = await asyncio.gather( - *( - discovery.fetch( - url, - name, - development=url in development_sources, - ) - for url, name in sources.items() - ) - ) - successful = [registry for registry in registries if registry.success] - if not successful: - errors = "; ".join( - f"{registry.name}: {registry.error}" for registry in registries - ) - msg = ( - f"Failed to inspect VOSpace registry records for IDP '{idp}': {errors}" - ) - raise ServerFetchError(msg) - - resources = [ - resource - for registry in successful - for resource in discovery.extract(registry, dev=dev) - ] + if not evidence.available: + errors = "; ".join(evidence.errors) + msg = f"Failed to inspect VOSpace registry records for IDP '{idp}': {errors}" + raise ServerFetchError(msg) endpoints = [ resource - for resource in resources + for resource in evidence.resources if resource.uri == str(server.uri) and resource.uri.endswith("/skaha") ] matching_urls = [ @@ -591,8 +653,8 @@ async def _discover_storage_resource( storage_resources = [ resource - for resource in resources - if resource.uri.endswith(f"/{search.preferred_storage_leaf}") + for resource in evidence.resources + if resource.uri.endswith(f"/{evidence.preferred_storage_leaf}") ] return _select_storage_resource(endpoint, storage_resources, strict=True) @@ -611,6 +673,17 @@ def _configured_storage_resource(server: Server) -> RegistryResource | None: ) +def _registry_resource_to_server(endpoint: RegistryResource, idp: str) -> Server: + """Convert registry endpoint identity without performing capability I/O.""" + uri = AnyUrl(endpoint.uri) + return Server( + idp=idp, + name=endpoint.name or _host_slug(uri), + uri=uri, + url=AnyHttpUrl(endpoint.url), + ) + + def _discovered_to_server( endpoint: RegistryResource, idp: str, @@ -634,13 +707,7 @@ def _discovered_to_server( Returns: Server: Persisted server model with capabilities metadata when available. """ - uri = AnyUrl(endpoint.uri) - server = Server( - idp=idp, - name=endpoint.name or _host_slug(uri), - uri=uri, - url=AnyHttpUrl(endpoint.url), - ) + server = _registry_resource_to_server(endpoint, idp) return enrich( server, config=config, diff --git a/canfar/utils/discover.py b/canfar/utils/discover.py index b45dc032..2a26f1eb 100644 --- a/canfar/utils/discover.py +++ b/canfar/utils/discover.py @@ -115,11 +115,12 @@ def extract(self, registry: IVOARegistry, dev: bool = False) -> list[Server]: url = _without_terminal_capabilities(url) if url is None: continue - # Apply exclusion filters - if not dev and any( + record_development = any( word in uri.lower() or word in url.lower() for word in self.config.excluded - ): + ) + # Apply exclusion filters + if not dev and record_development: continue # Apply omit filters @@ -127,7 +128,7 @@ def extract(self, registry: IVOARegistry, dev: bool = False) -> list[Server]: continue endpoint = Server( registry=registry.source or registry.name, - development=registry.development, + development=registry.development or record_development, uri=uri, url=url, name=self.config.names.get(uri) if leaf == "skaha" else None, diff --git a/tests/test_server.py b/tests/test_server.py index 79be2254..603fbaac 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -670,6 +670,28 @@ def test_storage_pairing_never_crosses_prod_and_dev_sources(self) -> None: assert _select_storage_resource(endpoint, [dev_storage], strict=False) is None + @pytest.mark.asyncio + async def test_mixed_registry_records_keep_per_record_environment(self) -> None: + """Development-looking records stay isolated within a production source.""" + registry = IVOARegistry( + name="SRCNet", + source="https://registry.example/resources", + content=( + "ivo://example.org/skaha=" + "https://platform.example/skaha/capabilities\n" + "ivo://example.org/cavern=" + "https://storage.example/dev/capabilities" + ), + ) + search = IVOARegistrySearch(preferred_storage_leaf="cavern") + + async with Discover(search) as discovery: + endpoint, storage = discovery.extract(registry, dev=True) + + assert endpoint.development is False + assert storage.development is True + assert _select_storage_resource(endpoint, [storage], strict=False) is None + def test_ambiguous_cross_registry_storage_is_not_last_write_wins(self) -> None: """Multiple namespace fallbacks are omitted or actionable, never arbitrary.""" endpoint = DiscoveredServer( @@ -695,7 +717,7 @@ def test_ambiguous_cross_registry_storage_is_not_last_write_wins(self) -> None: async def test_capability_enrichment_runs_concurrently_off_event_loop( self, ) -> None: - """Blocking authenticated capability clients run in concurrent workers.""" + """Workers run concurrently with isolated, pre-materialized config copies.""" endpoints = [ DiscoveredServer( registry="SRCNet", @@ -716,12 +738,21 @@ async def test_capability_enrichment_runs_concurrently_off_event_loop( mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) mock_discovery.__aexit__ = AsyncMock(return_value=None) concurrent = Barrier(2, timeout=2) + worker_configs: list[Configuration] = [] + config = Configuration( + active=ActiveConfig(authentication="srcnet", server=None), + authentication={"srcnet": OIDCCredential(idp="srcnet")}, + servers={}, + ) def convert( endpoint: DiscoveredServer, idp: str, - **_kwargs: object, + **kwargs: object, ) -> Server: + worker_config = kwargs["config"] + assert isinstance(worker_config, Configuration) + worker_configs.append(worker_config) concurrent.wait() return Server( idp=idp, @@ -732,13 +763,21 @@ def convert( auths=["oidc"], ) + materialize = AsyncMock(return_value=("current-token", None)) with ( patch("canfar.server.Discover", return_value=mock_discovery), patch("canfar.server._discovered_to_server", side_effect=convert), + patch( + "canfar.client.HTTPClient._materialize_credentials", + new=materialize, + ), ): - servers = await _discover_for_idp("srcnet") + servers = await _discover_for_idp("srcnet", config=config) assert [server.name for server in servers] == ["site-1", "site-2"] + materialize.assert_awaited_once_with() + assert len({id(worker_config) for worker_config in worker_configs}) == 2 + assert all(worker_config is not config for worker_config in worker_configs) @pytest.mark.parametrize( "capabilities_case", From ac9a0be8a089bc7ce790d8c2a036cc97cadaf106 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 18:50:04 -0700 Subject: [PATCH 27/47] refactor(server): isolate discovery worker state --- canfar/_discovery.py | 230 ++++++++++++++++++++++++++++++++++++++++++ canfar/server.py | 233 ++++++++++--------------------------------- tests/test_server.py | 77 ++++++++++++-- 3 files changed, 355 insertions(+), 185 deletions(-) create mode 100644 canfar/_discovery.py diff --git a/canfar/_discovery.py b/canfar/_discovery.py new file mode 100644 index 00000000..6040b068 --- /dev/null +++ b/canfar/_discovery.py @@ -0,0 +1,230 @@ +"""Private registry evidence and discovery-worker preparation.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from pathlib import Path + +from pydantic import AnyHttpUrl + +from canfar import get_logger +from canfar.auth.x509 import CertificateError +from canfar.exceptions.context import AuthContextError, AuthExpiredError +from canfar.idp import get_idp, registry_sources +from canfar.models.config import Configuration +from canfar.models.registry import IVOARegistrySearch +from canfar.models.registry import Server as RegistryResource +from canfar.utils.discover import Discover + +log = get_logger(__name__) + + +class RegistryEvidenceError(RuntimeError): + """Raised when strict registry evidence is missing or ambiguous.""" + + +@dataclass(frozen=True) +class RegistryEvidence: + """Registry resources plus acquisition outcome for one IDP inspection.""" + + preferred_storage_leaf: str | None + resources: tuple[RegistryResource, ...] + errors: tuple[str, ...] + available: bool + + +@dataclass(frozen=True) +class EnrichmentWorkers: + """Isolated worker configs with one pre-materialized runtime credential.""" + + configs: tuple[Configuration, ...] + token: str | None = None + certificate: Path | None = None + + +async def load_registry_evidence( + idp: str, + *, + dev: bool, + timeout: int, + check_platforms: bool, +) -> RegistryEvidence: + """Acquire and extract registry records through one shared pipeline.""" + idp_info = get_idp(idp) + sources = registry_sources(idp, include_dev=dev) + development_sources = set(idp_info.dev_registries) + search = IVOARegistrySearch( + registries=sources, + preferred_storage_leaf=idp_info.preferred_storage_leaf, + ) + async with Discover(search, timeout=timeout) as discovery: + registries = await asyncio.gather( + *( + discovery.fetch( + url, + name, + development=url in development_sources, + ) + for url, name in sources.items() + ) + ) + successful = [registry for registry in registries if registry.success] + resources = [ + resource + for registry in successful + for resource in discovery.extract(registry, dev=dev) + ] + if check_platforms: + endpoints = [ + resource for resource in resources if resource.uri.endswith("/skaha") + ] + await asyncio.gather(*(discovery.check(endpoint) for endpoint in endpoints)) + + return RegistryEvidence( + preferred_storage_leaf=idp_info.preferred_storage_leaf, + resources=tuple(resources), + errors=tuple( + f"{registry.name}: {registry.error}" + for registry in registries + if not registry.success + ), + available=bool(successful), + ) + + +def _registry_namespace(uri: str) -> str: + """Return the IVOA registry URI namespace before its resource leaf.""" + return uri.rpartition("/")[0] + + +def select_storage_resource( + endpoint: RegistryResource, + resources: list[RegistryResource], + *, + strict: bool, +) -> RegistryResource | None: + """Pair an endpoint with one unambiguous same-environment VOSpace record.""" + candidates = [ + resource + for resource in resources + if _registry_namespace(resource.uri) == _registry_namespace(endpoint.uri) + and resource.development == endpoint.development + ] + same_registry = [ + resource for resource in candidates if resource.registry == endpoint.registry + ] + preferred = same_registry or candidates + if len(preferred) == 1: + return preferred[0] + if len(preferred) > 1: + message = ( + "Multiple preferred VOSpace registry records found for Science " + f"Platform Server '{endpoint.name or endpoint.uri}' in namespace " + f"'{_registry_namespace(endpoint.uri)}'." + ) + if strict: + raise RegistryEvidenceError(message) + log.debug("%s Omitting generated storage configuration.", message) + return None + + +async def discover_storage_resource( + server_uri: str | None, + server_url: str | None, + server_name: str | None, + idp: str, + *, + dev: bool, + timeout: int, +) -> RegistryResource | None: + """Return fresh registry evidence for a server's primary VOSpace service.""" + if server_uri is None: + message = "Server URI is required to inspect its VOSpace Service." + raise RegistryEvidenceError(message) + + evidence = await load_registry_evidence( + idp, + dev=dev, + timeout=timeout, + check_platforms=False, + ) + if not evidence.available: + errors = "; ".join(evidence.errors) + message = ( + f"Failed to inspect VOSpace registry records for IDP '{idp}': {errors}" + ) + raise RegistryEvidenceError(message) + + endpoints = [ + resource + for resource in evidence.resources + if resource.uri == server_uri and resource.uri.endswith("/skaha") + ] + matching_urls = [ + endpoint + for endpoint in endpoints + if server_url is not None and endpoint.url == server_url + ] + if len(matching_urls) == 1: + endpoint = matching_urls[0] + elif len(endpoints) == 1: + endpoint = endpoints[0] + elif not endpoints: + message = ( + f"No Science Platform registry record found for Server '{server_name}' " + f"with URI '{server_uri}'." + ) + raise RegistryEvidenceError(message) + else: + message = ( + f"Multiple Science Platform registry records found for Server " + f"'{server_name}' with URI '{server_uri}'." + ) + raise RegistryEvidenceError(message) + + storage_resources = [ + resource + for resource in evidence.resources + if resource.uri.endswith(f"/{evidence.preferred_storage_leaf}") + ] + return select_storage_resource(endpoint, storage_resources, strict=True) + + +async def prepare_enrichment_workers( + config: Configuration | None, + idp: str, + *, + endpoint: RegistryResource, + count: int, +) -> EnrichmentWorkers | None: + """Materialize credentials once, then isolate worker configuration state.""" + from canfar.client import HTTPClient # noqa: PLC0415 + + base_config = config or Configuration() + client = HTTPClient( + config=base_config, + authentication_idp=idp, + url=AnyHttpUrl(endpoint.url), + ) + token: str | None = None + certificate: Path | None = None + if client.authentication_record is not None: + try: + token, certfile = await client._materialize_credentials() # noqa: SLF001 + except ( + KeyError, + OSError, + AuthContextError, + AuthExpiredError, + CertificateError, + TypeError, + ValueError, + ) as exc: + log.debug("Skipping capability enrichment for IDP %s: %s", idp, exc) + return None + certificate = Path(certfile) if certfile is not None else None + + values = base_config.model_dump(mode="python") + configs = tuple(Configuration.model_validate(values) for _ in range(count)) + return EnrichmentWorkers(configs, token=token, certificate=certificate) diff --git a/canfar/server.py b/canfar/server.py index 9c56bc86..0d87c35d 100644 --- a/canfar/server.py +++ b/canfar/server.py @@ -4,7 +4,7 @@ import asyncio from dataclasses import dataclass -from typing import Literal +from typing import TYPE_CHECKING, Literal from xml.etree.ElementTree import ParseError import httpx @@ -12,11 +12,18 @@ from pydantic import AnyHttpUrl, AnyUrl, ValidationError from canfar import get_logger +from canfar._discovery import ( + RegistryEvidenceError, + discover_storage_resource, + load_registry_evidence, + prepare_enrichment_workers, + select_storage_resource, +) from canfar.auth.x509 import CertificateError from canfar.errors import ErrorCode, StructuredError from canfar.exceptions.context import AuthContextError, AuthExpiredError from canfar.hooks.httpx.auth import AuthenticationError as HTTPAuthenticationError -from canfar.idp import get_idp, registry_sources +from canfar.idp import get_idp from canfar.models.config import Configuration from canfar.models.http import ( DEFAULT_SERVER_CORES, @@ -25,10 +32,11 @@ Server, VOSpaceService, ) -from canfar.models.registry import IVOARegistrySearch from canfar.models.registry import Server as RegistryResource from canfar.utils import vosi -from canfar.utils.discover import Discover + +if TYPE_CHECKING: + from pathlib import Path log = get_logger(__name__) @@ -88,16 +96,6 @@ class ServerActivation: reason: Literal["active", "remembered", "single", "selected"] -@dataclass(frozen=True) -class _RegistryEvidence: - """Registry resources plus acquisition outcome for one IDP inspection.""" - - preferred_storage_leaf: str | None - resources: tuple[RegistryResource, ...] - errors: tuple[str, ...] - available: bool - - def discover( idp: str, *, @@ -394,91 +392,6 @@ def _resolve_selector( return None -async def _load_registry_evidence( - idp: str, - *, - dev: bool, - timeout: int, - check_platforms: bool, -) -> _RegistryEvidence: - """Acquire and extract registry records through one shared pipeline.""" - idp_info = get_idp(idp) - sources = registry_sources(idp, include_dev=dev) - development_sources = set(idp_info.dev_registries) - search = IVOARegistrySearch( - registries=sources, - preferred_storage_leaf=idp_info.preferred_storage_leaf, - ) - async with Discover(search, timeout=timeout) as discovery: - registries = await asyncio.gather( - *( - discovery.fetch( - url, - name, - development=url in development_sources, - ) - for url, name in sources.items() - ) - ) - successful = [registry for registry in registries if registry.success] - resources = [ - resource - for registry in successful - for resource in discovery.extract(registry, dev=dev) - ] - if check_platforms: - endpoints = [ - resource for resource in resources if resource.uri.endswith("/skaha") - ] - await asyncio.gather(*(discovery.check(endpoint) for endpoint in endpoints)) - - return _RegistryEvidence( - preferred_storage_leaf=idp_info.preferred_storage_leaf, - resources=tuple(resources), - errors=tuple( - f"{registry.name}: {registry.error}" - for registry in registries - if not registry.success - ), - available=bool(successful), - ) - - -async def _enrichment_config_copies( - config: Configuration | None, - idp: str, - *, - endpoint: RegistryResource, - count: int, -) -> list[Configuration] | None: - """Materialize credentials once, then isolate worker configuration state.""" - from canfar.client import HTTPClient # noqa: PLC0415 - - base_config = config or Configuration() - client = HTTPClient( - config=base_config, - authentication_idp=idp, - url=AnyHttpUrl(endpoint.url), - ) - if client.authentication_record is not None: - try: - await client._materialize_credentials() # noqa: SLF001 - except ( - KeyError, - OSError, - AuthContextError, - AuthExpiredError, - CertificateError, - TypeError, - ValueError, - ) as exc: - log.debug("Skipping capability enrichment for IDP %s: %s", idp, exc) - return None - - values = base_config.model_dump(mode="python") - return [Configuration.model_validate(values) for _ in range(count)] - - async def _discover_for_idp( idp: str, *, @@ -500,7 +413,7 @@ async def _discover_for_idp( Raises: ServerDiscoveryError: If registry retrieval fails. """ - evidence = await _load_registry_evidence( + evidence = await load_registry_evidence( idp, dev=dev, timeout=timeout, @@ -524,13 +437,13 @@ async def _discover_for_idp( for resource in evidence.resources if resource.uri.endswith(f"/{evidence.preferred_storage_leaf}") ] - worker_configs = await _enrichment_config_copies( + workers = await prepare_enrichment_workers( config, idp, endpoint=endpoints[0], count=len(endpoints), ) - if worker_configs is None: + if workers is None: return [_registry_resource_to_server(endpoint, idp) for endpoint in endpoints] return list( @@ -541,6 +454,8 @@ async def _discover_for_idp( endpoint, idp, config=worker_config, + token=workers.token, + certificate=workers.certificate, timeout=timeout, storage_resource=_select_storage_resource( endpoint, @@ -550,7 +465,7 @@ async def _discover_for_idp( ) for endpoint, worker_config in zip( endpoints, - worker_configs, + workers.configs, strict=True, ) ) @@ -565,40 +480,17 @@ def _host_slug(uri: AnyUrl) -> str | None: return uri.host.replace(".", "-") -def _registry_namespace(uri: str) -> str: - """Return the IVOA registry URI namespace before its resource leaf.""" - return uri.rpartition("/")[0] - - def _select_storage_resource( endpoint: RegistryResource, resources: list[RegistryResource], *, strict: bool, ) -> RegistryResource | None: - """Pair an endpoint with one unambiguous same-environment VOSpace record.""" - candidates = [ - resource - for resource in resources - if _registry_namespace(resource.uri) == _registry_namespace(endpoint.uri) - and resource.development == endpoint.development - ] - same_registry = [ - resource for resource in candidates if resource.registry == endpoint.registry - ] - preferred = same_registry or candidates - if len(preferred) == 1: - return preferred[0] - if len(preferred) > 1: - message = ( - "Multiple preferred VOSpace registry records found for Science " - f"Platform Server '{endpoint.name or endpoint.uri}' in namespace " - f"'{_registry_namespace(endpoint.uri)}'." - ) - if strict: - raise ServerFetchError(message) - log.debug("%s Omitting generated storage configuration.", message) - return None + """Map private registry ambiguity to the public server fetch error.""" + try: + return select_storage_resource(endpoint, resources, strict=strict) + except RegistryEvidenceError as exc: + raise ServerFetchError(str(exc)) from exc async def _discover_storage_resource( @@ -609,54 +501,17 @@ async def _discover_storage_resource( timeout: int, ) -> RegistryResource | None: """Return fresh registry evidence for a server's primary VOSpace service.""" - if server.uri is None: - msg = "Server URI is required to inspect its VOSpace Service." - raise ServerFetchError(msg) - - evidence = await _load_registry_evidence( - idp, - dev=dev, - timeout=timeout, - check_platforms=False, - ) - if not evidence.available: - errors = "; ".join(evidence.errors) - msg = f"Failed to inspect VOSpace registry records for IDP '{idp}': {errors}" - raise ServerFetchError(msg) - - endpoints = [ - resource - for resource in evidence.resources - if resource.uri == str(server.uri) and resource.uri.endswith("/skaha") - ] - matching_urls = [ - endpoint - for endpoint in endpoints - if server.url is not None and endpoint.url == str(server.url) - ] - if len(matching_urls) == 1: - endpoint = matching_urls[0] - elif len(endpoints) == 1: - endpoint = endpoints[0] - elif not endpoints: - msg = ( - f"No Science Platform registry record found for Server '{server.name}' " - f"with URI '{server.uri}'." - ) - raise ServerFetchError(msg) - else: - msg = ( - f"Multiple Science Platform registry records found for Server " - f"'{server.name}' with URI '{server.uri}'." + try: + return await discover_storage_resource( + str(server.uri) if server.uri is not None else None, + str(server.url) if server.url is not None else None, + server.name, + idp, + dev=dev, + timeout=timeout, ) - raise ServerFetchError(msg) - - storage_resources = [ - resource - for resource in evidence.resources - if resource.uri.endswith(f"/{evidence.preferred_storage_leaf}") - ] - return _select_storage_resource(endpoint, storage_resources, strict=True) + except RegistryEvidenceError as exc: + raise ServerFetchError(str(exc)) from exc def _configured_storage_resource(server: Server) -> RegistryResource | None: @@ -689,6 +544,8 @@ def _discovered_to_server( idp: str, *, config: Configuration | None = None, + token: str | None = None, + certificate: Path | None = None, timeout: int = 2, storage_resource: RegistryResource | None = None, ) -> Server: @@ -701,6 +558,8 @@ def _discovered_to_server( endpoint: Discovered registry endpoint. idp: Canonical IDP key. config: Configuration whose Authentication Record authorizes enrichment. + token: Pre-materialized runtime bearer token for worker isolation. + certificate: Pre-materialized runtime certificate for worker isolation. timeout: HTTP timeout in seconds for VOSI capabilities requests. storage_resource: Same-namespace preferred VOSpace registry record, if any. @@ -711,6 +570,8 @@ def _discovered_to_server( return enrich( server, config=config, + token=token, + certificate=certificate, strict=False, timeout=timeout, storage_resource=storage_resource, @@ -722,6 +583,8 @@ def enrich( *, config: Configuration | None = None, authentication_idp: str | None = None, + token: str | None = None, + certificate: Path | None = None, strict: bool = True, timeout: int = 2, storage_resource: RegistryResource | None | object = _STORAGE_RESOURCE_UNSET, @@ -735,6 +598,8 @@ def enrich( Authentication or Server Selection. authentication_idp: Optional Authentication Record selector. Defaults to the Server IDP, then the active Authentication. + token: Optional runtime bearer token for capability requests. + certificate: Optional runtime certificate for capability requests. strict: When ``False``, keep usable registry and existing storage data when session or storage capabilities cannot be retrieved or parsed. Other successful enrichment may still be returned, so the result can @@ -764,6 +629,8 @@ def enrich( ), config=base_config, authentication_idp=active_idp, + token=token, + certificate=certificate, strict=strict, timeout=timeout, ) @@ -776,6 +643,8 @@ def enrich( server.url, config=base_config, authentication_idp=active_idp, + token=token, + certificate=certificate, timeout=timeout, ) ) @@ -846,6 +715,8 @@ def _enrich_storage( storage_resource: RegistryResource | None, config: Configuration, authentication_idp: str, + token: str | None, + certificate: Path | None, strict: bool, timeout: int, ) -> Server: @@ -864,6 +735,8 @@ def _enrich_storage( AnyHttpUrl(storage_resource.url), config=config, authentication_idp=authentication_idp, + token=token, + certificate=certificate, timeout=timeout, ) valid = vosi.is_vospace_service(xml) @@ -915,6 +788,8 @@ def _fetch_capabilities( *, config: Configuration, authentication_idp: str, + token: str | None = None, + certificate: Path | None = None, timeout: int, ) -> str: """Fetch one VOSI capabilities document through the existing HTTP seam.""" @@ -923,6 +798,8 @@ def _fetch_capabilities( with HTTPClient( config=config, authentication_idp=authentication_idp, + token=token, + certificate=certificate, url=url, timeout=timeout, raise_http_errors=False, diff --git a/tests/test_server.py b/tests/test_server.py index 603fbaac..632c3613 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -565,7 +565,7 @@ def test_activation_freshly_inspects_storage_missing_during_discovery(self) -> N config = _anonymous_config() with ( - patch("canfar.server.Discover", return_value=mock_discovery), + patch("canfar._discovery.Discover", return_value=mock_discovery), patch( "canfar.client.Client", side_effect=_http_client_factory(transport), @@ -640,7 +640,7 @@ def enriched( ) with ( - patch("canfar.server.Discover", return_value=mock_discovery), + patch("canfar._discovery.Discover", return_value=mock_discovery), patch("canfar.server.enrich", side_effect=enriched), ): servers = await _discover_for_idp("srcnet") @@ -753,6 +753,8 @@ def convert( worker_config = kwargs["config"] assert isinstance(worker_config, Configuration) worker_configs.append(worker_config) + assert kwargs["token"] == "current-token" + assert kwargs["certificate"] is None concurrent.wait() return Server( idp=idp, @@ -765,7 +767,7 @@ def convert( materialize = AsyncMock(return_value=("current-token", None)) with ( - patch("canfar.server.Discover", return_value=mock_discovery), + patch("canfar._discovery.Discover", return_value=mock_discovery), patch("canfar.server._discovered_to_server", side_effect=convert), patch( "canfar.client.HTTPClient._materialize_credentials", @@ -779,6 +781,65 @@ def convert( assert len({id(worker_config) for worker_config in worker_configs}) == 2 assert all(worker_config is not config for worker_config in worker_configs) + @pytest.mark.asyncio + async def test_worker_requests_use_pre_materialized_runtime_token(self) -> None: + """Fan-out requests cannot invoke saved-record refresh or persistence.""" + endpoint = DiscoveredServer( + registry="SRCNet", + uri="ivo://site.example/skaha", + url="https://site.example/skaha", + status=200, + name="site", + ) + mock_discovery = AsyncMock() + mock_discovery.fetch.return_value = MagicMock(success=True, content="line") + mock_discovery.extract = MagicMock(return_value=[endpoint]) + mock_discovery.check = AsyncMock(side_effect=lambda item: item) + mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) + mock_discovery.__aexit__ = AsyncMock(return_value=None) + config = Configuration( + active=ActiveConfig(authentication="srcnet", server=None), + authentication={"srcnet": OIDCCredential(idp="srcnet")}, + servers={}, + ) + requests: list[httpx.Request] = [] + session_capabilities = """ + + + + https://site.example/skaha/v1 + + + + + """ + + def response(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, text=session_capabilities, request=request) + + materialize = AsyncMock(return_value=("runtime-token", None)) + with ( + patch("canfar._discovery.Discover", return_value=mock_discovery), + patch( + "canfar.client.HTTPClient._materialize_credentials", + new=materialize, + ), + patch( + "canfar.client.Client", + side_effect=_http_client_factory(httpx.MockTransport(response)), + ), + patch("canfar.auth.oidc.sync_refresh") as refresh, + ): + [server] = await _discover_for_idp("srcnet", config=config) + + assert server.version == "v1" + materialize.assert_awaited_once_with() + refresh.assert_not_called() + assert [request.headers["Authorization"] for request in requests] == [ + "Bearer runtime-token" + ] + @pytest.mark.parametrize( "capabilities_case", [ @@ -1020,7 +1081,7 @@ def test_discover_keys_unnamed_server_by_host_slug(self, tmp_path: Path) -> None with ( patch("canfar.models.config.CONFIG_PATH", config_path), - patch("canfar.server.Discover", return_value=mock_discovery), + patch("canfar._discovery.Discover", return_value=mock_discovery), patch( "canfar.server.enrich", side_effect=lambda item, **_kwargs: item.model_copy( @@ -1058,7 +1119,7 @@ async def test_discover_for_idp_converts_active_endpoints(self) -> None: mock_discovery.__aexit__ = AsyncMock(return_value=None) with ( - patch("canfar.server.Discover", return_value=mock_discovery), + patch("canfar._discovery.Discover", return_value=mock_discovery), patch( "canfar.server.enrich", side_effect=lambda item, **_kwargs: item.model_copy( @@ -1147,7 +1208,7 @@ async def test_discover_for_idp_raises_when_registry_fetch_fails(self) -> None: mock_discovery.__aexit__ = AsyncMock(return_value=None) with ( - patch("canfar.server.Discover", return_value=mock_discovery), + patch("canfar._discovery.Discover", return_value=mock_discovery), pytest.raises(ServerDiscoveryError, match="Failed to discover"), ): await _discover_for_idp("cadc") @@ -1164,7 +1225,9 @@ async def test_discover_for_idp_honors_dev_sources_and_timeout(self) -> None: mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) mock_discovery.__aexit__ = AsyncMock(return_value=None) - with patch("canfar.server.Discover", return_value=mock_discovery) as factory: + with patch( + "canfar._discovery.Discover", return_value=mock_discovery + ) as factory: servers = await _discover_for_idp("cadc", dev=True, timeout=11) assert servers == [] From afb822f0b1f9c47be3c336b1a6fb465576826ce0 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 18:55:20 -0700 Subject: [PATCH 28/47] fix(server): preserve environment credentials --- canfar/_discovery.py | 2 +- canfar/server.py | 23 ++++---- tests/test_server.py | 127 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 11 deletions(-) diff --git a/canfar/_discovery.py b/canfar/_discovery.py index 6040b068..ae437795 100644 --- a/canfar/_discovery.py +++ b/canfar/_discovery.py @@ -209,7 +209,7 @@ async def prepare_enrichment_workers( ) token: str | None = None certificate: Path | None = None - if client.authentication_record is not None: + if client.uses_runtime_credentials or client.authentication_record is not None: try: token, certfile = await client._materialize_credentials() # noqa: SLF001 except ( diff --git a/canfar/server.py b/canfar/server.py index 0d87c35d..2d11d58b 100644 --- a/canfar/server.py +++ b/canfar/server.py @@ -4,7 +4,7 @@ import asyncio from dataclasses import dataclass -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal from xml.etree.ElementTree import ParseError import httpx @@ -795,15 +795,18 @@ def _fetch_capabilities( """Fetch one VOSI capabilities document through the existing HTTP seam.""" from canfar.client import HTTPClient # noqa: PLC0415 - with HTTPClient( - config=config, - authentication_idp=authentication_idp, - token=token, - certificate=certificate, - url=url, - timeout=timeout, - raise_http_errors=False, - ) as client: + client_kwargs: dict[str, Any] = { + "config": config, + "authentication_idp": authentication_idp, + "url": url, + "timeout": timeout, + "raise_http_errors": False, + } + if token is not None: + client_kwargs["token"] = token + if certificate is not None: + client_kwargs["certificate"] = certificate + with HTTPClient(**client_kwargs) as client: request_client = client.client request_client.headers["Accept"] = "application/xml" request_client.headers.pop("Content-Type", None) diff --git a/tests/test_server.py b/tests/test_server.py index 632c3613..4da0a242 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -840,6 +840,133 @@ def response(request: httpx.Request) -> httpx.Response: "Bearer runtime-token" ] + @pytest.mark.asyncio + async def test_environment_token_materializes_without_saved_credential( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Environment bearer auth survives preparation and worker fan-out.""" + endpoint = DiscoveredServer( + registry="SRCNet", + uri="ivo://site.example/skaha", + url="https://site.example/skaha", + status=200, + name="site", + ) + mock_discovery = AsyncMock() + mock_discovery.fetch.return_value = MagicMock(success=True, content="line") + mock_discovery.extract = MagicMock(return_value=[endpoint]) + mock_discovery.check = AsyncMock(side_effect=lambda item: item) + mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) + mock_discovery.__aexit__ = AsyncMock(return_value=None) + requests: list[httpx.Request] = [] + session_capabilities = """ + + + + https://site.example/skaha/v1 + + + + + """ + + def response(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, text=session_capabilities, request=request) + + monkeypatch.delenv("CANFAR_CERTIFICATE", raising=False) + monkeypatch.setenv("CANFAR_TOKEN", "environment-token") + with ( + patch("canfar._discovery.Discover", return_value=mock_discovery), + patch( + "canfar.client.Client", + side_effect=_http_client_factory(httpx.MockTransport(response)), + ), + ): + [server] = await _discover_for_idp( + "srcnet", + config=_anonymous_config(idp="srcnet"), + ) + + assert server.version == "v1" + assert [request.headers["Authorization"] for request in requests] == [ + "Bearer environment-token" + ] + + @pytest.mark.asyncio + async def test_environment_certificate_materializes_without_saved_credential( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """Environment certificate auth survives preparation and worker fan-out.""" + endpoint = DiscoveredServer( + registry="SRCNet", + uri="ivo://site.example/skaha", + url="https://site.example/skaha", + status=200, + name="site", + ) + mock_discovery = AsyncMock() + mock_discovery.fetch.return_value = MagicMock(success=True, content="line") + mock_discovery.extract = MagicMock(return_value=[endpoint]) + mock_discovery.check = AsyncMock(side_effect=lambda item: item) + mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) + mock_discovery.__aexit__ = AsyncMock(return_value=None) + session_capabilities = """ + + + + https://site.example/skaha/v1 + + + + + """ + certificate = tmp_path / "environment.pem" + valid = MagicMock(return_value=certificate.as_posix()) + ssl_context = MagicMock() + + monkeypatch.delenv("CANFAR_TOKEN", raising=False) + monkeypatch.setenv("CANFAR_CERTIFICATE", certificate.as_posix()) + with ( + patch("canfar._discovery.Discover", return_value=mock_discovery), + patch( + "canfar.client.x509.inspect", + return_value={ + "path": certificate.as_posix(), + "expiry": 9_999_999_999.0, + }, + ), + patch("canfar.client.x509.valid", new=valid), + patch( + "canfar.client.HTTPClient._get_ssl_context", + return_value=ssl_context, + ) as get_ssl_context, + patch( + "canfar.client.Client", + side_effect=_http_client_factory( + httpx.MockTransport( + lambda request: httpx.Response( + 200, + text=session_capabilities, + request=request, + ) + ) + ), + ), + ): + [server] = await _discover_for_idp( + "srcnet", + config=_anonymous_config(idp="srcnet"), + ) + + assert server.version == "v1" + valid.assert_called_once_with(certificate) + get_ssl_context.assert_called_once_with(certificate) + @pytest.mark.parametrize( "capabilities_case", [ From 94c3b515e49fb4b98ebac80b15bf48659f68a65e Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 19:03:56 -0700 Subject: [PATCH 29/47] fix(ci): support pinned ty checks --- canfar/_discovery.py | 2 +- canfar/_storage.py | 2 +- canfar/cli/data.py | 2 +- canfar/server.py | 4 +++- pyproject.toml | 3 +++ 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/canfar/_discovery.py b/canfar/_discovery.py index ae437795..d8bbcc28 100644 --- a/canfar/_discovery.py +++ b/canfar/_discovery.py @@ -201,7 +201,7 @@ async def prepare_enrichment_workers( """Materialize credentials once, then isolate worker configuration state.""" from canfar.client import HTTPClient # noqa: PLC0415 - base_config = config or Configuration() + base_config = config or Configuration() # ty: ignore[missing-argument] client = HTTPClient( config=base_config, authentication_idp=idp, diff --git a/canfar/_storage.py b/canfar/_storage.py index 55627fd7..abb4231b 100644 --- a/canfar/_storage.py +++ b/canfar/_storage.py @@ -28,7 +28,7 @@ def _vospace_source( @asynccontextmanager async def source() -> AsyncIterator[AbstractFileSystem]: - config = Configuration() + config = Configuration() # ty: ignore[missing-argument] endpoint, idp = config._resolve_storage(storage_name) # noqa: SLF001 try: client_kwargs: dict[str, Any] = { diff --git a/canfar/cli/data.py b/canfar/cli/data.py index e3324052..9e6b4377 100644 --- a/canfar/cli/data.py +++ b/canfar/cli/data.py @@ -36,7 +36,7 @@ async def _local_source() -> AsyncIterator[AbstractFileSystem]: def _sources() -> dict[str, AsyncFilesystemSource]: """Build the mapped sources for one data command invocation.""" - config = Configuration() + config = Configuration() # ty: ignore[missing-argument] sources = { storage_name: _vospace_source(storage_name) for server in config.servers.values() diff --git a/canfar/server.py b/canfar/server.py index 2d11d58b..d23fc952 100644 --- a/canfar/server.py +++ b/canfar/server.py @@ -775,7 +775,9 @@ def _enrich_storage( ) assert storage_resource is not None assert server.name is not None - service = VOSpaceService(uri=storage_resource.uri, url=storage_resource.url) + service = VOSpaceService.model_validate( + {"uri": storage_resource.uri, "url": storage_resource.url} + ) return server.model_copy( update={"storage": {**server.storage, server.name: service}}, diff --git a/pyproject.toml b/pyproject.toml index 0532e437..5cb414fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -233,6 +233,9 @@ docstring-code-line-length = "dynamic" # ty configuration (replaces mypy) [tool.ty] +[tool.ty.rules] +unused-ignore-comment = "ignore" + [tool.ty.environment] python-version = "3.10" From dc5fc5fe3f0b27aee94b0c817d2bf9f3f6cc4ab4 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 19:10:11 -0700 Subject: [PATCH 30/47] test: support Python 3.10 TOML parsing --- tests/test_data_dependencies.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_data_dependencies.py b/tests/test_data_dependencies.py index a46001df..6bcad0d0 100644 --- a/tests/test_data_dependencies.py +++ b/tests/test_data_dependencies.py @@ -4,7 +4,10 @@ from pathlib import Path -import tomllib +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib def test_tagged_data_dependencies_are_standard_dependencies() -> None: From cb36468a2102126c7b600c230e9b8686e50a5dbf Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 22 Jul 2026 21:36:50 -0700 Subject: [PATCH 31/47] fix(auth): reuse valid CADC proxy certificates --- canfar/auth/x509.py | 12 ++++++- canfar/cli/login.py | 2 +- canfar/cli/login_auth.py | 28 ++++++++++++++-- tests/test_auth_x509.py | 12 +++++++ tests/test_cli_login.py | 1 + tests/test_cli_login_auth.py | 62 ++++++++++++++++++++++++++++++++---- 6 files changed, 106 insertions(+), 11 deletions(-) diff --git a/canfar/auth/x509.py b/canfar/auth/x509.py index 70875bdf..8a45637f 100644 --- a/canfar/auth/x509.py +++ b/canfar/auth/x509.py @@ -25,6 +25,16 @@ class CertificateError(ValueError): """Raised when an X.509 certificate cannot be used.""" + def __init__( + self, + message: str, + *, + expired_at: datetime | None = None, + ) -> None: + """Initialize a certificate error with optional structured expiry.""" + super().__init__(message) + self.expired_at = expired_at + def _to_utc(value: datetime) -> datetime: """Return timezone aware datetime. @@ -88,7 +98,7 @@ def assert_valid_dates( f"Certificate {destination} expired on {valid_until.isoformat()}; " f"current time {now_utc.isoformat()}." ) - raise CertificateError(msg) + raise CertificateError(msg, expired_at=valid_until) def gather( diff --git a/canfar/cli/login.py b/canfar/cli/login.py index 14f2d5f2..c1b47fa1 100644 --- a/canfar/cli/login.py +++ b/canfar/cli/login.py @@ -66,7 +66,7 @@ def _login_flow( idp_info = get_idp(idp) try: - credential = authenticate_for_cli(idp_info, timeout=timeout) + credential = authenticate_for_cli(idp_info, timeout=timeout, force=force) except (ValueError, RuntimeError) as exc: get_console(stderr=True).print(f"[bold red]{exc}[/bold red]") raise typer.Exit(1) from exc diff --git a/canfar/cli/login_auth.py b/canfar/cli/login_auth.py index 27ac8698..0f79f7a4 100644 --- a/canfar/cli/login_auth.py +++ b/canfar/cli/login_auth.py @@ -6,6 +6,7 @@ import webbrowser from typing import TYPE_CHECKING, Any +import humanize import segno from rich import progress as rich_progress @@ -98,12 +99,14 @@ def authenticate_for_cli( idp_info: IdpInfo, *, timeout: int | None = None, + force: bool = False, ) -> AuthenticationCredential: """Acquire Authentication credentials interactively for CLI login. Args: idp_info: Built-in Identity Provider metadata. timeout: HTTP timeout in seconds for OIDC requests. + force: Obtain a new X.509 certificate instead of reusing an existing one. Returns: Saved-ready authentication credential without embedded server. @@ -113,19 +116,38 @@ def authenticate_for_cli( RuntimeError: If OIDC discovery URL is missing for an OIDC IDP. """ if idp_info.auth_mode == "x509": - return _authenticate_x509(idp_info.key) + return _authenticate_x509(idp_info.key, force=force) return _authenticate_oidc(idp_info, timeout=timeout) -def _authenticate_x509(idp: str) -> X509Credential: - """Run interactive X509 certificate acquisition. +def _authenticate_x509(idp: str, *, force: bool = False) -> X509Credential: + """Reuse a valid X.509 certificate or acquire a replacement. Args: idp: Canonical Identity Provider key. + force: Obtain a new certificate without inspecting the existing one. Returns: X509 credential record for persisted configuration. """ + if not force: + try: + info = x509.inspect() + except x509.CertificateError as exc: + if exc.expired_at is not None: + age = humanize.naturaltime(exc.expired_at) + console_utils.get_console().print( + f"[yellow]x509 certificate expired {age}[/yellow]" + ) + except (FileNotFoundError, PermissionError, ValueError): + pass + else: + return X509Credential( + idp=idp, + path=info["path"], + expiry=float(info["expiry"] or 0.0), + ) + return x509.authenticate_credential(X509Credential(idp=idp, expiry=0.0)) diff --git a/tests/test_auth_x509.py b/tests/test_auth_x509.py index 2bac41ee..d59c058f 100644 --- a/tests/test_auth_x509.py +++ b/tests/test_auth_x509.py @@ -130,6 +130,18 @@ def test_expiry_not_yet_valid() -> None: x509_auth.expiry(cert_path) +def test_expiry_exposes_expiration_time(tmp_path) -> None: + """Expired certificates expose their end time for human-facing diagnostics.""" + cert_path = tmp_path / "expired.pem" + generate_cert(cert_path, expired=True) + + with pytest.raises(x509_auth.CertificateError) as excinfo: + x509_auth.expiry(cert_path) + + assert excinfo.value.expired_at is not None + assert excinfo.value.expired_at < datetime.datetime.now(datetime.timezone.utc) + + def test_expiry_with_invalid_content() -> None: """Test that `expiry` raises ValueError for a file with invalid content.""" with tempfile.NamedTemporaryFile(mode="w", suffix=".pem") as temp_cert: diff --git a/tests/test_cli_login.py b/tests/test_cli_login.py index 7e41cd02..9e2094a7 100644 --- a/tests/test_cli_login.py +++ b/tests/test_cli_login.py @@ -253,6 +253,7 @@ def discover( assert result.exit_code == 0 authenticate.assert_called_once() assert authenticate.call_args.kwargs["timeout"] == 9 + assert authenticate.call_args.kwargs["force"] is True validate.assert_called_once() validated = validate.call_args.args[0] assert str(validated.uri) == _CADC_URI diff --git a/tests/test_cli_login_auth.py b/tests/test_cli_login_auth.py index 2102e69b..133d66ee 100644 --- a/tests/test_cli_login_auth.py +++ b/tests/test_cli_login_auth.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -10,6 +11,7 @@ from authlib.integrations.httpx_client import AsyncOAuth2Client from pydantic import AnyHttpUrl, SecretStr +from canfar.auth.x509 import CertificateError from canfar.cli.login_auth import _interactive_device_flow, authenticate_for_cli from canfar.idp import IdpInfo, get_idp from canfar.models.auth import DeviceAuthorization, OIDCCredential, X509Credential @@ -122,18 +124,66 @@ def test_authenticate_for_cli_presents_oidc_device_challenge( assert credential.expiry.refresh is None -def test_authenticate_for_cli_builds_x509_record_from_gather(tmp_path) -> None: - """CLI X.509 acquisition maps gathered certificate data without legacy state.""" +def test_authenticate_for_cli_reuses_valid_x509_certificate(tmp_path) -> None: + """CLI login reuses a valid certificate from the default Authentication Record.""" certificate = tmp_path / "cadcproxy.pem" - gathered = {"path": str(certificate), "expiry": 1893456000.0} + inspected = {"path": str(certificate), "expiry": 1893456000.0} - with patch("canfar.auth.x509.gather", return_value=gathered) as gather: + with ( + patch("canfar.auth.x509.inspect", return_value=inspected) as inspect, + patch("canfar.auth.x509.gather") as gather, + ): result = authenticate_for_cli(get_idp("cadc")) assert isinstance(result, X509Credential) assert result.idp == "cadc" assert result.path == certificate assert result.expiry == 1893456000.0 + inspect.assert_called_once_with() + gather.assert_not_called() + + +def test_authenticate_for_cli_reports_expired_x509_before_renewal(tmp_path) -> None: + """CLI login explains how long ago the default certificate expired.""" + certificate = tmp_path / "cadcproxy.pem" + expired_at = datetime.now(timezone.utc) - timedelta(minutes=10) + renewed = {"path": str(certificate), "expiry": 1893456000.0} + console = MagicMock() + + with ( + patch( + "canfar.auth.x509.inspect", + side_effect=CertificateError("expired", expired_at=expired_at), + ), + patch("canfar.auth.x509.gather", return_value=renewed) as gather, + patch( + "canfar.cli.login_auth.humanize.naturaltime", + return_value="10 minutes ago", + ), + patch("canfar.cli.login_auth.console_utils.get_console", return_value=console), + ): + result = authenticate_for_cli(get_idp("cadc")) + + assert result.path == certificate + gather.assert_called_once_with() + console.print.assert_called_once_with( + "[yellow]x509 certificate expired 10 minutes ago[/yellow]" + ) + + +def test_authenticate_for_cli_force_renews_x509_certificate(tmp_path) -> None: + """Forced CLI login obtains a new certificate without inspecting the old one.""" + certificate = tmp_path / "cadcproxy.pem" + gathered = {"path": str(certificate), "expiry": 1893456000.0} + + with ( + patch("canfar.auth.x509.inspect") as inspect, + patch("canfar.auth.x509.gather", return_value=gathered) as gather, + ): + result = authenticate_for_cli(get_idp("cadc"), force=True) + + assert result.path == certificate + inspect.assert_not_called() gather.assert_called_once_with() @@ -149,7 +199,7 @@ def test_authenticate_for_cli_normalizes_x509_gather_failure() -> None: match=r"^Failed to authenticate with X509 certificate:", ), ): - authenticate_for_cli(get_idp("cadc")) + authenticate_for_cli(get_idp("cadc"), force=True) def test_authenticate_for_cli_normalizes_malformed_x509_gather_result() -> None: @@ -161,7 +211,7 @@ def test_authenticate_for_cli_normalizes_malformed_x509_gather_result() -> None: match=r"^Failed to authenticate with X509 certificate:", ), ): - authenticate_for_cli(get_idp("cadc")) + authenticate_for_cli(get_idp("cadc"), force=True) def test_authenticate_for_cli_oidc_requires_discovery_url() -> None: From db110035cb7560581f647d214fcff8fa94e6acb6 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 24 Jul 2026 20:34:42 -0700 Subject: [PATCH 32/47] refactor(discovery): use pydantic models and concise names Rename the discovery seam for clarity and drop the last dataclasses in favour of pydantic models, per repo convention. - _discovery: load_registry_evidence -> discover, discover_storage_resource -> discover_storage (server_uri/server_url/server_name -> uri/url/name), prepare_enrichment_workers -> enrich, select_storage_resource -> select_storage; inline the single-use _registry_namespace helper. - RegistryEvidence and EnrichmentWorkers are frozen pydantic models. - Rename preferred_storage_leaf -> leaf across IdpInfo, IVOARegistrySearch, and RegistryEvidence. - auth.oidc: _refresh_parameters -> _refresh, _persist_refreshed_credential -> _persist. - Move _storage.py to storages.py, relocate the local filesystem adapter out of the CLI, and expose sources() so cli/data.py only mounts the Typer app. - Google-style docstrings on all touched functions. No behaviour change. --- canfar/_discovery.py | 139 ++++++++++++++------ canfar/auth/oidc.py | 4 +- canfar/cli/data.py | 39 +----- canfar/client.py | 4 +- canfar/hooks/httpx/auth.py | 6 +- canfar/idp.py | 8 +- canfar/models/registry.py | 2 +- canfar/{_storage.py => storages.py} | 54 +++++++- canfar/utils/discover.py | 2 +- tests/test_idp.py | 2 +- tests/{test_storage.py => test_storages.py} | 28 ++-- 11 files changed, 182 insertions(+), 106 deletions(-) rename canfar/{_storage.py => storages.py} (57%) rename tests/{test_storage.py => test_storages.py} (94%) diff --git a/canfar/_discovery.py b/canfar/_discovery.py index d8bbcc28..d3efa782 100644 --- a/canfar/_discovery.py +++ b/canfar/_discovery.py @@ -3,10 +3,9 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass from pathlib import Path -from pydantic import AnyHttpUrl +from pydantic import AnyHttpUrl, BaseModel, ConfigDict from canfar import get_logger from canfar.auth.x509 import CertificateError @@ -24,39 +23,64 @@ class RegistryEvidenceError(RuntimeError): """Raised when strict registry evidence is missing or ambiguous.""" -@dataclass(frozen=True) -class RegistryEvidence: - """Registry resources plus acquisition outcome for one IDP inspection.""" +class RegistryEvidence(BaseModel): + """Registry resources plus acquisition outcome for one IDP inspection. - preferred_storage_leaf: str | None + Attributes: + leaf: Preferred primary VOSpace registry URI leaf. + resources: Registry resources extracted from every successful registry. + errors: Human-readable failures keyed by registry name. + available: Whether at least one registry responded successfully. + """ + + model_config = ConfigDict(frozen=True) + + leaf: str | None resources: tuple[RegistryResource, ...] errors: tuple[str, ...] available: bool -@dataclass(frozen=True) -class EnrichmentWorkers: - """Isolated worker configs with one pre-materialized runtime credential.""" +class EnrichmentWorkers(BaseModel): + """Isolated worker configs with one pre-materialized runtime credential. + + Attributes: + configs: One isolated Configuration per concurrent worker. + token: Materialized bearer token, when the credential provides one. + certificate: Materialized X.509 certificate path, when applicable. + """ + + model_config = ConfigDict(frozen=True) configs: tuple[Configuration, ...] token: str | None = None certificate: Path | None = None -async def load_registry_evidence( +async def discover( idp: str, *, dev: bool, timeout: int, check_platforms: bool, ) -> RegistryEvidence: - """Acquire and extract registry records through one shared pipeline.""" + """Acquire and extract registry records through one shared pipeline. + + Args: + idp: Canonical Identity Provider key. + dev: Include development registries and endpoints during discovery. + timeout: HTTP timeout in seconds for registry requests. + check_platforms: Probe Science Platform endpoints for reachability. + + Returns: + RegistryEvidence: Extracted resources plus the acquisition outcome. + """ idp_info = get_idp(idp) sources = registry_sources(idp, include_dev=dev) development_sources = set(idp_info.dev_registries) search = IVOARegistrySearch( registries=sources, - preferred_storage_leaf=idp_info.preferred_storage_leaf, + leaf=idp_info.leaf, ) async with Discover(search, timeout=timeout) as discovery: registries = await asyncio.gather( @@ -82,7 +106,7 @@ async def load_registry_evidence( await asyncio.gather(*(discovery.check(endpoint) for endpoint in endpoints)) return RegistryEvidence( - preferred_storage_leaf=idp_info.preferred_storage_leaf, + leaf=idp_info.leaf, resources=tuple(resources), errors=tuple( f"{registry.name}: {registry.error}" @@ -93,22 +117,30 @@ async def load_registry_evidence( ) -def _registry_namespace(uri: str) -> str: - """Return the IVOA registry URI namespace before its resource leaf.""" - return uri.rpartition("/")[0] - - -def select_storage_resource( +def select_storage( endpoint: RegistryResource, resources: list[RegistryResource], *, strict: bool, ) -> RegistryResource | None: - """Pair an endpoint with one unambiguous same-environment VOSpace record.""" + """Pair an endpoint with one unambiguous same-environment VOSpace record. + + Args: + endpoint: Science Platform registry record to pair. + resources: Candidate VOSpace registry records. + strict: Raise instead of logging when the pairing is ambiguous. + + Returns: + RegistryResource | None: The single paired record, or None. + + Raises: + RegistryEvidenceError: If ``strict`` and the pairing is ambiguous. + """ + namespace = endpoint.uri.rpartition("/")[0] candidates = [ resource for resource in resources - if _registry_namespace(resource.uri) == _registry_namespace(endpoint.uri) + if resource.uri.rpartition("/")[0] == namespace and resource.development == endpoint.development ] same_registry = [ @@ -121,7 +153,7 @@ def select_storage_resource( message = ( "Multiple preferred VOSpace registry records found for Science " f"Platform Server '{endpoint.name or endpoint.uri}' in namespace " - f"'{_registry_namespace(endpoint.uri)}'." + f"'{namespace}'." ) if strict: raise RegistryEvidenceError(message) @@ -129,21 +161,37 @@ def select_storage_resource( return None -async def discover_storage_resource( - server_uri: str | None, - server_url: str | None, - server_name: str | None, +async def discover_storage( + uri: str | None, + url: str | None, + name: str | None, idp: str, *, dev: bool, timeout: int, ) -> RegistryResource | None: - """Return fresh registry evidence for a server's primary VOSpace service.""" - if server_uri is None: + """Return fresh registry evidence for a Server's primary VOSpace Service. + + Args: + uri: IVOA URI of the Science Platform Server. + url: URL of the Science Platform Server, used to disambiguate. + name: Server Name, used only for diagnostics. + idp: Canonical Identity Provider key. + dev: Include development registries and endpoints during discovery. + timeout: HTTP timeout in seconds for registry requests. + + Returns: + RegistryResource | None: The paired VOSpace record, or None. + + Raises: + RegistryEvidenceError: If the URI is missing, the registry is + unavailable, or the Server record is missing or ambiguous. + """ + if uri is None: message = "Server URI is required to inspect its VOSpace Service." raise RegistryEvidenceError(message) - evidence = await load_registry_evidence( + evidence = await discover( idp, dev=dev, timeout=timeout, @@ -159,12 +207,10 @@ async def discover_storage_resource( endpoints = [ resource for resource in evidence.resources - if resource.uri == server_uri and resource.uri.endswith("/skaha") + if resource.uri == uri and resource.uri.endswith("/skaha") ] matching_urls = [ - endpoint - for endpoint in endpoints - if server_url is not None and endpoint.url == server_url + endpoint for endpoint in endpoints if url is not None and endpoint.url == url ] if len(matching_urls) == 1: endpoint = matching_urls[0] @@ -172,33 +218,44 @@ async def discover_storage_resource( endpoint = endpoints[0] elif not endpoints: message = ( - f"No Science Platform registry record found for Server '{server_name}' " - f"with URI '{server_uri}'." + f"No Science Platform registry record found for Server '{name}' " + f"with URI '{uri}'." ) raise RegistryEvidenceError(message) else: message = ( f"Multiple Science Platform registry records found for Server " - f"'{server_name}' with URI '{server_uri}'." + f"'{name}' with URI '{uri}'." ) raise RegistryEvidenceError(message) storage_resources = [ resource for resource in evidence.resources - if resource.uri.endswith(f"/{evidence.preferred_storage_leaf}") + if resource.uri.endswith(f"/{evidence.leaf}") ] - return select_storage_resource(endpoint, storage_resources, strict=True) + return select_storage(endpoint, storage_resources, strict=True) -async def prepare_enrichment_workers( +async def enrich( config: Configuration | None, idp: str, *, endpoint: RegistryResource, count: int, ) -> EnrichmentWorkers | None: - """Materialize credentials once, then isolate worker configuration state.""" + """Materialize credentials once, then isolate worker configuration state. + + Args: + config: Configuration to derive workers from. Defaults to loading config. + idp: Canonical Identity Provider key. + endpoint: Science Platform registry record used to build the client. + count: Number of isolated worker configurations to produce. + + Returns: + EnrichmentWorkers | None: Workers, or None when credentials are absent + or unusable. + """ from canfar.client import HTTPClient # noqa: PLC0415 base_config = config or Configuration() # ty: ignore[missing-argument] @@ -227,4 +284,4 @@ async def prepare_enrichment_workers( values = base_config.model_dump(mode="python") configs = tuple(Configuration.model_validate(values) for _ in range(count)) - return EnrichmentWorkers(configs, token=token, certificate=certificate) + return EnrichmentWorkers(configs=configs, token=token, certificate=certificate) diff --git a/canfar/auth/oidc.py b/canfar/auth/oidc.py index a9c36b0c..3c87ca72 100644 --- a/canfar/auth/oidc.py +++ b/canfar/auth/oidc.py @@ -195,7 +195,7 @@ def _validated_refresh(refreshed: Any) -> dict[str, Any]: return dict(refreshed) -def _refresh_parameters( +def _refresh( credential: OIDCCredential, ) -> tuple[str, str, str, str] | None: """Return complete literal refresh inputs when the record is eligible.""" @@ -209,7 +209,7 @@ def _refresh_parameters( ) -def _persist_refreshed_credential( +def _persist( config: Configuration, credential: OIDCCredential, refreshed: dict[str, Any], diff --git a/canfar/cli/data.py b/canfar/cli/data.py index 9e6b4377..ae5b23d2 100644 --- a/canfar/cli/data.py +++ b/canfar/cli/data.py @@ -2,55 +2,30 @@ from __future__ import annotations -from contextlib import asynccontextmanager from typing import TYPE_CHECKING import typer -from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper -from fsspec.implementations.local import LocalFileSystem from fsspec_cli import App from typer.core import TyperGroup from typer.main import get_group -from canfar._storage import _vospace_source -from canfar.models.config import Configuration +from canfar.storages import sources if TYPE_CHECKING: - from collections.abc import AsyncIterator - - from fsspec.spec import AbstractFileSystem - from fsspec_cli import AsyncFilesystemSource from typer._click.core import Command, Context _DATA_GROUP_META_KEY = "canfar.data_group" -@asynccontextmanager -async def _local_source() -> AsyncIterator[AbstractFileSystem]: - """Yield a fresh asynchronous wrapper around the local filesystem.""" - yield AsyncFileSystemWrapper( - LocalFileSystem(skip_instance_cache=True), - asynchronous=True, - ) - - -def _sources() -> dict[str, AsyncFilesystemSource]: - """Build the mapped sources for one data command invocation.""" - config = Configuration() # ty: ignore[missing-argument] - sources = { - storage_name: _vospace_source(storage_name) - for server in config.servers.values() - for storage_name in server.storage - } - sources["local"] = _local_source - return sources - - def _upstream_group() -> TyperGroup: - """Build the released upstream application with CANFAR policy.""" + """Build the released upstream application with CANFAR policy. + + Returns: + TyperGroup: The upstream command group bound to configured sources. + """ return get_group( App( - _sources(), + sources(), capabilities={"recursion": {"copy": True, "remove": False}}, ).typer_app ) diff --git a/canfar/client.py b/canfar/client.py index bd526abf..0829156d 100644 --- a/canfar/client.py +++ b/canfar/client.py @@ -250,11 +250,11 @@ async def _materialize_credentials(self) -> tuple[str | None, str | None]: raise TypeError if credential.expired: - parameters = oidc._refresh_parameters(credential) # noqa: SLF001 + parameters = oidc._refresh(credential) # noqa: SLF001 if parameters is None: raise ValueError refreshed = await oidc.refresh(*parameters) - credential = oidc._persist_refreshed_credential( # noqa: SLF001 + credential = oidc._persist( # noqa: SLF001 self.config, credential, refreshed, diff --git a/canfar/hooks/httpx/auth.py b/canfar/hooks/httpx/auth.py index 0f99b8b6..03e2e4f5 100644 --- a/canfar/hooks/httpx/auth.py +++ b/canfar/hooks/httpx/auth.py @@ -78,7 +78,7 @@ def _apply_refreshed_token( request: httpx.Request, ) -> None: """Atomically persist refreshed OIDC state, then update active headers.""" - updated = oidc._persist_refreshed_credential( # noqa: SLF001 + updated = oidc._persist( # noqa: SLF001 client.config, credential, refreshed ) log.debug("Authentication refreshed and configuration saved.") @@ -120,7 +120,7 @@ def hook(request: httpx.Request) -> None: log.debug("Skipping auth refresh, access token is not expired.") return - parameters = oidc._refresh_parameters(credential) # noqa: SLF001 + parameters = oidc._refresh(credential) # noqa: SLF001 if parameters is None: log.warning("OIDC Authentication Record cannot be refreshed.") return @@ -182,7 +182,7 @@ async def ahook(request: httpx.Request) -> None: ) return - parameters = oidc._refresh_parameters(credential) # noqa: SLF001 + parameters = oidc._refresh(credential) # noqa: SLF001 if parameters is None: log.warning("OIDC Authentication Record cannot be refreshed.") return diff --git a/canfar/idp.py b/canfar/idp.py index 6f715f05..5ec39fa1 100644 --- a/canfar/idp.py +++ b/canfar/idp.py @@ -19,7 +19,7 @@ class IdpInfo(BaseModel): auth_mode: Default authentication mode for the IDP. registry_url: IVOA registry resource-caps URL for server discovery. registry_name: Registry display name used for server discovery. - preferred_storage_leaf: Preferred primary VOSpace registry URI leaf. + leaf: Preferred primary VOSpace registry URI leaf. dev_registries: Development registry resource-caps URLs keyed by URL. oidc_discovery_url: OIDC discovery URL when ``auth_mode`` is ``oidc``. oidc_issuer: Exact issuer expected from OIDC discovery metadata. @@ -37,7 +37,7 @@ class IdpInfo(BaseModel): default=None, description="Registry display name used for server discovery.", ) - preferred_storage_leaf: str | None = Field( + leaf: str | None = Field( default=None, description="Preferred primary VOSpace registry URI leaf.", ) @@ -61,7 +61,7 @@ class IdpInfo(BaseModel): name="Canadian Astronomy Data Centre", auth_mode="x509", registry_name="CADC", - preferred_storage_leaf="arc", + leaf="arc", registry_url=AnyHttpUrl( "https://ws.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/reg/resource-caps" ), @@ -76,7 +76,7 @@ class IdpInfo(BaseModel): name="SKA Regional Centre Network", auth_mode="oidc", registry_name="SRCNet", - preferred_storage_leaf="cavern", + leaf="cavern", registry_url=AnyHttpUrl("https://spsrc27.iaa.csic.es/reg/resource-caps"), oidc_discovery_url=AnyHttpUrl( "https://ska-iam.stfc.ac.uk/.well-known/openid-configuration" diff --git a/canfar/models/registry.py b/canfar/models/registry.py index 0e7f8684..ba33aea4 100644 --- a/canfar/models/registry.py +++ b/canfar/models/registry.py @@ -25,7 +25,7 @@ class IVOARegistrySearch(BaseModel): } ) - preferred_storage_leaf: str | None = None + leaf: str | None = None names: dict[str, str] = Field( default={ diff --git a/canfar/_storage.py b/canfar/storages.py similarity index 57% rename from canfar/_storage.py rename to canfar/storages.py index abb4231b..e5a8f4aa 100644 --- a/canfar/_storage.py +++ b/canfar/storages.py @@ -1,10 +1,13 @@ -"""Private adapters for configured VOSpace Services.""" +"""Adapters for the configured VOSpace Services and the local filesystem.""" from __future__ import annotations from contextlib import asynccontextmanager from typing import TYPE_CHECKING, Any +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper +from fsspec.implementations.local import LocalFileSystem + from canfar.client import HTTPClient from canfar.exceptions.context import AuthContextError from canfar.models.config import Configuration @@ -18,18 +21,27 @@ from pydantic import SecretStr -def _vospace_source( - storage_name: str, +def _vospace( + name: str, *, token: str | SecretStr | None = None, certificate: Path | None = None, ) -> AsyncFilesystemSource: - """Return a fresh authenticated async filesystem source.""" + """Return a fresh authenticated async filesystem source. + + Args: + name: Storage Name of the configured VOSpace Service. + token: Runtime bearer token, preferred over any saved credential. + certificate: Runtime X.509 certificate path. + + Returns: + AsyncFilesystemSource: Factory yielding one authenticated filesystem. + """ @asynccontextmanager async def source() -> AsyncIterator[AbstractFileSystem]: config = Configuration() # ty: ignore[missing-argument] - endpoint, idp = config._resolve_storage(storage_name) # noqa: SLF001 + endpoint, idp = config._resolve_storage(name) # noqa: SLF001 try: client_kwargs: dict[str, Any] = { "config": config, @@ -71,3 +83,35 @@ async def source() -> AsyncIterator[AbstractFileSystem]: await filesystem.aclose() return source + + +@asynccontextmanager +async def _local() -> AsyncIterator[AbstractFileSystem]: + """Yield a fresh asynchronous wrapper around the local filesystem. + + Yields: + AbstractFileSystem: An async-wrapped local filesystem. + """ + yield AsyncFileSystemWrapper( + LocalFileSystem(skip_instance_cache=True), + asynchronous=True, + ) + + +def sources() -> dict[str, AsyncFilesystemSource]: + """Build the mapped storage sources for one data command invocation. + + Every configured VOSpace Service is mapped by its Storage Name, plus the + always-available ``local`` filesystem. + + Returns: + dict[str, AsyncFilesystemSource]: Sources keyed by Storage Name. + """ + config = Configuration() # ty: ignore[missing-argument] + mapped = { + name: _vospace(name) + for server in config.servers.values() + for name in server.storage + } + mapped["local"] = _local + return mapped diff --git a/canfar/utils/discover.py b/canfar/utils/discover.py index 2a26f1eb..7b630ecc 100644 --- a/canfar/utils/discover.py +++ b/canfar/utils/discover.py @@ -111,7 +111,7 @@ def extract(self, registry: IVOARegistry, dev: bool = False) -> list[Server]: uri, url = uri.strip(), url.strip() leaf = uri.rpartition("/")[2] - if leaf in {"skaha", self.config.preferred_storage_leaf}: + if leaf in {"skaha", self.config.leaf}: url = _without_terminal_capabilities(url) if url is None: continue diff --git a/tests/test_idp.py b/tests/test_idp.py index d37c5923..4fb3b6c1 100644 --- a/tests/test_idp.py +++ b/tests/test_idp.py @@ -25,7 +25,7 @@ def test_list_idps_returns_idp_info_instances(self) -> None: def test_builtin_primary_storage_preferences(self) -> None: """Each built-in IDP declares its primary VOSpace resource leaf.""" - assert {idp.key: idp.preferred_storage_leaf for idp in list_idps()} == { + assert {idp.key: idp.leaf for idp in list_idps()} == { "cadc": "arc", "srcnet": "cavern", } diff --git a/tests/test_storage.py b/tests/test_storages.py similarity index 94% rename from tests/test_storage.py rename to tests/test_storages.py index 2e0c4b45..d75490a8 100644 --- a/tests/test_storage.py +++ b/tests/test_storages.py @@ -1,4 +1,4 @@ -"""Tests for private authenticated VOSpace source adapters.""" +"""Tests for the VOSpace and local storage source adapters.""" from __future__ import annotations @@ -10,11 +10,11 @@ import vosfs from pydantic import AnyHttpUrl, AnyUrl -from canfar._storage import _vospace_source from canfar.exceptions.context import AuthContextError from canfar.models.active import ActiveConfig from canfar.models.config import Configuration from canfar.models.http import Server, VOSpaceService +from canfar.storages import _vospace from tests.helpers.config import oidc_credential, x509_credential if TYPE_CHECKING: @@ -76,7 +76,7 @@ async def test_source_reloads_config_and_runtime_token_wins( """Entry reloads endpoint state and keeps token-over-certificate precedence.""" config = _config(credential=oidc_credential("inactive")) config.save() - source = _vospace_source( + source = _vospace( "archive", token="runtime-token", certificate=tmp_path / "ignored.pem", @@ -115,7 +115,7 @@ def build(endpoint: str, **kwargs: Any) -> _Filesystem: return filesystem monkeypatch.setattr(vosfs, "VOSpaceFileSystem", build) - source = _vospace_source("archive") + source = _vospace("archive") async with source() as first: assert first.closed is False @@ -139,7 +139,7 @@ async def test_environment_token_preserves_runtime_precedence( monkeypatch.setenv("CANFAR_TOKEN", "environment-token") monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) - async with _vospace_source("archive")() as filesystem: + async with _vospace("archive")() as filesystem: assert filesystem.kwargs == { "token": "environment-token", "asynchronous": True, @@ -165,7 +165,7 @@ async def test_environment_certificate_preserves_runtime_precedence( monkeypatch.setattr("canfar.client.x509.valid", valid) monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) - async with _vospace_source("archive")() as filesystem: + async with _vospace("archive")() as filesystem: assert filesystem.kwargs == { "certfile": certificate.as_posix(), "asynchronous": True, @@ -198,7 +198,7 @@ async def test_expired_inactive_oidc_refreshes_once_and_persists( monkeypatch.setattr("canfar.client.oidc.refresh", refresh) monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) - async with _vospace_source("archive")() as filesystem: + async with _vospace("archive")() as filesystem: assert filesystem.kwargs["token"] == "new-access-secret" refresh.assert_awaited_once_with( @@ -224,7 +224,7 @@ async def test_valid_saved_oidc_access_token_is_reused( monkeypatch.setattr("canfar.client.oidc.refresh", refresh) monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) - async with _vospace_source("archive")() as filesystem: + async with _vospace("archive")() as filesystem: assert filesystem.kwargs["token"] == "current-token" refresh.assert_not_awaited() @@ -247,7 +247,7 @@ def inspect_certificate(path: Path) -> dict[str, object]: monkeypatch.setattr("canfar.client.x509.inspect", inspect_certificate) monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) - async with _vospace_source("archive")() as filesystem: + async with _vospace("archive")() as filesystem: assert filesystem.kwargs["certfile"] == certificate.as_posix() inspect.assert_called_once_with(certificate) @@ -269,7 +269,7 @@ async def test_runtime_x509_overrides_saved_authentication_record( monkeypatch.setattr("canfar.client.x509.valid", valid) monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) - async with _vospace_source("archive", certificate=certificate)() as filesystem: + async with _vospace("archive", certificate=certificate)() as filesystem: assert filesystem.kwargs["certfile"] == certificate.as_posix() valid.assert_called_once_with(certificate) @@ -291,7 +291,7 @@ async def test_invalid_saved_x509_fails_before_vospace( monkeypatch.setattr(vosfs, "VOSpaceFileSystem", constructor) with pytest.raises(AuthContextError, match="canfar login") as exc_info: - async with _vospace_source("archive")(): + async with _vospace("archive")(): pass assert "certificate parse detail" not in str(exc_info.value) @@ -316,7 +316,7 @@ def build(endpoint: str, **kwargs: Any) -> _Filesystem: monkeypatch.setattr(vosfs, "VOSpaceFileSystem", build) async def use_source() -> None: - async with _vospace_source("archive")(): + async with _vospace("archive")(): if exit_kind == "error": raise RuntimeError await asyncio.Event().wait() @@ -352,7 +352,7 @@ async def test_unrefreshable_oidc_fails_secret_safe_before_vospace( monkeypatch.setattr(vosfs, "VOSpaceFileSystem", constructor) with pytest.raises(AuthContextError, match="canfar login") as exc_info: - async with _vospace_source("archive")(): + async with _vospace("archive")(): pass message = str(exc_info.value) @@ -371,7 +371,7 @@ async def test_empty_saved_oidc_token_fails_cleanly( monkeypatch.setattr(vosfs, "VOSpaceFileSystem", constructor) with pytest.raises(AuthContextError, match="canfar login"): - async with _vospace_source("archive")(): + async with _vospace("archive")(): pass constructor.assert_not_called() From fe175f646745596a40c7a3179e5832de6cdff329 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 24 Jul 2026 20:34:52 -0700 Subject: [PATCH 33/47] feat(storage): map arc and vault VOSpace Services for CADC The CADC Science Platform Server shipped no default storage, and discovery named a discovered Service after its Server Name, so the arc Service was addressed as `canfar:` and vault was never configured at all. - Add both CADC VOSpace Services to the default Server, taken from the IVOA registry: arc (ws-uv.canfar.net/arc) and vault (cadc-west-01.canfar.net/vault). Both authenticate with the same cadc credentials, so `canfar data ls arc:/` and `vault:/` work out of the box. - Heal Servers saved by an older client on load: an entry keyed by Server Name is restored to its registry leaf and missing default Services are merged in. Scoped to the default Servers, so federated Servers keep their Server Name keys and deliberate Storage Names are left untouched. - Merge discovered Services by IVOA URI so rediscovery refreshes an existing entry in place instead of duplicating it under the Server Name. Verified against the live services with a CADC certificate. --- canfar/models/config.py | 41 +++++++++++++++++++- canfar/server.py | 77 ++++++++++++++++++++++++------------- tests/test_data_smoke.py | 6 +-- tests/test_models_config.py | 72 ++++++++++++++++++++++++++++++++-- tests/test_server.py | 48 +++++++++++++---------- 5 files changed, 190 insertions(+), 54 deletions(-) diff --git a/canfar/models/config.py b/canfar/models/config.py index 3e16d46d..95cec056 100644 --- a/canfar/models/config.py +++ b/canfar/models/config.py @@ -36,7 +36,7 @@ AuthenticationCredential, X509Credential, ) -from canfar.models.http import Server +from canfar.models.http import Server, VOSpaceService from canfar.models.registry import ContainerRegistry log = get_logger(__name__) @@ -65,6 +65,16 @@ url=AnyHttpUrl("https://ws-uv.canfar.net/skaha"), version="v1", auths=["x509"], + storage={ + "arc": VOSpaceService( + uri=AnyUrl("ivo://cadc.nrc.ca/arc"), + url=AnyHttpUrl("https://ws-uv.canfar.net/arc"), + ), + "vault": VOSpaceService( + uri=AnyUrl("ivo://cadc.nrc.ca/vault"), + url=AnyHttpUrl("https://cadc-west-01.canfar.net/vault"), + ), + }, ), } @@ -178,9 +188,38 @@ def settings_customise_sources( file_secret_settings, ) + def _heal_default_storage(self) -> None: + """Restore default Storage Names on Servers saved by an older client. + + Older clients keyed a discovered VOSpace Service by its Server Name, so + an existing configuration holds ``canfar`` instead of ``arc`` and never + gained later defaults such as ``vault``. Healing is scoped to the + default Servers so federated Servers keep their Server Name keys. + """ + for name, default in default_servers.items(): + server = self.servers.get(name) + if server is None or not default.storage or server.idp != default.idp: + continue + storage = dict(server.storage) + legacy = storage.pop(name, None) + if legacy is None and storage: + # Deliberate Storage Names are configuration, not stale defaults. + continue + if legacy is not None: + leaf = str(legacy.uri).rpartition("/")[2] or name + storage.setdefault(leaf, legacy) + for storage_name, service in default.storage.items(): + storage.setdefault(storage_name, service.model_copy(deep=True)) + if storage != server.storage: + self.servers[name] = server.model_copy( + update={"storage": storage}, + deep=True, + ) + @model_validator(mode="after") def _normalize_and_validate_servers(self) -> Configuration: """Inject Server Names and validate Server and Storage Name keys.""" + self._heal_default_storage() updated: dict[str, Server] = {} server_name_by_storage_name: dict[str, str] = {} for name, server in self.servers.items(): diff --git a/canfar/server.py b/canfar/server.py index d23fc952..fee7646d 100644 --- a/canfar/server.py +++ b/canfar/server.py @@ -3,22 +3,15 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal from xml.etree.ElementTree import ParseError import httpx from defusedxml.common import DefusedXmlException -from pydantic import AnyHttpUrl, AnyUrl, ValidationError - -from canfar import get_logger -from canfar._discovery import ( - RegistryEvidenceError, - discover_storage_resource, - load_registry_evidence, - prepare_enrichment_workers, - select_storage_resource, -) +from pydantic import AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, ValidationError + +from canfar import _discovery, get_logger +from canfar._discovery import RegistryEvidenceError from canfar.auth.x509 import CertificateError from canfar.errors import ErrorCode, StructuredError from canfar.exceptions.context import AuthContextError, AuthExpiredError @@ -88,14 +81,44 @@ def __init__(self, idp: str, servers: list[Server]) -> None: self.servers = servers -@dataclass(frozen=True) -class ServerActivation: - """Result from activating an Authentication and Server pair.""" +class ServerActivation(BaseModel): + """Result from activating an Authentication and Server pair. + + Attributes: + server: The activated Science Platform Server. + reason: Why this Server was chosen. + """ + + model_config = ConfigDict(frozen=True) server: Server reason: Literal["active", "remembered", "single", "selected"] +def _merge_storage( + known: dict[str, VOSpaceService], + found: dict[str, VOSpaceService], +) -> dict[str, VOSpaceService]: + """Merge discovered VOSpace Services into known ones, keyed by IVOA URI. + + Discovery names a new Service after its Server Name, so an existing Service + configured under its registry leaf (``arc``) is refreshed in place instead + of being duplicated under the Server Name on every rediscovery. + + Args: + known: Configured VOSpace Services keyed by Storage Name. + found: Newly discovered VOSpace Services keyed by Storage Name. + + Returns: + dict[str, VOSpaceService]: Merged Services keyed by Storage Name. + """ + merged = dict(known) + names = {str(service.uri): name for name, service in known.items()} + for name, service in found.items(): + merged[names.get(str(service.uri), name)] = service + return merged + + def discover( idp: str, *, @@ -147,7 +170,9 @@ def discover( if known is not None and known.version is not None and known.auths: if server.storage: known = known.model_copy( - update={"storage": {**known.storage, **server.storage}}, + update={ + "storage": _merge_storage(known.storage, server.storage) + }, deep=True, ) canonical[name] = known @@ -159,7 +184,7 @@ def discover( exclude_none=True, ) if server.storage: - updates["storage"] = {**known.storage, **server.storage} + updates["storage"] = _merge_storage(known.storage, server.storage) merged_server = known.model_copy(update=updates, deep=True) canonical[name] = merged_server @@ -413,7 +438,7 @@ async def _discover_for_idp( Raises: ServerDiscoveryError: If registry retrieval fails. """ - evidence = await load_registry_evidence( + evidence = await _discovery.discover( idp, dev=dev, timeout=timeout, @@ -435,9 +460,9 @@ async def _discover_for_idp( storage_resources = [ resource for resource in evidence.resources - if resource.uri.endswith(f"/{evidence.preferred_storage_leaf}") + if resource.uri.endswith(f"/{evidence.leaf}") ] - workers = await prepare_enrichment_workers( + workers = await _discovery.enrich( config, idp, endpoint=endpoints[0], @@ -457,7 +482,7 @@ async def _discover_for_idp( token=workers.token, certificate=workers.certificate, timeout=timeout, - storage_resource=_select_storage_resource( + storage_resource=_select_storage( endpoint, storage_resources, strict=False, @@ -480,7 +505,7 @@ def _host_slug(uri: AnyUrl) -> str | None: return uri.host.replace(".", "-") -def _select_storage_resource( +def _select_storage( endpoint: RegistryResource, resources: list[RegistryResource], *, @@ -488,12 +513,12 @@ def _select_storage_resource( ) -> RegistryResource | None: """Map private registry ambiguity to the public server fetch error.""" try: - return select_storage_resource(endpoint, resources, strict=strict) + return _discovery.select_storage(endpoint, resources, strict=strict) except RegistryEvidenceError as exc: raise ServerFetchError(str(exc)) from exc -async def _discover_storage_resource( +async def _discover_storage( server: Server, idp: str, *, @@ -502,7 +527,7 @@ async def _discover_storage_resource( ) -> RegistryResource | None: """Return fresh registry evidence for a server's primary VOSpace service.""" try: - return await discover_storage_resource( + return await _discovery.discover_storage( str(server.uri) if server.uri is not None else None, str(server.url) if server.url is not None else None, server.name, @@ -723,7 +748,7 @@ def _enrich_storage( """Validate and attach one retained primary VOSpace registry resource.""" error: BaseException | None = None if storage_resource is None: - leaf = get_idp(authentication_idp).preferred_storage_leaf + leaf = get_idp(authentication_idp).leaf subject = f"same-namespace '{leaf}' registry record" error = ValueError( f"No {subject} found for Science Platform Server '{server.name}'." @@ -862,7 +887,7 @@ def _validate_server( storage_resource = _configured_storage_resource(server) if storage_resource is None: storage_resource = asyncio.run( - _discover_storage_resource( + _discover_storage( server, active_idp, dev=dev, diff --git a/tests/test_data_smoke.py b/tests/test_data_smoke.py index 302a0ce3..555a3760 100644 --- a/tests/test_data_smoke.py +++ b/tests/test_data_smoke.py @@ -14,7 +14,7 @@ def _skip(reason: str) -> NoReturn: pytest.skip( - "live data smoke requires Storage Name 'canfar' and a valid saved " + "live data smoke requires Storage Name 'arc' and a valid saved " f"Authentication Record/certificate: {reason}" ) @@ -27,7 +27,7 @@ def _require_live_credentials() -> None: _skip("configuration is unavailable") try: - _endpoint, idp = config._resolve_storage("canfar") # noqa: SLF001 + _endpoint, idp = config._resolve_storage("arc") # noqa: SLF001 credential = config.get_credential(idp) except (KeyError, ValueError): _skip("the named service or its Authentication Record is unavailable") @@ -58,7 +58,7 @@ def test_canfar_data_lists_live_primary_storage() -> None: """List the configured live CADC primary VOSpace Service when available.""" _require_live_credentials() - result = CliRunner().invoke(cli, ["data", "ls", "-lh", "canfar:/"]) + result = CliRunner().invoke(cli, ["data", "ls", "-lh", "arc:/"]) assert result.exit_code == 0, result.output assert not result.stdout.startswith("@") diff --git a/tests/test_models_config.py b/tests/test_models_config.py index 33e11380..786db3db 100644 --- a/tests/test_models_config.py +++ b/tests/test_models_config.py @@ -66,6 +66,72 @@ def test_default_servers_dict_keyed_by_server_name(self, tmp_path: Path) -> None dumped = config.model_dump(mode="json", exclude_none=True) assert dumped["servers"]["canfar"]["name"] == "canfar" + def test_default_canfar_server_ships_arc_and_vault_storage( + self, + tmp_path: Path, + ) -> None: + """The CADC Science Platform Server exposes both VOSpace Services.""" + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + + assert set(config.servers["canfar"].storage) == {"arc", "vault"} + for name, endpoint in ( + ("arc", "https://ws-uv.canfar.net/arc"), + ("vault", "https://cadc-west-01.canfar.net/vault"), + ): + resolved, idp = config._resolve_storage(name) # noqa: SLF001 + assert resolved.rstrip("/") == endpoint + assert idp == "cadc" + + def test_legacy_server_named_storage_is_healed(self, tmp_path: Path) -> None: + """A Storage Name saved as the Server Name is restored to its leaf.""" + config_path = tmp_path / "config.yaml" + config_path.write_text( + "version: 1\n" + "servers:\n" + " canfar:\n" + " idp: cadc\n" + " uri: ivo://cadc.nrc.ca/skaha\n" + " url: https://ws-uv.canfar.net/skaha\n" + " version: v1\n" + " auths: [x509]\n" + " storage:\n" + " canfar:\n" + " uri: ivo://cadc.nrc.ca/arc\n" + " url: https://ws-uv.canfar.net/arc\n", + encoding="utf-8", + ) + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + + storage = config.servers["canfar"].storage + assert set(storage) == {"arc", "vault"} + assert str(storage["arc"].url).rstrip("/") == "https://ws-uv.canfar.net/arc" + + def test_custom_storage_names_are_not_healed(self, tmp_path: Path) -> None: + """Deliberate Storage Names are configuration, not stale defaults.""" + config_path = tmp_path / "config.yaml" + config_path.write_text( + "version: 1\n" + "servers:\n" + " canfar:\n" + " idp: cadc\n" + " uri: ivo://cadc.nrc.ca/skaha\n" + " url: https://ws-uv.canfar.net/skaha\n" + " version: v1\n" + " auths: [x509]\n" + " storage:\n" + " canSRC:\n" + " uri: ivo://cadc.nrc.ca/arc\n" + " url: https://ws-cadc.canfar.net/arc\n", + encoding="utf-8", + ) + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + + assert set(config.servers["canfar"].storage) == {"canSRC"} + def test_default_authentication_dict_keyed_by_idp(self, tmp_path: Path) -> None: """Default Authentication Records are keyed by IDP.""" config_path = tmp_path / "config.yaml" @@ -457,10 +523,10 @@ def test_v1_storage_json_and_yaml_round_trip(self, tmp_path: Path) -> None: assert loaded.servers["canfar"].idp == "cadc" assert loaded == config - def test_existing_v1_configuration_without_storage_loads_unchanged( + def test_existing_v1_configuration_without_storage_gains_defaults( self, tmp_path: Path ) -> None: - """The optional storage mapping does not require a schema migration.""" + """A storage-less Server gains defaults without a schema migration.""" config_path = tmp_path / "config.yaml" config_path.write_text(yaml.safe_dump(_sample_config()), encoding="utf-8") @@ -468,7 +534,7 @@ def test_existing_v1_configuration_without_storage_loads_unchanged( config = Configuration() assert config.version == 1 - assert config.servers["canfar"].storage == {} + assert set(config.servers["canfar"].storage) == {"arc", "vault"} def test_save_creates_directory(self, tmp_path: Path) -> None: """Save creates parent directories when missing.""" diff --git a/tests/test_server.py b/tests/test_server.py index 4da0a242..cbe7fb5b 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -15,7 +15,7 @@ from canfar.errors import ErrorCode from canfar.models.active import ActiveConfig from canfar.models.auth import OIDCCredential, X509Credential -from canfar.models.config import Configuration +from canfar.models.config import Configuration, default_servers from canfar.models.http import Server, VOSpaceService from canfar.models.registry import IVOARegistry, IVOARegistrySearch from canfar.models.registry import Server as DiscoveredServer @@ -25,7 +25,7 @@ ServerSelectionRequiredError, _discover_for_idp, _discovered_to_server, - _select_storage_resource, + _select_storage, activate, discover, enrich, @@ -378,7 +378,7 @@ async def test_registry_retains_only_skaha_and_preferred_storage_records( "https://platform.example/skaha-arc/capabilities" ), ) - search = IVOARegistrySearch(preferred_storage_leaf="arc") + search = IVOARegistrySearch(leaf="arc") async with Discover(search) as discovery: resources = discovery.extract(registry) @@ -393,7 +393,12 @@ def test_discover_refreshes_primary_storage_and_preserves_manual_entries( self, tmp_path: Path, ) -> None: - """Rediscovery updates only the generated Storage Name entry.""" + """Rediscovery updates only the generated Storage Name entry. + + A Server Name keyed entry saved by an older client is healed to its + registry leaf (``arc``) on load, so rediscovery refreshes that entry in + place instead of generating a second one. + """ manual = VOSpaceService( uri="ivo://cadc.nrc.ca/custom", url="https://manual.example/custom", @@ -461,17 +466,13 @@ def capabilities_response(request: httpx.Request) -> httpx.Response: persisted = Configuration().servers["canfar"] - assert ( - discovered.storage - == persisted.storage - == { - "canfar": VOSpaceService( - uri="ivo://cadc.nrc.ca/arc", - url="https://storage.example/arc", - ), - "archive": manual, - } + assert discovered.storage == persisted.storage + assert set(discovered.storage) == {"arc", "archive", "vault"} + assert discovered.storage["arc"] == VOSpaceService( + uri="ivo://cadc.nrc.ca/arc", + url="https://storage.example/arc", ) + assert discovered.storage["archive"] == manual @pytest.mark.parametrize("mode", ["missing", "malformed", "unreachable"]) def test_storage_inspection_fail_or_keep_outcomes( @@ -668,7 +669,7 @@ def test_storage_pairing_never_crosses_prod_and_dev_sources(self) -> None: url="https://storage.example/arc", ) - assert _select_storage_resource(endpoint, [dev_storage], strict=False) is None + assert _select_storage(endpoint, [dev_storage], strict=False) is None @pytest.mark.asyncio async def test_mixed_registry_records_keep_per_record_environment(self) -> None: @@ -683,14 +684,14 @@ async def test_mixed_registry_records_keep_per_record_environment(self) -> None: "https://storage.example/dev/capabilities" ), ) - search = IVOARegistrySearch(preferred_storage_leaf="cavern") + search = IVOARegistrySearch(leaf="cavern") async with Discover(search) as discovery: endpoint, storage = discovery.extract(registry, dev=True) assert endpoint.development is False assert storage.development is True - assert _select_storage_resource(endpoint, [storage], strict=False) is None + assert _select_storage(endpoint, [storage], strict=False) is None def test_ambiguous_cross_registry_storage_is_not_last_write_wins(self) -> None: """Multiple namespace fallbacks are omitted or actionable, never arbitrary.""" @@ -709,9 +710,9 @@ def test_ambiguous_cross_registry_storage_is_not_last_write_wins(self) -> None: for index in (1, 2) ] - assert _select_storage_resource(endpoint, storage, strict=False) is None + assert _select_storage(endpoint, storage, strict=False) is None with pytest.raises(ServerFetchError, match="Multiple preferred VOSpace"): - _select_storage_resource(endpoint, storage, strict=True) + _select_storage(endpoint, storage, strict=True) @pytest.mark.asyncio async def test_capability_enrichment_runs_concurrently_off_event_loop( @@ -1065,13 +1066,18 @@ def capabilities_response(request: httpx.Request) -> httpx.Response: persisted = Configuration().servers["canfar"] + # A storage-less saved Server gains the default VOSpace Services on load. + healed = known.model_copy( + update={"storage": default_servers["canfar"].storage}, + deep=True, + ) expected = ( - known.model_copy( + healed.model_copy( update={"version": "v2.1", "auths": ["oidc"]}, deep=True, ) if capabilities_case == "success" - else known + else healed ) assert discovered == [expected] assert config.servers["canfar"] == expected From 8a77ffc3059954ae03c0a8c9a2db796caabfc63a Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 24 Jul 2026 20:35:02 -0700 Subject: [PATCH 34/47] build(deps): upgrade vosfs to v0.7.0 and fsspec-cli to v0.6.0 fsspec-cli v0.6.0 hands parsing to Typer, a breaking upstream change. The consumed contract is unchanged: App(sources, capabilities=...), .typer_app, AsyncFilesystemSource, and the capability TypedDicts are all identical, and only the unused CommandExtension protocol was replaced. VOSpaceFileSystem keeps its token/certfile/asynchronous/storage_options constructor and aclose() through its v0.7.0 internal decomposition. Two upstream behaviours moved, so their tests were updated to match: - Disabled recursion.remove now omits the flag from `rm` entirely rather than registering and rejecting it, so `rm -R` reports `No such option: -R`. Verified the capability is still wired: `cp` registers -R/-r, `rm` does not. - `-h` is now a human-readable size flag for `ls` that requires a long listing, instead of an unsupported option. No production code changes. --- pyproject.toml | 4 +-- tests/test_cli_data.py | 61 ++++++++++++++++++------------- tests/test_data_dependencies.py | 4 +-- uv.lock | 64 ++++++++++++++++----------------- 4 files changed, 72 insertions(+), 61 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5cb414fa..047c8f57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,8 +59,8 @@ dependencies = [ "rich>=13.9.4", "segno>=1.6.6", "typer>=0.16.0", - "vosfs @ git+https://github.com/shinybrar/vosfs@v0.6.0", - "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.5.0#subdirectory=src/fsspec-cli", + "vosfs @ git+https://github.com/shinybrar/vosfs@v0.7.0", + "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.6.0#subdirectory=src/fsspec-cli", ] [tool.uv] diff --git a/tests/test_cli_data.py b/tests/test_cli_data.py index 29f3fa5a..2e42a8c2 100644 --- a/tests/test_cli_data.py +++ b/tests/test_cli_data.py @@ -15,6 +15,7 @@ from typer.testing import CliRunner import canfar.cli.data as data_cli +from canfar import storages from canfar.cli.main import cli if TYPE_CHECKING: @@ -83,8 +84,8 @@ def __init__( captured_capabilities.append(capabilities) self.typer_app = typer.Typer() - monkeypatch.setattr(data_cli, "Configuration", configuration) - monkeypatch.setattr(data_cli, "_vospace_source", source_factory) + monkeypatch.setattr(storages, "Configuration", configuration) + monkeypatch.setattr(storages, "_vospace", source_factory) monkeypatch.setattr(data_cli, "App", FakeApp) data_cli._upstream_group() # noqa: SLF001 @@ -95,7 +96,7 @@ def __init__( {"arc", "cavern", "vault", "local"}, ] assert all( - sources["local"] is data_cli._local_source # noqa: SLF001 + sources["local"] is storages._local # noqa: SLF001 for sources in captured_sources ) assert captured_capabilities == [ @@ -105,11 +106,11 @@ def __init__( @pytest.mark.asyncio -async def test_local_source_returns_fresh_async_wrappers() -> None: +async def test_local_storage_returns_fresh_async_wrappers() -> None: """Each local source entry owns a fresh asynchronous wrapper.""" - async with data_cli._local_source() as first: # noqa: SLF001 + async with storages._local() as first: # noqa: SLF001 assert first.asynchronous is True - async with data_cli._local_source() as second: # noqa: SLF001 + async with storages._local() as second: # noqa: SLF001 assert second.asynchronous is True assert first is not second @@ -123,7 +124,7 @@ def test_data_cat_delegates_without_active_server_banner( """Data stdout belongs byte-for-byte to the upstream command.""" payload = tmp_path / "payload.txt" payload.write_text("upstream output\n", encoding="utf-8") - monkeypatch.setattr(data_cli, "Configuration", _configuration) + monkeypatch.setattr(storages, "Configuration", _configuration) result = runner.invoke(cli, ["data", "cat", f"local:{payload}"]) @@ -138,14 +139,14 @@ def test_data_long_listing_delegates_unchanged( """The documented ``ls -lh name:/path`` form reaches upstream unchanged.""" (tmp_path / "payload.txt").write_text("listed", encoding="utf-8") monkeypatch.setattr( - data_cli, + storages, "Configuration", partial(_configuration, "canSRC"), ) monkeypatch.setattr( - data_cli, - "_vospace_source", - lambda _name: data_cli._local_source, # noqa: SLF001 + storages, + "_vospace", + lambda _name: storages._local, # noqa: SLF001 ) result = runner.invoke(cli, ["data", "ls", "-lh", f"canSRC:{tmp_path}"]) @@ -164,14 +165,14 @@ def test_data_recursive_copy_delegates_to_released_contract( (source / "payload.txt").write_text("copied", encoding="utf-8") destination = tmp_path / "destination" monkeypatch.setattr( - data_cli, + storages, "Configuration", partial(_configuration, "canSRC"), ) monkeypatch.setattr( - data_cli, - "_vospace_source", - lambda _name: data_cli._local_source, # noqa: SLF001 + storages, + "_vospace", + lambda _name: storages._local, # noqa: SLF001 ) result = runner.invoke( @@ -183,15 +184,25 @@ def test_data_recursive_copy_delegates_to_released_contract( assert (destination / "payload.txt").read_text(encoding="utf-8") == "copied" -def test_data_recursive_remove_is_disabled(monkeypatch, tmp_path: Path) -> None: - """CANFAR keeps upstream recursive removal source-free and disabled.""" - monkeypatch.setattr(data_cli, "Configuration", _configuration) +@pytest.mark.parametrize("flag", ["-R", "-r"]) +def test_data_recursive_remove_is_disabled( + monkeypatch, + tmp_path: Path, + flag: str, +) -> None: + """CANFAR keeps upstream recursive removal source-free and disabled. + + Upstream ``fsspec-cli`` v0.6.0 gave parsing to Typer, so the disabled + ``recursion.remove`` capability now omits the flag from ``rm`` entirely + instead of registering it and rejecting the invocation at runtime. + """ + monkeypatch.setattr(storages, "Configuration", _configuration) - result = runner.invoke(cli, ["data", "rm", "-R", f"local:{tmp_path}"]) + result = runner.invoke(cli, ["data", "rm", flag, f"local:{tmp_path}"]) assert result.exit_code == 2 assert result.stdout == "" - assert result.stderr == "rm: recursive removal disabled by application\n" + assert f"No such option: {flag}" in result.stderr def test_data_cross_source_move_retains_upstream_rejection(monkeypatch) -> None: @@ -203,8 +214,8 @@ async def unused_source() -> AsyncIterator[AbstractFileSystem]: raise AssertionError(msg) yield - monkeypatch.setattr(data_cli, "Configuration", partial(_configuration, "arc")) - monkeypatch.setattr(data_cli, "_vospace_source", lambda _name: unused_source) + monkeypatch.setattr(storages, "Configuration", partial(_configuration, "arc")) + monkeypatch.setattr(storages, "_vospace", lambda _name: unused_source) result = runner.invoke(cli, ["data", "mv", "arc:/a", "local:/b"]) @@ -219,7 +230,7 @@ def test_data_deprecated_operand_grammar_is_unsupported( operand: str, ) -> None: """Only the upstream explicit ``name:/absolute/path`` grammar is accepted.""" - monkeypatch.setattr(data_cli, "Configuration", _configuration) + monkeypatch.setattr(storages, "Configuration", _configuration) result = runner.invoke(cli, ["data", "ls", operand]) @@ -231,7 +242,7 @@ def test_data_deprecated_operand_grammar_is_unsupported( [ (["storage", "ls"], "No such command 'storage'"), (["data", "ls", "active:/"], "unknown filesystem"), - (["data", "ls", "-h", "local:/"], "-h: unsupported option"), + (["data", "ls", "-h", "local:/"], "-h: requires long listing"), ], ) def test_data_retired_aliases_and_standalone_h_are_unsupported( @@ -240,7 +251,7 @@ def test_data_retired_aliases_and_standalone_h_are_unsupported( diagnostic: str, ) -> None: """Retired host aliases do not widen the upstream mapped grammar.""" - monkeypatch.setattr(data_cli, "Configuration", _configuration) + monkeypatch.setattr(storages, "Configuration", _configuration) result = runner.invoke(cli, arguments) diff --git a/tests/test_data_dependencies.py b/tests/test_data_dependencies.py index 6bcad0d0..40f8aa5c 100644 --- a/tests/test_data_dependencies.py +++ b/tests/test_data_dependencies.py @@ -15,11 +15,11 @@ def test_tagged_data_dependencies_are_standard_dependencies() -> None: metadata = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) assert ( - "vosfs @ git+https://github.com/shinybrar/vosfs@v0.6.0" + "vosfs @ git+https://github.com/shinybrar/vosfs@v0.7.0" in metadata["project"]["dependencies"] ) assert ( - "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.5.0" + "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.6.0" "#subdirectory=src/fsspec-cli" in metadata["project"]["dependencies"] ) assert "data" not in metadata["project"].get("optional-dependencies", {}) diff --git a/uv.lock b/uv.lock index 4ffeb2b5..5d908c19 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14'", @@ -160,7 +160,7 @@ requires-dist = [ { name = "cadcutils", specifier = ">=1.5.4" }, { name = "click", specifier = ">=8.4.1" }, { name = "defusedxml", specifier = ">=0.7.1" }, - { name = "fsspec-cli", git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.5.0" }, + { name = "fsspec-cli", git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.6.0" }, { name = "httpx", extras = ["http2"], specifier = ">=0.28.1" }, { name = "humanize", specifier = ">=4.12.3" }, { name = "pydantic", specifier = ">=2.9.2" }, @@ -170,7 +170,7 @@ requires-dist = [ { name = "rich", specifier = ">=13.9.4" }, { name = "segno", specifier = ">=1.6.6" }, { name = "typer", specifier = ">=0.16.0" }, - { name = "vosfs", git = "https://github.com/shinybrar/vosfs?rev=v0.6.0" }, + { name = "vosfs", git = "https://github.com/shinybrar/vosfs?rev=v0.7.0" }, ] [package.metadata.requires-dev] @@ -632,7 +632,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -677,8 +677,8 @@ wheels = [ [[package]] name = "fsspec-cli" -version = "0.5.0" -source = { git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.5.0#ddef461b464d41811b6ddbad8eb1a64e37d2d85e" } +version = "0.6.0" +source = { git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.6.0#fa194eb18d35dbf2c5da67bc0f82ff085b9835ff" } dependencies = [ { name = "fsspec" }, { name = "typer" }, @@ -846,17 +846,17 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.11'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "jedi", marker = "python_full_version < '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, - { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "stack-data", marker = "python_full_version < '3.11'" }, - { name = "traitlets", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -872,18 +872,18 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" } wheels = [ @@ -895,7 +895,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -2191,8 +2191,8 @@ wheels = [ [[package]] name = "vosfs" -version = "0.6.0" -source = { git = "https://github.com/shinybrar/vosfs?rev=v0.6.0#200eae23fbc5e2bd58693b5c14ce12b2795d0636" } +version = "0.7.0" +source = { git = "https://github.com/shinybrar/vosfs?rev=v0.7.0#d4b8ccd1eee8b774bfb9a8f925b02db7dd09bbd9" } dependencies = [ { name = "defusedxml" }, { name = "fsspec" }, From 265b585574bf1fb2a48ef1565a2d9ade1c2fe270 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 24 Jul 2026 20:35:10 -0700 Subject: [PATCH 35/47] docs(data): correct storage names and verified command behaviour Every claim below was reproduced against the CLI before documenting it. - Use the real Storage Names arc and vault instead of the stale `canfar:`, including in quick-start and cli-help. - Add a real cross-source example copying a 166K public test cutout from vault into an arc home directory. - Drop the dependency-pin section and the audited-commit block; point at the current tagged releases instead. - Simplify the source-mapping paragraph. - Correct `ls -h`: it is a supported flag that requires a long listing, not an unsupported option. `ll` is retained, as it is an upstream command. - Correct the recursive-removal diagnostic to `No such option: -R`. --- docs/cli/cli-help.md | 4 +- docs/cli/data.md | 97 +++++++++++++++++++++-------------------- docs/cli/quick-start.md | 17 ++++---- 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/docs/cli/cli-help.md b/docs/cli/cli-help.md index 2561eb44..de496b6f 100644 --- a/docs/cli/cli-help.md +++ b/docs/cli/cli-help.md @@ -187,8 +187,8 @@ options. ## Data ```bash -canfar data ls -lh canfar:/ -canfar data cp local:/absolute/path/file.fits canfar:/folder/file.fits +canfar data ls -lh arc:/home/[username] +canfar data cp local:/absolute/path/file.fits arc:/home/[username]/file.fits ``` Data operands use explicit `Storage-Name:/absolute/path` or diff --git a/docs/cli/data.md b/docs/cli/data.md index 355ab682..2118c324 100644 --- a/docs/cli/data.md +++ b/docs/cli/data.md @@ -12,55 +12,56 @@ pip install canfar canfar login cadc ``` -CANFAR installs the audited upstream releases as normal dependencies through -tagged Git references: - -- `vosfs @ git+https://github.com/shinybrar/vosfs@v0.6.0` -- `fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.5.0#subdirectory=src/fsspec-cli` - -There is no separate data extra. - ## Address mapped sources Every remote source uses its configured Storage Name. The reserved `local` source addresses the machine where the command runs. Operands always use an explicit mapped name and absolute path: -On each CLI invocation, CANFAR rebuilds the source mapping from every Storage -Name on every configured Science Platform Server and always adds `local`. -Active Server Selection does not filter data sources. Each mapped source is a -factory that creates a fresh asynchronous filesystem when the command acquires -it. - ```text Storage-Name:/absolute/path local:/absolute/path ``` -For a default CADC login, the discovered primary Storage Name is `canfar`: +Every mapped source is available concurrently, regardless of the active Server +Selection. A default CADC login maps two VOSpace Services, `arc` and `vault`, +plus `local`: ```bash -canfar data ls -lh canfar:/ -canfar data ls -lh canfar:/folder +canfar data ls -lh arc:/ +canfar data ls -lh arc:/home/[username] +canfar data ls -lh vault:/ ``` -Use `ls -lh` (or `ll -h`) for a human-readable long listing. Standalone -`ls -h` is not supported. Empty `:/path`, bare local paths, `active:/path`, and -the retired `canfar storage` command are not aliases. +Use `ls -lh`, or the `ll -h` long-form command, for a human-readable listing. +The `-h` flag reports human-readable sizes and requires a long listing, so +`ls -h` on its own exits with `ls: -h: requires long listing`. + +Operands must name a mapped source. Empty `:/path`, bare local paths such as +`/tmp/file`, and `active:/path` are all rejected; there is no `active` alias +and no `canfar storage` command. ## Copy files and directories Copy one file between local and remote sources: ```bash -canfar data cp local:/absolute/path/file.fits canfar:/folder/file.fits -canfar data cp canfar:/folder/file.fits local:/absolute/path/file.fits +canfar data cp local:/absolute/path/file.fits arc:/home/[username]/file.fits +canfar data cp arc:/home/[username]/file.fits local:/absolute/path/file.fits +``` + +Copy between two VOSpace Services, for example a public test cutout from +`vault` into your `arc` home directory: + +```bash +canfar data cp vault:/ALMA/test-data/cutouts/test-4d-cube-cutout.fits arc:/home/[username]/test-4d-cube-cutout.fits +canfar data ls -lh arc:/home/[username]/test-4d-cube-cutout.fits ``` Recursive copy is enabled for admitted local and remote source pairs: ```bash -canfar data cp -R local:/absolute/path/dataset canfar:/folder/dataset +canfar data cp -R local:/absolute/path/dataset arc:/home/[username]/dataset ``` The tagged upstream implementation builds a bounded manifest, copies files @@ -70,24 +71,28 @@ the destination before removing any source data. ## Move data between sources -Cross-source `mv` is unsupported. Move a file explicitly by copying it, -verifying the destination, and only then issuing a separate source removal. In -this example, `archive` stands for a second Storage Name that you configured: +Cross-source `mv` is unsupported and exits with status 2: + +```text +mv: cross-source move unsupported +``` + +Move a file between sources explicitly by copying it, verifying the +destination, and only then issuing a separate source removal: ```bash -canfar data cp canfar:/folder/file.fits archive:/folder/file.fits -canfar data ls -lh archive:/folder/file.fits -canfar data rm canfar:/folder/file.fits +canfar data cp vault:/folder/file.fits arc:/home/[username]/file.fits +canfar data ls -lh arc:/home/[username]/file.fits +canfar data rm vault:/folder/file.fits ``` -Do not use this sequence as a one-command or atomic move. CANFAR does not -advertise same-source `mv` for current VOSpace sources either. +Do not use this sequence as a one-command or atomic move. -Recursive removal is disabled by application policy. `rm -R` exits with status -2 and reports: +Recursive removal is disabled by application policy, so `rm` accepts no `-R` or +`-r` flag at all and exits with status 2: ```text -rm: recursive removal disabled by application +No such option: -R ``` Because recursive directory removal is disabled, directory movement is not a @@ -104,17 +109,13 @@ stderr. This release intentionally provides no public CANFAR storage Python API, FUSE mount, signed-URL extension, progress display, confirmation prompt, `:/path` or bare-path shorthand, `active` alias, `canfar storage` alias, recursive removal, -or cross-source/current-VOSpace `mv` workflow. - -## Audited upstream releases - -CANFAR's integration is based on -[`shinybrar/vosfs` PR #294](https://github.com/shinybrar/vosfs/pull/294), -audited at commit -[`9e5314db4706894d31d54d245392f43b9556cfbb`](https://github.com/shinybrar/vosfs/commit/9e5314db4706894d31d54d245392f43b9556cfbb). -The installed pair is the tagged -[`vosfs` v0.6.0](https://github.com/shinybrar/vosfs/releases/tag/v0.6.0) and -[`fsspec-cli-v0.5.0`](https://github.com/shinybrar/vosfs/releases/tag/fsspec-cli-v0.5.0) -releases. CANFAR tests its composition, configuration, authentication, and -output seams; exhaustive filesystem-command and backend matrices remain in the -upstream project. +or cross-source `mv` workflow. + +## Upstream releases + +CANFAR installs pinned, tagged releases of +[`vosfs`](https://github.com/shinybrar/vosfs/releases/tag/v0.7.0) and +[`fsspec-cli`](https://github.com/shinybrar/vosfs/releases/tag/fsspec-cli-v0.6.0). +CANFAR tests its composition, configuration, authentication, and output seams; +exhaustive filesystem-command and backend matrices remain in the upstream +project. diff --git a/docs/cli/quick-start.md b/docs/cli/quick-start.md index 6ab6aabf..efc4c192 100644 --- a/docs/cli/quick-start.md +++ b/docs/cli/quick-start.md @@ -30,22 +30,21 @@ canfar server ls ## 3. Work with data The standard installation includes data commands. Use a configured Storage Name -or the reserved `local` name with an absolute path: +or the reserved `local` name with an absolute path. A default CADC login maps +`arc` and `vault`: ```bash -canfar data ls -lh canfar:/ -canfar data cp local:/absolute/path/file.fits canfar:/folder/file.fits +canfar data ls -lh arc:/home/[username] +canfar data cp local:/absolute/path/file.fits arc:/home/[username]/file.fits ``` Cross-source `mv` is unsupported. Copy the file, verify the destination, and -then remove the source with a separate command. In this example, `archive` is -a placeholder for a second Storage Name that you configured; replace it with -that Storage Name: +then remove the source with a separate command: ```bash -canfar data cp canfar:/folder/file.fits archive:/folder/file.fits -canfar data ls -lh archive:/folder/file.fits -canfar data rm canfar:/folder/file.fits +canfar data cp vault:/folder/file.fits arc:/home/[username]/file.fits +canfar data ls -lh arc:/home/[username]/file.fits +canfar data rm vault:/folder/file.fits ``` ## 4. Create a notebook From 20760e96d58b7df9754b6cc94aa83ba5aca44b90 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 24 Jul 2026 21:01:40 -0700 Subject: [PATCH 36/47] fix(tests): assert recursion policy without rich-rendered text `rm -R` is rejected by Typer, which renders its usage error through rich. The panel wraps to the terminal width and carries ANSI colour, so asserting the substring `No such option: -R` passed locally on a wide console and failed on every CI Python. Assert the width-independent behaviour instead (exit status 2, empty stdout) and pin the real invariant structurally: the recursion capability withholds -R/-r from `rm` while still registering them on `cp`. --- tests/test_cli_data.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/test_cli_data.py b/tests/test_cli_data.py index 2e42a8c2..fd029e9c 100644 --- a/tests/test_cli_data.py +++ b/tests/test_cli_data.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING from unittest.mock import patch +import click import pytest import typer from typer.testing import CliRunner @@ -202,7 +203,23 @@ def test_data_recursive_remove_is_disabled( assert result.exit_code == 2 assert result.stdout == "" - assert f"No such option: {flag}" in result.stderr + + +def test_recursion_capability_omits_rm_flags_but_keeps_cp_flags( + monkeypatch, +) -> None: + """The recursion capability registers ``cp`` flags and withholds ``rm`` ones.""" + monkeypatch.setattr(storages, "Configuration", _configuration) + group = data_cli._upstream_group() # noqa: SLF001 + context = click.Context(group) + + def options(command: str) -> set[str]: + resolved = group.get_command(context, command) + assert resolved is not None + return {option for param in resolved.params for option in param.opts} + + assert not options("rm") & {"-R", "-r"} + assert {"-R", "-r"} <= options("cp") def test_data_cross_source_move_retains_upstream_rejection(monkeypatch) -> None: From 501810479cd22f25e8ae9bcd22378384abbba09d Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 24 Jul 2026 21:01:40 -0700 Subject: [PATCH 37/47] perf(storage): cache VOSpace directory listings per command Build every VOSpace filesystem with fsspec's listing cache enabled, so a command that walks a tree stops re-requesting the same directories. A repeated listing drops from a round trip to a dictionary lookup. Each command builds and closes its own filesystem, so a cached listing cannot outlive the command that produced it and no cross-command staleness is possible. --- canfar/storages.py | 13 +++++++++++++ tests/test_storages.py | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/canfar/storages.py b/canfar/storages.py index e5a8f4aa..1c43baf5 100644 --- a/canfar/storages.py +++ b/canfar/storages.py @@ -21,6 +21,13 @@ from pydantic import SecretStr +_LISTINGS_EXPIRY_SECONDS = 30 +"""Seconds a cached directory listing stays valid within one command.""" + +_LISTINGS_MAX_PATHS = 1000 +"""Maximum directory listings retained by one filesystem.""" + + def _vospace( name: str, *, @@ -68,6 +75,9 @@ async def source() -> AsyncIterator[AbstractFileSystem]: token=token_value, asynchronous=True, skip_instance_cache=True, + use_listings_cache=True, + listings_expiry_time=_LISTINGS_EXPIRY_SECONDS, + max_paths=_LISTINGS_MAX_PATHS, ) else: assert certfile is not None @@ -76,6 +86,9 @@ async def source() -> AsyncIterator[AbstractFileSystem]: certfile=certfile, asynchronous=True, skip_instance_cache=True, + use_listings_cache=True, + listings_expiry_time=_LISTINGS_EXPIRY_SECONDS, + max_paths=_LISTINGS_MAX_PATHS, ) try: yield filesystem diff --git a/tests/test_storages.py b/tests/test_storages.py index d75490a8..cec18359 100644 --- a/tests/test_storages.py +++ b/tests/test_storages.py @@ -20,6 +20,13 @@ if TYPE_CHECKING: from pathlib import Path +_LISTINGS = { + "use_listings_cache": True, + "listings_expiry_time": 30, + "max_paths": 1000, +} +"""Directory-listing cache settings every VOSpace filesystem is built with.""" + class _Filesystem: """Record construction and cleanup without VOSpace I/O.""" @@ -94,6 +101,7 @@ async def test_source_reloads_config_and_runtime_token_wins( "token": "runtime-token", "asynchronous": True, "skip_instance_cache": True, + **_LISTINGS, } assert filesystem.asynchronous is True assert filesystem.closed is False @@ -144,6 +152,7 @@ async def test_environment_token_preserves_runtime_precedence( "token": "environment-token", "asynchronous": True, "skip_instance_cache": True, + **_LISTINGS, } @@ -170,6 +179,7 @@ async def test_environment_certificate_preserves_runtime_precedence( "certfile": certificate.as_posix(), "asynchronous": True, "skip_instance_cache": True, + **_LISTINGS, } valid.assert_called_once_with(certificate) From 63c6f72ed545d02f48e97ba5e4ee14996ad9e836 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 24 Jul 2026 21:01:40 -0700 Subject: [PATCH 38/47] docs(data): document listing, file, and byte-range caching Explain the automatic listing cache and show how to cache file contents from Python, pointing the cache at /scratch on a CANFAR session since it is fast local disk that is cleared with the session. VOSpace serves ranged reads, so document MMapCache for caching a large cube one block at a time instead of downloading it whole; reading a FITS header transfers a single block. Both snippets were run against the live service before being documented. --- docs/cli/data.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/docs/cli/data.md b/docs/cli/data.md index 2118c324..0bdfb52f 100644 --- a/docs/cli/data.md +++ b/docs/cli/data.md @@ -100,6 +100,73 @@ supported workflow in this release. Any future one-command relocation would be a separately named, opt-in orchestration feature with stronger destination verification and residual-state semantics—not portable `mv`. +## Cache data locally + +### Directory listings + +Directory listings are cached automatically. Each command builds and closes its +own filesystem, so a cached listing only ever serves the command that produced +it and can never return a listing that outlives it. Repeated lookups while one +command walks a tree are served without another round trip. + +### Files and byte ranges + +There is no CLI flag for caching file contents, but `vosfs` is a normal +[fsspec](https://filesystem-spec.readthedocs.io/) filesystem, so any fsspec +cache can wrap it from Python. On a CANFAR session, `/scratch` is fast local +disk and is the right place to point a cache; it is not backed up and is +cleared when the session ends, which is exactly what a cache wants. + +Cache whole files under a named directory. The first read fetches over the +network, and later reads come from `/scratch`: + +```python +from pathlib import Path + +from fsspec.implementations.cached import WholeFileCacheFileSystem +from vosfs import VOSpaceFileSystem + +vault = VOSpaceFileSystem( + "https://cadc-west-01.canfar.net/vault", + certfile=str(Path.home() / ".ssl" / "cadcproxy.pem"), +) +cached = WholeFileCacheFileSystem(fs=vault, cache_storage="/scratch/vault-cache") + +data = cached.cat_file("/ALMA/test-data/cutouts/test-4d-cube-cutout.fits") +``` + +Use `SimpleCacheFileSystem` instead when you do not need the expiry and +staleness metadata that `WholeFileCacheFileSystem` keeps. Passing +`cache_storage` a list of directories tries each in order and treats only the +last as writable, so a shared read-only cache can back your own. + +VOSpace supports ranged reads, so a large cube can be cached one block at a +time rather than downloaded whole. `MMapCache` keeps fetched blocks in a sparse +file, so only the blocks you touch occupy disk: + +```python +from fsspec.caching import MMapCache + +path = "/ALMA/test-data/cutouts/test-4d-cube.fits" +size = vault.info(path)["size"] +blocks = MMapCache( + blocksize=1 << 20, + fetcher=lambda start, end: vault.cat_file(path, start, end), + size=size, + location="/scratch/vault-cache/test-4d-cube.blocks", +) + +header = blocks._fetch(0, 2880) # one FITS header block, one 1 MiB fetch +``` + +Reading a FITS header this way transfers a single block instead of the whole +file, and a second read of the same range is served from `/scratch`. Ranged +reads help most on large files; on small ones the per-request VOSpace transfer +negotiation dominates. + +These caches use the synchronous filesystem interface. Build the filesystem +without `asynchronous=True`, as above. + ## Output and accepted omissions Data command stdout belongs to the embedded command; CANFAR does not prepend From ba5493fd0296244a9a03c62fa572a759b2ad385b Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 24 Jul 2026 23:28:46 -0700 Subject: [PATCH 39/47] refactor(discovery,cli): shorten Enrichment and group names Rename EnrichmentWorkers to Enrichment and cli.data._upstream_group to group. The group rename also renames the local variable it was assigned to inside _DataGroup._delegate, which would otherwise shadow the module-level function and raise UnboundLocalError. Making group public drops the private-access suppressions from its tests. No behaviour change. --- canfar/_discovery.py | 8 ++++---- canfar/cli/data.py | 14 +++++++------- tests/test_cli_data.py | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/canfar/_discovery.py b/canfar/_discovery.py index d3efa782..c7450637 100644 --- a/canfar/_discovery.py +++ b/canfar/_discovery.py @@ -41,7 +41,7 @@ class RegistryEvidence(BaseModel): available: bool -class EnrichmentWorkers(BaseModel): +class Enrichment(BaseModel): """Isolated worker configs with one pre-materialized runtime credential. Attributes: @@ -243,7 +243,7 @@ async def enrich( *, endpoint: RegistryResource, count: int, -) -> EnrichmentWorkers | None: +) -> Enrichment | None: """Materialize credentials once, then isolate worker configuration state. Args: @@ -253,7 +253,7 @@ async def enrich( count: Number of isolated worker configurations to produce. Returns: - EnrichmentWorkers | None: Workers, or None when credentials are absent + Enrichment | None: Workers, or None when credentials are absent or unusable. """ from canfar.client import HTTPClient # noqa: PLC0415 @@ -284,4 +284,4 @@ async def enrich( values = base_config.model_dump(mode="python") configs = tuple(Configuration.model_validate(values) for _ in range(count)) - return EnrichmentWorkers(configs=configs, token=token, certificate=certificate) + return Enrichment(configs=configs, token=token, certificate=certificate) diff --git a/canfar/cli/data.py b/canfar/cli/data.py index ae5b23d2..653b612b 100644 --- a/canfar/cli/data.py +++ b/canfar/cli/data.py @@ -17,7 +17,7 @@ _DATA_GROUP_META_KEY = "canfar.data_group" -def _upstream_group() -> TyperGroup: +def group() -> TyperGroup: """Build the released upstream application with CANFAR policy. Returns: @@ -36,12 +36,12 @@ class _DataGroup(TyperGroup): @staticmethod def _delegate(ctx: Context) -> TyperGroup: - group = ctx.meta.get(_DATA_GROUP_META_KEY) - if group is None: - group = _upstream_group() - ctx.meta[_DATA_GROUP_META_KEY] = group - assert isinstance(group, TyperGroup) - return group + resolved = ctx.meta.get(_DATA_GROUP_META_KEY) + if resolved is None: + resolved = group() + ctx.meta[_DATA_GROUP_META_KEY] = resolved + assert isinstance(resolved, TyperGroup) + return resolved def list_commands(self, ctx: Context) -> list[str]: """List the unchanged upstream commands.""" diff --git a/tests/test_cli_data.py b/tests/test_cli_data.py index fd029e9c..1002b437 100644 --- a/tests/test_cli_data.py +++ b/tests/test_cli_data.py @@ -89,8 +89,8 @@ def __init__( monkeypatch.setattr(storages, "_vospace", source_factory) monkeypatch.setattr(data_cli, "App", FakeApp) - data_cli._upstream_group() # noqa: SLF001 - data_cli._upstream_group() # noqa: SLF001 + data_cli.group() + data_cli.group() assert [set(sources) for sources in captured_sources] == [ {"arc", "cavern", "local"}, @@ -210,7 +210,7 @@ def test_recursion_capability_omits_rm_flags_but_keeps_cp_flags( ) -> None: """The recursion capability registers ``cp`` flags and withholds ``rm`` ones.""" monkeypatch.setattr(storages, "Configuration", _configuration) - group = data_cli._upstream_group() # noqa: SLF001 + group = data_cli.group() context = click.Context(group) def options(command: str) -> set[str]: From 511199fec2054dc57875a9de4f3a70834c4cd29d Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Sat, 25 Jul 2026 00:00:11 -0700 Subject: [PATCH 40/47] docs(data): call the operand prefix an identifier The operand grammar used `Storage-Name` as its placeholder, which reads as the glossary noun rather than the slot it occupies, and does not cover the reserved `local` prefix. Call the placeholder `identifier` and state that it is either a Storage Name or `local`. Keeps the `Storage Name` glossary term from CONTEXT.md intact, since runtime errors use it verbatim. --- docs/cli/cli-help.md | 2 +- docs/cli/data.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/cli/cli-help.md b/docs/cli/cli-help.md index de496b6f..67d2bafe 100644 --- a/docs/cli/cli-help.md +++ b/docs/cli/cli-help.md @@ -191,7 +191,7 @@ canfar data ls -lh arc:/home/[username] canfar data cp local:/absolute/path/file.fits arc:/home/[username]/file.fits ``` -Data operands use explicit `Storage-Name:/absolute/path` or +Data operands use explicit `identifier:/absolute/path` or `local:/absolute/path` syntax. See [Data commands](data.md) for recursive copy, cross-source copy and verification, recursive-removal policy, and the exact supported boundary. diff --git a/docs/cli/data.md b/docs/cli/data.md index 0bdfb52f..84c41ada 100644 --- a/docs/cli/data.md +++ b/docs/cli/data.md @@ -14,12 +14,12 @@ canfar login cadc ## Address mapped sources -Every remote source uses its configured Storage Name. The reserved `local` -source addresses the machine where the command runs. Operands always use an -explicit mapped name and absolute path: +Every operand starts with an `identifier`: the Storage Name of a configured +VOSpace Service, or the reserved `local` identifier for the machine where the +command runs. Operands always pair an identifier with an absolute path: ```text -Storage-Name:/absolute/path +identifier:/absolute/path local:/absolute/path ``` From b26261f58a875fe454ace903543fbc50bf62980a Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Sat, 25 Jul 2026 00:22:13 -0700 Subject: [PATCH 41/47] docs(data): correct the byte-range caching guidance The caching section claimed VOSpace serves ranged reads and recommended MMapCache for fetching a cube one block at a time. That is wrong and the advice was actively harmful. vosfs `_cat_file(path, start, end)` reads the whole object and slices it in memory (filesystem.py), because OpenCADC Cavern does not implement HTTP byte ranges (staging.py); the package sends no Range header anywhere. A block cache therefore issues one full download per block, so reading three headers out of one cube downloads that cube three times. Replace the recommendation with its opposite: cache whole files, use exactly one cache layer, and avoid block caches. Also record that chained cache layers do not compose - the inner layer is never filled and never serves. Adds the supporting research note under docs/agents/research/. --- .../research/2026-07-25-storage-python-api.md | 816 ++++++++++++++++++ docs/cli/data.md | 32 +- 2 files changed, 829 insertions(+), 19 deletions(-) create mode 100644 docs/agents/research/2026-07-25-storage-python-api.md diff --git a/docs/agents/research/2026-07-25-storage-python-api.md b/docs/agents/research/2026-07-25-storage-python-api.md new file mode 100644 index 00000000..f557f68d --- /dev/null +++ b/docs/agents/research/2026-07-25-storage-python-api.md @@ -0,0 +1,816 @@ +# A Python storage API for CANFAR VOSpace Services + +- **Date:** 2026-07-25 +- **Repository snapshot:** `ba5493fd` on `feat/vosfs-support` +- **fsspec evaluated:** `2026.6.0` +- **vosfs evaluated:** `0.7.0` +- **fsspec-cli evaluated:** `0.6.0` +- **Python:** `3.13.5` +- **Question:** How should `canfar` conveniently expose VOSpace storage for Python API access? + +## Verdict + +**Ship one module, `canfar/storage.py`, with three public functions and no new +dependency. Build sync by default, async on request, and cache with exactly one +`SimpleCacheFileSystem` layer. Do not build a tiered cache: over `vosfs` 0.7.0 the +second tier is either dead or fictional.** + +The recommended surface is: + +```python +from canfar.storage import filesystem, fetch, sources +``` + +- `filesystem(name, *, cache=False, asynchronous=False, token=None, certificate=None)` + returns a ready `AbstractFileSystem` for one Storage Name. +- `fetch(name, path, *, cache=True)` materializes one remote object and returns its + local `Path`. +- `sources()` is the existing fsspec-cli seam, moved unchanged. + +Three findings drive every other decision, and two of them contradict text currently +in `docs/cli/data.md`: + +1. **`vosfs` 0.7.0 has no server-side byte ranges.** `_cat_file(path, start, end)` + downloads the whole object and slices it in memory. A 2880-byte "range" read of a + 3.5 MB file costs the same as reading the file (VERIFIED, §2). Every byte-range + design — `MMapCache`, `blockcache`, `cache_type=` — is therefore either broken or a + pessimization. +2. **Only whole-file caching composes.** `simplecache` and `filecache` work. Stacking + two cache layers builds a stack in which the inner layer never fills and never + serves (VERIFIED, §5). `cache_storage=["a", "b"]` is a genuine two-location lookup + but performs no promotion (VERIFIED, §5). +3. **There is no fsspec-native RAM tier over `vosfs`.** `cache_storage="memory://…"` + is accepted and silently creates a local directory literally named `memory:` + (VERIFIED, §5). The real RAM tier is the OS page cache over the disk cache + directory, plus `/dev/shm` on Linux. + +The sync/async tension resolves cleanly and is already half-solved in the repository: +build `VOSpaceFileSystem` synchronously, wrap it in the sync cache, and re-expose it +with `AsyncFileSystemWrapper` — the same class `canfar/storages.py:108` already uses +for `local`. This was verified end to end (§4, §9). + +`docs/cli/data.md:143-165` should be corrected regardless of whether this API ships. + +## Evidence key + +Every finding below is marked: + +- **VERIFIED** — I executed it in this repository's environment against the live + `vault` VOSpace Service with `~/.ssl/cadcproxy.pem`, read-only, on 2026-07-25. +- **DOCUMENTED** — I read the source or the published documentation and did not + execute it. + +All experiments used `uv run --no-sync python` from the repository root, targeted only +the two public read-only test objects +(`vault:/ALMA/test-data/cutouts/test-4d-cube-cutout.fits`, 169 920 B, and +`.../test-4d-cube.fits`, 3 542 400 B), and wrote only into the session scratchpad. No +`canfar/` or `tests/` file was modified and no git state was touched. + +## 1. Baseline: what exists today + +| Element | Location | +| --- | --- | +| `_vospace(name, *, token, certificate)` factory | `canfar/storages.py:31-98` | +| `_local()` async wrapper over `LocalFileSystem` | `canfar/storages.py:101-111` | +| `sources() -> dict[str, AsyncFilesystemSource]` | `canfar/storages.py:114-130` | +| Only consumer | `canfar/cli/data.py:12,22-33` | +| `AsyncFilesystemSource = Callable[[], AbstractAsyncContextManager[AbstractFileSystem]]` | `.venv/lib/python3.13/site-packages/fsspec_cli/_app.py:41-43` | +| Storage Name → endpoint + IDP | `canfar/models/config.py:382-395` | +| Credential materialization (async) | `canfar/client.py:234-262` | +| Runtime dependencies | `pyproject.toml:48-65` | + +DOCUMENTED. `canfar/storages.py` is not exported from `canfar/__init__.py:16-31`; its +only non-test importer is `canfar/cli/data.py:12`. `docs/cli/data.md:176` states the +release "intentionally provides no public CANFAR storage Python API". The module is +therefore free to be renamed. + +## 2. The constraint that dominates everything: no byte ranges + +DOCUMENTED. `vosfs/staging.py:3-5` states it outright: + +> OpenCADC Cavern does not implement HTTP byte ranges, so a seekable read is a +> whole-object download into a disk-backed temporary file. + +DOCUMENTED. `vosfs/filesystem.py:547-556` implements `_cat_file` as a whole read +followed by an in-memory slice: + +```python +async def _cat_file(self, path, start=None, end=None, **_kwargs): + """Return one whole-object read sliced with Python half-open semantics.""" + data = await _transfer.read_whole(self, self._strip_protocol(path)) + return data[start:end] +``` + +`_transfer.read_whole` (`vosfs/_transfer.py:227-240`) joins the full streamed body. +`_transfer.open_read_stream` (`vosfs/_transfer.py:197-224`) sends +`GET` with `transport.IDENTITY_ENCODING` headers and no `Range` header. A grep for +`Range` across the whole installed `vosfs` package returns no hits. + +VERIFIED. Timings on `vault:/ALMA/test-data/cutouts/test-4d-cube.fits` (3 542 400 B), +one warm filesystem instance: + +| Call | Bytes returned | Wall time | +| --- | --- | --- | +| `cat_file(BIG)` | 3 542 400 | 1.41 s | +| `cat_file(BIG, 0, 2880)` | 2 880 | 1.34 s | +| `cat_file(BIG, size-2880, None)` | 2 880 | 1.21 s | +| `cat_ranges([BIG]*4, 4 starts, 4 ends)` | 4 × 2 880 | 1.16 s | + +A 2880-byte head read costs 0.95× a whole-file read. `cat_ranges` is the one genuine +optimization: `vosfs/filesystem.py:594-620` groups ranges by path and +`_read_staged_ranges` (`vosfs/filesystem.py:622-633`) performs **one** whole-object +download per distinct path per call, then slices from the staging file. Four ranges of +one object cost one download, not four. + +**Consequence.** Anything that reads a "block" pays for the whole object. This is not +a `vosfs` defect to route around in `canfar`; it is a service-side gap. + +### The doc claim that is wrong + +`docs/cli/data.md:143-144` says "VOSpace supports ranged reads, so a large cube can be +cached one block at a time rather than downloaded whole", and `:162-163` says "Reading +a FITS header this way transfers a single block instead of the whole file." + +VERIFIED false. Driving `MMapCache(blocksize=1<<20, fetcher=lambda s, e: fs.cat_file(BIG, s, e), size=3542400, location=…)`: + +| Operation | Wall time | What actually moved | +| --- | --- | --- | +| `mc._fetch(0, 2880)` (one FITS header block) | 1.84 s | full 3 542 400 B download | +| plain `fs.cat_file(BIG)` for comparison | 1.64 s | full 3 542 400 B download | +| three scattered 2880 B reads via `MMapCache` | 4.01 s | **three** full downloads | +| sparse backing file after touching 3 of 4 blocks | — | 3 543 040 B allocated (i.e. full) | + +The `MMapCache` mechanics are real — a repeated read of an already-fetched block was +0.000 s, and the on-disk file is genuinely sparse-capable — but over `vosfs` the +fetcher is a whole-object download, so the cache costs *more* network than a single +`cat_file`, and the sparse file fills up anyway. `location=None` (anonymous mmap, RAM) +behaves identically for network cost: 1.40 s for the same header read. + +## 3. Import surface: `canfar.storage`, singular + +**Recommendation: rename `canfar/storages.py` to `canfar/storage.py` and make it the +public module.** + +Rationale, all DOCUMENTED: + +- The package's own convention is a singular module name for a plural domain. + `canfar/__init__.py:16` exports `server` (not `servers`), and `canfar/server.py` + contains `discover`, `list_servers`, `activate`, `use`. `canfar/sessions.py` is the + exception, not the rule, and it is a class module. +- The maintainer's suggested import is `from canfar.storage import sources`. +- The module is currently private in every meaningful sense (§1), so the rename costs + exactly one import line in `canfar/cli/data.py:12` plus the `from canfar import + storages` references in `tests/test_cli_data.py`. No published API breaks, because + `docs/cli/data.md:176` says none exists. + +Do **not** add a compatibility shim re-exporting `canfar.storages`. There is nothing +to be compatible with. + +### What belongs in it — and what does not + +| Ask | Recommendation | +| --- | --- | +| Filesystem accessor | `filesystem()` — yes | +| Opener | No new function. `filesystem(name).open(path)` already is the opener. | +| Cache configurator | No object, no builder. One `cache=` parameter on `filesystem()`. | +| Path/URL resolver | **No.** See below. | +| Materialize to a local path | `fetch()` — yes, because the fsspec route is private API (§7) | +| fsspec-cli seam | `sources()`, moved unchanged | + +**Reject the path/URL resolver.** VERIFIED: `vos://` URLs do not round-trip. Because +`vosfs` treats everything after the protocol marker as path +(`vosfs/filesystem.py:69-71`, `_strip_protocol` at `:179-188`): + +```text +_strip_protocol('vos://cadc-west-01.canfar.net/vault/ALMA/…/test-4d-cube-cutout.fits') + -> '/cadc-west-01.canfar.net/vault/ALMA/…/test-4d-cube-cutout.fits' +``` + +The authority is silently swallowed into the path. A `canfar` resolver that emitted +`vos://` URLs would produce strings that look addressable and are not. In Python the +Storage Name and the path are already two arguments; a resolver adds a failure mode +and removes nothing. The CLI's `identifier:/path` operand syntax +(`docs/cli/data.md:17-24`) is a *command-line* affordance and should stay there. + +## 4. Getting a filesystem, sync and async + +The tension is real and I measured both horns. + +VERIFIED. **Cache wrappers reject an async `vosfs` instance.** +`SimpleCacheFileSystem(fs=VOSpaceFileSystem(..., asynchronous=True), cache_storage=…)` +constructs, then the first read fails: + +```text +RuntimeError: Loop is not running +``` + +VERIFIED. **A sync `vosfs` instance works inside a running event loop, but blocks +it.** `VOSpaceFileSystem(..., asynchronous=False).cat_file(path)` called from inside +`asyncio.run` succeeded — fsspec's `sync()` dispatches to its own dedicated IO-thread +loop (DOCUMENTED, ), so +it does not hit the "cannot call sync from a running loop" guard. But a ticker +coroutine sleeping every 50 ms recorded **0 ticks** during a 10.60 s call. The caller's +loop is frozen for the duration. + +VERIFIED. **`AsyncFileSystemWrapper` over the sync cached stack resolves it.** + +```python +inner = VOSpaceFileSystem(endpoint, certfile=…, asynchronous=False) +cached = SimpleCacheFileSystem(fs=inner, cache_storage=…) +wrapped = AsyncFileSystemWrapper(cached, asynchronous=True) +``` + +| Measurement | Result | +| --- | --- | +| `await wrapped._cat_file(SMALL)` cold | 1.87 s, 169 920 B | +| `await wrapped._cat_file(SMALL)` warm | 0.007 s | +| `await wrapped._ls(dir, detail=False)` | 2 entries, correct | +| `await wrapped._info(SMALL)["size"]` | 169 920 | +| ticker ticks during the 1.87 s cold call | **36** (non-blocking) | + +DOCUMENTED. `fsspec/implementations/asyn_wrapper.py:11-36` wraps each sync method with +`asyncio.to_thread`, which is why the loop stays responsive. Its class declaration is +`fsspec/implementations/asyn_wrapper.py:39`. `canfar/storages.py:108` already uses this +class for `local`, so it is not a new concept in this codebase. fsspec's own docs call +it "experimental" and warn "Users should not expect this wrapper to magically make +things faster" () — the +caution is about throughput, not correctness, and the measurements above are the +throughput answer for this workload. + +### The resulting rule + +| Request | Build | +| --- | --- | +| `asynchronous=False`, `cache=False` | `VOSpaceFileSystem(..., asynchronous=False)` | +| `asynchronous=False`, cache on | `SimpleCacheFileSystem(fs=, ...)` | +| `asynchronous=True`, `cache=False` | `VOSpaceFileSystem(..., asynchronous=True)` — native, best | +| `asynchronous=True`, cache on | `AsyncFileSystemWrapper(SimpleCacheFileSystem(fs=), asynchronous=True)` | + +Never construct a cache wrapper over an `asynchronous=True` instance. + +### Credential materialization from a sync entry point + +`canfar/client.py:234` `_materialize_credentials` is `async def`. VERIFIED: it can be +driven from a sync function both outside and inside a running loop using fsspec's own +IO loop, with no `asyncio.run` and no nested-loop guard: + +```python +from fsspec.asyn import get_loop, sync +token, certfile = sync(get_loop(), client._materialize_credentials) +``` + +Outside a loop: 0.57 s (includes X.509 validation). Inside `asyncio.run`: 0.00 s +(credential already warm). This is the smallest correct bridge and adds no dependency. + +### One instance is worth keeping + +VERIFIED. Per-call latency against `vault`: + +| Measurement | Observed | +| --- | --- | +| first `info()` on a fresh instance, 5 fresh instances | 1.65, 0.43, 0.61, 0.56, 0.60 s | +| second `info()` on the same instance | 0.30, 0.34, 0.30, 0.52, 0.36 s | +| whole-object read, 169 920 B, warm instance | ≈ 1.0 s | +| worst first-call outlier observed across all runs | 14.3 s | + +The current `sources()` design builds and closes a filesystem per command +(`canfar/storages.py:93-96`), which is right for a CLI. A Python API should hand back a +*reusable* instance so callers pay service-binding discovery once. Note the outliers: +first-call cost is not deterministic, so do not document a fixed number. + +DOCUMENTED. `vosfs/filesystem.py:73-75` sets `protocol = "vos"` and `cachable = True`. +VERIFIED: `fsspec.filesystem("vos", endpoint_url=…, certfile=…)` twice returns the +identical object; `skip_instance_cache=True` defeats it. `canfar/storages.py:76,88` +already passes `skip_instance_cache=True`. **Keep that** for the returned instance: +fsspec's instance cache is keyed on the storage options, and a cached instance would +outlive a token refresh. + +## 5. Tiered caching: what genuinely composes + +### The composition matrix + +| Layering | Result | Evidence | +| --- | --- | --- | +| `SimpleCacheFileSystem(fs=vosfs_sync)` | **works**; cold 1.58 s → warm 0.000 s | VERIFIED | +| `WholeFileCacheFileSystem(fs=vosfs_sync)` | **works**; warm 0.000 s | VERIFIED | +| `CachingFileSystem` (blockcache) | **fails**, see §10 | VERIFIED | +| `fs.open(path, cache_type=…)` | **silently ignored** | VERIFIED | +| `filecache::simplecache::vos://` | builds; **middle layer is dead** | VERIFIED | +| hand-stacked `SimpleCache(SimpleCache(vosfs))` | **inner layer never fills** | VERIFIED | +| `cache_storage=[a, b]` | two-location lookup, **no promotion** | VERIFIED | +| `cache_storage="memory://…"` | **silently creates a local dir named `memory:`** | VERIFIED | +| `DirFileSystem(path=…, fs=vosfs)` | works, `async_impl=True` | VERIFIED | +| `SimpleCache(DirFileSystem(vosfs))` | works, 1.34 s cold | VERIFIED | +| `AsyncFileSystemWrapper(SimpleCache(vosfs_sync))` | works, non-blocking | VERIFIED | +| `SimpleCache(fs=vosfs_async)` | `RuntimeError: Loop is not running` | VERIFIED | + +### `cache_type=` is accepted and discarded + +VERIFIED. Every value in fsspec's cache registry +(`list(fsspec.caching.caches)` = `[None, 'none', 'mmap', 'bytes', 'readahead', +'blockcache', 'first', 'all', 'parts', 'background']`) was passed to +`vosfs.open(path, "rb", cache_type=…)`. All six tested values returned a +`StagedReadFile` with **no `.cache` attribute at all** and identical behaviour. No +warning, no error. + +DOCUMENTED, and this is why: `vosfs/filesystem.py:683-690` accepts `block_size` and +`cache_options` marked `# noqa: ARG002 - accepted, non-behavioural`, and +`vosfs/staging.py:59` defines `StagedReadFile(io.BufferedReader)` — a plain buffered +reader, not an `fsspec.spec.AbstractBufferedFile`. fsspec's read-caching machinery only +ever attaches to `AbstractBufferedFile`. **Never document `cache_type` as a `canfar` +tuning knob.** + +### URL chaining works, but only with `endpoint_url`, and the middle tier is dead + +VERIFIED. `filecache::vos://…` with only `vos={"certfile": …}` fails: + +```text +TypeError: VOSpaceFileSystem.__init__() missing 1 required positional argument: 'endpoint_url' +``` + +It succeeds when the endpoint is supplied in the target options: + +```python +fs, path = fsspec.core.url_to_fs( + f"simplecache::vos://{remote_path}", + vos={"endpoint_url": endpoint, "certfile": cert, "skip_instance_cache": True}, + simplecache={"cache_storage": cache_dir}, +) +``` + +cold 1.59 s → warm 0.000 s. + +VERIFIED, and this is the trap: `filecache::simplecache::vos://` **builds the right +object graph and then bypasses the middle layer**. After a cold read: + +```text +outer (filecache) dir: ['c6de39f6…'] +middle (simplecache) dir: [] <- never written +``` + +Wiping only the outer tier and re-reading took 1.52 s — a network round trip — and the +middle tier stayed empty. Hand-stacking `SimpleCacheFileSystem(fs=SimpleCacheFileSystem(fs=vosfs))` +reproduces it exactly: outer dir populated, inner dir empty. + +DOCUMENTED cause: `WholeFileCacheFileSystem` fills itself with +`self.fs.get(getpaths, storepaths)` / `self.fs.get_file(path, fn)` +(`fsspec/implementations/cached.py:673,706`). `get`/`get_file` on the inner cache +filesystem is a straight pass-through to the remote; it does not route through the +inner cache's own `_open`/`cat` path, so the inner cache is never populated. **Chained +cache layers are not a tiering mechanism in fsspec 2026.6.0.** + +### `cache_storage` as a list is the only real two-location mechanism + +DOCUMENTED. `fsspec/implementations/cached.py:87-91`: + +> `cache_storage: str or list(str)` — Location to store files. […] If a list, each +> location will be tried in the order given, but only the last will be considered +> writable. + +`fsspec/implementations/cached.py:140` runs `os.makedirs(storage[-1], exist_ok=True)`; +`SimpleCacheFileSystem._check_file` (`:820-826`) scans `self.storage` in order and +returns the first hit. + +VERIFIED with `cache_storage=[slow, fast]`: + +| Step | Result | +| --- | --- | +| cold read | 1.47 s; written to `fast` (the **last** entry) only | +| move the object to `slow`, empty `fast`, re-read | 0.037 s — served locally from `slow` | +| `fast` after that read | **empty — no promotion** | + +So a list gives you *lookup across two directories*, not a hot/cold hierarchy. It is +exactly right for "a shared, pre-populated, read-only cache behind my own writable +one", and it is wrong for "keep hot objects on the fast tier". + +### The RAM tier is not real + +VERIFIED and worth flagging as a hazard: `SimpleCacheFileSystem(fs=…, +cache_storage="memory://ram")` **constructs without error** and creates a literal local +directory named `memory:` in the process working directory. Nothing about the +`MemoryFileSystem` is involved. (This experiment created such a directory in the +repository root; it was removed and `git status` is clean.) + +VERIFIED alternatives: + +- `MemoryFileSystem` (`fsspec/implementations/memory.py`) is not a cache wrapper; its + `store` is a process-global `dict`. Hand-rolling `mem.pipe_file(key, fs.cat_file(p))` + works (warm read 0.0000 s) but is just a dictionary with extra ceremony, and it is + process-global mutable state. +- `MMapCache(location=None)` is a genuine anonymous-mmap RAM cache, but its fetcher + over `vosfs` downloads whole objects (§2), so it is a pessimization here. +- `/dev/shm` does not exist on this Darwin host (VERIFIED); on Linux — including CANFAR + Sessions — it is a tmpfs and is a perfectly good `cache_storage` value. I could not + test this on the available machine. + +**The honest RAM tier is the OS page cache sitting over your disk cache directory.** +A warm `simplecache` read measured 0.0002 s, which is memory speed, because the kernel +is already holding the file. Adding an explicit RAM layer buys nothing and costs +correctness. + +### Per-scenario recommendation + +| Scenario | Recommendation | +| --- | --- | +| **(a) Laptop, RAM only** | `filesystem(name, cache=False)` and hold bytes yourself via `fs.cat_file` / `fs.cat_ranges`. If you want a cache, use `cache=True` and let the page cache be the RAM tier. Do not construct `MemoryFileSystem` or `MMapCache`. | +| **(b) Laptop, RAM + second NVMe tier** | One `SimpleCacheFileSystem` pointed at the NVMe: `filesystem(name, cache="/mnt/nvme/canfar/")`. RAM tier = page cache. **Do not stack two cache layers** — the inner one is dead (VERIFIED above). | +| **(c) CANFAR Session, `/scratch` + RAM** | `filesystem(name, cache=True)`, which auto-selects `/scratch/canfar/`. `/scratch` is per-Session NVMe and is cleared at Session end, which is exactly a cache lifetime. If a shared pre-populated read-only cache exists, use `cache=["/shared/canfar-cache", "/scratch/canfar/"]` and document that hits in the shared tier are **not** promoted. | + +### Cache-location auto-detection: stdlib only + +**Do not add `platformdirs`.** VERIFIED: it is not in `pyproject.toml:48-65`. It +appears in `uv.lock` only as a transitive dependency of `mkdocs-get-deps` (`uv.lock:1256`) +and `virtualenv` (`uv.lock:2183`) — docs and dev groups, not runtime. Adding it to +runtime dependencies to pick a *cache* directory is backwards anyway: `platformdirs` +returns durable user-data/user-cache locations, and what is wanted here is ephemeral +scratch. `tempfile.gettempdir()` already honours `TMPDIR` and is one stdlib call. + +VERIFIED sketch (ran correctly on this host, falling back to the temp dir because +`/scratch` is absent): + +```python +def _cache_location(name: str) -> Path: + """Return the default cache directory for one Storage Name. + + Prefers the CANFAR Science Platform Server's per-Session ``/scratch`` volume, + which is fast local disk and is cleared when the Session ends. + + Args: + name: Storage Name of the configured VOSpace Service. + + Returns: + Path: Writable directory for cached objects. + """ + scratch = Path("/scratch") + root = scratch if scratch.is_dir() and os.access(scratch, os.W_OK) else Path(tempfile.gettempdir()) + return root / "canfar" / name +``` + +Keying by Storage Name matters: `fsspec`'s cache mapper hashes the *stripped* path +(`fsspec/implementations/cached.py:627`), which does not include the endpoint, so +`arc:/x` and `vault:/x` would otherwise collide in one directory. + +## 6. `WholeFileCacheFileSystem` vs `SimpleCacheFileSystem` + +**Recommend `SimpleCacheFileSystem`.** + +DOCUMENTED. `fsspec/implementations/cached.py:791-805`: `SimpleCacheFileSystem` +"only copies whole files, and does not keep any metadata about the download time or +file details. It is therefore safer to use in multi-threaded/concurrent situations." +fsspec's feature docs state "Only 'simplecache' is guaranteed thread/process-safe" +(). +Its `__init__` (`:811-818`) forces `cache_check`, `expiry_time` and `check_files` to +`False`. + +VERIFIED: both work; both give warm reads at 0.000 s. `WholeFileCacheFileSystem` keeps +a JSON metadata sidecar (a `cache` file appears alongside the objects) and computes +`self.fs.ukey(path)` on every miss (`fsspec/implementations/cached.py:634`), which is +an extra `info` round trip. A `/scratch` cache that dies with the Session does not need +expiry metadata. Expose `WholeFileCacheFileSystem` only if a user asks for expiry — do +not put it in the first API. + +## 7. "Get a file location" + +VERIFIED results for materializing a remote object to a local path: + +| Approach | Works over `vosfs`? | +| --- | --- | +| `fsspec.open_local("vos://…")` | **No.** `ValueError: open_local can only be used on a filesystem which has attribute local_file=True` | +| `fsspec.open_local("simplecache::vos://…", vos={"endpoint_url": …}, simplecache={"cache_storage": …})` | **Yes** — 1.64 s, returns the cache path | +| `fs.get_file(remote, local)` | **Yes** — writes the exact 169 920 B; explicit destination | +| `SimpleCacheFileSystem.open(path).name` | **Yes** — is the local cache path, exists on disk | +| `SimpleCacheFileSystem._check_file(path)` | **Yes** — returns the local path directly, `None` before the first read | +| `WholeFileCacheFileSystem._check_file(path)` | Returns a `(detail, path)` **tuple**, not a string | +| `LocalFileOpener` | Not reachable — it is `LocalFileSystem`'s own file class, never produced by a cache layer over `vosfs` | + +DOCUMENTED. `fsspec/core.py:544-549` is the `local_file` gate; +`fsspec/implementations/cached.py:560,808` set `local_file = True` on +`WholeFileCacheFileSystem` and `SimpleCacheFileSystem`. Bare `vosfs` sets nothing, hence +the failure. `WholeFileCacheFileSystem._open` (`:730-738`) returns a builtin +`open(fn, mode)` — a real `_io.BufferedReader` on a real local path — which is why +`.name` is usable. + +**Recommendation.** Expose `fetch()`. The two working routes are `fsspec.open_local` +with a chained URL — which requires callers to hand-assemble `vos://` strings and +`endpoint_url`, exactly the resolver footgun rejected in §3 — and `_check_file`, which +is private API. A four-line public function is cheaper than teaching either: + +```python +def fetch(name: str, path: str, *, cache: bool | str | Path = True) -> Path: + """Materialize one VOSpace object locally and return its path. + + Args: + name: Storage Name of the configured VOSpace Service. + path: Absolute path within that VOSpace Service. + cache: Cache location, or ``True`` for the default (see ``filesystem``). + + Returns: + Path: Local path holding the object's bytes. + """ + fs = filesystem(name, cache=cache or True) + fs.cat_file(path) # populates the cache + return Path(fs._check_file(path)) +``` + +If reaching into `_check_file` is unacceptable, use `fs.get_file(path, destination)` +with an explicit destination instead and drop the cache-path form — but then the caller +owns cleanup, and repeat calls re-download. Prefer the cache-backed form; the private +attribute is one call site in one file and is trivially covered by a test. + +## 8. Sync-first, async-first, or both + +**Sync-first, with `asynchronous=True` as a keyword. Not two classes, not two modules.** + +Reasons: + +1. Caching only exists in sync form (§4, §5). A sync-first API means the common case — + "read this FITS file, cache it on `/scratch`" — is one call with no wrapper. +2. Astronomy consumers of a materialized path (`astropy.io.fits.open`, + `numpy.load`, `pandas.read_*`) are synchronous. +3. The async path is genuinely better when uncached — native `VOSpaceFileSystem(..., + asynchronous=True)` was VERIFIED working for `_info` and `_ls` — so it must remain + available, and `sources()` depends on it. +4. `canfar` already ships sync/async parity elsewhere (`Session` / `AsyncSession`, + `AGENTS.md:58`), but that parity is for a hand-written client. Here both paths are + the same fsspec object with one constructor flag; two classes would be pure + duplication. + +### Relationship to `sources()` + +`sources()` moves to `canfar/storage.py` **unchanged**. Its contract with fsspec-cli is +`Callable[[], AbstractAsyncContextManager[AbstractFileSystem]]` +(`fsspec_cli/_app.py:41-43`), it must yield an async filesystem, and it must build and +`aclose()` per command so a listing cache cannot outlive its command +(`canfar/storages.py:93-96`, `docs/cli/data.md:106-110`). + +Refactor `_vospace` to delegate to the new `filesystem()` rather than duplicating +construction: + +```python +def _vospace(name, *, token=None, certificate=None) -> AsyncFilesystemSource: + @asynccontextmanager + async def source() -> AsyncIterator[AbstractFileSystem]: + fs = filesystem(name, token=token, certificate=certificate, asynchronous=True) + try: + yield fs + finally: + await fs.aclose() + return source +``` + +Do **not** put a cache layer behind `sources()`. `fsspec-cli` performs writes +(`cp`, `mkdir`, `rm`); `SimpleCacheFileSystem` has write semantics of its own +(`fsspec/implementations/cached.py:800-803`, `WriteCachedTransaction`) that were not +evaluated here, and a CLI that silently served stale bytes would be a defect. The CLI's +existing per-command listing cache is the right amount of caching for it. + +## 9. Recommended API sketch + +`canfar/storage.py` — three public functions, one private helper, no new dependency. +Every element below was exercised in the prototype (VERIFIED, §4/§5/§7). + +```python +"""Public access to configured VOSpace Services and the local filesystem.""" + +from __future__ import annotations + +import os +import tempfile +from contextlib import asynccontextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from fsspec.asyn import get_loop, sync +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper +from fsspec.implementations.cached import SimpleCacheFileSystem +from fsspec.implementations.local import LocalFileSystem + +from canfar.client import HTTPClient +from canfar.exceptions.context import AuthContextError +from canfar.models.config import Configuration + +_LISTINGS_EXPIRY_SECONDS = 30 +_LISTINGS_MAX_PATHS = 1000 + +CacheLocation = bool | str | Path | Sequence[str | Path] + + +def filesystem( + name: str, + *, + cache: CacheLocation = False, + asynchronous: bool = False, + token: str | SecretStr | None = None, + certificate: Path | None = None, +) -> AbstractFileSystem: + """Return an authenticated filesystem for one Storage Name. + + Args: + name: Storage Name of the configured VOSpace Service. + cache: ``False`` for no caching, ``True`` for the default cache + directory, a path for an explicit one, or a sequence of paths + tried in order where only the last is writable. + asynchronous: Return an async filesystem whose coroutine hooks are + awaited directly. Sync methods are unavailable on the result. + token: Runtime bearer token, preferred over any saved credential. + certificate: Runtime X.509 certificate path. + + Returns: + AbstractFileSystem: A ready filesystem rooted at the VOSpace Service. + + Raises: + AuthContextError: If no usable credential exists for the Identity + Provider that owns this VOSpace Service. + """ + + +def fetch( + name: str, + path: str, + *, + cache: CacheLocation = True, +) -> Path: + """Materialize one VOSpace object locally and return its path.""" + + +def sources() -> dict[str, AsyncFilesystemSource]: + """Build the mapped storage sources for one data command invocation.""" + # unchanged from canfar/storages.py:114-130 +``` + +Construction body, matching the rule table in §4: + +```python + config = Configuration() + endpoint, idp = config._resolve_storage(name) + try: + client = HTTPClient(config=config, authentication_idp=idp, url=endpoint, ...) + token_value, certfile = sync(get_loop(), client._materialize_credentials) + except (KeyError, OSError, TypeError, ValueError): + raise AuthContextError(idp, "Credential cannot be used. Run 'canfar login' for this IDP.") from None + + from vosfs import VOSpaceFileSystem + + credential = {"token": token_value} if token_value is not None else {"certfile": certfile} + remote = VOSpaceFileSystem( + endpoint, + asynchronous=asynchronous and not cache, + skip_instance_cache=True, + use_listings_cache=True, + listings_expiry_time=_LISTINGS_EXPIRY_SECONDS, + max_paths=_LISTINGS_MAX_PATHS, + **credential, + ) + if not cache: + return remote + storage = _cache_location(name) if cache is True else cache + cached = SimpleCacheFileSystem(fs=remote, cache_storage=_as_storage(storage)) + return AsyncFileSystemWrapper(cached, asynchronous=True) if asynchronous else cached +``` + +Notes on the shape: + +- `cache` is a single parameter, not a configurator object or a `CacheConfig` model. + A Pydantic model would be a data model with no persisted state and no validation + beyond "is this a path", which the filesystem call already performs. +- `asynchronous=True` with `cache` truthy must build the *sync* `vosfs` and wrap + (`asynchronous=asynchronous and not cache` above) — this is the single most + error-prone line and deserves a comment and a test. +- `skip_instance_cache=True` is retained deliberately (§4). +- No `Storage`/`VOSpace` class. Every method a caller could want is already on + `AbstractFileSystem`. + +### Usage the docs should show + +```python +from canfar.storage import fetch, filesystem + +# One-off read. +vault = filesystem("vault") +header = vault.cat_file("/ALMA/test-data/cutouts/test-4d-cube.fits", 0, 2880) + +# Repeated reads on a Science Platform Server Session: cache under /scratch. +vault = filesystem("vault", cache=True) +data = vault.cat_file("/ALMA/test-data/cutouts/test-4d-cube.fits") # 1.6 s +again = vault.cat_file("/ALMA/test-data/cutouts/test-4d-cube.fits") # 0.0002 s + +# Hand a local path to a library that only takes paths. +from astropy.io import fits +with fits.open(fetch("vault", "/ALMA/test-data/cutouts/test-4d-cube.fits")) as hdul: + ... + +# Concurrent async reads, uncached. +arc = filesystem("arc", asynchronous=True) +try: + sizes = await asyncio.gather(*(arc._info(p) for p in paths)) +finally: + await arc.aclose() +``` + +The `cat_file(path, 0, 2880)` in the first example is honest — it returns the right +bytes — but §2 means it costs a full download. Say so in the docs. + +## 10. Upstream gaps + +### Gap 1 — `blockcache` is unusable (root cause: no ranged reads) + +VERIFIED exact error, from `CachingFileSystem(fs=vosfs, cache_storage=…)` on both +`.open(path).read(100)` and `.cat_file(path)`: + +```text +AttributeError: 'StagedReadFile' object has no attribute 'blocksize' +``` + +DOCUMENTED cause: `fsspec/implementations/cached.py:702` reads +`block = getattr(f, "blocksize", 5 * 2**20)` defensively, but the failing access is on +the `CachingFileSystem` path, which assumes the target's file object is an +`AbstractBufferedFile` with `.blocksize` and `.cache`. `vosfs/staging.py:59` returns an +`io.BufferedReader` subclass instead. + +**What would unblock it, in order:** (1) OpenCADC Cavern implementing HTTP `Range` on +the negotiated data endpoint; (2) `vosfs` sending `Range` headers from `_cat_file`; +(3) `vosfs._open` returning an `fsspec.spec.AbstractBufferedFile` subclass backed by +those ranged reads. Adding a `blocksize` attribute alone would convert a loud failure +into a silent whole-object-per-block pessimization — do not ask for that. + +**Until then, `canfar` should not offer a block-cache option at all.** + +### Gap 2 — fsspec accepts a URL as `cache_storage` + +VERIFIED. `cache_storage="memory://ram"` is `os.makedirs`'d verbatim +(`fsspec/implementations/cached.py:140`), creating a directory named `memory:`. No +error, no warning. Worth an upstream issue; in the meantime, `canfar` should reject +any `cache` value containing `://` with a clear message. + +### Gap 3 — chained cache layers silently no-op + +VERIFIED (§5). `filecache::simplecache::X` constructs a three-layer stack in which the +middle layer never caches. Worth an upstream issue. `canfar` should never document +chained cache URLs. + +## 11. Documentation changes required + +If this API ships: + +- `docs/cli/data.md:176` — "This release intentionally provides no public CANFAR + storage Python API" must be updated; the FUSE mount, signed-URL, progress display and + the other listed omissions still stand. +- A new `docs/client/storage.md` for `canfar.storage`. + +**Independently of this API**, `docs/cli/data.md:143-165` is factually wrong and should +be corrected now: + +- `:143-144` "VOSpace supports ranged reads, so a large cube can be cached one block at + a time rather than downloaded whole" — VERIFIED false for `vosfs` 0.7.0. +- `:162-163` "Reading a FITS header this way transfers a single block instead of the + whole file, and a second read of the same range is served from `/scratch`" — the + second clause is true; the first is VERIFIED false. The `MMapCache` example moves + 3 542 400 B to return 2 880 B, and three scattered header reads cost three full + downloads (4.01 s versus 1.64 s for reading the file once). +- `:164-165` "Ranged reads help most on large files; on small ones the per-request + VOSpace transfer negotiation dominates" — the ordering is inverted. Ranged reads help + on *no* file size today; larger files are strictly worse. + +The correct advice for large-cube access with `vosfs` 0.7.0 is: read the object once +into a `simplecache` directory on `/scratch` and slice locally, or use `cat_ranges`, +which does group multiple ranges of one object into a single download +(`vosfs/filesystem.py:594-620`, VERIFIED: 4 ranges in 1.16 s). + +## 12. What I could not verify + +- **`/dev/shm` as `cache_storage` on a CANFAR Session.** `/dev/shm` does not exist on + this Darwin host (VERIFIED absent). The mechanism is a plain directory path so there + is no fsspec reason it would fail, but the tmpfs sizing and eviction behaviour on a + Science Platform Server Session is untested. +- **`/scratch` on a real Session.** Auto-detection was exercised only through its + fallback branch; `Path("/scratch").is_dir()` was `False` here. +- **Write paths through a cache layer.** All experiments were read-only against `vault` + by design. `SimpleCacheFileSystem`'s write/commit semantics + (`fsspec/implementations/cached.py:800-803`) were read, not executed. This is why §8 + recommends keeping caching out of `sources()`. +- **OIDC credentials.** Only the X.509 branch of `_materialize_credentials` was + exercised (`~/.ssl/cadcproxy.pem`). The OIDC-refresh branch + (`canfar/client.py:252-259`) is `async` and was not driven through + `fsspec.asyn.sync`; it should be covered by a test before release. +- **`arc`.** Every live measurement used `vault`. `arc` was not touched, per the + read-only constraint. +- **First-call latency stability.** Observed first-I/O costs ranged from 0.43 s to + 14.3 s with no reproducible pattern. Cause not established; do not publish a number. + +## Primary sources + +- `canfar/storages.py`, `canfar/cli/data.py`, `canfar/client.py`, + `canfar/models/config.py`, `pyproject.toml`, `uv.lock`, `CONTEXT.md`, `AGENTS.md`, + `docs/cli/data.md` (repository snapshot `ba5493fd`) +- `.venv/lib/python3.13/site-packages/vosfs/filesystem.py`, `.../vosfs/staging.py`, + `.../vosfs/_transfer.py` (vosfs 0.7.0) +- `.venv/lib/python3.13/site-packages/fsspec/implementations/cached.py`, + `.../fsspec/implementations/asyn_wrapper.py`, `.../fsspec/implementations/dirfs.py`, + `.../fsspec/implementations/memory.py`, `.../fsspec/caching.py`, `.../fsspec/core.py` + (fsspec 2026.6.0) +- `.venv/lib/python3.13/site-packages/fsspec_cli/_app.py` (fsspec-cli 0.6.0) +- [fsspec: Features — caching files locally](https://filesystem-spec.readthedocs.io/en/latest/features.html#caching-files-locally) +- [fsspec: Features — URL chaining](https://filesystem-spec.readthedocs.io/en/latest/features.html#url-chaining) +- [fsspec: Async](https://filesystem-spec.readthedocs.io/en/latest/async.html) +- [vosfs v0.7.0](https://github.com/shinybrar/vosfs/releases/tag/v0.7.0) +- [fsspec-cli v0.6.0](https://github.com/shinybrar/vosfs/releases/tag/fsspec-cli-v0.6.0) diff --git a/docs/cli/data.md b/docs/cli/data.md index 84c41ada..1785a7b7 100644 --- a/docs/cli/data.md +++ b/docs/cli/data.md @@ -140,29 +140,23 @@ staleness metadata that `WholeFileCacheFileSystem` keeps. Passing `cache_storage` a list of directories tries each in order and treats only the last as writable, so a shared read-only cache can back your own. -VOSpace supports ranged reads, so a large cube can be cached one block at a -time rather than downloaded whole. `MMapCache` keeps fetched blocks in a sparse -file, so only the blocks you touch occupy disk: +### Do not cache byte ranges -```python -from fsspec.caching import MMapCache - -path = "/ALMA/test-data/cutouts/test-4d-cube.fits" -size = vault.info(path)["size"] -blocks = MMapCache( - blocksize=1 << 20, - fetcher=lambda start, end: vault.cat_file(path, start, end), - size=size, - location="/scratch/vault-cache/test-4d-cube.blocks", -) +Cache whole files, never blocks. OpenCADC Cavern does not implement HTTP byte +ranges, so every partial read downloads the whole object and slices it in +memory. A block cache therefore turns one download into one download *per +block*: reading three headers out of a cube fetches that cube three times. -header = blocks._fetch(0, 2880) # one FITS header block, one 1 MiB fetch +Avoid `MMapCache` and any block-cache layer over a VOSpace Service for this +reason. `blockcache` refuses outright, which is the safer failure: + +```text +AttributeError: 'StagedReadFile' object has no attribute 'blocksize' ``` -Reading a FITS header this way transfers a single block instead of the whole -file, and a second read of the same range is served from `/scratch`. Ranged -reads help most on large files; on small ones the per-request VOSpace transfer -negotiation dominates. +Stacked caches do not help either: chaining them (`filecache::simplecache::`) +builds the layers, but the inner layer is never filled and never serves, so use +exactly one cache layer on your fastest local disk. These caches use the synchronous filesystem interface. Build the filesystem without `asynchronous=True`, as above. From dfe368033ab57dea188f47fd8d4871593666d9cc Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Sat, 25 Jul 2026 00:28:26 -0700 Subject: [PATCH 42/47] docs(data): separate client and service byte-range limits The previous correction blamed the missing byte ranges on OpenCADC Cavern and generalised that to every VOSpace Service. Verified live against both services, that is wrong: vault is backed by minoc and answers a ranged GET with 206, Accept-Ranges: bytes and Content-Range: bytes 0-2879/3542400, while arc returns the whole body and advertises no range support. The limitation that matters is in the client: vosfs sends no Range header, so a partial read downloads the whole object on either service, and block caches stay a pessimization today. State it that way, and record that teaching vosfs to send Range would unlock real byte-range reads on vault but not on arc. Corrects the same conflation in the research note. --- .../research/2026-07-25-storage-python-api.md | 18 ++++++++++++++- docs/cli/data.md | 22 +++++++++++++------ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/docs/agents/research/2026-07-25-storage-python-api.md b/docs/agents/research/2026-07-25-storage-python-api.md index f557f68d..a558d473 100644 --- a/docs/agents/research/2026-07-25-storage-python-api.md +++ b/docs/agents/research/2026-07-25-storage-python-api.md @@ -86,7 +86,23 @@ therefore free to be renamed. ## 2. The constraint that dominates everything: no byte ranges -DOCUMENTED. `vosfs/staging.py:3-5` states it outright: +> **Correction (2026-07-25, after review).** The heading below overstates the +> finding: the limitation is in the **client**, not uniformly in the services. +> `vosfs` sends no `Range` header, so every partial read is a whole-object +> download — that part stands and is what makes block caching a pessimization +> today. But the services differ, VERIFIED live against both: +> +> | Service | Backend | Ranged `GET` | +> | --- | --- | --- | +> | `vault` | `minoc` (`ws-cadc.canfar.net/minoc/files/...`) | `206`, `Accept-Ranges: bytes`, `Content-Range: bytes 0-2879/3542400` | +> | `arc` | Cavern | `200`, whole body, no `Accept-Ranges` | +> +> So the Cavern quote below is accurate for `arc` and does not generalize to +> `vault`. Teaching `vosfs` to send `Range` would deliver real byte-range reads +> on `vault` (not on `arc`), which upgrades the upstream ask in section 7 from +> "cosmetic" to genuinely valuable. + +DOCUMENTED. `vosfs/staging.py:3-5` states it outright, for Cavern: > OpenCADC Cavern does not implement HTTP byte ranges, so a seekable read is a > whole-object download into a disk-backed temporary file. diff --git a/docs/cli/data.md b/docs/cli/data.md index 1785a7b7..61aa0ddf 100644 --- a/docs/cli/data.md +++ b/docs/cli/data.md @@ -142,13 +142,21 @@ last as writable, so a shared read-only cache can back your own. ### Do not cache byte ranges -Cache whole files, never blocks. OpenCADC Cavern does not implement HTTP byte -ranges, so every partial read downloads the whole object and slices it in -memory. A block cache therefore turns one download into one download *per -block*: reading three headers out of a cube fetches that cube three times. - -Avoid `MMapCache` and any block-cache layer over a VOSpace Service for this -reason. `blockcache` refuses outright, which is the safer failure: +Cache whole files, never blocks. `vosfs` sends no HTTP `Range` header, so a +partial read such as `cat_file(path, start, end)` downloads the whole object +and slices it in memory. A block cache therefore turns one download into one +download *per block*: reading three headers out of a cube fetches that cube +three times. + +This is a client limitation, and it is not uniform across services. The `vault` +backend does serve ranges — a direct request returns `206` with +`Accept-Ranges: bytes` — while `arc` returns the whole body and advertises no +range support. So genuine byte-range reads are possible against `vault` today +only by bypassing `vosfs`, and teaching `vosfs` to send `Range` would unlock +them properly for `vault` but not for `arc`. + +Avoid `MMapCache` and any block-cache layer over a VOSpace Service until then. +`blockcache` refuses outright, which is the safer failure: ```text AttributeError: 'StagedReadFile' object has no attribute 'blocksize' From 6bfd2d294a90a621325f78dd77f36c37ef6217ca Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Sat, 25 Jul 2026 00:40:29 -0700 Subject: [PATCH 43/47] refactor(storage): unify on storage and Storage Identifier The same concept was called three things: `data` in the CLI, "Storage Name" in configuration and errors, and `identifier` in the operand grammar. Settle on one vocabulary. - CONTEXT.md renames the term to Storage Identifier and records that the reserved `local` identifier addresses the machine the command runs on. - Rename canfar/storages.py to canfar/storage.py, matching the singular module convention and the noun the domain now uses. - Reword user-visible errors, docstrings, and docs to Storage Identifier, and make the operand placeholder `storage-identifier:/absolute/path` so it names the slot rather than borrowing the glossary noun. Deliberately unchanged: the `storage` key in the configuration file, which is already the settled word and whose format is an invariant; the `canfar data` command group, which names the action rather than the thing; and Server Name, which is a separate term. This is naming only. Verified the saved configuration still loads and both Storage Identifiers still resolve. --- CONTEXT.md | 8 ++-- canfar/cli/data.py | 2 +- canfar/models/config.py | 16 ++++--- canfar/models/http.py | 9 ++-- canfar/server.py | 6 +-- canfar/{storages.py => storage.py} | 6 +-- .../research/2026-07-25-storage-python-api.md | 44 +++++++++---------- docs/cli/cli-help.md | 2 +- docs/cli/data.md | 4 +- docs/cli/quick-start.md | 2 +- tests/test_cli_data.py | 38 ++++++++-------- tests/test_data_smoke.py | 2 +- tests/test_models_config.py | 20 +++++---- tests/test_server.py | 2 +- tests/{test_storages.py => test_storage.py} | 2 +- 15 files changed, 84 insertions(+), 79 deletions(-) rename canfar/{storages.py => storage.py} (96%) rename tests/{test_storages.py => test_storage.py} (99%) diff --git a/CONTEXT.md b/CONTEXT.md index d1608f28..8f4ba627 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -20,9 +20,9 @@ _Avoid_: Label, alias, key Remote astronomical storage service associated with a **Science Platform Server**. _Avoid_: Storage backend, filesystem -**Storage Name**: -Required, globally unique, user-facing handle for one **VOSpace Service**. -_Avoid_: Server Name, alias, key +**Storage Identifier**: +Required, globally unique, user-facing handle for one **VOSpace Service**, used to address it in configuration, commands, and operands such as `arc:/home/user`. The reserved identifier `local` addresses the machine where the command runs. +_Avoid_: Storage Name, Server Name, alias, key **Identity Provider (IDP)**: Organization that issues user identity for CANFAR authentication. @@ -78,7 +78,7 @@ _Avoid_: Resource profile, quota - A **CANFAR Science Platform** exposes one or more **Science Platform Servers**. - A **Science Platform Server** is identified by its **Server Name**; its IVOA URI is discovery metadata, and two Server Names may point at the same endpoint. - A **Science Platform Server** can expose multiple **VOSpace Services**. -- A **VOSpace Service** is identified by its **Storage Name** and uses its parent **Science Platform Server**'s **Identity Provider (IDP)**. +- A **VOSpace Service** is identified by its **Storage Identifier** and uses its parent **Science Platform Server**'s **Identity Provider (IDP)**. - An **Identity Provider (IDP)** can support one or more **Science Platform Servers**. - **Authentication** and **Platform** are separate seams with independent ownership. - An **Authentication Record** belongs to one **Identity Provider (IDP)**. diff --git a/canfar/cli/data.py b/canfar/cli/data.py index 653b612b..e3f3d3d9 100644 --- a/canfar/cli/data.py +++ b/canfar/cli/data.py @@ -9,7 +9,7 @@ from typer.core import TyperGroup from typer.main import get_group -from canfar.storages import sources +from canfar.storage import sources if TYPE_CHECKING: from typer._click.core import Command, Context diff --git a/canfar/models/config.py b/canfar/models/config.py index 95cec056..5bd5d01b 100644 --- a/canfar/models/config.py +++ b/canfar/models/config.py @@ -189,7 +189,7 @@ def settings_customise_sources( ) def _heal_default_storage(self) -> None: - """Restore default Storage Names on Servers saved by an older client. + """Restore default Storage Identifiers on Servers saved by an older client. Older clients keyed a discovered VOSpace Service by its Server Name, so an existing configuration holds ``canfar`` instead of ``arc`` and never @@ -203,7 +203,7 @@ def _heal_default_storage(self) -> None: storage = dict(server.storage) legacy = storage.pop(name, None) if legacy is None and storage: - # Deliberate Storage Names are configuration, not stale defaults. + # Deliberate Storage Identifiers are configuration, not stale defaults. continue if legacy is not None: leaf = str(legacy.uri).rpartition("/")[2] or name @@ -218,7 +218,7 @@ def _heal_default_storage(self) -> None: @model_validator(mode="after") def _normalize_and_validate_servers(self) -> Configuration: - """Inject Server Names and validate Server and Storage Name keys.""" + """Inject Server Names and validate Server and Storage Identifier keys.""" self._heal_default_storage() updated: dict[str, Server] = {} server_name_by_storage_name: dict[str, str] = {} @@ -234,7 +234,8 @@ def _normalize_and_validate_servers(self) -> Configuration: storage_name ): msg = ( - f"Duplicate Storage Name '{storage_name}' in Science Platform " + f"Duplicate Storage Identifier '{storage_name}' in " + "Science Platform " f"Servers '{previous_server_name}' and '{name}'." ) raise ValueError(msg) @@ -380,18 +381,19 @@ def _get_server_by_name(self, name: str) -> Server: return self.servers[name] def _resolve_storage(self, storage_name: str) -> tuple[str, str]: - """Resolve a Storage Name to its endpoint and parent server IDP.""" + """Resolve a Storage Identifier to its endpoint and parent server IDP.""" for server in self.servers.values(): service = server.storage.get(storage_name) if service is not None: if server.idp is None: msg = ( - f"Storage Name '{storage_name}' belongs to a Science Platform " + f"Storage Identifier '{storage_name}' belongs to a " + "Science Platform " "Server without an IDP." ) raise ValueError(msg) return str(service.url), server.idp - msg = f"Storage Name '{storage_name}' is not configured." + msg = f"Storage Identifier '{storage_name}' is not configured." raise KeyError(msg) def upsert_credential(self, credential: AuthenticationCredential) -> None: diff --git a/canfar/models/http.py b/canfar/models/http.py index 28824a98..8666a774 100644 --- a/canfar/models/http.py +++ b/canfar/models/http.py @@ -94,7 +94,7 @@ class Server(BaseModel): storage: dict[str, VOSpaceService] = Field( default_factory=dict, title="VOSpace Services", - description="VOSpace Services keyed by globally unique Storage Name.", + description="VOSpace Services keyed by globally unique Storage Identifier.", ) cores: int = Field( @@ -128,7 +128,7 @@ class Server(BaseModel): @field_validator("storage", mode="before") @classmethod def _validate_storage_names(cls, value: Any) -> Any: - """Normalize and validate Storage Names before Pydantic transforms keys.""" + """Normalize and validate Storage Identifiers before key transforms.""" if not isinstance(value, dict): return value @@ -143,7 +143,7 @@ def _validate_storage_names(cls, value: Any) -> Any: name = original_name.strip() if name is None or (not name or name == "local" or name.startswith("-")): msg = ( - f"Invalid Storage Name {original_name!r}: after whitespace " + f"Invalid Storage Identifier {original_name!r}: after whitespace " "normalization it must be non-empty, differ from reserved 'local', " "contain no colon, NUL, or newline, and not start with '-'." ) @@ -151,7 +151,8 @@ def _validate_storage_names(cls, value: Any) -> Any: if name in normalized: previous_original_name = original_name_by_normalized_name[name] msg = ( - f"Storage Names {previous_original_name!r} and {original_name!r} " + f"Storage Identifiers {previous_original_name!r} and " + f"{original_name!r} " f"both normalize to {name!r}; use unique names." ) raise ValueError(msg) diff --git a/canfar/server.py b/canfar/server.py index fee7646d..da162486 100644 --- a/canfar/server.py +++ b/canfar/server.py @@ -106,11 +106,11 @@ def _merge_storage( of being duplicated under the Server Name on every rediscovery. Args: - known: Configured VOSpace Services keyed by Storage Name. - found: Newly discovered VOSpace Services keyed by Storage Name. + known: Configured VOSpace Services keyed by Storage Identifier. + found: Newly discovered VOSpace Services keyed by Storage Identifier. Returns: - dict[str, VOSpaceService]: Merged Services keyed by Storage Name. + dict[str, VOSpaceService]: Merged Services keyed by Storage Identifier. """ merged = dict(known) names = {str(service.uri): name for name, service in known.items()} diff --git a/canfar/storages.py b/canfar/storage.py similarity index 96% rename from canfar/storages.py rename to canfar/storage.py index 1c43baf5..188665b6 100644 --- a/canfar/storages.py +++ b/canfar/storage.py @@ -37,7 +37,7 @@ def _vospace( """Return a fresh authenticated async filesystem source. Args: - name: Storage Name of the configured VOSpace Service. + name: Storage Identifier of the configured VOSpace Service. token: Runtime bearer token, preferred over any saved credential. certificate: Runtime X.509 certificate path. @@ -114,11 +114,11 @@ async def _local() -> AsyncIterator[AbstractFileSystem]: def sources() -> dict[str, AsyncFilesystemSource]: """Build the mapped storage sources for one data command invocation. - Every configured VOSpace Service is mapped by its Storage Name, plus the + Every configured VOSpace Service is mapped by its Storage Identifier, plus the always-available ``local`` filesystem. Returns: - dict[str, AsyncFilesystemSource]: Sources keyed by Storage Name. + dict[str, AsyncFilesystemSource]: Sources keyed by Storage Identifier. """ config = Configuration() # ty: ignore[missing-argument] mapped = { diff --git a/docs/agents/research/2026-07-25-storage-python-api.md b/docs/agents/research/2026-07-25-storage-python-api.md index a558d473..6f9f8211 100644 --- a/docs/agents/research/2026-07-25-storage-python-api.md +++ b/docs/agents/research/2026-07-25-storage-python-api.md @@ -22,7 +22,7 @@ from canfar.storage import filesystem, fetch, sources ``` - `filesystem(name, *, cache=False, asynchronous=False, token=None, certificate=None)` - returns a ready `AbstractFileSystem` for one Storage Name. + returns a ready `AbstractFileSystem` for one Storage Identifier. - `fetch(name, path, *, cache=True)` materializes one remote object and returns its local `Path`. - `sources()` is the existing fsspec-cli seam, moved unchanged. @@ -46,7 +46,7 @@ in `docs/cli/data.md`: The sync/async tension resolves cleanly and is already half-solved in the repository: build `VOSpaceFileSystem` synchronously, wrap it in the sync cache, and re-expose it -with `AsyncFileSystemWrapper` — the same class `canfar/storages.py:108` already uses +with `AsyncFileSystemWrapper` — the same class `canfar/storage.py:108` already uses for `local`. This was verified end to end (§4, §9). `docs/cli/data.md:143-165` should be corrected regardless of whether this API ships. @@ -70,16 +70,16 @@ the two public read-only test objects | Element | Location | | --- | --- | -| `_vospace(name, *, token, certificate)` factory | `canfar/storages.py:31-98` | -| `_local()` async wrapper over `LocalFileSystem` | `canfar/storages.py:101-111` | -| `sources() -> dict[str, AsyncFilesystemSource]` | `canfar/storages.py:114-130` | +| `_vospace(name, *, token, certificate)` factory | `canfar/storage.py:31-98` | +| `_local()` async wrapper over `LocalFileSystem` | `canfar/storage.py:101-111` | +| `sources() -> dict[str, AsyncFilesystemSource]` | `canfar/storage.py:114-130` | | Only consumer | `canfar/cli/data.py:12,22-33` | | `AsyncFilesystemSource = Callable[[], AbstractAsyncContextManager[AbstractFileSystem]]` | `.venv/lib/python3.13/site-packages/fsspec_cli/_app.py:41-43` | -| Storage Name → endpoint + IDP | `canfar/models/config.py:382-395` | +| Storage Identifier → endpoint + IDP | `canfar/models/config.py:382-395` | | Credential materialization (async) | `canfar/client.py:234-262` | | Runtime dependencies | `pyproject.toml:48-65` | -DOCUMENTED. `canfar/storages.py` is not exported from `canfar/__init__.py:16-31`; its +DOCUMENTED. `canfar/storage.py` is not exported from `canfar/__init__.py:16-31`; its only non-test importer is `canfar/cli/data.py:12`. `docs/cli/data.md:176` states the release "intentionally provides no public CANFAR storage Python API". The module is therefore free to be renamed. @@ -164,7 +164,7 @@ behaves identically for network cost: 1.40 s for the same header read. ## 3. Import surface: `canfar.storage`, singular -**Recommendation: rename `canfar/storages.py` to `canfar/storage.py` and make it the +**Recommendation: rename `canfar/storage.py` to `canfar/storage.py` and make it the public module.** Rationale, all DOCUMENTED: @@ -179,7 +179,7 @@ Rationale, all DOCUMENTED: storages` references in `tests/test_cli_data.py`. No published API breaks, because `docs/cli/data.md:176` says none exists. -Do **not** add a compatibility shim re-exporting `canfar.storages`. There is nothing +Do **not** add a compatibility shim re-exporting `canfar.storage`. There is nothing to be compatible with. ### What belongs in it — and what does not @@ -204,7 +204,7 @@ _strip_protocol('vos://cadc-west-01.canfar.net/vault/ALMA/…/test-4d-cube-cutou The authority is silently swallowed into the path. A `canfar` resolver that emitted `vos://` URLs would produce strings that look addressable and are not. In Python the -Storage Name and the path are already two arguments; a resolver adds a failure mode +Storage Identifier and the path are already two arguments; a resolver adds a failure mode and removes nothing. The CLI's `identifier:/path` operand syntax (`docs/cli/data.md:17-24`) is a *command-line* affordance and should stay there. @@ -246,7 +246,7 @@ wrapped = AsyncFileSystemWrapper(cached, asynchronous=True) DOCUMENTED. `fsspec/implementations/asyn_wrapper.py:11-36` wraps each sync method with `asyncio.to_thread`, which is why the loop stays responsive. Its class declaration is -`fsspec/implementations/asyn_wrapper.py:39`. `canfar/storages.py:108` already uses this +`fsspec/implementations/asyn_wrapper.py:39`. `canfar/storage.py:108` already uses this class for `local`, so it is not a new concept in this codebase. fsspec's own docs call it "experimental" and warn "Users should not expect this wrapper to magically make things faster" () — the @@ -290,13 +290,13 @@ VERIFIED. Per-call latency against `vault`: | worst first-call outlier observed across all runs | 14.3 s | The current `sources()` design builds and closes a filesystem per command -(`canfar/storages.py:93-96`), which is right for a CLI. A Python API should hand back a +(`canfar/storage.py:93-96`), which is right for a CLI. A Python API should hand back a *reusable* instance so callers pay service-binding discovery once. Note the outliers: first-call cost is not deterministic, so do not document a fixed number. DOCUMENTED. `vosfs/filesystem.py:73-75` sets `protocol = "vos"` and `cachable = True`. VERIFIED: `fsspec.filesystem("vos", endpoint_url=…, certfile=…)` twice returns the -identical object; `skip_instance_cache=True` defeats it. `canfar/storages.py:76,88` +identical object; `skip_instance_cache=True` defeats it. `canfar/storage.py:76,88` already passes `skip_instance_cache=True`. **Keep that** for the returned instance: fsspec's instance cache is keyed on the storage options, and a cached instance would outlive a token refresh. @@ -446,13 +446,13 @@ VERIFIED sketch (ran correctly on this host, falling back to the temp dir becaus ```python def _cache_location(name: str) -> Path: - """Return the default cache directory for one Storage Name. + """Return the default cache directory for one Storage Identifier. Prefers the CANFAR Science Platform Server's per-Session ``/scratch`` volume, which is fast local disk and is cleared when the Session ends. Args: - name: Storage Name of the configured VOSpace Service. + name: Storage Identifier of the configured VOSpace Service. Returns: Path: Writable directory for cached objects. @@ -462,7 +462,7 @@ def _cache_location(name: str) -> Path: return root / "canfar" / name ``` -Keying by Storage Name matters: `fsspec`'s cache mapper hashes the *stripped* path +Keying by Storage Identifier matters: `fsspec`'s cache mapper hashes the *stripped* path (`fsspec/implementations/cached.py:627`), which does not include the endpoint, so `arc:/x` and `vault:/x` would otherwise collide in one directory. @@ -516,7 +516,7 @@ def fetch(name: str, path: str, *, cache: bool | str | Path = True) -> Path: """Materialize one VOSpace object locally and return its path. Args: - name: Storage Name of the configured VOSpace Service. + name: Storage Identifier of the configured VOSpace Service. path: Absolute path within that VOSpace Service. cache: Cache location, or ``True`` for the default (see ``filesystem``). @@ -557,7 +557,7 @@ Reasons: `Callable[[], AbstractAsyncContextManager[AbstractFileSystem]]` (`fsspec_cli/_app.py:41-43`), it must yield an async filesystem, and it must build and `aclose()` per command so a listing cache cannot outlive its command -(`canfar/storages.py:93-96`, `docs/cli/data.md:106-110`). +(`canfar/storage.py:93-96`, `docs/cli/data.md:106-110`). Refactor `_vospace` to delegate to the new `filesystem()` rather than duplicating construction: @@ -619,10 +619,10 @@ def filesystem( token: str | SecretStr | None = None, certificate: Path | None = None, ) -> AbstractFileSystem: - """Return an authenticated filesystem for one Storage Name. + """Return an authenticated filesystem for one Storage Identifier. Args: - name: Storage Name of the configured VOSpace Service. + name: Storage Identifier of the configured VOSpace Service. cache: ``False`` for no caching, ``True`` for the default cache directory, a path for an explicit one, or a sequence of paths tried in order where only the last is writable. @@ -651,7 +651,7 @@ def fetch( def sources() -> dict[str, AsyncFilesystemSource]: """Build the mapped storage sources for one data command invocation.""" - # unchanged from canfar/storages.py:114-130 + # unchanged from canfar/storage.py:114-130 ``` Construction body, matching the rule table in §4: @@ -815,7 +815,7 @@ which does group multiple ranges of one object into a single download ## Primary sources -- `canfar/storages.py`, `canfar/cli/data.py`, `canfar/client.py`, +- `canfar/storage.py`, `canfar/cli/data.py`, `canfar/client.py`, `canfar/models/config.py`, `pyproject.toml`, `uv.lock`, `CONTEXT.md`, `AGENTS.md`, `docs/cli/data.md` (repository snapshot `ba5493fd`) - `.venv/lib/python3.13/site-packages/vosfs/filesystem.py`, `.../vosfs/staging.py`, diff --git a/docs/cli/cli-help.md b/docs/cli/cli-help.md index 67d2bafe..0f3f93db 100644 --- a/docs/cli/cli-help.md +++ b/docs/cli/cli-help.md @@ -191,7 +191,7 @@ canfar data ls -lh arc:/home/[username] canfar data cp local:/absolute/path/file.fits arc:/home/[username]/file.fits ``` -Data operands use explicit `identifier:/absolute/path` or +Data operands use explicit `storage-identifier:/absolute/path` or `local:/absolute/path` syntax. See [Data commands](data.md) for recursive copy, cross-source copy and verification, recursive-removal policy, and the exact supported boundary. diff --git a/docs/cli/data.md b/docs/cli/data.md index 61aa0ddf..7d83b7a0 100644 --- a/docs/cli/data.md +++ b/docs/cli/data.md @@ -14,12 +14,12 @@ canfar login cadc ## Address mapped sources -Every operand starts with an `identifier`: the Storage Name of a configured +Every operand starts with a Storage Identifier: the handle of a configured VOSpace Service, or the reserved `local` identifier for the machine where the command runs. Operands always pair an identifier with an absolute path: ```text -identifier:/absolute/path +storage-identifier:/absolute/path local:/absolute/path ``` diff --git a/docs/cli/quick-start.md b/docs/cli/quick-start.md index efc4c192..d5658afb 100644 --- a/docs/cli/quick-start.md +++ b/docs/cli/quick-start.md @@ -29,7 +29,7 @@ canfar server ls ## 3. Work with data -The standard installation includes data commands. Use a configured Storage Name +The standard installation includes data commands. Use a configured Storage Identifier or the reserved `local` name with an absolute path. A default CADC login maps `arc` and `vault`: diff --git a/tests/test_cli_data.py b/tests/test_cli_data.py index 1002b437..c0f29698 100644 --- a/tests/test_cli_data.py +++ b/tests/test_cli_data.py @@ -16,7 +16,7 @@ from typer.testing import CliRunner import canfar.cli.data as data_cli -from canfar import storages +from canfar import storage from canfar.cli.main import cli if TYPE_CHECKING: @@ -85,8 +85,8 @@ def __init__( captured_capabilities.append(capabilities) self.typer_app = typer.Typer() - monkeypatch.setattr(storages, "Configuration", configuration) - monkeypatch.setattr(storages, "_vospace", source_factory) + monkeypatch.setattr(storage, "Configuration", configuration) + monkeypatch.setattr(storage, "_vospace", source_factory) monkeypatch.setattr(data_cli, "App", FakeApp) data_cli.group() @@ -97,7 +97,7 @@ def __init__( {"arc", "cavern", "vault", "local"}, ] assert all( - sources["local"] is storages._local # noqa: SLF001 + sources["local"] is storage._local # noqa: SLF001 for sources in captured_sources ) assert captured_capabilities == [ @@ -109,9 +109,9 @@ def __init__( @pytest.mark.asyncio async def test_local_storage_returns_fresh_async_wrappers() -> None: """Each local source entry owns a fresh asynchronous wrapper.""" - async with storages._local() as first: # noqa: SLF001 + async with storage._local() as first: # noqa: SLF001 assert first.asynchronous is True - async with storages._local() as second: # noqa: SLF001 + async with storage._local() as second: # noqa: SLF001 assert second.asynchronous is True assert first is not second @@ -125,7 +125,7 @@ def test_data_cat_delegates_without_active_server_banner( """Data stdout belongs byte-for-byte to the upstream command.""" payload = tmp_path / "payload.txt" payload.write_text("upstream output\n", encoding="utf-8") - monkeypatch.setattr(storages, "Configuration", _configuration) + monkeypatch.setattr(storage, "Configuration", _configuration) result = runner.invoke(cli, ["data", "cat", f"local:{payload}"]) @@ -140,14 +140,14 @@ def test_data_long_listing_delegates_unchanged( """The documented ``ls -lh name:/path`` form reaches upstream unchanged.""" (tmp_path / "payload.txt").write_text("listed", encoding="utf-8") monkeypatch.setattr( - storages, + storage, "Configuration", partial(_configuration, "canSRC"), ) monkeypatch.setattr( - storages, + storage, "_vospace", - lambda _name: storages._local, # noqa: SLF001 + lambda _name: storage._local, # noqa: SLF001 ) result = runner.invoke(cli, ["data", "ls", "-lh", f"canSRC:{tmp_path}"]) @@ -166,14 +166,14 @@ def test_data_recursive_copy_delegates_to_released_contract( (source / "payload.txt").write_text("copied", encoding="utf-8") destination = tmp_path / "destination" monkeypatch.setattr( - storages, + storage, "Configuration", partial(_configuration, "canSRC"), ) monkeypatch.setattr( - storages, + storage, "_vospace", - lambda _name: storages._local, # noqa: SLF001 + lambda _name: storage._local, # noqa: SLF001 ) result = runner.invoke( @@ -197,7 +197,7 @@ def test_data_recursive_remove_is_disabled( ``recursion.remove`` capability now omits the flag from ``rm`` entirely instead of registering it and rejecting the invocation at runtime. """ - monkeypatch.setattr(storages, "Configuration", _configuration) + monkeypatch.setattr(storage, "Configuration", _configuration) result = runner.invoke(cli, ["data", "rm", flag, f"local:{tmp_path}"]) @@ -209,7 +209,7 @@ def test_recursion_capability_omits_rm_flags_but_keeps_cp_flags( monkeypatch, ) -> None: """The recursion capability registers ``cp`` flags and withholds ``rm`` ones.""" - monkeypatch.setattr(storages, "Configuration", _configuration) + monkeypatch.setattr(storage, "Configuration", _configuration) group = data_cli.group() context = click.Context(group) @@ -231,8 +231,8 @@ async def unused_source() -> AsyncIterator[AbstractFileSystem]: raise AssertionError(msg) yield - monkeypatch.setattr(storages, "Configuration", partial(_configuration, "arc")) - monkeypatch.setattr(storages, "_vospace", lambda _name: unused_source) + monkeypatch.setattr(storage, "Configuration", partial(_configuration, "arc")) + monkeypatch.setattr(storage, "_vospace", lambda _name: unused_source) result = runner.invoke(cli, ["data", "mv", "arc:/a", "local:/b"]) @@ -247,7 +247,7 @@ def test_data_deprecated_operand_grammar_is_unsupported( operand: str, ) -> None: """Only the upstream explicit ``name:/absolute/path`` grammar is accepted.""" - monkeypatch.setattr(storages, "Configuration", _configuration) + monkeypatch.setattr(storage, "Configuration", _configuration) result = runner.invoke(cli, ["data", "ls", operand]) @@ -268,7 +268,7 @@ def test_data_retired_aliases_and_standalone_h_are_unsupported( diagnostic: str, ) -> None: """Retired host aliases do not widen the upstream mapped grammar.""" - monkeypatch.setattr(storages, "Configuration", _configuration) + monkeypatch.setattr(storage, "Configuration", _configuration) result = runner.invoke(cli, arguments) diff --git a/tests/test_data_smoke.py b/tests/test_data_smoke.py index 555a3760..69d2fe66 100644 --- a/tests/test_data_smoke.py +++ b/tests/test_data_smoke.py @@ -14,7 +14,7 @@ def _skip(reason: str) -> NoReturn: pytest.skip( - "live data smoke requires Storage Name 'arc' and a valid saved " + "live data smoke requires Storage Identifier 'arc' and a valid saved " f"Authentication Record/certificate: {reason}" ) diff --git a/tests/test_models_config.py b/tests/test_models_config.py index 786db3db..4bd5c046 100644 --- a/tests/test_models_config.py +++ b/tests/test_models_config.py @@ -85,7 +85,7 @@ def test_default_canfar_server_ships_arc_and_vault_storage( assert idp == "cadc" def test_legacy_server_named_storage_is_healed(self, tmp_path: Path) -> None: - """A Storage Name saved as the Server Name is restored to its leaf.""" + """A Storage Identifier saved as the Server Name is restored to its leaf.""" config_path = tmp_path / "config.yaml" config_path.write_text( "version: 1\n" @@ -110,7 +110,7 @@ def test_legacy_server_named_storage_is_healed(self, tmp_path: Path) -> None: assert str(storage["arc"].url).rstrip("/") == "https://ws-uv.canfar.net/arc" def test_custom_storage_names_are_not_healed(self, tmp_path: Path) -> None: - """Deliberate Storage Names are configuration, not stale defaults.""" + """Deliberate Storage Identifiers are configuration, not stale defaults.""" config_path = tmp_path / "config.yaml" config_path.write_text( "version: 1\n" @@ -221,10 +221,12 @@ def test_invalid_idp_key_rejected(self, idp_key: str) -> None: def test_invalid_storage_name_rejected( self, storage_name: str, tmp_path: Path ) -> None: - """Invalid Storage Names fail with the rejected name and constraints.""" + """Invalid Storage Identifiers fail with the rejected name and constraints.""" with ( patch("canfar.models.config.CONFIG_PATH", tmp_path / "config.yaml"), - pytest.raises(ValidationError, match="Invalid Storage Name") as exc_info, + pytest.raises( + ValidationError, match="Invalid Storage Identifier" + ) as exc_info, ): Configuration.model_validate( { @@ -244,7 +246,7 @@ def test_invalid_storage_name_rejected( assert repr(storage_name) in str(exc_info.value) def test_long_storage_name_allowed(self) -> None: - """Storage Names do not inherit Server field length limits.""" + """Storage Identifiers do not inherit Server field length limits.""" storage_name = "a" * 257 service = { "uri": "ivo://cadc.nrc.ca/arc", @@ -267,7 +269,7 @@ def test_storage_name_whitespace_is_trimmed(self) -> None: def test_duplicate_storage_name_across_servers_rejected( self, tmp_path: Path ) -> None: - """A Storage Name identifies one service across Science Platform Servers.""" + """A Storage Identifier names one service across Science Platform Servers.""" service = VOSpaceService( uri="ivo://cadc.nrc.ca/arc", url="https://ws-cadc.canfar.net/arc", @@ -278,7 +280,7 @@ def test_duplicate_storage_name_across_servers_rejected( pytest.raises( ValidationError, match=( - "Duplicate Storage Name 'shared' in Science Platform Servers " + "Duplicate Storage Identifier 'shared' in Science Platform Servers " "'canfar' and 'srcnet'" ), ), @@ -301,7 +303,7 @@ def test_normalized_storage_name_collision_rejected(self) -> None: with pytest.raises( ValidationError, - match="Storage Names 'arc' and ' arc ' both normalize to 'arc'", + match="Storage Identifiers 'arc' and ' arc ' both normalize to 'arc'", ): Server(storage={"arc": service, " arc ": service}) @@ -485,7 +487,7 @@ def test_complex_round_trip_serialization(self, tmp_path: Path) -> None: assert loaded_oidc.expiry.refresh is None def test_v1_storage_json_and_yaml_round_trip(self, tmp_path: Path) -> None: - """Multiple VOSpace Services retain stable nested Storage Names.""" + """Multiple VOSpace Services retain stable nested Storage Identifiers.""" config = Configuration.model_validate( _sample_config( servers={ diff --git a/tests/test_server.py b/tests/test_server.py index cbe7fb5b..e270aec6 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -393,7 +393,7 @@ def test_discover_refreshes_primary_storage_and_preserves_manual_entries( self, tmp_path: Path, ) -> None: - """Rediscovery updates only the generated Storage Name entry. + """Rediscovery updates only the generated Storage Identifier entry. A Server Name keyed entry saved by an older client is healed to its registry leaf (``arc``) on load, so rediscovery refreshes that entry in diff --git a/tests/test_storages.py b/tests/test_storage.py similarity index 99% rename from tests/test_storages.py rename to tests/test_storage.py index cec18359..53dc1798 100644 --- a/tests/test_storages.py +++ b/tests/test_storage.py @@ -14,7 +14,7 @@ from canfar.models.active import ActiveConfig from canfar.models.config import Configuration from canfar.models.http import Server, VOSpaceService -from canfar.storages import _vospace +from canfar.storage import _vospace from tests.helpers.config import oidc_credential, x509_credential if TYPE_CHECKING: From 14dadf8cff2523a5d0b820734781568279d54118 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Sat, 25 Jul 2026 01:57:20 -0700 Subject: [PATCH 44/47] build(deps): upgrade vosfs to v0.8.0 and fsspec-cli to v0.7.0 Upstream shinybrar/vosfs#335 teaches vosfs to send an HTTP Range header and use the response when the byte endpoint answers 206, closing the gap this project reported in shinybrar/vosfs#334. Verified against the live services: - vault (minoc) now serves real partial reads. A cat_file(path, 0, 2880) on a 3542400 byte cube sends `Range: bytes=0-2879` and receives `HTTP 206` with `Content-Range: bytes 0-2879/3542400`, transferring 2880 bytes. - MMapCache over a cat_file fetcher is genuinely partial: one ranged request, one block of four materialised, no whole-object fallback. On 0.7.0 the same operation downloaded the whole object per block. - arc (Cavern) serves no ranges and still falls back to a whole-object read that is sliced; slices remain correct, so nothing regresses there. - blockcache still refuses, because Range is honoured for byte reads and not through the file-object path: StagedReadFile has no blocksize attribute. Docs replace the "do not cache byte ranges" guidance with per-backend guidance, and the research note carries an update banner recording which of its 0.7.0 conclusions are superseded. --- .../research/2026-07-25-storage-python-api.md | 32 +++++++++- docs/cli/data.md | 58 ++++++++++++++----- pyproject.toml | 4 +- tests/test_data_dependencies.py | 4 +- uv.lock | 12 ++-- 5 files changed, 82 insertions(+), 28 deletions(-) diff --git a/docs/agents/research/2026-07-25-storage-python-api.md b/docs/agents/research/2026-07-25-storage-python-api.md index 6f9f8211..23304d20 100644 --- a/docs/agents/research/2026-07-25-storage-python-api.md +++ b/docs/agents/research/2026-07-25-storage-python-api.md @@ -3,11 +3,39 @@ - **Date:** 2026-07-25 - **Repository snapshot:** `ba5493fd` on `feat/vosfs-support` - **fsspec evaluated:** `2026.6.0` -- **vosfs evaluated:** `0.7.0` -- **fsspec-cli evaluated:** `0.6.0` +- **vosfs evaluated:** `0.7.0` (see the 0.8.0 update below) +- **fsspec-cli evaluated:** `0.6.0` (see the 0.7.0 update below) - **Python:** `3.13.5` - **Question:** How should `canfar` conveniently expose VOSpace storage for Python API access? +## Update, 2026-07-25 (later the same day): byte ranges now work on `vault` + +This note was written against `vosfs` 0.7.0. Upstream +[shinybrar/vosfs#334](https://github.com/shinybrar/vosfs/issues/334), filed from +this research, was fixed in +[#335](https://github.com/shinybrar/vosfs/issues/335) and released as +`vosfs` 0.8.0 / `fsspec-cli` 0.7.0, which `canfar` now pins. + +What changed, VERIFIED against the live service on 0.8.0: + +- `cat_file(path, start, end)` now sends `Range` and uses a `206` response. + Observed on `vault`: `Range: bytes=0-2879` -> `HTTP 206`, + `Content-Range: bytes 0-2879/3542400`, 2880 bytes returned. +- `MMapCache` over a `cat_file` fetcher is now genuinely partial: one ranged + request, one block of four materialised, zero whole-object fallbacks. Section + 4 below measured the opposite on 0.7.0 and is superseded for `vault`. +- `arc` (Cavern) still serves no ranges and falls back to a whole-object read + that is sliced. Slices remain correct (VERIFIED), so the "cache whole files" + guidance still holds there. +- `blockcache` / `CachingFileSystem` still fails with + `AttributeError: 'StagedReadFile' object has no attribute 'blocksize'`. + `Range` is honoured for byte reads, not through the file-object path. + +Sections 2 and 4 describe the 0.7.0 behaviour and are kept as the historical +record. The single-cache-layer recommendation in the Verdict stands, but its +stated reason ("the second tier is either dead or fictional") now applies only +to `arc`; on `vault` a block cache over `MMapCache` is a real option. + ## Verdict **Ship one module, `canfar/storage.py`, with three public functions and no new diff --git a/docs/cli/data.md b/docs/cli/data.md index 7d83b7a0..38b40c07 100644 --- a/docs/cli/data.md +++ b/docs/cli/data.md @@ -140,23 +140,49 @@ staleness metadata that `WholeFileCacheFileSystem` keeps. Passing `cache_storage` a list of directories tries each in order and treats only the last as writable, so a shared read-only cache can back your own. -### Do not cache byte ranges +### Cache byte ranges -Cache whole files, never blocks. `vosfs` sends no HTTP `Range` header, so a -partial read such as `cat_file(path, start, end)` downloads the whole object -and slices it in memory. A block cache therefore turns one download into one -download *per block*: reading three headers out of a cube fetches that cube -three times. +`vosfs` sends an HTTP `Range` header and uses the response when the byte +endpoint answers `206`, so a partial read such as `cat_file(path, start, end)` +transfers only the bytes you asked for. Range support is per-backend: -This is a client limitation, and it is not uniform across services. The `vault` -backend does serve ranges — a direct request returns `206` with -`Accept-Ranges: bytes` — while `arc` returns the whole body and advertises no -range support. So genuine byte-range reads are possible against `vault` today -only by bypassing `vosfs`, and teaching `vosfs` to send `Range` would unlock -them properly for `vault` but not for `arc`. +| Storage Identifier | Backend | Ranged reads | +| --- | --- | --- | +| `vault` | `minoc` | Yes — a partial read returns `206` and transfers only that slice | +| `arc` | Cavern | No — the whole object is fetched and sliced, which is correct but not cheaper | -Avoid `MMapCache` and any block-cache layer over a VOSpace Service until then. -`blockcache` refuses outright, which is the safer failure: +Because a range is now a real partial transfer against `vault`, a block cache +is worth using there. `MMapCache` keeps fetched blocks in a sparse file, so +only the blocks you touch occupy disk: + +```python +from fsspec.caching import MMapCache + +path = "/ALMA/test-data/cutouts/test-4d-cube.fits" +size = vault.info(path)["size"] +blocks = MMapCache( + blocksize=1 << 20, + fetcher=lambda start, end: vault.cat_file(path, start, end), + size=size, + location="/scratch/vault-cache/test-4d-cube.blocks", +) + +header = blocks._fetch(0, 2880) # one FITS header block, one 1 MiB range request +``` + +Reading a FITS header from a 3.4 MB cube this way issues a single ranged +request and materialises one block of four; a second read of the same range is +served from `/scratch`. The saving is in bytes transferred rather than seconds +on small files, because VOSpace transfer negotiation dominates a short request. +It grows with file size, and matters most when many reads hit different parts +of one large cube. + +Against `arc` a block cache still costs a whole download per block, so cache +whole files there instead. + +The `blockcache` filesystem remains unavailable over a VOSpace Service. `Range` +is honoured for byte reads, not through the file-object path, so wrapping +`CachingFileSystem` still fails: ```text AttributeError: 'StagedReadFile' object has no attribute 'blocksize' @@ -183,8 +209,8 @@ or cross-source `mv` workflow. ## Upstream releases CANFAR installs pinned, tagged releases of -[`vosfs`](https://github.com/shinybrar/vosfs/releases/tag/v0.7.0) and -[`fsspec-cli`](https://github.com/shinybrar/vosfs/releases/tag/fsspec-cli-v0.6.0). +[`vosfs`](https://github.com/shinybrar/vosfs/releases/tag/v0.8.0) and +[`fsspec-cli`](https://github.com/shinybrar/vosfs/releases/tag/fsspec-cli-v0.7.0). CANFAR tests its composition, configuration, authentication, and output seams; exhaustive filesystem-command and backend matrices remain in the upstream project. diff --git a/pyproject.toml b/pyproject.toml index 047c8f57..3dbf489a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,8 +59,8 @@ dependencies = [ "rich>=13.9.4", "segno>=1.6.6", "typer>=0.16.0", - "vosfs @ git+https://github.com/shinybrar/vosfs@v0.7.0", - "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.6.0#subdirectory=src/fsspec-cli", + "vosfs @ git+https://github.com/shinybrar/vosfs@v0.8.0", + "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.7.0#subdirectory=src/fsspec-cli", ] [tool.uv] diff --git a/tests/test_data_dependencies.py b/tests/test_data_dependencies.py index 40f8aa5c..ffaabbb0 100644 --- a/tests/test_data_dependencies.py +++ b/tests/test_data_dependencies.py @@ -15,11 +15,11 @@ def test_tagged_data_dependencies_are_standard_dependencies() -> None: metadata = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) assert ( - "vosfs @ git+https://github.com/shinybrar/vosfs@v0.7.0" + "vosfs @ git+https://github.com/shinybrar/vosfs@v0.8.0" in metadata["project"]["dependencies"] ) assert ( - "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.6.0" + "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.7.0" "#subdirectory=src/fsspec-cli" in metadata["project"]["dependencies"] ) assert "data" not in metadata["project"].get("optional-dependencies", {}) diff --git a/uv.lock b/uv.lock index 5d908c19..60e6648c 100644 --- a/uv.lock +++ b/uv.lock @@ -160,7 +160,7 @@ requires-dist = [ { name = "cadcutils", specifier = ">=1.5.4" }, { name = "click", specifier = ">=8.4.1" }, { name = "defusedxml", specifier = ">=0.7.1" }, - { name = "fsspec-cli", git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.6.0" }, + { name = "fsspec-cli", git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.7.0" }, { name = "httpx", extras = ["http2"], specifier = ">=0.28.1" }, { name = "humanize", specifier = ">=4.12.3" }, { name = "pydantic", specifier = ">=2.9.2" }, @@ -170,7 +170,7 @@ requires-dist = [ { name = "rich", specifier = ">=13.9.4" }, { name = "segno", specifier = ">=1.6.6" }, { name = "typer", specifier = ">=0.16.0" }, - { name = "vosfs", git = "https://github.com/shinybrar/vosfs?rev=v0.7.0" }, + { name = "vosfs", git = "https://github.com/shinybrar/vosfs?rev=v0.8.0" }, ] [package.metadata.requires-dev] @@ -677,8 +677,8 @@ wheels = [ [[package]] name = "fsspec-cli" -version = "0.6.0" -source = { git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.6.0#fa194eb18d35dbf2c5da67bc0f82ff085b9835ff" } +version = "0.7.0" +source = { git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.7.0#a2ef38d4014a10d5ddcf9a1f8eeaf682fde23ed8" } dependencies = [ { name = "fsspec" }, { name = "typer" }, @@ -2191,8 +2191,8 @@ wheels = [ [[package]] name = "vosfs" -version = "0.7.0" -source = { git = "https://github.com/shinybrar/vosfs?rev=v0.7.0#d4b8ccd1eee8b774bfb9a8f925b02db7dd09bbd9" } +version = "0.8.0" +source = { git = "https://github.com/shinybrar/vosfs?rev=v0.8.0#1e2e4a9835693d7894916f5ee8cdd721bed7f853" } dependencies = [ { name = "defusedxml" }, { name = "fsspec" }, From 39286e45391f8d32728bc0a35bac3836a90fa833 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Sat, 25 Jul 2026 02:26:13 -0700 Subject: [PATCH 45/47] docs(client): add a Data Access guide for the Python client Document how to reach a VOSpace Service from Python: resolve the endpoint and credential from configuration, then use the vosfs fsspec filesystem. Covers filesystem operations, materialising a local path, the three cache tiers (whole file, byte range, RAM), the per-backend range table, integration with astropy, numpy, pandas, and dask, and the async form. The page states plainly that no canfar.storage module exists yet and points at the CLI guide for the equivalent surface, so it documents what ships rather than a planned API. Every example was executed against the live services before being written: astropy reports NAXIS=4 and shape (1, 96, 26, 16) for the test cutout, the APASS table converts to a 2373-row DataFrame, and dask's lazy mean matches numpy's to the last digit. --- docs/client/data.md | 284 ++++++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 285 insertions(+) create mode 100644 docs/client/data.md diff --git a/docs/client/data.md b/docs/client/data.md new file mode 100644 index 00000000..be40a74b --- /dev/null +++ b/docs/client/data.md @@ -0,0 +1,284 @@ +# Data Access + +Read and write CANFAR VOSpace Services from Python. `canfar` resolves the +endpoints and credentials; [`vosfs`](https://github.com/shinybrar/vosfs) is the +[fsspec](https://filesystem-spec.readthedocs.io/) filesystem that talks to them, +so every tool that already speaks fsspec — astropy, pandas, dask, zarr — works +without an adapter. + +!!! note "There is no `canfar.storage` module yet" + + This release ships no public CANFAR storage API. The supported way to reach + a VOSpace Service from Python is to resolve it from your configuration and + hand the endpoint to `vosfs`, as below. The equivalent CLI surface is + documented in [Data commands](../cli/data.md). + +## Open a Storage Service + +A Storage Identifier (`arc`, `vault`, or the reserved `local`) names one +service. Resolve its endpoint and your saved credential from configuration +rather than hard-coding them: + +```python +from canfar.models.config import Configuration +from vosfs import VOSpaceFileSystem + +config = Configuration() +service = config.servers["canfar"].storage["vault"] +credential = config.get_credential("cadc") + +vault = VOSpaceFileSystem(str(service.url), certfile=str(credential.path)) +``` + +For a token Identity Provider pass `token=` instead of `certfile=`. Run +`canfar login` first; the credential is whatever that stored. + +Swap `"vault"` for `"arc"` to reach the other service. To see what is +configured: + +```python +for server in config.servers.values(): + for identifier, vospace in server.storage.items(): + print(identifier, vospace.url) +``` + +## Filesystem operations + +The object is a standard fsspec filesystem, so the usual verbs apply: + +```python +cutouts = "/ALMA/test-data/cutouts" +target = f"{cutouts}/test-4d-cube-cutout.fits" + +vault.ls(cutouts, detail=False) # ['/ALMA/.../test-4d-cube-cutout.fits', ...] +vault.info(target)["size"] # 169920 +vault.exists(target) # True +vault.isdir(cutouts) # True +vault.glob(f"{cutouts}/*cutout.fits") +vault.find(cutouts) # recursive listing +vault.du(cutouts) # 3712320 +``` + +Reads come in whole-object, ranged, and file-like forms: + +```python +whole = vault.cat_file(target) # 169920 bytes +header = vault.cat_file(target, 0, 2880) # first 2880 bytes only +first, second = vault.cat_ranges([target, target], [0, 100], [80, 180]) + +with vault.open(target, "rb") as handle: + handle.read(80) + +vault.head(target, 100) +vault.tail(target, 100) +``` + +Writes use `put_file`, `pipe_file`, `mkdir`, and `rm`. Directory listings are +cached in memory for the lifetime of the filesystem object, so a long-lived +object can serve a stale listing; build a fresh one, or pass +`use_listings_cache=False`, when you need to observe another writer's changes. + +## Get a local path + +Some libraries want a real path rather than a file object — anything that +memory-maps, or a C extension that opens by name. Materialise the file: + +```python +vault.get_file(target, "/scratch/cutout.fits") +``` + +## Cache + +Nothing is cached to disk unless you ask. On a CANFAR session `/scratch` is +fast local NVMe, is not backed up, and is cleared when the session ends, which +is exactly what a cache wants. Locally, any directory works. + +### Whole files + +```python +from fsspec.implementations.cached import WholeFileCacheFileSystem + +cached = WholeFileCacheFileSystem(fs=vault, cache_storage="/scratch/vault-cache") + +cached.cat_file(target) # cold: fetched over the network +cached.cat_file(target) # warm: served from /scratch +``` + +A cold read of the 166 KiB cutout took 2.96 s; the warm read took 0.4 ms. + +Use `SimpleCacheFileSystem` when you do not need the expiry and staleness +metadata `WholeFileCacheFileSystem` keeps. Passing `cache_storage` a list of +directories tries each in order and treats only the last as writable, so a +shared read-only cache can back your own. + +### Byte ranges + +`vosfs` sends an HTTP `Range` header and uses the response when the byte +endpoint answers `206`, so a partial read transfers only the bytes you asked +for. Support is per-backend: + +| Storage Identifier | Backend | Ranged reads | +| --- | --- | --- | +| `vault` | `minoc` | Yes — a partial read returns `206` and transfers only that slice | +| `arc` | Cavern | No — the whole object is fetched and sliced, correct but not cheaper | + +Against `vault`, cache blocks instead of whole files when you touch small parts +of a large cube. `MMapCache` keeps fetched blocks in a sparse file: + +```python +from fsspec.caching import MMapCache + +cube = "/ALMA/test-data/cutouts/test-4d-cube.fits" +size = vault.info(cube)["size"] + +blocks = MMapCache( + blocksize=1 << 20, + fetcher=lambda start, end: vault.cat_file(cube, start, end), + size=size, + location="/scratch/vault-cache/cube.blocks", +) + +header = blocks._fetch(0, 2880) # one ranged request, one block of four +``` + +Reading a FITS header from the 3.4 MB cube materialises one block of four. The +saving is in bytes transferred rather than seconds on small files, because +VOSpace transfer negotiation dominates a short request; it grows with file +size. Against `arc` a block cache still costs a whole download per block, so +cache whole files there. + +### RAM + +Omit `location` and `MMapCache` uses an anonymous memory map — a RAM cache that +never touches disk, for when `/scratch` is absent or you want the data gone +when the process exits: + +```python +blocks = MMapCache( + blocksize=1 << 20, + fetcher=lambda start, end: vault.cat_file(cube, start, end), + size=size, +) +``` + +A warm re-read of a cached range returned in 18 µs. + +### What does not work + +`blockcache` (`CachingFileSystem`) cannot wrap a VOSpace Service. `Range` is +honoured for byte reads, not through the file-object path: + +```text +AttributeError: 'StagedReadFile' object has no attribute 'blocksize' +``` + +Stacked caches do not help either: chaining them (`filecache::simplecache::`) +builds the layers, but the inner layer is never filled and never serves. Use +exactly one cache layer, on your fastest local disk. + +## Scientific tools + +### astropy + +Read a header without downloading the file, using one ranged request: + +```python +from astropy.io import fits + +raw = vault.cat_file(target, 0, 2880) +header = fits.Header.fromstring(raw.decode("latin-1")) +header["NAXIS"], header["OBJECT"] # 4, 'hers1' +``` + +Or hand the file object straight to astropy: + +```python +with vault.open(target, "rb") as handle, fits.open(handle) as hdul: + hdul[0].data.shape # (1, 96, 26, 16) +``` + +To memory-map, materialise the file first — `memmap=True` needs a real path: + +```python +vault.get_file(target, "/scratch/cutout.fits") + +with fits.open("/scratch/cutout.fits", memmap=True) as hdul: + data = hdul[0].data # paged in on demand, not loaded up front +``` + +### numpy + +```python +import numpy as np + +raw = vault.cat_file(target) +values = np.frombuffer(raw[2880:2880 + 64], dtype=">f4") +``` + +### pandas + +Astronomy tables usually arrive as FITS rather than CSV. Read one remotely and +convert: + +```python +from astropy.table import Table + +with vault.open("/APASS/north/091106/n091106.0101.cat", "rb") as handle: + table = Table.read(handle, format="fits") + +frame = table.to_pandas() # 2373 rows +frame.columns[:3] # ['NUMBER', 'MAG_AUTO', 'MAGERR_AUTO'] +``` + +For delimited text, pass the file object to pandas directly: + +```python +import pandas as pd + +with vault.open("/path/to/table.csv", "rb") as handle: + frame = pd.read_csv(handle) +``` + +### dask + +Memory-map a materialised cube and chunk it, so only the blocks a computation +touches are paged in: + +```python +import dask.array as da +from astropy.io import fits + +with fits.open("/scratch/cutout.fits", memmap=True) as hdul: + array = da.from_array(hdul[0].data, chunks=(1, 24, 26, 16)) + array.mean().compute() +``` + +## Async + +Every read has an async twin. Build the filesystem with `asynchronous=True`, +use the underscore-prefixed coroutines, and close it when done: + +```python +import asyncio +from vosfs import VOSpaceFileSystem + + +async def main() -> None: + vault = VOSpaceFileSystem( + str(service.url), + certfile=str(credential.path), + asynchronous=True, + ) + try: + entries = await vault._ls(cutouts, detail=False) + header = await vault._cat_file(cube, 0, 2880) + finally: + await vault.aclose() + + +asyncio.run(main()) +``` + +Async filesystems cannot open file objects — `open()` is rejected, and the +fsspec caches are synchronous — so use the synchronous form for caching and for +libraries that want a file handle. diff --git a/mkdocs.yml b/mkdocs.yml index e8391982..78e8f629 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -248,6 +248,7 @@ nav: - Examples: - Basic: client/examples.md - Advanced: client/advanced-examples.md + - Data Access: client/data.md - Reference: - Python API: - AsyncSession: client/async_session.md From 239861dcd5f93a2a7b37a4e610f2522d4c2b17b7 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Sat, 25 Jul 2026 02:39:51 -0700 Subject: [PATCH 46/47] feat(storage): expose Storage Identifiers as importable filesystems Add the public Python surface to canfar.storage so a Storage Identifier is importable by name: from canfar.storage import arc, vault, local A module-level __getattr__ resolves any configured Storage Identifier to a ready, authenticated synchronous filesystem, and __dir__ offers them for tab completion. Alongside it: identifiers() lists what is addressable, filesystem() builds one explicitly with an optional runtime credential, and fetch() copies one object to local disk and returns its Path. The existing sources() seam used by the data command is unchanged. Credentials are materialized through fsspec's background event loop, so the synchronous builder also works from inside a running loop. __getattr__ checks membership before building rather than catching KeyError from the build, so a failure to authenticate surfaces as itself instead of as a misleading "no attribute" error. Docs use the new imports, and the CLI guide no longer claims the release ships no public storage API. --- canfar/storage.py | 142 +++++++++++++++++- .../research/2026-07-25-storage-python-api.md | 5 + docs/cli/data.md | 11 +- docs/client/data.md | 59 +++++--- tests/test_storage.py | 59 +++++++- 5 files changed, 245 insertions(+), 31 deletions(-) diff --git a/canfar/storage.py b/canfar/storage.py index 188665b6..5311206c 100644 --- a/canfar/storage.py +++ b/canfar/storage.py @@ -3,8 +3,10 @@ from __future__ import annotations from contextlib import asynccontextmanager +from pathlib import Path from typing import TYPE_CHECKING, Any +from fsspec.asyn import get_loop, sync from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper from fsspec.implementations.local import LocalFileSystem @@ -14,13 +16,15 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator - from pathlib import Path from fsspec.spec import AbstractFileSystem from fsspec_cli import AsyncFilesystemSource from pydantic import SecretStr +LOCAL = "local" +"""Reserved Storage Identifier for the machine where the code runs.""" + _LISTINGS_EXPIRY_SECONDS = 30 """Seconds a cached directory listing stays valid within one command.""" @@ -126,5 +130,139 @@ def sources() -> dict[str, AsyncFilesystemSource]: for server in config.servers.values() for name in server.storage } - mapped["local"] = _local + mapped[LOCAL] = _local return mapped + + +def identifiers() -> list[str]: + """Return every Storage Identifier this configuration can address. + + Returns: + list[str]: Configured Storage Identifiers plus the reserved ``local``. + """ + config = Configuration() # ty: ignore[missing-argument] + names = {name for server in config.servers.values() for name in server.storage} + return [*sorted(names), LOCAL] + + +def filesystem( + identifier: str, + *, + token: str | SecretStr | None = None, + certificate: Path | str | None = None, +) -> AbstractFileSystem: + """Return a synchronous filesystem for one Storage Identifier. + + Args: + identifier: Storage Identifier, or ``local`` for the running machine. + token: Runtime bearer token, preferred over any saved credential. + certificate: Runtime X.509 certificate path. + + Returns: + AbstractFileSystem: A ready, authenticated filesystem. + + Raises: + KeyError: If ``identifier`` is not configured. + AuthContextError: If the saved credential cannot be used. + """ + if identifier == LOCAL: + return LocalFileSystem() + + config = Configuration() # ty: ignore[missing-argument] + endpoint, idp = config._resolve_storage(identifier) # noqa: SLF001 + try: + client_kwargs: dict[str, Any] = { + "config": config, + "authentication_idp": idp, + "url": endpoint, + } + if token is not None: + client_kwargs["token"] = token + if certificate is not None: + client_kwargs["certificate"] = certificate + client = HTTPClient(**client_kwargs) + # fsspec's background loop, so this works inside a running loop too. + token_value, certfile = sync( + get_loop(), + client._materialize_credentials, # noqa: SLF001 + ) + except (KeyError, OSError, TypeError, ValueError): + reason = "Credential cannot be used. Run 'canfar login' for this IDP." + raise AuthContextError(idp, reason) from None + + from vosfs import VOSpaceFileSystem # noqa: PLC0415 + + credential: dict[str, Any] = ( + {"token": token_value} if token_value is not None else {"certfile": certfile} + ) + return VOSpaceFileSystem( + endpoint, + **credential, + skip_instance_cache=True, + use_listings_cache=True, + listings_expiry_time=_LISTINGS_EXPIRY_SECONDS, + max_paths=_LISTINGS_MAX_PATHS, + ) + + +def fetch(identifier: str, path: str, destination: Path | str | None = None) -> Path: + """Copy one remote object to local disk and return its path. + + Args: + identifier: Storage Identifier holding ``path``. + path: Absolute path of the object within that Storage Service. + destination: Local path to write. Defaults to the object's name in the + current working directory. + + Returns: + Path: The local path now holding the object. + """ + target = Path(destination) if destination is not None else Path(path).name + target = Path(target) + if target.is_dir(): + target = target / Path(path).name + filesystem(identifier).get_file(path, str(target)) + return target + + +def __getattr__(name: str) -> AbstractFileSystem: + """Return a filesystem for a Storage Identifier accessed as an attribute. + + Makes ``from canfar.storage import vault`` resolve to a ready filesystem for + the ``vault`` Storage Identifier. + + Args: + name: Attribute name, treated as a Storage Identifier. + + Returns: + AbstractFileSystem: A ready, authenticated filesystem. + + Raises: + AttributeError: If ``name`` is not a configured Storage Identifier. + """ + if name.startswith("_"): + message = f"module {__name__!r} has no attribute {name!r}" + raise AttributeError(message) + known = identifiers() + if name not in known: + message = ( + f"module {__name__!r} has no attribute {name!r}; " + f"configured Storage Identifiers are: {', '.join(known)}" + ) + raise AttributeError(message) + # Build outside the membership check so a failure to authenticate surfaces + # as itself rather than as a missing attribute. + return filesystem(name) + + +def __dir__() -> list[str]: + """List the module's own names plus every Storage Identifier. + + Returns: + list[str]: Names available on this module, for tab completion. + """ + static = ["fetch", "filesystem", "identifiers", "sources", LOCAL] + try: + return sorted({*static, *identifiers()}) + except (OSError, ValueError): # pragma: no cover - unreadable configuration + return sorted(static) diff --git a/docs/agents/research/2026-07-25-storage-python-api.md b/docs/agents/research/2026-07-25-storage-python-api.md index 23304d20..38c4c4af 100644 --- a/docs/agents/research/2026-07-25-storage-python-api.md +++ b/docs/agents/research/2026-07-25-storage-python-api.md @@ -31,6 +31,11 @@ What changed, VERIFIED against the live service on 0.8.0: `AttributeError: 'StagedReadFile' object has no attribute 'blocksize'`. `Range` is honoured for byte reads, not through the file-object path. +The recommended module has since shipped as `canfar/storage.py`, adding +attribute access (`from canfar.storage import vault, arc`) on top of the +`filesystem` / `fetch` / `sources` surface proposed below, so any text quoting +the old "no public CANFAR storage Python API" wording is historical. + Sections 2 and 4 describe the 0.7.0 behaviour and are kept as the historical record. The single-cache-layer recommendation in the Verdict stands, but its stated reason ("the second tier is either dead or fictional") now applies only diff --git a/docs/cli/data.md b/docs/cli/data.md index 38b40c07..322af2bb 100644 --- a/docs/cli/data.md +++ b/docs/cli/data.md @@ -201,10 +201,13 @@ Data command stdout belongs to the embedded command; CANFAR does not prepend the active-Server banner or add JSON/YAML envelopes. Diagnostics are written to stderr. -This release intentionally provides no public CANFAR storage Python API, FUSE -mount, signed-URL extension, progress display, confirmation prompt, `:/path` or -bare-path shorthand, `active` alias, `canfar storage` alias, recursive removal, -or cross-source `mv` workflow. +The Python equivalent of these commands is documented in +[Data Access](../client/data.md). + +This release intentionally provides no FUSE mount, signed-URL extension, +progress display, confirmation prompt, `:/path` or bare-path shorthand, +`active` alias, `canfar storage` alias, recursive removal, or cross-source `mv` +workflow. ## Upstream releases diff --git a/docs/client/data.md b/docs/client/data.md index be40a74b..46a01c10 100644 --- a/docs/client/data.md +++ b/docs/client/data.md @@ -6,42 +6,39 @@ endpoints and credentials; [`vosfs`](https://github.com/shinybrar/vosfs) is the so every tool that already speaks fsspec — astropy, pandas, dask, zarr — works without an adapter. -!!! note "There is no `canfar.storage` module yet" - - This release ships no public CANFAR storage API. The supported way to reach - a VOSpace Service from Python is to resolve it from your configuration and - hand the endpoint to `vosfs`, as below. The equivalent CLI surface is - documented in [Data commands](../cli/data.md). +The same Storage Identifiers the CLI uses are importable by name. ## Open a Storage Service -A Storage Identifier (`arc`, `vault`, or the reserved `local`) names one -service. Resolve its endpoint and your saved credential from configuration -rather than hard-coding them: +Import a Storage Identifier and you get a ready, authenticated filesystem. Run +`canfar login` first; the credential resolution is the same one the CLI uses. ```python -from canfar.models.config import Configuration -from vosfs import VOSpaceFileSystem +from canfar.storage import arc, vault, local +``` -config = Configuration() -service = config.servers["canfar"].storage["vault"] -credential = config.get_credential("cadc") +Any configured Storage Identifier works this way. To see which are available: -vault = VOSpaceFileSystem(str(service.url), certfile=str(credential.path)) -``` +```python +from canfar.storage import identifiers -For a token Identity Provider pass `token=` instead of `certfile=`. Run -`canfar login` first; the credential is whatever that stored. +identifiers() # ['arc', 'vault', 'local'] +``` -Swap `"vault"` for `"arc"` to reach the other service. To see what is -configured: +Import binds the filesystem once, which is what you usually want. To build one +explicitly — to override a credential, or to name an identifier held in a +variable — use `filesystem`: ```python -for server in config.servers.values(): - for identifier, vospace in server.storage.items(): - print(identifier, vospace.url) +from canfar.storage import filesystem + +vault = filesystem("vault") +staging = filesystem("vault", token="...") # runtime bearer token +archive = filesystem("arc", certificate="/path/to/proxy.pem") ``` +`local` is reserved for the machine your code runs on and needs no credential. + ## Filesystem operations The object is a standard fsspec filesystem, so the usual verbs apply: @@ -83,6 +80,16 @@ object can serve a stale listing; build a fresh one, or pass Some libraries want a real path rather than a file object — anything that memory-maps, or a C extension that opens by name. Materialise the file: +```python +from canfar.storage import fetch + +path = fetch("vault", target, "/scratch/cutout.fits") +``` + +`fetch` returns the local `Path`. Omit the destination to write into the +current directory under the object's own name. The underlying filesystem +method works too: + ```python vault.get_file(target, "/scratch/cutout.fits") ``` @@ -260,13 +267,17 @@ use the underscore-prefixed coroutines, and close it when done: ```python import asyncio + +from canfar.models.config import Configuration from vosfs import VOSpaceFileSystem async def main() -> None: + config = Configuration() + service = config.servers["canfar"].storage["vault"] vault = VOSpaceFileSystem( str(service.url), - certfile=str(credential.path), + certfile=str(config.get_credential("cadc").path), asynchronous=True, ) try: diff --git a/tests/test_storage.py b/tests/test_storage.py index 53dc1798..c6b12688 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -8,8 +8,10 @@ import pytest import vosfs +from fsspec.implementations.local import LocalFileSystem from pydantic import AnyHttpUrl, AnyUrl +from canfar import storage from canfar.exceptions.context import AuthContextError from canfar.models.active import ActiveConfig from canfar.models.config import Configuration @@ -34,7 +36,7 @@ class _Filesystem: def __init__(self, endpoint: str, **kwargs: Any) -> None: self.endpoint = endpoint self.kwargs = kwargs - self.asynchronous = kwargs["asynchronous"] + self.asynchronous = kwargs.get("asynchronous", False) self.closed = False async def aclose(self) -> None: @@ -385,3 +387,58 @@ async def test_empty_saved_oidc_token_fails_cleanly( pass constructor.assert_not_called() + + +class TestPublicSurface: + """Tests for the public Storage Identifier accessors.""" + + def test_identifiers_lists_configured_services_and_local(self) -> None: + """Every configured Storage Identifier is listed, with local last.""" + _config(credential=x509_credential("inactive")).save() + + assert storage.identifiers() == ["archive", "local"] + + def test_local_identifier_returns_a_local_filesystem(self) -> None: + """The reserved local identifier needs no credential.""" + assert isinstance(storage.filesystem("local"), LocalFileSystem) + assert isinstance(storage.local, LocalFileSystem) + + def test_attribute_access_builds_a_filesystem( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """A Storage Identifier resolves as a module attribute.""" + certificate = tmp_path / "saved.pem" + _config(credential=x509_credential("inactive", path=certificate)).save() + monkeypatch.setattr( + "canfar.client.x509.inspect", + lambda path: {"path": path.as_posix(), "expiry": 9_999_999_999.0}, + ) + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + filesystem = storage.archive + + assert filesystem.endpoint == "https://inactive.example/vospace" + assert filesystem.kwargs["certfile"] == certificate.as_posix() + assert filesystem.kwargs["use_listings_cache"] is True + + def test_unknown_identifier_raises_attribute_error(self) -> None: + """An unconfigured name is an AttributeError naming what is available.""" + _config(credential=x509_credential("inactive")).save() + + with pytest.raises(AttributeError, match="archive"): + _ = storage.missing + + def test_private_names_are_not_treated_as_identifiers(self) -> None: + """Dunder lookups must not attempt a filesystem build.""" + with pytest.raises(AttributeError): + _ = storage.__wrapped__ + + def test_dir_offers_identifiers_for_completion(self) -> None: + """Tab completion exposes identifiers alongside the module functions.""" + _config(credential=x509_credential("inactive")).save() + + listed = dir(storage) + + assert {"archive", "local", "filesystem", "fetch", "identifiers"} <= set(listed) From 54a5fc659168dfcbcff3ddcb0b04b7a901c60443 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Sat, 25 Jul 2026 23:46:12 -0700 Subject: [PATCH 47/47] refactor(storage): address standards review across the vosfs integration Standards axis findings from the two-axis review, plus the Storage Identifier vocabulary sweep. Documented-standard violations: - Glossary: no `storage_name` identifier remains. `_resolve_storage(identifier)`, `server_by_identifier`, `_validate_storage_identifiers`, and the "Storage Service" docstring now use CONTEXT.md terms. - Home: `canfar/_discovery.py` moves to `canfar/utils/registry.py`, the location AGENTS.md and the architecture Module Map document, and no longer sits beside the near-homonymous `canfar/utils/discover.py`. - Models over dicts: `HTTPClient.build()` replaces the three hand-built `client_kwargs` dicts, still omitting unset credentials so settings sources keep filling them. Smells: - `RuntimeCredential` replaces `tuple[str | None, str | None]`, so the four consumers no longer re-switch on which half is None. - `_vospace` and `filesystem` share `_resolve` and `_build`; the synchronous path reuses the asynchronous resolution through fsspec's loop. - `Configuration.storage_identifiers()` owns the server/storage walk that `sources()` and `identifiers()` both duplicated. - `Workers` replaces the misleading `Enrichment`; `evidence`/`workers` no longer collide with the public `server.discover`/`enrich`. - `LOCAL` and `RESERVED_IDENTIFIERS` live once in `models/http.py`; the validator now rejects every identifier that would shadow the module surface. - `__all__` drives `__dir__`, so completion cannot drift. Also drops `fetch()` from the public surface in favour of fsspec's `get_file`, and gives `filesystem("local")` the `skip_instance_cache=True` every other construction site uses. Adds docs/agents/upstream-provenance.md recording that both distributions are pinned to tagged Git refs on a personal fork, with the pin history. --- canfar/client.py | 55 ++++- canfar/models/auth.py | 17 ++ canfar/models/config.py | 42 ++-- canfar/models/http.py | 21 +- canfar/server.py | 37 ++- canfar/storage.py | 243 +++++++++----------- canfar/{_discovery.py => utils/registry.py} | 35 +-- docs/agents/upstream-provenance.md | 30 +++ docs/client/data.md | 15 +- mkdocs.yml | 1 + tests/test_cli_data.py | 7 +- tests/test_server.py | 26 +-- tests/test_storage.py | 2 +- 13 files changed, 315 insertions(+), 216 deletions(-) rename canfar/{_discovery.py => utils/registry.py} (91%) create mode 100644 docs/agents/upstream-provenance.md diff --git a/canfar/client.py b/canfar/client.py index 0829156d..5ff1bd53 100644 --- a/canfar/client.py +++ b/canfar/client.py @@ -26,6 +26,7 @@ from canfar.models.auth import ( AuthenticationCredential, OIDCCredential, + RuntimeCredential, X509Credential, ) from canfar.models.config import Configuration @@ -231,21 +232,63 @@ def _resolved_authentication_record(self) -> AuthenticationCredential | None: ) return credential - async def _materialize_credentials(self) -> tuple[str | None, str | None]: - """Return one literal token or validated certificate path.""" + @classmethod + def build( + cls, + *, + config: Configuration, + authentication_idp: str, + url: Any, + token: Any = None, + certificate: Any = None, + **extra: Any, + ) -> HTTPClient: + """Build a client, omitting runtime credentials that were not supplied. + + Unset credentials are omitted rather than passed as ``None`` so the + settings sources stay free to supply them. + + Args: + config: Configuration backing the client. + authentication_idp: Canonical Identity Provider key. + url: Base URL for the client. + token: Runtime bearer token, when overriding the saved record. + certificate: Runtime X.509 certificate path, when overriding it. + **extra: Further client settings passed through unchanged. + + Returns: + HTTPClient: A client bound to the supplied credentials. + """ + supplied = {"token": token, "certificate": certificate} + return cls( + config=config, + authentication_idp=authentication_idp, + url=url, + **{key: value for key, value in supplied.items() if value is not None}, + **extra, + ) + + async def _materialize_credentials(self) -> RuntimeCredential: + """Return one literal token or validated certificate path. + + Returns: + RuntimeCredential: Exactly one of a bearer token or a certificate. + """ if self.token is not None: token = self.token.get_secret_value() if token: - return token, None + return RuntimeCredential(token=token) raise ValueError if self.certificate is not None: - return None, x509.valid(self.certificate) + return RuntimeCredential(certificate=x509.valid(self.certificate)) credential = self.authentication_record if isinstance(credential, X509Credential): if credential.path is None: raise ValueError - return None, str(x509.inspect(credential.path)["path"]) + return RuntimeCredential( + certificate=str(x509.inspect(credential.path)["path"]) + ) if not isinstance(credential, OIDCCredential): raise TypeError @@ -264,7 +307,7 @@ async def _materialize_credentials(self) -> tuple[str | None, str | None]: token = credential.token.access.get_secret_value() if not token: raise ValueError - return token, None + return RuntimeCredential(token=token) def _get_base_url(self) -> URL: """Get the base URL for the client. diff --git a/canfar/models/auth.py b/canfar/models/auth.py index 543a3e98..40a1de42 100644 --- a/canfar/models/auth.py +++ b/canfar/models/auth.py @@ -166,6 +166,23 @@ class DeviceAuthorization(BaseModel): ] +class RuntimeCredential(BaseModel): + """One materialized credential, ready to authenticate a request. + + Exactly one field is set: an Authentication Record resolves to either a + bearer token or a validated X.509 certificate path, never both. + + Attributes: + token: Bearer token value, when the record provides one. + certificate: Validated X.509 certificate path, when the record is X.509. + """ + + model_config = ConfigDict(frozen=True) + + token: str | None = None + certificate: str | None = None + + class X509Credential(BaseModel): """X.509 authentication credential decoupled from server selection.""" diff --git a/canfar/models/config.py b/canfar/models/config.py index 5bd5d01b..7b7ed71a 100644 --- a/canfar/models/config.py +++ b/canfar/models/config.py @@ -36,7 +36,7 @@ AuthenticationCredential, X509Credential, ) -from canfar.models.http import Server, VOSpaceService +from canfar.models.http import LOCAL, Server, VOSpaceService from canfar.models.registry import ContainerRegistry log = get_logger(__name__) @@ -195,6 +195,9 @@ def _heal_default_storage(self) -> None: an existing configuration holds ``canfar`` instead of ``arc`` and never gained later defaults such as ``vault``. Healing is scoped to the default Servers so federated Servers keep their Server Name keys. + + This normalizes the loaded Configuration in memory only; the file on + disk is rewritten when something independently calls ``save()``. """ for name, default in default_servers.items(): server = self.servers.get(name) @@ -208,8 +211,8 @@ def _heal_default_storage(self) -> None: if legacy is not None: leaf = str(legacy.uri).rpartition("/")[2] or name storage.setdefault(leaf, legacy) - for storage_name, service in default.storage.items(): - storage.setdefault(storage_name, service.model_copy(deep=True)) + for identifier, service in default.storage.items(): + storage.setdefault(identifier, service.model_copy(deep=True)) if storage != server.storage: self.servers[name] = server.model_copy( update={"storage": storage}, @@ -221,7 +224,7 @@ def _normalize_and_validate_servers(self) -> Configuration: """Inject Server Names and validate Server and Storage Identifier keys.""" self._heal_default_storage() updated: dict[str, Server] = {} - server_name_by_storage_name: dict[str, str] = {} + server_by_identifier: dict[str, str] = {} for name, server in self.servers.items(): if not _SERVER_NAME_PATTERN.match(name): msg = ( @@ -229,17 +232,15 @@ def _normalize_and_validate_servers(self) -> Configuration: r"^[A-Za-z][A-Za-z0-9_-]*$" ) raise ValueError(msg) - for storage_name in server.storage: - if previous_server_name := server_name_by_storage_name.get( - storage_name - ): + for identifier in server.storage: + if previous_server_name := server_by_identifier.get(identifier): msg = ( - f"Duplicate Storage Identifier '{storage_name}' in " + f"Duplicate Storage Identifier '{identifier}' in " "Science Platform " f"Servers '{previous_server_name}' and '{name}'." ) raise ValueError(msg) - server_name_by_storage_name[storage_name] = name + server_by_identifier[identifier] = name updated[name] = server.model_copy(update={"name": name}, deep=True) self.servers = updated return self @@ -380,20 +381,33 @@ def _get_server_by_name(self, name: str) -> Server: raise KeyError(msg) return self.servers[name] - def _resolve_storage(self, storage_name: str) -> tuple[str, str]: + def storage_identifiers(self) -> list[str]: + """Return every addressable Storage Identifier, ``local`` last. + + Returns: + list[str]: Configured Storage Identifiers plus reserved ``local``. + """ + configured = { + identifier + for server in self.servers.values() + for identifier in server.storage + } + return [*sorted(configured), LOCAL] + + def _resolve_storage(self, identifier: str) -> tuple[str, str]: """Resolve a Storage Identifier to its endpoint and parent server IDP.""" for server in self.servers.values(): - service = server.storage.get(storage_name) + service = server.storage.get(identifier) if service is not None: if server.idp is None: msg = ( - f"Storage Identifier '{storage_name}' belongs to a " + f"Storage Identifier '{identifier}' belongs to a " "Science Platform " "Server without an IDP." ) raise ValueError(msg) return str(service.url), server.idp - msg = f"Storage Identifier '{storage_name}' is not configured." + msg = f"Storage Identifier '{identifier}' is not configured." raise KeyError(msg) def upsert_credential(self, credential: AuthenticationCredential) -> None: diff --git a/canfar/models/http.py b/canfar/models/http.py index 8666a774..3a08b71f 100644 --- a/canfar/models/http.py +++ b/canfar/models/http.py @@ -16,6 +16,15 @@ """Default GPU count when context enrichment is unavailable.""" +LOCAL = "local" +"""Reserved Storage Identifier for the machine where the code runs.""" + +RESERVED_IDENTIFIERS = frozenset( + {LOCAL, "filesystem", "identifiers", "sources"}, +) +"""Storage Identifiers that would shadow the ``canfar.storage`` module surface.""" + + class VOSpaceService(BaseModel): """VOSpace Service discovered through an IVOA registry.""" @@ -127,7 +136,7 @@ class Server(BaseModel): @field_validator("storage", mode="before") @classmethod - def _validate_storage_names(cls, value: Any) -> Any: + def _validate_storage_identifiers(cls, value: Any) -> Any: """Normalize and validate Storage Identifiers before key transforms.""" if not isinstance(value, dict): return value @@ -141,11 +150,15 @@ def _validate_storage_names(cls, value: Any) -> Any: name = None else: name = original_name.strip() - if name is None or (not name or name == "local" or name.startswith("-")): + if name is None or ( + not name or name in RESERVED_IDENTIFIERS or name.startswith("-") + ): + reserved = ", ".join(sorted(RESERVED_IDENTIFIERS)) msg = ( f"Invalid Storage Identifier {original_name!r}: after whitespace " - "normalization it must be non-empty, differ from reserved 'local', " - "contain no colon, NUL, or newline, and not start with '-'." + f"normalization it must be non-empty, avoid the reserved names " + f"({reserved}), contain no colon, NUL, or newline, and not start " + "with '-'." ) raise ValueError(msg) if name in normalized: diff --git a/canfar/server.py b/canfar/server.py index da162486..ac77c592 100644 --- a/canfar/server.py +++ b/canfar/server.py @@ -3,15 +3,14 @@ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Literal from xml.etree.ElementTree import ParseError import httpx from defusedxml.common import DefusedXmlException from pydantic import AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, ValidationError -from canfar import _discovery, get_logger -from canfar._discovery import RegistryEvidenceError +from canfar import get_logger from canfar.auth.x509 import CertificateError from canfar.errors import ErrorCode, StructuredError from canfar.exceptions.context import AuthContextError, AuthExpiredError @@ -26,7 +25,8 @@ VOSpaceService, ) from canfar.models.registry import Server as RegistryResource -from canfar.utils import vosi +from canfar.utils import registry, vosi +from canfar.utils.registry import RegistryEvidenceError if TYPE_CHECKING: from pathlib import Path @@ -438,7 +438,7 @@ async def _discover_for_idp( Raises: ServerDiscoveryError: If registry retrieval fails. """ - evidence = await _discovery.discover( + evidence = await registry.evidence( idp, dev=dev, timeout=timeout, @@ -462,7 +462,7 @@ async def _discover_for_idp( for resource in evidence.resources if resource.uri.endswith(f"/{evidence.leaf}") ] - workers = await _discovery.enrich( + workers = await registry.workers( config, idp, endpoint=endpoints[0], @@ -513,7 +513,7 @@ def _select_storage( ) -> RegistryResource | None: """Map private registry ambiguity to the public server fetch error.""" try: - return _discovery.select_storage(endpoint, resources, strict=strict) + return registry.select_storage(endpoint, resources, strict=strict) except RegistryEvidenceError as exc: raise ServerFetchError(str(exc)) from exc @@ -527,7 +527,7 @@ async def _discover_storage( ) -> RegistryResource | None: """Return fresh registry evidence for a server's primary VOSpace service.""" try: - return await _discovery.discover_storage( + return await registry.discover_storage( str(server.uri) if server.uri is not None else None, str(server.url) if server.url is not None else None, server.name, @@ -822,18 +822,15 @@ def _fetch_capabilities( """Fetch one VOSI capabilities document through the existing HTTP seam.""" from canfar.client import HTTPClient # noqa: PLC0415 - client_kwargs: dict[str, Any] = { - "config": config, - "authentication_idp": authentication_idp, - "url": url, - "timeout": timeout, - "raise_http_errors": False, - } - if token is not None: - client_kwargs["token"] = token - if certificate is not None: - client_kwargs["certificate"] = certificate - with HTTPClient(**client_kwargs) as client: + with HTTPClient.build( + config=config, + authentication_idp=authentication_idp, + url=url, + token=token, + certificate=certificate, + timeout=timeout, + raise_http_errors=False, + ) as client: request_client = client.client request_client.headers["Accept"] = "application/xml" request_client.headers.pop("Content-Type", None) diff --git a/canfar/storage.py b/canfar/storage.py index 5311206c..c41095a4 100644 --- a/canfar/storage.py +++ b/canfar/storage.py @@ -3,8 +3,7 @@ from __future__ import annotations from contextlib import asynccontextmanager -from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from fsspec.asyn import get_loop, sync from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper @@ -13,35 +12,114 @@ from canfar.client import HTTPClient from canfar.exceptions.context import AuthContextError from canfar.models.config import Configuration +from canfar.models.http import LOCAL if TYPE_CHECKING: from collections.abc import AsyncIterator + from pathlib import Path from fsspec.spec import AbstractFileSystem from fsspec_cli import AsyncFilesystemSource from pydantic import SecretStr + from vosfs import VOSpaceFileSystem + from canfar.models.auth import RuntimeCredential -LOCAL = "local" -"""Reserved Storage Identifier for the machine where the code runs.""" +__all__ = ["LOCAL", "filesystem", "identifiers", "sources"] +"""Public surface; Storage Identifiers resolve through ``__getattr__``.""" _LISTINGS_EXPIRY_SECONDS = 30 -"""Seconds a cached directory listing stays valid within one command.""" +"""Seconds a cached directory listing stays valid on one filesystem.""" _LISTINGS_MAX_PATHS = 1000 """Maximum directory listings retained by one filesystem.""" +async def _resolve( + identifier: str, + token: str | SecretStr | None = None, + certificate: Path | str | None = None, +) -> tuple[str, RuntimeCredential]: + """Resolve one Storage Identifier to its endpoint and a usable credential. + + Args: + identifier: Storage Identifier of the configured VOSpace Service. + token: Runtime bearer token, preferred over any saved credential. + certificate: Runtime X.509 certificate path. + + Returns: + tuple[str, RuntimeCredential]: The endpoint and its credential. + + Raises: + AuthContextError: If the credential cannot be materialized. + """ + config = Configuration() # ty: ignore[missing-argument] + endpoint, idp = config._resolve_storage(identifier) # noqa: SLF001 + try: + client = HTTPClient.build( + config=config, + authentication_idp=idp, + url=endpoint, + token=token, + certificate=certificate, + ) + credential = await client._materialize_credentials() # noqa: SLF001 + except (KeyError, OSError, TypeError, ValueError): + reason = "Credential cannot be used. Run 'canfar login' for this IDP." + raise AuthContextError(idp, reason) from None + return endpoint, credential + + +def _build( + endpoint: str, + credential: RuntimeCredential, + *, + asynchronous: bool, +) -> VOSpaceFileSystem: + """Construct one VOSpace filesystem for an endpoint and credential. + + Args: + endpoint: URL of the VOSpace Service. + credential: Materialized token or certificate for that Service. + asynchronous: Build the filesystem in fsspec's asynchronous mode. + + Returns: + VOSpaceFileSystem: A ready, authenticated VOSpace filesystem. + """ + from vosfs import VOSpaceFileSystem as _VOSpaceFileSystem # noqa: PLC0415 + + # Passed explicitly rather than unpacked so the credential kwarg stays typed. + if credential.token is not None: + return _VOSpaceFileSystem( + endpoint, + token=credential.token, + asynchronous=asynchronous, + skip_instance_cache=True, + use_listings_cache=True, + listings_expiry_time=_LISTINGS_EXPIRY_SECONDS, + max_paths=_LISTINGS_MAX_PATHS, + ) + return _VOSpaceFileSystem( + endpoint, + certfile=credential.certificate, + asynchronous=asynchronous, + skip_instance_cache=True, + use_listings_cache=True, + listings_expiry_time=_LISTINGS_EXPIRY_SECONDS, + max_paths=_LISTINGS_MAX_PATHS, + ) + + def _vospace( - name: str, + identifier: str, *, token: str | SecretStr | None = None, - certificate: Path | None = None, + certificate: Path | str | None = None, ) -> AsyncFilesystemSource: """Return a fresh authenticated async filesystem source. Args: - name: Storage Identifier of the configured VOSpace Service. + identifier: Storage Identifier of the configured VOSpace Service. token: Runtime bearer token, preferred over any saved credential. certificate: Runtime X.509 certificate path. @@ -51,49 +129,8 @@ def _vospace( @asynccontextmanager async def source() -> AsyncIterator[AbstractFileSystem]: - config = Configuration() # ty: ignore[missing-argument] - endpoint, idp = config._resolve_storage(name) # noqa: SLF001 - try: - client_kwargs: dict[str, Any] = { - "config": config, - "authentication_idp": idp, - "url": endpoint, - } - if token is not None: - client_kwargs["token"] = token - if certificate is not None: - client_kwargs["certificate"] = certificate - client = HTTPClient( - **client_kwargs, - ) - token_value, certfile = await client._materialize_credentials() # noqa: SLF001 - except (KeyError, OSError, TypeError, ValueError): - reason = "Credential cannot be used. Run 'canfar login' for this IDP." - raise AuthContextError(idp, reason) from None - - from vosfs import VOSpaceFileSystem # noqa: PLC0415 - - if token_value is not None: - filesystem = VOSpaceFileSystem( - endpoint, - token=token_value, - asynchronous=True, - skip_instance_cache=True, - use_listings_cache=True, - listings_expiry_time=_LISTINGS_EXPIRY_SECONDS, - max_paths=_LISTINGS_MAX_PATHS, - ) - else: - assert certfile is not None - filesystem = VOSpaceFileSystem( - endpoint, - certfile=certfile, - asynchronous=True, - skip_instance_cache=True, - use_listings_cache=True, - listings_expiry_time=_LISTINGS_EXPIRY_SECONDS, - max_paths=_LISTINGS_MAX_PATHS, - ) + endpoint, credential = await _resolve(identifier, token, certificate) + filesystem = _build(endpoint, credential, asynchronous=True) try: yield filesystem finally: @@ -118,17 +155,17 @@ async def _local() -> AsyncIterator[AbstractFileSystem]: def sources() -> dict[str, AsyncFilesystemSource]: """Build the mapped storage sources for one data command invocation. - Every configured VOSpace Service is mapped by its Storage Identifier, plus the - always-available ``local`` filesystem. + Every configured VOSpace Service is mapped by its Storage Identifier, plus + the always-available ``local`` filesystem. Returns: dict[str, AsyncFilesystemSource]: Sources keyed by Storage Identifier. """ config = Configuration() # ty: ignore[missing-argument] - mapped = { - name: _vospace(name) - for server in config.servers.values() - for name in server.storage + mapped: dict[str, AsyncFilesystemSource] = { + identifier: _vospace(identifier) + for identifier in config.storage_identifiers() + if identifier != LOCAL } mapped[LOCAL] = _local return mapped @@ -141,8 +178,7 @@ def identifiers() -> list[str]: list[str]: Configured Storage Identifiers plus the reserved ``local``. """ config = Configuration() # ty: ignore[missing-argument] - names = {name for server in config.servers.values() for name in server.storage} - return [*sorted(names), LOCAL] + return config.storage_identifiers() def filesystem( @@ -166,93 +202,41 @@ def filesystem( AuthContextError: If the saved credential cannot be used. """ if identifier == LOCAL: - return LocalFileSystem() - - config = Configuration() # ty: ignore[missing-argument] - endpoint, idp = config._resolve_storage(identifier) # noqa: SLF001 - try: - client_kwargs: dict[str, Any] = { - "config": config, - "authentication_idp": idp, - "url": endpoint, - } - if token is not None: - client_kwargs["token"] = token - if certificate is not None: - client_kwargs["certificate"] = certificate - client = HTTPClient(**client_kwargs) - # fsspec's background loop, so this works inside a running loop too. - token_value, certfile = sync( - get_loop(), - client._materialize_credentials, # noqa: SLF001 - ) - except (KeyError, OSError, TypeError, ValueError): - reason = "Credential cannot be used. Run 'canfar login' for this IDP." - raise AuthContextError(idp, reason) from None - - from vosfs import VOSpaceFileSystem # noqa: PLC0415 - - credential: dict[str, Any] = ( - {"token": token_value} if token_value is not None else {"certfile": certfile} - ) - return VOSpaceFileSystem( - endpoint, - **credential, - skip_instance_cache=True, - use_listings_cache=True, - listings_expiry_time=_LISTINGS_EXPIRY_SECONDS, - max_paths=_LISTINGS_MAX_PATHS, - ) - - -def fetch(identifier: str, path: str, destination: Path | str | None = None) -> Path: - """Copy one remote object to local disk and return its path. - - Args: - identifier: Storage Identifier holding ``path``. - path: Absolute path of the object within that Storage Service. - destination: Local path to write. Defaults to the object's name in the - current working directory. - - Returns: - Path: The local path now holding the object. - """ - target = Path(destination) if destination is not None else Path(path).name - target = Path(target) - if target.is_dir(): - target = target / Path(path).name - filesystem(identifier).get_file(path, str(target)) - return target + return LocalFileSystem(skip_instance_cache=True) + # fsspec's background loop, so this works inside a running loop too. + endpoint, credential = sync(get_loop(), _resolve, identifier, token, certificate) + return _build(endpoint, credential, asynchronous=False) -def __getattr__(name: str) -> AbstractFileSystem: +def __getattr__(identifier: str) -> AbstractFileSystem: """Return a filesystem for a Storage Identifier accessed as an attribute. - Makes ``from canfar.storage import vault`` resolve to a ready filesystem for - the ``vault`` Storage Identifier. + Makes ``from canfar.storage import vault`` resolve to a ready filesystem + for the ``vault`` Storage Identifier. Args: - name: Attribute name, treated as a Storage Identifier. + identifier: Attribute name, treated as a Storage Identifier. Returns: AbstractFileSystem: A ready, authenticated filesystem. Raises: - AttributeError: If ``name`` is not a configured Storage Identifier. + AttributeError: If ``identifier`` is not a configured Storage + Identifier. """ - if name.startswith("_"): - message = f"module {__name__!r} has no attribute {name!r}" + if identifier.startswith("_"): + message = f"module {__name__!r} has no attribute {identifier!r}" raise AttributeError(message) known = identifiers() - if name not in known: + if identifier not in known: message = ( - f"module {__name__!r} has no attribute {name!r}; " + f"module {__name__!r} has no attribute {identifier!r}; " f"configured Storage Identifiers are: {', '.join(known)}" ) raise AttributeError(message) - # Build outside the membership check so a failure to authenticate surfaces + # Built outside the membership check so a failure to authenticate surfaces # as itself rather than as a missing attribute. - return filesystem(name) + return filesystem(identifier) def __dir__() -> list[str]: @@ -261,8 +245,7 @@ def __dir__() -> list[str]: Returns: list[str]: Names available on this module, for tab completion. """ - static = ["fetch", "filesystem", "identifiers", "sources", LOCAL] try: - return sorted({*static, *identifiers()}) + return sorted({*__all__, *identifiers()}) except (OSError, ValueError): # pragma: no cover - unreadable configuration - return sorted(static) + return sorted(__all__) diff --git a/canfar/_discovery.py b/canfar/utils/registry.py similarity index 91% rename from canfar/_discovery.py rename to canfar/utils/registry.py index c7450637..79f78a3a 100644 --- a/canfar/_discovery.py +++ b/canfar/utils/registry.py @@ -1,4 +1,4 @@ -"""Private registry evidence and discovery-worker preparation.""" +"""Registry evidence acquisition and discovery-worker preparation.""" from __future__ import annotations @@ -41,7 +41,7 @@ class RegistryEvidence(BaseModel): available: bool -class Enrichment(BaseModel): +class Workers(BaseModel): """Isolated worker configs with one pre-materialized runtime credential. Attributes: @@ -57,7 +57,7 @@ class Enrichment(BaseModel): certificate: Path | None = None -async def discover( +async def evidence( idp: str, *, dev: bool, @@ -191,14 +191,14 @@ async def discover_storage( message = "Server URI is required to inspect its VOSpace Service." raise RegistryEvidenceError(message) - evidence = await discover( + found = await evidence( idp, dev=dev, timeout=timeout, check_platforms=False, ) - if not evidence.available: - errors = "; ".join(evidence.errors) + if not found.available: + errors = "; ".join(found.errors) message = ( f"Failed to inspect VOSpace registry records for IDP '{idp}': {errors}" ) @@ -206,7 +206,7 @@ async def discover_storage( endpoints = [ resource - for resource in evidence.resources + for resource in found.resources if resource.uri == uri and resource.uri.endswith("/skaha") ] matching_urls = [ @@ -231,19 +231,19 @@ async def discover_storage( storage_resources = [ resource - for resource in evidence.resources - if resource.uri.endswith(f"/{evidence.leaf}") + for resource in found.resources + if resource.uri.endswith(f"/{found.leaf}") ] return select_storage(endpoint, storage_resources, strict=True) -async def enrich( +async def workers( config: Configuration | None, idp: str, *, endpoint: RegistryResource, count: int, -) -> Enrichment | None: +) -> Workers | None: """Materialize credentials once, then isolate worker configuration state. Args: @@ -253,13 +253,13 @@ async def enrich( count: Number of isolated worker configurations to produce. Returns: - Enrichment | None: Workers, or None when credentials are absent + Workers | None: Workers, or None when credentials are absent or unusable. """ from canfar.client import HTTPClient # noqa: PLC0415 base_config = config or Configuration() # ty: ignore[missing-argument] - client = HTTPClient( + client = HTTPClient.build( config=base_config, authentication_idp=idp, url=AnyHttpUrl(endpoint.url), @@ -268,7 +268,7 @@ async def enrich( certificate: Path | None = None if client.uses_runtime_credentials or client.authentication_record is not None: try: - token, certfile = await client._materialize_credentials() # noqa: SLF001 + credential = await client._materialize_credentials() # noqa: SLF001 except ( KeyError, OSError, @@ -280,8 +280,11 @@ async def enrich( ) as exc: log.debug("Skipping capability enrichment for IDP %s: %s", idp, exc) return None - certificate = Path(certfile) if certfile is not None else None + token = credential.token + certificate = ( + Path(credential.certificate) if credential.certificate is not None else None + ) values = base_config.model_dump(mode="python") configs = tuple(Configuration.model_validate(values) for _ in range(count)) - return Enrichment(configs=configs, token=token, certificate=certificate) + return Workers(configs=configs, token=token, certificate=certificate) diff --git a/docs/agents/upstream-provenance.md b/docs/agents/upstream-provenance.md new file mode 100644 index 00000000..ab6b20ce --- /dev/null +++ b/docs/agents/upstream-provenance.md @@ -0,0 +1,30 @@ +# Upstream provenance + +`canfar` depends on two upstream distributions that are **pinned to tagged Git +refs on a personal fork**, not to PyPI releases. Record the pin and its basis +here whenever it moves. + +## Current pins + +| Distribution | Pin | Source | +| --- | --- | --- | +| `vosfs` | `v0.8.0` | `git+https://github.com/shinybrar/vosfs@v0.8.0` | +| `fsspec-cli` | `fsspec-cli-v0.7.0` | same repository, `subdirectory=src/fsspec-cli` | + +## Why this matters + +- The source is a personal repository, so the usual PyPI ownership and + yanking guarantees do not apply. Review the tag before moving a pin. +- Tags are immutable by convention only. `uv.lock` records the resolved commit, + which is the real integrity anchor. +- `vosfs` v0.8.0 added HTTP `Range` support, honoured by the `vault` byte + endpoint and not by Cavern behind `arc`. Behaviour therefore differs per + Storage Identifier; see [Data Access](../client/data.md). + +## History + +- `v0.6.0` / `fsspec-cli-v0.5.0` — first integration, audited at upstream + PR #294, commit `9e5314db4706894d31d54d245392f43b9556cfbb`. +- `v0.7.0` / `fsspec-cli-v0.6.0` — Typer-owned commands, a breaking upstream + change to command parsing. +- `v0.8.0` / `fsspec-cli-v0.7.0` — server-side `Range` support (current). diff --git a/docs/client/data.md b/docs/client/data.md index 46a01c10..78c6377d 100644 --- a/docs/client/data.md +++ b/docs/client/data.md @@ -8,7 +8,7 @@ without an adapter. The same Storage Identifiers the CLI uses are importable by name. -## Open a Storage Service +## Open a VOSpace Service Import a Storage Identifier and you get a ready, authenticated filesystem. Run `canfar login` first; the credential resolution is the same one the CLI uses. @@ -80,20 +80,13 @@ object can serve a stale listing; build a fresh one, or pass Some libraries want a real path rather than a file object — anything that memory-maps, or a C extension that opens by name. Materialise the file: -```python -from canfar.storage import fetch - -path = fetch("vault", target, "/scratch/cutout.fits") -``` - -`fetch` returns the local `Path`. Omit the destination to write into the -current directory under the object's own name. The underlying filesystem -method works too: - ```python vault.get_file(target, "/scratch/cutout.fits") ``` +`get_file` is the standard fsspec verb; `put_file` is its counterpart for +uploads. + ## Cache Nothing is cached to disk unless you ask. On a CANFAR session `/scratch` is diff --git a/mkdocs.yml b/mkdocs.yml index 78e8f629..a7df070d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -279,6 +279,7 @@ nav: - Domain Docs: agents/domain.md - Issue Tracker: agents/issue-tracker.md - Triage Statuses: agents/triage-labels.md + - Upstream Provenance: agents/upstream-provenance.md - Operations: - Platform Deployments: https://www.opencadc.org/deployments/ - Security Issues: security.md diff --git a/tests/test_cli_data.py b/tests/test_cli_data.py index c0f29698..0ee1c871 100644 --- a/tests/test_cli_data.py +++ b/tests/test_cli_data.py @@ -31,7 +31,10 @@ def _configuration(*storage_names: str) -> SimpleNamespace: storage = dict.fromkeys(storage_names, object()) - return SimpleNamespace(servers={"server": SimpleNamespace(storage=storage)}) + return SimpleNamespace( + servers={"server": SimpleNamespace(storage=storage)}, + storage_identifiers=lambda: [*storage_names, "local"], + ) def test_root_help_advertises_data() -> None: @@ -58,6 +61,7 @@ def source_factory(_name: str) -> object: "first": SimpleNamespace(storage={"arc": object()}), "second": SimpleNamespace(storage={"cavern": object()}), }, + storage_identifiers=lambda: ["arc", "cavern", "local"], ), SimpleNamespace( active=SimpleNamespace(server="second"), @@ -67,6 +71,7 @@ def source_factory(_name: str) -> object: storage={"cavern": object(), "vault": object()} ), }, + storage_identifiers=lambda: ["arc", "cavern", "vault", "local"], ), ] ) diff --git a/tests/test_server.py b/tests/test_server.py index e270aec6..6d0caae9 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -14,7 +14,7 @@ from canfar.errors import ErrorCode from canfar.models.active import ActiveConfig -from canfar.models.auth import OIDCCredential, X509Credential +from canfar.models.auth import OIDCCredential, RuntimeCredential, X509Credential from canfar.models.config import Configuration, default_servers from canfar.models.http import Server, VOSpaceService from canfar.models.registry import IVOARegistry, IVOARegistrySearch @@ -566,7 +566,7 @@ def test_activation_freshly_inspects_storage_missing_during_discovery(self) -> N config = _anonymous_config() with ( - patch("canfar._discovery.Discover", return_value=mock_discovery), + patch("canfar.utils.registry.Discover", return_value=mock_discovery), patch( "canfar.client.Client", side_effect=_http_client_factory(transport), @@ -641,7 +641,7 @@ def enriched( ) with ( - patch("canfar._discovery.Discover", return_value=mock_discovery), + patch("canfar.utils.registry.Discover", return_value=mock_discovery), patch("canfar.server.enrich", side_effect=enriched), ): servers = await _discover_for_idp("srcnet") @@ -766,9 +766,9 @@ def convert( auths=["oidc"], ) - materialize = AsyncMock(return_value=("current-token", None)) + materialize = AsyncMock(return_value=RuntimeCredential(token="current-token")) with ( - patch("canfar._discovery.Discover", return_value=mock_discovery), + patch("canfar.utils.registry.Discover", return_value=mock_discovery), patch("canfar.server._discovered_to_server", side_effect=convert), patch( "canfar.client.HTTPClient._materialize_credentials", @@ -819,9 +819,9 @@ def response(request: httpx.Request) -> httpx.Response: requests.append(request) return httpx.Response(200, text=session_capabilities, request=request) - materialize = AsyncMock(return_value=("runtime-token", None)) + materialize = AsyncMock(return_value=RuntimeCredential(token="runtime-token")) with ( - patch("canfar._discovery.Discover", return_value=mock_discovery), + patch("canfar.utils.registry.Discover", return_value=mock_discovery), patch( "canfar.client.HTTPClient._materialize_credentials", new=materialize, @@ -879,7 +879,7 @@ def response(request: httpx.Request) -> httpx.Response: monkeypatch.delenv("CANFAR_CERTIFICATE", raising=False) monkeypatch.setenv("CANFAR_TOKEN", "environment-token") with ( - patch("canfar._discovery.Discover", return_value=mock_discovery), + patch("canfar.utils.registry.Discover", return_value=mock_discovery), patch( "canfar.client.Client", side_effect=_http_client_factory(httpx.MockTransport(response)), @@ -933,7 +933,7 @@ async def test_environment_certificate_materializes_without_saved_credential( monkeypatch.delenv("CANFAR_TOKEN", raising=False) monkeypatch.setenv("CANFAR_CERTIFICATE", certificate.as_posix()) with ( - patch("canfar._discovery.Discover", return_value=mock_discovery), + patch("canfar.utils.registry.Discover", return_value=mock_discovery), patch( "canfar.client.x509.inspect", return_value={ @@ -1214,7 +1214,7 @@ def test_discover_keys_unnamed_server_by_host_slug(self, tmp_path: Path) -> None with ( patch("canfar.models.config.CONFIG_PATH", config_path), - patch("canfar._discovery.Discover", return_value=mock_discovery), + patch("canfar.utils.registry.Discover", return_value=mock_discovery), patch( "canfar.server.enrich", side_effect=lambda item, **_kwargs: item.model_copy( @@ -1252,7 +1252,7 @@ async def test_discover_for_idp_converts_active_endpoints(self) -> None: mock_discovery.__aexit__ = AsyncMock(return_value=None) with ( - patch("canfar._discovery.Discover", return_value=mock_discovery), + patch("canfar.utils.registry.Discover", return_value=mock_discovery), patch( "canfar.server.enrich", side_effect=lambda item, **_kwargs: item.model_copy( @@ -1341,7 +1341,7 @@ async def test_discover_for_idp_raises_when_registry_fetch_fails(self) -> None: mock_discovery.__aexit__ = AsyncMock(return_value=None) with ( - patch("canfar._discovery.Discover", return_value=mock_discovery), + patch("canfar.utils.registry.Discover", return_value=mock_discovery), pytest.raises(ServerDiscoveryError, match="Failed to discover"), ): await _discover_for_idp("cadc") @@ -1359,7 +1359,7 @@ async def test_discover_for_idp_honors_dev_sources_and_timeout(self) -> None: mock_discovery.__aexit__ = AsyncMock(return_value=None) with patch( - "canfar._discovery.Discover", return_value=mock_discovery + "canfar.utils.registry.Discover", return_value=mock_discovery ) as factory: servers = await _discover_for_idp("cadc", dev=True, timeout=11) diff --git a/tests/test_storage.py b/tests/test_storage.py index c6b12688..e4e7a454 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -441,4 +441,4 @@ def test_dir_offers_identifiers_for_completion(self) -> None: listed = dir(storage) - assert {"archive", "local", "filesystem", "fetch", "identifiers"} <= set(listed) + assert {"archive", "local", "filesystem", "identifiers"} <= set(listed)