diff --git a/CONTEXT.md b/CONTEXT.md index 847238d1..8f4ba627 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -16,6 +16,14 @@ _Avoid_: Base URL, host, cluster Required, unique, user-facing handle for one **Science Platform Server**, used to reference it in configuration and commands. _Avoid_: Label, alias, key +**VOSpace Service**: +Remote astronomical storage service associated with a **Science Platform Server**. +_Avoid_: Storage backend, filesystem + +**Storage Identifier**: +Required, globally unique, user-facing handle for one **VOSpace Service**, used to address it in configuration, commands, and operands such as `arc:/home/user`. The reserved identifier `local` addresses the machine where the command runs. +_Avoid_: Storage Name, Server Name, alias, key + **Identity Provider (IDP)**: Organization that issues user identity for CANFAR authentication. Initial IDPs are `Canadian Astronomy Data Centre (CADC)` and `SKA Regional Centre Network (SRCNet)`. @@ -69,6 +77,8 @@ _Avoid_: Resource profile, quota - A **CANFAR Science Platform** exposes one or more **Science Platform Servers**. - A **Science Platform Server** is identified by its **Server Name**; its IVOA URI is discovery metadata, and two Server Names may point at the same endpoint. +- A **Science Platform Server** can expose multiple **VOSpace Services**. +- A **VOSpace Service** is identified by its **Storage Identifier** and uses its parent **Science Platform Server**'s **Identity Provider (IDP)**. - An **Identity Provider (IDP)** can support one or more **Science Platform Servers**. - **Authentication** and **Platform** are separate seams with independent ownership. - An **Authentication Record** belongs to one **Identity Provider (IDP)**. diff --git a/canfar/auth/oidc.py b/canfar/auth/oidc.py index 3679e46d..3c87ca72 100644 --- a/canfar/auth/oidc.py +++ b/canfar/auth/oidc.py @@ -7,7 +7,7 @@ import time from collections.abc import Awaitable, Callable, Generator from contextlib import contextmanager -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import httpx from authlib.integrations.base_client.errors import OAuthError @@ -20,6 +20,8 @@ if TYPE_CHECKING: from authlib.integrations.httpx_client import AsyncOAuth2Client + from canfar.models.config import Configuration + log = get_logger(__name__) _BASIC_AUTH_METHOD = "client_secret_basic" @@ -193,6 +195,67 @@ def _validated_refresh(refreshed: Any) -> dict[str, Any]: return dict(refreshed) +def _refresh( + credential: OIDCCredential, +) -> tuple[str, str, str, str] | None: + """Return complete literal refresh inputs when the record is eligible.""" + if not credential.refreshable: + return None + return ( + cast("str", credential.endpoints.token), + cast("str", credential.client.identity), + cast("SecretStr", credential.client.secret).get_secret_value(), + cast("SecretStr", credential.token.refresh).get_secret_value(), + ) + + +def _persist( + config: Configuration, + credential: OIDCCredential, + refreshed: dict[str, Any], +) -> OIDCCredential: + """Validate and atomically persist refreshed OIDC state.""" + previous_refresh = credential.token.refresh + previous_refresh_value = ( + previous_refresh.get_secret_value() if previous_refresh is not None else None + ) + returned_refresh = refreshed.get("refresh_token") + refresh_value = ( + returned_refresh + if isinstance(returned_refresh, str) and returned_refresh + else previous_refresh_value + ) + access_value = refreshed.get("access_token") + if not isinstance(access_value, str) or not access_value: + msg = "OIDC token refresh failed: malformed token response" + raise ValueError(msg) + try: + token = Token( + access=SecretStr(access_value), + refresh=SecretStr(refresh_value) if refresh_value is not None else None, + token_type=refreshed.get("token_type") or credential.token.token_type, + scope=refreshed.get("scope") or credential.token.scope, + ) + expiry = Expiry( + access=refreshed.get("expires_at"), + refresh=( + None + if refresh_value != previous_refresh_value + else credential.expiry.refresh + ), + ) + except (KeyError, TypeError, ValidationError): + msg = "OIDC token refresh failed: malformed token response" + raise ValueError(msg) from None + + updated = credential.model_copy(update={"token": token, "expiry": expiry}) + candidate = config.model_copy(deep=True) + candidate.update_credential(updated) + candidate.save() + config.update_credential(updated) + return updated + + async def refresh( url: str, identity: str, diff --git a/canfar/auth/x509.py b/canfar/auth/x509.py index 70875bdf..8a45637f 100644 --- a/canfar/auth/x509.py +++ b/canfar/auth/x509.py @@ -25,6 +25,16 @@ class CertificateError(ValueError): """Raised when an X.509 certificate cannot be used.""" + def __init__( + self, + message: str, + *, + expired_at: datetime | None = None, + ) -> None: + """Initialize a certificate error with optional structured expiry.""" + super().__init__(message) + self.expired_at = expired_at + def _to_utc(value: datetime) -> datetime: """Return timezone aware datetime. @@ -88,7 +98,7 @@ def assert_valid_dates( f"Certificate {destination} expired on {valid_until.isoformat()}; " f"current time {now_utc.isoformat()}." ) - raise CertificateError(msg) + raise CertificateError(msg, expired_at=valid_until) def gather( diff --git a/canfar/cli/data.py b/canfar/cli/data.py new file mode 100644 index 00000000..e3f3d3d9 --- /dev/null +++ b/canfar/cli/data.py @@ -0,0 +1,60 @@ +"""Mount the upstream data command application.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import typer +from fsspec_cli import App +from typer.core import TyperGroup +from typer.main import get_group + +from canfar.storage import sources + +if TYPE_CHECKING: + from typer._click.core import Command, Context + +_DATA_GROUP_META_KEY = "canfar.data_group" + + +def group() -> TyperGroup: + """Build the released upstream application with CANFAR policy. + + Returns: + TyperGroup: The upstream command group bound to configured sources. + """ + return get_group( + App( + sources(), + capabilities={"recursion": {"copy": True, "remove": False}}, + ).typer_app + ) + + +class _DataGroup(TyperGroup): + """Resolve the embedded app lazily so imports perform no configuration I/O.""" + + @staticmethod + def _delegate(ctx: Context) -> TyperGroup: + resolved = ctx.meta.get(_DATA_GROUP_META_KEY) + if resolved is None: + resolved = group() + ctx.meta[_DATA_GROUP_META_KEY] = resolved + assert isinstance(resolved, TyperGroup) + return resolved + + def list_commands(self, ctx: Context) -> list[str]: + """List the unchanged upstream commands.""" + return self._delegate(ctx).list_commands(ctx) + + def get_command(self, ctx: Context, cmd_name: str) -> Command | None: + """Resolve an unchanged upstream command.""" + return self._delegate(ctx).get_command(ctx, cmd_name) + + +data = typer.Typer( + cls=_DataGroup, + help="Operate on configured data sources.", + add_completion=False, + no_args_is_help=True, +) diff --git a/canfar/cli/login.py b/canfar/cli/login.py index 14f2d5f2..c1b47fa1 100644 --- a/canfar/cli/login.py +++ b/canfar/cli/login.py @@ -66,7 +66,7 @@ def _login_flow( idp_info = get_idp(idp) try: - credential = authenticate_for_cli(idp_info, timeout=timeout) + credential = authenticate_for_cli(idp_info, timeout=timeout, force=force) except (ValueError, RuntimeError) as exc: get_console(stderr=True).print(f"[bold red]{exc}[/bold red]") raise typer.Exit(1) from exc diff --git a/canfar/cli/login_auth.py b/canfar/cli/login_auth.py index 27ac8698..0f79f7a4 100644 --- a/canfar/cli/login_auth.py +++ b/canfar/cli/login_auth.py @@ -6,6 +6,7 @@ import webbrowser from typing import TYPE_CHECKING, Any +import humanize import segno from rich import progress as rich_progress @@ -98,12 +99,14 @@ def authenticate_for_cli( idp_info: IdpInfo, *, timeout: int | None = None, + force: bool = False, ) -> AuthenticationCredential: """Acquire Authentication credentials interactively for CLI login. Args: idp_info: Built-in Identity Provider metadata. timeout: HTTP timeout in seconds for OIDC requests. + force: Obtain a new X.509 certificate instead of reusing an existing one. Returns: Saved-ready authentication credential without embedded server. @@ -113,19 +116,38 @@ def authenticate_for_cli( RuntimeError: If OIDC discovery URL is missing for an OIDC IDP. """ if idp_info.auth_mode == "x509": - return _authenticate_x509(idp_info.key) + return _authenticate_x509(idp_info.key, force=force) return _authenticate_oidc(idp_info, timeout=timeout) -def _authenticate_x509(idp: str) -> X509Credential: - """Run interactive X509 certificate acquisition. +def _authenticate_x509(idp: str, *, force: bool = False) -> X509Credential: + """Reuse a valid X.509 certificate or acquire a replacement. Args: idp: Canonical Identity Provider key. + force: Obtain a new certificate without inspecting the existing one. Returns: X509 credential record for persisted configuration. """ + if not force: + try: + info = x509.inspect() + except x509.CertificateError as exc: + if exc.expired_at is not None: + age = humanize.naturaltime(exc.expired_at) + console_utils.get_console().print( + f"[yellow]x509 certificate expired {age}[/yellow]" + ) + except (FileNotFoundError, PermissionError, ValueError): + pass + else: + return X509Credential( + idp=idp, + path=info["path"], + expiry=float(info["expiry"] or 0.0), + ) + return x509.authenticate_credential(X509Credential(idp=idp, expiry=0.0)) diff --git a/canfar/cli/main.py b/canfar/cli/main.py index 7e41834d..a7c6f0c0 100644 --- a/canfar/cli/main.py +++ b/canfar/cli/main.py @@ -11,6 +11,7 @@ from canfar.cli.auth import auth from canfar.cli.config import config from canfar.cli.create import create +from canfar.cli.data import data from canfar.cli.delete import delete from canfar.cli.events import events from canfar.cli.image import image @@ -115,7 +116,8 @@ def warning_writer(error: StructuredError) -> None: if ctx.invoked_subcommand is None: get_console().print(ctx.get_help()) raise typer.Exit(0) - set_before_command(ctx, _emit_banner_for_command) + if ctx.invoked_subcommand != "data": + set_before_command(ctx, _emit_banner_for_command) cli: typer.Typer = typer.Typer( @@ -161,6 +163,12 @@ def warning_writer(error: StructuredError) -> None: rich_help_panel="Auth Management", ) +cli.add_typer( + data, + name="data", + rich_help_panel="Data Management", +) + cli.add_typer( create, no_args_is_help=True, diff --git a/canfar/client.py b/canfar/client.py index a6f63408..5ff1bd53 100644 --- a/canfar/client.py +++ b/canfar/client.py @@ -20,12 +20,13 @@ from typing_extensions import Self from canfar import __version__, get_logger -from canfar.auth import x509 +from canfar.auth import oidc, x509 from canfar.exceptions.context import AuthContextError from canfar.hooks.httpx import auth, debug, errors, expiry from canfar.models.auth import ( AuthenticationCredential, OIDCCredential, + RuntimeCredential, X509Credential, ) from canfar.models.config import Configuration @@ -231,6 +232,83 @@ def _resolved_authentication_record(self) -> AuthenticationCredential | None: ) return credential + @classmethod + def build( + cls, + *, + config: Configuration, + authentication_idp: str, + url: Any, + token: Any = None, + certificate: Any = None, + **extra: Any, + ) -> HTTPClient: + """Build a client, omitting runtime credentials that were not supplied. + + Unset credentials are omitted rather than passed as ``None`` so the + settings sources stay free to supply them. + + Args: + config: Configuration backing the client. + authentication_idp: Canonical Identity Provider key. + url: Base URL for the client. + token: Runtime bearer token, when overriding the saved record. + certificate: Runtime X.509 certificate path, when overriding it. + **extra: Further client settings passed through unchanged. + + Returns: + HTTPClient: A client bound to the supplied credentials. + """ + supplied = {"token": token, "certificate": certificate} + return cls( + config=config, + authentication_idp=authentication_idp, + url=url, + **{key: value for key, value in supplied.items() if value is not None}, + **extra, + ) + + async def _materialize_credentials(self) -> RuntimeCredential: + """Return one literal token or validated certificate path. + + Returns: + RuntimeCredential: Exactly one of a bearer token or a certificate. + """ + if self.token is not None: + token = self.token.get_secret_value() + if token: + return RuntimeCredential(token=token) + raise ValueError + if self.certificate is not None: + return RuntimeCredential(certificate=x509.valid(self.certificate)) + + credential = self.authentication_record + if isinstance(credential, X509Credential): + if credential.path is None: + raise ValueError + return RuntimeCredential( + certificate=str(x509.inspect(credential.path)["path"]) + ) + if not isinstance(credential, OIDCCredential): + raise TypeError + + if credential.expired: + parameters = oidc._refresh(credential) # noqa: SLF001 + if parameters is None: + raise ValueError + refreshed = await oidc.refresh(*parameters) + credential = oidc._persist( # noqa: SLF001 + self.config, + credential, + refreshed, + ) + if credential.token.access is None: + raise ValueError + token = credential.token.access.get_secret_value() + if not token: + raise ValueError + return RuntimeCredential(token=token) + def _get_base_url(self) -> URL: """Get the base URL for the client. diff --git a/canfar/hooks/httpx/auth.py b/canfar/hooks/httpx/auth.py index 391d5079..03e2e4f5 100644 --- a/canfar/hooks/httpx/auth.py +++ b/canfar/hooks/httpx/auth.py @@ -30,18 +30,17 @@ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING, Any, Callable, cast - -from pydantic import SecretStr, ValidationError +from typing import TYPE_CHECKING, Any, Callable from canfar import get_logger from canfar.auth import oidc -from canfar.models.auth import Expiry, OIDCCredential, Token +from canfar.models.auth import OIDCCredential if TYPE_CHECKING: from collections.abc import Awaitable, MutableMapping import httpx + from pydantic import SecretStr from canfar.client import HTTPClient @@ -71,24 +70,6 @@ def _apply_access_header( request.headers["Authorization"] = header -def _refresh_parameters( - credential: OIDCCredential, -) -> tuple[str, str, str, str] | None: - """Return complete refresh inputs when the record is eligible.""" - if not credential.refreshable: - return None - token_url = cast("str", credential.endpoints.token) - identity = cast("str", credential.client.identity) - client_secret = cast("SecretStr", credential.client.secret) - refresh_token = cast("SecretStr", credential.token.refresh) - return ( - token_url, - identity, - client_secret.get_secret_value(), - refresh_token.get_secret_value(), - ) - - def _apply_refreshed_token( client: HTTPClient, credential: OIDCCredential, @@ -97,51 +78,13 @@ def _apply_refreshed_token( request: httpx.Request, ) -> None: """Atomically persist refreshed OIDC state, then update active headers.""" - previous_refresh = credential.token.refresh - previous_refresh_value = ( - previous_refresh.get_secret_value() if previous_refresh is not None else None - ) - returned_refresh = refreshed.get("refresh_token") - refresh_value = ( - returned_refresh - if isinstance(returned_refresh, str) and returned_refresh - else previous_refresh_value + updated = oidc._persist( # noqa: SLF001 + client.config, credential, refreshed ) - rotated = refresh_value != previous_refresh_value - access_value = refreshed.get("access_token") - if not isinstance(access_value, str) or not access_value: - msg = "OIDC token refresh failed: malformed token response" - raise ValueError(msg) - access_token = SecretStr(access_value) - token_type = refreshed.get("token_type") - if token_type is None or token_type == "": - token_type = credential.token.token_type - scope = refreshed.get("scope") - if scope is None or scope == "": - scope = credential.token.scope - try: - token = Token( - access=access_token, - refresh=SecretStr(refresh_value) if refresh_value is not None else None, - token_type=token_type, - scope=scope, - ) - expiry = Expiry( - access=refreshed.get("expires_at"), - refresh=None if rotated else credential.expiry.refresh, - ) - except (KeyError, TypeError, ValidationError): - msg = "OIDC token refresh failed: malformed token response" - raise ValueError(msg) from None - - updated = credential.model_copy(update={"token": token, "expiry": expiry}) - candidate = client.config.model_copy(deep=True) - candidate.update_credential(updated) - candidate.save() - client.config.update_credential(updated) log.debug("Authentication refreshed and configuration saved.") - _apply_access_header(access_token, httpx_client_headers, request) + assert updated.token.access is not None + _apply_access_header(updated.token.access, httpx_client_headers, request) log.debug("HTTP request headers updated with new token.") log.info("OIDC Access Token Refreshed.") @@ -177,7 +120,7 @@ def hook(request: httpx.Request) -> None: log.debug("Skipping auth refresh, access token is not expired.") return - parameters = _refresh_parameters(credential) + parameters = oidc._refresh(credential) # noqa: SLF001 if parameters is None: log.warning("OIDC Authentication Record cannot be refreshed.") return @@ -239,7 +182,7 @@ async def ahook(request: httpx.Request) -> None: ) return - parameters = _refresh_parameters(credential) + parameters = oidc._refresh(credential) # noqa: SLF001 if parameters is None: log.warning("OIDC Authentication Record cannot be refreshed.") return diff --git a/canfar/idp.py b/canfar/idp.py index 25f37030..5ec39fa1 100644 --- a/canfar/idp.py +++ b/canfar/idp.py @@ -19,6 +19,7 @@ class IdpInfo(BaseModel): auth_mode: Default authentication mode for the IDP. registry_url: IVOA registry resource-caps URL for server discovery. registry_name: Registry display name used for server discovery. + leaf: Preferred primary VOSpace registry URI leaf. dev_registries: Development registry resource-caps URLs keyed by URL. oidc_discovery_url: OIDC discovery URL when ``auth_mode`` is ``oidc``. oidc_issuer: Exact issuer expected from OIDC discovery metadata. @@ -36,6 +37,10 @@ class IdpInfo(BaseModel): default=None, description="Registry display name used for server discovery.", ) + leaf: str | None = Field( + default=None, + description="Preferred primary VOSpace registry URI leaf.", + ) dev_registries: dict[str, str] = Field( default_factory=dict, description="Development registry resource-caps URLs keyed by URL.", @@ -56,6 +61,7 @@ class IdpInfo(BaseModel): name="Canadian Astronomy Data Centre", auth_mode="x509", registry_name="CADC", + leaf="arc", registry_url=AnyHttpUrl( "https://ws.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/reg/resource-caps" ), @@ -70,6 +76,7 @@ class IdpInfo(BaseModel): name="SKA Regional Centre Network", auth_mode="oidc", registry_name="SRCNet", + leaf="cavern", registry_url=AnyHttpUrl("https://spsrc27.iaa.csic.es/reg/resource-caps"), oidc_discovery_url=AnyHttpUrl( "https://ska-iam.stfc.ac.uk/.well-known/openid-configuration" diff --git a/canfar/models/auth.py b/canfar/models/auth.py index 543a3e98..40a1de42 100644 --- a/canfar/models/auth.py +++ b/canfar/models/auth.py @@ -166,6 +166,23 @@ class DeviceAuthorization(BaseModel): ] +class RuntimeCredential(BaseModel): + """One materialized credential, ready to authenticate a request. + + Exactly one field is set: an Authentication Record resolves to either a + bearer token or a validated X.509 certificate path, never both. + + Attributes: + token: Bearer token value, when the record provides one. + certificate: Validated X.509 certificate path, when the record is X.509. + """ + + model_config = ConfigDict(frozen=True) + + token: str | None = None + certificate: str | None = None + + class X509Credential(BaseModel): """X.509 authentication credential decoupled from server selection.""" diff --git a/canfar/models/config.py b/canfar/models/config.py index f5473217..7b7ed71a 100644 --- a/canfar/models/config.py +++ b/canfar/models/config.py @@ -36,7 +36,7 @@ AuthenticationCredential, X509Credential, ) -from canfar.models.http import Server +from canfar.models.http import LOCAL, Server, VOSpaceService from canfar.models.registry import ContainerRegistry log = get_logger(__name__) @@ -65,6 +65,16 @@ url=AnyHttpUrl("https://ws-uv.canfar.net/skaha"), version="v1", auths=["x509"], + storage={ + "arc": VOSpaceService( + uri=AnyUrl("ivo://cadc.nrc.ca/arc"), + url=AnyHttpUrl("https://ws-uv.canfar.net/arc"), + ), + "vault": VOSpaceService( + uri=AnyUrl("ivo://cadc.nrc.ca/vault"), + url=AnyHttpUrl("https://cadc-west-01.canfar.net/vault"), + ), + }, ), } @@ -178,10 +188,43 @@ def settings_customise_sources( file_secret_settings, ) + def _heal_default_storage(self) -> None: + """Restore default Storage Identifiers on Servers saved by an older client. + + Older clients keyed a discovered VOSpace Service by its Server Name, so + an existing configuration holds ``canfar`` instead of ``arc`` and never + gained later defaults such as ``vault``. Healing is scoped to the + default Servers so federated Servers keep their Server Name keys. + + This normalizes the loaded Configuration in memory only; the file on + disk is rewritten when something independently calls ``save()``. + """ + for name, default in default_servers.items(): + server = self.servers.get(name) + if server is None or not default.storage or server.idp != default.idp: + continue + storage = dict(server.storage) + legacy = storage.pop(name, None) + if legacy is None and storage: + # Deliberate Storage Identifiers are configuration, not stale defaults. + continue + if legacy is not None: + leaf = str(legacy.uri).rpartition("/")[2] or name + storage.setdefault(leaf, legacy) + for identifier, service in default.storage.items(): + storage.setdefault(identifier, service.model_copy(deep=True)) + if storage != server.storage: + self.servers[name] = server.model_copy( + update={"storage": storage}, + deep=True, + ) + @model_validator(mode="after") - def _inject_server_names(self) -> Configuration: - """Inject dict keys into each server record and validate Server Names.""" + def _normalize_and_validate_servers(self) -> Configuration: + """Inject Server Names and validate Server and Storage Identifier keys.""" + self._heal_default_storage() updated: dict[str, Server] = {} + server_by_identifier: dict[str, str] = {} for name, server in self.servers.items(): if not _SERVER_NAME_PATTERN.match(name): msg = ( @@ -189,6 +232,15 @@ def _inject_server_names(self) -> Configuration: r"^[A-Za-z][A-Za-z0-9_-]*$" ) raise ValueError(msg) + for identifier in server.storage: + if previous_server_name := server_by_identifier.get(identifier): + msg = ( + f"Duplicate Storage Identifier '{identifier}' in " + "Science Platform " + f"Servers '{previous_server_name}' and '{name}'." + ) + raise ValueError(msg) + server_by_identifier[identifier] = name updated[name] = server.model_copy(update={"name": name}, deep=True) self.servers = updated return self @@ -329,6 +381,35 @@ def _get_server_by_name(self, name: str) -> Server: raise KeyError(msg) return self.servers[name] + def storage_identifiers(self) -> list[str]: + """Return every addressable Storage Identifier, ``local`` last. + + Returns: + list[str]: Configured Storage Identifiers plus reserved ``local``. + """ + configured = { + identifier + for server in self.servers.values() + for identifier in server.storage + } + return [*sorted(configured), LOCAL] + + def _resolve_storage(self, identifier: str) -> tuple[str, str]: + """Resolve a Storage Identifier to its endpoint and parent server IDP.""" + for server in self.servers.values(): + service = server.storage.get(identifier) + if service is not None: + if server.idp is None: + msg = ( + f"Storage Identifier '{identifier}' belongs to a " + "Science Platform " + "Server without an IDP." + ) + raise ValueError(msg) + return str(service.url), server.idp + msg = f"Storage Identifier '{identifier}' is not configured." + raise KeyError(msg) + def upsert_credential(self, credential: AuthenticationCredential) -> None: """Insert or replace a validated Authentication Record. diff --git a/canfar/models/http.py b/canfar/models/http.py index dd09dea3..3a08b71f 100644 --- a/canfar/models/http.py +++ b/canfar/models/http.py @@ -2,7 +2,9 @@ from __future__ import annotations -from pydantic import AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, Field +from typing import Annotated, Any + +from pydantic import AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, Field, field_validator DEFAULT_SERVER_CORES = 2 """Default CPU core limit when context enrichment is unavailable.""" @@ -14,6 +16,33 @@ """Default GPU count when context enrichment is unavailable.""" +LOCAL = "local" +"""Reserved Storage Identifier for the machine where the code runs.""" + +RESERVED_IDENTIFIERS = frozenset( + {LOCAL, "filesystem", "identifiers", "sources"}, +) +"""Storage Identifiers that would shadow the ``canfar.storage`` module surface.""" + + +class VOSpaceService(BaseModel): + """VOSpace Service discovered through an IVOA registry.""" + + model_config = ConfigDict(extra="forbid") + + uri: AnyUrl + url: AnyHttpUrl + + @field_validator("url") + @classmethod + def _reject_capabilities_endpoint(cls, url: AnyHttpUrl) -> AnyHttpUrl: + """Require the VOSpace base endpoint rather than its capabilities URL.""" + if (url.path or "").rstrip("/").endswith("/capabilities"): + msg = "VOSpace Service base URL must not end with /capabilities." + raise ValueError(msg) + return url + + class Server(BaseModel): """Science Platform Server Details.""" @@ -22,7 +51,6 @@ class Server(BaseModel): extra="forbid", json_schema_mode_override="serialization", str_strip_whitespace=True, - str_max_length=256, str_min_length=1, ) @@ -59,7 +87,7 @@ class Server(BaseModel): min_length=2, max_length=8, ) - auths: list[str] | None = Field( + auths: list[Annotated[str, Field(max_length=256)]] | None = Field( default=None, title="Supported Auth Modes", description="Authentication modes supported by the Server", @@ -72,6 +100,12 @@ class Server(BaseModel): min_length=1, max_length=64, ) + storage: dict[str, VOSpaceService] = Field( + default_factory=dict, + title="VOSpace Services", + description="VOSpace Services keyed by globally unique Storage Identifier.", + ) + cores: int = Field( default=DEFAULT_SERVER_CORES, title="Default CPU Core Limit", @@ -97,4 +131,44 @@ class Server(BaseModel): "Persisted compatibility field for discovery reachability status " "when known." ), + max_length=256, ) + + @field_validator("storage", mode="before") + @classmethod + def _validate_storage_identifiers(cls, value: Any) -> Any: + """Normalize and validate Storage Identifiers before key transforms.""" + if not isinstance(value, dict): + return value + + normalized: dict[str, Any] = {} + original_name_by_normalized_name: dict[str, str] = {} + for original_name, service in value.items(): + if not isinstance(original_name, str) or any( + character in original_name for character in ":\x00\r\n" + ): + name = None + else: + name = original_name.strip() + if name is None or ( + not name or name in RESERVED_IDENTIFIERS or name.startswith("-") + ): + reserved = ", ".join(sorted(RESERVED_IDENTIFIERS)) + msg = ( + f"Invalid Storage Identifier {original_name!r}: after whitespace " + f"normalization it must be non-empty, avoid the reserved names " + f"({reserved}), contain no colon, NUL, or newline, and not start " + "with '-'." + ) + raise ValueError(msg) + if name in normalized: + previous_original_name = original_name_by_normalized_name[name] + msg = ( + f"Storage Identifiers {previous_original_name!r} and " + f"{original_name!r} " + f"both normalize to {name!r}; use unique names." + ) + raise ValueError(msg) + normalized[name] = service + original_name_by_normalized_name[name] = original_name + return normalized diff --git a/canfar/models/registry.py b/canfar/models/registry.py index e0797480..ba33aea4 100644 --- a/canfar/models/registry.py +++ b/canfar/models/registry.py @@ -25,6 +25,8 @@ class IVOARegistrySearch(BaseModel): } ) + leaf: str | None = None + names: dict[str, str] = Field( default={ "ivo://canfar.net/src/skaha": "canSRC", @@ -67,6 +69,8 @@ class IVOARegistry(BaseModel): name: str content: str + source: str | None = None + development: bool = False success: bool = True error: str | None = None @@ -75,6 +79,7 @@ class Server(BaseModel): """Model to store Canfar Server endpoint information.""" registry: str + development: bool = False uri: str url: str status: int | None = None diff --git a/canfar/server.py b/canfar/server.py index d788110c..ac77c592 100644 --- a/canfar/server.py +++ b/canfar/server.py @@ -3,35 +3,37 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass from typing import TYPE_CHECKING, Literal from xml.etree.ElementTree import ParseError import httpx from defusedxml.common import DefusedXmlException -from pydantic import AnyHttpUrl, AnyUrl, ValidationError +from pydantic import AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, ValidationError from canfar import get_logger from canfar.auth.x509 import CertificateError from canfar.errors import ErrorCode, StructuredError from canfar.exceptions.context import AuthContextError, AuthExpiredError from canfar.hooks.httpx.auth import AuthenticationError as HTTPAuthenticationError -from canfar.idp import registry_sources +from canfar.idp import get_idp from canfar.models.config import Configuration from canfar.models.http import ( DEFAULT_SERVER_CORES, DEFAULT_SERVER_GPUS, DEFAULT_SERVER_RAM_GB, Server, + VOSpaceService, ) -from canfar.models.registry import IVOARegistrySearch -from canfar.utils import vosi -from canfar.utils.discover import Discover +from canfar.models.registry import Server as RegistryResource +from canfar.utils import registry, vosi +from canfar.utils.registry import RegistryEvidenceError + +if TYPE_CHECKING: + from pathlib import Path log = get_logger(__name__) -if TYPE_CHECKING: - from canfar.models.registry import Server as DiscoveredServer +_STORAGE_RESOURCE_UNSET = object() class ServerSelectorError(ValueError): @@ -79,14 +81,44 @@ def __init__(self, idp: str, servers: list[Server]) -> None: self.servers = servers -@dataclass(frozen=True) -class ServerActivation: - """Result from activating an Authentication and Server pair.""" +class ServerActivation(BaseModel): + """Result from activating an Authentication and Server pair. + + Attributes: + server: The activated Science Platform Server. + reason: Why this Server was chosen. + """ + + model_config = ConfigDict(frozen=True) server: Server reason: Literal["active", "remembered", "single", "selected"] +def _merge_storage( + known: dict[str, VOSpaceService], + found: dict[str, VOSpaceService], +) -> dict[str, VOSpaceService]: + """Merge discovered VOSpace Services into known ones, keyed by IVOA URI. + + Discovery names a new Service after its Server Name, so an existing Service + configured under its registry leaf (``arc``) is refreshed in place instead + of being duplicated under the Server Name on every rediscovery. + + Args: + known: Configured VOSpace Services keyed by Storage Identifier. + found: Newly discovered VOSpace Services keyed by Storage Identifier. + + Returns: + dict[str, VOSpaceService]: Merged Services keyed by Storage Identifier. + """ + merged = dict(known) + names = {str(service.uri): name for name, service in known.items()} + for name, service in found.items(): + merged[names.get(str(service.uri), name)] = service + return merged + + def discover( idp: str, *, @@ -136,6 +168,13 @@ def discover( known = canonical.get(name, known_servers.get(name)) if server.version is None or not server.auths: if known is not None and known.version is not None and known.auths: + if server.storage: + known = known.model_copy( + update={ + "storage": _merge_storage(known.storage, server.storage) + }, + deep=True, + ) canonical[name] = known continue merged_server = server @@ -144,6 +183,8 @@ def discover( include={"idp", "name", "uri", "url", "version", "auths"}, exclude_none=True, ) + if server.storage: + updates["storage"] = _merge_storage(known.storage, server.storage) merged_server = known.model_copy(update=updates, deep=True) canonical[name] = merged_server @@ -246,6 +287,7 @@ def activate( resolved, config=target_config, idp=idp, + dev=dev, timeout=timeout, ) target_config.set_active_selection(idp, validated) @@ -396,41 +438,64 @@ async def _discover_for_idp( Raises: ServerDiscoveryError: If registry retrieval fails. """ - sources = registry_sources(idp, include_dev=dev) - search = IVOARegistrySearch(registries=sources) - async with Discover(search, timeout=timeout) as discovery: - registries = await asyncio.gather( - *(discovery.fetch(url, name) for url, name in sources.items()) - ) - successful_registries = [ - registry for registry in registries if registry.success - ] - if not successful_registries: - errors = "; ".join( - f"{registry.name}: {registry.error}" for registry in registries - ) - msg = f"Failed to discover servers for IDP '{idp}': {errors}" - raise ServerDiscoveryError(msg) - - endpoints = [] - for registry in successful_registries: - endpoints.extend(discovery.extract(registry, dev=dev)) - if not endpoints: - return [] + evidence = await registry.evidence( + idp, + dev=dev, + timeout=timeout, + check_platforms=True, + ) + if not evidence.available: + errors = "; ".join(evidence.errors) + msg = f"Failed to discover servers for IDP '{idp}': {errors}" + raise ServerDiscoveryError(msg) + + endpoints = [ + resource + for resource in evidence.resources + if resource.uri.endswith("/skaha") and resource.status == 200 + ] + if not endpoints: + return [] - checked = await asyncio.gather( - *(discovery.check(endpoint) for endpoint in endpoints) - ) - return [ - _discovered_to_server( - endpoint, - idp, - config=config, - timeout=timeout, + storage_resources = [ + resource + for resource in evidence.resources + if resource.uri.endswith(f"/{evidence.leaf}") + ] + workers = await registry.workers( + config, + idp, + endpoint=endpoints[0], + count=len(endpoints), + ) + if workers is None: + return [_registry_resource_to_server(endpoint, idp) for endpoint in endpoints] + + return list( + await asyncio.gather( + *( + asyncio.to_thread( + _discovered_to_server, + endpoint, + idp, + config=worker_config, + token=workers.token, + certificate=workers.certificate, + timeout=timeout, + storage_resource=_select_storage( + endpoint, + storage_resources, + strict=False, + ), + ) + for endpoint, worker_config in zip( + endpoints, + workers.configs, + strict=True, + ) ) - for endpoint in checked - if endpoint.status == 200 - ] + ) + ) def _host_slug(uri: AnyUrl) -> str | None: @@ -440,12 +505,74 @@ def _host_slug(uri: AnyUrl) -> str | None: return uri.host.replace(".", "-") +def _select_storage( + endpoint: RegistryResource, + resources: list[RegistryResource], + *, + strict: bool, +) -> RegistryResource | None: + """Map private registry ambiguity to the public server fetch error.""" + try: + return registry.select_storage(endpoint, resources, strict=strict) + except RegistryEvidenceError as exc: + raise ServerFetchError(str(exc)) from exc + + +async def _discover_storage( + server: Server, + idp: str, + *, + dev: bool, + timeout: int, +) -> RegistryResource | None: + """Return fresh registry evidence for a server's primary VOSpace service.""" + try: + return await registry.discover_storage( + str(server.uri) if server.uri is not None else None, + str(server.url) if server.url is not None else None, + server.name, + idp, + dev=dev, + timeout=timeout, + ) + except RegistryEvidenceError as exc: + raise ServerFetchError(str(exc)) from exc + + +def _configured_storage_resource(server: Server) -> RegistryResource | None: + """Convert the persisted primary VOSpace service to inspection evidence.""" + if server.name is None: + return None + service = server.storage.get(server.name) + if service is None: + return None + return RegistryResource( + registry="configuration", + uri=str(service.uri), + url=str(service.url), + ) + + +def _registry_resource_to_server(endpoint: RegistryResource, idp: str) -> Server: + """Convert registry endpoint identity without performing capability I/O.""" + uri = AnyUrl(endpoint.uri) + return Server( + idp=idp, + name=endpoint.name or _host_slug(uri), + uri=uri, + url=AnyHttpUrl(endpoint.url), + ) + + def _discovered_to_server( - endpoint: DiscoveredServer, + endpoint: RegistryResource, idp: str, *, config: Configuration | None = None, + token: str | None = None, + certificate: Path | None = None, timeout: int = 2, + storage_resource: RegistryResource | None = None, ) -> Server: """Convert a registry discovery record to a persisted HTTP server model. @@ -456,23 +583,23 @@ def _discovered_to_server( endpoint: Discovered registry endpoint. idp: Canonical IDP key. config: Configuration whose Authentication Record authorizes enrichment. + token: Pre-materialized runtime bearer token for worker isolation. + certificate: Pre-materialized runtime certificate for worker isolation. timeout: HTTP timeout in seconds for VOSI capabilities requests. + storage_resource: Same-namespace preferred VOSpace registry record, if any. Returns: Server: Persisted server model with capabilities metadata when available. """ - uri = AnyUrl(endpoint.uri) - server = Server( - idp=idp, - name=endpoint.name or _host_slug(uri), - uri=uri, - url=AnyHttpUrl(endpoint.url), - ) + server = _registry_resource_to_server(endpoint, idp) return enrich( server, config=config, + token=token, + certificate=certificate, strict=False, timeout=timeout, + storage_resource=storage_resource, ) @@ -481,8 +608,11 @@ def enrich( *, config: Configuration | None = None, authentication_idp: str | None = None, + token: str | None = None, + certificate: Path | None = None, strict: bool = True, timeout: int = 2, + storage_resource: RegistryResource | None | object = _STORAGE_RESOURCE_UNSET, ) -> Server: """Return a validated Server enriched from its VOSI capabilities. @@ -493,10 +623,17 @@ def enrich( Authentication or Server Selection. authentication_idp: Optional Authentication Record selector. Defaults to the Server IDP, then the active Authentication. - strict: When ``False``, return the original server if capabilities - cannot be retrieved or parsed. Discovery uses non-strict mode so - one malformed endpoint does not abort listing for an IDP. + token: Optional runtime bearer token for capability requests. + certificate: Optional runtime certificate for capability requests. + strict: When ``False``, keep usable registry and existing storage data + when session or storage capabilities cannot be retrieved or parsed. + Other successful enrichment may still be returned, so the result can + be partial. Discovery uses non-strict mode so one malformed endpoint + does not abort listing for an IDP. timeout: HTTP timeout in seconds for VOSI capabilities requests. + storage_resource: Retained same-namespace VOSpace registry record. Passing + ``None`` records that the preferred resource was absent; omitting the + argument leaves storage outside this inspection. Returns: Server: Copy with version and auth modes populated when discoverable. @@ -505,29 +642,37 @@ def enrich( ServerFetchError: If ``strict`` is ``True`` and capabilities cannot be retrieved, parsed, or contain no session capabilities. """ - if server.url is None: - msg = "Server URL is required to inspect capabilities." - raise ServerFetchError(msg) - base_config = config or Configuration() # ty: ignore[missing-argument] active_idp = authentication_idp or server.idp or base_config.active.authentication - try: - from canfar.client import HTTPClient # noqa: PLC0415 - - with HTTPClient( + if storage_resource is not _STORAGE_RESOURCE_UNSET: + server = _enrich_storage( + server, + storage_resource=( + storage_resource + if isinstance(storage_resource, RegistryResource) + else None + ), config=base_config, authentication_idp=active_idp, - url=server.url, + token=token, + certificate=certificate, + strict=strict, timeout=timeout, - raise_http_errors=False, - ) as client: - request_client = client.client - request_client.headers["Accept"] = "application/xml" - request_client.headers.pop("Content-Type", None) - request_client.headers.pop("X-Skaha-Registry-Auth", None) - response = request_client.get("capabilities") - response.raise_for_status() - capabilities = vosi.capabilities(xml=response.text) + ) + if server.url is None: + msg = "Server URL is required to inspect capabilities." + raise ServerFetchError(msg) + try: + capabilities = vosi.capabilities( + xml=_fetch_capabilities( + server.url, + config=base_config, + authentication_idp=active_idp, + token=token, + certificate=certificate, + timeout=timeout, + ) + ) except ( httpx.HTTPError, OSError, @@ -589,6 +734,112 @@ def enrich( ) +def _enrich_storage( + server: Server, + *, + storage_resource: RegistryResource | None, + config: Configuration, + authentication_idp: str, + token: str | None, + certificate: Path | None, + strict: bool, + timeout: int, +) -> Server: + """Validate and attach one retained primary VOSpace registry resource.""" + error: BaseException | None = None + if storage_resource is None: + leaf = get_idp(authentication_idp).leaf + subject = f"same-namespace '{leaf}' registry record" + error = ValueError( + f"No {subject} found for Science Platform Server '{server.name}'." + ) + else: + subject = storage_resource.uri + try: + xml = _fetch_capabilities( + AnyHttpUrl(storage_resource.url), + config=config, + authentication_idp=authentication_idp, + token=token, + certificate=certificate, + timeout=timeout, + ) + valid = vosi.is_vospace_service(xml) + except ( + httpx.HTTPError, + OSError, + AuthContextError, + AuthExpiredError, + CertificateError, + HTTPAuthenticationError, + ParseError, + DefusedXmlException, + ValueError, + ) as exc: + error = exc + else: + if not valid: + error = ValueError( + "required VOSpace node capability is missing or malformed" + ) + elif server.name is None: + error = ValueError("Science Platform Server has no Server Name") + + if error is not None: + message = ( + f"Failed to inspect VOSpace Service '{subject}' for Science " + f"Platform Server '{server.name}': {error}" + ) + return _keep_or_raise( + server, + strict=strict, + error=message, + cause=error, + debug="Skipping VOSpace Service %s during discovery: %s", + args=(subject, error), + ) + assert storage_resource is not None + assert server.name is not None + service = VOSpaceService.model_validate( + {"uri": storage_resource.uri, "url": storage_resource.url} + ) + + return server.model_copy( + update={"storage": {**server.storage, server.name: service}}, + deep=True, + ) + + +def _fetch_capabilities( + url: AnyHttpUrl, + *, + config: Configuration, + authentication_idp: str, + token: str | None = None, + certificate: Path | None = None, + timeout: int, +) -> str: + """Fetch one VOSI capabilities document through the existing HTTP seam.""" + from canfar.client import HTTPClient # noqa: PLC0415 + + with HTTPClient.build( + config=config, + authentication_idp=authentication_idp, + url=url, + token=token, + certificate=certificate, + timeout=timeout, + raise_http_errors=False, + ) as client: + request_client = client.client + request_client.headers["Accept"] = "application/xml" + request_client.headers.pop("Content-Type", None) + request_client.headers.pop("X-Skaha-Registry-Auth", None) + response = request_client.get("capabilities") + response.raise_for_status() + return response.text + + def _keep_or_raise( server: Server, *, @@ -610,6 +861,7 @@ def _validate_server( *, config: Configuration | None = None, idp: str | None = None, + dev: bool = False, timeout: int = 2, ) -> Server: """Fetch and validate a server before persisting it as active. @@ -618,6 +870,7 @@ def _validate_server( server: Candidate server record. config: Configuration to use while validating the candidate selection. idp: Authentication IDP to pair with the candidate server. + dev: Include development registry evidence during validation. timeout: HTTP timeout in seconds for validation requests. Returns: @@ -628,12 +881,23 @@ def _validate_server( """ base_config = config or Configuration() # ty: ignore[missing-argument] active_idp = idp or server.idp or base_config.active.authentication + storage_resource = _configured_storage_resource(server) + if storage_resource is None: + storage_resource = asyncio.run( + _discover_storage( + server, + active_idp, + dev=dev, + timeout=timeout, + ) + ) enriched = enrich( server, config=base_config, authentication_idp=active_idp, strict=True, timeout=timeout, + storage_resource=storage_resource, ) if enriched.url is None or enriched.version is None: msg = "Server URL and version are required before activation." diff --git a/canfar/storage.py b/canfar/storage.py new file mode 100644 index 00000000..c41095a4 --- /dev/null +++ b/canfar/storage.py @@ -0,0 +1,251 @@ +"""Adapters for the configured VOSpace Services and the local filesystem.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING + +from fsspec.asyn import get_loop, sync +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper +from fsspec.implementations.local import LocalFileSystem + +from canfar.client import HTTPClient +from canfar.exceptions.context import AuthContextError +from canfar.models.config import Configuration +from canfar.models.http import LOCAL + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + from pathlib import Path + + from fsspec.spec import AbstractFileSystem + from fsspec_cli import AsyncFilesystemSource + from pydantic import SecretStr + from vosfs import VOSpaceFileSystem + + from canfar.models.auth import RuntimeCredential + +__all__ = ["LOCAL", "filesystem", "identifiers", "sources"] +"""Public surface; Storage Identifiers resolve through ``__getattr__``.""" + +_LISTINGS_EXPIRY_SECONDS = 30 +"""Seconds a cached directory listing stays valid on one filesystem.""" + +_LISTINGS_MAX_PATHS = 1000 +"""Maximum directory listings retained by one filesystem.""" + + +async def _resolve( + identifier: str, + token: str | SecretStr | None = None, + certificate: Path | str | None = None, +) -> tuple[str, RuntimeCredential]: + """Resolve one Storage Identifier to its endpoint and a usable credential. + + Args: + identifier: Storage Identifier of the configured VOSpace Service. + token: Runtime bearer token, preferred over any saved credential. + certificate: Runtime X.509 certificate path. + + Returns: + tuple[str, RuntimeCredential]: The endpoint and its credential. + + Raises: + AuthContextError: If the credential cannot be materialized. + """ + config = Configuration() # ty: ignore[missing-argument] + endpoint, idp = config._resolve_storage(identifier) # noqa: SLF001 + try: + client = HTTPClient.build( + config=config, + authentication_idp=idp, + url=endpoint, + token=token, + certificate=certificate, + ) + credential = await client._materialize_credentials() # noqa: SLF001 + except (KeyError, OSError, TypeError, ValueError): + reason = "Credential cannot be used. Run 'canfar login' for this IDP." + raise AuthContextError(idp, reason) from None + return endpoint, credential + + +def _build( + endpoint: str, + credential: RuntimeCredential, + *, + asynchronous: bool, +) -> VOSpaceFileSystem: + """Construct one VOSpace filesystem for an endpoint and credential. + + Args: + endpoint: URL of the VOSpace Service. + credential: Materialized token or certificate for that Service. + asynchronous: Build the filesystem in fsspec's asynchronous mode. + + Returns: + VOSpaceFileSystem: A ready, authenticated VOSpace filesystem. + """ + from vosfs import VOSpaceFileSystem as _VOSpaceFileSystem # noqa: PLC0415 + + # Passed explicitly rather than unpacked so the credential kwarg stays typed. + if credential.token is not None: + return _VOSpaceFileSystem( + endpoint, + token=credential.token, + asynchronous=asynchronous, + skip_instance_cache=True, + use_listings_cache=True, + listings_expiry_time=_LISTINGS_EXPIRY_SECONDS, + max_paths=_LISTINGS_MAX_PATHS, + ) + return _VOSpaceFileSystem( + endpoint, + certfile=credential.certificate, + asynchronous=asynchronous, + skip_instance_cache=True, + use_listings_cache=True, + listings_expiry_time=_LISTINGS_EXPIRY_SECONDS, + max_paths=_LISTINGS_MAX_PATHS, + ) + + +def _vospace( + identifier: str, + *, + token: str | SecretStr | None = None, + certificate: Path | str | None = None, +) -> AsyncFilesystemSource: + """Return a fresh authenticated async filesystem source. + + Args: + identifier: Storage Identifier of the configured VOSpace Service. + token: Runtime bearer token, preferred over any saved credential. + certificate: Runtime X.509 certificate path. + + Returns: + AsyncFilesystemSource: Factory yielding one authenticated filesystem. + """ + + @asynccontextmanager + async def source() -> AsyncIterator[AbstractFileSystem]: + endpoint, credential = await _resolve(identifier, token, certificate) + filesystem = _build(endpoint, credential, asynchronous=True) + try: + yield filesystem + finally: + await filesystem.aclose() + + return source + + +@asynccontextmanager +async def _local() -> AsyncIterator[AbstractFileSystem]: + """Yield a fresh asynchronous wrapper around the local filesystem. + + Yields: + AbstractFileSystem: An async-wrapped local filesystem. + """ + yield AsyncFileSystemWrapper( + LocalFileSystem(skip_instance_cache=True), + asynchronous=True, + ) + + +def sources() -> dict[str, AsyncFilesystemSource]: + """Build the mapped storage sources for one data command invocation. + + Every configured VOSpace Service is mapped by its Storage Identifier, plus + the always-available ``local`` filesystem. + + Returns: + dict[str, AsyncFilesystemSource]: Sources keyed by Storage Identifier. + """ + config = Configuration() # ty: ignore[missing-argument] + mapped: dict[str, AsyncFilesystemSource] = { + identifier: _vospace(identifier) + for identifier in config.storage_identifiers() + if identifier != LOCAL + } + mapped[LOCAL] = _local + return mapped + + +def identifiers() -> list[str]: + """Return every Storage Identifier this configuration can address. + + Returns: + list[str]: Configured Storage Identifiers plus the reserved ``local``. + """ + config = Configuration() # ty: ignore[missing-argument] + return config.storage_identifiers() + + +def filesystem( + identifier: str, + *, + token: str | SecretStr | None = None, + certificate: Path | str | None = None, +) -> AbstractFileSystem: + """Return a synchronous filesystem for one Storage Identifier. + + Args: + identifier: Storage Identifier, or ``local`` for the running machine. + token: Runtime bearer token, preferred over any saved credential. + certificate: Runtime X.509 certificate path. + + Returns: + AbstractFileSystem: A ready, authenticated filesystem. + + Raises: + KeyError: If ``identifier`` is not configured. + AuthContextError: If the saved credential cannot be used. + """ + if identifier == LOCAL: + return LocalFileSystem(skip_instance_cache=True) + # fsspec's background loop, so this works inside a running loop too. + endpoint, credential = sync(get_loop(), _resolve, identifier, token, certificate) + return _build(endpoint, credential, asynchronous=False) + + +def __getattr__(identifier: str) -> AbstractFileSystem: + """Return a filesystem for a Storage Identifier accessed as an attribute. + + Makes ``from canfar.storage import vault`` resolve to a ready filesystem + for the ``vault`` Storage Identifier. + + Args: + identifier: Attribute name, treated as a Storage Identifier. + + Returns: + AbstractFileSystem: A ready, authenticated filesystem. + + Raises: + AttributeError: If ``identifier`` is not a configured Storage + Identifier. + """ + if identifier.startswith("_"): + message = f"module {__name__!r} has no attribute {identifier!r}" + raise AttributeError(message) + known = identifiers() + if identifier not in known: + message = ( + f"module {__name__!r} has no attribute {identifier!r}; " + f"configured Storage Identifiers are: {', '.join(known)}" + ) + raise AttributeError(message) + # Built outside the membership check so a failure to authenticate surfaces + # as itself rather than as a missing attribute. + return filesystem(identifier) + + +def __dir__() -> list[str]: + """List the module's own names plus every Storage Identifier. + + Returns: + list[str]: Names available on this module, for tab completion. + """ + try: + return sorted({*__all__, *identifiers()}) + except (OSError, ValueError): # pragma: no cover - unreadable configuration + return sorted(__all__) diff --git a/canfar/utils/discover.py b/canfar/utils/discover.py index baceb574..7b630ecc 100644 --- a/canfar/utils/discover.py +++ b/canfar/utils/discover.py @@ -3,6 +3,7 @@ from __future__ import annotations import time +from urllib.parse import urlsplit, urlunsplit import httpx from typing_extensions import Self @@ -50,12 +51,19 @@ async def __aexit__( """ await self.client.aclose() - async def fetch(self, url: str, name: str) -> IVOARegistry: + async def fetch( + self, + url: str, + name: str, + *, + development: bool = False, + ) -> IVOARegistry: """Fetch registry contents. Args: url (str): Registry URL. name (str): Common name for the registry. + development: Whether this source contains development records. Returns: RegistryInfo: Registry information. @@ -69,13 +77,26 @@ async def fetch(self, url: str, name: str) -> IVOARegistry: f"[dim]Fetched {name} in {elapsed:.2f}s[/dim]" ) - return IVOARegistry(name=name, content=response.text, success=True) + return IVOARegistry( + name=name, + source=url, + development=development, + content=response.text, + success=True, + ) except httpx.HTTPError as error: error_msg = str(error) - return IVOARegistry(name=name, content="", success=False, error=error_msg) + return IVOARegistry( + name=name, + source=url, + development=development, + content="", + success=False, + error=error_msg, + ) def extract(self, registry: IVOARegistry, dev: bool = False) -> list[Server]: - """Extract capabilities endpoints from registry content.""" + """Extract Science Platform and preferred VOSpace registry records.""" if not registry.success or not registry.content: return [] @@ -89,23 +110,28 @@ def extract(self, registry: IVOARegistry, dev: bool = False) -> list[Server]: uri, url = line.split("=", 1) uri, url = uri.strip(), url.strip() - if url.endswith("/skaha/capabilities") and uri.endswith("/skaha"): - url = url.replace("/capabilities", "") - # Apply exclusion filters - if not dev and any( + leaf = uri.rpartition("/")[2] + if leaf in {"skaha", self.config.leaf}: + url = _without_terminal_capabilities(url) + if url is None: + continue + record_development = any( word in uri.lower() or word in url.lower() for word in self.config.excluded - ): + ) + # Apply exclusion filters + if not dev and record_development: continue # Apply omit filters if (registry.name, uri) in self.config.omit: continue endpoint = Server( - registry=registry.name, + registry=registry.source or registry.name, + development=registry.development or record_development, uri=uri, url=url, - name=self.config.names.get(uri), + name=self.config.names.get(uri) if leaf == "skaha" else None, ) endpoints.append(endpoint) @@ -119,3 +145,11 @@ async def check(self, endpoint: Server) -> Server: except httpx.HTTPError: endpoint.status = None return endpoint + + +def _without_terminal_capabilities(url: str) -> str | None: + """Remove only a terminal ``/capabilities`` path component.""" + parsed = urlsplit(url) + if not parsed.path.endswith("/capabilities"): + return None + return urlunsplit(parsed._replace(path=parsed.path.removesuffix("/capabilities"))) diff --git a/canfar/utils/registry.py b/canfar/utils/registry.py new file mode 100644 index 00000000..79f78a3a --- /dev/null +++ b/canfar/utils/registry.py @@ -0,0 +1,290 @@ +"""Registry evidence acquisition and discovery-worker preparation.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +from pydantic import AnyHttpUrl, BaseModel, ConfigDict + +from canfar import get_logger +from canfar.auth.x509 import CertificateError +from canfar.exceptions.context import AuthContextError, AuthExpiredError +from canfar.idp import get_idp, registry_sources +from canfar.models.config import Configuration +from canfar.models.registry import IVOARegistrySearch +from canfar.models.registry import Server as RegistryResource +from canfar.utils.discover import Discover + +log = get_logger(__name__) + + +class RegistryEvidenceError(RuntimeError): + """Raised when strict registry evidence is missing or ambiguous.""" + + +class RegistryEvidence(BaseModel): + """Registry resources plus acquisition outcome for one IDP inspection. + + Attributes: + leaf: Preferred primary VOSpace registry URI leaf. + resources: Registry resources extracted from every successful registry. + errors: Human-readable failures keyed by registry name. + available: Whether at least one registry responded successfully. + """ + + model_config = ConfigDict(frozen=True) + + leaf: str | None + resources: tuple[RegistryResource, ...] + errors: tuple[str, ...] + available: bool + + +class Workers(BaseModel): + """Isolated worker configs with one pre-materialized runtime credential. + + Attributes: + configs: One isolated Configuration per concurrent worker. + token: Materialized bearer token, when the credential provides one. + certificate: Materialized X.509 certificate path, when applicable. + """ + + model_config = ConfigDict(frozen=True) + + configs: tuple[Configuration, ...] + token: str | None = None + certificate: Path | None = None + + +async def evidence( + idp: str, + *, + dev: bool, + timeout: int, + check_platforms: bool, +) -> RegistryEvidence: + """Acquire and extract registry records through one shared pipeline. + + Args: + idp: Canonical Identity Provider key. + dev: Include development registries and endpoints during discovery. + timeout: HTTP timeout in seconds for registry requests. + check_platforms: Probe Science Platform endpoints for reachability. + + Returns: + RegistryEvidence: Extracted resources plus the acquisition outcome. + """ + idp_info = get_idp(idp) + sources = registry_sources(idp, include_dev=dev) + development_sources = set(idp_info.dev_registries) + search = IVOARegistrySearch( + registries=sources, + leaf=idp_info.leaf, + ) + async with Discover(search, timeout=timeout) as discovery: + registries = await asyncio.gather( + *( + discovery.fetch( + url, + name, + development=url in development_sources, + ) + for url, name in sources.items() + ) + ) + successful = [registry for registry in registries if registry.success] + resources = [ + resource + for registry in successful + for resource in discovery.extract(registry, dev=dev) + ] + if check_platforms: + endpoints = [ + resource for resource in resources if resource.uri.endswith("/skaha") + ] + await asyncio.gather(*(discovery.check(endpoint) for endpoint in endpoints)) + + return RegistryEvidence( + leaf=idp_info.leaf, + resources=tuple(resources), + errors=tuple( + f"{registry.name}: {registry.error}" + for registry in registries + if not registry.success + ), + available=bool(successful), + ) + + +def select_storage( + endpoint: RegistryResource, + resources: list[RegistryResource], + *, + strict: bool, +) -> RegistryResource | None: + """Pair an endpoint with one unambiguous same-environment VOSpace record. + + Args: + endpoint: Science Platform registry record to pair. + resources: Candidate VOSpace registry records. + strict: Raise instead of logging when the pairing is ambiguous. + + Returns: + RegistryResource | None: The single paired record, or None. + + Raises: + RegistryEvidenceError: If ``strict`` and the pairing is ambiguous. + """ + namespace = endpoint.uri.rpartition("/")[0] + candidates = [ + resource + for resource in resources + if resource.uri.rpartition("/")[0] == namespace + and resource.development == endpoint.development + ] + same_registry = [ + resource for resource in candidates if resource.registry == endpoint.registry + ] + preferred = same_registry or candidates + if len(preferred) == 1: + return preferred[0] + if len(preferred) > 1: + message = ( + "Multiple preferred VOSpace registry records found for Science " + f"Platform Server '{endpoint.name or endpoint.uri}' in namespace " + f"'{namespace}'." + ) + if strict: + raise RegistryEvidenceError(message) + log.debug("%s Omitting generated storage configuration.", message) + return None + + +async def discover_storage( + uri: str | None, + url: str | None, + name: str | None, + idp: str, + *, + dev: bool, + timeout: int, +) -> RegistryResource | None: + """Return fresh registry evidence for a Server's primary VOSpace Service. + + Args: + uri: IVOA URI of the Science Platform Server. + url: URL of the Science Platform Server, used to disambiguate. + name: Server Name, used only for diagnostics. + idp: Canonical Identity Provider key. + dev: Include development registries and endpoints during discovery. + timeout: HTTP timeout in seconds for registry requests. + + Returns: + RegistryResource | None: The paired VOSpace record, or None. + + Raises: + RegistryEvidenceError: If the URI is missing, the registry is + unavailable, or the Server record is missing or ambiguous. + """ + if uri is None: + message = "Server URI is required to inspect its VOSpace Service." + raise RegistryEvidenceError(message) + + found = await evidence( + idp, + dev=dev, + timeout=timeout, + check_platforms=False, + ) + if not found.available: + errors = "; ".join(found.errors) + message = ( + f"Failed to inspect VOSpace registry records for IDP '{idp}': {errors}" + ) + raise RegistryEvidenceError(message) + + endpoints = [ + resource + for resource in found.resources + if resource.uri == uri and resource.uri.endswith("/skaha") + ] + matching_urls = [ + endpoint for endpoint in endpoints if url is not None and endpoint.url == url + ] + if len(matching_urls) == 1: + endpoint = matching_urls[0] + elif len(endpoints) == 1: + endpoint = endpoints[0] + elif not endpoints: + message = ( + f"No Science Platform registry record found for Server '{name}' " + f"with URI '{uri}'." + ) + raise RegistryEvidenceError(message) + else: + message = ( + f"Multiple Science Platform registry records found for Server " + f"'{name}' with URI '{uri}'." + ) + raise RegistryEvidenceError(message) + + storage_resources = [ + resource + for resource in found.resources + if resource.uri.endswith(f"/{found.leaf}") + ] + return select_storage(endpoint, storage_resources, strict=True) + + +async def workers( + config: Configuration | None, + idp: str, + *, + endpoint: RegistryResource, + count: int, +) -> Workers | None: + """Materialize credentials once, then isolate worker configuration state. + + Args: + config: Configuration to derive workers from. Defaults to loading config. + idp: Canonical Identity Provider key. + endpoint: Science Platform registry record used to build the client. + count: Number of isolated worker configurations to produce. + + Returns: + Workers | None: Workers, or None when credentials are absent + or unusable. + """ + from canfar.client import HTTPClient # noqa: PLC0415 + + base_config = config or Configuration() # ty: ignore[missing-argument] + client = HTTPClient.build( + config=base_config, + authentication_idp=idp, + url=AnyHttpUrl(endpoint.url), + ) + token: str | None = None + certificate: Path | None = None + if client.uses_runtime_credentials or client.authentication_record is not None: + try: + credential = await client._materialize_credentials() # noqa: SLF001 + except ( + KeyError, + OSError, + AuthContextError, + AuthExpiredError, + CertificateError, + TypeError, + ValueError, + ) as exc: + log.debug("Skipping capability enrichment for IDP %s: %s", idp, exc) + return None + token = credential.token + certificate = ( + Path(credential.certificate) if credential.certificate is not None else None + ) + + values = base_config.model_dump(mode="python") + configs = tuple(Configuration.model_validate(values) for _ in range(count)) + return Workers(configs=configs, token=token, certificate=certificate) diff --git a/canfar/utils/vosi.py b/canfar/utils/vosi.py index fc699972..7d2f1946 100644 --- a/canfar/utils/vosi.py +++ b/canfar/utils/vosi.py @@ -9,6 +9,8 @@ LEGACY_SESSIONS_STDID = "vos://cadc.nrc.ca~vospace/CADC/std/Proc#sessions-1.0" PLATFORM_PREFIX = "http://www.opencadc.org/std/platform#session-" +VOSPACE_NODES_STDID = "ivo://ivoa.net/std/VOSpace/v2.0#nodes" +_XSI_TYPE = "{http://www.w3.org/2001/XMLSchema-instance}type" # Auth mode rename map AUTH_RENAME = { @@ -199,3 +201,23 @@ def capabilities( # noqa: PLR0912 too many branches (clarity) reverse=True, ) return items + + +def is_vospace_service(xml: str) -> bool: + """Return whether VOSI XML exposes the OpenCADC VOSpace node binding.""" + root = ElementTree.fromstring(xml) + for capability in root.findall(".//{*}capability"): + if capability.get("standardID") != VOSPACE_NODES_STDID: + continue + for interface in capability.findall("{*}interface"): + if interface.get("role") != "std": + continue + if interface.get(_XSI_TYPE, "").rsplit(":", 1)[-1] != "ParamHTTP": + continue + if any( + (access.get("use") in {None, "base"}) + and bool((access.text or "").strip()) + for access in interface.findall("{*}accessURL") + ): + return True + return False diff --git a/docs/agents/research/2026-07-25-storage-python-api.md b/docs/agents/research/2026-07-25-storage-python-api.md new file mode 100644 index 00000000..38c4c4af --- /dev/null +++ b/docs/agents/research/2026-07-25-storage-python-api.md @@ -0,0 +1,865 @@ +# A Python storage API for CANFAR VOSpace Services + +- **Date:** 2026-07-25 +- **Repository snapshot:** `ba5493fd` on `feat/vosfs-support` +- **fsspec evaluated:** `2026.6.0` +- **vosfs evaluated:** `0.7.0` (see the 0.8.0 update below) +- **fsspec-cli evaluated:** `0.6.0` (see the 0.7.0 update below) +- **Python:** `3.13.5` +- **Question:** How should `canfar` conveniently expose VOSpace storage for Python API access? + +## Update, 2026-07-25 (later the same day): byte ranges now work on `vault` + +This note was written against `vosfs` 0.7.0. Upstream +[shinybrar/vosfs#334](https://github.com/shinybrar/vosfs/issues/334), filed from +this research, was fixed in +[#335](https://github.com/shinybrar/vosfs/issues/335) and released as +`vosfs` 0.8.0 / `fsspec-cli` 0.7.0, which `canfar` now pins. + +What changed, VERIFIED against the live service on 0.8.0: + +- `cat_file(path, start, end)` now sends `Range` and uses a `206` response. + Observed on `vault`: `Range: bytes=0-2879` -> `HTTP 206`, + `Content-Range: bytes 0-2879/3542400`, 2880 bytes returned. +- `MMapCache` over a `cat_file` fetcher is now genuinely partial: one ranged + request, one block of four materialised, zero whole-object fallbacks. Section + 4 below measured the opposite on 0.7.0 and is superseded for `vault`. +- `arc` (Cavern) still serves no ranges and falls back to a whole-object read + that is sliced. Slices remain correct (VERIFIED), so the "cache whole files" + guidance still holds there. +- `blockcache` / `CachingFileSystem` still fails with + `AttributeError: 'StagedReadFile' object has no attribute 'blocksize'`. + `Range` is honoured for byte reads, not through the file-object path. + +The recommended module has since shipped as `canfar/storage.py`, adding +attribute access (`from canfar.storage import vault, arc`) on top of the +`filesystem` / `fetch` / `sources` surface proposed below, so any text quoting +the old "no public CANFAR storage Python API" wording is historical. + +Sections 2 and 4 describe the 0.7.0 behaviour and are kept as the historical +record. The single-cache-layer recommendation in the Verdict stands, but its +stated reason ("the second tier is either dead or fictional") now applies only +to `arc`; on `vault` a block cache over `MMapCache` is a real option. + +## Verdict + +**Ship one module, `canfar/storage.py`, with three public functions and no new +dependency. Build sync by default, async on request, and cache with exactly one +`SimpleCacheFileSystem` layer. Do not build a tiered cache: over `vosfs` 0.7.0 the +second tier is either dead or fictional.** + +The recommended surface is: + +```python +from canfar.storage import filesystem, fetch, sources +``` + +- `filesystem(name, *, cache=False, asynchronous=False, token=None, certificate=None)` + returns a ready `AbstractFileSystem` for one Storage Identifier. +- `fetch(name, path, *, cache=True)` materializes one remote object and returns its + local `Path`. +- `sources()` is the existing fsspec-cli seam, moved unchanged. + +Three findings drive every other decision, and two of them contradict text currently +in `docs/cli/data.md`: + +1. **`vosfs` 0.7.0 has no server-side byte ranges.** `_cat_file(path, start, end)` + downloads the whole object and slices it in memory. A 2880-byte "range" read of a + 3.5 MB file costs the same as reading the file (VERIFIED, §2). Every byte-range + design — `MMapCache`, `blockcache`, `cache_type=` — is therefore either broken or a + pessimization. +2. **Only whole-file caching composes.** `simplecache` and `filecache` work. Stacking + two cache layers builds a stack in which the inner layer never fills and never + serves (VERIFIED, §5). `cache_storage=["a", "b"]` is a genuine two-location lookup + but performs no promotion (VERIFIED, §5). +3. **There is no fsspec-native RAM tier over `vosfs`.** `cache_storage="memory://…"` + is accepted and silently creates a local directory literally named `memory:` + (VERIFIED, §5). The real RAM tier is the OS page cache over the disk cache + directory, plus `/dev/shm` on Linux. + +The sync/async tension resolves cleanly and is already half-solved in the repository: +build `VOSpaceFileSystem` synchronously, wrap it in the sync cache, and re-expose it +with `AsyncFileSystemWrapper` — the same class `canfar/storage.py:108` already uses +for `local`. This was verified end to end (§4, §9). + +`docs/cli/data.md:143-165` should be corrected regardless of whether this API ships. + +## Evidence key + +Every finding below is marked: + +- **VERIFIED** — I executed it in this repository's environment against the live + `vault` VOSpace Service with `~/.ssl/cadcproxy.pem`, read-only, on 2026-07-25. +- **DOCUMENTED** — I read the source or the published documentation and did not + execute it. + +All experiments used `uv run --no-sync python` from the repository root, targeted only +the two public read-only test objects +(`vault:/ALMA/test-data/cutouts/test-4d-cube-cutout.fits`, 169 920 B, and +`.../test-4d-cube.fits`, 3 542 400 B), and wrote only into the session scratchpad. No +`canfar/` or `tests/` file was modified and no git state was touched. + +## 1. Baseline: what exists today + +| Element | Location | +| --- | --- | +| `_vospace(name, *, token, certificate)` factory | `canfar/storage.py:31-98` | +| `_local()` async wrapper over `LocalFileSystem` | `canfar/storage.py:101-111` | +| `sources() -> dict[str, AsyncFilesystemSource]` | `canfar/storage.py:114-130` | +| Only consumer | `canfar/cli/data.py:12,22-33` | +| `AsyncFilesystemSource = Callable[[], AbstractAsyncContextManager[AbstractFileSystem]]` | `.venv/lib/python3.13/site-packages/fsspec_cli/_app.py:41-43` | +| Storage Identifier → endpoint + IDP | `canfar/models/config.py:382-395` | +| Credential materialization (async) | `canfar/client.py:234-262` | +| Runtime dependencies | `pyproject.toml:48-65` | + +DOCUMENTED. `canfar/storage.py` is not exported from `canfar/__init__.py:16-31`; its +only non-test importer is `canfar/cli/data.py:12`. `docs/cli/data.md:176` states the +release "intentionally provides no public CANFAR storage Python API". The module is +therefore free to be renamed. + +## 2. The constraint that dominates everything: no byte ranges + +> **Correction (2026-07-25, after review).** The heading below overstates the +> finding: the limitation is in the **client**, not uniformly in the services. +> `vosfs` sends no `Range` header, so every partial read is a whole-object +> download — that part stands and is what makes block caching a pessimization +> today. But the services differ, VERIFIED live against both: +> +> | Service | Backend | Ranged `GET` | +> | --- | --- | --- | +> | `vault` | `minoc` (`ws-cadc.canfar.net/minoc/files/...`) | `206`, `Accept-Ranges: bytes`, `Content-Range: bytes 0-2879/3542400` | +> | `arc` | Cavern | `200`, whole body, no `Accept-Ranges` | +> +> So the Cavern quote below is accurate for `arc` and does not generalize to +> `vault`. Teaching `vosfs` to send `Range` would deliver real byte-range reads +> on `vault` (not on `arc`), which upgrades the upstream ask in section 7 from +> "cosmetic" to genuinely valuable. + +DOCUMENTED. `vosfs/staging.py:3-5` states it outright, for Cavern: + +> OpenCADC Cavern does not implement HTTP byte ranges, so a seekable read is a +> whole-object download into a disk-backed temporary file. + +DOCUMENTED. `vosfs/filesystem.py:547-556` implements `_cat_file` as a whole read +followed by an in-memory slice: + +```python +async def _cat_file(self, path, start=None, end=None, **_kwargs): + """Return one whole-object read sliced with Python half-open semantics.""" + data = await _transfer.read_whole(self, self._strip_protocol(path)) + return data[start:end] +``` + +`_transfer.read_whole` (`vosfs/_transfer.py:227-240`) joins the full streamed body. +`_transfer.open_read_stream` (`vosfs/_transfer.py:197-224`) sends +`GET` with `transport.IDENTITY_ENCODING` headers and no `Range` header. A grep for +`Range` across the whole installed `vosfs` package returns no hits. + +VERIFIED. Timings on `vault:/ALMA/test-data/cutouts/test-4d-cube.fits` (3 542 400 B), +one warm filesystem instance: + +| Call | Bytes returned | Wall time | +| --- | --- | --- | +| `cat_file(BIG)` | 3 542 400 | 1.41 s | +| `cat_file(BIG, 0, 2880)` | 2 880 | 1.34 s | +| `cat_file(BIG, size-2880, None)` | 2 880 | 1.21 s | +| `cat_ranges([BIG]*4, 4 starts, 4 ends)` | 4 × 2 880 | 1.16 s | + +A 2880-byte head read costs 0.95× a whole-file read. `cat_ranges` is the one genuine +optimization: `vosfs/filesystem.py:594-620` groups ranges by path and +`_read_staged_ranges` (`vosfs/filesystem.py:622-633`) performs **one** whole-object +download per distinct path per call, then slices from the staging file. Four ranges of +one object cost one download, not four. + +**Consequence.** Anything that reads a "block" pays for the whole object. This is not +a `vosfs` defect to route around in `canfar`; it is a service-side gap. + +### The doc claim that is wrong + +`docs/cli/data.md:143-144` says "VOSpace supports ranged reads, so a large cube can be +cached one block at a time rather than downloaded whole", and `:162-163` says "Reading +a FITS header this way transfers a single block instead of the whole file." + +VERIFIED false. Driving `MMapCache(blocksize=1<<20, fetcher=lambda s, e: fs.cat_file(BIG, s, e), size=3542400, location=…)`: + +| Operation | Wall time | What actually moved | +| --- | --- | --- | +| `mc._fetch(0, 2880)` (one FITS header block) | 1.84 s | full 3 542 400 B download | +| plain `fs.cat_file(BIG)` for comparison | 1.64 s | full 3 542 400 B download | +| three scattered 2880 B reads via `MMapCache` | 4.01 s | **three** full downloads | +| sparse backing file after touching 3 of 4 blocks | — | 3 543 040 B allocated (i.e. full) | + +The `MMapCache` mechanics are real — a repeated read of an already-fetched block was +0.000 s, and the on-disk file is genuinely sparse-capable — but over `vosfs` the +fetcher is a whole-object download, so the cache costs *more* network than a single +`cat_file`, and the sparse file fills up anyway. `location=None` (anonymous mmap, RAM) +behaves identically for network cost: 1.40 s for the same header read. + +## 3. Import surface: `canfar.storage`, singular + +**Recommendation: rename `canfar/storage.py` to `canfar/storage.py` and make it the +public module.** + +Rationale, all DOCUMENTED: + +- The package's own convention is a singular module name for a plural domain. + `canfar/__init__.py:16` exports `server` (not `servers`), and `canfar/server.py` + contains `discover`, `list_servers`, `activate`, `use`. `canfar/sessions.py` is the + exception, not the rule, and it is a class module. +- The maintainer's suggested import is `from canfar.storage import sources`. +- The module is currently private in every meaningful sense (§1), so the rename costs + exactly one import line in `canfar/cli/data.py:12` plus the `from canfar import + storages` references in `tests/test_cli_data.py`. No published API breaks, because + `docs/cli/data.md:176` says none exists. + +Do **not** add a compatibility shim re-exporting `canfar.storage`. There is nothing +to be compatible with. + +### What belongs in it — and what does not + +| Ask | Recommendation | +| --- | --- | +| Filesystem accessor | `filesystem()` — yes | +| Opener | No new function. `filesystem(name).open(path)` already is the opener. | +| Cache configurator | No object, no builder. One `cache=` parameter on `filesystem()`. | +| Path/URL resolver | **No.** See below. | +| Materialize to a local path | `fetch()` — yes, because the fsspec route is private API (§7) | +| fsspec-cli seam | `sources()`, moved unchanged | + +**Reject the path/URL resolver.** VERIFIED: `vos://` URLs do not round-trip. Because +`vosfs` treats everything after the protocol marker as path +(`vosfs/filesystem.py:69-71`, `_strip_protocol` at `:179-188`): + +```text +_strip_protocol('vos://cadc-west-01.canfar.net/vault/ALMA/…/test-4d-cube-cutout.fits') + -> '/cadc-west-01.canfar.net/vault/ALMA/…/test-4d-cube-cutout.fits' +``` + +The authority is silently swallowed into the path. A `canfar` resolver that emitted +`vos://` URLs would produce strings that look addressable and are not. In Python the +Storage Identifier and the path are already two arguments; a resolver adds a failure mode +and removes nothing. The CLI's `identifier:/path` operand syntax +(`docs/cli/data.md:17-24`) is a *command-line* affordance and should stay there. + +## 4. Getting a filesystem, sync and async + +The tension is real and I measured both horns. + +VERIFIED. **Cache wrappers reject an async `vosfs` instance.** +`SimpleCacheFileSystem(fs=VOSpaceFileSystem(..., asynchronous=True), cache_storage=…)` +constructs, then the first read fails: + +```text +RuntimeError: Loop is not running +``` + +VERIFIED. **A sync `vosfs` instance works inside a running event loop, but blocks +it.** `VOSpaceFileSystem(..., asynchronous=False).cat_file(path)` called from inside +`asyncio.run` succeeded — fsspec's `sync()` dispatches to its own dedicated IO-thread +loop (DOCUMENTED, ), so +it does not hit the "cannot call sync from a running loop" guard. But a ticker +coroutine sleeping every 50 ms recorded **0 ticks** during a 10.60 s call. The caller's +loop is frozen for the duration. + +VERIFIED. **`AsyncFileSystemWrapper` over the sync cached stack resolves it.** + +```python +inner = VOSpaceFileSystem(endpoint, certfile=…, asynchronous=False) +cached = SimpleCacheFileSystem(fs=inner, cache_storage=…) +wrapped = AsyncFileSystemWrapper(cached, asynchronous=True) +``` + +| Measurement | Result | +| --- | --- | +| `await wrapped._cat_file(SMALL)` cold | 1.87 s, 169 920 B | +| `await wrapped._cat_file(SMALL)` warm | 0.007 s | +| `await wrapped._ls(dir, detail=False)` | 2 entries, correct | +| `await wrapped._info(SMALL)["size"]` | 169 920 | +| ticker ticks during the 1.87 s cold call | **36** (non-blocking) | + +DOCUMENTED. `fsspec/implementations/asyn_wrapper.py:11-36` wraps each sync method with +`asyncio.to_thread`, which is why the loop stays responsive. Its class declaration is +`fsspec/implementations/asyn_wrapper.py:39`. `canfar/storage.py:108` already uses this +class for `local`, so it is not a new concept in this codebase. fsspec's own docs call +it "experimental" and warn "Users should not expect this wrapper to magically make +things faster" () — the +caution is about throughput, not correctness, and the measurements above are the +throughput answer for this workload. + +### The resulting rule + +| Request | Build | +| --- | --- | +| `asynchronous=False`, `cache=False` | `VOSpaceFileSystem(..., asynchronous=False)` | +| `asynchronous=False`, cache on | `SimpleCacheFileSystem(fs=, ...)` | +| `asynchronous=True`, `cache=False` | `VOSpaceFileSystem(..., asynchronous=True)` — native, best | +| `asynchronous=True`, cache on | `AsyncFileSystemWrapper(SimpleCacheFileSystem(fs=), asynchronous=True)` | + +Never construct a cache wrapper over an `asynchronous=True` instance. + +### Credential materialization from a sync entry point + +`canfar/client.py:234` `_materialize_credentials` is `async def`. VERIFIED: it can be +driven from a sync function both outside and inside a running loop using fsspec's own +IO loop, with no `asyncio.run` and no nested-loop guard: + +```python +from fsspec.asyn import get_loop, sync +token, certfile = sync(get_loop(), client._materialize_credentials) +``` + +Outside a loop: 0.57 s (includes X.509 validation). Inside `asyncio.run`: 0.00 s +(credential already warm). This is the smallest correct bridge and adds no dependency. + +### One instance is worth keeping + +VERIFIED. Per-call latency against `vault`: + +| Measurement | Observed | +| --- | --- | +| first `info()` on a fresh instance, 5 fresh instances | 1.65, 0.43, 0.61, 0.56, 0.60 s | +| second `info()` on the same instance | 0.30, 0.34, 0.30, 0.52, 0.36 s | +| whole-object read, 169 920 B, warm instance | ≈ 1.0 s | +| worst first-call outlier observed across all runs | 14.3 s | + +The current `sources()` design builds and closes a filesystem per command +(`canfar/storage.py:93-96`), which is right for a CLI. A Python API should hand back a +*reusable* instance so callers pay service-binding discovery once. Note the outliers: +first-call cost is not deterministic, so do not document a fixed number. + +DOCUMENTED. `vosfs/filesystem.py:73-75` sets `protocol = "vos"` and `cachable = True`. +VERIFIED: `fsspec.filesystem("vos", endpoint_url=…, certfile=…)` twice returns the +identical object; `skip_instance_cache=True` defeats it. `canfar/storage.py:76,88` +already passes `skip_instance_cache=True`. **Keep that** for the returned instance: +fsspec's instance cache is keyed on the storage options, and a cached instance would +outlive a token refresh. + +## 5. Tiered caching: what genuinely composes + +### The composition matrix + +| Layering | Result | Evidence | +| --- | --- | --- | +| `SimpleCacheFileSystem(fs=vosfs_sync)` | **works**; cold 1.58 s → warm 0.000 s | VERIFIED | +| `WholeFileCacheFileSystem(fs=vosfs_sync)` | **works**; warm 0.000 s | VERIFIED | +| `CachingFileSystem` (blockcache) | **fails**, see §10 | VERIFIED | +| `fs.open(path, cache_type=…)` | **silently ignored** | VERIFIED | +| `filecache::simplecache::vos://` | builds; **middle layer is dead** | VERIFIED | +| hand-stacked `SimpleCache(SimpleCache(vosfs))` | **inner layer never fills** | VERIFIED | +| `cache_storage=[a, b]` | two-location lookup, **no promotion** | VERIFIED | +| `cache_storage="memory://…"` | **silently creates a local dir named `memory:`** | VERIFIED | +| `DirFileSystem(path=…, fs=vosfs)` | works, `async_impl=True` | VERIFIED | +| `SimpleCache(DirFileSystem(vosfs))` | works, 1.34 s cold | VERIFIED | +| `AsyncFileSystemWrapper(SimpleCache(vosfs_sync))` | works, non-blocking | VERIFIED | +| `SimpleCache(fs=vosfs_async)` | `RuntimeError: Loop is not running` | VERIFIED | + +### `cache_type=` is accepted and discarded + +VERIFIED. Every value in fsspec's cache registry +(`list(fsspec.caching.caches)` = `[None, 'none', 'mmap', 'bytes', 'readahead', +'blockcache', 'first', 'all', 'parts', 'background']`) was passed to +`vosfs.open(path, "rb", cache_type=…)`. All six tested values returned a +`StagedReadFile` with **no `.cache` attribute at all** and identical behaviour. No +warning, no error. + +DOCUMENTED, and this is why: `vosfs/filesystem.py:683-690` accepts `block_size` and +`cache_options` marked `# noqa: ARG002 - accepted, non-behavioural`, and +`vosfs/staging.py:59` defines `StagedReadFile(io.BufferedReader)` — a plain buffered +reader, not an `fsspec.spec.AbstractBufferedFile`. fsspec's read-caching machinery only +ever attaches to `AbstractBufferedFile`. **Never document `cache_type` as a `canfar` +tuning knob.** + +### URL chaining works, but only with `endpoint_url`, and the middle tier is dead + +VERIFIED. `filecache::vos://…` with only `vos={"certfile": …}` fails: + +```text +TypeError: VOSpaceFileSystem.__init__() missing 1 required positional argument: 'endpoint_url' +``` + +It succeeds when the endpoint is supplied in the target options: + +```python +fs, path = fsspec.core.url_to_fs( + f"simplecache::vos://{remote_path}", + vos={"endpoint_url": endpoint, "certfile": cert, "skip_instance_cache": True}, + simplecache={"cache_storage": cache_dir}, +) +``` + +cold 1.59 s → warm 0.000 s. + +VERIFIED, and this is the trap: `filecache::simplecache::vos://` **builds the right +object graph and then bypasses the middle layer**. After a cold read: + +```text +outer (filecache) dir: ['c6de39f6…'] +middle (simplecache) dir: [] <- never written +``` + +Wiping only the outer tier and re-reading took 1.52 s — a network round trip — and the +middle tier stayed empty. Hand-stacking `SimpleCacheFileSystem(fs=SimpleCacheFileSystem(fs=vosfs))` +reproduces it exactly: outer dir populated, inner dir empty. + +DOCUMENTED cause: `WholeFileCacheFileSystem` fills itself with +`self.fs.get(getpaths, storepaths)` / `self.fs.get_file(path, fn)` +(`fsspec/implementations/cached.py:673,706`). `get`/`get_file` on the inner cache +filesystem is a straight pass-through to the remote; it does not route through the +inner cache's own `_open`/`cat` path, so the inner cache is never populated. **Chained +cache layers are not a tiering mechanism in fsspec 2026.6.0.** + +### `cache_storage` as a list is the only real two-location mechanism + +DOCUMENTED. `fsspec/implementations/cached.py:87-91`: + +> `cache_storage: str or list(str)` — Location to store files. […] If a list, each +> location will be tried in the order given, but only the last will be considered +> writable. + +`fsspec/implementations/cached.py:140` runs `os.makedirs(storage[-1], exist_ok=True)`; +`SimpleCacheFileSystem._check_file` (`:820-826`) scans `self.storage` in order and +returns the first hit. + +VERIFIED with `cache_storage=[slow, fast]`: + +| Step | Result | +| --- | --- | +| cold read | 1.47 s; written to `fast` (the **last** entry) only | +| move the object to `slow`, empty `fast`, re-read | 0.037 s — served locally from `slow` | +| `fast` after that read | **empty — no promotion** | + +So a list gives you *lookup across two directories*, not a hot/cold hierarchy. It is +exactly right for "a shared, pre-populated, read-only cache behind my own writable +one", and it is wrong for "keep hot objects on the fast tier". + +### The RAM tier is not real + +VERIFIED and worth flagging as a hazard: `SimpleCacheFileSystem(fs=…, +cache_storage="memory://ram")` **constructs without error** and creates a literal local +directory named `memory:` in the process working directory. Nothing about the +`MemoryFileSystem` is involved. (This experiment created such a directory in the +repository root; it was removed and `git status` is clean.) + +VERIFIED alternatives: + +- `MemoryFileSystem` (`fsspec/implementations/memory.py`) is not a cache wrapper; its + `store` is a process-global `dict`. Hand-rolling `mem.pipe_file(key, fs.cat_file(p))` + works (warm read 0.0000 s) but is just a dictionary with extra ceremony, and it is + process-global mutable state. +- `MMapCache(location=None)` is a genuine anonymous-mmap RAM cache, but its fetcher + over `vosfs` downloads whole objects (§2), so it is a pessimization here. +- `/dev/shm` does not exist on this Darwin host (VERIFIED); on Linux — including CANFAR + Sessions — it is a tmpfs and is a perfectly good `cache_storage` value. I could not + test this on the available machine. + +**The honest RAM tier is the OS page cache sitting over your disk cache directory.** +A warm `simplecache` read measured 0.0002 s, which is memory speed, because the kernel +is already holding the file. Adding an explicit RAM layer buys nothing and costs +correctness. + +### Per-scenario recommendation + +| Scenario | Recommendation | +| --- | --- | +| **(a) Laptop, RAM only** | `filesystem(name, cache=False)` and hold bytes yourself via `fs.cat_file` / `fs.cat_ranges`. If you want a cache, use `cache=True` and let the page cache be the RAM tier. Do not construct `MemoryFileSystem` or `MMapCache`. | +| **(b) Laptop, RAM + second NVMe tier** | One `SimpleCacheFileSystem` pointed at the NVMe: `filesystem(name, cache="/mnt/nvme/canfar/")`. RAM tier = page cache. **Do not stack two cache layers** — the inner one is dead (VERIFIED above). | +| **(c) CANFAR Session, `/scratch` + RAM** | `filesystem(name, cache=True)`, which auto-selects `/scratch/canfar/`. `/scratch` is per-Session NVMe and is cleared at Session end, which is exactly a cache lifetime. If a shared pre-populated read-only cache exists, use `cache=["/shared/canfar-cache", "/scratch/canfar/"]` and document that hits in the shared tier are **not** promoted. | + +### Cache-location auto-detection: stdlib only + +**Do not add `platformdirs`.** VERIFIED: it is not in `pyproject.toml:48-65`. It +appears in `uv.lock` only as a transitive dependency of `mkdocs-get-deps` (`uv.lock:1256`) +and `virtualenv` (`uv.lock:2183`) — docs and dev groups, not runtime. Adding it to +runtime dependencies to pick a *cache* directory is backwards anyway: `platformdirs` +returns durable user-data/user-cache locations, and what is wanted here is ephemeral +scratch. `tempfile.gettempdir()` already honours `TMPDIR` and is one stdlib call. + +VERIFIED sketch (ran correctly on this host, falling back to the temp dir because +`/scratch` is absent): + +```python +def _cache_location(name: str) -> Path: + """Return the default cache directory for one Storage Identifier. + + Prefers the CANFAR Science Platform Server's per-Session ``/scratch`` volume, + which is fast local disk and is cleared when the Session ends. + + Args: + name: Storage Identifier of the configured VOSpace Service. + + Returns: + Path: Writable directory for cached objects. + """ + scratch = Path("/scratch") + root = scratch if scratch.is_dir() and os.access(scratch, os.W_OK) else Path(tempfile.gettempdir()) + return root / "canfar" / name +``` + +Keying by Storage Identifier matters: `fsspec`'s cache mapper hashes the *stripped* path +(`fsspec/implementations/cached.py:627`), which does not include the endpoint, so +`arc:/x` and `vault:/x` would otherwise collide in one directory. + +## 6. `WholeFileCacheFileSystem` vs `SimpleCacheFileSystem` + +**Recommend `SimpleCacheFileSystem`.** + +DOCUMENTED. `fsspec/implementations/cached.py:791-805`: `SimpleCacheFileSystem` +"only copies whole files, and does not keep any metadata about the download time or +file details. It is therefore safer to use in multi-threaded/concurrent situations." +fsspec's feature docs state "Only 'simplecache' is guaranteed thread/process-safe" +(). +Its `__init__` (`:811-818`) forces `cache_check`, `expiry_time` and `check_files` to +`False`. + +VERIFIED: both work; both give warm reads at 0.000 s. `WholeFileCacheFileSystem` keeps +a JSON metadata sidecar (a `cache` file appears alongside the objects) and computes +`self.fs.ukey(path)` on every miss (`fsspec/implementations/cached.py:634`), which is +an extra `info` round trip. A `/scratch` cache that dies with the Session does not need +expiry metadata. Expose `WholeFileCacheFileSystem` only if a user asks for expiry — do +not put it in the first API. + +## 7. "Get a file location" + +VERIFIED results for materializing a remote object to a local path: + +| Approach | Works over `vosfs`? | +| --- | --- | +| `fsspec.open_local("vos://…")` | **No.** `ValueError: open_local can only be used on a filesystem which has attribute local_file=True` | +| `fsspec.open_local("simplecache::vos://…", vos={"endpoint_url": …}, simplecache={"cache_storage": …})` | **Yes** — 1.64 s, returns the cache path | +| `fs.get_file(remote, local)` | **Yes** — writes the exact 169 920 B; explicit destination | +| `SimpleCacheFileSystem.open(path).name` | **Yes** — is the local cache path, exists on disk | +| `SimpleCacheFileSystem._check_file(path)` | **Yes** — returns the local path directly, `None` before the first read | +| `WholeFileCacheFileSystem._check_file(path)` | Returns a `(detail, path)` **tuple**, not a string | +| `LocalFileOpener` | Not reachable — it is `LocalFileSystem`'s own file class, never produced by a cache layer over `vosfs` | + +DOCUMENTED. `fsspec/core.py:544-549` is the `local_file` gate; +`fsspec/implementations/cached.py:560,808` set `local_file = True` on +`WholeFileCacheFileSystem` and `SimpleCacheFileSystem`. Bare `vosfs` sets nothing, hence +the failure. `WholeFileCacheFileSystem._open` (`:730-738`) returns a builtin +`open(fn, mode)` — a real `_io.BufferedReader` on a real local path — which is why +`.name` is usable. + +**Recommendation.** Expose `fetch()`. The two working routes are `fsspec.open_local` +with a chained URL — which requires callers to hand-assemble `vos://` strings and +`endpoint_url`, exactly the resolver footgun rejected in §3 — and `_check_file`, which +is private API. A four-line public function is cheaper than teaching either: + +```python +def fetch(name: str, path: str, *, cache: bool | str | Path = True) -> Path: + """Materialize one VOSpace object locally and return its path. + + Args: + name: Storage Identifier of the configured VOSpace Service. + path: Absolute path within that VOSpace Service. + cache: Cache location, or ``True`` for the default (see ``filesystem``). + + Returns: + Path: Local path holding the object's bytes. + """ + fs = filesystem(name, cache=cache or True) + fs.cat_file(path) # populates the cache + return Path(fs._check_file(path)) +``` + +If reaching into `_check_file` is unacceptable, use `fs.get_file(path, destination)` +with an explicit destination instead and drop the cache-path form — but then the caller +owns cleanup, and repeat calls re-download. Prefer the cache-backed form; the private +attribute is one call site in one file and is trivially covered by a test. + +## 8. Sync-first, async-first, or both + +**Sync-first, with `asynchronous=True` as a keyword. Not two classes, not two modules.** + +Reasons: + +1. Caching only exists in sync form (§4, §5). A sync-first API means the common case — + "read this FITS file, cache it on `/scratch`" — is one call with no wrapper. +2. Astronomy consumers of a materialized path (`astropy.io.fits.open`, + `numpy.load`, `pandas.read_*`) are synchronous. +3. The async path is genuinely better when uncached — native `VOSpaceFileSystem(..., + asynchronous=True)` was VERIFIED working for `_info` and `_ls` — so it must remain + available, and `sources()` depends on it. +4. `canfar` already ships sync/async parity elsewhere (`Session` / `AsyncSession`, + `AGENTS.md:58`), but that parity is for a hand-written client. Here both paths are + the same fsspec object with one constructor flag; two classes would be pure + duplication. + +### Relationship to `sources()` + +`sources()` moves to `canfar/storage.py` **unchanged**. Its contract with fsspec-cli is +`Callable[[], AbstractAsyncContextManager[AbstractFileSystem]]` +(`fsspec_cli/_app.py:41-43`), it must yield an async filesystem, and it must build and +`aclose()` per command so a listing cache cannot outlive its command +(`canfar/storage.py:93-96`, `docs/cli/data.md:106-110`). + +Refactor `_vospace` to delegate to the new `filesystem()` rather than duplicating +construction: + +```python +def _vospace(name, *, token=None, certificate=None) -> AsyncFilesystemSource: + @asynccontextmanager + async def source() -> AsyncIterator[AbstractFileSystem]: + fs = filesystem(name, token=token, certificate=certificate, asynchronous=True) + try: + yield fs + finally: + await fs.aclose() + return source +``` + +Do **not** put a cache layer behind `sources()`. `fsspec-cli` performs writes +(`cp`, `mkdir`, `rm`); `SimpleCacheFileSystem` has write semantics of its own +(`fsspec/implementations/cached.py:800-803`, `WriteCachedTransaction`) that were not +evaluated here, and a CLI that silently served stale bytes would be a defect. The CLI's +existing per-command listing cache is the right amount of caching for it. + +## 9. Recommended API sketch + +`canfar/storage.py` — three public functions, one private helper, no new dependency. +Every element below was exercised in the prototype (VERIFIED, §4/§5/§7). + +```python +"""Public access to configured VOSpace Services and the local filesystem.""" + +from __future__ import annotations + +import os +import tempfile +from contextlib import asynccontextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from fsspec.asyn import get_loop, sync +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper +from fsspec.implementations.cached import SimpleCacheFileSystem +from fsspec.implementations.local import LocalFileSystem + +from canfar.client import HTTPClient +from canfar.exceptions.context import AuthContextError +from canfar.models.config import Configuration + +_LISTINGS_EXPIRY_SECONDS = 30 +_LISTINGS_MAX_PATHS = 1000 + +CacheLocation = bool | str | Path | Sequence[str | Path] + + +def filesystem( + name: str, + *, + cache: CacheLocation = False, + asynchronous: bool = False, + token: str | SecretStr | None = None, + certificate: Path | None = None, +) -> AbstractFileSystem: + """Return an authenticated filesystem for one Storage Identifier. + + Args: + name: Storage Identifier of the configured VOSpace Service. + cache: ``False`` for no caching, ``True`` for the default cache + directory, a path for an explicit one, or a sequence of paths + tried in order where only the last is writable. + asynchronous: Return an async filesystem whose coroutine hooks are + awaited directly. Sync methods are unavailable on the result. + token: Runtime bearer token, preferred over any saved credential. + certificate: Runtime X.509 certificate path. + + Returns: + AbstractFileSystem: A ready filesystem rooted at the VOSpace Service. + + Raises: + AuthContextError: If no usable credential exists for the Identity + Provider that owns this VOSpace Service. + """ + + +def fetch( + name: str, + path: str, + *, + cache: CacheLocation = True, +) -> Path: + """Materialize one VOSpace object locally and return its path.""" + + +def sources() -> dict[str, AsyncFilesystemSource]: + """Build the mapped storage sources for one data command invocation.""" + # unchanged from canfar/storage.py:114-130 +``` + +Construction body, matching the rule table in §4: + +```python + config = Configuration() + endpoint, idp = config._resolve_storage(name) + try: + client = HTTPClient(config=config, authentication_idp=idp, url=endpoint, ...) + token_value, certfile = sync(get_loop(), client._materialize_credentials) + except (KeyError, OSError, TypeError, ValueError): + raise AuthContextError(idp, "Credential cannot be used. Run 'canfar login' for this IDP.") from None + + from vosfs import VOSpaceFileSystem + + credential = {"token": token_value} if token_value is not None else {"certfile": certfile} + remote = VOSpaceFileSystem( + endpoint, + asynchronous=asynchronous and not cache, + skip_instance_cache=True, + use_listings_cache=True, + listings_expiry_time=_LISTINGS_EXPIRY_SECONDS, + max_paths=_LISTINGS_MAX_PATHS, + **credential, + ) + if not cache: + return remote + storage = _cache_location(name) if cache is True else cache + cached = SimpleCacheFileSystem(fs=remote, cache_storage=_as_storage(storage)) + return AsyncFileSystemWrapper(cached, asynchronous=True) if asynchronous else cached +``` + +Notes on the shape: + +- `cache` is a single parameter, not a configurator object or a `CacheConfig` model. + A Pydantic model would be a data model with no persisted state and no validation + beyond "is this a path", which the filesystem call already performs. +- `asynchronous=True` with `cache` truthy must build the *sync* `vosfs` and wrap + (`asynchronous=asynchronous and not cache` above) — this is the single most + error-prone line and deserves a comment and a test. +- `skip_instance_cache=True` is retained deliberately (§4). +- No `Storage`/`VOSpace` class. Every method a caller could want is already on + `AbstractFileSystem`. + +### Usage the docs should show + +```python +from canfar.storage import fetch, filesystem + +# One-off read. +vault = filesystem("vault") +header = vault.cat_file("/ALMA/test-data/cutouts/test-4d-cube.fits", 0, 2880) + +# Repeated reads on a Science Platform Server Session: cache under /scratch. +vault = filesystem("vault", cache=True) +data = vault.cat_file("/ALMA/test-data/cutouts/test-4d-cube.fits") # 1.6 s +again = vault.cat_file("/ALMA/test-data/cutouts/test-4d-cube.fits") # 0.0002 s + +# Hand a local path to a library that only takes paths. +from astropy.io import fits +with fits.open(fetch("vault", "/ALMA/test-data/cutouts/test-4d-cube.fits")) as hdul: + ... + +# Concurrent async reads, uncached. +arc = filesystem("arc", asynchronous=True) +try: + sizes = await asyncio.gather(*(arc._info(p) for p in paths)) +finally: + await arc.aclose() +``` + +The `cat_file(path, 0, 2880)` in the first example is honest — it returns the right +bytes — but §2 means it costs a full download. Say so in the docs. + +## 10. Upstream gaps + +### Gap 1 — `blockcache` is unusable (root cause: no ranged reads) + +VERIFIED exact error, from `CachingFileSystem(fs=vosfs, cache_storage=…)` on both +`.open(path).read(100)` and `.cat_file(path)`: + +```text +AttributeError: 'StagedReadFile' object has no attribute 'blocksize' +``` + +DOCUMENTED cause: `fsspec/implementations/cached.py:702` reads +`block = getattr(f, "blocksize", 5 * 2**20)` defensively, but the failing access is on +the `CachingFileSystem` path, which assumes the target's file object is an +`AbstractBufferedFile` with `.blocksize` and `.cache`. `vosfs/staging.py:59` returns an +`io.BufferedReader` subclass instead. + +**What would unblock it, in order:** (1) OpenCADC Cavern implementing HTTP `Range` on +the negotiated data endpoint; (2) `vosfs` sending `Range` headers from `_cat_file`; +(3) `vosfs._open` returning an `fsspec.spec.AbstractBufferedFile` subclass backed by +those ranged reads. Adding a `blocksize` attribute alone would convert a loud failure +into a silent whole-object-per-block pessimization — do not ask for that. + +**Until then, `canfar` should not offer a block-cache option at all.** + +### Gap 2 — fsspec accepts a URL as `cache_storage` + +VERIFIED. `cache_storage="memory://ram"` is `os.makedirs`'d verbatim +(`fsspec/implementations/cached.py:140`), creating a directory named `memory:`. No +error, no warning. Worth an upstream issue; in the meantime, `canfar` should reject +any `cache` value containing `://` with a clear message. + +### Gap 3 — chained cache layers silently no-op + +VERIFIED (§5). `filecache::simplecache::X` constructs a three-layer stack in which the +middle layer never caches. Worth an upstream issue. `canfar` should never document +chained cache URLs. + +## 11. Documentation changes required + +If this API ships: + +- `docs/cli/data.md:176` — "This release intentionally provides no public CANFAR + storage Python API" must be updated; the FUSE mount, signed-URL, progress display and + the other listed omissions still stand. +- A new `docs/client/storage.md` for `canfar.storage`. + +**Independently of this API**, `docs/cli/data.md:143-165` is factually wrong and should +be corrected now: + +- `:143-144` "VOSpace supports ranged reads, so a large cube can be cached one block at + a time rather than downloaded whole" — VERIFIED false for `vosfs` 0.7.0. +- `:162-163` "Reading a FITS header this way transfers a single block instead of the + whole file, and a second read of the same range is served from `/scratch`" — the + second clause is true; the first is VERIFIED false. The `MMapCache` example moves + 3 542 400 B to return 2 880 B, and three scattered header reads cost three full + downloads (4.01 s versus 1.64 s for reading the file once). +- `:164-165` "Ranged reads help most on large files; on small ones the per-request + VOSpace transfer negotiation dominates" — the ordering is inverted. Ranged reads help + on *no* file size today; larger files are strictly worse. + +The correct advice for large-cube access with `vosfs` 0.7.0 is: read the object once +into a `simplecache` directory on `/scratch` and slice locally, or use `cat_ranges`, +which does group multiple ranges of one object into a single download +(`vosfs/filesystem.py:594-620`, VERIFIED: 4 ranges in 1.16 s). + +## 12. What I could not verify + +- **`/dev/shm` as `cache_storage` on a CANFAR Session.** `/dev/shm` does not exist on + this Darwin host (VERIFIED absent). The mechanism is a plain directory path so there + is no fsspec reason it would fail, but the tmpfs sizing and eviction behaviour on a + Science Platform Server Session is untested. +- **`/scratch` on a real Session.** Auto-detection was exercised only through its + fallback branch; `Path("/scratch").is_dir()` was `False` here. +- **Write paths through a cache layer.** All experiments were read-only against `vault` + by design. `SimpleCacheFileSystem`'s write/commit semantics + (`fsspec/implementations/cached.py:800-803`) were read, not executed. This is why §8 + recommends keeping caching out of `sources()`. +- **OIDC credentials.** Only the X.509 branch of `_materialize_credentials` was + exercised (`~/.ssl/cadcproxy.pem`). The OIDC-refresh branch + (`canfar/client.py:252-259`) is `async` and was not driven through + `fsspec.asyn.sync`; it should be covered by a test before release. +- **`arc`.** Every live measurement used `vault`. `arc` was not touched, per the + read-only constraint. +- **First-call latency stability.** Observed first-I/O costs ranged from 0.43 s to + 14.3 s with no reproducible pattern. Cause not established; do not publish a number. + +## Primary sources + +- `canfar/storage.py`, `canfar/cli/data.py`, `canfar/client.py`, + `canfar/models/config.py`, `pyproject.toml`, `uv.lock`, `CONTEXT.md`, `AGENTS.md`, + `docs/cli/data.md` (repository snapshot `ba5493fd`) +- `.venv/lib/python3.13/site-packages/vosfs/filesystem.py`, `.../vosfs/staging.py`, + `.../vosfs/_transfer.py` (vosfs 0.7.0) +- `.venv/lib/python3.13/site-packages/fsspec/implementations/cached.py`, + `.../fsspec/implementations/asyn_wrapper.py`, `.../fsspec/implementations/dirfs.py`, + `.../fsspec/implementations/memory.py`, `.../fsspec/caching.py`, `.../fsspec/core.py` + (fsspec 2026.6.0) +- `.venv/lib/python3.13/site-packages/fsspec_cli/_app.py` (fsspec-cli 0.6.0) +- [fsspec: Features — caching files locally](https://filesystem-spec.readthedocs.io/en/latest/features.html#caching-files-locally) +- [fsspec: Features — URL chaining](https://filesystem-spec.readthedocs.io/en/latest/features.html#url-chaining) +- [fsspec: Async](https://filesystem-spec.readthedocs.io/en/latest/async.html) +- [vosfs v0.7.0](https://github.com/shinybrar/vosfs/releases/tag/v0.7.0) +- [fsspec-cli v0.6.0](https://github.com/shinybrar/vosfs/releases/tag/fsspec-cli-v0.6.0) diff --git a/docs/agents/upstream-provenance.md b/docs/agents/upstream-provenance.md new file mode 100644 index 00000000..ab6b20ce --- /dev/null +++ b/docs/agents/upstream-provenance.md @@ -0,0 +1,30 @@ +# Upstream provenance + +`canfar` depends on two upstream distributions that are **pinned to tagged Git +refs on a personal fork**, not to PyPI releases. Record the pin and its basis +here whenever it moves. + +## Current pins + +| Distribution | Pin | Source | +| --- | --- | --- | +| `vosfs` | `v0.8.0` | `git+https://github.com/shinybrar/vosfs@v0.8.0` | +| `fsspec-cli` | `fsspec-cli-v0.7.0` | same repository, `subdirectory=src/fsspec-cli` | + +## Why this matters + +- The source is a personal repository, so the usual PyPI ownership and + yanking guarantees do not apply. Review the tag before moving a pin. +- Tags are immutable by convention only. `uv.lock` records the resolved commit, + which is the real integrity anchor. +- `vosfs` v0.8.0 added HTTP `Range` support, honoured by the `vault` byte + endpoint and not by Cavern behind `arc`. Behaviour therefore differs per + Storage Identifier; see [Data Access](../client/data.md). + +## History + +- `v0.6.0` / `fsspec-cli-v0.5.0` — first integration, audited at upstream + PR #294, commit `9e5314db4706894d31d54d245392f43b9556cfbb`. +- `v0.7.0` / `fsspec-cli-v0.6.0` — Typer-owned commands, a breaking upstream + change to command parsing. +- `v0.8.0` / `fsspec-cli-v0.7.0` — server-side `Range` support (current). diff --git a/docs/cli/cli-help.md b/docs/cli/cli-help.md index dd4e9ca1..0f3f93db 100644 --- a/docs/cli/cli-help.md +++ b/docs/cli/cli-help.md @@ -184,6 +184,18 @@ canfar stats Use `canfar image --help` and `canfar stats --help` for command-specific options. +## Data + +```bash +canfar data ls -lh arc:/home/[username] +canfar data cp local:/absolute/path/file.fits arc:/home/[username]/file.fits +``` + +Data operands use explicit `storage-identifier:/absolute/path` or +`local:/absolute/path` syntax. See [Data commands](data.md) for recursive copy, +cross-source copy and verification, recursive-removal policy, and the exact +supported boundary. + ## Client configuration ```bash diff --git a/docs/cli/data.md b/docs/cli/data.md new file mode 100644 index 00000000..322af2bb --- /dev/null +++ b/docs/cli/data.md @@ -0,0 +1,219 @@ +# Data commands + +Use `canfar data` to work with configured VOSpace Services and the local +filesystem through the embedded `fsspec-cli` command application. + +## Install and authenticate + +Data commands are included in the standard installation: + +```bash +pip install canfar +canfar login cadc +``` + +## Address mapped sources + +Every operand starts with a Storage Identifier: the handle of a configured +VOSpace Service, or the reserved `local` identifier for the machine where the +command runs. Operands always pair an identifier with an absolute path: + +```text +storage-identifier:/absolute/path +local:/absolute/path +``` + +Every mapped source is available concurrently, regardless of the active Server +Selection. A default CADC login maps two VOSpace Services, `arc` and `vault`, +plus `local`: + +```bash +canfar data ls -lh arc:/ +canfar data ls -lh arc:/home/[username] +canfar data ls -lh vault:/ +``` + +Use `ls -lh`, or the `ll -h` long-form command, for a human-readable listing. +The `-h` flag reports human-readable sizes and requires a long listing, so +`ls -h` on its own exits with `ls: -h: requires long listing`. + +Operands must name a mapped source. Empty `:/path`, bare local paths such as +`/tmp/file`, and `active:/path` are all rejected; there is no `active` alias +and no `canfar storage` command. + +## Copy files and directories + +Copy one file between local and remote sources: + +```bash +canfar data cp local:/absolute/path/file.fits arc:/home/[username]/file.fits +canfar data cp arc:/home/[username]/file.fits local:/absolute/path/file.fits +``` + +Copy between two VOSpace Services, for example a public test cutout from +`vault` into your `arc` home directory: + +```bash +canfar data cp vault:/ALMA/test-data/cutouts/test-4d-cube-cutout.fits arc:/home/[username]/test-4d-cube-cutout.fits +canfar data ls -lh arc:/home/[username]/test-4d-cube-cutout.fits +``` + +Recursive copy is enabled for admitted local and remote source pairs: + +```bash +canfar data cp -R local:/absolute/path/dataset arc:/home/[username]/dataset +``` + +The tagged upstream implementation builds a bounded manifest, copies files +through host-local staging when sources differ, and verifies destination +metadata. Recursive copy is not atomic and does not create a snapshot; inspect +the destination before removing any source data. + +## Move data between sources + +Cross-source `mv` is unsupported and exits with status 2: + +```text +mv: cross-source move unsupported +``` + +Move a file between sources explicitly by copying it, verifying the +destination, and only then issuing a separate source removal: + +```bash +canfar data cp vault:/folder/file.fits arc:/home/[username]/file.fits +canfar data ls -lh arc:/home/[username]/file.fits +canfar data rm vault:/folder/file.fits +``` + +Do not use this sequence as a one-command or atomic move. + +Recursive removal is disabled by application policy, so `rm` accepts no `-R` or +`-r` flag at all and exits with status 2: + +```text +No such option: -R +``` + +Because recursive directory removal is disabled, directory movement is not a +supported workflow in this release. Any future one-command relocation would be a +separately named, opt-in orchestration feature with stronger destination +verification and residual-state semantics—not portable `mv`. + +## Cache data locally + +### Directory listings + +Directory listings are cached automatically. Each command builds and closes its +own filesystem, so a cached listing only ever serves the command that produced +it and can never return a listing that outlives it. Repeated lookups while one +command walks a tree are served without another round trip. + +### Files and byte ranges + +There is no CLI flag for caching file contents, but `vosfs` is a normal +[fsspec](https://filesystem-spec.readthedocs.io/) filesystem, so any fsspec +cache can wrap it from Python. On a CANFAR session, `/scratch` is fast local +disk and is the right place to point a cache; it is not backed up and is +cleared when the session ends, which is exactly what a cache wants. + +Cache whole files under a named directory. The first read fetches over the +network, and later reads come from `/scratch`: + +```python +from pathlib import Path + +from fsspec.implementations.cached import WholeFileCacheFileSystem +from vosfs import VOSpaceFileSystem + +vault = VOSpaceFileSystem( + "https://cadc-west-01.canfar.net/vault", + certfile=str(Path.home() / ".ssl" / "cadcproxy.pem"), +) +cached = WholeFileCacheFileSystem(fs=vault, cache_storage="/scratch/vault-cache") + +data = cached.cat_file("/ALMA/test-data/cutouts/test-4d-cube-cutout.fits") +``` + +Use `SimpleCacheFileSystem` instead when you do not need the expiry and +staleness metadata that `WholeFileCacheFileSystem` keeps. Passing +`cache_storage` a list of directories tries each in order and treats only the +last as writable, so a shared read-only cache can back your own. + +### Cache byte ranges + +`vosfs` sends an HTTP `Range` header and uses the response when the byte +endpoint answers `206`, so a partial read such as `cat_file(path, start, end)` +transfers only the bytes you asked for. Range support is per-backend: + +| Storage Identifier | Backend | Ranged reads | +| --- | --- | --- | +| `vault` | `minoc` | Yes — a partial read returns `206` and transfers only that slice | +| `arc` | Cavern | No — the whole object is fetched and sliced, which is correct but not cheaper | + +Because a range is now a real partial transfer against `vault`, a block cache +is worth using there. `MMapCache` keeps fetched blocks in a sparse file, so +only the blocks you touch occupy disk: + +```python +from fsspec.caching import MMapCache + +path = "/ALMA/test-data/cutouts/test-4d-cube.fits" +size = vault.info(path)["size"] +blocks = MMapCache( + blocksize=1 << 20, + fetcher=lambda start, end: vault.cat_file(path, start, end), + size=size, + location="/scratch/vault-cache/test-4d-cube.blocks", +) + +header = blocks._fetch(0, 2880) # one FITS header block, one 1 MiB range request +``` + +Reading a FITS header from a 3.4 MB cube this way issues a single ranged +request and materialises one block of four; a second read of the same range is +served from `/scratch`. The saving is in bytes transferred rather than seconds +on small files, because VOSpace transfer negotiation dominates a short request. +It grows with file size, and matters most when many reads hit different parts +of one large cube. + +Against `arc` a block cache still costs a whole download per block, so cache +whole files there instead. + +The `blockcache` filesystem remains unavailable over a VOSpace Service. `Range` +is honoured for byte reads, not through the file-object path, so wrapping +`CachingFileSystem` still fails: + +```text +AttributeError: 'StagedReadFile' object has no attribute 'blocksize' +``` + +Stacked caches do not help either: chaining them (`filecache::simplecache::`) +builds the layers, but the inner layer is never filled and never serves, so use +exactly one cache layer on your fastest local disk. + +These caches use the synchronous filesystem interface. Build the filesystem +without `asynchronous=True`, as above. + +## Output and accepted omissions + +Data command stdout belongs to the embedded command; CANFAR does not prepend +the active-Server banner or add JSON/YAML envelopes. Diagnostics are written to +stderr. + +The Python equivalent of these commands is documented in +[Data Access](../client/data.md). + +This release intentionally provides no FUSE mount, signed-URL extension, +progress display, confirmation prompt, `:/path` or bare-path shorthand, +`active` alias, `canfar storage` alias, recursive removal, or cross-source `mv` +workflow. + +## Upstream releases + +CANFAR installs pinned, tagged releases of +[`vosfs`](https://github.com/shinybrar/vosfs/releases/tag/v0.8.0) and +[`fsspec-cli`](https://github.com/shinybrar/vosfs/releases/tag/fsspec-cli-v0.7.0). +CANFAR tests its composition, configuration, authentication, and output seams; +exhaustive filesystem-command and backend matrices remain in the upstream +project. diff --git a/docs/cli/quick-start.md b/docs/cli/quick-start.md index dcfb005e..d5658afb 100644 --- a/docs/cli/quick-start.md +++ b/docs/cli/quick-start.md @@ -27,7 +27,27 @@ canfar auth show canfar server ls ``` -## 3. Create a notebook +## 3. Work with data + +The standard installation includes data commands. Use a configured Storage Identifier +or the reserved `local` name with an absolute path. A default CADC login maps +`arc` and `vault`: + +```bash +canfar data ls -lh arc:/home/[username] +canfar data cp local:/absolute/path/file.fits arc:/home/[username]/file.fits +``` + +Cross-source `mv` is unsupported. Copy the file, verify the destination, and +then remove the source with a separate command: + +```bash +canfar data cp vault:/folder/file.fits arc:/home/[username]/file.fits +canfar data ls -lh arc:/home/[username]/file.fits +canfar data rm vault:/folder/file.fits +``` + +## 4. Create a notebook ```bash canfar create notebook skaha/astroml:latest @@ -40,7 +60,7 @@ registry. Use the full image name when you want to be explicit: canfar create notebook images.canfar.net/skaha/astroml:latest ``` -## 4. Check status +## 5. Check status ```bash canfar ps @@ -53,7 +73,7 @@ Use machine output in scripts: canfar ps --json ``` -## 5. Open the notebook +## 6. Open the notebook ```bash canfar open $(canfar ps -q) @@ -61,14 +81,14 @@ canfar open $(canfar ps -q) Notebook Sessions usually take 60-120 seconds to become reachable. -## 6. Inspect startup events +## 7. Inspect startup events ```bash canfar events $(canfar ps -q) canfar logs $(canfar ps -q) ``` -## 7. Clean up +## 8. Clean up ```bash canfar delete $(canfar ps -q) diff --git a/docs/client/data.md b/docs/client/data.md new file mode 100644 index 00000000..78c6377d --- /dev/null +++ b/docs/client/data.md @@ -0,0 +1,288 @@ +# Data Access + +Read and write CANFAR VOSpace Services from Python. `canfar` resolves the +endpoints and credentials; [`vosfs`](https://github.com/shinybrar/vosfs) is the +[fsspec](https://filesystem-spec.readthedocs.io/) filesystem that talks to them, +so every tool that already speaks fsspec — astropy, pandas, dask, zarr — works +without an adapter. + +The same Storage Identifiers the CLI uses are importable by name. + +## Open a VOSpace Service + +Import a Storage Identifier and you get a ready, authenticated filesystem. Run +`canfar login` first; the credential resolution is the same one the CLI uses. + +```python +from canfar.storage import arc, vault, local +``` + +Any configured Storage Identifier works this way. To see which are available: + +```python +from canfar.storage import identifiers + +identifiers() # ['arc', 'vault', 'local'] +``` + +Import binds the filesystem once, which is what you usually want. To build one +explicitly — to override a credential, or to name an identifier held in a +variable — use `filesystem`: + +```python +from canfar.storage import filesystem + +vault = filesystem("vault") +staging = filesystem("vault", token="...") # runtime bearer token +archive = filesystem("arc", certificate="/path/to/proxy.pem") +``` + +`local` is reserved for the machine your code runs on and needs no credential. + +## Filesystem operations + +The object is a standard fsspec filesystem, so the usual verbs apply: + +```python +cutouts = "/ALMA/test-data/cutouts" +target = f"{cutouts}/test-4d-cube-cutout.fits" + +vault.ls(cutouts, detail=False) # ['/ALMA/.../test-4d-cube-cutout.fits', ...] +vault.info(target)["size"] # 169920 +vault.exists(target) # True +vault.isdir(cutouts) # True +vault.glob(f"{cutouts}/*cutout.fits") +vault.find(cutouts) # recursive listing +vault.du(cutouts) # 3712320 +``` + +Reads come in whole-object, ranged, and file-like forms: + +```python +whole = vault.cat_file(target) # 169920 bytes +header = vault.cat_file(target, 0, 2880) # first 2880 bytes only +first, second = vault.cat_ranges([target, target], [0, 100], [80, 180]) + +with vault.open(target, "rb") as handle: + handle.read(80) + +vault.head(target, 100) +vault.tail(target, 100) +``` + +Writes use `put_file`, `pipe_file`, `mkdir`, and `rm`. Directory listings are +cached in memory for the lifetime of the filesystem object, so a long-lived +object can serve a stale listing; build a fresh one, or pass +`use_listings_cache=False`, when you need to observe another writer's changes. + +## Get a local path + +Some libraries want a real path rather than a file object — anything that +memory-maps, or a C extension that opens by name. Materialise the file: + +```python +vault.get_file(target, "/scratch/cutout.fits") +``` + +`get_file` is the standard fsspec verb; `put_file` is its counterpart for +uploads. + +## Cache + +Nothing is cached to disk unless you ask. On a CANFAR session `/scratch` is +fast local NVMe, is not backed up, and is cleared when the session ends, which +is exactly what a cache wants. Locally, any directory works. + +### Whole files + +```python +from fsspec.implementations.cached import WholeFileCacheFileSystem + +cached = WholeFileCacheFileSystem(fs=vault, cache_storage="/scratch/vault-cache") + +cached.cat_file(target) # cold: fetched over the network +cached.cat_file(target) # warm: served from /scratch +``` + +A cold read of the 166 KiB cutout took 2.96 s; the warm read took 0.4 ms. + +Use `SimpleCacheFileSystem` when you do not need the expiry and staleness +metadata `WholeFileCacheFileSystem` keeps. Passing `cache_storage` a list of +directories tries each in order and treats only the last as writable, so a +shared read-only cache can back your own. + +### Byte ranges + +`vosfs` sends an HTTP `Range` header and uses the response when the byte +endpoint answers `206`, so a partial read transfers only the bytes you asked +for. Support is per-backend: + +| Storage Identifier | Backend | Ranged reads | +| --- | --- | --- | +| `vault` | `minoc` | Yes — a partial read returns `206` and transfers only that slice | +| `arc` | Cavern | No — the whole object is fetched and sliced, correct but not cheaper | + +Against `vault`, cache blocks instead of whole files when you touch small parts +of a large cube. `MMapCache` keeps fetched blocks in a sparse file: + +```python +from fsspec.caching import MMapCache + +cube = "/ALMA/test-data/cutouts/test-4d-cube.fits" +size = vault.info(cube)["size"] + +blocks = MMapCache( + blocksize=1 << 20, + fetcher=lambda start, end: vault.cat_file(cube, start, end), + size=size, + location="/scratch/vault-cache/cube.blocks", +) + +header = blocks._fetch(0, 2880) # one ranged request, one block of four +``` + +Reading a FITS header from the 3.4 MB cube materialises one block of four. The +saving is in bytes transferred rather than seconds on small files, because +VOSpace transfer negotiation dominates a short request; it grows with file +size. Against `arc` a block cache still costs a whole download per block, so +cache whole files there. + +### RAM + +Omit `location` and `MMapCache` uses an anonymous memory map — a RAM cache that +never touches disk, for when `/scratch` is absent or you want the data gone +when the process exits: + +```python +blocks = MMapCache( + blocksize=1 << 20, + fetcher=lambda start, end: vault.cat_file(cube, start, end), + size=size, +) +``` + +A warm re-read of a cached range returned in 18 µs. + +### What does not work + +`blockcache` (`CachingFileSystem`) cannot wrap a VOSpace Service. `Range` is +honoured for byte reads, not through the file-object path: + +```text +AttributeError: 'StagedReadFile' object has no attribute 'blocksize' +``` + +Stacked caches do not help either: chaining them (`filecache::simplecache::`) +builds the layers, but the inner layer is never filled and never serves. Use +exactly one cache layer, on your fastest local disk. + +## Scientific tools + +### astropy + +Read a header without downloading the file, using one ranged request: + +```python +from astropy.io import fits + +raw = vault.cat_file(target, 0, 2880) +header = fits.Header.fromstring(raw.decode("latin-1")) +header["NAXIS"], header["OBJECT"] # 4, 'hers1' +``` + +Or hand the file object straight to astropy: + +```python +with vault.open(target, "rb") as handle, fits.open(handle) as hdul: + hdul[0].data.shape # (1, 96, 26, 16) +``` + +To memory-map, materialise the file first — `memmap=True` needs a real path: + +```python +vault.get_file(target, "/scratch/cutout.fits") + +with fits.open("/scratch/cutout.fits", memmap=True) as hdul: + data = hdul[0].data # paged in on demand, not loaded up front +``` + +### numpy + +```python +import numpy as np + +raw = vault.cat_file(target) +values = np.frombuffer(raw[2880:2880 + 64], dtype=">f4") +``` + +### pandas + +Astronomy tables usually arrive as FITS rather than CSV. Read one remotely and +convert: + +```python +from astropy.table import Table + +with vault.open("/APASS/north/091106/n091106.0101.cat", "rb") as handle: + table = Table.read(handle, format="fits") + +frame = table.to_pandas() # 2373 rows +frame.columns[:3] # ['NUMBER', 'MAG_AUTO', 'MAGERR_AUTO'] +``` + +For delimited text, pass the file object to pandas directly: + +```python +import pandas as pd + +with vault.open("/path/to/table.csv", "rb") as handle: + frame = pd.read_csv(handle) +``` + +### dask + +Memory-map a materialised cube and chunk it, so only the blocks a computation +touches are paged in: + +```python +import dask.array as da +from astropy.io import fits + +with fits.open("/scratch/cutout.fits", memmap=True) as hdul: + array = da.from_array(hdul[0].data, chunks=(1, 24, 26, 16)) + array.mean().compute() +``` + +## Async + +Every read has an async twin. Build the filesystem with `asynchronous=True`, +use the underscore-prefixed coroutines, and close it when done: + +```python +import asyncio + +from canfar.models.config import Configuration +from vosfs import VOSpaceFileSystem + + +async def main() -> None: + config = Configuration() + service = config.servers["canfar"].storage["vault"] + vault = VOSpaceFileSystem( + str(service.url), + certfile=str(config.get_credential("cadc").path), + asynchronous=True, + ) + try: + entries = await vault._ls(cutouts, detail=False) + header = await vault._cat_file(cube, 0, 2880) + finally: + await vault.aclose() + + +asyncio.run(main()) +``` + +Async filesystems cannot open file objects — `open()` is rejected, and the +fsspec caches are synchronous — so use the synchronous form for caching and for +libraries that want a file handle. diff --git a/docs/presentations/srcnet-june-2026-demo.typ b/docs/presentations/srcnet-june-2026-demo.typ index a554f59e..b7c6b534 100644 --- a/docs/presentations/srcnet-june-2026-demo.typ +++ b/docs/presentations/srcnet-june-2026-demo.typ @@ -90,17 +90,17 @@ for node in ["canSRC", "sweSRC"]: await session.create(image="skaha/base-notebook:latest", cmd="env") ``` -== Next: storage, same front door +== Data, same front door -Coming in #link("https://github.com/opencadc/canfar/pull/83")[#raw("#83")] — file management over VOSpace, no second login: +File management over named VOSpace Services, using their configured Authentication Records: ```bash -canfar server use sweSRC -canfar storage ls -canfar storage mv /data/filename @canSRC:/data/ # Syntax TBD +canfar data ls -lh canSRC:/ +canfar data cp local:/absolute/path/filename canSRC:/data/filename ``` -- Familiar verbs: `ls` · `cp` · `mkdir` · `mv` · `rm` -- Reuses your active identity — *no extra auth* +- Cross-source movement is explicit `cp`, verify, then separate `rm` +- Each VOSpace Service uses its parent Science Platform Server's IDP +- CANFAR uses that IDP's saved Authentication Record, even when the Science Platform Server is inactive #align(center)[_Your files, your compute, your platform — one door, one login._] diff --git a/mkdocs.yml b/mkdocs.yml index c0e93c09..a7df070d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -248,6 +248,7 @@ nav: - Examples: - Basic: client/examples.md - Advanced: client/advanced-examples.md + - Data Access: client/data.md - Reference: - Python API: - AsyncSession: client/async_session.md @@ -259,6 +260,7 @@ nav: - Helpers: client/helpers.md - CLI: - 5-Minute Tutorial: cli/quick-start.md + - Data Commands: cli/data.md - Authentication and Servers: cli/authentication-contexts.md - Logging: cli/logging.md - Reference: cli/cli-help.md @@ -277,6 +279,7 @@ nav: - Domain Docs: agents/domain.md - Issue Tracker: agents/issue-tracker.md - Triage Statuses: agents/triage-labels.md + - Upstream Provenance: agents/upstream-provenance.md - Operations: - Platform Deployments: https://www.opencadc.org/deployments/ - Security Issues: security.md diff --git a/pyproject.toml b/pyproject.toml index 48b07818..3dbf489a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,9 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel.force-include] "canfar/utils/data/scientists.txt" = "canfar/utils/data/scientists.txt" +[tool.hatch.metadata] +allow-direct-references = true + [project] name = "canfar" version = "1.4.1" @@ -56,6 +59,8 @@ dependencies = [ "rich>=13.9.4", "segno>=1.6.6", "typer>=0.16.0", + "vosfs @ git+https://github.com/shinybrar/vosfs@v0.8.0", + "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.7.0#subdirectory=src/fsspec-cli", ] [tool.uv] @@ -228,6 +233,9 @@ docstring-code-line-length = "dynamic" # ty configuration (replaces mypy) [tool.ty] +[tool.ty.rules] +unused-ignore-comment = "ignore" + [tool.ty.environment] python-version = "3.10" diff --git a/tests/test_auth_x509.py b/tests/test_auth_x509.py index 2bac41ee..d59c058f 100644 --- a/tests/test_auth_x509.py +++ b/tests/test_auth_x509.py @@ -130,6 +130,18 @@ def test_expiry_not_yet_valid() -> None: x509_auth.expiry(cert_path) +def test_expiry_exposes_expiration_time(tmp_path) -> None: + """Expired certificates expose their end time for human-facing diagnostics.""" + cert_path = tmp_path / "expired.pem" + generate_cert(cert_path, expired=True) + + with pytest.raises(x509_auth.CertificateError) as excinfo: + x509_auth.expiry(cert_path) + + assert excinfo.value.expired_at is not None + assert excinfo.value.expired_at < datetime.datetime.now(datetime.timezone.utc) + + def test_expiry_with_invalid_content() -> None: """Test that `expiry` raises ValueError for a file with invalid content.""" with tempfile.NamedTemporaryFile(mode="w", suffix=".pem") as temp_cert: diff --git a/tests/test_cli_data.py b/tests/test_cli_data.py new file mode 100644 index 00000000..0ee1c871 --- /dev/null +++ b/tests/test_cli_data.py @@ -0,0 +1,297 @@ +"""Tests for the embedded upstream data command group.""" + +from __future__ import annotations + +import importlib +import sys +from contextlib import asynccontextmanager +from functools import partial +from types import SimpleNamespace +from typing import TYPE_CHECKING +from unittest.mock import patch + +import click +import pytest +import typer +from typer.testing import CliRunner + +import canfar.cli.data as data_cli +from canfar import storage +from canfar.cli.main import cli + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Mapping + from pathlib import Path + + from fsspec.spec import AbstractFileSystem + from fsspec_cli import AsyncFilesystemSource + +runner = CliRunner() + + +def _configuration(*storage_names: str) -> SimpleNamespace: + storage = dict.fromkeys(storage_names, object()) + return SimpleNamespace( + servers={"server": SimpleNamespace(storage=storage)}, + storage_identifiers=lambda: [*storage_names, "local"], + ) + + +def test_root_help_advertises_data() -> None: + """The standard CANFAR command surface contains the data group.""" + result = runner.invoke(cli, ["--help"]) + + assert result.exit_code == 0 + assert "data" in result.output + + +def test_upstream_app_receives_configured_sources_and_policy(monkeypatch) -> None: + """Every app build maps all configured Servers plus local, not only active.""" + captured_sources: list[dict[str, object]] = [] + captured_capabilities: list[object] = [] + + def source_factory(_name: str) -> object: + return object() + + configurations = iter( + [ + SimpleNamespace( + active=SimpleNamespace(server="first"), + servers={ + "first": SimpleNamespace(storage={"arc": object()}), + "second": SimpleNamespace(storage={"cavern": object()}), + }, + storage_identifiers=lambda: ["arc", "cavern", "local"], + ), + SimpleNamespace( + active=SimpleNamespace(server="second"), + servers={ + "first": SimpleNamespace(storage={"arc": object()}), + "second": SimpleNamespace( + storage={"cavern": object(), "vault": object()} + ), + }, + storage_identifiers=lambda: ["arc", "cavern", "vault", "local"], + ), + ] + ) + + def configuration() -> SimpleNamespace: + return next(configurations) + + class FakeApp: + def __init__( + self, + sources: Mapping[str, AsyncFilesystemSource], + *, + capabilities: object, + ) -> None: + captured_sources.append(dict(sources)) + captured_capabilities.append(capabilities) + self.typer_app = typer.Typer() + + monkeypatch.setattr(storage, "Configuration", configuration) + monkeypatch.setattr(storage, "_vospace", source_factory) + monkeypatch.setattr(data_cli, "App", FakeApp) + + data_cli.group() + data_cli.group() + + assert [set(sources) for sources in captured_sources] == [ + {"arc", "cavern", "local"}, + {"arc", "cavern", "vault", "local"}, + ] + assert all( + sources["local"] is storage._local # noqa: SLF001 + for sources in captured_sources + ) + assert captured_capabilities == [ + {"recursion": {"copy": True, "remove": False}}, + {"recursion": {"copy": True, "remove": False}}, + ] + + +@pytest.mark.asyncio +async def test_local_storage_returns_fresh_async_wrappers() -> None: + """Each local source entry owns a fresh asynchronous wrapper.""" + async with storage._local() as first: # noqa: SLF001 + assert first.asynchronous is True + async with storage._local() as second: # noqa: SLF001 + assert second.asynchronous is True + + assert first is not second + assert first.sync_fs is not second.sync_fs + + +def test_data_cat_delegates_without_active_server_banner( + monkeypatch, + tmp_path: Path, +) -> None: + """Data stdout belongs byte-for-byte to the upstream command.""" + payload = tmp_path / "payload.txt" + payload.write_text("upstream output\n", encoding="utf-8") + monkeypatch.setattr(storage, "Configuration", _configuration) + + result = runner.invoke(cli, ["data", "cat", f"local:{payload}"]) + + assert result.exit_code == 0, result.output + assert result.stdout == "upstream output\n" + + +def test_data_long_listing_delegates_unchanged( + monkeypatch, + tmp_path: Path, +) -> None: + """The documented ``ls -lh name:/path`` form reaches upstream unchanged.""" + (tmp_path / "payload.txt").write_text("listed", encoding="utf-8") + monkeypatch.setattr( + storage, + "Configuration", + partial(_configuration, "canSRC"), + ) + monkeypatch.setattr( + storage, + "_vospace", + lambda _name: storage._local, # noqa: SLF001 + ) + + result = runner.invoke(cli, ["data", "ls", "-lh", f"canSRC:{tmp_path}"]) + + assert result.exit_code == 0, result.output + assert "payload.txt" in result.stdout + + +def test_data_recursive_copy_delegates_to_released_contract( + monkeypatch, + tmp_path: Path, +) -> None: + """The embedded app retains its enabled recursive-copy behavior.""" + source = tmp_path / "source" + source.mkdir() + (source / "payload.txt").write_text("copied", encoding="utf-8") + destination = tmp_path / "destination" + monkeypatch.setattr( + storage, + "Configuration", + partial(_configuration, "canSRC"), + ) + monkeypatch.setattr( + storage, + "_vospace", + lambda _name: storage._local, # noqa: SLF001 + ) + + result = runner.invoke( + cli, + ["data", "cp", "-R", f"local:{source}", f"canSRC:{destination}"], + ) + + assert result.exit_code == 0, result.output + assert (destination / "payload.txt").read_text(encoding="utf-8") == "copied" + + +@pytest.mark.parametrize("flag", ["-R", "-r"]) +def test_data_recursive_remove_is_disabled( + monkeypatch, + tmp_path: Path, + flag: str, +) -> None: + """CANFAR keeps upstream recursive removal source-free and disabled. + + Upstream ``fsspec-cli`` v0.6.0 gave parsing to Typer, so the disabled + ``recursion.remove`` capability now omits the flag from ``rm`` entirely + instead of registering it and rejecting the invocation at runtime. + """ + monkeypatch.setattr(storage, "Configuration", _configuration) + + result = runner.invoke(cli, ["data", "rm", flag, f"local:{tmp_path}"]) + + assert result.exit_code == 2 + assert result.stdout == "" + + +def test_recursion_capability_omits_rm_flags_but_keeps_cp_flags( + monkeypatch, +) -> None: + """The recursion capability registers ``cp`` flags and withholds ``rm`` ones.""" + monkeypatch.setattr(storage, "Configuration", _configuration) + group = data_cli.group() + context = click.Context(group) + + def options(command: str) -> set[str]: + resolved = group.get_command(context, command) + assert resolved is not None + return {option for param in resolved.params for option in param.opts} + + assert not options("rm") & {"-R", "-r"} + assert {"-R", "-r"} <= options("cp") + + +def test_data_cross_source_move_retains_upstream_rejection(monkeypatch) -> None: + """CANFAR does not reinterpret or implement cross-source move.""" + + @asynccontextmanager + async def unused_source() -> AsyncIterator[AbstractFileSystem]: + msg = "source-free rejection must not acquire filesystems" + raise AssertionError(msg) + yield + + monkeypatch.setattr(storage, "Configuration", partial(_configuration, "arc")) + monkeypatch.setattr(storage, "_vospace", lambda _name: unused_source) + + result = runner.invoke(cli, ["data", "mv", "arc:/a", "local:/b"]) + + assert result.exit_code == 2 + assert result.stdout == "" + assert result.stderr == "mv: cross-source move unsupported\n" + + +@pytest.mark.parametrize("operand", [":/path", "/bare/local/path"]) +def test_data_deprecated_operand_grammar_is_unsupported( + monkeypatch, + operand: str, +) -> None: + """Only the upstream explicit ``name:/absolute/path`` grammar is accepted.""" + monkeypatch.setattr(storage, "Configuration", _configuration) + + result = runner.invoke(cli, ["data", "ls", operand]) + + assert result.exit_code != 0 + + +@pytest.mark.parametrize( + ("arguments", "diagnostic"), + [ + (["storage", "ls"], "No such command 'storage'"), + (["data", "ls", "active:/"], "unknown filesystem"), + (["data", "ls", "-h", "local:/"], "-h: requires long listing"), + ], +) +def test_data_retired_aliases_and_standalone_h_are_unsupported( + monkeypatch, + arguments: list[str], + diagnostic: str, +) -> None: + """Retired host aliases do not widen the upstream mapped grammar.""" + monkeypatch.setattr(storage, "Configuration", _configuration) + + result = runner.invoke(cli, arguments) + + assert result.exit_code == 2 + assert diagnostic in result.stderr + + +def test_importing_data_module_does_not_load_configuration() -> None: + """Registering the command performs no configuration filesystem I/O.""" + package = sys.modules["canfar.cli"] + original_attribute = package.data + original = sys.modules.pop("canfar.cli.data") + try: + with patch( + "canfar.models.config.Configuration", + side_effect=AssertionError("configuration loaded during import"), + ): + importlib.import_module("canfar.cli.data") + finally: + sys.modules["canfar.cli.data"] = original + package.data = original_attribute diff --git a/tests/test_cli_login.py b/tests/test_cli_login.py index 7e41cd02..9e2094a7 100644 --- a/tests/test_cli_login.py +++ b/tests/test_cli_login.py @@ -253,6 +253,7 @@ def discover( assert result.exit_code == 0 authenticate.assert_called_once() assert authenticate.call_args.kwargs["timeout"] == 9 + assert authenticate.call_args.kwargs["force"] is True validate.assert_called_once() validated = validate.call_args.args[0] assert str(validated.uri) == _CADC_URI diff --git a/tests/test_cli_login_auth.py b/tests/test_cli_login_auth.py index 2102e69b..133d66ee 100644 --- a/tests/test_cli_login_auth.py +++ b/tests/test_cli_login_auth.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -10,6 +11,7 @@ from authlib.integrations.httpx_client import AsyncOAuth2Client from pydantic import AnyHttpUrl, SecretStr +from canfar.auth.x509 import CertificateError from canfar.cli.login_auth import _interactive_device_flow, authenticate_for_cli from canfar.idp import IdpInfo, get_idp from canfar.models.auth import DeviceAuthorization, OIDCCredential, X509Credential @@ -122,18 +124,66 @@ def test_authenticate_for_cli_presents_oidc_device_challenge( assert credential.expiry.refresh is None -def test_authenticate_for_cli_builds_x509_record_from_gather(tmp_path) -> None: - """CLI X.509 acquisition maps gathered certificate data without legacy state.""" +def test_authenticate_for_cli_reuses_valid_x509_certificate(tmp_path) -> None: + """CLI login reuses a valid certificate from the default Authentication Record.""" certificate = tmp_path / "cadcproxy.pem" - gathered = {"path": str(certificate), "expiry": 1893456000.0} + inspected = {"path": str(certificate), "expiry": 1893456000.0} - with patch("canfar.auth.x509.gather", return_value=gathered) as gather: + with ( + patch("canfar.auth.x509.inspect", return_value=inspected) as inspect, + patch("canfar.auth.x509.gather") as gather, + ): result = authenticate_for_cli(get_idp("cadc")) assert isinstance(result, X509Credential) assert result.idp == "cadc" assert result.path == certificate assert result.expiry == 1893456000.0 + inspect.assert_called_once_with() + gather.assert_not_called() + + +def test_authenticate_for_cli_reports_expired_x509_before_renewal(tmp_path) -> None: + """CLI login explains how long ago the default certificate expired.""" + certificate = tmp_path / "cadcproxy.pem" + expired_at = datetime.now(timezone.utc) - timedelta(minutes=10) + renewed = {"path": str(certificate), "expiry": 1893456000.0} + console = MagicMock() + + with ( + patch( + "canfar.auth.x509.inspect", + side_effect=CertificateError("expired", expired_at=expired_at), + ), + patch("canfar.auth.x509.gather", return_value=renewed) as gather, + patch( + "canfar.cli.login_auth.humanize.naturaltime", + return_value="10 minutes ago", + ), + patch("canfar.cli.login_auth.console_utils.get_console", return_value=console), + ): + result = authenticate_for_cli(get_idp("cadc")) + + assert result.path == certificate + gather.assert_called_once_with() + console.print.assert_called_once_with( + "[yellow]x509 certificate expired 10 minutes ago[/yellow]" + ) + + +def test_authenticate_for_cli_force_renews_x509_certificate(tmp_path) -> None: + """Forced CLI login obtains a new certificate without inspecting the old one.""" + certificate = tmp_path / "cadcproxy.pem" + gathered = {"path": str(certificate), "expiry": 1893456000.0} + + with ( + patch("canfar.auth.x509.inspect") as inspect, + patch("canfar.auth.x509.gather", return_value=gathered) as gather, + ): + result = authenticate_for_cli(get_idp("cadc"), force=True) + + assert result.path == certificate + inspect.assert_not_called() gather.assert_called_once_with() @@ -149,7 +199,7 @@ def test_authenticate_for_cli_normalizes_x509_gather_failure() -> None: match=r"^Failed to authenticate with X509 certificate:", ), ): - authenticate_for_cli(get_idp("cadc")) + authenticate_for_cli(get_idp("cadc"), force=True) def test_authenticate_for_cli_normalizes_malformed_x509_gather_result() -> None: @@ -161,7 +211,7 @@ def test_authenticate_for_cli_normalizes_malformed_x509_gather_result() -> None: match=r"^Failed to authenticate with X509 certificate:", ), ): - authenticate_for_cli(get_idp("cadc")) + authenticate_for_cli(get_idp("cadc"), force=True) def test_authenticate_for_cli_oidc_requires_discovery_url() -> None: diff --git a/tests/test_cli_server.py b/tests/test_cli_server.py index 9604bd43..58999efc 100644 --- a/tests/test_cli_server.py +++ b/tests/test_cli_server.py @@ -149,6 +149,7 @@ def test_server_ls_json_output(tmp_path: Path) -> None: "version", "auths", "idp", + "storage", "cores", "ram", "gpus", diff --git a/tests/test_data_dependencies.py b/tests/test_data_dependencies.py new file mode 100644 index 00000000..ffaabbb0 --- /dev/null +++ b/tests/test_data_dependencies.py @@ -0,0 +1,25 @@ +"""Package metadata tests for the standard data dependencies.""" + +from __future__ import annotations + +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib + + +def test_tagged_data_dependencies_are_standard_dependencies() -> None: + """A standard CANFAR install includes both immutable upstream releases.""" + metadata = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + + assert ( + "vosfs @ git+https://github.com/shinybrar/vosfs@v0.8.0" + in metadata["project"]["dependencies"] + ) + assert ( + "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.7.0" + "#subdirectory=src/fsspec-cli" in metadata["project"]["dependencies"] + ) + assert "data" not in metadata["project"].get("optional-dependencies", {}) diff --git a/tests/test_data_smoke.py b/tests/test_data_smoke.py new file mode 100644 index 00000000..69d2fe66 --- /dev/null +++ b/tests/test_data_smoke.py @@ -0,0 +1,64 @@ +"""Optional authenticated smoke test for the configured CADC VOSpace Service.""" + +from __future__ import annotations + +from typing import NoReturn + +import pytest +from typer.testing import CliRunner + +from canfar.auth import x509 +from canfar.cli.main import cli +from canfar.models.config import Configuration + + +def _skip(reason: str) -> NoReturn: + pytest.skip( + "live data smoke requires Storage Identifier 'arc' and a valid saved " + f"Authentication Record/certificate: {reason}" + ) + + +def _require_live_credentials() -> None: + """Skip unless persisted configuration can authenticate the live source.""" + try: + config = Configuration() # ty: ignore[missing-argument] + except Exception: # noqa: BLE001 - optional environment preflight. + _skip("configuration is unavailable") + + try: + _endpoint, idp = config._resolve_storage("arc") # noqa: SLF001 + credential = config.get_credential(idp) + except (KeyError, ValueError): + _skip("the named service or its Authentication Record is unavailable") + + if credential.mode == "x509": + if credential.path is None: + _skip("the saved X.509 certificate path is missing") + try: + x509.valid(credential.path) + x509.expiry(credential.path) + except (OSError, ValueError): + _skip("the saved X.509 certificate is unavailable or invalid") + return + + access = credential.token.access + has_current_access = ( + access is not None + and bool(access.get_secret_value()) + and not credential.expired + ) + if not has_current_access and not credential.refreshable: + _skip("the saved OIDC credential cannot authenticate or refresh") + + +@pytest.mark.integration +@pytest.mark.slow +def test_canfar_data_lists_live_primary_storage() -> None: + """List the configured live CADC primary VOSpace Service when available.""" + _require_live_credentials() + + result = CliRunner().invoke(cli, ["data", "ls", "-lh", "arc:/"]) + + assert result.exit_code == 0, result.output + assert not result.stdout.startswith("@") diff --git a/tests/test_idp.py b/tests/test_idp.py index 938a4c47..4fb3b6c1 100644 --- a/tests/test_idp.py +++ b/tests/test_idp.py @@ -23,6 +23,13 @@ def test_list_idps_returns_idp_info_instances(self) -> None: assert len(idps) == 2 assert all(isinstance(idp, IdpInfo) for idp in idps) + def test_builtin_primary_storage_preferences(self) -> None: + """Each built-in IDP declares its primary VOSpace resource leaf.""" + assert {idp.key: idp.leaf for idp in list_idps()} == { + "cadc": "arc", + "srcnet": "cavern", + } + class TestGetIdpCadc: """Tests for get_idp() with the cadc entry.""" diff --git a/tests/test_models_config.py b/tests/test_models_config.py index dd010c28..4bd5c046 100644 --- a/tests/test_models_config.py +++ b/tests/test_models_config.py @@ -20,7 +20,7 @@ X509Credential, ) from canfar.models.config import Configuration, _CanfarEnvSettingsSource -from canfar.models.http import Server +from canfar.models.http import Server, VOSpaceService from canfar.models.registry import ContainerRegistry @@ -66,6 +66,72 @@ def test_default_servers_dict_keyed_by_server_name(self, tmp_path: Path) -> None dumped = config.model_dump(mode="json", exclude_none=True) assert dumped["servers"]["canfar"]["name"] == "canfar" + def test_default_canfar_server_ships_arc_and_vault_storage( + self, + tmp_path: Path, + ) -> None: + """The CADC Science Platform Server exposes both VOSpace Services.""" + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + + assert set(config.servers["canfar"].storage) == {"arc", "vault"} + for name, endpoint in ( + ("arc", "https://ws-uv.canfar.net/arc"), + ("vault", "https://cadc-west-01.canfar.net/vault"), + ): + resolved, idp = config._resolve_storage(name) # noqa: SLF001 + assert resolved.rstrip("/") == endpoint + assert idp == "cadc" + + def test_legacy_server_named_storage_is_healed(self, tmp_path: Path) -> None: + """A Storage Identifier saved as the Server Name is restored to its leaf.""" + config_path = tmp_path / "config.yaml" + config_path.write_text( + "version: 1\n" + "servers:\n" + " canfar:\n" + " idp: cadc\n" + " uri: ivo://cadc.nrc.ca/skaha\n" + " url: https://ws-uv.canfar.net/skaha\n" + " version: v1\n" + " auths: [x509]\n" + " storage:\n" + " canfar:\n" + " uri: ivo://cadc.nrc.ca/arc\n" + " url: https://ws-uv.canfar.net/arc\n", + encoding="utf-8", + ) + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + + storage = config.servers["canfar"].storage + assert set(storage) == {"arc", "vault"} + assert str(storage["arc"].url).rstrip("/") == "https://ws-uv.canfar.net/arc" + + def test_custom_storage_names_are_not_healed(self, tmp_path: Path) -> None: + """Deliberate Storage Identifiers are configuration, not stale defaults.""" + config_path = tmp_path / "config.yaml" + config_path.write_text( + "version: 1\n" + "servers:\n" + " canfar:\n" + " idp: cadc\n" + " uri: ivo://cadc.nrc.ca/skaha\n" + " url: https://ws-uv.canfar.net/skaha\n" + " version: v1\n" + " auths: [x509]\n" + " storage:\n" + " canSRC:\n" + " uri: ivo://cadc.nrc.ca/arc\n" + " url: https://ws-cadc.canfar.net/arc\n", + encoding="utf-8", + ) + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + + assert set(config.servers["canfar"].storage) == {"canSRC"} + def test_default_authentication_dict_keyed_by_idp(self, tmp_path: Path) -> None: """Default Authentication Records are keyed by IDP.""" config_path = tmp_path / "config.yaml" @@ -137,6 +203,110 @@ def test_invalid_idp_key_rejected(self, idp_key: str) -> None: }, ) + @pytest.mark.parametrize( + "storage_name", + [ + "", + "local", + " local ", + "archive:old", + "archive\x00old", + "archive\nold", + "archive\n", + "archive\r", + "archive\r\n", + "-archive", + ], + ) + def test_invalid_storage_name_rejected( + self, storage_name: str, tmp_path: Path + ) -> None: + """Invalid Storage Identifiers fail with the rejected name and constraints.""" + with ( + patch("canfar.models.config.CONFIG_PATH", tmp_path / "config.yaml"), + pytest.raises( + ValidationError, match="Invalid Storage Identifier" + ) as exc_info, + ): + Configuration.model_validate( + { + "servers": { + "canfar": { + "storage": { + storage_name: { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + } + } + } + } + } + ) + + assert repr(storage_name) in str(exc_info.value) + + def test_long_storage_name_allowed(self) -> None: + """Storage Identifiers do not inherit Server field length limits.""" + storage_name = "a" * 257 + service = { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + } + + server = Server(storage={storage_name: service}) + + assert list(server.storage) == [storage_name] + + def test_storage_name_whitespace_is_trimmed(self) -> None: + """Valid surrounding whitespace remains normalized.""" + service = { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + } + + assert list(Server(storage={" arc ": service}).storage) == ["arc"] + + def test_duplicate_storage_name_across_servers_rejected( + self, tmp_path: Path + ) -> None: + """A Storage Identifier names one service across Science Platform Servers.""" + service = VOSpaceService( + uri="ivo://cadc.nrc.ca/arc", + url="https://ws-cadc.canfar.net/arc", + ) + + with ( + patch("canfar.models.config.CONFIG_PATH", tmp_path / "config.yaml"), + pytest.raises( + ValidationError, + match=( + "Duplicate Storage Identifier 'shared' in Science Platform Servers " + "'canfar' and 'srcnet'" + ), + ), + ): + Configuration.model_validate( + { + "servers": { + "canfar": Server(storage={"shared": service}), + "srcnet": Server(storage={"shared": service}), + } + } + ) + + def test_normalized_storage_name_collision_rejected(self) -> None: + """Whitespace normalization cannot silently replace a VOSpace Service.""" + service = { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + } + + with pytest.raises( + ValidationError, + match="Storage Identifiers 'arc' and ' arc ' both normalize to 'arc'", + ): + Server(storage={"arc": service, " arc ": service}) + def test_valid_active_references(self) -> None: """Validation passes when active authentication and server exist.""" config = Configuration( @@ -316,6 +486,58 @@ def test_complex_round_trip_serialization(self, tmp_path: Path) -> None: assert loaded_oidc.expiry.access == 1893456000 assert loaded_oidc.expiry.refresh is None + def test_v1_storage_json_and_yaml_round_trip(self, tmp_path: Path) -> None: + """Multiple VOSpace Services retain stable nested Storage Identifiers.""" + config = Configuration.model_validate( + _sample_config( + servers={ + "canfar": { + **_sample_config()["servers"]["canfar"], + "storage": { + "canSRC": { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + }, + "canSRCs3": { + "uri": "ivo://cadc.nrc.ca/arc-s3", + "url": "https://ws-cadc.canfar.net/arc-s3", + }, + }, + } + } + ) + ) + + json_data = config.model_dump(mode="json") + assert list(json_data["servers"]["canfar"]["storage"]) == [ + "canSRC", + "canSRCs3", + ] + assert "name" not in json_data["servers"]["canfar"]["storage"]["canSRC"] + assert Configuration.model_validate_json(config.model_dump_json()) == config + + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config.save() + loaded = Configuration() + + assert list(loaded.servers["canfar"].storage) == ["canSRC", "canSRCs3"] + assert loaded.servers["canfar"].idp == "cadc" + assert loaded == config + + def test_existing_v1_configuration_without_storage_gains_defaults( + self, tmp_path: Path + ) -> None: + """A storage-less Server gains defaults without a schema migration.""" + config_path = tmp_path / "config.yaml" + config_path.write_text(yaml.safe_dump(_sample_config()), encoding="utf-8") + + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + + assert config.version == 1 + assert set(config.servers["canfar"].storage) == {"arc", "vault"} + def test_save_creates_directory(self, tmp_path: Path) -> None: """Save creates parent directories when missing.""" config = Configuration() diff --git a/tests/test_models_http.py b/tests/test_models_http.py index ffb28de4..ff7ed3c1 100644 --- a/tests/test_models_http.py +++ b/tests/test_models_http.py @@ -3,7 +3,49 @@ import pytest from pydantic import ValidationError -from canfar.models.http import Server +from canfar.models.http import Server, VOSpaceService + + +class TestVOSpaceService: + """Test VOSpace Service configuration.""" + + def test_serializes_registry_uri_and_http_endpoint(self) -> None: + """A VOSpace Service carries only its registry URI and base URL.""" + service = VOSpaceService( + uri="ivo://cadc.nrc.ca/arc", + url="https://ws-cadc.canfar.net/arc", + ) + + assert service.model_dump(mode="json") == { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + } + + @pytest.mark.parametrize( + "payload", + [ + {"uri": "not-a-url", "url": "https://ws-cadc.canfar.net/arc"}, + {"uri": "ivo://cadc.nrc.ca/arc", "url": "vos://example.com/arc"}, + {"url": "https://ws-cadc.canfar.net/arc"}, + {"uri": "ivo://cadc.nrc.ca/arc"}, + ], + ) + def test_requires_valid_registry_uri_and_http_endpoint(self, payload: dict) -> None: + """Both VOSpace Service identifiers are required and validated.""" + with pytest.raises(ValidationError): + VOSpaceService.model_validate(payload) + + @pytest.mark.parametrize( + "url", + [ + "https://ws-cadc.canfar.net/arc/capabilities", + "https://ws-cadc.canfar.net/arc/capabilities/", + ], + ) + def test_rejects_capabilities_endpoint_as_base_url(self, url: str) -> None: + """A VOSpace Service URL names the base service, not capabilities.""" + with pytest.raises(ValidationError, match="must not end with /capabilities"): + VOSpaceService(uri="ivo://cadc.nrc.ca/arc", url=url) class TestServer: @@ -33,6 +75,7 @@ def test_default_values(self) -> None: assert server.url is None assert server.version is None assert server.status is None + assert server.storage == {} def test_with_all_values(self) -> None: """Test Server with all custom values.""" diff --git a/tests/test_platform_enrichment.py b/tests/test_platform_enrichment.py index cfc659b2..8d578b6f 100644 --- a/tests/test_platform_enrichment.py +++ b/tests/test_platform_enrichment.py @@ -23,7 +23,7 @@ X509Credential, ) from canfar.models.config import Configuration -from canfar.models.http import Server +from canfar.models.http import Server, VOSpaceService from canfar.models.registry import ContainerRegistry from tests.test_auth_x509 import generate_cert @@ -39,6 +39,25 @@ "memoryGB": {"defaultLimit": 192}, "gpus": {"options": [0, 1, 2, 4]}, } +_VOSPACE_CAPABILITIES = """ + + + + https://storage.example/arc/nodes + + + +""" + + +def _primary_storage(name: str, *, leaf: str = "arc") -> dict[str, VOSpaceService]: + """Return persisted primary storage evidence for activation tests.""" + return { + name: VOSpaceService( + uri=AnyUrl(f"ivo://storage.example/{leaf}"), + url=AnyHttpUrl(f"https://storage.example/{leaf}"), + ) + } def _http_client_factory( @@ -149,6 +168,7 @@ def test_activation_enriches_resources_and_round_trips_server_name( """Activation owns HTTP enrichment while Server remains persisted data.""" known = _server( name="Stable-Name", + storage=_primary_storage("Stable-Name"), cores=8, ram=64, gpus=1, @@ -159,6 +179,8 @@ def test_activation_enriches_resources_and_round_trips_server_name( def response(request: httpx.Request) -> httpx.Response: """Serve capabilities and context for activation enrichment.""" requests.append(request) + if request.url.host == "storage.example": + return httpx.Response(200, text=_VOSPACE_CAPABILITIES, request=request) if request.url.path.endswith("/capabilities"): return httpx.Response( 200, text=_capabilities(_CADC_URL), request=request @@ -189,12 +211,14 @@ def response(request: httpx.Request) -> httpx.Response: assert persisted.active.server == "Stable-Name" assert persisted.servers["Stable-Name"] == activated.server assert [request.url.path for request in requests] == [ + "/arc/capabilities", "/skaha/capabilities", "/skaha/v2/context", ] assert [request.extensions["timeout"] for request in requests] == [ {"connect": 7, "read": 7, "write": 7, "pool": 7}, {"connect": 7, "read": 7, "write": 7, "pool": 7}, + {"connect": 7, "read": 7, "write": 7, "pool": 7}, ] @pytest.mark.parametrize("failure", ["transport", "parse", "validation"]) @@ -204,10 +228,19 @@ def test_activation_uses_resource_defaults_when_context_is_unusable( failure: str, ) -> None: """Expected context failures keep server identity and safe defaults.""" - known = _server(name="Stable-Name", cores=8, ram=64, gpus=1, status="reachable") + known = _server( + name="Stable-Name", + storage=_primary_storage("Stable-Name"), + cores=8, + ram=64, + gpus=1, + status="reachable", + ) def response(request: httpx.Request) -> httpx.Response: """Serve capabilities and a parametrized unusable context reply.""" + if request.url.host == "storage.example": + return httpx.Response(200, text=_VOSPACE_CAPABILITIES, request=request) if request.url.path.endswith("/capabilities"): return httpx.Response( 200, text=_capabilities(_CADC_URL), request=request @@ -445,6 +478,7 @@ def test_refresh_can_persist_without_candidate_platform_state( name="SRCNet", uri=AnyUrl("ivo://srcnet.example/skaha"), url=AnyHttpUrl("https://srcnet.example/skaha"), + storage=_primary_storage("SRCNet", leaf="cavern"), version=None, auths=None, ) @@ -455,6 +489,8 @@ def test_refresh_can_persist_without_candidate_platform_state( def platform_response(request: httpx.Request) -> httpx.Response: """Serve capabilities then a failed context fetch during refresh.""" platform_requests.append(request) + if request.url.host == "storage.example": + return httpx.Response(200, text=_VOSPACE_CAPABILITIES, request=request) if request.url.path.endswith("/capabilities"): return httpx.Response(200, text=capabilities, request=request) if request.url.path.endswith("/v2/context"): @@ -520,6 +556,7 @@ def token_response(request: httpx.Request) -> httpx.Response: assert [request.headers["Authorization"] for request in platform_requests] == [ f"Bearer {_REFRESHED_TOKEN}", f"Bearer {_REFRESHED_TOKEN}", + f"Bearer {_REFRESHED_TOKEN}", ] assert config.active.model_dump(mode="json") == before_active assert { diff --git a/tests/test_server.py b/tests/test_server.py index c22ee7a1..6d0caae9 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -3,6 +3,7 @@ from __future__ import annotations from pathlib import Path +from threading import Barrier from typing import TYPE_CHECKING from unittest.mock import AsyncMock, MagicMock, patch @@ -13,9 +14,10 @@ from canfar.errors import ErrorCode from canfar.models.active import ActiveConfig -from canfar.models.auth import OIDCCredential, X509Credential -from canfar.models.config import Configuration -from canfar.models.http import Server +from canfar.models.auth import OIDCCredential, RuntimeCredential, X509Credential +from canfar.models.config import Configuration, default_servers +from canfar.models.http import Server, VOSpaceService +from canfar.models.registry import IVOARegistry, IVOARegistrySearch from canfar.models.registry import Server as DiscoveredServer from canfar.server import ( ServerDiscoveryError, @@ -23,13 +25,16 @@ ServerSelectionRequiredError, _discover_for_idp, _discovered_to_server, + _select_storage, activate, discover, + enrich, use, ) from canfar.server import ( list_servers as server_list, ) +from canfar.utils.discover import Discover from tests.helpers.config import assign_servers if TYPE_CHECKING: @@ -38,6 +43,16 @@ _CADC_URI = "ivo://cadc.nrc.ca/skaha" _CADC_URL = "https://ws-uv.canfar.net/skaha" +_VOSPACE_CAPABILITIES = """ + + + + https://storage.example/arc/nodes + + + +""" + def _http_client_factory( transport: httpx.BaseTransport, @@ -345,6 +360,614 @@ def test_discover_merges_servers_through_public_api(self, tmp_path: Path) -> Non "https://cadc.example/skaha" ) + @pytest.mark.asyncio + async def test_registry_retains_only_skaha_and_preferred_storage_records( + self, + ) -> None: + """CADC discovery retains ARC despite Vault and preserves registry URLs.""" + registry = IVOARegistry( + name="CADC", + content=( + "ivo://cadc.nrc.ca/skaha=" + "https://platform.example/skaha/capabilities\n" + "ivo://cadc.nrc.ca/arc=" + "https://storage.example/custom/capabilities\n" + "ivo://cadc.nrc.ca/vault=" + "https://storage.example/vault/capabilities\n" + "ivo://other.example/arc=" + "https://platform.example/skaha-arc/capabilities" + ), + ) + search = IVOARegistrySearch(leaf="arc") + + async with Discover(search) as discovery: + resources = discovery.extract(registry) + + assert [(resource.uri, resource.url) for resource in resources] == [ + ("ivo://cadc.nrc.ca/skaha", "https://platform.example/skaha"), + ("ivo://cadc.nrc.ca/arc", "https://storage.example/custom"), + ("ivo://other.example/arc", "https://platform.example/skaha-arc"), + ] + + def test_discover_refreshes_primary_storage_and_preserves_manual_entries( + self, + tmp_path: Path, + ) -> None: + """Rediscovery updates only the generated Storage Identifier entry. + + A Server Name keyed entry saved by an older client is healed to its + registry leaf (``arc``) on load, so rediscovery refreshes that entry in + place instead of generating a second one. + """ + manual = VOSpaceService( + uri="ivo://cadc.nrc.ca/custom", + url="https://manual.example/custom", + ) + old_primary = VOSpaceService( + uri="ivo://cadc.nrc.ca/arc", + url="https://old.example/arc", + ) + known = _cadc_server( + name="canfar", + storage={"canfar": old_primary, "archive": manual}, + ) + registry_body = "\n".join( + ( + f"{_CADC_URI}={_CADC_URL}/capabilities", + "ivo://cadc.nrc.ca/arc=https://storage.example/arc/capabilities", + "ivo://cadc.nrc.ca/vault=https://storage.example/vault/capabilities", + ) + ) + + def registry_response(request: httpx.Request) -> httpx.Response: + if request.method == "GET": + return httpx.Response(200, text=registry_body, request=request) + return httpx.Response(200, request=request) + + session_capabilities = """ + + + + https://ws-uv.canfar.net/skaha/v1 + + + + + """ + + def capabilities_response(request: httpx.Request) -> httpx.Response: + content = ( + _VOSPACE_CAPABILITIES + if str(request.url) == "https://storage.example/arc/capabilities" + else session_capabilities + ) + return httpx.Response(200, text=content, request=request) + + real_async_client = httpx.AsyncClient + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = _anonymous_config(known) + with ( + patch( + "canfar.utils.discover.httpx.AsyncClient", + side_effect=lambda **_kwargs: real_async_client( + transport=httpx.MockTransport(registry_response) + ), + ), + patch( + "canfar.client.Client", + side_effect=_http_client_factory( + httpx.MockTransport(capabilities_response) + ), + ), + ): + [discovered] = discover("cadc", config=config) + + persisted = Configuration().servers["canfar"] + + assert discovered.storage == persisted.storage + assert set(discovered.storage) == {"arc", "archive", "vault"} + assert discovered.storage["arc"] == VOSpaceService( + uri="ivo://cadc.nrc.ca/arc", + url="https://storage.example/arc", + ) + assert discovered.storage["archive"] == manual + + @pytest.mark.parametrize("mode", ["missing", "malformed", "unreachable"]) + def test_storage_inspection_fail_or_keep_outcomes( + self, mode: str, tmp_path: Path + ) -> None: + """Storage failures are kept non-strict and actionable when strict.""" + storage_resource = ( + None + if mode == "missing" + else DiscoveredServer( + registry="CADC", + uri="ivo://cadc.nrc.ca/arc", + url="https://storage.example/arc", + ) + ) + + def response(request: httpx.Request) -> httpx.Response: + if mode == "unreachable": + message = "storage unavailable" + raise httpx.ConnectError(message, request=request) + return httpx.Response(200, text="", request=request) + + server = _cadc_server() + transport = httpx.MockTransport(response) + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = _anonymous_config() + with patch( + "canfar.client.Client", side_effect=_http_client_factory(transport) + ): + assert ( + enrich( + server, + config=config, + storage_resource=storage_resource, + strict=False, + ) + == server + ) + + with ( + patch( + "canfar.client.Client", + side_effect=_http_client_factory(transport), + ), + pytest.raises(ServerFetchError, match="VOSpace Service") as exc_info, + ): + enrich( + server, + config=config, + storage_resource=storage_resource, + strict=True, + ) + + if mode == "malformed": + assert isinstance(exc_info.value.__cause__, ValueError) + + def test_activation_freshly_inspects_storage_missing_during_discovery(self) -> None: + """Activation obtains fresh evidence without transient Configuration state.""" + endpoint = DiscoveredServer( + registry="CADC source", + uri=_CADC_URI, + url=_CADC_URL, + status=200, + name="CADC-CANFAR", + ) + mock_discovery = AsyncMock() + mock_discovery.fetch.return_value = MagicMock(success=True, content="line") + mock_discovery.extract = MagicMock(return_value=[endpoint]) + mock_discovery.check = AsyncMock(side_effect=lambda item: item) + mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) + mock_discovery.__aexit__ = AsyncMock(return_value=None) + session_capabilities = """ + + + + https://ws-uv.canfar.net/skaha/v1 + + + + + """ + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, + text=session_capabilities, + request=request, + ) + ) + config = _anonymous_config() + + with ( + patch("canfar.utils.registry.Discover", return_value=mock_discovery), + patch( + "canfar.client.Client", + side_effect=_http_client_factory(transport), + ), + ): + [discovered] = discover("cadc", config=config, save=False) + reloaded = Configuration.model_validate(config.model_dump(mode="python")) + with pytest.raises( + ServerFetchError, + match="same-namespace 'arc' registry record", + ): + activate("cadc", _CADC_URI, config=reloaded) + + assert discovered.version == "v1" + assert discovered.storage == {} + assert "_storage_discovery_errors" not in Configuration.__private_attributes__ + + @pytest.mark.asyncio + async def test_cross_registry_singletons_pair_by_namespace(self) -> None: + """A lone same-environment fallback may cross registry provenance.""" + resources = [ + DiscoveredServer( + registry="SRCNet", + uri="ivo://canfar.net/src/skaha", + url="https://one.example/skaha", + status=200, + name="canSRC", + ), + DiscoveredServer( + registry="SRCNet", + uri="ivo://swesrc.chalmers.se/skaha", + url="https://two.example/skaha", + status=200, + name="sweSRC", + ), + DiscoveredServer( + registry="SRCNet mirror B", + uri="ivo://swesrc.chalmers.se/cavern", + url="https://storage.example/two", + ), + DiscoveredServer( + registry="SRCNet mirror A", + uri="ivo://canfar.net/src/cavern", + url="https://storage.example/one", + ), + ] + mock_discovery = AsyncMock() + mock_discovery.fetch.return_value = MagicMock(success=True, content="line") + mock_discovery.extract = MagicMock(return_value=resources) + mock_discovery.check = AsyncMock(side_effect=lambda item: item) + mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) + mock_discovery.__aexit__ = AsyncMock(return_value=None) + + def enriched( + server: Server, + *, + storage_resource: DiscoveredServer, + **_kwargs: object, + ) -> Server: + return server.model_copy( + update={ + "version": "v1", + "auths": ["oidc"], + "storage": { + server.name: VOSpaceService( + uri=storage_resource.uri, + url=storage_resource.url, + ) + }, + }, + deep=True, + ) + + with ( + patch("canfar.utils.registry.Discover", return_value=mock_discovery), + patch("canfar.server.enrich", side_effect=enriched), + ): + servers = await _discover_for_idp("srcnet") + + assert { + server.name: str(server.storage[server.name].uri) for server in servers + } == { + "canSRC": "ivo://canfar.net/src/cavern", + "sweSRC": "ivo://swesrc.chalmers.se/cavern", + } + + def test_storage_pairing_never_crosses_prod_and_dev_sources(self) -> None: + """A same-namespace record from another environment is not a fallback.""" + endpoint = DiscoveredServer( + registry="https://registry.example/prod", + development=False, + uri="ivo://cadc.nrc.ca/skaha", + url="https://platform.example/skaha", + name="canfar", + ) + dev_storage = DiscoveredServer( + registry="https://registry.example/dev", + development=True, + uri="ivo://cadc.nrc.ca/arc", + url="https://storage.example/arc", + ) + + assert _select_storage(endpoint, [dev_storage], strict=False) is None + + @pytest.mark.asyncio + async def test_mixed_registry_records_keep_per_record_environment(self) -> None: + """Development-looking records stay isolated within a production source.""" + registry = IVOARegistry( + name="SRCNet", + source="https://registry.example/resources", + content=( + "ivo://example.org/skaha=" + "https://platform.example/skaha/capabilities\n" + "ivo://example.org/cavern=" + "https://storage.example/dev/capabilities" + ), + ) + search = IVOARegistrySearch(leaf="cavern") + + async with Discover(search) as discovery: + endpoint, storage = discovery.extract(registry, dev=True) + + assert endpoint.development is False + assert storage.development is True + assert _select_storage(endpoint, [storage], strict=False) is None + + def test_ambiguous_cross_registry_storage_is_not_last_write_wins(self) -> None: + """Multiple namespace fallbacks are omitted or actionable, never arbitrary.""" + endpoint = DiscoveredServer( + registry="https://registry.example/platform", + uri="ivo://example.org/skaha", + url="https://platform.example/skaha", + name="example", + ) + storage = [ + DiscoveredServer( + registry=f"https://registry.example/storage-{index}", + uri="ivo://example.org/cavern", + url=f"https://storage-{index}.example/cavern", + ) + for index in (1, 2) + ] + + assert _select_storage(endpoint, storage, strict=False) is None + with pytest.raises(ServerFetchError, match="Multiple preferred VOSpace"): + _select_storage(endpoint, storage, strict=True) + + @pytest.mark.asyncio + async def test_capability_enrichment_runs_concurrently_off_event_loop( + self, + ) -> None: + """Workers run concurrently with isolated, pre-materialized config copies.""" + endpoints = [ + DiscoveredServer( + registry="SRCNet", + uri=f"ivo://site-{index}.example/skaha", + url=f"https://site-{index}.example/skaha", + status=200, + name=f"site-{index}", + ) + for index in (1, 2) + ] + mock_discovery = AsyncMock() + mock_discovery.fetch.return_value = MagicMock( + success=True, + content="line", + ) + mock_discovery.extract = MagicMock(return_value=endpoints) + mock_discovery.check = AsyncMock(side_effect=lambda item: item) + mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) + mock_discovery.__aexit__ = AsyncMock(return_value=None) + concurrent = Barrier(2, timeout=2) + worker_configs: list[Configuration] = [] + config = Configuration( + active=ActiveConfig(authentication="srcnet", server=None), + authentication={"srcnet": OIDCCredential(idp="srcnet")}, + servers={}, + ) + + def convert( + endpoint: DiscoveredServer, + idp: str, + **kwargs: object, + ) -> Server: + worker_config = kwargs["config"] + assert isinstance(worker_config, Configuration) + worker_configs.append(worker_config) + assert kwargs["token"] == "current-token" + assert kwargs["certificate"] is None + concurrent.wait() + return Server( + idp=idp, + name=endpoint.name, + uri=AnyUrl(endpoint.uri), + url=AnyHttpUrl(endpoint.url), + version="v1", + auths=["oidc"], + ) + + materialize = AsyncMock(return_value=RuntimeCredential(token="current-token")) + with ( + patch("canfar.utils.registry.Discover", return_value=mock_discovery), + patch("canfar.server._discovered_to_server", side_effect=convert), + patch( + "canfar.client.HTTPClient._materialize_credentials", + new=materialize, + ), + ): + servers = await _discover_for_idp("srcnet", config=config) + + assert [server.name for server in servers] == ["site-1", "site-2"] + materialize.assert_awaited_once_with() + assert len({id(worker_config) for worker_config in worker_configs}) == 2 + assert all(worker_config is not config for worker_config in worker_configs) + + @pytest.mark.asyncio + async def test_worker_requests_use_pre_materialized_runtime_token(self) -> None: + """Fan-out requests cannot invoke saved-record refresh or persistence.""" + endpoint = DiscoveredServer( + registry="SRCNet", + uri="ivo://site.example/skaha", + url="https://site.example/skaha", + status=200, + name="site", + ) + mock_discovery = AsyncMock() + mock_discovery.fetch.return_value = MagicMock(success=True, content="line") + mock_discovery.extract = MagicMock(return_value=[endpoint]) + mock_discovery.check = AsyncMock(side_effect=lambda item: item) + mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) + mock_discovery.__aexit__ = AsyncMock(return_value=None) + config = Configuration( + active=ActiveConfig(authentication="srcnet", server=None), + authentication={"srcnet": OIDCCredential(idp="srcnet")}, + servers={}, + ) + requests: list[httpx.Request] = [] + session_capabilities = """ + + + + https://site.example/skaha/v1 + + + + + """ + + def response(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, text=session_capabilities, request=request) + + materialize = AsyncMock(return_value=RuntimeCredential(token="runtime-token")) + with ( + patch("canfar.utils.registry.Discover", return_value=mock_discovery), + patch( + "canfar.client.HTTPClient._materialize_credentials", + new=materialize, + ), + patch( + "canfar.client.Client", + side_effect=_http_client_factory(httpx.MockTransport(response)), + ), + patch("canfar.auth.oidc.sync_refresh") as refresh, + ): + [server] = await _discover_for_idp("srcnet", config=config) + + assert server.version == "v1" + materialize.assert_awaited_once_with() + refresh.assert_not_called() + assert [request.headers["Authorization"] for request in requests] == [ + "Bearer runtime-token" + ] + + @pytest.mark.asyncio + async def test_environment_token_materializes_without_saved_credential( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Environment bearer auth survives preparation and worker fan-out.""" + endpoint = DiscoveredServer( + registry="SRCNet", + uri="ivo://site.example/skaha", + url="https://site.example/skaha", + status=200, + name="site", + ) + mock_discovery = AsyncMock() + mock_discovery.fetch.return_value = MagicMock(success=True, content="line") + mock_discovery.extract = MagicMock(return_value=[endpoint]) + mock_discovery.check = AsyncMock(side_effect=lambda item: item) + mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) + mock_discovery.__aexit__ = AsyncMock(return_value=None) + requests: list[httpx.Request] = [] + session_capabilities = """ + + + + https://site.example/skaha/v1 + + + + + """ + + def response(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, text=session_capabilities, request=request) + + monkeypatch.delenv("CANFAR_CERTIFICATE", raising=False) + monkeypatch.setenv("CANFAR_TOKEN", "environment-token") + with ( + patch("canfar.utils.registry.Discover", return_value=mock_discovery), + patch( + "canfar.client.Client", + side_effect=_http_client_factory(httpx.MockTransport(response)), + ), + ): + [server] = await _discover_for_idp( + "srcnet", + config=_anonymous_config(idp="srcnet"), + ) + + assert server.version == "v1" + assert [request.headers["Authorization"] for request in requests] == [ + "Bearer environment-token" + ] + + @pytest.mark.asyncio + async def test_environment_certificate_materializes_without_saved_credential( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """Environment certificate auth survives preparation and worker fan-out.""" + endpoint = DiscoveredServer( + registry="SRCNet", + uri="ivo://site.example/skaha", + url="https://site.example/skaha", + status=200, + name="site", + ) + mock_discovery = AsyncMock() + mock_discovery.fetch.return_value = MagicMock(success=True, content="line") + mock_discovery.extract = MagicMock(return_value=[endpoint]) + mock_discovery.check = AsyncMock(side_effect=lambda item: item) + mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) + mock_discovery.__aexit__ = AsyncMock(return_value=None) + session_capabilities = """ + + + + https://site.example/skaha/v1 + + + + + """ + certificate = tmp_path / "environment.pem" + valid = MagicMock(return_value=certificate.as_posix()) + ssl_context = MagicMock() + + monkeypatch.delenv("CANFAR_TOKEN", raising=False) + monkeypatch.setenv("CANFAR_CERTIFICATE", certificate.as_posix()) + with ( + patch("canfar.utils.registry.Discover", return_value=mock_discovery), + patch( + "canfar.client.x509.inspect", + return_value={ + "path": certificate.as_posix(), + "expiry": 9_999_999_999.0, + }, + ), + patch("canfar.client.x509.valid", new=valid), + patch( + "canfar.client.HTTPClient._get_ssl_context", + return_value=ssl_context, + ) as get_ssl_context, + patch( + "canfar.client.Client", + side_effect=_http_client_factory( + httpx.MockTransport( + lambda request: httpx.Response( + 200, + text=session_capabilities, + request=request, + ) + ) + ), + ), + ): + [server] = await _discover_for_idp( + "srcnet", + config=_anonymous_config(idp="srcnet"), + ) + + assert server.version == "v1" + valid.assert_called_once_with(certificate) + get_ssl_context.assert_called_once_with(certificate) + @pytest.mark.parametrize( "capabilities_case", [ @@ -443,13 +1066,18 @@ def capabilities_response(request: httpx.Request) -> httpx.Response: persisted = Configuration().servers["canfar"] + # A storage-less saved Server gains the default VOSpace Services on load. + healed = known.model_copy( + update={"storage": default_servers["canfar"].storage}, + deep=True, + ) expected = ( - known.model_copy( + healed.model_copy( update={"version": "v2.1", "auths": ["oidc"]}, deep=True, ) if capabilities_case == "success" - else known + else healed ) assert discovered == [expected] assert config.servers["canfar"] == expected @@ -586,7 +1214,7 @@ def test_discover_keys_unnamed_server_by_host_slug(self, tmp_path: Path) -> None with ( patch("canfar.models.config.CONFIG_PATH", config_path), - patch("canfar.server.Discover", return_value=mock_discovery), + patch("canfar.utils.registry.Discover", return_value=mock_discovery), patch( "canfar.server.enrich", side_effect=lambda item, **_kwargs: item.model_copy( @@ -624,7 +1252,7 @@ async def test_discover_for_idp_converts_active_endpoints(self) -> None: mock_discovery.__aexit__ = AsyncMock(return_value=None) with ( - patch("canfar.server.Discover", return_value=mock_discovery), + patch("canfar.utils.registry.Discover", return_value=mock_discovery), patch( "canfar.server.enrich", side_effect=lambda item, **_kwargs: item.model_copy( @@ -713,7 +1341,7 @@ async def test_discover_for_idp_raises_when_registry_fetch_fails(self) -> None: mock_discovery.__aexit__ = AsyncMock(return_value=None) with ( - patch("canfar.server.Discover", return_value=mock_discovery), + patch("canfar.utils.registry.Discover", return_value=mock_discovery), pytest.raises(ServerDiscoveryError, match="Failed to discover"), ): await _discover_for_idp("cadc") @@ -730,7 +1358,9 @@ async def test_discover_for_idp_honors_dev_sources_and_timeout(self) -> None: mock_discovery.__aenter__ = AsyncMock(return_value=mock_discovery) mock_discovery.__aexit__ = AsyncMock(return_value=None) - with patch("canfar.server.Discover", return_value=mock_discovery) as factory: + with patch( + "canfar.utils.registry.Discover", return_value=mock_discovery + ) as factory: servers = await _discover_for_idp("cadc", dev=True, timeout=11) assert servers == [] diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 00000000..e4e7a454 --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,444 @@ +"""Tests for the VOSpace and local storage source adapters.""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Any +from unittest.mock import AsyncMock, Mock + +import pytest +import vosfs +from fsspec.implementations.local import LocalFileSystem +from pydantic import AnyHttpUrl, AnyUrl + +from canfar import storage +from canfar.exceptions.context import AuthContextError +from canfar.models.active import ActiveConfig +from canfar.models.config import Configuration +from canfar.models.http import Server, VOSpaceService +from canfar.storage import _vospace +from tests.helpers.config import oidc_credential, x509_credential + +if TYPE_CHECKING: + from pathlib import Path + +_LISTINGS = { + "use_listings_cache": True, + "listings_expiry_time": 30, + "max_paths": 1000, +} +"""Directory-listing cache settings every VOSpace filesystem is built with.""" + + +class _Filesystem: + """Record construction and cleanup without VOSpace I/O.""" + + def __init__(self, endpoint: str, **kwargs: Any) -> None: + self.endpoint = endpoint + self.kwargs = kwargs + self.asynchronous = kwargs.get("asynchronous", False) + self.closed = False + + async def aclose(self) -> None: + self.closed = True + + +def _config( + *, + credential: Any, + endpoint: str = "https://inactive.example/vospace", +) -> Configuration: + server = Server( + idp=credential.idp, + uri=AnyUrl("ivo://inactive.example/skaha"), + url=AnyHttpUrl("https://inactive.example/skaha"), + version="v1", + storage={ + "archive": VOSpaceService( + uri=AnyUrl("ivo://inactive.example/arc"), + url=AnyHttpUrl(endpoint), + ) + }, + ) + return Configuration( + active=ActiveConfig(authentication="active", server=None), + authentication={ + "active": x509_credential("active"), + credential.idp: credential, + }, + servers={"inactive": server}, + ) + + +@pytest.fixture(autouse=True) +def _isolate_configuration(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Isolate every source test from the developer's configuration.""" + config_path = tmp_path / "config.yaml" + monkeypatch.setattr("canfar.models.config.CONFIG_PATH", config_path) + + +@pytest.mark.asyncio +async def test_source_reloads_config_and_runtime_token_wins( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Entry reloads endpoint state and keeps token-over-certificate precedence.""" + config = _config(credential=oidc_credential("inactive")) + config.save() + source = _vospace( + "archive", + token="runtime-token", + certificate=tmp_path / "ignored.pem", + ) + + config.servers["inactive"].storage["archive"].url = AnyHttpUrl( + "https://changed.example/vospace" + ) + config.save() + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with source() as filesystem: + assert filesystem.endpoint == "https://changed.example/vospace" + assert filesystem.kwargs == { + "token": "runtime-token", + "asynchronous": True, + "skip_instance_cache": True, + **_LISTINGS, + } + assert filesystem.asynchronous is True + assert filesystem.closed is False + + assert filesystem.closed is True + + +@pytest.mark.asyncio +async def test_source_factory_constructs_fresh_filesystem_per_acquisition( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Repeated acquisition constructs and closes distinct VOSpace filesystems.""" + _config(credential=oidc_credential("inactive", access="current-token")).save() + filesystems: list[_Filesystem] = [] + + def build(endpoint: str, **kwargs: Any) -> _Filesystem: + filesystem = _Filesystem(endpoint, **kwargs) + filesystems.append(filesystem) + return filesystem + + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", build) + source = _vospace("archive") + + async with source() as first: + assert first.closed is False + assert first.closed is True + + async with source() as second: + assert second.closed is False + assert second.closed is True + + assert first is not second + assert filesystems == [first, second] + + +@pytest.mark.asyncio +async def test_environment_token_preserves_runtime_precedence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The source leaves an omitted token open to HTTPClient environment settings.""" + _config(credential=x509_credential("inactive")).save() + monkeypatch.delenv("CANFAR_CERTIFICATE", raising=False) + monkeypatch.setenv("CANFAR_TOKEN", "environment-token") + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with _vospace("archive")() as filesystem: + assert filesystem.kwargs == { + "token": "environment-token", + "asynchronous": True, + "skip_instance_cache": True, + **_LISTINGS, + } + + +@pytest.mark.asyncio +async def test_environment_certificate_preserves_runtime_precedence( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The source leaves an omitted certificate open to settings sources.""" + certificate = tmp_path / "environment.pem" + _config(credential=oidc_credential("inactive")).save() + monkeypatch.delenv("CANFAR_TOKEN", raising=False) + monkeypatch.setenv("CANFAR_CERTIFICATE", certificate.as_posix()) + monkeypatch.setattr( + "canfar.client.x509.inspect", + lambda path: {"path": path.as_posix(), "expiry": 9_999_999_999.0}, + ) + valid = Mock(return_value=certificate.as_posix()) + monkeypatch.setattr("canfar.client.x509.valid", valid) + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with _vospace("archive")() as filesystem: + assert filesystem.kwargs == { + "certfile": certificate.as_posix(), + "asynchronous": True, + "skip_instance_cache": True, + **_LISTINGS, + } + + valid.assert_called_once_with(certificate) + + +@pytest.mark.asyncio +async def test_expired_inactive_oidc_refreshes_once_and_persists( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An inactive Science Platform Server uses its IDP and persists refresh.""" + config = _config( + credential=oidc_credential( + "inactive", + access="old-access-secret", + refresh="refresh-secret", + access_expiry=1.0, + ) + ) + config.save() + refresh = AsyncMock( + return_value={ + "access_token": "new-access-secret", + "expires_at": 9_999_999_999.0, + } + ) + monkeypatch.setattr("canfar.client.oidc.refresh", refresh) + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with _vospace("archive")() as filesystem: + assert filesystem.kwargs["token"] == "new-access-secret" + + refresh.assert_awaited_once_with( + "https://oidc.example.com/token", + "test-client", + "test-secret", + "refresh-secret", + ) + persisted = Configuration() # ty: ignore[missing-argument] + saved = persisted.get_credential("inactive") + assert saved.mode == "oidc" + assert saved.token.access is not None + assert saved.token.access.get_secret_value() == "new-access-secret" + + +@pytest.mark.asyncio +async def test_valid_saved_oidc_access_token_is_reused( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A current saved OIDC Authentication Record needs no refresh.""" + _config(credential=oidc_credential("inactive", access="current-token")).save() + refresh = AsyncMock() + monkeypatch.setattr("canfar.client.oidc.refresh", refresh) + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with _vospace("archive")() as filesystem: + assert filesystem.kwargs["token"] == "current-token" + + refresh.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_saved_x509_is_validated_before_construction( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Saved X.509 material becomes only an inspected literal certfile path.""" + certificate = tmp_path / "saved.pem" + _config(credential=x509_credential("inactive", path=certificate)).save() + inspect = Mock() + + def inspect_certificate(path: Path) -> dict[str, object]: + inspect(path) + return {"path": path.as_posix(), "expiry": 9_999_999_999.0} + + monkeypatch.setattr("canfar.client.x509.inspect", inspect_certificate) + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with _vospace("archive")() as filesystem: + assert filesystem.kwargs["certfile"] == certificate.as_posix() + + inspect.assert_called_once_with(certificate) + + +@pytest.mark.asyncio +async def test_runtime_x509_overrides_saved_authentication_record( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A validated runtime certificate wins over the saved Authentication Record.""" + certificate = tmp_path / "runtime.pem" + _config(credential=oidc_credential("inactive")).save() + monkeypatch.setattr( + "canfar.client.x509.inspect", + lambda path: {"path": path.as_posix(), "expiry": 9_999_999_999.0}, + ) + valid = Mock(return_value=certificate.as_posix()) + monkeypatch.setattr("canfar.client.x509.valid", valid) + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + async with _vospace("archive", certificate=certificate)() as filesystem: + assert filesystem.kwargs["certfile"] == certificate.as_posix() + + valid.assert_called_once_with(certificate) + + +@pytest.mark.asyncio +async def test_invalid_saved_x509_fails_before_vospace( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """An invalid X.509 Authentication Record fails with a clean login hint.""" + certificate = tmp_path / "invalid.pem" + _config(credential=x509_credential("inactive", path=certificate)).save() + monkeypatch.setattr( + "canfar.client.x509.inspect", + Mock(side_effect=ValueError("certificate parse detail")), + ) + constructor = Mock() + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", constructor) + + with pytest.raises(AuthContextError, match="canfar login") as exc_info: + async with _vospace("archive")(): + pass + + assert "certificate parse detail" not in str(exc_info.value) + constructor.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("exit_kind", ["error", "cancel"]) +async def test_source_closes_on_failure_and_cancellation( + exit_kind: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Context exit always closes a yielded filesystem.""" + _config(credential=oidc_credential("inactive")).save() + filesystems: list[_Filesystem] = [] + + def build(endpoint: str, **kwargs: Any) -> _Filesystem: + filesystem = _Filesystem(endpoint, **kwargs) + filesystems.append(filesystem) + return filesystem + + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", build) + + async def use_source() -> None: + async with _vospace("archive")(): + if exit_kind == "error": + raise RuntimeError + await asyncio.Event().wait() + + task = asyncio.create_task(use_source()) + await asyncio.sleep(0) + if exit_kind == "error": + with pytest.raises(RuntimeError): + await task + else: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert filesystems[0].closed is True + + +@pytest.mark.asyncio +async def test_unrefreshable_oidc_fails_secret_safe_before_vospace( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Bad saved Authentication Record reports a secret-safe login hint.""" + _config( + credential=oidc_credential( + "inactive", + access="old-access-secret", + refresh="refresh-secret", + access_expiry=1.0, + refresh_expiry=1.0, + ) + ).save() + constructor = AsyncMock() + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", constructor) + + with pytest.raises(AuthContextError, match="canfar login") as exc_info: + async with _vospace("archive")(): + pass + + message = str(exc_info.value) + assert "old-access-secret" not in message + assert "refresh-secret" not in message + constructor.assert_not_called() + + +@pytest.mark.asyncio +async def test_empty_saved_oidc_token_fails_cleanly( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An empty saved token cannot fall through to certificate construction.""" + _config(credential=oidc_credential("inactive", access="")).save() + constructor = Mock() + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", constructor) + + with pytest.raises(AuthContextError, match="canfar login"): + async with _vospace("archive")(): + pass + + constructor.assert_not_called() + + +class TestPublicSurface: + """Tests for the public Storage Identifier accessors.""" + + def test_identifiers_lists_configured_services_and_local(self) -> None: + """Every configured Storage Identifier is listed, with local last.""" + _config(credential=x509_credential("inactive")).save() + + assert storage.identifiers() == ["archive", "local"] + + def test_local_identifier_returns_a_local_filesystem(self) -> None: + """The reserved local identifier needs no credential.""" + assert isinstance(storage.filesystem("local"), LocalFileSystem) + assert isinstance(storage.local, LocalFileSystem) + + def test_attribute_access_builds_a_filesystem( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """A Storage Identifier resolves as a module attribute.""" + certificate = tmp_path / "saved.pem" + _config(credential=x509_credential("inactive", path=certificate)).save() + monkeypatch.setattr( + "canfar.client.x509.inspect", + lambda path: {"path": path.as_posix(), "expiry": 9_999_999_999.0}, + ) + monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) + + filesystem = storage.archive + + assert filesystem.endpoint == "https://inactive.example/vospace" + assert filesystem.kwargs["certfile"] == certificate.as_posix() + assert filesystem.kwargs["use_listings_cache"] is True + + def test_unknown_identifier_raises_attribute_error(self) -> None: + """An unconfigured name is an AttributeError naming what is available.""" + _config(credential=x509_credential("inactive")).save() + + with pytest.raises(AttributeError, match="archive"): + _ = storage.missing + + def test_private_names_are_not_treated_as_identifiers(self) -> None: + """Dunder lookups must not attempt a filesystem build.""" + with pytest.raises(AttributeError): + _ = storage.__wrapped__ + + def test_dir_offers_identifiers_for_completion(self) -> None: + """Tab completion exposes identifiers alongside the module functions.""" + _config(credential=x509_credential("inactive")).save() + + listed = dir(storage) + + assert {"archive", "local", "filesystem", "identifiers"} <= set(listed) diff --git a/uv.lock b/uv.lock index 94ce2ee3..60e6648c 100644 --- a/uv.lock +++ b/uv.lock @@ -118,6 +118,7 @@ dependencies = [ { name = "cadcutils" }, { name = "click" }, { name = "defusedxml" }, + { name = "fsspec-cli" }, { name = "httpx", extra = ["http2"] }, { name = "humanize" }, { name = "pydantic" }, @@ -127,6 +128,7 @@ dependencies = [ { name = "rich" }, { name = "segno" }, { name = "typer" }, + { name = "vosfs" }, ] [package.dev-dependencies] @@ -158,6 +160,7 @@ requires-dist = [ { name = "cadcutils", specifier = ">=1.5.4" }, { name = "click", specifier = ">=8.4.1" }, { name = "defusedxml", specifier = ">=0.7.1" }, + { name = "fsspec-cli", git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.7.0" }, { name = "httpx", extras = ["http2"], specifier = ">=0.28.1" }, { name = "humanize", specifier = ">=4.12.3" }, { name = "pydantic", specifier = ">=2.9.2" }, @@ -167,6 +170,7 @@ requires-dist = [ { name = "rich", specifier = ">=13.9.4" }, { name = "segno", specifier = ">=1.6.6" }, { name = "typer", specifier = ">=0.16.0" }, + { name = "vosfs", git = "https://github.com/shinybrar/vosfs?rev=v0.8.0" }, ] [package.metadata.requires-dev] @@ -628,7 +632,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -662,6 +666,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/02/be4a57b60c7149b55b9e3b3c13f609cd8eb5307c751f22bd8fb8d262e75b/filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51", size = 46036, upload-time = "2026-07-08T05:46:57.53Z" }, ] +[[package]] +name = "fsspec" +version = "2026.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, +] + +[[package]] +name = "fsspec-cli" +version = "0.7.0" +source = { git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.7.0#a2ef38d4014a10d5ddcf9a1f8eeaf682fde23ed8" } +dependencies = [ + { name = "fsspec" }, + { name = "typer" }, +] + [[package]] name = "ghp-import" version = "2.1.0" @@ -824,17 +846,17 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.11'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "jedi", marker = "python_full_version < '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, - { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "stack-data", marker = "python_full_version < '3.11'" }, - { name = "traitlets", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -850,18 +872,18 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" } wheels = [ @@ -873,7 +895,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -2167,6 +2189,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/7c/4e7225d46d634a0d8d534dd8a6ce0c319d09b4d0cf0337eb314ca4789d8c/virtualenv-21.6.1-py3-none-any.whl", hash = "sha256:afe991df855715a2b2f60edfcc0107ef95a79fdfd8cb4cdaa71603d1c12e463b", size = 5506392, upload-time = "2026-07-10T19:33:51.629Z" }, ] +[[package]] +name = "vosfs" +version = "0.8.0" +source = { git = "https://github.com/shinybrar/vosfs?rev=v0.8.0#1e2e4a9835693d7894916f5ee8cdd721bed7f853" } +dependencies = [ + { name = "defusedxml" }, + { name = "fsspec" }, + { name = "httpx" }, +] + [[package]] name = "watchdog" version = "6.0.0"