From a1410924ec804a102eb4bc131b3d78d25ad9f836 Mon Sep 17 00:00:00 2001 From: Tik Date: Sat, 13 Jun 2026 22:30:54 +0300 Subject: [PATCH 1/3] bugfix - ZarrCollectionProvider not working with data_cols setting. --- .../collections_providers/zarr_collection_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydggsapi/dependencies/collections_providers/zarr_collection_provider.py b/pydggsapi/dependencies/collections_providers/zarr_collection_provider.py index 040df04..e027776 100644 --- a/pydggsapi/dependencies/collections_providers/zarr_collection_provider.py +++ b/pydggsapi/dependencies/collections_providers/zarr_collection_provider.py @@ -105,7 +105,7 @@ def get_data(self, zoneIds: List[Any], res: int, datasource_id: str, sql = f"""select {cols} from ds where ("{id_col}" in ({', '.join(f"'{z}'" for z in zoneIds)})) and ({cql_sql}) """ zarr_result = xr.Dataset.from_dataframe(ctx.sql(sql).to_pandas().set_index(id_col)) else: - cols = OrderedSet(ds.data_vars) if ("*" in datasource.data_cols) else OrderdSet(datasource.data_cols) + cols = OrderedSet(ds.data_vars) if ("*" in datasource.data_cols) else OrderedSet(datasource.data_cols) cols = list(cols - OrderedSet(datasource.exclude_data_cols)) idx_mask = ds[id_col].isin(np.array(zoneIds, dtype=ds[id_col].dtype)) zarr_result = ds.sel({id_col: idx_mask}) From 22f886a58875ec65923e209adbf1832a3fa84c5d Mon Sep 17 00:00:00 2001 From: Tik Date: Sat, 13 Jun 2026 22:57:44 +0300 Subject: [PATCH 2/3] ZarrCollectionProvider: a faster way to select data from dataset --- .../collections_providers/zarr_collection_provider.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pydggsapi/dependencies/collections_providers/zarr_collection_provider.py b/pydggsapi/dependencies/collections_providers/zarr_collection_provider.py index e027776..f8bde10 100644 --- a/pydggsapi/dependencies/collections_providers/zarr_collection_provider.py +++ b/pydggsapi/dependencies/collections_providers/zarr_collection_provider.py @@ -107,8 +107,10 @@ def get_data(self, zoneIds: List[Any], res: int, datasource_id: str, else: cols = OrderedSet(ds.data_vars) if ("*" in datasource.data_cols) else OrderedSet(datasource.data_cols) cols = list(cols - OrderedSet(datasource.exclude_data_cols)) - idx_mask = ds[id_col].isin(np.array(zoneIds, dtype=ds[id_col].dtype)) - zarr_result = ds.sel({id_col: idx_mask}) + #idx_mask = ds[id_col].isin(np.array(zoneIds, dtype=ds[id_col].dtype)) + #zarr_result = ds.sel({id_col: idx_mask}) + zarr_result = ds.sel({id_col: np.array(zoneIds, dtype=ds[id_col].dtype)}, method="pad") + zarr_result = zarr_result.drop_duplicates(id_col, keep='first') zarr_result = zarr_result[cols] except Exception as e: # Zarr will raise exception if nothing matched From b285ccc92079fbb7cf9c675e9665457f5ad576f3 Mon Sep 17 00:00:00 2001 From: Tik Date: Sun, 14 Jun 2026 00:56:50 +0300 Subject: [PATCH 3/3] ZarrCollectionProvider: changed to use xql to select data for performance --- .../collections_providers/zarr_collection_provider.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pydggsapi/dependencies/collections_providers/zarr_collection_provider.py b/pydggsapi/dependencies/collections_providers/zarr_collection_provider.py index f8bde10..7b600ba 100644 --- a/pydggsapi/dependencies/collections_providers/zarr_collection_provider.py +++ b/pydggsapi/dependencies/collections_providers/zarr_collection_provider.py @@ -109,8 +109,12 @@ def get_data(self, zoneIds: List[Any], res: int, datasource_id: str, cols = list(cols - OrderedSet(datasource.exclude_data_cols)) #idx_mask = ds[id_col].isin(np.array(zoneIds, dtype=ds[id_col].dtype)) #zarr_result = ds.sel({id_col: idx_mask}) - zarr_result = ds.sel({id_col: np.array(zoneIds, dtype=ds[id_col].dtype)}, method="pad") - zarr_result = zarr_result.drop_duplicates(id_col, keep='first') + # zarr_result = ds.sel({id_col: np.array(zoneIds, dtype=ds[id_col].dtype)}, method="nearest", tolerance=0.5) + #zarr_result = zarr_result.drop_duplicates(id_col, keep='first') + ctx = xql.XarrayContext() + ctx.from_dataset('ds', ds) + sql = f"""select * from ds where ("{id_col}" in ({', '.join(f"'{z}'" for z in zoneIds)}))""" + zarr_result = xr.Dataset.from_dataframe(ctx.sql(sql).to_pandas().set_index(id_col)) zarr_result = zarr_result[cols] except Exception as e: # Zarr will raise exception if nothing matched