Skip to content

move input coordinates to extra subsection #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
observingClouds wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

move input coordinates to extra subsection #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
442490b
move input coordinates to extra subsection
observingClouds Oct 30, 2024
ff09707
update schema_version in tests
observingClouds Oct 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 16 additions & 12 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,13 +149,15 @@ inputs:
dims: [time, x, y, altitude]
variables:
u:
altitude:
values: [100,]
units: m
coordinates:
altitude:
values: [100,]
units: m
v:
altitude:
values: [100, ]
units: m
coordinates:
altitude:
values: [100, ]
units: m
dim_mapping:
time:
method: rename
Expand Down Expand Up @@ -256,13 +258,15 @@ inputs:
dims: [time, x, y, altitude]
variables:
u:
altitude:
values: [100,]
units: m
coordinates:
altitude:
values: [100,]
units: m
v:
altitude:
values: [100, ]
units: m
coordinates:
altitude:
values: [100, ]
units: m
dim_mapping:
time:
method: rename
Expand Down
14 changes: 8 additions & 6 deletions example.danra.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,15 @@ inputs:
dims: [time, x, y, altitude]
variables:
u:
altitude:
values: [100,]
units: m
coordinates:
altitude:
values: [100,]
units: m
v:
altitude:
values: [100, ]
units: m
coordinates:
altitude:
values: [100, ]
units: m
dim_mapping:
time:
method: rename
Expand Down
35 changes: 18 additions & 17 deletions mllam_data_prep/ops/loading.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,24 +25,25 @@ def load_and_subset_dataset(fp, variables):
ds_subset = xr.Dataset()
ds_subset.attrs.update(ds.attrs)
if isinstance(variables, dict):
for var, coords_to_sample in variables.items():
for var, var_metadata in variables.items():
da = ds[var]
for coord, sampling in coords_to_sample.items():
coord_values = sampling.values
try:
da = da.sel(**{coord: coord_values})
except KeyError as ex:
raise KeyError(
f"Could not find the all coordinate values `{coord_values}` in "
f"coordinate `{coord}` in the dataset"
) from ex
expected_units = sampling.units
coord_units = da[coord].attrs.get("units", None)
if coord_units is not None and coord_units != expected_units:
raise ValueError(
f"Expected units {expected_units} for coordinate {coord}"
f" in variable {var} but got {coord_units}"
)
if var_metadata.get("coordinates", False):
for coord, sampling in var_metadata["coordinates"].items():
coord_values = sampling.values
try:
da = da.sel(**{coord: coord_values})
except KeyError as ex:
raise KeyError(
f"Could not find the all coordinate values `{coord_values}` in "
f"coordinate `{coord}` in the dataset"
) from ex
expected_units = sampling.units
coord_units = da[coord].attrs.get("units", None)
if coord_units is not None and coord_units != expected_units:
raise ValueError(
f"Expected units {expected_units} for coordinate {coord}"
f" in variable {var} but got {coord_units}"
)
ds_subset[var] = da
elif isinstance(variables, list):
try:
Expand Down
20 changes: 11 additions & 9 deletions tests/test_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import mllam_data_prep as mdp

INVALID_EXTRA_FIELDS_CONFIG_YAML = """
schema_version: v0.1.0
schema_version: v0.2.0
dataset_version: v0.1.0

output:
Expand All @@ -21,7 +21,7 @@
"""

MISSING_FIELDS_CONFIG_YAML = """
schema_version: v0.1.0
schema_version: v0.2.0
dataset_version: v0.1.0
"""

Expand All @@ -36,7 +36,7 @@ def test_get_config_issues():


VALID_EXAMPLE_CONFIG_YAML = """
schema_version: v0.1.0
schema_version: v0.2.0
dataset_version: v0.1.0

output:
Expand Down Expand Up @@ -71,13 +71,15 @@ def test_get_config_issues():
dims: [time, x, y, altitude]
variables:
u:
altitude:
values: [100, ]
units: m
coordinates:
altitude:
values: [100, ]
units: m
v:
altitude:
values: [100, ]
units: m
coordinates:
altitude:
values: [100, ]
units: m
dim_mapping:
time:
method: rename
Expand Down