Skip to content

Commit 476b8dc

Browse files
mauzey1mauzey1
authored
Merge pull request #906 from PCMDI/895-user-defined-chunking
Add `cmor_set_chunking` function
2 parents 66ea3fe + 6b20775 commit 476b8dc

10 files changed

Lines changed: 226 additions & 134 deletions

‎Lib/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
has_cur_dataset_attribute, set_variable_attribute, get_variable_attribute,
1414
has_variable_attribute, get_final_filename, set_deflate, set_zstandard,
1515
set_quantize, set_furtherinfourl, set_climatology, get_climatology,
16-
set_terminate_signal, get_terminate_signal)
16+
set_terminate_signal, get_terminate_signal, set_chunking)
1717

1818
try:
1919
from check_CMOR_compliant import checkCMOR

‎Lib/pywrapper.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1112,6 +1112,26 @@ def set_quantize(var_id, quantize_mode, quantize_nsd):
11121112
return _cmor.set_quantize(var_id, quantize_mode, quantize_nsd)
11131113

11141114

1115+
def set_chunking(var_id, chunking_dimensions):
1116+
"""Sets chunking dimensions on a cmor variable
1117+
Usage:
1118+
cmor.set_chunking(var_id, chunking_dimensions)
1119+
Where:
1120+
var_id: is cmor variable id
1121+
chunking_dimensions: A list of NetCDF chunking dimensions for
1122+
the variable. The chunking dimensions must be a list of
1123+
positive integers that is the same length as the number of axes
1124+
for the variable. The list must follow the order of the
1125+
variable's list of axes.
1126+
1127+
Any negative values in chunking_dimensions will make CMOR warn
1128+
that the value will be replaced with zero.
1129+
1130+
"""
1131+
chunk_dims = numpy.ascontiguousarray(chunking_dimensions, dtype=numpy.int32)
1132+
return _cmor.set_chunking(var_id, chunk_dims)
1133+
1134+
11151135
def has_variable_attribute(var_id, name):
11161136
"""determines if the a cmor variable has an attribute
11171137
Usage:

‎Src/_cmormodule.c‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,53 @@ static PyObject *PyCMOR_set_quantize(PyObject * self, PyObject * args)
197197
return (Py_BuildValue("i", ierr));
198198
}
199199

200+
/************************************************************************/
201+
/* PyCMOR_set_chunking() */
202+
/************************************************************************/
203+
static PyObject *PyCMOR_set_chunking(PyObject * self, PyObject * args)
204+
{
205+
signal(signal_to_catch, signal_handler);
206+
int ierr, var_id;
207+
PyObject *chunk_dims_obj;
208+
PyArrayObject *chunk_dims_arr = NULL;
209+
int *chunk_dims;
210+
char *name, *text_vals;
211+
212+
if (!PyArg_ParseTuple
213+
(args, "iO", &var_id, &chunk_dims_obj))
214+
return NULL;
215+
216+
217+
if (chunk_dims_obj == Py_None) {
218+
chunk_dims = NULL;
219+
} else {
220+
chunk_dims_arr =
221+
(PyArrayObject *) PyArray_ContiguousFromObject(chunk_dims_obj,
222+
NPY_INT, 1, 0);
223+
224+
if (PyArray_NDIM(chunk_dims_arr) != 1) {
225+
printf("ok we need to pass contiguous flattened arrays only!\n");
226+
return NULL;
227+
}
228+
229+
chunk_dims = (int *)PyArray_DATA(chunk_dims_arr);
230+
}
231+
232+
ierr = cmor_set_chunking(var_id, chunk_dims);
233+
234+
if (chunk_dims_arr != NULL) {
235+
Py_DECREF(chunk_dims_arr);
236+
}
237+
238+
if (ierr != 0 || raise_exception) {
239+
raise_exception = 0;
240+
PyErr_Format(CMORError, exception_message, "set_chunking");
241+
return NULL;
242+
}
243+
244+
return (Py_BuildValue("i", ierr));
245+
}
246+
200247
/************************************************************************/
201248
/* PyCMOR_set_variable_attribute() */
202249
/************************************************************************/
@@ -1246,6 +1293,7 @@ static PyMethodDef MyExtractMethods[] = {
12461293
{"set_deflate", PyCMOR_set_deflate, METH_VARARGS},
12471294
{"set_zstandard", PyCMOR_set_zstandard, METH_VARARGS},
12481295
{"set_quantize", PyCMOR_set_quantize, METH_VARARGS},
1296+
{"set_chunking", PyCMOR_set_chunking, METH_VARARGS},
12491297
{"set_terminate_signal", PyCMOR_set_terminate_signal, METH_VARARGS},
12501298
{"get_terminate_signal", PyCMOR_get_terminate_signal, METH_VARARGS},
12511299
{NULL, NULL} /*sentinel */

‎Src/cmor.c‎

Lines changed: 48 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -814,8 +814,12 @@ void cmor_reset_variable(int var_id)
814814
if (cmor_vars[var_id].values != NULL) {
815815
free(cmor_vars[var_id].values);
816816
}
817+
if (cmor_vars[var_id].chunking_dimensions != NULL) {
818+
free(cmor_vars[var_id].chunking_dimensions);
819+
}
817820

818821
cmor_vars[var_id].values = NULL;
822+
cmor_vars[var_id].chunking_dimensions = NULL;
819823
cmor_vars[var_id].first_time = -999.;
820824
cmor_vars[var_id].last_time = -999.;
821825
cmor_vars[var_id].first_bound = 1.e20;
@@ -3654,8 +3658,7 @@ void cmor_define_dimensions(int var_id, int ncid,
36543658
int ncafid, double *time_bounds,
36553659
int *nc_dim,
36563660
int *nc_vars, int *nc_bnds_vars,
3657-
int *nc_vars_af,
3658-
size_t * nc_dim_chunking, int *dim_bnds,
3661+
int *nc_vars_af, int *dim_bnds,
36593662
int *zfactors, int *nc_zfactors,
36603663
int *nc_dim_af, int *nzfactors)
36613664
{
@@ -3698,15 +3701,6 @@ void cmor_define_dimensions(int var_id, int ncid,
36983701
if ((i == 0) && (cmor_axes[nAxisID].axis == 'T'))
36993702
j = NC_UNLIMITED;
37003703

3701-
if ((cmor_axes[nAxisID].axis == 'X')
3702-
|| (cmor_axes[nAxisID].axis == 'Y')) {
3703-
nc_dim_chunking[i] = j;
3704-
} else if (cmor_axes[nAxisID].isgridaxis == 1) {
3705-
nc_dim_chunking[i] = j;
3706-
} else {
3707-
nc_dim_chunking[i] = 1;
3708-
}
3709-
37103704
ierr = nc_def_dim(ncid, cmor_axes[nAxisID].id, j, &nc_dim[i]);
37113705
if (ierr != NC_NOERR) {
37123706
ierr = nc_enddef(ncid);
@@ -4909,7 +4903,6 @@ int cmor_write(int var_id, void *data, char type, char *file_suffix,
49094903
char ctmp2[CMOR_MAX_STRING];
49104904
char msg[CMOR_MAX_STRING];
49114905
char appending_to[CMOR_MAX_STRING];
4912-
size_t nc_dim_chunking[CMOR_MAX_AXES];
49134906
int nc_vars[CMOR_MAX_VARIABLES];
49144907
int nc_vars_af[CMOR_MAX_VARIABLES];
49154908
int nc_dim_af[CMOR_MAX_DIMENSIONS];
@@ -5159,7 +5152,7 @@ int cmor_write(int var_id, void *data, char type, char *file_suffix,
51595152
/* define dimensions in NetCDF file */
51605153
/* -------------------------------------------------------------------- */
51615154
cmor_define_dimensions(var_id, ncid, ncafid, time_bounds, nc_dim,
5162-
nc_vars, nc_bnds_vars, nc_vars_af, nc_dim_chunking,
5155+
nc_vars, nc_bnds_vars, nc_vars_af,
51635156
&dim_bnds, zfactors, nc_zfactors, nc_dim_af, &nzfactors);
51645157
/* -------------------------------------------------------------------- */
51655158
/* Store the dimension id for reuse when writing */
@@ -5219,7 +5212,7 @@ int cmor_write(int var_id, void *data, char type, char *file_suffix,
52195212
cmor_create_var_attributes(var_id, ncid, ncafid, nc_vars,
52205213
nc_bnds_vars, nc_vars_af, nc_associated_vars, nc_singletons,
52215214
nc_singletons_bnds, nc_zfactors, zfactors, nzfactors,
5222-
nc_dim_chunking, outname);
5215+
outname);
52235216

52245217

52255218
}
@@ -5327,7 +5320,7 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid,
53275320
int *nc_vars_af, int *nc_associated_vars,
53285321
int *nc_singletons, int *nc_singletons_bnds,
53295322
int *nc_zfactors, int *zfactors, int nzfactors,
5330-
size_t * nc_dim_chunking, char *outname)
5323+
char *outname)
53315324
{
53325325

53335326
size_t starts[2], counts[2];
@@ -5604,41 +5597,47 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid,
56045597
}
56055598
}
56065599

5607-
size_t bytes_per_elem = 0;
5608-
if (pVar->type == 'c')
5609-
bytes_per_elem = sizeof(char);
5610-
else if (pVar->type == 'f')
5611-
bytes_per_elem = sizeof(float);
5612-
else if (pVar->type == 'd')
5613-
bytes_per_elem = sizeof(double);
5614-
else if (pVar->type == 'i')
5615-
bytes_per_elem = sizeof(int);
5616-
else if (pVar->type == 'l')
5617-
bytes_per_elem = sizeof(long);
5618-
56195600
size_t chunking_dims[pVar->ndims];
56205601

5621-
// Create chunking dimensions where multiple timesteps can fit
5622-
// if the chunk size stays under a maximum size.
5623-
size_t bytes_per_timestep = bytes_per_elem;
5624-
for (i = 0; i < pVar->ndims; i++) {
5625-
if(cmor_axes[pVar->axes_ids[i]].axis != 'T') {
5626-
bytes_per_timestep *= cmor_axes[pVar->axes_ids[i]].length;
5602+
if (pVar->chunking_dimensions != NULL) {
5603+
for (i = 0; i < pVar->ndims; i++) {
5604+
chunking_dims[i] = (size_t)pVar->chunking_dimensions[i];
56275605
}
5628-
}
5629-
size_t timesteps_per_chunk = CMOR_TIMESTEP_CHUNK_MAX_BYTES / bytes_per_timestep;
5630-
for (i = 0; i < pVar->ndims; i++) {
5631-
if(cmor_axes[pVar->axes_ids[i]].axis == 'T') {
5632-
if (timesteps_per_chunk > cmor_axes[pVar->axes_ids[i]].length) {
5633-
chunking_dims[i] = cmor_axes[pVar->axes_ids[i]].length;
5634-
} else {
5635-
chunking_dims[i] = timesteps_per_chunk;
5606+
} else {
5607+
size_t bytes_per_elem = 0;
5608+
if (pVar->type == 'c')
5609+
bytes_per_elem = sizeof(char);
5610+
else if (pVar->type == 'f')
5611+
bytes_per_elem = sizeof(float);
5612+
else if (pVar->type == 'd')
5613+
bytes_per_elem = sizeof(double);
5614+
else if (pVar->type == 'i')
5615+
bytes_per_elem = sizeof(int);
5616+
else if (pVar->type == 'l')
5617+
bytes_per_elem = sizeof(long);
5618+
5619+
// Create chunking dimensions where multiple timesteps can fit
5620+
// if the chunk size stays under a maximum size.
5621+
size_t bytes_per_timestep = bytes_per_elem;
5622+
for (i = 0; i < pVar->ndims; i++) {
5623+
if(cmor_axes[pVar->axes_ids[i]].axis != 'T') {
5624+
bytes_per_timestep *= cmor_axes[pVar->axes_ids[i]].length;
56365625
}
5637-
} else {
5638-
if (timesteps_per_chunk == 0) {
5639-
chunking_dims[i] = 0;
5626+
}
5627+
size_t timesteps_per_chunk = CMOR_TIMESTEP_CHUNK_MAX_BYTES / bytes_per_timestep;
5628+
for (i = 0; i < pVar->ndims; i++) {
5629+
if(cmor_axes[pVar->axes_ids[i]].axis == 'T') {
5630+
if (timesteps_per_chunk > cmor_axes[pVar->axes_ids[i]].length) {
5631+
chunking_dims[i] = cmor_axes[pVar->axes_ids[i]].length;
5632+
} else {
5633+
chunking_dims[i] = timesteps_per_chunk;
5634+
}
56405635
} else {
5641-
chunking_dims[i] = cmor_axes[pVar->axes_ids[i]].length;
5636+
if (timesteps_per_chunk == 0) {
5637+
chunking_dims[i] = 0;
5638+
} else {
5639+
chunking_dims[i] = cmor_axes[pVar->axes_ids[i]].length;
5640+
}
56425641
}
56435642
}
56445643
}
@@ -6940,6 +6939,10 @@ int cmor_close_variable(int var_id, char *file_name, int *preserve)
69406939
if (cmor_vars[var_id].values != NULL) {
69416940
free(cmor_vars[var_id].values);
69426941
}
6942+
if (cmor_vars[var_id].chunking_dimensions != NULL) {
6943+
free(cmor_vars[var_id].chunking_dimensions);
6944+
cmor_vars[var_id].chunking_dimensions = NULL;
6945+
}
69436946
for (i = 0; i < cmor_vars[var_id].nattributes; i++) {
69446947
if (strcmp(cmor_vars[var_id].attributes[i], "cell_methods")
69456948
== 0) {

‎Src/cmor_cfortran_interface.c‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,14 @@ int cmor_set_quantize_cff_(int *var_id, int *quantize_mode,
109109
return (cmor_set_quantize(*var_id, *quantize_mode, *quantize_level));
110110
}
111111

112+
/************************************************************************/
113+
/* cmor_set_chunking_cff_() */
114+
/************************************************************************/
115+
int cmor_set_chunking_cff_(int *var_id, int *chunking_dims)
116+
{
117+
return (cmor_set_chunking(*var_id, chunking_dims));
118+
}
119+
112120
/************************************************************************/
113121
/* cmor_get_variable_attribute_cff_() */
114122
/************************************************************************/

‎Src/cmor_fortran_interface.f90‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,15 @@ function cmor_set_quantize_cff(var_id, quantize_mode, quantize_nsd )result (ierr
104104
end function cmor_set_quantize_cff
105105
end interface
106106

107+
interface
108+
function cmor_set_chunking_cff(var_id, chunking_dims )result (ierr)
109+
use iso_c_binding
110+
integer, intent(in) :: var_id
111+
integer, intent(in) :: chunking_dims
112+
integer :: ierr
113+
end function cmor_set_chunking_cff
114+
end interface
115+
107116
interface
108117
function cmor_setup_cff_nolog(path,ncmode,verbosity,mode,crsub) result (j)
109118
integer ncmode,verbosity,mode, j, crsub
@@ -7183,6 +7192,14 @@ function cmor_set_quantize(var_id, quantize_mode, quantize_nsd) result (ierr)
71837192
ierr = cmor_set_quantize_cff(var_id, quantize_mode, quantize_nsd)
71847193
end function cmor_set_quantize
71857194

7195+
function cmor_set_chunking(var_id, chunking_dims) result (ierr)
7196+
implicit none
7197+
integer, intent (in) :: var_id
7198+
integer, intent (in) :: chunking_dims(:)
7199+
integer ierr
7200+
ierr = cmor_set_chunking_cff(var_id, chunking_dims(1))
7201+
end function cmor_set_chunking
7202+
71867203
function cmor_setup_ints(inpath,netcdf_file_action, set_verbosity,&
71877204
exit_control, logfile, create_subdirectories) result(ierr)
71887205
implicit none
@@ -7308,6 +7325,12 @@ function cmor_setup_nc_char(inpath,netcdf_file_action, set_verbosity,&
73087325
nc = CMOR_APPEND_3
73097326
else if (similar(netcdf_file_action,"replace_3") ) then
73107327
nc = CMOR_REPLACE_3
7328+
else if (similar(netcdf_file_action,"preserve_4") ) then
7329+
nc = CMOR_PRESERVE_4
7330+
else if (similar(netcdf_file_action,"append_4") ) then
7331+
nc = CMOR_APPEND_4
7332+
else if (similar(netcdf_file_action,"replace_4") ) then
7333+
nc = CMOR_REPLACE_4
73117334
else !dummy value to generate error from C
73127335
nc =1000
73137336
endif

0 commit comments

Comments
 (0)