diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py index 95067328..c6198e27 100755 --- a/scripts/ccpp_prebuild.py +++ b/scripts/ccpp_prebuild.py @@ -13,6 +13,7 @@ import sys # CCPP framework imports +from common import lowercase_keys_and_values from common import encode_container, decode_container, decode_container_as_dict from common import CCPP_STAGES, CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE from common import STANDARD_VARIABLE_TYPES, STANDARD_INTEGER_TYPE, CCPP_TYPE @@ -361,11 +362,12 @@ def check_schemes_in_suites(arguments, suites): """Check that all schemes that are requested in the suites exist""" success = True logging.info("Checking for existence of schemes in suites ...") + argument_keys = [x.lower() for x in arguments.keys()] for suite in suites: for group in suite.groups: for subcycle in group.subcycles: for scheme_name in subcycle.schemes: - if not scheme_name in arguments.keys(): + if not scheme_name in argument_keys: success = False logging.critical("Scheme {} in suite {} cannot be found".format(scheme_name, suite.name)) return success @@ -401,19 +403,19 @@ def filter_metadata(metadata, arguments, dependencies, schemes_in_files, suites) # Filter argument lists for scheme in arguments.keys(): for suite in suites: - if scheme in suite.all_schemes_called: + if scheme.lower() in suite.all_schemes_called: arguments_filtered[scheme] = arguments[scheme] break # Filter dependencies for scheme in dependencies.keys(): for suite in suites: - if scheme in suite.all_schemes_called: + if scheme.lower() in suite.all_schemes_called: dependencies_filtered[scheme] = dependencies[scheme] break # Filter schemes_in_files for scheme in schemes_in_files.keys(): for suite in suites: - if scheme in suite.all_schemes_called: + if scheme.lower() in suite.all_schemes_called: schemes_in_files_filtered[scheme] = schemes_in_files[scheme] return (success, metadata_filtered, arguments_filtered, dependencies_filtered, schemes_in_files_filtered) @@ -749,6 +751,9 @@ def main(): logging.info('CCPP prebuild clean completed successfully, exiting.') sys.exit(0) + # Convert TYPEDEFS_NEW_METATA config to lowercase + config['typedefs_new_metadata'] = lowercase_keys_and_values(config['typedefs_new_metadata']) + # If no suite definition files were given, get all of them if not sdfs: (success, sdfs) = get_all_suites(config['suites_dir']) diff --git a/scripts/ccpp_track_variables.py b/scripts/ccpp_track_variables.py index 64dfc518..6d3f6d0b 100755 --- a/scripts/ccpp_track_variables.py +++ b/scripts/ccpp_track_variables.py @@ -10,6 +10,7 @@ from metadata_table import find_scheme_names, parse_metadata_file from ccpp_prebuild import import_config, gather_variable_definitions from mkstatic import Suite +from common import lowercase_keys from parse_checkers import registered_fortran_ddt_names from parse_tools import init_log, set_log_level from framework_env import CCPPFrameworkEnv @@ -78,7 +79,7 @@ def create_metadata_filename_dict(metapath): # The above returns a list of schemes in each filename, but # we want a dictionary of schemes associated with filenames: for scheme in schemes: - metadata_dict[scheme] = scheme_fn + metadata_dict[scheme.lower()] = scheme_fn return metadata_dict diff --git a/scripts/common.py b/scripts/common.py index 84520222..234e2521 100755 --- a/scripts/common.py +++ b/scripts/common.py @@ -101,13 +101,14 @@ def split_var_name_and_array_reference(var_name): def encode_container(*args): """Encodes a container, i.e. the location of a metadata table for CCPP. Currently, there are three possibilities with different numbers of input - arguments: module, module+typedef, module+scheme+subroutine.""" + arguments: module, module+typedef, module+scheme+subroutine. Convert all + names to lowercase to support the new case-insensitive capgen parser.""" if len(args)==3: - container = 'MODULE_{0} SCHEME_{1} SUBROUTINE_{2}'.format(*args) + container = 'MODULE_{0} SCHEME_{1} SUBROUTINE_{2}'.format(*[arg.lower() for arg in args]) elif len(args)==2: - container = 'MODULE_{0} TYPE_{1}'.format(*args) + container = 'MODULE_{0} TYPE_{1}'.format(*[arg.lower() for arg in args]) elif len(args)==1: - container = 'MODULE_{0}'.format(*args) + container = 'MODULE_{0}'.format(*[arg.lower() for arg in args]) else: raise Exception("encode_container not implemented for {0} arguments".format(len(args))) return container @@ -184,3 +185,49 @@ def string_to_python_identifier(string): return string else: raise Exception("Resulting string '{0}' is not a valid Python identifier".format(string)) + +# New utilities added 2025/07/25 for dealing with case-insensitivity changes from capgen. Used to convert XML data read with the xml library and other ccpp_prebuild dictionaries + +def lowercase_keys_and_values(d): + """Recursively convert all keys and values in a regular dictionary to lowercase""" + if isinstance(d, dict): + return { + (k.lower() if isinstance(k, str) else k): + lowercase_keys_and_values(v) + for k, v in d.items() + } + elif isinstance(d, list): + return [lowercase_keys_and_values(item) for item in d] + elif isinstance(d, str): + return d.lower() + else: + return d + +def lowercase_keys(d): + """Recursively convert all keys in an OrderedDict to lowercase""" + if isinstance(d, OrderedDict): + new_dict = OrderedDict() + for k, v in d.items(): + new_key = k.lower() if isinstance(k, str) else k + new_dict[new_key] = lowercase_keys(v) + return new_dict + elif isinstance(d, list): + return [lowercase_keys(item) for item in d] + else: + return d + +def lowercase_xml(element): + """Recursively convert XML elements to lowercase""" + # Lowercase the tag name + element.tag = element.tag.lower() + # Lowercase the text content, if it exists + if element.text: + element.text = element.text.lower() + if element.tail: + element.tail = element.tail.lower() + # Lowercase attribute keys and values + element.attrib = {k.lower(): v.lower() for k, v in element.attrib.items()} + # Recurse into child elements + for i in range(len(element)): + element[i] = lowercase_xml(element[i]) + return element diff --git a/scripts/metadata_parser.py b/scripts/metadata_parser.py index e8c3f987..31078b64 100755 --- a/scripts/metadata_parser.py +++ b/scripts/metadata_parser.py @@ -236,12 +236,12 @@ def read_new_metadata(filename, module_name, table_name, scheme_name = None, sub # units from capgen to the "+"-format (i.e. "m2 s-2" --> "m+2 s-2") units = insert_plus_sign_for_positive_exponents(new_var.get_prop_value('units')) - var = Var(standard_name = standard_name, + var = Var(standard_name = standard_name.lower(), long_name = new_var.get_prop_value('long_name') + legacy_note, units = units, - local_name = new_var.get_prop_value('local_name'), + local_name = new_var.get_prop_value('local_name').lower(), type = new_var.get_prop_value('type').lower(), - dimensions = dimensions, + dimensions = [dim.lower() for dim in dimensions], container = container, kind = kind, intent = new_var.get_prop_value('intent'), diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 618d3a86..e33164a3 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -13,6 +13,7 @@ import xml.etree.ElementTree as ET from common import encode_container +from common import lowercase_keys, lowercase_xml from common import CCPP_STAGES from common import CCPP_T_INSTANCE_VARIABLE, CCPP_ERROR_CODE_VARIABLE, CCPP_ERROR_MSG_VARIABLE, CCPP_LOOP_COUNTER, CCPP_LOOP_EXTENT from common import CCPP_BLOCK_NUMBER, CCPP_BLOCK_COUNT, CCPP_BLOCK_SIZES, CCPP_THREAD_NUMBER, CCPP_THREAD_COUNT, CCPP_INTERNAL_VARIABLES @@ -291,6 +292,24 @@ class API(object): public :: {subroutines} contains + + ! Necessary to convert incoming suite and group names to lowercase + function to_lower(str) result(lower) + implicit none + character(len=*), intent(in) :: str + character(len=len(str)) :: lower + integer, parameter :: upper_to_lower = ichar('a') - ichar('A') + integer :: i, ichar_val + + do i = 1, len(str) + ichar_val = ichar(str(i:i)) + if (ichar_val >= ichar('A') .and. ichar_val <= ichar('Z')) then + lower(i:i) = char(ichar_val + upper_to_lower) + else + lower(i:i) = str(i:i) + end if + end do + end function to_lower ''' sub = ''' @@ -310,7 +329,7 @@ class API(object): {suite_switch} else - write({ccpp_var_name}%errmsg,'(*(a))') 'Invalid suite ' // trim(suite_name) + write({ccpp_var_name}%errmsg,'(*(a))') 'Invalid suite ' // to_lower(trim(suite_name)) ierr = 1 end if @@ -436,7 +455,7 @@ def write(self): clause = 'else if' argument_list_group = create_argument_list_wrapped_explicit(group.arguments[ccpp_stage]) group_calls += ''' - {clause} (trim(group_name)=="{group_name}") then + {clause} (to_lower(trim(group_name))=="{group_name}") then ierr = {suite_name}_{group_name}_{stage}_cap({arguments})'''.format(clause=clause, suite_name=group.suite, group_name=group.name, @@ -444,7 +463,7 @@ def write(self): arguments=argument_list_group) group_calls += ''' else - write({ccpp_var_name}%errmsg, '(*(a))') 'Group ' // trim(group_name) // ' not found' + write({ccpp_var_name}%errmsg, '(*(a))') 'Group ' // to_lower(trim(group_name)) // ' not found' ierr = 1 end if '''.format(ccpp_var_name=ccpp_var.local_name, group_name=group.name) @@ -463,7 +482,7 @@ def write(self): else: clause = 'else if' suite_switch += ''' - {clause} (trim(suite_name)=="{suite_name}") then + {clause} (to_lower(trim(suite_name))=="{suite_name}") then if (present(group_name)) then {group_calls} @@ -690,19 +709,8 @@ def parse(self, make_call_tree=False): return success tree = ET.parse(self._sdf_name) - suite_xml = tree.getroot() + suite_xml = lowercase_xml(tree.getroot()) self._name = suite_xml.get('name') - # Validate name of suite in XML tag against filename; could be moved to common.py - if not (os.path.basename(self._sdf_name) == '{}.xml'.format(self._name)): - if (os.path.basename(self._sdf_name) == 'suite_{}.xml'.format(self._name)): - logging.debug("Parsing suite using legacy naming convention") - logging.debug(f"Filename {os.path.basename(self._sdf_name)}") - logging.debug(f"Suite name {format(self._name)}") - else: - logging.critical("Invalid suite name {0} in suite definition file {1}.".format( - self._name, self._sdf_name)) - success = False - return success # Check if suite name is too long if len(self._name) > SUITE_NAME_MAX_CHARS: @@ -726,15 +734,15 @@ def parse(self, make_call_tree=False): self._call_tree[group_xml.attrib['name']] = [] # Add suite-wide init scheme to group 'init', similar for finalize - if group_xml.tag.lower() == 'init' or group_xml.tag.lower() == 'finalize': + if group_xml.tag == 'init' or group_xml.tag == 'finalize': self._all_schemes_called.append(group_xml.text) - self._all_subroutines_called.append(group_xml.text + '_' + group_xml.tag.lower()) + self._all_subroutines_called.append(group_xml.text + '_' + group_xml.tag) schemes = [group_xml.text] subcycles.append(Subcycle(loop=1, schemes=schemes)) - if group_xml.tag.lower() == 'init': - self._groups.append(Group(name=group_xml.tag.lower(), subcycles=subcycles, suite=self._name, init=True)) - elif group_xml.tag.lower() == 'finalize': - self._groups.append(Group(name=group_xml.tag.lower(), subcycles=subcycles, suite=self._name, finalize=True)) + if group_xml.tag == 'init': + self._groups.append(Group(name=group_xml.tag, subcycles=subcycles, suite=self._name, init=True)) + elif group_xml.tag == 'finalize': + self._groups.append(Group(name=group_xml.tag, subcycles=subcycles, suite=self._name, finalize=True)) continue # Parse subcycles of all regular groups @@ -761,7 +769,6 @@ def parse(self, make_call_tree=False): self._all_schemes_called = list(set(self._all_schemes_called)) self._all_subroutines_called = list(set(self._all_subroutines_called)) - return success def print_debug(self): @@ -1058,10 +1065,13 @@ def __init__(self, **kwargs): for key, value in kwargs.items(): setattr(self, "_"+key, value) - def write(self, metadata_request, metadata_define, arguments, debug): + def write(self, metadata_request, metadata_define, arguments_in, debug): """Create caps for all stages of this group. Add additional code for debugging if debug flag is True.""" + # First, convert all keys in arguments_in to lowercase (recursively) + arguments = lowercase_keys(arguments_in) + # Create an inverse lookup table of local variable names defined (by the host model) and standard names standard_name_by_local_name_define = collections.OrderedDict() for standard_name in metadata_define.keys(): @@ -1127,8 +1137,7 @@ def write(self, metadata_request, metadata_define, arguments, debug): # First, add a few mandatory variables to the list of required # variables. This is mostly for handling horizontal dimensions - # correctly for the different CCPP phases and for cases when - # blocked data structures or chunked arrays are used. + # correctly for the different CCPP phases and for chunked arrays additional_variables_required = [] if CCPP_HORIZONTAL_LOOP_EXTENT in metadata_define.keys(): for add_var in [ CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_LOOP_EXTENT]: diff --git a/test/unit_tests/test_common.py b/test/unit_tests/test_common.py index e95b8184..aa81ee8d 100755 --- a/test/unit_tests/test_common.py +++ b/test/unit_tests/test_common.py @@ -47,18 +47,18 @@ def test_encode_container(self): typename = "COMPLEX" schemename = "testscheme" subroutinename = "testsubroutine" - self.assertEqual(common.encode_container(modulename),f"MODULE_{modulename}") - self.assertEqual(common.encode_container(modulename,typename),f"MODULE_{modulename} TYPE_{typename}") + self.assertEqual(common.encode_container(modulename),f"MODULE_{modulename.lower()}") + self.assertEqual(common.encode_container(modulename,typename),f"MODULE_{modulename.lower()} TYPE_{typename.lower()}") self.assertEqual(common.encode_container(modulename,schemename,subroutinename), - f"MODULE_{modulename} SCHEME_{schemename} SUBROUTINE_{subroutinename}") + f"MODULE_{modulename.lower()} SCHEME_{schemename.lower()} SUBROUTINE_{subroutinename.lower()}") self.assertRaises(Exception,common.encode_container,modulename,typename,schemename,subroutinename) self.assertRaises(Exception,common.encode_container) def test_decode_container(self): """Test decode_container() function""" - modulename = "ABCD1234" - typename = "COMPLEX" + modulename = "abcd1234" + typename = "complex" schemename = "testscheme" subroutinename = "testsubroutine" self.assertEqual(common.decode_container(f"MODULE_{modulename}"),f"MODULE {modulename}")