diff --git a/README.md b/README.md index 5c7e0408..265b0ac3 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,17 @@ A tool for building a scientific software stack from a recipe for vClusters on C Read the [documentation](https://eth-cscs.github.io/stackinator/) to get started. Create a ticket in our [GitHub issues](https://github.com/eth-cscs/stackinator/issues) if you find a bug, have a feature request or have a question. + +## running tests: + +Use uv to run the tests, which will in turn ensure that the correct dependencies from `pyproject.toml` are used: + +``` +uv run pytest +``` + +Before pushing, apply the linting rules (this calls uv under the hood): + +``` +./lint +``` diff --git a/bin/stack-config b/bin/stack-config index 66496991..02f48700 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -2,6 +2,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ +# "python-magic", # "jinja2", # "jsonschema", # "pyYAML", diff --git a/pyproject.toml b/pyproject.toml index 583f5d46..25fd1ddf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ license-files = ["LICENSE"] dynamic = ["version"] requires-python = ">=3.12" dependencies = [ + "python-magic", "Jinja2", "jsonschema", "PyYAML", diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..d0de0ac4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +attrs==25.4.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +packaging==26.0 +pluggy==1.6.0 +pygments==2.19.2 +pytest==9.0.2 +python-magic==0.4.27 +pyyaml==6.0.3 +referencing==0.37.0 +rpds-py==0.30.0 diff --git a/stackinator/builder.py b/stackinator/builder.py index 2c05e3c1..47a73b05 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, cache, root_logger, spack_util +from . import VERSION, root_logger, spack_util, mirror def install(src, dst, *, ignore=None, symlinks=False): @@ -164,6 +164,7 @@ def environment_meta(self, recipe): self._environment_meta = meta def generate(self, recipe): + """Setup the recipe build environment.""" # make the paths, in case bwrap is not used, directly write to recipe.mount store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) tmp_path = self.path / "tmp" @@ -226,12 +227,13 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( - cache=recipe.mirror, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, + gpg_keys=recipe.mirrors.keys, + cache=recipe.build_cache_mirror, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -312,11 +314,12 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - if recipe.mirror: - dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the build cache mirror: {dst}") - with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") + try: + recipe.mirrors.setup_configs(config_path) + except mirror.MirrorError as err: + self._logger.error(f"Could not set up mirrors.\n{err}") + return 1 # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/cache.py b/stackinator/cache.py deleted file mode 100644 index 24177e33..00000000 --- a/stackinator/cache.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import pathlib - -import yaml - -from . import schema - - -def configuration_from_file(file, mount): - with file.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) - - # validate the yaml - schema.CacheValidator.validate(raw) - - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(raw["root"])) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - raw["root"] = path - - # Put the build cache in a sub-directory named after the mount point. - # This avoids relocation issues. - raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) - - # verify that the key file exists if it was specified - key = raw["key"] - if key is not None: - key = pathlib.Path(os.path.expandvars(key)) - if not key.is_absolute(): - raise FileNotFoundError(f"The build cache key '{key}' is not absolute") - if not key.is_file(): - raise FileNotFoundError(f"The build cache key '{key}' does not exist") - raw["key"] = key - - return raw - - -def generate_mirrors_yaml(config): - path = config["path"].as_posix() - mirrors = { - "mirrors": { - "alpscache": { - "fetch": { - "url": f"file://{path}", - }, - "push": { - "url": f"file://{path}", - }, - } - } - } - - return yaml.dump(mirrors, default_flow_style=False) diff --git a/stackinator/main.py b/stackinator/main.py index 44406215..ec384561 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -81,13 +81,31 @@ def log_header(args): def make_argparser(): parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") - parser.add_argument("-b", "--build", required=True, type=str) + parser.add_argument( + "-b", + "--build", + required=True, + type=str, + help="Where to set up the stackinator build directory. ('/tmp' is not allowed, use '/var/tmp')", + ) parser.add_argument("--no-bwrap", action="store_true", required=False) - parser.add_argument("-r", "--recipe", required=True, type=str) - parser.add_argument("-s", "--system", required=True, type=str) + parser.add_argument( + "-r", "--recipe", required=True, type=str, help="Name of (and/or path to) the Stackinator recipe." + ) + parser.add_argument( + "-s", "--system", required=True, type=str, help="Name of (and/or path to) the Stackinator system configuration." + ) parser.add_argument("-d", "--debug", action="store_true") - parser.add_argument("-m", "--mount", required=False, type=str) - parser.add_argument("-c", "--cache", required=False, type=str) + parser.add_argument( + "-m", "--mount", required=False, type=str, help="The mount point where the environment will be located." + ) + parser.add_argument( + "-c", + "--cache", + required=False, + type=str, + help="Buildcache location or name (from system config's mirrors.yaml).", + ) parser.add_argument("--develop", action="store_true", required=False) return parser diff --git a/stackinator/mirror.py b/stackinator/mirror.py new file mode 100644 index 00000000..e6ba268b --- /dev/null +++ b/stackinator/mirror.py @@ -0,0 +1,281 @@ +from typing import Optional, List, Dict +import base64 +import io +import magic +import os +import pathlib +import urllib.error +import urllib.request +import yaml + +from . import schema, root_logger + + +class MirrorError(RuntimeError): + """Exception class for errors thrown by mirror configuration problems.""" + + +class Mirrors: + """Manage the definition of mirrors in a recipe.""" + + KEY_STORE_DIR = "key_store" + MIRRORS_YAML = "mirrors.yaml" + CMDLINE_CACHE = "cmdline_cache" + + def __init__( + self, + system_config_root: pathlib.Path, + cmdline_cache: Optional[pathlib.Path] = None, + mount_point: Optional[pathlib.Path] = None, + ): + """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + + self._system_config_root = system_config_root + self._mount_point = mount_point + + self._logger = root_logger + + self.mirrors = self._load_mirrors(cmdline_cache) + self._check_mirrors() + + # Always use the cache given on the command line + if self.CMDLINE_CACHE in self.mirrors: + self.build_cache_mirror = self.CMDLINE_CACHE + else: + # Otherwise, grab the configured cache (or None) + self.build_cache_mirror: Optional[str] = ( + [name for name, mirror in self.mirrors.items() if mirror.get("cache", False)] + [None] + ).pop(0) + + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", False)] + + # Will hold a list of all the gpg keys (public and private) + self._keys: Optional[List[pathlib.Path]] = [] + + def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: + """Load the mirrors file, if one exists.""" + path = self._system_config_root / "mirrors.yaml" + if path.exists(): + try: + with path.open() as fid: + # load the raw yaml input + mirrors = yaml.load(fid, Loader=yaml.SafeLoader) + except (OSError, PermissionError) as err: + raise MirrorError(f"Could not open/read mirrors.yaml file.\n{err}") + else: + mirrors = {} + + try: + schema.MirrorsValidator.validate(mirrors) + except ValueError as err: + raise MirrorError(f"Mirror config does not comply with schema.\n{err}") + + caches = [name for name, mirror in mirrors.items() if mirror["cache"]] + if len(caches) > 1: + raise MirrorError( + "Mirror config has more than one mirror specified as the build cache destination.\n" + f"{self._pp_yaml(caches)}" + ) + elif caches: + cache = mirrors[caches[0]] + if not cache.get("private_key"): + raise MirrorError(f"Mirror build cache config '{caches[0]}' missing a required 'private_key' path.") + + # Load the cache as defined by the deprecated 'cache.yaml' file. + if cmdline_cache is not None: + mirrors[self.CMDLINE_CACHE] = self._load_cmdline_cache(cmdline_cache) + + return {name: mirror for name, mirror in mirrors.items() if mirror["enabled"]} + + @staticmethod + def _pp_yaml(object): + """Pretty print the given object as yaml.""" + + example_yaml_stream = io.StringIO() + yaml.dump(object, example_yaml_stream, default_flow_style=False) + return example_yaml_stream.getvalue() + + def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: + """Load the mirror definition from the legacy 'cache.yaml' file.""" + + if not cache_config_path.is_file(): + raise MirrorError( + f"Binary cache configuration path given on the command line '{cache_config_path}' does not exist." + ) + + with cache_config_path.open("r") as file: + try: + raw = yaml.load(file, Loader=yaml.SafeLoader) + except ValueError as err: + raise MirrorError(f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + + try: + schema.CacheValidator.validate(raw) + except ValueError as err: + raise MirrorError(f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") + + mirror_cfg = { + "url": raw["root"], + "description": "Buildcache dest loaded from legacy cache.yaml", + "cache": True, + "enabled": True, + "bootstrap": False, + "mount_specific": True, + "private_key": raw["key"], + } + + self._logger.warning( + "Configuring the buildcache from the system cache.yaml file.\n" + "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" + f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}" + ) + + return mirror_cfg + + def _check_mirrors(self): + """Validate the mirror config entries.""" + + for name, mirror in self.mirrors.items(): + url = mirror["url"] + if url.startswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not absolute") + if not path.is_dir(): + raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not a directory") + + mirror["url"] = path + + elif url.startswith("https://"): + try: + request = urllib.request.Request(url, method="HEAD") + urllib.request.urlopen(request) + except urllib.error.URLError as e: + raise MirrorError( + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}" + ) + + @property + def keys(self): + """Return the list of public and private key file paths.""" + + if self._keys is None: + raise RuntimeError("The mirror.keys method was accessed before setup_configs() was called.") + + return self._keys + + def setup_configs(self, config_root: pathlib.Path): + """Setup all mirror configs in the given config_root.""" + + self._key_setup(config_root / self.KEY_STORE_DIR) + self._create_spack_mirrors_yaml(config_root / self.MIRRORS_YAML) + self._create_bootstrap_configs(config_root) + + def _create_spack_mirrors_yaml(self, dest: pathlib.Path): + """Generate the mirrors.yaml for our build directory.""" + + raw = {"mirrors": {}} + + for name, mirror in self.mirrors.items(): + url = mirror["url"] + + # Make the mirror path specific to the mount point + if mirror["mount_specific"] and self._mount_point is not None: + url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") + + raw["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, + } + + with dest.open("w") as file: + yaml.dump(raw, file, default_flow_style=False) + + def _create_bootstrap_configs(self, config_root: pathlib.Path): + """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" + + if not self.bootstrap_mirrors: + return + + bootstrap_yaml = { + "sources": [], + "trusted": {}, + } + + for name in self.bootstrap_mirrors: + bs_mirror_path = config_root / f"bootstrap/{name}" + mirror = self.mirrors[name] + # Tell spack where to find the metadata for each bootstrap mirror. + bootstrap_yaml["sources"].append( + { + "name": name, + "metadata": str(bs_mirror_path), + } + ) + # And trust each one + bootstrap_yaml["trusted"][name] = True + + # Create the metadata dir and metadata.yaml + bs_mirror_path.mkdir(parents=True, exist_ok=True) + bs_mirror_yaml = { + "type": "install", + "info": mirror["url"], + } + with (bs_mirror_path / "metadata.yaml").open("w") as file: + yaml.dump(bs_mirror_yaml, file, default_flow_style=False) + + with (config_root / "bootstrap.yaml").open("w") as file: + yaml.dump(bootstrap_yaml, file, default_flow_style=False) + + def _key_setup(self, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + + self._keys = [] + key_store.mkdir(exist_ok=True) + + for name, mirror in self.mirrors.items(): + if mirror.get("public_key") is None: + continue + + key = mirror["public_key"] + + # key will be saved under key_store/mirror_name.gpg + + dest = pathlib.Path(key_store / f"{name}.gpg") + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if not path.is_absolute(): + # try prepending system config path + path = self._system_config_root / path + + if path.is_file(): + with open(path, "rb") as reader: + binary_key = reader.read() + + # convert base64 key to binary + else: + try: + binary_key = base64.b64decode(key) + except ValueError: + raise MirrorError( + f"Key for mirror '{name}' is not valid: '{path}'. \n" + f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" + f"Check the key listed in mirrors.yaml in system config." + ) + + file_type = magic.from_buffer(binary_key, mime=True) + if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): + raise MirrorError( + f"Key for mirror {name} is not a valid GPG key. \n" + f"The file (or base64) was readable, but the data itself was not a PGP key.\n" + f"Check the key listed in mirrors.yaml in system config." + ) + + # copy key to new destination in key store + with open(dest, "wb") as writer: + writer.write(binary_key) + + self._keys.append(dest) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index ca8d2b3d..ff3d8e27 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -5,7 +5,7 @@ import jinja2 import yaml -from . import cache, root_logger, schema, spack_util +from . import root_logger, schema, spack_util, mirror from .etc import envvars @@ -169,15 +169,10 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - # optional mirror configurtion - mirrors_path = self.path / "mirrors.yaml" - if mirrors_path.is_file(): - self._logger.warning( - "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." - ) - raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - - self.mirror = (args.cache, self.mount) + # load the optional mirrors.yaml from system config, and add any additional + # mirrors specified on the command line. + self._logger.debug("Configuring mirrors.") + self.mirrors = mirror.Mirrors(self.system_config_path, pathlib.Path(args.cache)) # optional post install hook if self.post_install_hook is not None: @@ -206,6 +201,13 @@ def spack_repo(self): return repo_path return None + # Returns: + # Path: if the recipe specified a build cache mirror + # None: if no build cache mirror is used + @property + def build_cache_mirror(self): + return self.mirrors.build_cache_mirror + # Returns: # Path: of the recipe extra path if it exists # None: if there is no user-provided extra path in the recipe @@ -236,32 +238,6 @@ def pre_install_hook(self): return hook_path return None - # Returns a dictionary with the following fields - # - # root: /path/to/cache - # path: /path/to/cache/user-environment - # key: /path/to/private-pgp-key - @property - def mirror(self): - return self._mirror - - # configuration is a tuple with two fields: - # - a Path of the yaml file containing the cache configuration - # - the mount point of the image - @mirror.setter - def mirror(self, configuration): - self._logger.debug(f"configuring build cache mirror with {configuration}") - self._mirror = None - - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The cache configuration '{file}' is not a file") - - self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - @property def config(self): return self._config @@ -541,7 +517,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirror is not None + push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, @@ -572,7 +548,7 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.mirror is not None + push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( environments=self.environments, push_to_cache=push_to_cache, diff --git a/stackinator/schema.py b/stackinator/schema.py index 3a2a9842..d461ff0e 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -121,3 +121,4 @@ def check_module_paths(instance): EnvironmentsValidator = SchemaValidator(prefix / "schema/environments.json") CacheValidator = SchemaValidator(prefix / "schema/cache.json") ModulesValidator = SchemaValidator(prefix / "schema/modules.json", check_module_paths) +MirrorsValidator = SchemaValidator(prefix / "schema/mirror.json") diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json new file mode 100644 index 00000000..89770831 --- /dev/null +++ b/stackinator/schema/mirror.json @@ -0,0 +1,46 @@ +{ + "type" : "object", + "additionalProperties": { + "type": "object", + "required": ["url"], + "additionalProperties": false, + "properties": { + "url": { + "type": "string", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." + }, + "description": { + "type": "string", + "description": "What this mirror is for." + }, + "enabled": { + "type": "boolean", + "default": true, + "description": "Whether this mirror is enabled." + }, + "bootstrap": { + "type": "boolean", + "default": false, + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." + }, + "cache": { + "type": "boolean", + "default": false, + "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." + }, + "public_key": { + "type": "string", + "description": "Public PGP key for validating binary cache packages. A path or base64 encoded key." + }, + "private_key": { + "type": "string", + "description": "Private PGP key for signing binary cache packages. (Path only)" + }, + "mount_specific": { + "type": "boolean", + "default": false, + "description": "Use a mount specific buildcache path (specified path + recipe mount point)." + } + } + } +} diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10a0ea58..f0b90ebe 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -33,12 +33,14 @@ pre-install: spack-setup mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - {% if cache %} + @echo "Pulling and trusting keys from configured buildcaches." $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} - {% endif %} + @echo "Adding mirror gpg keys." + {% for key_path in gpg_keys %} + $(SANDBOX) $(SPACK) gpg trust {{ key_path }} + {% endfor %} + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list touch mirror-setup compilers: mirror-setup @@ -77,14 +79,14 @@ store.squashfs: post-install # Force push all built packages to the build cache cache-force: mirror-setup -{% if cache.key %} +{% if cache %} $(warning ================================================================================) $(warning Generate the config in order to force push partially built compiler environments) $(warning if this step is performed with partially built compiler envs, you will) $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package alpscache \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache.name \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ diff --git a/unittests/__init__.py b/unittests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/unittests/data/systems/mirror-bad-key/bad_key.gpg b/unittests/data/systems/mirror-bad-key/bad_key.gpg new file mode 100644 index 00000000..d7980bbf --- /dev/null +++ b/unittests/data/systems/mirror-bad-key/bad_key.gpg @@ -0,0 +1 @@ +This is a bad key \ No newline at end of file diff --git a/unittests/data/systems/mirror-bad-key/mirrors.yaml b/unittests/data/systems/mirror-bad-key/mirrors.yaml new file mode 100644 index 00000000..d5154dd3 --- /dev/null +++ b/unittests/data/systems/mirror-bad-key/mirrors.yaml @@ -0,0 +1,3 @@ +bad-key: + url: https://mirror.spack.io + public_key: bad_key.gpg diff --git a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml new file mode 100644 index 00000000..3433e04a --- /dev/null +++ b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml @@ -0,0 +1,3 @@ +bad-key-path: + url: https://mirror.spack.io + public_key: /path/doesnt/exist diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml new file mode 100644 index 00000000..8ffce331 --- /dev/null +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -0,0 +1,2 @@ +bad-url: + url: https://www.testsite.io/services \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok/cache.yaml b/unittests/data/systems/mirror-ok/cache.yaml new file mode 100644 index 00000000..ad7de37c --- /dev/null +++ b/unittests/data/systems/mirror-ok/cache.yaml @@ -0,0 +1,2 @@ +root: /tmp/foo +key: ../../test-gpg-priv.asc diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml new file mode 100644 index 00000000..0a1b8434 --- /dev/null +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -0,0 +1,55 @@ +fake-mirror: + url: https://github.com + public_key: ../../test-gpg-pub.asc +disabled-mirror: + url: https://github.com + enabled: false +buildcache-mirror: + url: https://mirror.spack.io + private_key: '../test-gpg-priv.asc' + public_key: "\ + mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ + eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ + 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ + kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ + IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ + WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ + jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ + TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ + qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ + KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ + CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ + CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ + LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ + qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ + HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ + WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ + /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ + glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ + PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ + lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ + kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ + mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ + HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ + Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ + UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ + zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ + Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ + 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ + sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ + 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ + OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ + NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ + Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ + +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ + 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ + xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ + lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ + eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ + eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ + gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" + + cache: true +bootstrap-mirror: + url: https://mirror.spack.io + bootstrap: true diff --git a/unittests/data/test-gpg-priv.asc b/unittests/data/test-gpg-priv.asc new file mode 100644 index 00000000..eaa2dc19 Binary files /dev/null and b/unittests/data/test-gpg-priv.asc differ diff --git a/unittests/data/test-gpg-pub.asc b/unittests/data/test-gpg-pub.asc new file mode 100644 index 00000000..aa72b028 Binary files /dev/null and b/unittests/data/test-gpg-pub.asc differ diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py new file mode 100644 index 00000000..d262cd1b --- /dev/null +++ b/unittests/test_mirrors.py @@ -0,0 +1,191 @@ +import base64 +import pathlib +import pytest +import stackinator.mirror as mirror +import yaml + + +@pytest.fixture +def test_path(): + return pathlib.Path(__file__).parent.resolve() + + +@pytest.fixture +def systems_path(test_path): + return test_path / "data" / "systems" + + +def test_mirror_init(systems_path): + """Check that Mirror objects are initialized correctly.""" + path = systems_path / "mirror-ok" + mirrors_obj = mirror.Mirrors(path) + + valid_mirrors = { + "fake-mirror": { + "url": "https://github.com", + "enabled": True, + "bootstrap": False, + "cache": False, + "public_key": "../../test-gpg-pub.asc", + "mount_specific": False, + }, + "buildcache-mirror": { + "url": "https://mirror.spack.io", + "enabled": True, + "bootstrap": False, + "cache": True, + "private_key": "../test-gpg-priv.asc", + "mount_specific": False, + }, + "bootstrap-mirror": { + "url": "https://mirror.spack.io", + "enabled": True, + "bootstrap": True, + "cache": False, + "mount_specific": False, + }, + } + + with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: + key = base64.b64encode(pub_key_file.read()).decode() + valid_mirrors["buildcache-mirror"]["public_key"] = key + + assert mirrors_obj.mirrors == valid_mirrors + assert mirrors_obj.bootstrap_mirrors == [ + name for name in valid_mirrors.keys() if valid_mirrors[name].get("bootstrap") + ] + assert mirrors_obj.build_cache_mirror == [ + name for name in valid_mirrors.keys() if valid_mirrors[name].get("cache") + ].pop(0) + + for mir in mirrors_obj.mirrors: + assert mirrors_obj.mirrors[mir].get("enabled") + + +def test_mirror_init_bad_url(systems_path): + """Check that MirrorError is raised for a bad url.""" + + path = systems_path / "mirror-bad-url" + + with pytest.raises(mirror.MirrorError): + mirror.Mirrors(path) + + +def test_setup_configs(tmp_path, systems_path): + """Test general config setup.""" + + mir = mirror.Mirrors(systems_path / "mirror-ok") + mir.setup_configs(tmp_path) + + assert (tmp_path / "mirrors.yaml").is_file() + assert (tmp_path / "bootstrap").is_dir() + assert (tmp_path / "key_store").is_dir() + + +def test_command_line_cache(systems_path): + """Check that adding a cache from the command line works.""" + + mirrors = mirror.Mirrors(systems_path / "mirror-ok", cmdline_cache=systems_path / "mirror-ok/cache.yaml") + + assert len(mirrors.mirrors) == 4 + # This should always be the build cache even though one is already defined. + assert mirrors.build_cache_mirror == "cmdline_cache" + cache_mirror = mirrors.mirrors["cmdline_cache"] + assert cache_mirror["url"] == "/tmp/foo" + assert cache_mirror["enabled"] + assert cache_mirror["cache"] + assert not cache_mirror["bootstrap"] + assert cache_mirror["mount_specific"] + + +def test_create_spack_mirrors_yaml(tmp_path, systems_path): + """Check that the mirrors.yaml passed to spack is correct""" + + valid_spack_yaml = { + "mirrors": { + "fake-mirror": { + "fetch": {"url": "https://github.com"}, + "push": {"url": "https://github.com"}, + }, + "buildcache-mirror": { + "fetch": {"url": "https://mirror.spack.io"}, + "push": {"url": "https://mirror.spack.io"}, + }, + "bootstrap-mirror": { + "fetch": {"url": "https://mirror.spack.io"}, + "push": {"url": "https://mirror.spack.io"}, + }, + } + } + + dest = tmp_path / "test_output.yaml" + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") + mirrors_obj._create_spack_mirrors_yaml(dest) + + with dest.open() as f: + data = yaml.safe_load(f) + + assert data == valid_spack_yaml + + +def test_create_bootstrap_configs(tmp_path, systems_path): + """Check that spack bootstrap configs are generated correctly""" + + valid_yaml = { + "sources": [ + { + "name": "bootstrap-mirror", + "metadata": str(tmp_path / "bootstrap/bootstrap-mirror"), + } + ], + "trusted": {"bootstrap-mirror": True}, + } + valid_metadata = { + "type": "install", + "info": "https://mirror.spack.io", + } + + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") + mirrors_obj._create_bootstrap_configs(tmp_path) + + with (tmp_path / "bootstrap.yaml").open() as f: + bs_data = yaml.safe_load(f) + print(bs_data) + print(valid_yaml) + assert bs_data == valid_yaml + + with (tmp_path / "bootstrap/bootstrap-mirror/metadata.yaml").open() as f: + metadata = yaml.safe_load(f) + assert metadata == valid_metadata + + +def test_key_setup(systems_path, tmp_path): + """Check that public keys are set up properly.""" + + mirrors = mirror.Mirrors(systems_path / "mirror-ok") + + mirrors._key_setup(tmp_path) + + key_files = list(tmp_path.iterdir()) + assert {key_file.name for key_file in key_files} == {"buildcache-mirror.gpg", "fake-mirror.gpg"} + # The two files should be identical in content + key_file_data = [] + for key_file in key_files: + with key_file.open("rb") as file: + key_file_data.append(file.read()) + assert key_file_data[0] == key_file_data[1] + + +@pytest.mark.parametrize( + "system_name", + [ + "mirror-bad-key", + "mirror-bad-keypath", + ], +) +def test_key_setup_bad_key(tmp_path, systems_path, system_name): + """asdfasdf""" + + mirrors = mirror.Mirrors(systems_path / system_name) + with pytest.raises(mirror.MirrorError): + mirrors._key_setup(tmp_path)