diff --git a/.github/workflows/t-pull.yml b/.github/workflows/t-pull.yml index 5f5dddcbfb..7951c122f8 100644 --- a/.github/workflows/t-pull.yml +++ b/.github/workflows/t-pull.yml @@ -1,3 +1,4 @@ +--- name: Check integration tests with "none" device on: @@ -37,12 +38,29 @@ jobs: for file in netsim/extra/*/plugin.py; do python3 -m mypy $file done - - name: Run transformation tests + - name: Consolidate system files for faster testing + id: consolidate + run: | + # Pre-consolidate all system/package YAML files + # This cache will be used by all _read.load() calls in pytest tests + CACHE_FILE="/tmp/netlab-system-cache.json" + ./netlab consolidate -o "$CACHE_FILE" + echo "✅ System cache created: $(ls -lh "$CACHE_FILE" | awk '{print $5}')" + cache_info=$(python3 -c "import json; data=json.load(open('$CACHE_FILE')); \ + print(f\"{len(data['files'])} files, netlab \ + {data.get('netlab_version', 'unknown')}\")") + echo "📦 Cache contains: $cache_info" + echo "cache_file=$CACHE_FILE" >> $GITHUB_OUTPUT + - name: Run transformation tests (with JSON cache) if: ${{ github.event.pull_request.head.repo.full_name != 'ipspace/netlab' }} + env: + NETLAB_JSON_CACHE: ${{ steps.consolidate.outputs.cache_file }} run: | cd tests PYTHONPATH="../" pytest - - name: Check integration tests + - name: Check integration tests (using JSON cache) + env: + NETLAB_JSON_CACHE: ${{ steps.consolidate.outputs.cache_file }} run: | cd tests ./check-integration-tests.sh diff --git a/MANIFEST.in b/MANIFEST.in index a594b362bd..5dbd957e5e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,7 @@ recursive-include netsim/templates * recursive-include netsim/daemons * recursive-include netsim/install * recursive-include netsim/tools * +recursive-include netsim/yang * recursive-include netsim *.yml include requirements.txt include netsim/cli/usage.txt diff --git a/docs/defaults.md b/docs/defaults.md index e7e2b6386a..399772892b 100644 --- a/docs/defaults.md +++ b/docs/defaults.md @@ -129,7 +129,7 @@ You can change _netlab_ defaults with environment variables starting with `netla For example, the `NETLAB_DEVICE` variable sets the **defaults.device** parameter and the `NETLAB_BGP_AS` variable sets the **defaults.bgp.as** parameter. -You can use the environment variables instead of the `--device`, `--provider`, or `--set` arguments of the **netlab up** command. For example, the following command sequence starts a lab topology using Arista EOS containers: +You can use the environment variables instead of the `--device`, `--provider`, `--json-cache`, or `--set` arguments of the **netlab up** and **netlab create** commands. For example, the following command sequence starts a lab topology using Arista EOS containers: ``` $ export NETLAB_DEVICE=eos @@ -137,6 +137,13 @@ $ export NETLAB_PROVIDER=clab $ netlab up ``` +You can also use `NETLAB_JSON_CACHE` to specify a consolidated JSON cache file: + +``` +$ export NETLAB_JSON_CACHE=/path/to/cache.json +$ netlab create topology.yml +``` + Some _netlab_ defaults have an underscore in their names. To set those parameters with environment variables, use a double underscore. For example, to set the *libvirt* **batch_size** parameter, use: ``` diff --git a/docs/netlab/consolidate.md b/docs/netlab/consolidate.md new file mode 100644 index 0000000000..830c8c90d5 --- /dev/null +++ b/docs/netlab/consolidate.md @@ -0,0 +1,314 @@ +(netlab-consolidate)= +# Consolidate YAML Files into JSON Cache + +The **netlab consolidate** command collects all YAML files (topology, defaults, modules, devices, providers) that would be loaded during a `netlab create` operation and consolidates them into a single JSON file. This JSON cache can then be used with the `--json-cache` flag to significantly speed up subsequent `netlab create` operations. + +## Why Use Consolidation? + +When `netlab create` runs, it reads many YAML files: +- Topology file and any included files +- Default settings files (project, user, system) +- Module definitions +- Device configurations +- Provider configurations + +Each file requires: +- File system I/O operations +- YAML parsing (slower than JSON parsing) +- Path resolution and file lookups + +By consolidating these files into a single JSON cache, you can: +- **Reduce I/O operations**: One file read instead of 80+ separate files +- **Faster parsing**: JSON parsing is significantly faster than YAML parsing +- **Eliminate file lookups**: All data is pre-resolved and ready to use +- **Improve CI/CD performance**: Pre-consolidate once, use cache for all tests + +## Performance Benefits + +Based on EVPN integration test results: +- **44.6% faster** execution time (1.81x speedup) +- **19.59 seconds saved** per full test suite run +- Consistent improvement across all topology files + +For example, a test that takes 2.44 seconds without cache takes only 1.35 seconds with cache. + +## Usage + +### Consolidate a Specific Topology + +To consolidate all YAML files for a specific topology: + +```bash +netlab consolidate topology.yml -o cache.json +``` + +This will: +1. Load the topology file and all its dependencies +2. Track all YAML files that get read (defaults, modules, devices, providers) +3. Consolidate them into a single JSON file +4. Validate the structure against a JSON schema (if `jsonschema` is installed) + +### Consolidate All System Files + +To consolidate all system/package YAML files without a topology (useful for CI/CD): + +```bash +netlab consolidate -o system-cache.json +``` + +This consolidates: +- All system defaults files +- All module definitions +- All device configurations +- All provider configurations +- Package files + +This is particularly useful for integration test suites where you want to cache all default files once and reuse them across multiple labs. + +### Using the JSON Cache + +After creating a consolidated JSON file, use it with `netlab create` or `netlab up`: + +**Option 1: Using CLI argument** +```bash +# Use with netlab create +netlab create --json-cache cache.json topology.yml + +# Use with netlab up +netlab up --json-cache cache.json topology.yml +``` + +**Option 2: Using environment variable** +```bash +# Set environment variable +export NETLAB_JSON_CACHE=cache.json + +# Use with netlab create or netlab up +netlab create topology.yml +netlab up topology.yml +``` + +The CLI argument `--json-cache` takes precedence over the `NETLAB_JSON_CACHE` environment variable (same precedence pattern as `NETLAB_PROVIDER`). + +The JSON cache tells `netlab create` or `netlab up` to: +1. Load the consolidated JSON file instead of reading individual YAML files +2. Use the pre-parsed content directly +3. Skip YAML parsing and file I/O operations + +## Command Syntax + +```text +usage: netlab consolidate [-h] [--log] [-q] [-v] [--defaults DEFAULTS] + [-d DEVICE] [-p PROVIDER] [-s SETTINGS] + [--plugin PLUGIN] [-o OUTPUT] [topology] + +Consolidate all YAML files into a single JSON file for faster loading + +positional arguments: + topology Topology file to consolidate (optional: if omitted, + consolidates all system/package YAML files) + +optional arguments: + -h, --help show this help message and exit + --log Enable basic logging + -q, --quiet Report only major errors + -v, --verbose Verbose logging + --defaults DEFAULTS Local topology defaults file + -d DEVICE, --device DEVICE + Default device type + -p PROVIDER, --provider PROVIDER + Override virtualization provider + --plugin PLUGIN Additional plugin(s) + -s SETTINGS, --set SETTINGS + Additional parameters added to topology file + -o OUTPUT, --output OUTPUT + Output JSON file (default: netlab.consolidated.json) +``` + +## JSON Cache Structure + +The consolidated JSON file has the following structure: + +```json +{ + "version": "1.0", + "netlab_version": "25.12.02", + "topology_file": "topology.yml", + "files": { + "/path/to/topology.yml": { + "content": { ... }, + "source": "topology.yml", + "package": false + }, + "package:topology-defaults.yml": { + "content": { ... }, + "source": "package:topology-defaults.yml", + "package": true + }, + ... + }, + "file_count": 95 +} +``` + +Each file entry contains: +- **content**: The parsed YAML content as a dictionary/object +- **source**: The original filename/path +- **package**: Whether this is a package file (starts with `package:`) + +## Version Compatibility + +The consolidated JSON cache includes the **netlab version** that created it. When loading a cache file, netlab checks if the cache version matches the current netlab version: + +- **Version matches**: Cache is used normally +- **Version mismatch**: Cache is rejected with an error message, and you must regenerate it + +This ensures that cache files are only used with compatible netlab versions, preventing issues from: +- Changes in YAML file structure between versions +- Modified default settings or module definitions +- Updated device or provider configurations + +**Example error when versions don't match:** +``` +ERROR: JSON cache cache.json was created with netlab version 25.12.01 +ERROR: Current netlab version is 25.12.02 +ERROR: Cache file is incompatible and must be regenerated +ERROR: Please run "netlab consolidate" again to create a new cache +``` + +## Schema Validation + +The consolidated JSON cache is validated against a JSON schema to ensure data integrity. The schema validates: +- Overall structure (version, netlab_version, files, file_count) +- File entry structure (content, source, package) +- Content type (must be an object/dictionary) + +The schema uses `additionalProperties: true` for content objects, allowing any YAML structure to be stored. This provides flexibility while still ensuring the cache has the correct overall structure. + +Schema validation is **optional** and requires the `jsonschema` package. If `jsonschema` is not installed, consolidation will still work, but validation will be skipped. + +To install schema validation support: + +```bash +pip install jsonschema +``` + +## Use Cases + +### Development Workflow + +1. Consolidate once at the start of your session: + ```bash + netlab consolidate -o cache.json + ``` + +2. Use the cache for all subsequent `netlab create` operations: + ```bash + netlab create --json-cache cache.json topology.yml + ``` + +3. Re-consolidate if you modify defaults or system files + +### CI/CD Pipelines + +1. Pre-consolidate system files once: + ```bash + netlab consolidate -o system-cache.json + ``` + +2. Use the cache for all integration tests: + ```bash + for topo in tests/integration/*/*.yml; do + netlab create --json-cache system-cache.json "$topo" + done + ``` + +This provides significant time savings when running large test suites. + +### Integration Testing + +For integration test suites with many topology files: + +1. Create a system cache once: + ```bash + netlab consolidate -o system-cache.json + ``` + +2. Run all tests with the cache: + ```bash + netlab create --json-cache system-cache.json test-topology.yml + ``` + +This can reduce test execution time by 40-50%. + +## Limitations + +- The JSON cache must be regenerated if: + - **Netlab version changes** (automatic check - cache will be rejected) + - System defaults files change + - Module definitions change + - Device or provider configurations change + - The topology file or its includes change + +- **Version compatibility**: The cache includes the netlab version that created it. If you upgrade netlab, the cache will be automatically rejected and you'll need to regenerate it. + +- Schema validation requires the `jsonschema` package, but is optional. + +## Examples + +### Example 1: Consolidate and Use for Single Topology + +```bash +# Consolidate topology and all dependencies +netlab consolidate my-topology.yml -o my-cache.json + +# Use the cache for faster creation (CLI argument) +netlab create --json-cache my-cache.json my-topology.yml + +# Or use the cache via environment variable +export NETLAB_JSON_CACHE=my-cache.json +netlab create my-topology.yml + +# Or use the cache when starting the lab +netlab up --json-cache my-cache.json my-topology.yml +``` + +### Example 2: CI/CD Pipeline + +```bash +# Pre-consolidate system files (run once) +netlab consolidate -o system-cache.json + +# Use cache for all tests (via environment variable) +export NETLAB_JSON_CACHE=system-cache.json +for test in tests/integration/**/*.yml; do + netlab create "$test" -p clab -d frr +done + +# Or use CLI argument +for test in tests/integration/**/*.yml; do + netlab create --json-cache system-cache.json "$test" -p clab -d frr +done +``` + +### Example 3: Development Iteration + +```bash +# Start of session: consolidate system files +netlab consolidate -o dev-cache.json + +# During development: use cache for quick iterations (via environment variable) +export NETLAB_JSON_CACHE=dev-cache.json +netlab create topology.yml +netlab up topology.yml # Much faster! + +# Or use CLI argument +netlab create --json-cache dev-cache.json topology.yml +netlab up --json-cache dev-cache.json topology.yml # Much faster! +``` + +## Related Commands + +- **[netlab create](netlab-create)**: Create lab configuration files (supports `--json-cache` flag) +- **[netlab defaults](netlab-defaults)**: Manage default settings files + diff --git a/docs/netlab/up.md b/docs/netlab/up.md index 183d9fd77d..bd18f7f40c 100644 --- a/docs/netlab/up.md +++ b/docs/netlab/up.md @@ -9,6 +9,12 @@ You can skip this step and reuse existing configuration files with the `--snapshot` flag ([more details](netlab-up-restart)); ``` +```{tip} +For faster execution, especially in CI/CD pipelines or when running multiple `netlab up` operations, you can use the `--json-cache` flag or `NETLAB_JSON_CACHE` environment variable with a consolidated JSON cache file created by **[netlab consolidate](netlab-consolidate)**. This can improve performance by 40-50% by eliminating YAML parsing and reducing file I/O operations. + +The CLI argument takes precedence over the environment variable. +``` + * Checks the [virtualization provider](../providers.md) installation; * Creates the lab management network ([more details](libvirt-mgmt)) * Starts the virtual lab using the [selected virtualization provider](topology-reference-top-elements); @@ -35,7 +41,7 @@ You can use `netlab up` to create configuration files and start the lab, or use usage: netlab up [-h] [--log] [-v] [-q] [--defaults [DEFAULTS ...]] [-d DEVICE] [-p PROVIDER] [--plugin PLUGIN] [-s SETTINGS] [--no-config] [-r RELOAD] [--no-tools] [--dry-run] [--fast-config] [--snapshot [SNAPSHOT]] - [topology] + [--json-cache JSON_CACHE] [topology] Create configuration files, start a virtual lab, and configure it @@ -65,6 +71,9 @@ options: --fast-config Use fast device configuration (Ansible strategy = free) --snapshot [SNAPSHOT] Use netlab snapshot file created by a previous lab run + --json-cache JSON_CACHE + Use consolidated JSON cache file instead of reading + YAML files (see [netlab consolidate](netlab-consolidate)) ``` ```{tip} diff --git a/netsim/cli/consolidate.py b/netsim/cli/consolidate.py new file mode 100644 index 0000000000..4cb0317521 --- /dev/null +++ b/netsim/cli/consolidate.py @@ -0,0 +1,84 @@ +# +# netlab consolidate command +# +# Consolidates all YAML files into a single JSON file for faster loading +# +import argparse +import typing +from pathlib import Path + +from box import Box + +from ..utils import consolidate, log +from . import common_parse_args, error_and_exit, topology_parse_args + + +def consolidate_parse(args: typing.List[str]) -> argparse.Namespace: + """Parse arguments for consolidate command""" + parents = [common_parse_args(True), topology_parse_args()] + parser = argparse.ArgumentParser( + parents=parents, + prog='netlab consolidate', + description='Consolidate all YAML files into a single JSON file for faster loading') + + parser.add_argument( + 'topology', + nargs='?', + action='store', + default=None, + help='Topology file to consolidate (optional: if omitted, consolidates all system/package YAML files)') + + parser.add_argument( + '-o', '--output', + dest='output', + action='store', + default='netlab.consolidated.json', + help='Output JSON file (default: netlab.consolidated.json)') + + return parser.parse_args(args) + +def run(cli_args: typing.List[str]) -> None: + """Run the consolidate command""" + args = consolidate_parse(cli_args) + + try: + if args.topology: + # Consolidate specific topology file + topology_file = args.topology + if not Path(topology_file).exists(): + error_and_exit(f'Topology file {topology_file} does not exist', module='consolidate') + + # Get defaults lists if specified + user_defaults = None + system_defaults = None + + # Build defaults list similar to load function + from ..utils import read as _read + temp_topology = Box() + defaults_list = _read.build_defaults_list( + temp_topology, + user_defaults=user_defaults, + system_defaults=system_defaults + ) + + consolidate.consolidate_to_json( + topology_file=topology_file, + output_file=args.output, + user_defaults=defaults_list if user_defaults is None else user_defaults, + system_defaults=defaults_list if system_defaults is None else system_defaults + ) + log.status_green('CONSOLIDATED', '') + print(f'All YAML files consolidated into {args.output}') + print(f'Use --json-cache {args.output} with netlab create to use this cache') + else: + # Consolidate all system/package YAML files + consolidate.consolidate_to_json( + topology_file=None, + output_file=args.output + ) + log.status_green('CONSOLIDATED', '') + print(f'All system/package YAML files consolidated into {args.output}') + print(f'Use --json-cache {args.output} with netlab create to use this cache') + except Exception as ex: + error_and_exit(f'Error consolidating files: {ex}', module='consolidate') + diff --git a/netsim/cli/create.py b/netsim/cli/create.py index 4af51eca0e..751c50bd43 100644 --- a/netsim/cli/create.py +++ b/netsim/cli/create.py @@ -17,9 +17,37 @@ from .. import augment from ..outputs import _TopologyOutput from ..utils import log, strings +from ..utils import read as _read from . import common_parse_args, error_and_exit, lab_status_log, load_topology, topology_parse_args +def get_json_cache_path(args: typing.Union[argparse.Namespace, Box]) -> typing.Optional[str]: + """ + Get JSON cache path from CLI argument or environment variable. + + Precedence: + 1. CLI argument (--json-cache) + 2. Environment variable (NETLAB_JSON_CACHE) + + Args: + args: Parsed command-line arguments + + Returns: + JSON cache file path if found, None otherwise + """ + # Check CLI argument first (highest precedence) + if hasattr(args, 'json_cache') and args.json_cache: + return args.json_cache + + # Fall back to environment variable + json_cache_path = os.environ.get('NETLAB_JSON_CACHE') + if json_cache_path: + return json_cache_path + + return None + + + # # CLI parser for create-topology script # @@ -62,6 +90,10 @@ def create_topology_parse( if cmd == 'create': parser.add_argument('-o','--output',dest='output', action='append',help='Output format(s): format:option=filename') parser.add_argument('--devices',dest='devices', action='store_true',help='Create provider configuration file and netlab-devices.yml') + + # Add json-cache option for both 'create' and 'up' commands + if cmd in ['create', 'up']: + parser.add_argument('--json-cache',dest='json_cache', action='store',help='Use consolidated JSON cache file instead of reading YAML files') return parser.parse_args(args) @@ -137,6 +169,11 @@ def run(cli_args: typing.List[str], if not tpath.is_file(): log.fatal(f'The specified lab topology ({args.topology}) is not a file',module='create') + # Set JSON cache if requested (from CLI argument or environment variable) + json_cache_path = get_json_cache_path(args) + if json_cache_path: + _read.set_json_cache(json_cache_path) + topology = load_topology(args) augment.main.transform(topology) log.exit_on_error() diff --git a/netsim/cli/yang.py b/netsim/cli/yang.py new file mode 100644 index 0000000000..a9b9cbd898 --- /dev/null +++ b/netsim/cli/yang.py @@ -0,0 +1,117 @@ +# +# netlab yang command +# +# Validate topology file using YANG model with MUST statements +# +import argparse +import json +import os +import sys +import typing + +from .. import augment +from ..augment.main import transform_setup +from ..utils import log +from ..utils import read as _read +from . import common_parse_args, parser_add_debug, topology_parse_args +from .yang_validator import validate_topology_yang + + +def yang_parse(args: typing.List[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + parents=[ common_parse_args(), topology_parse_args() ], + prog="netlab yang", + description='Validate topology file using YANG model with MUST statements') + + parser.add_argument( + dest='topology', + action='store', + nargs='?', + default='topology.yml', + help='Topology file to validate (default: topology.yml)') + + parser.add_argument( + '--model', + dest='yang_model', + action='store', + default='package:yang/netlab-topology.yang', + help='YANG model file to use for validation (default: package:yang/netlab-topology.yang)') + + parser.add_argument( + '--output', + dest='output', + action='store', + choices=['text', 'json'], + default='text', + help='Output format (default: text)') + + parser.add_argument( + '--json-cache', + dest='json_cache', + action='store', + help='Use consolidated JSON cache file instead of reading YAML files') + + parser_add_debug(parser) + return parser.parse_args(args) + + +def run(cli_args: typing.List[str]) -> None: + args = yang_parse(cli_args) + log.set_logging_flags(args) + + # Set JSON cache if provided + if hasattr(args, 'json_cache') and args.json_cache: + _read.set_json_cache(args.json_cache) + else: + # Check environment variable as fallback + json_cache_path = os.environ.get('NETLAB_JSON_CACHE') + if json_cache_path: + _read.set_json_cache(json_cache_path) + + # Load and transform topology + # Get topology file - argparse should set this from positional argument + topology_file = getattr(args, 'topology', 'topology.yml') + if not topology_file: + topology_file = 'topology.yml' + + topology = _read.load(topology_file, getattr(args, 'defaults', None)) + if topology is None: + log.fatal(f'Cannot read topology file: {topology_file}') + + # Apply CLI settings (device, provider, etc.) before transformation + # This allows test files without explicit device types to work with -d flag + if hasattr(args, 'device') and args.device: + topology.nodes = augment.nodes.create_node_dict(topology.nodes) + _read.add_cli_args(topology, args) + + log.exit_on_error() + + # Transform topology to get full data structure + transform_setup(topology) + log.exit_on_error() + + # Validate using YANG model + try: + errors = validate_topology_yang(topology, args.yang_model) + + if errors: + if args.output == 'json': + output = json.dumps({'errors': errors}, indent=2) + print(output) + else: + print(f"\nYANG validation found {len(errors)} error(s):\n") + for error in errors: + print(f" - {error}") + print() + + sys.exit(1) + else: + if args.output == 'text': + print("Topology validation passed: all YANG MUST statements satisfied") + else: + print(json.dumps({'status': 'valid', 'errors': []}, indent=2)) + sys.exit(0) + + except Exception as ex: + log.fatal(f'YANG validation error: {ex}') + diff --git a/netsim/cli/yang_validator.py b/netsim/cli/yang_validator.py new file mode 100644 index 0000000000..552d0d1065 --- /dev/null +++ b/netsim/cli/yang_validator.py @@ -0,0 +1,197 @@ +# +# YANG validation module for topology files +# +import json +import re +import traceback +import typing +from pathlib import Path + +from box import Box + +from ..utils import files as _files +from ..utils import log + + +def topology_to_json(topology: Box) -> dict: + """ + Convert topology Box structure to JSON-compatible dictionary for YANG validation + + yangson expects JSON data matching the YANG model structure. The data should + be wrapped in the namespace-qualified container name. + """ + # Convert Box to dict and handle special cases + topo_dict = topology.to_dict() + + # Remove all internal/private keys starting with '_' before YANG validation + # Recursively remove underscore-prefixed keys from nested structures + def remove_underscore_keys(obj: typing.Any, depth: int = 0) -> None: + if depth > 10: # Prevent infinite recursion + return + if isinstance(obj, dict): + keys_to_remove = [] + for key in obj.keys(): + if key.startswith('_'): + keys_to_remove.append(key) + else: + # Recursively process nested structures + remove_underscore_keys(obj[key], depth + 1) + # Remove underscore-prefixed keys + for key in keys_to_remove: + del obj[key] + elif isinstance(obj, list): + for item in obj: + remove_underscore_keys(item, depth + 1) + + remove_underscore_keys(topo_dict) + + # Transform nodes from dictionary to list format for YANG validation + # YANG lists must be arrays, but netlab uses dictionaries keyed by node name + if 'nodes' in topo_dict and isinstance(topo_dict['nodes'], dict): + nodes_list = [] + for node_name, node_data in topo_dict['nodes'].items(): + if isinstance(node_data, dict): + node_data['name'] = node_name + nodes_list.append(node_data) + else: + # If node_data is not a dict, create a simple node entry + nodes_list.append({'name': node_name}) + topo_dict['nodes'] = nodes_list + + + # Wrap in namespace container for YANG validation + # yangson expects the data under the namespace-qualified container name + return { + 'netlab-topology:topology': topo_dict + } + + +def load_yang_model_path(model_path: str) -> Path: + """ + Get Path object for YANG model file + """ + if 'package:' in model_path: + pkg_files = _files.get_traversable_path('package:') + model_file = pkg_files.joinpath(model_path.replace('package:', '')) + if not model_file.exists(): + raise FileNotFoundError(f'YANG model not found: {model_path}') + return model_file + else: + model_file = Path(model_path) + if not model_file.exists(): + raise FileNotFoundError(f'YANG model not found: {model_path}') + return model_file + + +def _create_yang_library(yang_content: str) -> str: + """ + Extract module metadata from YANG content and create YANG library JSON. + + Returns JSON string in ietf-yang-library format. + """ + # Extract module name, revision, and namespace from YANG content + mod_match = re.search(r'module\s+(\S+)\s*\{', yang_content) + rev_match = re.search(r'revision\s+(\S+)\s*\{', yang_content) + ns_match = re.search(r'namespace\s+"([^"]+)"', yang_content) + + if not mod_match: + raise ValueError("Failed to extract module name from YANG file") + if not rev_match: + raise ValueError("Failed to extract revision from YANG file") + if not ns_match: + raise ValueError("Failed to extract namespace from YANG file") + + mod_name = mod_match.group(1) + mod_revision = rev_match.group(1) + mod_namespace = ns_match.group(1) + + # Create YANG library JSON (ietf-yang-library format) + yang_library = { + "ietf-yang-library:modules-state": { + "module-set-id": "netlab-topology-set", + "module": [ + { + "name": mod_name, + "revision": mod_revision, + "namespace": mod_namespace, + "conformance-type": "implement" + } + ] + } + } + + return json.dumps(yang_library) + + +def validate_topology_yang(topology: Box, yang_model_path: str) -> typing.List[str]: + """ + Validate topology against YANG model using actual YANG MUST statements + + Uses yangson library to parse the YANG model and validate the topology data, + including evaluation of MUST statements. + + Returns list of error messages, empty list if validation passes + """ + errors: typing.List[str] = [] + + # Check if yangson is available + try: + from yangson import DataModel # type: ignore[import-untyped] + from yangson.enumerations import ContentType, ValidationScope # type: ignore[import-untyped] + from yangson.exceptions import SchemaError, SemanticError, YangTypeError # type: ignore[import-untyped] + except ImportError as ex: + log.fatal( + f'yangson library not found: {ex}. Install it with: pip install yangson', + module='yang' + ) + return errors + + # Load YANG model file + try: + yang_model_file = load_yang_model_path(yang_model_path) + except FileNotFoundError as ex: + log.fatal(f'YANG model not found: {ex}', module='yang') + return errors + + # Read YANG model file content + try: + with yang_model_file.open('r') as f: + yang_content = f.read() + except Exception as ex: + log.fatal(f'Failed to read YANG model file: {ex}', module='yang') + return errors + + # Create data model from YANG file + # yangson requires YANG library JSON format, not raw YANG files + yang_dir = str(yang_model_file.parent) + + try: + yang_library_json = _create_yang_library(yang_content) + dm = DataModel(yang_library_json, [yang_dir]) + except SchemaError as ex: + log.fatal(f'Failed to load YANG model: {ex}', module='yang') + return errors + except ValueError as ex: + log.fatal(f'Invalid YANG model format: {ex}', module='yang') + return errors + except Exception as ex: + log.fatal(f'Failed to create YANG data model: {ex}', module='yang') + return errors + + # Convert topology to JSON format and validate + try: + topo_json = topology_to_json(topology) + instance = dm.from_raw(topo_json) + instance.validate(ValidationScope.all, ContentType.all) + except (SemanticError, YangTypeError) as ex: + # These are expected validation errors - return them as error messages + errors.append(f"YANG validation failed: {str(ex)}") + except Exception as ex: + errors.append(f"YANG validation failed: {str(ex)}") + if log.debug_active('yang'): + errors.append(f"Traceback: {traceback.format_exc()}") + + return errors + + + diff --git a/netsim/utils/consolidate.py b/netsim/utils/consolidate.py new file mode 100644 index 0000000000..b5f63dfa81 --- /dev/null +++ b/netsim/utils/consolidate.py @@ -0,0 +1,398 @@ +# +# Consolidate all Netlab YAML files into a single JSON file +# +import importlib.util +import json +import os +import typing +from pathlib import Path + +from box import Box + +from .. import __version__ as netlab_version +from . import files as _files +from . import log +from . import read as _read + + +def _get_schema_path() -> Path: + """Get the path to the JSON schema file""" + return Path(__file__).parent / 'consolidate_schema.json' + +def _load_schema() -> typing.Optional[dict]: + """Load the JSON schema for validation""" + # Check if jsonschema is available + if importlib.util.find_spec('jsonschema') is None: + # jsonschema not available, skip validation + return None + + schema_path = _get_schema_path() + if not schema_path.exists(): + log.warning(text=f'JSON schema not found at {schema_path}', module='consolidate') + return None + + try: + with open(schema_path, 'r') as f: + return json.load(f) + except Exception as ex: + log.warning(text=f'Error loading JSON schema: {ex}', module='consolidate') + return None + +def _validate_json_cache(data: dict, schema: dict) -> bool: + """Validate JSON cache data against schema""" + # Check if jsonschema is available + if importlib.util.find_spec('jsonschema') is None: + # jsonschema not available, skip validation + return True # Don't fail if jsonschema is not available + + # Handle both old and new jsonschema API + try: + # jsonschema v4+ API - validate is a function, not a method + from jsonschema import validate # type: ignore[import-untyped] + from jsonschema.exceptions import ValidationError # type: ignore[import-untyped] + except (ImportError, AttributeError): + # Fallback for older jsonschema versions (< v4) + import jsonschema # type: ignore[import-untyped] + validate = getattr(jsonschema, 'validate', None) + if validate is None: + # Last resort: try validators module + from jsonschema.validators import validate # type: ignore[import-untyped,no-redef] + try: + from jsonschema.exceptions import ValidationError # type: ignore[import-untyped] + except ImportError: + ValidationError = getattr(jsonschema, 'ValidationError', Exception) # type: ignore[attr-defined] + + try: + validate(instance=data, schema=schema) + return True + except ValidationError as ex: + log.error(f'JSON cache validation failed: {ex.message}', module='consolidate') + if ex.path: + log.error(f' Path: {".".join(str(p) for p in ex.path)}', module='consolidate') + return False + except Exception as ex: + log.warning(text=f'Error during JSON cache validation: {ex}', module='consolidate') + return True # Don't fail on validation errors, just warn + +def _collect_yaml_files( + topology_file: str, + user_defaults: typing.Optional[list] = None, + system_defaults: typing.Optional[list] = None) -> dict: + """ + Collect all YAML files that would be loaded for a topology. + Returns a dictionary mapping file paths to their parsed content. + + This simulates what the load() function does - it loads the topology + and all defaults files, collecting all YAML files that get read. + """ + collected = {} + visited = set() + + def collect_file(fname: str, source: typing.Optional[str] = None) -> None: + """Recursively collect a YAML file and all its includes""" + # Normalize the filename + if fname.startswith('package:'): + cache_key = fname + else: + try: + cache_key = str(_files.absolute_path(fname)) + except: + cache_key = fname + + # Skip if already collected + if cache_key in visited: + return + + visited.add(cache_key) + + # Read the YAML file (this will process includes automatically) + try: + yaml_data = _read.read_yaml(filename=fname) + if yaml_data is None: + return + + # Store the file content (includes are already processed) + collected[cache_key] = { + 'content': yaml_data.to_dict(), + 'source': fname, + 'package': fname.startswith('package:') + } + except Exception as ex: + log.warning(text=f'Error collecting file {fname}: {ex}', module='consolidate') + + # Normalize topology file path + if not topology_file.startswith('package:'): + topology_file = str(_files.absolute_path(topology_file)) + + # Collect the main topology file (this will also collect all its includes) + collect_file(topology_file) + + # Now collect all defaults files that would be loaded + # We need to simulate what load() does - it reads topology first, then defaults + try: + # Temporarily read topology to get defaults list + temp_topology = _read.read_yaml(filename=topology_file) + if temp_topology: + defaults_list = _read.build_defaults_list( + temp_topology, + user_defaults=user_defaults, + system_defaults=system_defaults + ) + + # Collect all defaults files + for dfname in defaults_list: + if dfname.find('package:') != 0: + abs_dfname = str(_files.absolute_path(dfname, topology_file)) + if os.path.isfile(abs_dfname): + collect_file(abs_dfname) + elif os.path.isfile(dfname): + collect_file(dfname) + else: + collect_file(dfname) + except Exception as ex: + log.warning(text=f'Error collecting defaults files: {ex}', module='consolidate') + # Fallback: try to collect common defaults + from ..utils.read import SYSTEM_DEFAULTS, USER_DEFAULTS + all_defaults = (user_defaults or USER_DEFAULTS) + (system_defaults or SYSTEM_DEFAULTS) + for dfname in all_defaults: + if dfname.find('package:') != 0: + abs_dfname = str(_files.absolute_path(dfname, topology_file)) + if os.path.isfile(abs_dfname): + collect_file(abs_dfname) + else: + collect_file(dfname) + + return collected + +def consolidate_to_json( + topology_file: typing.Optional[str] = None, + output_file: str = 'netlab.consolidated.json', + user_defaults: typing.Optional[list] = None, + system_defaults: typing.Optional[list] = None) -> None: + """ + Consolidate all YAML files into a single JSON file. + + This works by actually loading the topology (which loads all files) + and tracking all files that get read during the process. + + If topology_file is None, consolidates all system/package YAML files. + This is useful for integration test suites where you want to cache all default + files, modules, devices, and providers that would be used across multiple labs. + + Args: + topology_file: Path to the topology YAML file (optional: if None, consolidates all system/package files) + output_file: Path where the JSON file should be written + user_defaults: Optional list of user defaults files + system_defaults: Optional list of system defaults files + """ + if topology_file: + log.info(f'Consolidating YAML files for {topology_file}...') + else: + log.info('Consolidating all system/package YAML files...') + + # Track all files that get read + files_tracked = {} + original_read_yaml = _read.read_yaml + + def tracking_read_yaml(filename: typing.Optional[str] = None, string: typing.Optional[str] = None) -> typing.Optional[Box]: + """Wrapper around read_yaml that tracks all files read""" + if filename and not string: + # Normalize the filename for tracking + if filename.startswith('package:'): + cache_key = filename + else: + try: + cache_key = str(_files.absolute_path(filename)) + except: + cache_key = filename + + # Call original read_yaml + result = original_read_yaml(filename=filename, string=string) + + # Track this file if we haven't seen it yet + if result and cache_key not in files_tracked: + # Handle both Box objects (dicts) and lists + if isinstance(result, Box): + content = result.to_dict() + elif isinstance(result, (list, dict)): + content = result + else: + content = {} + + files_tracked[cache_key] = { + 'content': content, + 'source': filename, + 'package': filename.startswith('package:') + } + + return result + else: + return original_read_yaml(filename=filename, string=string) + + # Temporarily replace read_yaml with our tracking version + _read.read_yaml = tracking_read_yaml + + try: + if topology_file: + # Actually load the topology - this will read all files + from ..utils.read import load + load( + topology_file, + user_defaults=user_defaults, + system_defaults=system_defaults + ) + else: + # Load all system defaults + from ..utils.read import SYSTEM_DEFAULTS + + # Try to load system defaults + for default_file in SYSTEM_DEFAULTS: + try: + _read.read_yaml(filename=default_file) + except: + pass # Some defaults may not exist, that's OK + + # Load package defaults + try: + _read.read_yaml(filename='package:topology-defaults.yml') + except: + pass + + # Load all defaults from netsim/defaults + defaults_dir = Path(__file__).parent.parent / 'defaults' + if defaults_dir.exists(): + for yaml_file in defaults_dir.rglob('*.yml'): + try: + result = _read.read_yaml(filename=str(yaml_file)) + # Some files might be sequences, skip those that fail + if result is None: + continue + except Exception: + pass # Skip files that can't be read as dicts (e.g., sequences) + + # Load all modules + modules_dir = Path(__file__).parent.parent / 'modules' + if modules_dir.exists(): + for yaml_file in modules_dir.rglob('*.yml'): + try: + result = _read.read_yaml(filename=str(yaml_file)) + if result is None: + continue + except Exception: + pass # Skip files that can't be read as dicts + + # Load all devices + devices_dir = Path(__file__).parent.parent / 'devices' + if devices_dir.exists(): + for yaml_file in devices_dir.rglob('*.yml'): + try: + result = _read.read_yaml(filename=str(yaml_file)) + if result is None: + continue + except Exception: + pass # Skip files that can't be read as dicts + + # Load all providers + providers_dir = Path(__file__).parent.parent / 'providers' + if providers_dir.exists(): + for yaml_file in providers_dir.rglob('*.yml'): + try: + result = _read.read_yaml(filename=str(yaml_file)) + if result is None: + continue + except Exception: + pass # Skip files that can't be read as dicts + + # Load extra modules (skip deploy files which are often sequences) + extra_dir = Path(__file__).parent.parent / 'extra' + if extra_dir.exists(): + for yaml_file in extra_dir.rglob('*.yml'): + # Skip deploy files which are typically sequences + if 'deploy' in str(yaml_file): + continue + try: + result = _read.read_yaml(filename=str(yaml_file)) + if result is None: + continue + except Exception: + pass # Skip files that can't be read as dicts + finally: + # Restore original read_yaml + _read.read_yaml = original_read_yaml + + # Create the consolidated structure + consolidated = { + 'version': '1.0', + 'netlab_version': netlab_version, + 'topology_file': topology_file, + 'files': files_tracked, + 'file_count': len(files_tracked) + } + + # Validate against schema before writing + schema = _load_schema() + if schema: + if not _validate_json_cache(consolidated, schema): + log.error('Generated JSON cache does not match schema, but writing anyway', module='consolidate') + # Schema validation passed if we get here + + # Write to JSON file + output_path = Path(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + with open(output_path, 'w') as f: + json.dump(consolidated, f, indent=2, default=str) + + if topology_file: + log.info(f'Consolidated {len(files_tracked)} files into {output_file}') + else: + log.info(f'Consolidated {len(files_tracked)} system/package files into {output_file}') + +def load_from_json(json_file: str, validate: bool = True) -> typing.Optional[dict]: + """ + Load the consolidated JSON file. + + Args: + json_file: Path to the JSON cache file + validate: Whether to validate against schema (default: True) + + Returns: + Full consolidated dictionary (with version, files, etc.) or None if file doesn't exist or is invalid + """ + if not os.path.isfile(json_file): + return None + + try: + with open(json_file, 'r') as f: + consolidated = json.load(f) + + # Check netlab version compatibility + cache_version = consolidated.get('netlab_version') + if cache_version != netlab_version: + log.error(f'JSON cache {json_file} was created with netlab version {cache_version}', module='consolidate') + log.error(f'Current netlab version is {netlab_version}', module='consolidate') + log.error('Cache file is incompatible and must be regenerated', module='consolidate') + log.error('Please run "netlab consolidate" again to create a new cache', module='consolidate') + return None + + # Validate against schema if requested + if validate: + schema = _load_schema() + if schema: + if not _validate_json_cache(consolidated, schema): + log.error(f'JSON cache {json_file} does not match schema', module='consolidate') + log.error('Cache file may be corrupted or from an incompatible version', module='consolidate') + return None + # Note: log.debug may not be available, so we skip debug messages + else: + # Schema validation requested but schema not found (jsonschema may not be available) + pass + + return consolidated + except json.JSONDecodeError as ex: + log.error(f'Invalid JSON in cache file {json_file}: {ex}', module='consolidate') + return None + except Exception as ex: + log.warning(text=f'Error loading JSON cache {json_file}: {ex}', module='consolidate') + return None + diff --git a/netsim/utils/consolidate_schema.json b/netsim/utils/consolidate_schema.json new file mode 100644 index 0000000000..36380bed87 --- /dev/null +++ b/netsim/utils/consolidate_schema.json @@ -0,0 +1,60 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Netlab Consolidated JSON Cache Schema", + "description": "Schema for validating consolidated JSON cache files created by netlab consolidate", + "type": "object", + "required": ["version", "netlab_version", "files", "file_count"], + "properties": { + "version": { + "type": "string", + "description": "Schema version of the consolidated cache", + "pattern": "^\\d+\\.\\d+$" + }, + "netlab_version": { + "type": "string", + "description": "Netlab version that created this cache (e.g., '25.12.02'). Cache is invalid if this doesn't match the current netlab version.", + "pattern": "^\\d+\\.\\d+\\.\\d+$" + }, + "topology_file": { + "oneOf": [ + {"type": "string"}, + {"type": "null"} + ], + "description": "Path to the original topology file that was consolidated (null if consolidating all system files)" + }, + "files": { + "type": "object", + "description": "Dictionary mapping file paths to their content", + "patternProperties": { + ".*": { + "type": "object", + "required": ["content", "source"], + "properties": { + "content": { + "description": "Parsed YAML content as a dictionary/object", + "type": "object", + "additionalProperties": true + }, + "source": { + "type": "string", + "description": "Original source filename/path" + }, + "package": { + "type": "boolean", + "description": "Whether this is a package file (package: prefix)" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "file_count": { + "type": "integer", + "description": "Number of files in the cache", + "minimum": 0 + } + }, + "additionalProperties": false +} + diff --git a/netsim/utils/read.py b/netsim/utils/read.py index 5aa924b90f..6f90775bfc 100644 --- a/netsim/utils/read.py +++ b/netsim/utils/read.py @@ -184,6 +184,73 @@ def save_to_pickle(path: str, data: Box) -> None: # read_cache: dict = {} +# JSON cache support +_json_cache: typing.Optional[dict] = None + +def set_json_cache(json_file: str) -> None: + """ + Set the JSON cache file to use for loading YAML files. + This will cause read_yaml() to check the JSON cache before reading YAML files. + + Args: + json_file: Path to the consolidated JSON cache file + """ + global _json_cache + from . import consolidate as _consolidate + + consolidated = _consolidate.load_from_json(json_file, validate=True) + if consolidated is None: + log.warning(text=f'Failed to load JSON cache from {json_file}, falling back to YAML files', module='read') + _json_cache = None + else: + # Extract just the files dictionary for use in read_yaml + _json_cache = consolidated.get('files') or {} + file_count = len(_json_cache) + cache_version = consolidated.get('netlab_version', 'unknown') + log.info(f'Using JSON cache from {json_file} (netlab {cache_version}, {file_count} files)', module='read') + +def _read_from_json_cache(filename: str) -> typing.Optional[Box]: + """ + Try to read a file from the JSON cache. + + Args: + filename: The filename to look up in the cache + + Returns: + Box object with the file content if found, None otherwise + + Note: + Assumes _json_cache is not None. Caller should check this first. + """ + global _json_cache + + # Type guard: ensure _json_cache is not None (caller should check, but mypy needs this) + if _json_cache is None: + return None + + # Normalize the filename to match cache keys + if filename.startswith('package:'): + cache_key = filename + else: + try: + cache_key = str(_files.absolute_path(filename)) + except: + cache_key = filename + + # Try exact match first + file_data = _json_cache.get(cache_key) + if file_data: + content = file_data.get('content', {}) + return Box(content, default_box=True, box_dots=True, default_box_none_transform=False) + + # Try to find by source filename (for package: files or relative paths) + for file_data in _json_cache.values(): + if file_data.get('source') == filename: + content = file_data.get('content', {}) + return Box(content, default_box=True, box_dots=True, default_box_none_transform=False) + + return None + class UniqueKeyLoader(yaml.SafeLoader): def construct_mapping(self, node : yaml.MappingNode, deep : bool = False) -> dict: mapping = [] @@ -196,7 +263,7 @@ def construct_mapping(self, node : yaml.MappingNode, deep : bool = False) -> dic return super().construct_mapping(node, deep) def read_yaml(filename: typing.Optional[str] = None, string: typing.Optional[str] = None) -> typing.Optional[Box]: - global read_cache + global read_cache, _json_cache if string is not None: try: @@ -212,6 +279,13 @@ def read_yaml(filename: typing.Optional[str] = None, string: typing.Optional[str if log.debug_active('defaults'): print(f"Reading {filename}") + # Check JSON cache first if it's available + if _json_cache is not None: + cached_result = _read_from_json_cache(filename) + if cached_result is not None: + return cached_result + + # Fall back to normal cache if filename in read_cache: return Box(read_cache[filename],default_box=True,box_dots=True,default_box_none_transform=False) diff --git a/netsim/yang/netlab-topology.yang b/netsim/yang/netlab-topology.yang new file mode 100644 index 0000000000..6e73b3bb52 --- /dev/null +++ b/netsim/yang/netlab-topology.yang @@ -0,0 +1,760 @@ +module netlab-topology { + yang-version 1.1; + namespace "urn:netlab:topology"; + prefix "netlab"; + + description + "YANG model for netlab lab topology validation with MUST statements + to check internal consistency. The topology is described in a YAML file + using a dictionary format."; + + revision 2025-12-12 { + description "Initial version"; + } + + container topology { + description + "Network lab topology structure. The three major components that should + be present in every topology file are: nodes (lab devices), links + (between the lab devices), and defaults (topology-wide defaults like + default device type)."; + + container defaults { + description + "Topology-wide defaults. System defaults are specified in the global + topology-defaults.yml file and imported into the lab topology under + the defaults top-level attribute. They can be overwritten with the + defaults element within the lab topology or from user- or system + defaults files."; + leaf device { + type string; + description + "Default device type used for all nodes unless overridden at the + node level. See supported platforms documentation for available + device types."; + } + leaf provider { + type string; + description + "Virtualization provider (libvirt, clab, or external). Default + value: libvirt."; + } + leaf name { + type string; + description + "Default topology name. If not specified, the first twelve + characters of the directory name are used (spaces and dots replaced + with underscores)."; + } + leaf-list module { + type string; + description + "Default list of optional configuration modules used by this + network topology. Can be overridden at the node level."; + } + // Use anydata for complex nested structures that can have arbitrary keys + // This allows providers, devices, and other defaults to have flexible structures + anydata providers { + description "Provider-specific settings (flexible structure)"; + } + container sources { + description + "Default source files. Source files can be specified in user + defaults, system defaults, or topology defaults."; + leaf-list extra { + type string; + description "Additional source files to include"; + } + leaf-list list { + type string; + description "List of source files"; + } + leaf-list user { + type string; + description "User source files (from ~/.netlab.yml)"; + } + leaf-list system { + type string; + description "System source files (from /etc/netlab/defaults.yml)"; + } + anydata other { + description "Other source settings (flexible structure)"; + } + } + anydata devices { + description "Device-specific default settings (flexible structure)"; + } + anydata daemons { + description "Daemon-specific default settings (flexible structure)"; + } + anydata outputs { + description "Output module default settings (flexible structure)"; + } + anydata modules { + description "Module default settings (flexible structure)"; + } + anydata attributes { + description "Attribute validation settings (flexible structure)"; + } + anydata paths { + description "Search path settings (flexible structure)"; + } + anydata tools { + description "External tools default settings (flexible structure)"; + } + anydata warnings { + description "Warning settings (flexible structure)"; + } + anydata const { + description "Constants and limits (flexible structure)"; + } + anydata addressing { + description "Addressing pool defaults (flexible structure)"; + } + anydata automation { + description "Automation settings (flexible structure)"; + } + anydata hints { + description "Error hints settings (flexible structure)"; + } + anydata multilab { + description "Multi-lab settings (flexible structure)"; + } + anydata pools { + description "Address pools (flexible structure)"; + } + anydata plugin { + description "Plugin settings (flexible structure)"; + } + anydata roles { + description "Role settings (flexible structure)"; + } + anydata vlan { + description "VLAN module defaults (flexible structure)"; + } + anydata ospf { + description "OSPF module defaults (flexible structure)"; + } + anydata isis { + description "ISIS module defaults (flexible structure)"; + } + anydata evpn { + description "EVPN module defaults (flexible structure)"; + } + anydata vrf { + description "VRF module defaults (flexible structure)"; + } + anydata sr { + description "Segment Routing defaults (flexible structure)"; + } + anydata mpls { + description "MPLS defaults (flexible structure)"; + } + anydata srv6 { + description "SRv6 defaults (flexible structure)"; + } + anydata bfd { + description "BFD defaults (flexible structure)"; + } + anydata gateway { + description "Gateway defaults (flexible structure)"; + } + anydata lag { + description "LAG defaults (flexible structure)"; + } + anydata vxlan { + description "VXLAN defaults (flexible structure)"; + } + anydata netlab { + description "Netlab-specific defaults (flexible structure)"; + } + anydata ports { + description "Port mappings (flexible structure)"; + } + anydata routing { + description "Routing policy defaults (flexible structure)"; + } + anydata ripv2 { + description "RIP defaults (flexible structure)"; + } + anydata bgp { + description "BGP module defaults (flexible structure)"; + } + anydata dhcp { + description "DHCP defaults (flexible structure)"; + } + anydata linux { + description "Linux defaults (flexible structure)"; + } + anydata docker { + description "Docker defaults (flexible structure)"; + } + anydata ansible { + description "Ansible defaults (flexible structure)"; + } + anydata vagrant { + description "Vagrant defaults (flexible structure)"; + } + anydata eigrp { + description "EIGRP defaults (flexible structure)"; + } + anydata stp { + description "STP defaults (flexible structure)"; + } + anydata lldp { + description "LLDP defaults (flexible structure)"; + } + anydata ntp { + description "NTP defaults (flexible structure)"; + } + anydata snmp { + description "SNMP defaults (flexible structure)"; + } + anydata syslog { + description "Syslog defaults (flexible structure)"; + } + anydata initial { + description "Initial configuration defaults (flexible structure)"; + } + anydata custom_config { + description "Custom configuration defaults (flexible structure)"; + } + anydata deploy { + description "Deployment defaults (flexible structure)"; + } + anydata prefix { + description "Prefix defaults (flexible structure)"; + } + anydata graph { + description "Graph defaults (flexible structure)"; + } + anydata report { + description "Report defaults (flexible structure)"; + } + // Note: Additional default fields may need to be added as they are encountered + // YANG doesn't support true catch-all for container children + } + + list nodes { + key "name"; + description + "Lab devices (nodes). Nodes can be specified as a list of strings + (node names) or a dictionary of node names and node attributes. + Individual nodes can override the default device type or add + additional node attributes."; + + leaf name { + type string; + mandatory true; + description + "Node name. Node names can have up to 16 characters by default. + The maximum length can be changed with defaults.const.MAX_NODE_ID_LENGTH."; + } + + leaf device { + type string; + description + "Device type for this node. If not specified, the default device + type from defaults.device is used. See supported platforms + documentation for available device types."; + } + + leaf provider { + type string; + description + "Virtualization provider for this node. Allows mixing providers + within a single topology (e.g., some nodes using libvirt, others + using clab)."; + } + + leaf box { + type string; + description + "Vagrant box or Docker container identifier. Default images for + individual device types are defined in system defaults and can be + changed with defaults.devices... settings."; + } + + leaf id { + type uint32; + description + "Static node identifier (integer between 1 and MAX_NODE_ID, default + 250). If not specified, node ID is assigned based on node's position + in the nodes dictionary, starting with 1 and skipping static IDs + used by other nodes."; + } + + leaf role { + type enumeration { + enum router { + description "Router role - device gets loopback IP and participates in routing"; + } + enum host { + description "Host role - device does not get loopback IP and uses static routing"; + } + enum bridge { + description "Bridge role - device acts as a bridge"; + } + enum gateway { + description "Gateway role - device acts as a gateway"; + } + } + description + "Node role. When set to 'host', the device does not get a loopback IP + address and uses static routing toward the default gateway. Default + role is 'router'."; + } + + leaf mtu { + type uint32 { + range "64..9216"; + } + description + "Device-wide (system) MTU. This MTU is applied to all interfaces that + don't have an explicit MTU. Valid range: 64-9216 bytes."; + } + + anydata loopback { + description + "Loopback interface configuration. Can be a boolean 'true' to enable + loopback, or a dictionary with custom IPv4/IPv6 addresses and other + loopback settings (flexible structure)."; + } + + leaf-list module { + type string; + description + "List of optional configuration modules used by this node. Overrides + the global module list if specified. Host nodes do not inherit + global configuration modules."; + } + + list interfaces { + key "ifindex"; + description + "Node interfaces. Interfaces are created as needed during the link + transformation phase and collected in the interfaces list. Each + interface can have addressing, MTU, and module-specific attributes."; + leaf ifindex { + type uint32; + description + "Interface index used to generate the interface/port name. Can be + specified per-node in link definitions with the ifindex attribute."; + } + anydata interface_data { + description + "Interface data including addressing, MTU, module-specific + attributes, and other interface parameters (flexible structure)"; + } + } + + anydata clab { + description + "Containerlab-specific node settings. Can include attributes like + clab.type (device type when supported by containerlab) and + clab.license (license file needed for network devices)."; + } + anydata libvirt { + description + "Libvirt-specific node settings. Can include attributes like + libvirt.nic_model (virtual NIC model: virtio, e1000, rtl8139, etc.)."; + } + anydata external { + description + "External provider-specific node settings. Used when integrating + netlab topologies with external devices or management systems."; + } + + // Device-specific attributes (eos, frr, ios, etc.) + anydata eos { + description + "Arista EOS-specific node attributes (e.g., eos.systemmacaddr, eos.serialnumber)."; + } + anydata frr { + description + "FRR-specific node attributes."; + } + anydata ios { + description + "Cisco IOS-specific node attributes."; + } + // Add other device-specific attributes as needed + + // Module-specific attributes (routing, bgp, ospf, etc.) + anydata routing { + description + "Routing module node attributes (e.g., routing.static for static routes)."; + } + anydata bridge { + description + "Bridge module node attributes."; + } + anydata dhcp { + description + "DHCP module node attributes."; + } + // Add other module-specific attributes as needed + + + container bgp { + when "../module[.='bgp']"; + description + "BGP configuration. This container can only be present when the BGP + module is enabled on the node."; + must "../module[.='bgp']" { + error-message "BGP container can only be present when BGP module is enabled"; + } + leaf as { + type uint32; + mandatory true; + description + "BGP AS number. Nodes with the same AS number are automatically + grouped into an AS group (e.g., as65000)."; + } + } + + must "device or ../../defaults/device" { + error-message "Node must have a device type or a default device must be set"; + } + } + + list links { + description + "Links between lab devices. Links can be specified in multiple + formats: as a string (node-node format), as a list of node names + (for multi-access links), as a dictionary of node names and link + attributes, or as a dictionary with a list of node interfaces. + Each link definition is converted into a dictionary+list of + interfaces format during topology transformation."; + key "linkindex"; + + leaf linkindex { + type uint32; + description + "Link sequence number (starting with one), used to generate default + bridge names in libvirt. This is a read-only attribute set during + topology transformation."; + } + + list interfaces { + key "node"; + description + "Interfaces connected by this link. Each interface entry represents + a node attachment to the link."; + min-elements 1; + + leaf node { + type leafref { + path "../../../nodes/name"; + require-instance true; + } + mandatory true; + description + "Node name. Must reference a node defined in the nodes list."; + } + + leaf ipv4 { + type union { + type boolean; + type string; + } + description + "IPv4 address for this interface. Can be specified as an address + string or as a boolean 'true' for unnumbered interfaces (where + the interface uses the node's loopback address)."; + } + + leaf ipv6 { + type union { + type boolean; + type string; + } + description + "IPv6 address for this interface. Can be specified as an address + string or as a boolean 'true' for unnumbered interfaces (where + the interface uses the node's loopback address)."; + } + + leaf shutdown { + type boolean; + description + "Shut down this interface during initial configuration."; + } + + leaf mtu { + type uint32 { + range "64..9216"; + } + description + "Interface MTU. Valid range: 64-9216 bytes."; + } + + anydata interface_attrs { + description + "Additional interface attributes (flexible structure)"; + } + } + + leaf bandwidth { + type uint32; + description + "Link bandwidth in Mbps. Used to configure interface bandwidth + when supported by the connected device(s)."; + } + + leaf type { + type enumeration { + enum p2p { + description "Point-to-point link between two nodes"; + } + enum lan { + description "Multi-access link (LAN)"; + } + enum stub { + description "Stub link (single node attachment)"; + } + enum loopback { + description "Loopback link"; + } + enum tunnel { + description "Tunnel link"; + } + enum vlan_member { + description "VLAN member link"; + } + enum lag { + description "Link aggregation group (LAG)"; + } + } + description + "Link type. Valid values: p2p (point-to-point), lan (multi-access), + stub (single node attachment), loopback, tunnel, vlan_member, lag. + When missing, link type is selected based on the number of devices + connected to the link."; + } + + leaf bridge { + type string; + description + "Bridge name for this link. Used in libvirt provider to specify + the Linux bridge name for the link."; + } + + leaf pool { + type string; + description + "Address pool name to use for this link. References a pool defined + in the addressing element."; + } + + leaf prefix { + type string; + description + "Prefix name to use for this link. References a prefix defined + in the prefix element."; + } + + leaf name { + type string; + description + "Link name. Used for identification and in some provider configurations."; + } + + leaf mtu { + type uint32 { + range "64..9216"; + } + description + "Link MTU. Valid range: 64-9216 bytes."; + } + + container ra { + description + "IPv6 Router Advertisement (RA) configuration for this link."; + leaf disable { + type boolean; + description + "Disable Router Advertisements on this link."; + } + leaf slaac { + type boolean; + description + "Enable Stateless Address Autoconfiguration (SLAAC)."; + } + leaf dhcp { + type string; + description + "DHCP configuration: 'all' or 'other'."; + } + leaf onlink { + type boolean; + description + "Advertise on-link prefix."; + } + anydata ra_config { + description + "Additional RA configuration attributes (flexible structure)"; + } + } + + anydata link_attrs { + description + "Additional link attributes (flexible structure)"; + } + + must "count(interfaces/node) >= 2 or type = 'stub' or type = 'loopback' or not(type)" { + error-message "Link must connect at least two nodes (unless it's a stub or loopback link, or type is not specified)"; + } + } + + leaf-list module { + type string; + description + "Default list of optional configuration modules used by this network + topology. You can use device-level module attribute to override this + setting for individual nodes. Host nodes do not inherit global + configuration modules."; + } + + leaf message { + type string; + description + "Help message to display after successful 'netlab initial' or + 'netlab up' commands. You can use this message to tell the end-user + how to use the lab."; + } + + leaf name { + type string; + description + "Topology name used to name Linux bridges when using libvirt Vagrant + plugin. If not specified, the first twelve characters of the directory + name are used (spaces and dots replaced with underscores)."; + } + + leaf provider { + type string; + description + "Virtualization provider (libvirt, clab, or external). Default value: + libvirt."; + } + + leaf version { + type string; + description + "Minimum netlab version required to use this lab topology. You can + use either a simple version number or Python module version + specification (example: >= 1.6.3)."; + } + + anydata addressing { + description + "IPv4 and IPv6 address pools used to address management, loopback, + LAN, P2P and stub interfaces. Default pools include: mgmt (management + addresses), loopback (loopback addresses), lan (LAN link addresses), + p2p (point-to-point link addresses), and others. You can override or + augment them in the topology addressing element."; + } + + anydata pools { + description + "Address pools (internal representation, flexible structure). This + is the processed form of the addressing element after topology + transformation."; + } + + anydata groups { + description + "Groups of lab objects (nodes, VLANs, or VRFs) with similar + attributes. Groups are specified as a dictionary with group names + as keys. Dictionary values can be either a list of member nodes or a + dictionary specifying members. Groups can be used to set node/VLAN/VRF + attributes, set Ansible group variables, or limit the scope of netlab + commands."; + } + + anydata vlans { + description + "VLAN definitions. VLANs are specified as a dictionary with VLAN + names as keys. Each VLAN can have attributes like mode (bridge, irb, + route), links (list of links associated with the VLAN), and other + VLAN-specific settings."; + } + + anydata validate { + description + "Lab validation tests. Validation tests are specified as a dictionary + with test names as keys. Each test can define nodes to test, wait + conditions, plugins to execute, and expected results."; + } + + anydata prefix { + description + "Named IP prefixes that can be used to address links and VLANs or in + prefix lists or validation expressions. The dictionary keys are + prefix names (e.g., lan_1, lan_2, loopback); the values are dictionaries + defining individual prefixes with ipv4, ipv6, pool, and allocation + attributes (flexible structure)."; + } + + container tools { + description + "External network management tools deployed after the lab has been + started and configured. Tools are started as Docker containers and + can access the management network and management interfaces of lab + devices."; + anydata tool_definitions { + description + "Tool definitions. Each tool can have runtime, enabled, and + tool-specific configuration parameters (flexible structure)"; + } + } + + leaf-list plugin { + type string; + description + "List of plugins used by this topology. Plugins extend netlab + functionality with custom configuration templates, validation tests, + or other features."; + } + + anydata Plugin { + description + "Plugin internal data (flexible structure). Internal representation + of plugin configuration after topology transformation."; + } + + anydata Provider { + description + "Provider internal data (flexible structure). Internal representation + of provider configuration after topology transformation."; + } + + anydata libvirt { + description + "Libvirt-specific topology settings. Can include libvirt-specific + configuration like bridge names, port mappings, and other + virtualization provider settings."; + } + + anydata clab { + description + "Containerlab-specific topology settings. Can include containerlab-specific + configuration like network names, container settings, and other + containerlab provider settings."; + } + + leaf-list input { + type string; + description + "Input topology file paths (internal). Tracks the source topology + files used during transformation."; + } + + // Global MUST statements + must "count(nodes) > 0" { + error-message "Topology must contain at least one node"; + } + } +} + diff --git a/requirements.txt b/requirements.txt index daa8e916f1..e0b39e4352 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ filelock >= 3.16.1 packaging requests rich +yangson >= 1.4.19 diff --git a/tests/check-integration-tests.sh b/tests/check-integration-tests.sh index f45c56b402..d0adc36b72 100755 --- a/tests/check-integration-tests.sh +++ b/tests/check-integration-tests.sh @@ -3,6 +3,8 @@ # Run transformation code on integration tests for an additional # verification before merging pull requests # + +err_cnt=0 for file in integration/**/[0-9]*.yml platform-integration/**/[0-9]*.yml; do ../netlab create -o none -d none $file 2>/dev/null || ( echo "Errors found in $file" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..be30c5a011 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,31 @@ +# +# Pytest configuration and fixtures +# +import os + +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def setup_json_cache(): + """ + Set up JSON cache for all tests if available. + + This fixture runs once per test session and sets the JSON cache + if the NETLAB_JSON_CACHE environment variable is set and points + to an existing file. This provides significant performance + improvements (44.6% faster) for tests that load topology files. + """ + json_cache = os.environ.get('NETLAB_JSON_CACHE') + if json_cache and os.path.exists(json_cache): + from netsim.utils import read as _read + _read.set_json_cache(json_cache) + print(f"\n✅ Using JSON cache: {json_cache}") + print(f" This will speed up all topology loading operations") + else: + if json_cache: + print(f"\n⚠️ JSON cache specified but not found: {json_cache}") + print(f" Running tests without JSON cache (slower)") + yield + # Cleanup (if needed) happens here +