From 74344f542700bbc8f095f71111ad6990278f77c7 Mon Sep 17 00:00:00 2001 From: Mohammad Tayyab Date: Mon, 20 Jul 2026 20:17:25 +0200 Subject: [PATCH 1/3] feat: add data fetching from meter, inverter, component and get the display name from assets api Signed-off-by: Mohammad Tayyab --- .../lib/notebooks/reporting/__init__.py | 15 +- .../notebooks/reporting/data_processing.py | 234 +++++++++---- .../reporting/utils/component_metadata.py | 26 ++ .../lib/notebooks/reporting/utils/helpers.py | 273 ++++++++++++--- .../reporting/utils/reporting_nb_functions.py | 310 +++++++++++++++++- 5 files changed, 731 insertions(+), 127 deletions(-) create mode 100644 src/frequenz/lib/notebooks/reporting/utils/component_metadata.py diff --git a/src/frequenz/lib/notebooks/reporting/__init__.py b/src/frequenz/lib/notebooks/reporting/__init__.py index 9bcf8d92..4ad18b2e 100644 --- a/src/frequenz/lib/notebooks/reporting/__init__.py +++ b/src/frequenz/lib/notebooks/reporting/__init__.py @@ -3,6 +3,17 @@ """Initialise the reporting related modules.""" -from .data_processing import create_battery_usecase_df, create_energy_report_df +from .data_processing import ( + build_energy_report, + create_battery_usecase_df, + create_energy_report_df, +) +from .utils.component_metadata import ComponentMetadata, EnergyReport -__all__ = ["create_energy_report_df", "create_battery_usecase_df"] +__all__ = [ + "build_energy_report", + "create_energy_report_df", + "create_battery_usecase_df", + "ComponentMetadata", + "EnergyReport", +] diff --git a/src/frequenz/lib/notebooks/reporting/data_processing.py b/src/frequenz/lib/notebooks/reporting/data_processing.py index 42d28d7d..dc68508a 100644 --- a/src/frequenz/lib/notebooks/reporting/data_processing.py +++ b/src/frequenz/lib/notebooks/reporting/data_processing.py @@ -24,20 +24,169 @@ tables for KPIs, dashboards, and stakeholder reporting. """ +import warnings + import pandas as pd +from frequenz.client.base.exception import ApiClientError from frequenz.gridpool import MicrogridConfig from frequenz.lib.notebooks.reporting.utils.column_mapper import ColumnMapper +from frequenz.lib.notebooks.reporting.utils.component_metadata import ( + ComponentMetadata, + EnergyReport, +) from frequenz.lib.notebooks.reporting.utils.helpers import ( AggregatedComponentConfig, add_energy_flows, convert_timezone, fill_aggregated_component_columns, + get_component_ids, get_energy_report_columns, - label_component_columns, + get_meter_display_names, ) +def _resolve_component_metadata( + energy_report_df: pd.DataFrame, + component_types: list[str], + mcfg: MicrogridConfig, + component_display_names: dict[str, str] | None, + *, + include_display_names: bool, +) -> tuple[ComponentMetadata, list[str]]: + """Resolve component display metadata and canonical component columns.""" + resolved_display_names = component_display_names if include_display_names else {} + if include_display_names and resolved_display_names is None: + microgrid_id = getattr(getattr(mcfg, "meta", None), "microgrid_id", None) + if microgrid_id is not None: + try: + resolved_display_names = get_meter_display_names(int(microgrid_id)) + except ValueError: + resolved_display_names = None + except ApiClientError as exc: # pragma: no cover - defensive fallback + warnings.warn( + f"Could not fetch meter display names from the Assets API: {exc}", + RuntimeWarning, + stacklevel=2, + ) + resolved_display_names = None + + component_ids_by_type = { + component_type: [ + component_id + for component_id in get_component_ids(mcfg, component_type) + if component_id in energy_report_df.columns + ] + for component_type in component_types + } + single_components = [ + component_id + for component_type in component_types + for component_id in component_ids_by_type.get(component_type, []) + ] + + return ( + ComponentMetadata( + display_names=resolved_display_names or {}, + ids_by_type=component_ids_by_type, + ), + single_components, + ) + + +# pylint: disable=too-many-arguments +def _build_energy_report_dataframe( + df: pd.DataFrame, + component_types: list[str], + mapper: ColumnMapper, + *, + tz_name: str, + assume_tz: str, + fill_missing_values: bool, + aggregated_component_config: AggregatedComponentConfig | None, + component_metadata: ComponentMetadata, +) -> pd.DataFrame: + """Apply the dataframe transformation pipeline for energy reporting.""" + energy_report_df = df.copy() + + if isinstance(energy_report_df.index, (pd.DatetimeIndex, pd.PeriodIndex)): + if "timestamp" not in energy_report_df.columns: + energy_report_df = energy_report_df.reset_index(names="timestamp") + + energy_report_df = add_energy_flows( + energy_report_df, + production_cols=["pv", "chp", "wind"], + consumption_cols=["consumption"], + grid_cols=["grid"], + battery_cols=["battery"], + ) + energy_report_df = mapper.to_canonical(energy_report_df) + energy_report_df["timestamp"] = pd.to_datetime( + energy_report_df["timestamp"], errors="coerce", utc=True + ) + energy_report_df["timestamp"] = convert_timezone( + energy_report_df["timestamp"], + target_tz=tz_name, + assume_tz=assume_tz, + ) + + energy_report_df_cols = get_energy_report_columns( + component_types, + [ + component_id + for component_ids in component_metadata.ids_by_type.values() + for component_id in component_ids + ], + ) + energy_report_df = energy_report_df[energy_report_df_cols] + + if fill_missing_values: + energy_report_df = fill_aggregated_component_columns( + energy_report_df, + component_types, + aggregated_component_config, + component_columns_by_type=component_metadata.ids_by_type, + ) + + return energy_report_df + + +# pylint: disable=too-many-arguments +def build_energy_report( + df: pd.DataFrame, + component_types: list[str], + mcfg: MicrogridConfig, + mapper: ColumnMapper, + *, + tz_name: str = "Europe/Berlin", + assume_tz: str = "UTC", + fill_missing_values: bool = True, + aggregated_component_config: AggregatedComponentConfig | None = None, + component_display_names: dict[str, str] | None = None, + include_component_metadata: bool = True, +) -> EnergyReport: + """Build the normalized energy report and optional component metadata.""" + metadata, _ = _resolve_component_metadata( + df, + component_types, + mcfg, + component_display_names, + include_display_names=include_component_metadata, + ) + energy_report_df = _build_energy_report_dataframe( + df, + component_types, + mapper, + tz_name=tz_name, + assume_tz=assume_tz, + fill_missing_values=fill_missing_values, + aggregated_component_config=aggregated_component_config, + component_metadata=metadata, + ) + + return EnergyReport(df=energy_report_df, metadata=metadata) + + # pylint: disable=too-many-arguments, too-many-locals def create_battery_usecase_df( energy_report_df: pd.DataFrame, @@ -135,13 +284,11 @@ def create_energy_report_df( assume_tz: str = "UTC", fill_missing_values: bool = True, aggregated_component_config: AggregatedComponentConfig | None = None, + component_display_names: dict[str, str] | None = None, ) -> pd.DataFrame: """Create a normalized Energy Report DataFrame with selected columns. - Makes a copy of the input, converts the timestamp column to the configured - timezone, renames standard columns to unified names, adds the net import - column, renames numeric component IDs to labeled names, and returns a - reduced DataFrame containing only relevant columns. + This is a dataframe-only compatibility wrapper around ``build_energy_report()``. Args: df: Raw input table containing energy data. @@ -156,70 +303,25 @@ def create_energy_report_df( aggregated_component_config: Optional mapping of component types to aggregated column metadata used when filling missing aggregates. Defaults to the shared `DEFAULT_AGGREGATED_COMPONENT_CONFIG`. + component_display_names: Optional mapping from numeric component IDs to + display names that are surfaced by ``build_energy_report()`` when + callers need metadata as well. Returns: - The Energy Report DataFrame with standardized and selected columns. + The Energy Report DataFrame. Notes: - Component IDs are renamed to labeled names via ``label_component_columns()``. + Use ``build_energy_report()`` when callers also need component metadata. """ - energy_report_df = df.copy() - - # Only reset index if it's a datetime or period index and 'timestamp' column is missing - if isinstance(energy_report_df.index, (pd.DatetimeIndex, pd.PeriodIndex)): - if "timestamp" not in energy_report_df.columns: - energy_report_df = energy_report_df.reset_index(names="timestamp") - - # Add Energy flow columns - energy_report_df = add_energy_flows( - energy_report_df, - production_cols=["pv", "chp", "wind"], - consumption_cols=["consumption"], - grid_cols=["grid"], - battery_cols=["battery"], - ) - - # Standardize column names (from raw to canonical) - energy_report_df = mapper.to_canonical(energy_report_df) - - # Convert timestamp to datetime if not already - energy_report_df["timestamp"] = pd.to_datetime( - energy_report_df["timestamp"], errors="coerce", utc=True - ) - - # Convert timezone - energy_report_df["timestamp"] = convert_timezone( - energy_report_df["timestamp"], - target_tz=tz_name, - assume_tz=assume_tz, - ) - - # Helper to rename numeric component IDs to labeled names like PV #250, Battery #219 - # (casing matches output format) - energy_report_df, single_components = label_component_columns( - energy_report_df, + return build_energy_report( + df, + component_types, mcfg, - column_battery="battery", - column_pv="pv", - column_chp="chp", - column_ev="ev", - column_wind="wind", - ) - - # Determine relevant columns based on component types - energy_report_df_cols = get_energy_report_columns( - component_types, single_components - ) - - # Select only the relevant columns - energy_report_df = energy_report_df[energy_report_df_cols] - - if fill_missing_values: - # Fill in missing aggregate component columns from per-component sums - energy_report_df = fill_aggregated_component_columns( - energy_report_df, - component_types, - aggregated_component_config, - ) - - return energy_report_df + mapper, + tz_name=tz_name, + assume_tz=assume_tz, + fill_missing_values=fill_missing_values, + aggregated_component_config=aggregated_component_config, + component_display_names=component_display_names, + include_component_metadata=False, + ).df diff --git a/src/frequenz/lib/notebooks/reporting/utils/component_metadata.py b/src/frequenz/lib/notebooks/reporting/utils/component_metadata.py new file mode 100644 index 00000000..45d39727 --- /dev/null +++ b/src/frequenz/lib/notebooks/reporting/utils/component_metadata.py @@ -0,0 +1,26 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Typed metadata for component-oriented reporting outputs.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import pandas as pd + + +@dataclass(frozen=True) +class ComponentMetadata: + """Dataset-level metadata for component display and grouping.""" + + display_names: dict[str, str] = field(default_factory=dict) + ids_by_type: dict[str, list[str]] = field(default_factory=dict) + + +@dataclass(frozen=True) +class EnergyReport: + """Normalized energy report dataframe plus resolved component metadata.""" + + df: pd.DataFrame + metadata: ComponentMetadata = field(default_factory=ComponentMetadata) diff --git a/src/frequenz/lib/notebooks/reporting/utils/helpers.py b/src/frequenz/lib/notebooks/reporting/utils/helpers.py index 70a4d8a5..8b234e40 100644 --- a/src/frequenz/lib/notebooks/reporting/utils/helpers.py +++ b/src/frequenz/lib/notebooks/reporting/utils/helpers.py @@ -36,15 +36,20 @@ from __future__ import annotations +import asyncio +import os import warnings +from concurrent.futures import ThreadPoolExecutor from datetime import date, datetime, time -from typing import Any, Literal, Mapping, cast +from typing import Any, Callable, Iterable, Literal, Mapping, cast from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import matplotlib.colors as mcolors import pandas as pd import plotly.express as px import yaml +from frequenz.client.assets import AssetsApiClient +from frequenz.client.common.microgrid import MicrogridId from frequenz.gridpool import MicrogridConfig from frequenz.lib.notebooks.reporting.metrics.reporting_metrics import ( @@ -61,6 +66,7 @@ from frequenz.lib.notebooks.reporting.utils.colors import COLOR_DICT AggregatedComponentConfig = Mapping[str, tuple[str, str]] +ComponentColumnsByType = Mapping[str, list[str]] DEFAULT_AGGREGATED_COMPONENT_CONFIG: AggregatedComponentConfig = { "battery": ("battery_power_flow", "Battery #"), @@ -70,6 +76,185 @@ } +def _extract_component_id(component_id: Any) -> str | None: + """Extract the numeric part from a component identifier.""" + digits = "".join(ch for ch in str(component_id) if ch.isdigit()) + return digits or None + + +def _configured_component_ids(mcfg: MicrogridConfig, component_type: str) -> set[str]: + """Return component IDs from the detailed component-type config when available.""" + component_configs = getattr(mcfg, "ctype", None) + if not isinstance(component_configs, dict): + return set() + + component_config = component_configs.get(component_type) + if component_config is None: + return set() + + resolved_ids: set[str] = set() + + for attr_name in ("meter", "inverter", "component"): + attr_value = getattr(component_config, attr_name, None) + if isinstance(attr_value, Iterable) and not isinstance( + attr_value, (str, bytes) + ): + resolved_ids.update(map(str, attr_value)) + + return resolved_ids + + +def _has_component_type(mcfg: MicrogridConfig, component_type: str) -> bool: + """Return whether the config exposes the requested component type.""" + component_types_method = getattr(mcfg, "component_types", None) + if callable(component_types_method): + return component_type in set(component_types_method()) + + component_configs = getattr(mcfg, "ctype", None) + return isinstance(component_configs, dict) and component_type in component_configs + + +def _fallback_component_ids(mcfg: MicrogridConfig, component_type: str) -> list[str]: + """Return component IDs from the simpler component_type_ids interface.""" + return sorted( + str(component_id) for component_id in mcfg.component_type_ids(component_type) + ) + + +def get_component_ids(mcfg: MicrogridConfig, component_type: str) -> list[str]: + """Return the configured component IDs for a given component type.""" + if not _has_component_type(mcfg, component_type): + return [] + + configured_ids = _configured_component_ids(mcfg, component_type) + if configured_ids: + return sorted(configured_ids) + + try: + return _fallback_component_ids(mcfg, component_type) + except ValueError as exc: + if str(exc) == "No IDs available": + print( + f"Component IDs are not available for component type '{component_type}'. " + "Skipping component-level metadata for this type." + ) + return [] + raise + + +def _resolved_assets_api_config( + server_url: str | None, + auth_key: str | None, + sign_secret: str | None, +) -> tuple[str, str | None, str | None]: + """Resolve the Assets API connection settings from args and environment.""" + resolved_server_url = server_url or os.getenv("ASSETS_API_URL") + resolved_auth_key = auth_key or os.getenv("API_KEY") + resolved_sign_secret = sign_secret or os.getenv("API_SECRET") + + if not resolved_server_url: + raise ValueError( + "Assets API URL not configured. Set ASSETS_API_URL or pass server_url." + ) + + return resolved_server_url, resolved_auth_key, resolved_sign_secret + + +def _is_meter_component(component: Any) -> bool: + """Return whether an electrical component should be treated as a meter.""" + category_name = getattr(getattr(component, "category", None), "name", None) + return category_name == "METER" or component.__class__.__name__ == "Meter" + + +def _component_display_name(component: Any) -> tuple[str, str] | None: + """Extract a component-id/display-name pair from an asset component.""" + component_key = _extract_component_id(getattr(component, "id", None)) + component_name = getattr(component, "name", None) + if component_key and component_name: + return component_key, str(component_name) + return None + + +async def _fetch_meter_display_names( + microgrid_id: int, + *, + server_url: str | None = None, + auth_key: str | None = None, + sign_secret: str | None = None, +) -> dict[str, str]: + """Fetch meter display names asynchronously from the Assets API.""" + ( + resolved_server_url, + resolved_auth_key, + resolved_sign_secret, + ) = _resolved_assets_api_config(server_url, auth_key, sign_secret) + + client = AssetsApiClient( + server_url=resolved_server_url, + auth_key=resolved_auth_key, + sign_secret=resolved_sign_secret, + ) + components = await client.list_microgrid_electrical_components( + MicrogridId(microgrid_id) + ) + + meter_names: dict[str, str] = {} + for component in components: + if not _is_meter_component(component): + continue + + component_pair = _component_display_name(component) + if component_pair is not None: + component_id, component_name = component_pair + meter_names[component_id] = component_name + + return meter_names + + +def _run_coro_in_thread(result_factory: Callable[[], dict[str, str]]) -> dict[str, str]: + """Run a coroutine-producing callable in a dedicated thread.""" + with ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(result_factory).result() + + +def get_meter_display_names( + microgrid_id: int, + *, + server_url: str | None = None, + auth_key: str | None = None, + sign_secret: str | None = None, +) -> dict[str, str]: + """Fetch meter display names for a microgrid from the Assets API. + + Args: + microgrid_id: The ID of the microgrid for which to fetch meter names. + server_url: Optional server URL for the Assets API. + auth_key: Optional authentication key for the Assets API. + sign_secret: Optional signing secret for the Assets API. + + Returns: + A dictionary mapping component IDs to their display names. + """ + + def run_fetch() -> dict[str, str]: + return asyncio.run( + _fetch_meter_display_names( + microgrid_id, + server_url=server_url, + auth_key=auth_key, + sign_secret=sign_secret, + ) + ) + + try: + asyncio.get_running_loop() + except RuntimeError: + return run_fetch() + + # Jupyter/IPython already runs an event loop. + return _run_coro_in_thread(run_fetch) + + def _get_numeric_series(df: pd.DataFrame, col: str | None) -> pd.Series: """Safely extract a numeric Series or return zeros if missing. @@ -212,6 +397,7 @@ def label_component_columns( column_chp: str = "chp", column_ev: str = "ev", column_wind: str = "wind", + component_display_names: Mapping[str, str] | None = None, ) -> tuple[pd.DataFrame, list[str]]: """Rename numeric single-component columns to labeled names. @@ -228,52 +414,43 @@ def label_component_columns( column_chp: Key name for CHP component type. column_ev: Key name for EV component type. column_wind: Key name for wind component type. + component_display_names: Optional mapping from numeric component IDs to + human-readable display names fetched from the Assets API. Returns: Tuple containing the renamed DataFrame and the list of applied labels """ - # Numeric component columns present in df - single_components = [str(c) for c in df.columns if str(c).isdigit()] - available_types = set(mcfg.component_types()) - - # From config (empty set if missing) - def ids_if_available(t: str) -> set[str]: - return ( - {str(x) for x in mcfg.component_type_ids(t)} - if t in available_types - else set() - ) - battery_ids = ids_if_available(column_battery) - pv_ids = ids_if_available(column_pv) - chp_ids = ids_if_available(column_chp) - ev_ids = ids_if_available(column_ev) - wind_ids = ids_if_available(column_wind) + def display_label(prefix: str, component_id: str) -> str: + component_name = display_names.get(component_id) + suffix = f" {component_name}" if component_name else "" + return f"{prefix} #{component_id}{suffix}" + + display_names = component_display_names or {} + + component_columns = {str(column) for column in df.columns if str(column).isdigit()} + + component_types = { + column_battery: column_battery.capitalize(), + column_pv: column_pv.upper(), + column_ev: column_ev.upper(), + column_chp: column_chp.upper(), + column_wind: column_wind.capitalize(), + } rename: dict[str, str] = {} - rename.update( - { - c: f"{column_battery.capitalize()} #{c}" - for c in single_components - if c in battery_ids - } - ) - rename.update( - {c: f"{column_pv.upper()} #{c}" for c in single_components if c in pv_ids} - ) - rename.update( - {c: f"{column_ev.upper()} #{c}" for c in single_components if c in ev_ids} - ) - rename.update( - {c: f"{column_chp.upper()} #{c}" for c in single_components if c in chp_ids} - ) - rename.update( - { - c: f"{column_wind.capitalize()} #{c}" - for c in single_components - if c in wind_ids - } - ) + + for component_type, label_prefix in component_types.items(): + matching_ids = sorted( + component_columns & set(get_component_ids(mcfg, component_type)) + ) + + rename.update( + { + component_id: display_label(label_prefix, component_id) + for component_id in matching_ids + } + ) return df.rename(columns=rename), list(rename.values()) @@ -704,6 +881,7 @@ def fill_aggregated_component_columns( df: pd.DataFrame, component_types: list[str], config: AggregatedComponentConfig | None = None, + component_columns_by_type: ComponentColumnsByType | None = None, ) -> pd.DataFrame: """Populate missing aggregate columns by summing labeled component columns. @@ -717,6 +895,9 @@ def fill_aggregated_component_columns( config: Mapping of component types to tuples containing the aggregated column name and the prefix used to identify individual component columns. + component_columns_by_type: Optional explicit mapping from component types + to canonical per-component columns. When provided, these columns are + used instead of matching legacy display prefixes. Returns: DataFrame with missing aggregated component columns filled in by summing @@ -729,7 +910,17 @@ def fill_aggregated_component_columns( if comp_type not in normalized_types or agg_col not in df.columns: continue - component_cols = [col for col in df.columns if col.startswith(prefix)] + explicit_component_cols = ( + component_columns_by_type.get(comp_type, []) + if component_columns_by_type is not None + else None + ) + legacy_component_cols = [col for col in df.columns if col.startswith(prefix)] + component_cols = ( + explicit_component_cols + if explicit_component_cols + else legacy_component_cols + ) # Only proceed if there are components to sum and missing values to fill if component_cols and df[agg_col].isna().any(): diff --git a/src/frequenz/lib/notebooks/reporting/utils/reporting_nb_functions.py b/src/frequenz/lib/notebooks/reporting/utils/reporting_nb_functions.py index 69f40122..5323d0b8 100644 --- a/src/frequenz/lib/notebooks/reporting/utils/reporting_nb_functions.py +++ b/src/frequenz/lib/notebooks/reporting/utils/reporting_nb_functions.py @@ -48,18 +48,198 @@ """ from datetime import datetime, timedelta -from typing import Iterable, Union, cast +from typing import Any, Iterable, Literal, Mapping, Union, cast import pandas as pd from .column_mapper import ColumnMapper +from .component_metadata import ComponentMetadata +from .helpers import get_component_ids +def _component_analysis_display_names( + component_metadata: ComponentMetadata | None, + component_display_names: Mapping[str, str] | None, +) -> Mapping[str, str]: + """Resolve component display names from args or explicit metadata.""" + if component_display_names is not None: + return component_display_names + + if component_metadata is not None: + return component_metadata.display_names + + return {} + + +def _component_analysis_ids( + component_metadata: ComponentMetadata | None, + component_label: str, + component_ids: set[str] | None, +) -> set[str]: + """Resolve canonical component IDs from args or explicit metadata.""" + if component_ids is not None: + return component_ids + + if component_metadata is not None: + return set(component_metadata.ids_by_type.get(component_label.lower(), [])) + + return set() + + +def _extract_component_id(component_label: str, column_name: str) -> str | None: + """Extract the component ID from either canonical or legacy component columns.""" + if column_name.isdigit(): + return column_name + + prefix_with_hash = f"{component_label} #" + if not column_name.startswith(prefix_with_hash): + return None + + remainder = column_name[len(prefix_with_hash) :] + component_id = remainder.split(" ", maxsplit=1)[0] + return component_id if component_id.isdigit() else None + + +def _format_component_value( + component_label: str, + column_name: str, + display_names: Mapping[str, str], +) -> str: + """Convert a component column name into the final display value.""" + component_id = _extract_component_id(component_label, column_name) + if component_id is None: + return column_name + + if column_name.isdigit(): + return display_names.get(component_id, f"#{component_id}") + + remainder = column_name.split(f"#{component_id}", maxsplit=1)[1].strip() + return remainder or f"#{component_id}" + + +def _normalize_selected_component( + selection: str, + display_names: Mapping[str, str], +) -> str | None: + """Normalize a component selector to its canonical numeric component ID.""" + normalized = str(selection).strip() + if normalized.startswith("#") and normalized[1:].isdigit(): + return normalized[1:] + if normalized.isdigit(): + return normalized + + for component_id, component_name in display_names.items(): + if component_name == normalized: + return component_id + + return None + + +def _select_all_component_columns( + energy_report_df: pd.DataFrame, + component_label: str, + raw_component_ids: set[str], +) -> list[str]: + """Select all component columns for a label, preferring canonical IDs.""" + prefix = f"{component_label} #" + raw_columns = [ + component_id + for component_id in sorted(raw_component_ids) + if component_id in energy_report_df.columns + ] + legacy_columns = [col for col in energy_report_df.columns if col.startswith(prefix)] + return raw_columns or legacy_columns + + +def _select_filtered_component_columns( + energy_report_df: pd.DataFrame, + selection_filter: Iterable[str], + component_label: str, + display_names: Mapping[str, str], +) -> list[str]: + """Select component columns matching the provided filter values.""" + comp_columns: list[str] = [] + seen_columns: set[str] = set() + + for selected_component in selection_filter: + normalized_selection = str(selected_component).strip() + selected_component_id = _normalize_selected_component( + normalized_selection, display_names + ) + if ( + selected_component_id is not None + and selected_component_id in energy_report_df.columns + ): + if selected_component_id not in seen_columns: + comp_columns.append(selected_component_id) + seen_columns.add(selected_component_id) + continue + + if normalized_selection.startswith(f"{component_label} #"): + selected_prefix = normalized_selection + else: + selected_prefix = f"{component_label} {normalized_selection}" + for col in energy_report_df.columns: + if col in seen_columns: + continue + if col == selected_prefix or col.startswith(f"{selected_prefix} "): + comp_columns.append(col) + seen_columns.add(col) + continue + + if _format_component_value(component_label, col, display_names) == ( + normalized_selection + ): + comp_columns.append(col) + seen_columns.add(col) + + return comp_columns + + +# pylint: disable=too-many-arguments, too-many-positional-arguments +def _selected_component_columns( + energy_report_df: pd.DataFrame, + selection_filter: Iterable[str], + component_label: str, + raw_component_ids: set[str], + display_names: Mapping[str, str], + allowed_component_ids: set[str] | None, +) -> list[str]: + """Resolve the component columns to include in the analysis dataframe.""" + if any(str(x).lower() == "all" for x in selection_filter): + comp_columns = _select_all_component_columns( + energy_report_df, + component_label, + raw_component_ids, + ) + else: + comp_columns = _select_filtered_component_columns( + energy_report_df, + selection_filter, + component_label, + display_names, + ) + + if allowed_component_ids is None: + return comp_columns + + return [ + col + for col in comp_columns + if _extract_component_id(component_label, col) in allowed_component_ids + ] + + +# pylint: disable=too-many-arguments, too-many-positional-arguments def build_component_analysis( energy_report_df: pd.DataFrame, selection_filter: Iterable[str], component_label: str, value_col_name: str, + allowed_component_ids: set[str] | None = None, + component_ids: set[str] | None = None, + component_display_names: Mapping[str, str] | None = None, + component_metadata: ComponentMetadata | None = None, ) -> pd.DataFrame: """Build a long-format analysis table for a single component type. @@ -82,6 +262,15 @@ def build_component_analysis( value_col_name: Name of the output column containing the selected component data (e.g., "battery", "chp", "ev"). + allowed_component_ids: + Optional set of numeric component IDs allowed for selection. When + provided, the selection is restricted to matching IDs only. + component_ids: + Optional set of numeric component IDs to include in the analysis. + component_display_names: + Optional mapping of component IDs to their display names. + component_metadata: + Optional metadata about the components, including their types and other attributes. Returns: pd.DataFrame: @@ -93,19 +282,23 @@ def build_component_analysis( If no matching columns are found, returns an empty DataFrame with the appropriate columns. """ - prefix = f"{component_label} #" - - # Select columns - if any(str(x).lower() == "all" for x in selection_filter): - comp_columns = [ - col for col in energy_report_df.columns if col.startswith(prefix) - ] - else: - comp_columns = [ - f"{component_label} {x}" - for x in selection_filter - if f"{component_label} {x}" in energy_report_df.columns - ] + raw_component_ids = _component_analysis_ids( + component_metadata, + component_label, + component_ids, + ) + display_names = _component_analysis_display_names( + component_metadata, + component_display_names, + ) + comp_columns = _selected_component_columns( + energy_report_df, + selection_filter, + component_label, + raw_component_ids, + display_names, + allowed_component_ids, + ) if not comp_columns: return pd.DataFrame(columns=["timestamp", component_label, value_col_name]) @@ -122,15 +315,65 @@ def build_component_analysis( value_name=value_col_name, ) - # Keep only the number after " " - analyse_df[component_label] = analyse_df[component_label].str.replace( - f"{component_label} ", "", regex=False + analyse_df[component_label] = analyse_df[component_label].map( + lambda column_name: _format_component_value( + component_label, + column_name, + display_names, + ) ) return analyse_df -# pylint: disable=too-many-arguments, too-many-positional-arguments +def resolve_component_analysis_ids( + mcfg: Any | None, + component_key: str, + component_id_source: Literal["meter", "inverter", "component", "all"] | None, +) -> set[str] | None: + """Resolve allowed component IDs for analysis from the microgrid config. + + Args: + mcfg: + Microgrid config exposing a ``ctype`` mapping with component type + configuration entries. + component_key: + Component type key such as ``"pv"`` or ``"battery"``. + component_id_source: + Which ID set to use. ``None`` and ``"all"`` disable filtering. + + Returns: + A set of string IDs when a specific source is requested, otherwise + ``None``. + """ + if mcfg is None or component_id_source in (None, "all"): + return None + + component_configs = getattr(mcfg, "ctype", None) + if not isinstance(component_configs, dict): + return None + + component_config = component_configs.get(component_key) + if component_config is None: + return None + + if component_id_source == "meter": + attr_name: Literal["meter", "inverter", "component"] = "meter" + elif component_id_source == "inverter": + attr_name = "inverter" + elif component_id_source == "component": + attr_name = "component" + else: + return None + + component_ids = getattr(component_config, attr_name, None) + if component_ids is None: + return set() + + return {str(component_id) for component_id in component_ids} + + +# pylint: disable=too-many-arguments, too-many-positional-arguments, too-many-locals def assemble_component_analysis( component_filter: list[str], component_key: str, @@ -142,6 +385,9 @@ def assemble_component_analysis( value_col_name: str, invert_sign: bool = False, trunc_values: bool = False, + mcfg: Any | None = None, + component_id_source: Literal["meter", "inverter", "component", "all"] | None = None, + component_metadata: ComponentMetadata | None = None, ) -> tuple[pd.DataFrame, float, str]: """Assemble a component-level analysis table and compute its energy total. @@ -180,6 +426,15 @@ def assemble_component_analysis( trunc_values: If ``True``, truncate values to zero (i.e., set negative values to zero) after scaling (dataframe values are kept unchanged). + mcfg: + Optional microgrid config used to resolve the allowed component IDs. + component_id_source: + Optional ID source selector. Use ``"inverter"`` to analyze inverter + IDs, ``"meter"`` for meter IDs, ``"component"`` for component IDs, + or ``"all"``/``None`` to keep the previous unrestricted behavior. + component_metadata: + Optional dataset-level component metadata containing display names and + canonical component IDs. Returns: - The long-form analysis DataFrame with energy values in kWh. @@ -205,12 +460,31 @@ def assemble_component_analysis( ): return pd.DataFrame(), 0, filter_text + allowed_component_ids = resolve_component_analysis_ids( + mcfg, + component_key, + component_id_source, + ) + component_ids = ( + set(get_component_ids(cast(Any, mcfg), component_key)) + if mcfg is not None + else None + ) + # Build Analysis analyse_df = build_component_analysis( energy_report_df, component_filter, component_label=component_label, value_col_name=value_col_name, + allowed_component_ids=allowed_component_ids, + component_ids=component_ids, + component_metadata=component_metadata, + ) + + analyse_df[value_col_name] = pd.to_numeric( + analyse_df[value_col_name], + errors="coerce", ) # Calculate timestep scaling according to resolution From be30e2088adc2c71cffed3cedfacb711f5b938a7 Mon Sep 17 00:00:00 2001 From: Mohammad Tayyab Date: Mon, 20 Jul 2026 20:18:02 +0200 Subject: [PATCH 2/3] tests: update the test cases Signed-off-by: Mohammad Tayyab --- tests/test_helpers.py | 119 +++++++--- tests/test_reporting_data_processing.py | 303 +++++++++++++++++++++--- tests/test_reporting_helpers.py | 54 +++++ tests/test_reporting_nb_functions.py | 274 +++++++++++++++++++++ 4 files changed, 691 insertions(+), 59 deletions(-) create mode 100644 tests/test_reporting_helpers.py diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 17e04393..bb7e25a1 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -5,6 +5,7 @@ from __future__ import annotations +import asyncio from dataclasses import dataclass from datetime import UTC, date, datetime from typing import cast @@ -12,7 +13,7 @@ import pandas as pd import pytest -from frequenz.gridpool import MicrogridConfig +from frequenz.client.common.microgrid import MicrogridId from pandas.testing import assert_frame_equal, assert_series_equal from frequenz.lib.notebooks.reporting.utils.helpers import ( @@ -23,7 +24,7 @@ convert_timezone, fill_aggregated_component_columns, fmt_to_de_system, - label_component_columns, + get_meter_display_names, long_to_wide, set_date_to_midnight, ) @@ -112,42 +113,62 @@ class _DummyMicrogridConfig: """Minimal config stub exposing component type helpers.""" mapping: dict[str, list[str]] + ctype: dict[str, object] | None = None def component_types(self) -> list[str]: + if self.ctype: + return list(self.ctype.keys()) return list(self.mapping.keys()) def component_type_ids(self, component_type: str) -> list[str]: return self.mapping.get(component_type, []) -def test_label_component_columns_applies_expected_prefixes() -> None: - """Numeric columns are renamed with component labels while others stay untouched.""" - df = pd.DataFrame( - { - "1": [10], - "2": [20], - "3": [30], - "4": [40], - "constant": [99], - } - ) - config = _DummyMicrogridConfig( - {"battery": ["1"], "pv": ["2"], "ev": ["3"], "chp": ["4"]} - ) - - renamed, labels = label_component_columns( - df, - cast(MicrogridConfig, config), - ) - - assert renamed.columns.tolist() == [ - "Battery #1", - "PV #2", - "EV #3", - "CHP #4", - "constant", - ] - assert labels == ["Battery #1", "PV #2", "EV #3", "CHP #4"] +@dataclass +class _DummyComponentConfig: + """Minimal component config stub with meter/inverter/component groups.""" + + meter: list[str] | None = None + inverter: list[str] | None = None + component: list[str] | None = None + + +def test_get_meter_display_names_runs_inside_active_event_loop() -> None: + """The sync helper should work even when an event loop is already active.""" + + class _Category: + def __init__(self, name: str) -> None: + self.name = name + + class _Component: + def __init__(self, component_id: str, name: str, category: str) -> None: + self.id = component_id + self.name = name + self.category = _Category(category) + + class _FakeClient: + async def list_microgrid_electrical_components( + self, microgrid_id: MicrogridId + ) -> list[_Component]: + assert microgrid_id == MicrogridId(241) + return [ + _Component("ElectricalComponentId(1179)", "meter_pq_0", "METER"), + _Component("ElectricalComponentId(1188)", "meter_GT", "METER"), + _Component("ElectricalComponentId(1189)", "GT_1", "CHP"), + ] + + async def _run_test() -> None: + result = get_meter_display_names( + 241, server_url="grpc://assets.example.com:443" + ) + assert result == {"1179": "meter_pq_0", "1188": "meter_GT"} + + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setattr( + "frequenz.lib.notebooks.reporting.utils.helpers.AssetsApiClient", + lambda server_url, auth_key=None, sign_secret=None: _FakeClient(), + ) + asyncio.run(_run_test()) def test_set_date_to_midnight_creates_timezone_aware_midnight() -> None: @@ -258,6 +279,44 @@ def test_fill_aggregated_component_columns_accepts_custom_config() -> None: assert result["custom_power"].tolist() == [3.0, 4.0] +def test_fill_aggregated_component_columns_accepts_explicit_component_columns() -> None: + """Explicit canonical component columns can drive aggregation fill-ins.""" + df = pd.DataFrame( + { + "battery_power_flow": [float("nan"), 4.0], + "1532": [1.0, 2.0], + "1533": [2.0, 3.0], + } + ) + + result = fill_aggregated_component_columns( + df.copy(), + component_types=["battery"], + component_columns_by_type={"battery": ["1532", "1533"]}, + ) + + assert result["battery_power_flow"].tolist() == [3.0, 4.0] + + +def test_fill_aggregated_component_columns_falls_back_to_legacy_prefixes() -> None: + """Legacy prefixed columns should still fill aggregates when explicit ids miss.""" + df = pd.DataFrame( + { + "battery_power_flow": [float("nan"), 4.0], + "Battery #1532": [1.0, 2.0], + "Battery #1533": [2.0, 3.0], + } + ) + + result = fill_aggregated_component_columns( + df.copy(), + component_types=["battery"], + component_columns_by_type={"battery": []}, + ) + + assert result["battery_power_flow"].tolist() == [3.0, 4.0] + + def test_long_to_wide_custom_sum_column_name() -> None: """Custom sum column name is respected when provided.""" df = pd.DataFrame( diff --git a/tests/test_reporting_data_processing.py b/tests/test_reporting_data_processing.py index f7496908..e694ceeb 100644 --- a/tests/test_reporting_data_processing.py +++ b/tests/test_reporting_data_processing.py @@ -5,50 +5,78 @@ from __future__ import annotations +from typing import cast + import pandas as pd import pytest +from frequenz.gridpool import MicrogridConfig from pandas.testing import assert_frame_equal -from frequenz.lib.notebooks.reporting.data_processing import create_battery_usecase_df +from frequenz.lib.notebooks.reporting.data_processing import ( + build_energy_report, + create_battery_usecase_df, + create_energy_report_df, +) +from frequenz.lib.notebooks.reporting.utils.column_mapper import ColumnMapper -def test_create_battery_usecase_df_builds_expected_columns() -> None: - """Battery usecase helper derives the expected canonical columns.""" - energy_report_df = pd.DataFrame( - { - "timestamp": pd.to_datetime(["2026-01-09 06:30:00", "2026-01-09 07:00:00"]), - "mid_consumption": [35.0, 21.0], - "grid_consumption": [30.0, 25.0], - "battery_power_flow": [5.0, -4.0], - } - ) - result = create_battery_usecase_df(energy_report_df) +class _DummyMeta: + """Minimal config metadata stub with only the microgrid id.""" - expected = pd.DataFrame( - { - "timestamp": pd.to_datetime(["2026-01-09 06:30:00", "2026-01-09 07:00:00"]), - "consumption": [35.0, 21.0], - "grid_consumption": [30.0, 25.0], - "battery_power_flow": [5.0, -4.0], - "peak_before_optimization": [35.0, 35.0], - "peak_after_optimization": [30.0, 30.0], - "battery_charge": [5.0, 0.0], - "battery_discharge": [0.0, -4.0], - } - ) + def __init__(self, microgrid_id: int) -> None: + self.microgrid_id = microgrid_id - assert_frame_equal(result, expected) +class _DummyMicrogridConfig: + """Minimal config stub exposing component type helpers and metadata.""" + + def __init__( + self, + mapping: dict[str, list[str]], + microgrid_id: int = 241, + ctype: dict[str, object] | None = None, + ) -> None: + self.mapping = mapping + self.meta = _DummyMeta(microgrid_id) + self.ctype = ctype -def test_create_battery_usecase_df_preserves_pv_when_available() -> None: - """Optional PV production is retained in the standardized output.""" + def component_types(self) -> list[str]: + if self.ctype: + return list(self.ctype.keys()) + return list(self.mapping.keys()) + + def component_type_ids(self, component_type: str) -> list[str]: + return self.mapping.get(component_type, []) + + +class _DummyComponentConfig: + """Minimal component config stub with meter/inverter/component groups.""" + + def __init__( + self, + *, + meter: list[str] | None = None, + inverter: list[str] | None = None, + component: list[str] | None = None, + ) -> None: + self.meter = meter + self.inverter = inverter + self.component = component + + +def _as_microgrid_config(mcfg: _DummyMicrogridConfig) -> MicrogridConfig: + """Cast a lightweight test double to the production config type.""" + return cast(MicrogridConfig, mcfg) + + +def test_create_battery_usecase_df_builds_expected_columns() -> None: + """Battery usecase helper derives the expected canonical columns.""" energy_report_df = pd.DataFrame( { "timestamp": pd.to_datetime(["2026-01-09 06:30:00", "2026-01-09 07:00:00"]), "mid_consumption": [35.0, 21.0], "grid_consumption": [30.0, 25.0], "battery_power_flow": [5.0, -4.0], - "pv_asset_production": [12.0, 10.0], } ) result = create_battery_usecase_df(energy_report_df) @@ -59,7 +87,6 @@ def test_create_battery_usecase_df_preserves_pv_when_available() -> None: "consumption": [35.0, 21.0], "grid_consumption": [30.0, 25.0], "battery_power_flow": [5.0, -4.0], - "pv": [12.0, 10.0], "peak_before_optimization": [35.0, 35.0], "peak_after_optimization": [30.0, 30.0], "battery_charge": [5.0, 0.0], @@ -119,3 +146,221 @@ def test_create_battery_usecase_df_requires_configured_input_columns() -> None: with pytest.raises(KeyError, match="battery_power_flow"): create_battery_usecase_df(energy_report_df) + + +def test_build_energy_report_includes_meter_names_when_available( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Meter names are stored as display metadata when fetched from assets.""" + raw_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "grid": [30.0], + "pv": [-12.0], + "1179": [-12.0], + } + ) + mapper = ColumnMapper.from_default(locale="en") + mcfg = _DummyMicrogridConfig({"pv": ["1179"]}, microgrid_id=241) + + monkeypatch.setattr( + "frequenz.lib.notebooks.reporting.data_processing.get_meter_display_names", + lambda microgrid_id: {"1179": "PV Roof Meter"}, + ) + + result = build_energy_report( + raw_df, + component_types=["pv"], + mcfg=_as_microgrid_config(mcfg), + mapper=mapper, + ) + + assert "1179" in result.df.columns + assert result.metadata.display_names == {"1179": "PV Roof Meter"} + assert result.metadata.ids_by_type == {"pv": ["1179"]} + + +def test_build_energy_report_accepts_explicit_component_display_names() -> None: + """Explicit display-name mappings should be stored as display metadata.""" + raw_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "grid": [30.0], + "battery": [5.0], + "1532": [5.0], + } + ) + mapper = ColumnMapper.from_default(locale="en") + mcfg = _DummyMicrogridConfig({"battery": ["1532"]}, microgrid_id=241) + + result = build_energy_report( + raw_df, + component_types=["battery"], + mcfg=_as_microgrid_config(mcfg), + mapper=mapper, + component_display_names={"1532": "Battery East"}, + ) + + assert "1532" in result.df.columns + assert result.metadata.display_names == {"1532": "Battery East"} + assert result.metadata.ids_by_type == {"battery": ["1532"]} + + +def test_build_energy_report_keeps_inverter_component_columns() -> None: + """Per-inverter PV columns should survive into the energy report dataframe.""" + raw_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "grid": [30.0], + "pv": [-12.0], + "1134": [-7.0], + "1135": [-5.0], + } + ) + mapper = ColumnMapper.from_default(locale="en") + mcfg = _DummyMicrogridConfig( + {"pv": []}, + microgrid_id=241, + ctype={ + "pv": _DummyComponentConfig(inverter=["1134", "1135"]), + }, + ) + + result = build_energy_report( + raw_df, + component_types=["pv"], + mcfg=_as_microgrid_config(mcfg), + mapper=mapper, + component_display_names={}, + ) + + assert "1134" in result.df.columns + assert "1135" in result.df.columns + assert result.metadata.ids_by_type == {"pv": ["1134", "1135"]} + assert "pv_asset_production" in result.df.columns + + +def test_create_energy_report_df_returns_dataframe_by_default() -> None: + """The public API should continue to return only the dataframe by default.""" + raw_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "grid": [30.0], + "battery": [5.0], + "1532": [5.0], + } + ) + mapper = ColumnMapper.from_default(locale="en") + mcfg = _DummyMicrogridConfig({"battery": ["1532"]}, microgrid_id=241) + + result = create_energy_report_df( + raw_df, + component_types=["battery"], + mcfg=_as_microgrid_config(mcfg), + mapper=mapper, + ) + + assert isinstance(result, pd.DataFrame) + assert "1532" in result.columns + + +def test_create_energy_report_df_skips_meter_lookup_without_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Default dataframe-only calls should not fetch display names.""" + raw_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "grid": [30.0], + "battery": [5.0], + "1532": [5.0], + } + ) + mapper = ColumnMapper.from_default(locale="en") + mcfg = _DummyMicrogridConfig({"battery": ["1532"]}, microgrid_id=241) + + def fail_lookup(_: int) -> dict[str, str]: + raise AssertionError("meter lookup should not run") + + monkeypatch.setattr( + "frequenz.lib.notebooks.reporting.data_processing.get_meter_display_names", + fail_lookup, + ) + + result = create_energy_report_df( + raw_df, + component_types=["battery"], + mcfg=_as_microgrid_config(mcfg), + mapper=mapper, + ) + + assert isinstance(result, pd.DataFrame) + assert "1532" in result.columns + + +def test_create_energy_report_df_skips_component_types_without_ids() -> None: + """Misconfigured component types should not fail dataframe creation.""" + raw_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "grid": [30.0], + "pv": [-12.0], + "battery": [5.0], + "1532": [5.0], + } + ) + mapper = ColumnMapper.from_default(locale="en") + mcfg = _DummyMicrogridConfig( + { + "grid": [], + "pv": [], + "battery": ["1532"], + }, + microgrid_id=241, + ) + + result = create_energy_report_df( + raw_df, + component_types=["grid", "pv", "battery"], + mcfg=_as_microgrid_config(mcfg), + mapper=mapper, + ) + + assert isinstance(result, pd.DataFrame) + assert "1532" in result.columns + assert "timestamp" in result.columns + + +def test_build_energy_report_can_skip_meter_lookup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Metadata lookups can be disabled while keeping the stable return shape.""" + raw_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "grid": [30.0], + "battery": [5.0], + "1532": [5.0], + } + ) + mapper = ColumnMapper.from_default(locale="en") + mcfg = _DummyMicrogridConfig({"battery": ["1532"]}, microgrid_id=241) + + def fail_lookup(_: int) -> dict[str, str]: + raise AssertionError("meter lookup should not run") + + monkeypatch.setattr( + "frequenz.lib.notebooks.reporting.data_processing.get_meter_display_names", + fail_lookup, + ) + + result = build_energy_report( + raw_df, + component_types=["battery"], + mcfg=_as_microgrid_config(mcfg), + mapper=mapper, + include_component_metadata=False, + ) + + assert isinstance(result.df, pd.DataFrame) + assert result.metadata.ids_by_type == {"battery": ["1532"]} diff --git a/tests/test_reporting_helpers.py b/tests/test_reporting_helpers.py new file mode 100644 index 00000000..1fd08314 --- /dev/null +++ b/tests/test_reporting_helpers.py @@ -0,0 +1,54 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Tests for reporting helpers.""" + +from __future__ import annotations + +from typing import cast + +from frequenz.gridpool import MicrogridConfig + +from frequenz.lib.notebooks.reporting.utils.helpers import get_component_ids + + +class _DummyMicrogridConfig: + """Minimal config stub exposing component type helpers.""" + + def __init__(self, mapping: dict[str, list[str] | Exception]) -> None: + self.mapping = mapping + + def component_types(self) -> list[str]: + return list(self.mapping.keys()) + + def component_type_ids(self, component_type: str) -> list[str]: + result = self.mapping[component_type] + if isinstance(result, Exception): + raise result + return result + + +def _as_microgrid_config(mcfg: _DummyMicrogridConfig) -> MicrogridConfig: + """Cast a lightweight test double to the production config type.""" + return cast(MicrogridConfig, mcfg) + + +def test_get_component_ids_skips_types_without_configured_ids() -> None: + """Missing ids in gridpool config should be treated as no components.""" + mcfg = _DummyMicrogridConfig({"grid": ValueError("No IDs available")}) + + result = get_component_ids(_as_microgrid_config(mcfg), "grid") + + assert result == [] + + +def test_get_component_ids_reraises_unrelated_value_errors() -> None: + """Only the known missing-id sentinel should be swallowed.""" + mcfg = _DummyMicrogridConfig({"grid": ValueError("unexpected config issue")}) + + try: + get_component_ids(_as_microgrid_config(mcfg), "grid") + except ValueError as err: + assert str(err) == "unexpected config issue" + else: # pragma: no cover - defensive assertion + raise AssertionError("Expected ValueError to be reraised") diff --git a/tests/test_reporting_nb_functions.py b/tests/test_reporting_nb_functions.py index cffd8a99..7dd046ce 100644 --- a/tests/test_reporting_nb_functions.py +++ b/tests/test_reporting_nb_functions.py @@ -12,15 +12,41 @@ from pandas.testing import assert_frame_equal from frequenz.lib.notebooks.reporting.utils.column_mapper import ColumnMapper +from frequenz.lib.notebooks.reporting.utils.component_metadata import ( + ComponentMetadata, +) from frequenz.lib.notebooks.reporting.utils.reporting_nb_functions import ( aggregate_metrics, assemble_component_analysis, build_component_analysis, build_overview_df, compute_energy_summary, + resolve_component_analysis_ids, ) +class _DummyComponentConfig: + """Minimal component config stub with id groups.""" + + def __init__( + self, + *, + meter: list[int] | None = None, + inverter: list[int] | None = None, + component: list[int] | None = None, + ) -> None: + self.meter = meter + self.inverter = inverter + self.component = component + + +class _DummyMicrogridConfig: + """Minimal microgrid config stub exposing the ctype mapping.""" + + def __init__(self, ctype: dict[str, _DummyComponentConfig]) -> None: + self.ctype = ctype + + def test_build_component_analysis_selects_all_components_and_melts() -> None: """All matching component columns are reshaped into a long-format table.""" energy_report_df = pd.DataFrame( @@ -57,6 +83,107 @@ def test_build_component_analysis_selects_all_components_and_melts() -> None: assert_frame_equal(result, expected) +def test_build_component_analysis_uses_raw_component_ids_with_display_metadata() -> ( + None +): + """Raw component ID columns should render meter names from dataframe metadata.""" + energy_report_df = pd.DataFrame( + { + "timestamp": pd.to_datetime( + ["2026-01-09 06:30:00", "2026-01-09 07:00:00"], utc=True + ), + "1179": [1.2, 1.5], + "1188": [0.7, 0.8], + } + ) + component_metadata = ComponentMetadata( + display_names={ + "1179": "PV Roof Meter", + "1188": "PV Yard Meter", + }, + ids_by_type={"pv": ["1179", "1188"]}, + ) + + result = build_component_analysis( + energy_report_df, + selection_filter=["All"], + component_label="PV", + value_col_name="pv_asset_production", + component_metadata=component_metadata, + ) + + assert result["PV"].tolist() == [ + "PV Roof Meter", + "PV Roof Meter", + "PV Yard Meter", + "PV Yard Meter", + ] + assert result["pv_asset_production"].tolist() == [1.2, 1.5, 0.7, 0.8] + + +def test_build_component_analysis_keeps_id_when_no_display_name_is_present() -> None: + """Legacy `#id` labels should still work when no meter name is available.""" + energy_report_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "Battery #1532": [5.0], + } + ) + + result = build_component_analysis( + energy_report_df, + selection_filter=["#1532"], + component_label="Battery", + value_col_name="battery_power_flow", + ) + + assert result["Battery"].tolist() == ["#1532"] + + +def test_build_component_analysis_can_restrict_to_allowed_component_ids() -> None: + """Allowed ids should constrain the selected columns for `All` filters.""" + energy_report_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "PV #1133 PV Meter": [2.0], + "PV #1134 PV Inverter": [1.0], + } + ) + + result = build_component_analysis( + energy_report_df, + selection_filter=["All"], + component_label="PV", + value_col_name="pv_asset_production", + allowed_component_ids={"1134"}, + ) + + assert result["PV"].tolist() == ["PV Inverter"] + assert result["pv_asset_production"].tolist() == [1.0] + + +def test_build_component_analysis_can_reselect_legacy_component_by_display_value() -> ( + None +): + """Legacy labels returned by the formatter should round-trip through selection.""" + energy_report_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "PV #1134 PV Inverter": [1.0], + } + ) + + result = build_component_analysis( + energy_report_df, + selection_filter=["PV Inverter"], + component_label="PV", + value_col_name="pv_asset_production", + ) + + assert result["PV"].tolist() == ["PV Inverter"] + assert result["pv_asset_production"].tolist() == [1.0] + + def test_build_component_analysis_returns_empty_when_columns_missing() -> None: """A missing component selection should return a typed empty frame.""" energy_report_df = pd.DataFrame( @@ -76,6 +203,29 @@ def test_build_component_analysis_returns_empty_when_columns_missing() -> None: ) +def test_resolve_component_analysis_ids_supports_meter_and_inverter() -> None: + """The helper should expose the requested id group from the config.""" + mcfg = _DummyMicrogridConfig( + { + "pv": _DummyComponentConfig(meter=[1133, 1141], inverter=[1134, 1135]), + } + ) + + assert resolve_component_analysis_ids(mcfg, "pv", "meter") == {"1133", "1141"} + assert resolve_component_analysis_ids(mcfg, "pv", "inverter") == { + "1134", + "1135", + } + assert resolve_component_analysis_ids(mcfg, "pv", "all") is None + + +def test_resolve_component_analysis_ids_returns_empty_set_for_missing_group() -> None: + """A missing id group should produce an empty allowed-id set.""" + mcfg = _DummyMicrogridConfig({"pv": _DummyComponentConfig(meter=[1133])}) + + assert resolve_component_analysis_ids(mcfg, "pv", "component") == set() + + def test_assemble_component_analysis_scales_and_truncates_component_sum() -> None: """Scaled values are returned and the optional truncated sum is respected.""" mapper = ColumnMapper.from_default(locale="en") @@ -120,6 +270,130 @@ def test_assemble_component_analysis_scales_and_truncates_component_sum() -> Non assert filter_text == "All" +def test_assemble_component_analysis_coerces_object_values_to_numeric() -> None: + """Object-typed component values should still be scaled and rounded.""" + analyse_df, component_sum, filter_text = assemble_component_analysis( + component_filter=["#1179"], + component_key="pv", + component_types=["pv"], + energy_report_df=pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "PV #1179 PV Roof Meter": ["-1.2345"], + } + ), + timestep_hours=1.0, + mapper=ColumnMapper.from_default(locale="en"), + component_label="PV", + value_col_name="pv_asset_production", + invert_sign=True, + trunc_values=True, + ) + + assert analyse_df["PV"].tolist() == ["PV Roof Meter"] + assert analyse_df["PV-Production"].tolist() == [1.234] + assert component_sum == 1.234 + assert filter_text == "#1179" + + +def test_assemble_component_analysis_can_pick_inverter_ids() -> None: + """Analysis should select inverter-labelled columns when requested.""" + energy_report_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "PV #1133 PV Meter": [-2.0], + "PV #1134 PV Inverter": [-1.0], + } + ) + mcfg = _DummyMicrogridConfig( + { + "pv": _DummyComponentConfig(meter=[1133], inverter=[1134]), + } + ) + + analyse_df, component_sum, _ = assemble_component_analysis( + component_filter=["All"], + component_key="pv", + component_types=["pv"], + energy_report_df=energy_report_df, + timestep_hours=1.0, + mapper=ColumnMapper.from_default(locale="en"), + component_label="PV", + value_col_name="pv_asset_production", + invert_sign=True, + trunc_values=True, + mcfg=mcfg, + component_id_source="inverter", + ) + + assert analyse_df["PV"].tolist() == ["PV Inverter"] + assert analyse_df["PV-Production"].tolist() == [1.0] + assert component_sum == 1.0 + + +def test_assemble_component_analysis_can_pick_meter_ids() -> None: + """Analysis should select meter-labelled columns when requested.""" + energy_report_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "Battery #1532 Battery Meter": [5.0], + "Battery #1533 Battery Inverter": [4.0], + } + ) + mcfg = _DummyMicrogridConfig( + { + "battery": _DummyComponentConfig(meter=[1532], inverter=[1533]), + } + ) + + analyse_df, component_sum, _ = assemble_component_analysis( + component_filter=["All"], + component_key="battery", + component_types=["battery"], + energy_report_df=energy_report_df, + timestep_hours=1.0, + mapper=ColumnMapper.from_default(locale="en"), + component_label="Battery", + value_col_name="battery_power_flow", + mcfg=mcfg, + component_id_source="meter", + ) + + assert analyse_df["Battery"].tolist() == ["Battery Meter"] + assert analyse_df["Battery Power Flow"].tolist() == [5.0] + assert component_sum == 5.0 + + +def test_assemble_component_analysis_returns_empty_when_id_source_has_no_matches() -> ( + None +): + """Selecting an id source with no matching columns should return empty output.""" + energy_report_df = pd.DataFrame( + { + "timestamp": pd.to_datetime(["2026-01-09 06:30:00"], utc=True), + "PV #1134 PV Inverter": [-1.0], + } + ) + mcfg = _DummyMicrogridConfig({"pv": _DummyComponentConfig(meter=[1133])}) + + analyse_df, component_sum, filter_text = assemble_component_analysis( + component_filter=["All"], + component_key="pv", + component_types=["pv"], + energy_report_df=energy_report_df, + timestep_hours=1.0, + mapper=ColumnMapper.from_default(locale="en"), + component_label="PV", + value_col_name="pv_asset_production", + mcfg=mcfg, + component_id_source="meter", + ) + + assert analyse_df.empty + assert component_sum == 0 + assert filter_text == "All" + + def test_build_overview_df_keeps_expected_optional_columns() -> None: """Overview output should preserve order and ignore unavailable columns.""" energy_report_df = pd.DataFrame( From 2c95d82e04453dd9f6891b8f44ee908bc19368a9 Mon Sep 17 00:00:00 2001 From: Mohammad Tayyab Date: Mon, 20 Jul 2026 20:20:17 +0200 Subject: [PATCH 3/3] docs: update release notes Signed-off-by: Mohammad Tayyab --- RELEASE_NOTES.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 18039c63..8f461baa 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -16,6 +16,14 @@ - Add day ahead prices fetching. - `aggregate_metrics` now exposes grid import cost and grid feed-in revenue totals when price data is present in the reporting dataframe. +- Reporting notebooks can now build component-level analysis for configured + meters, inverters, and other component groups selected from the microgrid + configuration. +- Added `build_energy_report`, `ComponentMetadata`, and `EnergyReport` to + expose canonical component IDs and optional meter display names from the + Assets API to downstream reporting workflows. +- `create_energy_report_df` and downstream component analysis now preserve + canonical component IDs, making notebook selections and plots easier to read. - Add `plot_power` to the asset optimization plotly visualizations, stacking each component in the passive sign convention so the top of the stack meets the grid line. Unlike `plot_power_flow`, production is not clipped. - `init_microgrid_data` accepts several dotenv files, so shared API URLs can live apart from per-microgrid credentials. - `plot_time_series_battery_usecase` now supports selectable stack modes for