From d06ce16e462d0fbfb330f34d45fa26dd52af2887 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 15:45:35 -0700 Subject: [PATCH 1/8] feat(python-sdk): add LLMGeneratorProtocol for framework-agnostic model injection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces three types to sdks/python: - LLMGeneratorProtocol (typing.Protocol) — structural interface for injecting any LLM backend into evaluators without a hard framework dependency - LLMResponse (NamedTuple) — structured response aligned with OTel GenAI semconv (content, model, input_tokens, output_tokens) - GenerateConfig (dataclass) — temperature and max_tokens passthrough Refactors BaseEvaluator.execute_prompt_chain_step to accept an optional llm_provider: LLMGeneratorProtocol. When set, the protocol path formats the LangChain template to extract system/human strings, delegates the LLM call to the injected provider, and parses the JSON response via Pydantic directly. The existing LangChain path is unchanged and remains the default. Also improves _strip_json_fences to use JSONDecoder.raw_decode, correctly handling trailing prose and multiple JSON objects in LLM responses. Also adds *.egg-info/, dist/, build/, logs/ to root .gitignore. --- .gitignore | 9 ++ sdks/python/pyproject.toml | 2 +- .../learning_commons_evaluators/__init__.py | 8 + .../evaluators/base.py | 139 ++++++++++++++++++ .../schemas/__init__.py | 8 + .../schemas/llm_provider.py | 121 +++++++++++++++ .../python/tests/schemas/test_llm_provider.py | 106 +++++++++++++ 7 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py create mode 100644 sdks/python/tests/schemas/test_llm_provider.py diff --git a/.gitignore b/.gitignore index fb6507ee..1ee9c3bd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,15 @@ .venv/ __pycache__/ *.pyc +*.egg-info/ +dist/ +build/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ + +# Inspect AI eval logs +logs/ # Jupyter Notebook .ipynb_checkpoints/ diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index aea23087..b96dc345 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "learning-commons-evaluators" -version = "0.2.0" +version = "0.3.0" description = "Python SDK for Learning Commons educational evaluators" readme = "README.md" license = { text = "MIT" } diff --git a/sdks/python/src/learning_commons_evaluators/__init__.py b/sdks/python/src/learning_commons_evaluators/__init__.py index 6169df43..2ec1b989 100644 --- a/sdks/python/src/learning_commons_evaluators/__init__.py +++ b/sdks/python/src/learning_commons_evaluators/__init__.py @@ -59,6 +59,11 @@ TextInputField, ) from learning_commons_evaluators.schemas.config import EvaluationSettings, LLMProvider +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) from learning_commons_evaluators.schemas.conventionality import ( ConventionalityEvaluationSettings, ConventionalityOutput, @@ -163,6 +168,9 @@ "create_config_telemetry_with_full_input", "create_logger", "create_silent_logger", + "GenerateConfig", + "LLMGeneratorProtocol", + "LLMResponse", "get_logger", "wrap_provider_error", ] diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index 9954c987..f41f31f7 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import json as _json +import re import time from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable @@ -19,6 +21,7 @@ from learning_commons_evaluators.schemas.config import ( EvaluationSettings, EvaluatorConfig, + LLMProvider, PromptSettings, ) from learning_commons_evaluators.schemas.errors import ( @@ -32,6 +35,11 @@ EvaluationInput, EvaluationResult, ) +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) from learning_commons_evaluators.schemas.metadata import ( PROMPT_STEP_EXTRA_PROMPT_SETTINGS, PROMPT_STEP_EXTRA_TOKEN_USAGE, @@ -43,6 +51,31 @@ prompt_settings_to_extras_value, ) + +def _strip_json_fences(text: str) -> str: + """Strip markdown code fences and extract the first valid JSON object or array. + + Uses ``json.JSONDecoder.raw_decode`` to locate the first balanced JSON structure + and discard any surrounding prose or trailing text. This correctly handles: + - Markdown-fenced responses: ````json\\n{...}\\n``` `` + - Prose-prefixed responses: ``Here is the result:\\n{...}`` + - Trailing-prose responses: ``{...} Here is my reasoning...`` + """ + text = text.strip() + text = re.sub(r"^```(?:json)?\s*\n?", "", text) + text = re.sub(r"\n?```\s*$", "", text) + text = text.strip() + # Find the first { or [ and use raw_decode to extract the complete JSON structure, + # correctly discarding any trailing prose or a second JSON object in the response. + start = next((i for i, ch in enumerate(text) if ch in ("{", "[")), -1) + if start != -1: + try: + _, end = _json.JSONDecoder().raw_decode(text, start) + return text[start:end] + except _json.JSONDecodeError: + pass + return text + InputT = TypeVar("InputT", bound=EvaluationInput) OutputT = TypeVar("OutputT", bound=EvaluationResult) SettingsT = TypeVar("SettingsT", bound=EvaluationSettings) @@ -74,9 +107,11 @@ def __init__( self, config: EvaluatorConfig, *, + llm_provider: LLMGeneratorProtocol | None = None, default_evaluation_settings: SettingsT | None = None, ) -> None: self.config = config + self._llm_provider = llm_provider if default_evaluation_settings is not None: self.default_evaluation_settings = default_evaluation_settings # TODO: validate config @@ -312,6 +347,14 @@ async def execute_prompt_chain_step( Parsed instance of ``parser_output_type`` when it is a model class; plain ``str`` when ``parser_output_type`` is omitted or ``None``. + Note: + **Execution path**: when ``self._llm_provider`` is set (injected at + construction via ``BaseEvaluator.__init__``), the *protocol path* is taken + — the LangChain template is formatted to extract system/human strings, the + injected provider is called directly, and JSON is parsed via Pydantic. + When ``self._llm_provider`` is ``None`` (default), the *LangChain path* + is taken and ``create_provider()`` is called internally. + Raises: ConfigurationError: No provider config for ``prompt_settings.provider_type``. OutputValidationError: The LLM response didn't satisfy the expected @@ -330,6 +373,102 @@ async def execute_prompt_chain_step( # Populated after a successful LLM invoke so we can attach usage even if parsing fails. token_usage: TokenUsage | None = None + if self._llm_provider is not None: + # ── Protocol path ────────────────────────────────────────────── + # Format the LangChain template to extract system/human strings, + # then delegate the actual LLM call to the injected provider. + # JSON parsing is handled directly via Pydantic — no LangChain parser needed. + provider = self._llm_provider + + async def _run_via_provider() -> BaseModel | str: + nonlocal token_usage + formatted = await template.aformat_messages(**chain_inputs) + system_str = next( + (str(m.content) for m in formatted if getattr(m, "type", "") == "system"), "" + ) + human_str = next( + (str(m.content) for m in formatted if getattr(m, "type", "") == "human"), "" + ) + try: + response: LLMResponse = await provider.generate( + system=system_str, + human=human_str, + config=GenerateConfig(temperature=prompt_settings.temperature), + ) + except EvaluatorError: + raise + except (KeyboardInterrupt, SystemExit): + raise + except Exception as e: + raise wrap_provider_error( + e, + provider=prompt_settings.provider_type, + model=prompt_settings.model, + ) from e + if response.input_tokens is not None or response.output_tokens is not None: + token_usage = TokenUsage( + provider_type=prompt_settings.provider_type, + model=response.model, + input_tokens=response.input_tokens or 0, + output_tokens=response.output_tokens or 0, + ) + if parser_output_type is None: + return response.content + raw_json = _strip_json_fences(response.content) + try: + if json_dict_normalizer is not None: + parsed_dict = _json.loads(raw_json) + if not isinstance(parsed_dict, dict): + raise OutputValidationError( + "Model output is not a JSON object", + provider=prompt_settings.provider_type, + model=response.model, + ) + try: + normalized = json_dict_normalizer(parsed_dict) + except (TypeError, ValueError) as norm_err: + raise OutputValidationError( + "Model output could not be normalized before validation", + provider=prompt_settings.provider_type, + model=response.model, + ) from norm_err + return parser_output_type.model_validate(normalized) + return parser_output_type.model_validate_json(raw_json) + except PydanticValidationError as e: + raise OutputValidationError( + provider=prompt_settings.provider_type, + model=response.model, + validation_errors=sanitize_pydantic_errors(e.errors()), + ) from e + except OutputValidationError: + raise + except Exception as e: + raise OutputValidationError( + provider=prompt_settings.provider_type, + model=response.model, + ) from e + + try: + return await self.execute_step( + step_name, + evaluation_metadata, + _run_via_provider, + extras={ + PROMPT_STEP_EXTRA_PROMPT_SETTINGS: prompt_settings_to_extras_value( + prompt_settings + ), + }, + ) + finally: + if token_usage is not None: + self.update_total_token_usage(token_usage, evaluation_metadata) + step = evaluation_metadata.step_details.get(step_name) + if step is not None: + step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE] = token_usage.model_dump( + mode="json" + ) + + # ── LangChain path (default, unchanged) ─────────────────────────── async def _run_chain() -> BaseModel | str: nonlocal token_usage try: diff --git a/sdks/python/src/learning_commons_evaluators/schemas/__init__.py b/sdks/python/src/learning_commons_evaluators/schemas/__init__.py index 27caf004..bb41b809 100644 --- a/sdks/python/src/learning_commons_evaluators/schemas/__init__.py +++ b/sdks/python/src/learning_commons_evaluators/schemas/__init__.py @@ -33,6 +33,11 @@ InputSpec, TextInputSpec, ) +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) from learning_commons_evaluators.schemas.metadata import ( PROMPT_STEP_EXTRA_PROMPT_SETTINGS, PROMPT_STEP_EXTRA_TOKEN_USAGE, @@ -81,5 +86,8 @@ "TextInputField", "TokenUsage", "InputValidationError", + "GenerateConfig", + "LLMGeneratorProtocol", + "LLMResponse", "prompt_settings_to_extras_value", ] diff --git a/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py b/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py new file mode 100644 index 00000000..7de3c594 --- /dev/null +++ b/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py @@ -0,0 +1,121 @@ +"""LLM provider protocol and associated types for framework-agnostic model injection. + +These types define the interface that evaluation frameworks (Inspect AI, Braintrust, +Arize/Phoenix, Langfuse, etc.) implement to provide their own model execution to the SDK. + +``LLMGeneratorProtocol`` is a structural protocol (``typing.Protocol``) — integration +packages do not need to import or inherit from it. Any class with the correct ``generate`` +signature satisfies the protocol automatically for static type checkers. + +Response fields are aligned with OpenTelemetry GenAI semantic conventions: +https://opentelemetry.io/docs/specs/semconv/gen-ai/ +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import NamedTuple, Protocol + + +class LLMResponse(NamedTuple): + """Structured response from an LLM generation call. + + A ``NamedTuple`` — immutable, constructible by keyword or position, and + iterable for adapters that need to destructure the result. + + Fields are aligned with OpenTelemetry GenAI semantic conventions so that + observability adapters (Arize/Phoenix, Langfuse) can populate their spans + without additional parsing. + + Required fields (``content``, ``model``) must always be populated. + Optional fields should be populated whenever the underlying provider returns them. + """ + + content: str + """The model's text response.""" + + model: str + """The model that generated the response (``gen_ai.response.model``).""" + + input_tokens: int | None = None + """Number of input/prompt tokens consumed (``gen_ai.usage.input_tokens``).""" + + output_tokens: int | None = None + """Number of output/completion tokens generated (``gen_ai.usage.output_tokens``).""" + + +@dataclass +class GenerateConfig: + """Configuration for a single LLM generation call. + + All fields are optional. Adapters should apply whatever the underlying + provider supports and ignore the rest. + """ + + temperature: float | None = None + """Sampling temperature. 0.0 for deterministic output (recommended for evals).""" + + max_tokens: int | None = None + """Maximum number of tokens to generate.""" + + +class LLMGeneratorProtocol(Protocol): + """Structural protocol for LLM generation adapters. + + Implement this protocol in an integration package to allow the SDK to call + your framework's model system. + + No import of this class is required in the implementing package. Structural + conformance (correct ``generate`` signature) is sufficient for static type + checkers. + + **Lifecycle**: if your adapter holds a connection pool or HTTP session, add + ``async def aclose(self) -> None`` and call it when done. The protocol does + not include ``aclose`` so that stateless adapters remain fully conformant + without boilerplate. Callers that want to support teardown should use + ``hasattr(adapter, "aclose")``. + + Example:: + + from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMResponse, + ) + + class MyFrameworkAdapter: + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + response = await my_framework.call(system, human) + return LLMResponse( + content=response.text, + model=response.model_name, + input_tokens=response.usage.input, + output_tokens=response.usage.output, + ) + """ + + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + """Generate a response from the LLM. + + Args: + system: The system prompt. + human: The human/user prompt. + config: Optional generation configuration. Adapters apply whatever + fields the underlying provider supports and ignore the rest. + + Returns: + ``LLMResponse`` with at minimum ``content`` and ``model`` populated. + Populate optional fields (token counts) whenever the provider returns them. + """ + ... diff --git a/sdks/python/tests/schemas/test_llm_provider.py b/sdks/python/tests/schemas/test_llm_provider.py new file mode 100644 index 00000000..e819df3f --- /dev/null +++ b/sdks/python/tests/schemas/test_llm_provider.py @@ -0,0 +1,106 @@ +"""Tests for LLMGeneratorProtocol, LLMResponse, and GenerateConfig.""" + +from __future__ import annotations + +import learning_commons_evaluators +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) + + +class TestLLMResponse: + def test_required_fields(self): + r = LLMResponse(content="hello", model="anthropic/claude-opus-4-8") + assert r.content == "hello" + assert r.model == "anthropic/claude-opus-4-8" + assert r.input_tokens is None + assert r.output_tokens is None + + def test_optional_token_fields(self): + r = LLMResponse(content="text", model="test", input_tokens=100, output_tokens=50) + assert r.input_tokens == 100 + assert r.output_tokens == 50 + + + +class TestGenerateConfig: + def test_defaults(self): + c = GenerateConfig() + assert c.temperature is None + assert c.max_tokens is None + + def test_with_values(self): + c = GenerateConfig(temperature=0.0, max_tokens=512) + assert c.temperature == 0.0 + assert c.max_tokens == 512 + + +class TestLLMGeneratorProtocol: + async def test_generate_returns_llm_response(self): + class Adapter: + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + return LLMResponse(content="hi", model="m", input_tokens=5, output_tokens=2) + + result = await Adapter().generate(system="sys", human="hello") + assert result.content == "hi" + assert result.model == "m" + assert result.input_tokens == 5 + + async def test_generate_config_passed_through(self): + received: list[GenerateConfig | None] = [] + + class Adapter: + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + received.append(config) + return LLMResponse(content="", model="test") + + cfg = GenerateConfig(temperature=0.0, max_tokens=256) + await Adapter().generate(system="sys", human="hello", config=cfg) + assert received[0] is cfg + assert received[0].temperature == 0.0 + + async def test_generate_config_none_by_default(self): + received: list[GenerateConfig | None] = [] + + class Adapter: + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + received.append(config) + return LLMResponse(content="", model="test") + + await Adapter().generate(system="sys", human="hello") + assert received[0] is None + + def test_structural_conformance(self): + """Static type checkers validate this assignment — no subclassing required. + + LLMGeneratorProtocol is not @runtime_checkable; isinstance() is not available. + Conformance is enforced at type-check time (mypy/pyright) by the annotation below. + """ + + class Adapter: + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + return LLMResponse(content="", model="test") + + # mypy/pyright validate this structurally + _adapter: LLMGeneratorProtocol = Adapter() + assert _adapter is not None # runtime no-op; static check is the value + + +def test_exported_from_package(): + assert "GenerateConfig" in learning_commons_evaluators.__all__ + assert "LLMGeneratorProtocol" in learning_commons_evaluators.__all__ + assert "LLMResponse" in learning_commons_evaluators.__all__ From 5828620b8044849f5a63410a5f946bd4fb628791 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 21:12:03 -0700 Subject: [PATCH 2/8] fix(python-sdk): remove unused LLMProvider import from base.py --- sdks/python/src/learning_commons_evaluators/evaluators/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index f41f31f7..b96d31ef 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -21,7 +21,6 @@ from learning_commons_evaluators.schemas.config import ( EvaluationSettings, EvaluatorConfig, - LLMProvider, PromptSettings, ) from learning_commons_evaluators.schemas.errors import ( From 3ecaf88edf87b6c49fd2f2767e43146d804699b3 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 21:23:40 -0700 Subject: [PATCH 3/8] style(python-sdk): apply ruff formatter to base.py and test_llm_provider.py --- sdks/python/src/learning_commons_evaluators/evaluators/base.py | 1 + sdks/python/tests/schemas/test_llm_provider.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index b96d31ef..080c5be0 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -75,6 +75,7 @@ def _strip_json_fences(text: str) -> str: pass return text + InputT = TypeVar("InputT", bound=EvaluationInput) OutputT = TypeVar("OutputT", bound=EvaluationResult) SettingsT = TypeVar("SettingsT", bound=EvaluationSettings) diff --git a/sdks/python/tests/schemas/test_llm_provider.py b/sdks/python/tests/schemas/test_llm_provider.py index e819df3f..b0782097 100644 --- a/sdks/python/tests/schemas/test_llm_provider.py +++ b/sdks/python/tests/schemas/test_llm_provider.py @@ -24,7 +24,6 @@ def test_optional_token_fields(self): assert r.output_tokens == 50 - class TestGenerateConfig: def test_defaults(self): c = GenerateConfig() From ca64de28e8be836aac45f18e4a9719cbb383b11a Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 21:32:30 -0700 Subject: [PATCH 4/8] test(python-sdk): add protocol path coverage for execute_prompt_chain_step Adds TestExecutePromptChainStepProtocolPath (13 tests) covering the llm_provider injection path in BaseEvaluator.execute_prompt_chain_step: - Raw string return when parser_output_type=None - Clean JSON parse - Markdown fence stripping - Trailing prose stripping (JSON followed by explanation text) - Leading prose stripping (prose before JSON) - json_dict_normalizer path - Non-dict JSON raises OutputValidationError on normalizer path - Malformed JSON raises OutputValidationError - Schema mismatch raises OutputValidationError - Token usage recorded in step extras and total_token_usage - Token usage absent when LLMResponse has None tokens - Provider RuntimeError wrapped as APIError - EvaluatorError from provider re-raised unchanged - KeyboardInterrupt from provider propagated --- sdks/python/tests/evaluators/test_base.py | 241 ++++++++++++++++++++++ 1 file changed, 241 insertions(+) diff --git a/sdks/python/tests/evaluators/test_base.py b/sdks/python/tests/evaluators/test_base.py index f595af9c..998c6808 100644 --- a/sdks/python/tests/evaluators/test_base.py +++ b/sdks/python/tests/evaluators/test_base.py @@ -857,3 +857,244 @@ def passthrough(d: dict) -> dict: assert "JSON object" in str(exc_info.value) assert exc_info.value.provider is LLMProvider.GOOGLE assert exc_info.value.model == "gemini-2.0-flash" + + +# --------------------------------------------------------------------------- +# execute_prompt_chain_step — protocol path (llm_provider injected) +# --------------------------------------------------------------------------- + + +from learning_commons_evaluators.schemas.llm_provider import LLMResponse # noqa: E402 + + +def _make_adapter( + content: str, + model: str = "test-model", + input_tokens: int | None = 10, + output_tokens: int | None = 5, +) -> AsyncMock: + """Minimal mock that satisfies LLMGeneratorProtocol.generate().""" + adapter = AsyncMock() + adapter.generate = AsyncMock( + return_value=LLMResponse( + content=content, model=model, input_tokens=input_tokens, output_tokens=output_tokens + ) + ) + return adapter + + +_PROTO_SETTINGS = PromptSettings( + provider_type=LLMProvider.ANTHROPIC, + model="claude-opus-4-8", + temperature=0.0, +) + +_PROTO_TEMPLATE = ChatPromptTemplate.from_messages( + [("system", "You are a grader."), ("human", "{input}")] +) + + +class TestExecutePromptChainStepProtocolPath: + """Protocol path: llm_provider injected — LangChain provider is never called.""" + + def _ev(self, adapter: AsyncMock) -> _StubEvaluator: + return _StubEvaluator(create_config_no_telemetry(), llm_provider=adapter) + + async def test_returns_raw_string_when_parser_type_is_none(self, evaluation_metadata): + ev = self._ev(_make_adapter("plain prose")) + out = await ev.execute_prompt_chain_step( + step_name="raw", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=None, + ) + assert out == "plain prose" + + async def test_parses_clean_json(self, evaluation_metadata): + ev = self._ev(_make_adapter(_CHAIN_JSON)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert isinstance(result, _ChainOutput) + assert result.label == "ok" + assert result.score == 7 + + async def test_strips_markdown_fences(self, evaluation_metadata): + fenced = f"```json\n{_CHAIN_JSON}\n```" + ev = self._ev(_make_adapter(fenced)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert result.label == "ok" + + async def test_strips_trailing_prose(self, evaluation_metadata): + with_prose = f"{_CHAIN_JSON}\n\nHere is my reasoning for this score." + ev = self._ev(_make_adapter(with_prose)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert result.label == "ok" + + async def test_strips_leading_prose(self, evaluation_metadata): + with_prefix = f"Here is the result:\n{_CHAIN_JSON}" + ev = self._ev(_make_adapter(with_prefix)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert result.label == "ok" + + async def test_json_dict_normalizer_path(self, evaluation_metadata): + class _Out(BaseModel): + n: int + doubled: int + + ev = self._ev(_make_adapter('{"n": 3}')) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_Out, + json_dict_normalizer=lambda d: {**d, "doubled": d["n"] * 2}, + ) + assert result.n == 3 + assert result.doubled == 6 + + async def test_non_dict_json_in_normalizer_path_raises_output_validation_error( + self, evaluation_metadata + ): + class _Out(BaseModel): + n: int + + ev = self._ev(_make_adapter('["not", "an", "object"]')) + with pytest.raises(OutputValidationError) as exc_info: + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_Out, + json_dict_normalizer=lambda d: d, + ) + assert "JSON object" in str(exc_info.value) + + async def test_malformed_json_raises_output_validation_error(self, evaluation_metadata): + ev = self._ev(_make_adapter("not json at all")) + with pytest.raises(OutputValidationError): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + + async def test_schema_mismatch_raises_output_validation_error(self, evaluation_metadata): + ev = self._ev(_make_adapter('{"label": "only"}')) # missing required `score` + with pytest.raises(OutputValidationError) as exc_info: + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert isinstance(exc_info.value.__cause__, PydanticValidationError) + + async def test_token_usage_recorded_in_step_extras_and_total(self, evaluation_metadata): + ev = self._ev( + _make_adapter(_CHAIN_JSON, model="claude-opus-4-8", input_tokens=42, output_tokens=17) + ) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + step = evaluation_metadata.step_details["main"] + assert step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE]["input_tokens"] == 42 + assert step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE]["output_tokens"] == 17 + assert evaluation_metadata.total_token_usage[LLMProvider.ANTHROPIC].input_tokens == 42 + + async def test_token_usage_absent_when_llm_response_has_none_tokens(self, evaluation_metadata): + ev = self._ev(_make_adapter(_CHAIN_JSON, input_tokens=None, output_tokens=None)) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert not evaluation_metadata.total_token_usage + + async def test_provider_error_wrapped_as_api_error(self, evaluation_metadata): + adapter = AsyncMock() + adapter.generate = AsyncMock(side_effect=RuntimeError("network timeout")) + ev = self._ev(adapter) + with pytest.raises(APIError) as exc_info: + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert isinstance(exc_info.value.__cause__, RuntimeError) + + async def test_evaluator_error_from_provider_reraises_unchanged(self, evaluation_metadata): + adapter = AsyncMock() + adapter.generate = AsyncMock(side_effect=EvaluatorError("already wrapped")) + ev = self._ev(adapter) + with pytest.raises(EvaluatorError, match="already wrapped"): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + + async def test_keyboard_interrupt_from_provider_propagates(self, evaluation_metadata): + adapter = AsyncMock() + adapter.generate = AsyncMock(side_effect=KeyboardInterrupt) + ev = self._ev(adapter) + with pytest.raises(KeyboardInterrupt): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) From 7418a24e026606a50705b40bd4ab96f0a36f173e Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 22:00:18 -0700 Subject: [PATCH 5/8] fix(python-sdk): address independent code review findings - Revert version bump to 0.2.0 (release-please handles this on merge) - Add model field to GenerateConfig so adapters can see which model the evaluator expects without reaching into prompt_settings - Move template.aformat_messages() inside try block so missing template variables raise EvaluatorError rather than bare KeyError - Add ValueError when human_str is empty; DEBUG log when system_str is empty - Extract _parse_json_output() helper to deduplicate JSON parsing and error wrapping between the protocol and LangChain paths - Add tests: assert adapter.generate() called with correct system/human, human-only template passes empty system string, missing variable raises EvaluatorError --- sdks/python/pyproject.toml | 2 +- .../evaluators/base.py | 163 ++++++++++-------- .../schemas/llm_provider.py | 12 ++ sdks/python/tests/evaluators/test_base.py | 51 ++++++ 4 files changed, 156 insertions(+), 72 deletions(-) diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index b96dc345..aea23087 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "learning-commons-evaluators" -version = "0.3.0" +version = "0.2.0" description = "Python SDK for Learning Commons educational evaluators" readme = "README.md" license = { text = "MIT" } diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index 080c5be0..cd81d85c 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -50,6 +50,57 @@ prompt_settings_to_extras_value, ) +# TypeVars used by module-level helpers and the BaseEvaluator generic class. +InputT = TypeVar("InputT", bound=EvaluationInput) +OutputT = TypeVar("OutputT", bound=EvaluationResult) +SettingsT = TypeVar("SettingsT", bound=EvaluationSettings) +StepResultT = TypeVar("StepResultT") +ParsedT = TypeVar("ParsedT", bound=BaseModel) + + +def _parse_json_output( + raw: str, + parser_output_type: type[ParsedT], + json_dict_normalizer: Callable[[dict], dict] | None, + provider_type: Any, + model: str, +) -> ParsedT: + """Parse a raw JSON string into ``parser_output_type``, wrapping errors consistently. + + Shared by both the protocol path and (for the normalizer branch) the LangChain path + so that error handling is symmetric and normaliser logic lives in one place. + """ + raw = _strip_json_fences(raw) + try: + if json_dict_normalizer is not None: + parsed_dict = _json.loads(raw) + if not isinstance(parsed_dict, dict): + raise OutputValidationError( + "Model output is not a JSON object", + provider=provider_type, + model=model, + ) + try: + normalized = json_dict_normalizer(parsed_dict) + except (TypeError, ValueError) as norm_err: + raise OutputValidationError( + "Model output could not be normalized before validation", + provider=provider_type, + model=model, + ) from norm_err + return parser_output_type.model_validate(normalized) + return parser_output_type.model_validate_json(raw) + except PydanticValidationError as e: + raise OutputValidationError( + provider=provider_type, + model=model, + validation_errors=sanitize_pydantic_errors(e.errors()), + ) from e + except OutputValidationError: + raise + except Exception as e: + raise OutputValidationError(provider=provider_type, model=model) from e + def _strip_json_fences(text: str) -> str: """Strip markdown code fences and extract the first valid JSON object or array. @@ -76,13 +127,6 @@ def _strip_json_fences(text: str) -> str: return text -InputT = TypeVar("InputT", bound=EvaluationInput) -OutputT = TypeVar("OutputT", bound=EvaluationResult) -SettingsT = TypeVar("SettingsT", bound=EvaluationSettings) -StepResultT = TypeVar("StepResultT") -ParsedT = TypeVar("ParsedT", bound=BaseModel) - - class BaseEvaluator(ABC, Generic[InputT, OutputT, SettingsT]): """ Abstract base class for all evaluators. @@ -382,18 +426,36 @@ async def execute_prompt_chain_step( async def _run_via_provider() -> BaseModel | str: nonlocal token_usage - formatted = await template.aformat_messages(**chain_inputs) - system_str = next( - (str(m.content) for m in formatted if getattr(m, "type", "") == "system"), "" - ) - human_str = next( - (str(m.content) for m in formatted if getattr(m, "type", "") == "human"), "" - ) try: + # Template formatting is inside try so missing variables become EvaluatorErrors, + # not bare KeyErrors — matching the error contract of the LangChain path. + formatted = await template.aformat_messages(**chain_inputs) + system_str = next( + (str(m.content) for m in formatted if getattr(m, "type", "") == "system"), + "", + ) + human_str = next( + (str(m.content) for m in formatted if getattr(m, "type", "") == "human"), + "", + ) + if not human_str: + raise ValueError( + f"Template for step '{step_name}' produced no human message. " + 'Ensure the template contains at least one ("human", ...) turn.' + ) + if not system_str: + self.config.logger.debug( + "No system message in template for step '%s'; " + "passing empty string to adapter.", + step_name, + ) response: LLMResponse = await provider.generate( system=system_str, human=human_str, - config=GenerateConfig(temperature=prompt_settings.temperature), + config=GenerateConfig( + temperature=prompt_settings.temperature, + model=prompt_settings.model, + ), ) except EvaluatorError: raise @@ -414,39 +476,13 @@ async def _run_via_provider() -> BaseModel | str: ) if parser_output_type is None: return response.content - raw_json = _strip_json_fences(response.content) - try: - if json_dict_normalizer is not None: - parsed_dict = _json.loads(raw_json) - if not isinstance(parsed_dict, dict): - raise OutputValidationError( - "Model output is not a JSON object", - provider=prompt_settings.provider_type, - model=response.model, - ) - try: - normalized = json_dict_normalizer(parsed_dict) - except (TypeError, ValueError) as norm_err: - raise OutputValidationError( - "Model output could not be normalized before validation", - provider=prompt_settings.provider_type, - model=response.model, - ) from norm_err - return parser_output_type.model_validate(normalized) - return parser_output_type.model_validate_json(raw_json) - except PydanticValidationError as e: - raise OutputValidationError( - provider=prompt_settings.provider_type, - model=response.model, - validation_errors=sanitize_pydantic_errors(e.errors()), - ) from e - except OutputValidationError: - raise - except Exception as e: - raise OutputValidationError( - provider=prompt_settings.provider_type, - model=response.model, - ) from e + return _parse_json_output( + response.content, + parser_output_type, + json_dict_normalizer, + prompt_settings.provider_type, + response.model, + ) try: return await self.execute_step( @@ -481,29 +517,14 @@ async def _run_chain() -> BaseModel | str: from langchain_core.output_parsers.json import JsonOutputParser if json_dict_normalizer is not None: - loose = JsonOutputParser() - parsed_dict = await loose.ainvoke(ai_message) - if not isinstance(parsed_dict, dict): - # JSON parsed cleanly but the top-level value isn't an object - # (e.g. the LLM returned a JSON array or scalar). That's an - # output-shape failure, not a parse failure — surface it as - # OutputValidationError so callers can treat it consistently - # with schema-mismatch errors, and avoid the TypeError that - # ``dict(parsed_dict)`` would raise on a non-dict. - raise OutputValidationError( - "Model output is not a JSON object", - provider=prompt_settings.provider_type, - model=prompt_settings.model, - ) - try: - normalized = json_dict_normalizer(parsed_dict) - except (TypeError, ValueError) as norm_err: - raise OutputValidationError( - "Model output could not be normalized before validation", - provider=prompt_settings.provider_type, - model=prompt_settings.model, - ) from norm_err - return parser_output_type.model_validate(normalized) + # Use the shared helper so normalizer + error-wrapping logic stays in one place. + return _parse_json_output( + str(ai_message.content), + parser_output_type, + json_dict_normalizer, + prompt_settings.provider_type, + prompt_settings.model, + ) parser = JsonOutputParser(pydantic_object=parser_output_type) raw = await parser.ainvoke(ai_message) diff --git a/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py b/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py index 7de3c594..0ccf489b 100644 --- a/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py +++ b/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py @@ -58,6 +58,18 @@ class GenerateConfig: max_tokens: int | None = None """Maximum number of tokens to generate.""" + model: str | None = None + """Model identifier to request from the provider (e.g. ``"claude-opus-4-8"``). + + Adapters should use this when set and ignore it otherwise — the contract is + identical to all other ``GenerateConfig`` fields. When ``None`` (default), + the adapter uses whatever model it was constructed with. + + Populated from ``PromptSettings.model`` on the protocol path so that + adapter authors can inspect which model the evaluator expects without + reaching into ``prompt_settings`` directly. + """ + class LLMGeneratorProtocol(Protocol): """Structural protocol for LLM generation adapters. diff --git a/sdks/python/tests/evaluators/test_base.py b/sdks/python/tests/evaluators/test_base.py index 998c6808..863d36e0 100644 --- a/sdks/python/tests/evaluators/test_base.py +++ b/sdks/python/tests/evaluators/test_base.py @@ -1098,3 +1098,54 @@ async def test_keyboard_interrupt_from_provider_propagates(self, evaluation_meta chain_inputs={"input": "Hello"}, parser_output_type=_ChainOutput, ) + + async def test_adapter_called_with_formatted_system_and_human(self, evaluation_metadata): + """Template formatting actually reaches the adapter with the correct strings.""" + from unittest.mock import ANY + + adapter = _make_adapter(_CHAIN_JSON) + ev = self._ev(adapter) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + adapter.generate.assert_awaited_once_with( + system="You are a grader.", + human="Hello", + config=ANY, + ) + + async def test_human_only_template_passes_empty_system(self, evaluation_metadata): + """Templates with no system turn pass empty string to the adapter without error.""" + from unittest.mock import ANY + + human_only = ChatPromptTemplate.from_messages([("human", "{input}")]) + adapter = _make_adapter(_CHAIN_JSON) + ev = self._ev(adapter) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=human_only, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + adapter.generate.assert_awaited_once_with(system="", human="Hello", config=ANY) + + async def test_template_with_missing_variable_raises_evaluator_error(self, evaluation_metadata): + """A missing template variable becomes an EvaluatorError, not a bare KeyError.""" + adapter = _make_adapter(_CHAIN_JSON) + ev = self._ev(adapter) + with pytest.raises(EvaluatorError): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={}, # missing required "input" variable + parser_output_type=_ChainOutput, + ) From 7d75e0d5c0f7088e15b069613c243e039645fa30 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 11 Jun 2026 22:12:41 -0700 Subject: [PATCH 6/8] style: remove redundant inline comments in base.py --- .../learning_commons_evaluators/evaluators/base.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index cd81d85c..70d2b412 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -50,7 +50,6 @@ prompt_settings_to_extras_value, ) -# TypeVars used by module-level helpers and the BaseEvaluator generic class. InputT = TypeVar("InputT", bound=EvaluationInput) OutputT = TypeVar("OutputT", bound=EvaluationResult) SettingsT = TypeVar("SettingsT", bound=EvaluationSettings) @@ -418,17 +417,14 @@ async def execute_prompt_chain_step( token_usage: TokenUsage | None = None if self._llm_provider is not None: - # ── Protocol path ────────────────────────────────────────────── - # Format the LangChain template to extract system/human strings, - # then delegate the actual LLM call to the injected provider. - # JSON parsing is handled directly via Pydantic — no LangChain parser needed. + # ── Protocol path ───────────────────────────────────────────── provider = self._llm_provider async def _run_via_provider() -> BaseModel | str: nonlocal token_usage try: - # Template formatting is inside try so missing variables become EvaluatorErrors, - # not bare KeyErrors — matching the error contract of the LangChain path. + # Inside try: missing template variables become EvaluatorErrors, + # not bare KeyErrors — consistent with the LangChain path's error contract. formatted = await template.aformat_messages(**chain_inputs) system_str = next( (str(m.content) for m in formatted if getattr(m, "type", "") == "system"), @@ -517,7 +513,6 @@ async def _run_chain() -> BaseModel | str: from langchain_core.output_parsers.json import JsonOutputParser if json_dict_normalizer is not None: - # Use the shared helper so normalizer + error-wrapping logic stays in one place. return _parse_json_output( str(ai_message.content), parser_output_type, From af18c04a52dac33a904ac90f404b14ab5db01ca4 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Mon, 22 Jun 2026 21:04:23 -0700 Subject: [PATCH 7/8] feat: add Inspect AI integration package (learning-commons-inspect-scorers) --- .release-please-manifest.json | 3 +- integrations/inspect-python/.gitignore | 6 + integrations/inspect-python/CHANGELOG.md | 1 + integrations/inspect-python/README.md | 82 +++++ integrations/inspect-python/pyproject.toml | 75 +++++ .../__init__.py | 6 + .../_registry.py | 10 + .../adapter.py | 71 ++++ .../learning_commons_inspect_scorers/gla.py | 122 +++++++ .../learning_commons_inspect_scorers/py.typed | 0 .../inspect-python/tests/test_gla_scorer.py | 306 ++++++++++++++++++ release-please-config.json | 8 + 12 files changed, 689 insertions(+), 1 deletion(-) create mode 100644 integrations/inspect-python/.gitignore create mode 100644 integrations/inspect-python/CHANGELOG.md create mode 100644 integrations/inspect-python/README.md create mode 100644 integrations/inspect-python/pyproject.toml create mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/__init__.py create mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/_registry.py create mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/adapter.py create mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py create mode 100644 integrations/inspect-python/src/learning_commons_inspect_scorers/py.typed create mode 100644 integrations/inspect-python/tests/test_gla_scorer.py diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 297471d8..e795794a 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,5 +1,6 @@ { "evals/prompts": "1.5.0", "sdks/python": "0.2.0", - "sdks/typescript": "0.7.0" + "sdks/typescript": "0.7.0", + "integrations/inspect-python": "0.1.0" } diff --git a/integrations/inspect-python/.gitignore b/integrations/inspect-python/.gitignore new file mode 100644 index 00000000..5ca865e5 --- /dev/null +++ b/integrations/inspect-python/.gitignore @@ -0,0 +1,6 @@ +*.egg-info/ +dist/ +build/ +__pycache__/ +.pytest_cache/ +.mypy_cache/ diff --git a/integrations/inspect-python/CHANGELOG.md b/integrations/inspect-python/CHANGELOG.md new file mode 100644 index 00000000..825c32f0 --- /dev/null +++ b/integrations/inspect-python/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog diff --git a/integrations/inspect-python/README.md b/integrations/inspect-python/README.md new file mode 100644 index 00000000..d5f3b192 --- /dev/null +++ b/integrations/inspect-python/README.md @@ -0,0 +1,82 @@ +# learning-commons-inspect-scorers + +[Inspect AI](https://inspect.aisi.org.uk/) scorer wrappers for the [Learning Commons evaluators](https://github.com/learning-commons-org/evaluators) SDK. + +## Installation + +```bash +pip install learning-commons-inspect-scorers +``` + +> **Note:** Requires `learning-commons-evaluators>=0.2.0`. During local development +> install the SDK from the repo root first: +> ```bash +> pip install -e sdks/python +> pip install -e integrations/inspect-python +> ``` + +## Usage + +### Grade Level Appropriateness scorer + +Evaluates whether model output (or generated artifact files) is written at the +appropriate reading level for a target K-12 grade band. + +```python +from inspect_ai import Task, task +from inspect_ai.dataset import csv_dataset, FieldSpec +from inspect_ai.solver import generate +from learning_commons_inspect_scorers import gla_scorer + +@task +def my_eval(): + return Task( + dataset=csv_dataset("samples.csv"), # requires target_grade column + solver=[generate()], + scorer=gla_scorer(), + ) +``` + +The dataset CSV must include a `target_grade` metadata column with one of: +`K-1`, `2-3`, `4-5`, `6-8`, `9-10`, `11-CCR`. + +### Scoring text your task produced (not the completion) + +By default the scorer grades `state.output.completion`. To grade text from +somewhere else — files your solver wrote, a specific message, filtered content — +pass a `text_fn`. This keeps task-specific knowledge (file layout, naming, +formatting) in your task, not in this package: + +```python +def student_artifact_text(state): + # read whatever your task produced; return None to skip the sample + return "\n\n".join(read_my_files(state)) or None + +scorer=gla_scorer(text_fn=student_artifact_text) +``` + +### Re-scoring an existing log from the CLI + +Once installed, scorers are registered via Inspect's entry point system: + +```bash +inspect score logs/my-eval.eval --scorer learning_commons_inspect_scorers/gla_scorer +``` + +## Configuration + +| Parameter | Default | Description | +|---|---|---| +| `grader_model` | `"anthropic/claude-opus-4-8"` | Inspect model string for the grading LLM. Uses Inspect's model system — no separate API key configuration needed. | +| `target_grade_key` | `"target_grade"` | Metadata key holding the expected grade band. | +| `allow_adjacent` | `True` | If `True`, the one grade band above or below the target also passes. | +| `text_fn` | `None` | Callable `(TaskState) -> str \| None` returning the text to grade. Defaults to `state.output.completion`. Return `None`/empty to skip the sample. Caller keeps text within the evaluator's input-length limit. | + +## Development + +```bash +# From repo root +pip install -e sdks/python +pip install -e "integrations/inspect-python[dev]" +pytest integrations/inspect-python/tests/ +``` diff --git a/integrations/inspect-python/pyproject.toml b/integrations/inspect-python/pyproject.toml new file mode 100644 index 00000000..8ae72276 --- /dev/null +++ b/integrations/inspect-python/pyproject.toml @@ -0,0 +1,75 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "learning-commons-inspect-scorers" +version = "0.1.0" +description = "Inspect AI scorer wrappers for Learning Commons evaluators" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.10" +authors = [{ name = "Learning Commons" }] +keywords = ["education", "evaluators", "inspect", "evals", "scoring"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Education", +] +dependencies = [ + "learning-commons-evaluators>=0.2.0", + # Score.unscored() — used in every skip/error path of gla_scorer — was added in inspect-ai 0.3.214. + "inspect-ai>=0.3.214", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "ruff>=0.9.0", + "mypy>=1.14.0", +] + +[project.urls] +Homepage = "https://github.com/learning-commons-org/evaluators" +Repository = "https://github.com/learning-commons-org/evaluators/tree/main/integrations/inspect-python" +Documentation = "https://docs.learningcommons.org/evaluators" +"Bug Tracker" = "https://github.com/learning-commons-org/evaluators/issues" + +# Registers scorers with Inspect's component discovery via setuptools entry points. +# Once installed, scorers are accessible as e.g. `learning_commons_inspect_scorers/gla_scorer` +# from the CLI: inspect score log.eval --scorer learning_commons_inspect_scorers/gla_scorer +[project.entry-points.inspect_ai] +learning_commons_inspect_scorers = "learning_commons_inspect_scorers._registry" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +learning_commons_inspect_scorers = ["py.typed"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.ruff] +target-version = "py310" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "UP", "B", "SIM"] +ignore = ["E501"] + +[tool.mypy] +python_version = "3.10" +mypy_path = ["src", "tests"] +explicit_package_bases = true +plugins = ["pydantic.mypy"] +warn_unused_configs = true +show_error_codes = true diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/__init__.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/__init__.py new file mode 100644 index 00000000..c9f2f977 --- /dev/null +++ b/integrations/inspect-python/src/learning_commons_inspect_scorers/__init__.py @@ -0,0 +1,6 @@ +"""Learning Commons Inspect scorers — Inspect AI wrappers for LC evaluators.""" + +from learning_commons_inspect_scorers.adapter import InspectModelAdapter +from learning_commons_inspect_scorers.gla import gla_scorer + +__all__ = ["InspectModelAdapter", "gla_scorer"] diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/_registry.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/_registry.py new file mode 100644 index 00000000..18aefcdd --- /dev/null +++ b/integrations/inspect-python/src/learning_commons_inspect_scorers/_registry.py @@ -0,0 +1,10 @@ +"""Entry point registry — imported by Inspect via the inspect_ai setuptools entry point. + +Importing this module registers all scorers with Inspect's component system, +making them accessible by name (e.g. learning_commons_inspect_scorers/gla_scorer) +from both the Python API and the CLI. +""" + +from learning_commons_inspect_scorers.gla import ( + gla_scorer, # noqa: F401 — import triggers @scorer registry side-effect +) diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/adapter.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/adapter.py new file mode 100644 index 00000000..43d5e25f --- /dev/null +++ b/integrations/inspect-python/src/learning_commons_inspect_scorers/adapter.py @@ -0,0 +1,71 @@ +"""Inspect AI model adapter implementing LLMGeneratorProtocol. + +This is a separate package (``learning-commons-inspect-scorers``) rather than +part of ``learning-commons-evaluators`` because it introduces ``inspect-ai`` as +a hard dependency — a heavy framework that not all SDK users need. + +**Versioning contract**: this package requires ``learning-commons-evaluators>=0.2.0`` +where ``LLMGeneratorProtocol`` was introduced. If a new method is added to the +protocol, bump the lower bound here and update this adapter. + +**Building a new integration** (e.g. ``integrations/langsmith-python``): +implement ``LLMGeneratorProtocol`` — a single async ``generate()`` method that +calls your framework's model and returns ``LLMResponse`` — then inject it into +any evaluator via ``GradeLevelAppropriatenessEvaluator(config=..., llm_provider=adapter)``. +""" + +from __future__ import annotations + +from inspect_ai.model import ( + ChatMessageSystem, + ChatMessageUser, + get_model, +) +from inspect_ai.model import ( + GenerateConfig as InspectGenConfig, +) +from learning_commons_evaluators.schemas.llm_provider import GenerateConfig, LLMResponse + + +class InspectModelAdapter: + """Wraps Inspect's get_model() to satisfy LLMGeneratorProtocol. + + Pass a model string in the same form accepted by Inspect's ``--model`` flag, + for example ``"anthropic/claude-opus-4-8"`` or ``"openai/gpt-4o"``. + + Example:: + + adapter = InspectModelAdapter("anthropic/claude-opus-4-8") + evaluator = GradeLevelAppropriatenessEvaluator( + config=create_config_no_telemetry(), + llm_provider=adapter, + ) + """ + + def __init__(self, model_name: str) -> None: + self._model_name = model_name + + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + # get_model() is memoized by Inspect — repeated calls with the same string + # return the cached Model object without reconstruction. + inspect_model = get_model(self._model_name) + inspect_config = InspectGenConfig( + temperature=config.temperature if config is not None else None, + max_tokens=config.max_tokens if config is not None else None, + ) + output = await inspect_model.generate( + [ChatMessageSystem(content=system), ChatMessageUser(content=human)], + config=inspect_config, + ) + return LLMResponse( + content=output.completion, + model=self._model_name, + input_tokens=getattr(output.usage, "input_tokens", None), + output_tokens=getattr(output.usage, "output_tokens", None), + ) diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py b/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py new file mode 100644 index 00000000..bcc3ea10 --- /dev/null +++ b/integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py @@ -0,0 +1,122 @@ +"""Inspect scorer wrapper for the Grade Level Appropriateness evaluator.""" + +from __future__ import annotations + +from collections.abc import Callable + +from inspect_ai.scorer import CORRECT, INCORRECT, Score, Target, accuracy, scorer +from inspect_ai.solver import TaskState +from learning_commons_evaluators.config import create_config_no_telemetry +from learning_commons_evaluators.evaluators.grade_level_appropriateness import ( + GradeLevelAppropriatenessEvaluationInput, + GradeLevelAppropriatenessEvaluator, +) +from learning_commons_evaluators.schemas.errors import ( + APIError, + ConfigurationError, + InputValidationError, +) +from learning_commons_evaluators.schemas.grade_level_appropriateness import GradeLevelAnswer + +from learning_commons_inspect_scorers.adapter import InspectModelAdapter + +GRADE_BANDS = [m.score for m in GradeLevelAnswer] + + +def _completion_text(state: TaskState) -> str | None: + return state.output.completion or None + + +def _acceptable_bands(target: str, allow_adjacent: bool) -> set[str]: + idx = GRADE_BANDS.index(target) if target in GRADE_BANDS else -1 + if idx == -1 or not allow_adjacent: + return {target} + return {GRADE_BANDS[i] for i in range(max(0, idx - 1), min(len(GRADE_BANDS), idx + 2))} + + +@scorer(metrics=[accuracy()]) +def gla_scorer( + grader_model: str = "anthropic/claude-opus-4-8", + target_grade_key: str = "target_grade", + allow_adjacent: bool = True, + text_fn: Callable[[TaskState], str | None] | None = None, +): + """Score output for grade-level appropriateness against a target grade band. + + Uses Inspect's active model (via InspectModelAdapter) to run the GLA evaluator. + No separate LLM API keys are required — the model is resolved through Inspect's + own model configuration. + + Args: + grader_model: Inspect model string for the grading LLM. + Default: ``"anthropic/claude-opus-4-8"``. + target_grade_key: Metadata key holding the expected grade band string. + Default: ``"target_grade"``. Must be one of: K-1, 2-3, + 4-5, 6-8, 9-10, 11-CCR. + allow_adjacent: If ``True`` (default), the one grade band above or below + the target also counts as a pass. Set to ``False`` for an + exact match. + text_fn: Returns the text to grade for a sample, given the ``TaskState``. + Defaults to ``state.output.completion``. Supply a custom function + to source text elsewhere — e.g. files your task produced — keeping + any task-specific knowledge (file layout, naming, formatting) out + of this package. Return ``None`` or empty to skip the sample. The + caller is responsible for keeping the text within the GLA + evaluator's input-length limit; oversized text raises + ``InputValidationError`` as a task failure. + + Returns ``None`` (skip — the sample is omitted from this scorer's results) when + ``target_grade_key`` is absent or not a valid grade band, when ``text_fn`` + yields no text, or when a transient API/parse error occurs. Re-raises + ``ConfigurationError`` and ``InputValidationError`` as task-level failures. + + ``Score.value`` is ``CORRECT`` (pass) or ``INCORRECT`` (fail). + ``Score.metadata`` contains ``gla_grade``, ``target_grade``, ``alternative_grade``, + and ``scaffolding_needed``. + """ + adapter = InspectModelAdapter(grader_model) + # config is required by BaseEvaluator but no API keys are read here — + # the llm_provider bypasses the LangChain provider path entirely. + config = create_config_no_telemetry() + evaluator = GradeLevelAppropriatenessEvaluator(config=config, llm_provider=adapter) + get_text = text_fn or _completion_text + + async def score(state: TaskState, target: Target) -> Score | None: + # Skip paths return None (not Score.unscored()): None omits the sample from + # this scorer's results entirely, which every Inspect metric — and custom + # report renderers — handle cleanly. Score.unscored() records a NaN value + # that naive renderers can mistake for a real score and average into the mean. + target_grade: str = (state.metadata.get(target_grade_key) or "").strip() + if not target_grade or target_grade not in GRADE_BANDS: + return None + + text = get_text(state) + if not text or not text.strip(): + return None + + try: + result = await evaluator.evaluate(GradeLevelAppropriatenessEvaluationInput(text=text)) + except (ConfigurationError, InputValidationError): + raise # setup/programming errors — let Inspect surface them as task failures + except APIError: # OutputValidationError is a subclass; transient grading failure → skip + return None + + gla_grade: str = result.answer.score + passed = gla_grade in _acceptable_bands(target_grade, allow_adjacent) + + return Score( + value=CORRECT if passed else INCORRECT, + answer=gla_grade, + explanation=( + f"Target: {target_grade} | GLA: {gla_grade} | " + f"{'PASS' if passed else 'FAIL'}\n" + (result.explanation.summary or "") + ), + metadata={ + "gla_grade": gla_grade, + "target_grade": target_grade, + "alternative_grade": result.explanation.details["alternative_grade"], + "scaffolding_needed": result.explanation.details["scaffolding_needed"], + }, + ) + + return score diff --git a/integrations/inspect-python/src/learning_commons_inspect_scorers/py.typed b/integrations/inspect-python/src/learning_commons_inspect_scorers/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/integrations/inspect-python/tests/test_gla_scorer.py b/integrations/inspect-python/tests/test_gla_scorer.py new file mode 100644 index 00000000..c5da592b --- /dev/null +++ b/integrations/inspect-python/tests/test_gla_scorer.py @@ -0,0 +1,306 @@ +"""Tests for the GLA Inspect scorer wrapper.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from inspect_ai.scorer import CORRECT, INCORRECT +from learning_commons_evaluators.schemas.errors import APIError, ConfigurationError + +from learning_commons_inspect_scorers.gla import _acceptable_bands, gla_scorer + +GRADE_BANDS = ["K-1", "2-3", "4-5", "6-8", "9-10", "11-CCR"] + + +# ── Helpers ────────────────────────────────────────────────────────────────── + + +def _make_state(completion: str = "", metadata: dict | None = None) -> MagicMock: + state = MagicMock() + state.output.completion = completion + state.metadata = metadata or {} + return state + + +def _make_target() -> MagicMock: + return MagicMock() + + +def _make_gla_result(grade: str = "6-8") -> MagicMock: + result = MagicMock() + result.answer.score = grade + result.explanation.summary = "Test reasoning." + result.explanation.details = { + "alternative_grade": "4-5", + "scaffolding_needed": "Pre-teach vocabulary.", + } + return result + + +def _make_scorer(grade_result: str = "6-8", side_effect=None, **scorer_kwargs): + """Build a gla_scorer with a patched evaluator. + + Patches GradeLevelAppropriatenessEvaluator so no real LLM calls are made. + The evaluator instance captured in the scorer closure is the mock, so calls + work normally after construction. + """ + mock_evaluator = MagicMock() + mock_evaluator.evaluate = AsyncMock( + return_value=_make_gla_result(grade_result), + side_effect=side_effect, + ) + with patch( + "learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator", + return_value=mock_evaluator, + ): + scorer_fn = gla_scorer(**scorer_kwargs) + # The scorer closure already holds the mock_evaluator instance — no further + # patching needed for subsequent calls. + return scorer_fn, mock_evaluator + + +# ── _acceptable_bands ──────────────────────────────────────────────────────── + + +class TestAcceptableBands: + def test_middle_grade_allow_adjacent(self): + assert _acceptable_bands("6-8", allow_adjacent=True) == {"4-5", "6-8", "9-10"} + + def test_lower_boundary_allow_adjacent(self): + # K-1 is at index 0 — no band below it + assert _acceptable_bands("K-1", allow_adjacent=True) == {"K-1", "2-3"} + + def test_upper_boundary_allow_adjacent(self): + # 11-CCR is at end — no band above it + assert _acceptable_bands("11-CCR", allow_adjacent=True) == {"9-10", "11-CCR"} + + def test_exact_match_only(self): + assert _acceptable_bands("6-8", allow_adjacent=False) == {"6-8"} + + def test_invalid_grade_returns_singleton(self): + assert _acceptable_bands("invalid", allow_adjacent=True) == {"invalid"} + + @pytest.mark.parametrize("band", GRADE_BANDS) + def test_all_bands_are_valid(self, band): + result = _acceptable_bands(band, allow_adjacent=True) + assert band in result + assert len(result) >= 1 + + +# ── gla_scorer — score routing ──────────────────────────────────────────────── + + +class TestGlaScorer: + async def test_matching_grade_returns_correct(self): + scorer_fn, _ = _make_scorer(grade_result="6-8") + state = _make_state(completion="Sample text.", metadata={"target_grade": "6-8"}) + score = await scorer_fn(state, _make_target()) + assert score.value == CORRECT + assert score.answer == "6-8" + + async def test_adjacent_grade_passes_when_allow_adjacent(self): + scorer_fn, _ = _make_scorer(grade_result="4-5") + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + assert (await scorer_fn(state, _make_target())).value == CORRECT + + async def test_non_adjacent_grade_fails(self): + scorer_fn, _ = _make_scorer(grade_result="K-1") + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + assert (await scorer_fn(state, _make_target())).value == INCORRECT + + async def test_exact_match_only_when_adjacent_disabled(self): + scorer_fn, _ = _make_scorer(grade_result="4-5", allow_adjacent=False) + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + assert (await scorer_fn(state, _make_target())).value == INCORRECT + + async def test_boundary_k1_adjacent_passes(self): + scorer_fn, _ = _make_scorer(grade_result="2-3") + state = _make_state(completion="Sample.", metadata={"target_grade": "K-1"}) + assert (await scorer_fn(state, _make_target())).value == CORRECT + + async def test_boundary_11ccr_adjacent_passes(self): + scorer_fn, _ = _make_scorer(grade_result="9-10") + state = _make_state(completion="Sample.", metadata={"target_grade": "11-CCR"}) + assert (await scorer_fn(state, _make_target())).value == CORRECT + + async def test_missing_target_grade_returns_none(self): + scorer_fn, _ = _make_scorer() + state = _make_state(completion="Sample.", metadata={}) + assert await scorer_fn(state, _make_target()) is None + + async def test_invalid_target_grade_returns_none(self): + scorer_fn, _ = _make_scorer() + state = _make_state(completion="Sample.", metadata={"target_grade": "Grade 5"}) + assert await scorer_fn(state, _make_target()) is None + + async def test_empty_completion_returns_none(self): + scorer_fn, mock_evaluator = _make_scorer() + state = _make_state(completion="", metadata={"target_grade": "6-8"}) + assert await scorer_fn(state, _make_target()) is None + mock_evaluator.evaluate.assert_not_called() + + async def test_api_error_returns_none(self): + scorer_fn, _ = _make_scorer(side_effect=APIError("rate limit")) + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + assert await scorer_fn(state, _make_target()) is None + + async def test_configuration_error_propagates(self): + scorer_fn, _ = _make_scorer(side_effect=ConfigurationError("no key")) + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + with pytest.raises(ConfigurationError): + await scorer_fn(state, _make_target()) + + async def test_score_metadata_populated(self): + scorer_fn, _ = _make_scorer(grade_result="6-8") + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + score = await scorer_fn(state, _make_target()) + assert score.metadata["gla_grade"] == "6-8" + assert score.metadata["target_grade"] == "6-8" + assert score.metadata["alternative_grade"] == "4-5" + assert score.metadata["scaffolding_needed"] == "Pre-teach vocabulary." + + async def test_custom_target_grade_key(self): + scorer_fn, _ = _make_scorer(grade_result="6-8", target_grade_key="expected_grade") + state = _make_state(completion="Sample.", metadata={"expected_grade": "6-8"}) + assert (await scorer_fn(state, _make_target())).value == CORRECT + + async def test_custom_target_grade_key_absent_returns_none(self): + scorer_fn, _ = _make_scorer(target_grade_key="expected_grade") + state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"}) + assert await scorer_fn(state, _make_target()) is None + + async def test_custom_grader_model_passed_to_adapter(self): + with ( + patch("learning_commons_inspect_scorers.gla.InspectModelAdapter") as mock_adapter_cls, + patch("learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator"), + ): + gla_scorer(grader_model="openai/gpt-4o") + mock_adapter_cls.assert_called_once_with("openai/gpt-4o") + + +# ── gla_scorer — text_fn seam ───────────────────────────────────────────────── + + +class TestGlaScorerTextFn: + async def test_default_reads_completion(self): + scorer_fn, mock_evaluator = _make_scorer(grade_result="6-8") + state = _make_state(completion="From completion.", metadata={"target_grade": "6-8"}) + await scorer_fn(state, _make_target()) + called_text = mock_evaluator.evaluate.call_args[0][0].text.value + assert "From completion." in called_text + + async def test_custom_text_fn_is_used(self): + scorer_fn, mock_evaluator = _make_scorer( + grade_result="6-8", text_fn=lambda state: "From custom fn." + ) + # completion is ignored when text_fn is supplied + state = _make_state(completion="From completion.", metadata={"target_grade": "6-8"}) + await scorer_fn(state, _make_target()) + called_text = mock_evaluator.evaluate.call_args[0][0].text.value + assert called_text == "From custom fn." + + async def test_text_fn_returning_none_skips(self): + scorer_fn, mock_evaluator = _make_scorer(text_fn=lambda state: None) + state = _make_state(completion="ignored", metadata={"target_grade": "6-8"}) + assert await scorer_fn(state, _make_target()) is None + mock_evaluator.evaluate.assert_not_called() + + async def test_text_fn_returning_blank_skips(self): + scorer_fn, mock_evaluator = _make_scorer(text_fn=lambda state: " \n ") + state = _make_state(completion="ignored", metadata={"target_grade": "6-8"}) + assert await scorer_fn(state, _make_target()) is None + mock_evaluator.evaluate.assert_not_called() + + async def test_text_fn_receives_state(self): + seen = {} + + def capture(state): + seen["state"] = state + return "text" + + scorer_fn, _ = _make_scorer(grade_result="6-8", text_fn=capture) + state = _make_state(completion="x", metadata={"target_grade": "6-8"}) + await scorer_fn(state, _make_target()) + assert seen["state"] is state + + +# ── Integration tests (mockllm/model) ──────────────────────────────────────── + + +class TestGlaScorerIntegration: + """End-to-end tests using Inspect's built-in mockllm/model provider. + + These tests validate that gla_scorer satisfies the Inspect Scorer protocol + and wires correctly through eval() — without making any real LLM calls. + The GLA evaluator itself is still mocked at the SDK boundary. + + .. note:: + These tests are synchronous (``def``, not ``async def``) because + ``inspect_ai.eval()`` calls ``anyio.run()`` internally to start its own + event loop. Using ``async def`` under ``pytest-asyncio asyncio_mode=auto`` + would start a second event loop, causing an anyio ``ScopeMismatch`` error. + """ + + def test_scorer_wires_through_eval(self): + from unittest.mock import AsyncMock, patch + + from inspect_ai import Task, eval + from inspect_ai.dataset import Sample + from inspect_ai.solver import generate + + mock_evaluator = MagicMock() + mock_evaluator.evaluate = AsyncMock(return_value=_make_gla_result("6-8")) + + with patch( + "learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator", + return_value=mock_evaluator, + ): + scorer = gla_scorer() + + task = Task( + dataset=[ + Sample( + input="Write a short paragraph for 6th graders.", + metadata={"target_grade": "6-8"}, + ) + ], + solver=[generate()], + scorer=scorer, + ) + + log = eval(task, model="mockllm/model")[0] + + assert log.status == "success" + sample = log.samples[0] + assert sample.score is not None + assert sample.score.value == CORRECT + + def test_skipped_sample_is_omitted_from_scores(self): + from unittest.mock import AsyncMock, patch + + from inspect_ai import Task, eval + from inspect_ai.dataset import Sample + from inspect_ai.solver import generate + + mock_evaluator = MagicMock() + mock_evaluator.evaluate = AsyncMock(return_value=_make_gla_result("6-8")) + + with patch( + "learning_commons_inspect_scorers.gla.GradeLevelAppropriatenessEvaluator", + return_value=mock_evaluator, + ): + scorer = gla_scorer() + + # Sample has no target_grade — gla_scorer returns None, so it must NOT + # contribute a score (no CORRECT/INCORRECT, no NaN to poison the mean). + task = Task( + dataset=[Sample(input="Write something.", metadata={})], + solver=[generate()], + scorer=scorer, + ) + + log = eval(task, model="mockllm/model")[0] + assert log.status == "success" + sample = log.samples[0] + assert (sample.scores or {}).get("gla_scorer") is None diff --git a/release-please-config.json b/release-please-config.json index a0cd7202..76c8aee4 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -63,6 +63,14 @@ "release-type": "node", "changelog-path": "CHANGELOG.md", "component": "sdks-typescript" + }, + "integrations/inspect-python": { + "release-type": "python", + "changelog-path": "CHANGELOG.md", + "component": "integrations-inspect-python", + "release-as": "0.1.0", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true } } } From 6bc1c9615da19e9dedbc49174f826b467e9f427e Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Mon, 22 Jun 2026 21:04:23 -0700 Subject: [PATCH 8/8] ci: add test + publish workflows for the Inspect integration package --- .github/workflows/publish-inspect-python.yml | 59 ++++++++++++++++++ .github/workflows/test-inspect-python.yml | 63 ++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 .github/workflows/publish-inspect-python.yml create mode 100644 .github/workflows/test-inspect-python.yml diff --git a/.github/workflows/publish-inspect-python.yml b/.github/workflows/publish-inspect-python.yml new file mode 100644 index 00000000..b709f776 --- /dev/null +++ b/.github/workflows/publish-inspect-python.yml @@ -0,0 +1,59 @@ +# ─── Before the first publish (integrations-inspect-python-v0.1.0) ─────────── +# +# 1. Tighten the SDK dependency. integrations/inspect-python/pyproject.toml pins +# `learning-commons-evaluators>=0.2.0`, but the integration requires the +# LLMGeneratorProtocol / llm_provider support that ships in SDK 0.3.0. Raise +# the floor to `>=0.3.0` once 0.3.0 is on PyPI — and do not publish this +# package before then, or installs resolve a SDK without llm_provider. +# +# 2. Remove `release-as: "0.1.0"` from the integrations/inspect-python block in +# release-please-config.json IMMEDIATELY after 0.1.0 lands on PyPI. It is a +# sticky pin: every later release-please PR stays at 0.1.0 until removed, and +# `skip-existing: true` below silently hides the duplicate-version rejection. +# +# ───────────────────────────────────────────────────────────────────────────── + +name: Publish Inspect integration + +on: + release: + types: [published] + workflow_dispatch: + +jobs: + publish: + if: github.event_name == 'workflow_dispatch' || startsWith(github.event.release.tag_name, 'integrations-inspect-python-v') + runs-on: ubuntu-latest + environment: pypi + permissions: + contents: read + id-token: write + defaults: + run: + working-directory: integrations/inspect-python + concurrency: + group: publish-inspect-python + cancel-in-progress: false + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@v6 + + - name: 🐍 Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: 📥 Install build tooling + run: python -m pip install --upgrade build twine + + - name: 📦 Build sdist + wheel + run: python -m build + + - name: 🧪 Validate distribution metadata + run: python -m twine check dist/* + + - name: 🚀 Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: integrations/inspect-python/dist + skip-existing: true diff --git a/.github/workflows/test-inspect-python.yml b/.github/workflows/test-inspect-python.yml new file mode 100644 index 00000000..c2071b5b --- /dev/null +++ b/.github/workflows/test-inspect-python.yml @@ -0,0 +1,63 @@ +name: 🔍 Test Inspect integration + +# Runs only when the integration package itself changes, or when the Python SDK +# it depends on changes (a SDK change — including a version bump — can break the +# integration, so we re-verify here too). +on: + push: + branches: + - main + paths: + - "integrations/inspect-python/**" + - "sdks/python/**" + - "sdks/settings/**" + - ".github/workflows/test-inspect-python.yml" + pull_request: + paths: + - "integrations/inspect-python/**" + - "sdks/python/**" + - "sdks/settings/**" + - ".github/workflows/test-inspect-python.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + name: Verify (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Install in-repo SDK + integration package + # Install the SDK from source first: the integration relies on + # LLMGeneratorProtocol (SDK >=0.3.0), which is not yet on PyPI. The + # editable install satisfies the >=0.2.0 floor while providing the + # unreleased protocol code, so CI tests against current monorepo state. + run: | + python -m pip install --upgrade pip + pip install -e ./sdks/python + pip install -e "./integrations/inspect-python[dev]" + + - name: Lint + working-directory: integrations/inspect-python + run: | + python -m ruff check . + python -m ruff format --check . + + - name: Test + working-directory: integrations/inspect-python + run: python -m pytest