feat: add Organization evaluator (early access)

evals/literacy/qualitative-text-complexity/organizational-structure/config.json

@@ -0,0 +1,105 @@
+{
+  "$schema": "../../../_schemas/config.schema.json",
+  "evaluator": {
+    "id": "literacy.gla.organizational_structure",
+    "name": "Organizational Structure Dimension Text Complexity Evaluator",
+    "description": "Evaluates the Organizational Structure dimension of qualitative text complexity for grades 3-12 reading assessment, producing a 4-level rubric rating with structured pedagogical detail.",
+    "supported_grades": [
+      "3",
+      "4",
+      "5",
+      "6",
+      "7",
+      "8",
+      "9",
+      "10",
+      "11",
+      "12"
+    ]
+  },
+  "input_schema": { "$ref": "input_schema.json" },
+  "preprocessing": [
+    {
+      "id": "fk_score",
+      "type": "computation",
+      "kind": "flesch_kincaid_grade",
+      "description": "Compute the Flesch-Kincaid Grade Level for the input text and bind it to {fk_score} in the prompt.",
+      "input": "text",
+      "output": "fk_score",
+      "implementation": {
+        "python": {
+          "library": "textstat",
+          "function": "flesch_kincaid_grade",
+          "post_transform": {
+            "type": "round",
+            "precision": 2
+          }
+        },
+        "typescript": {
+          "library": "text-readability",
+          "function": "fleschKincaidGrade",
+          "post_transform": {
+            "type": "round",
+            "precision": 2
+          }
+        }
+      }
+    }
+  ],
+  "steps": [
+    {
+      "id": "evaluate_organizational_structure",
+      "type": "llm",
+      "description": "Single-call LLM step that produces the EvaluatorOutput JSON.",
+      "prompt": {
+        "messages": [
+          {
+            "role": "system",
+            "source_path": "system.txt",
+            "sha256": "cf8c0dbee1721259b22835157daeb4ef9078542d55a3bd961fd697fbfb5e075d"
+          },
+          {
+            "role": "user",
+            "source_path": "user.txt",
+            "sha256": "cd8e6347db1a55d104e34436f8f66e833bd6583645d4786a554aaefdd26479b2"
+          }
+        ],
+        "placeholders": {
+          "text": {
+            "required": true,
+            "source": "input"
+          },
+          "grade_level": {
+            "required": true,
+            "source": "input"
+          },
+          "fk_score": {
+            "required": true,
+            "source": "preprocessing.fk_score"
+          }
+        }
+      },
+      "model": {
+        "provider": "google",
+        "name": "gemini-3-flash-preview",
+        "alias": "organizational-structure-evaluator-default"
+      },
+      "generation": {
+        "temperature": 1
+      },
+      "parser": {
+        "kind": "structured_output"
+      }
+    }
+  ],
+  "output_schema": {
+    "$ref": "output_schema.json"
+  },
+  "fixtures": {
+    "path": "fixtures.json",
+    "tolerance": {
+      "allow_adjacent_levels": true,
+      "notes": "If allow_adjacent_levels is true, predictions within one rubric step of the expected label count as a pass."
+    }
+  }
+}

evals/literacy/qualitative-text-complexity/organizational-structure/example_notebook.ipynb

@@ -0,0 +1,78 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": "# Organizational Structure Evaluator\n\n**The Organizational Structure Evaluator** assesses how demanding a text's structure is for students at a given grade level. It identifies the organizational pattern(s) a text uses to arrange ideas (chronological, sequential, cause-and-effect, compare-and-contrast, problem-solution, or more intricate discipline-specific structures) and how explicitly it signals the connections between them, then evaluates those structural demands against what students at the target grade are expected to navigate. When you run a passage through the evaluator, it returns a structured output with the following top-level fields:\n\n* **complexity_score**: The Organizational Structure complexity level (Slightly to Exceedingly Complex).\n* **reasoning**: A synthesis of why the text fits the chosen complexity level.\n* **details**: Keyed instructional detail, containing:\n    * **detailed_summary**: Individual organizational complexity factors that drive the rating, with descriptions and their effect on the dimension.\n    * **adjustment_and_scaffolding**: Scaffolding strategies to make the text's structure accessible at the target grade.\n    * **recommended_use_cases**: Additional instructional opportunities for using the text's structure.\n\nThis gives you a clear signal about the organizational demands of a passage, helping ensure AI-generated content is appropriate for the target grade."
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": "%pip install -qU langchain-google-genai langchain pydantic textstat typing_extensions"
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": "import getpass\nimport os\nfrom dotenv import load_dotenv\nimport json\nimport hashlib\nfrom pathlib import Path\nfrom typing import List\nfrom enum import Enum\nfrom langchain_google_genai import ChatGoogleGenerativeAI\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.output_parsers import JsonOutputParser\nfrom pydantic import BaseModel, Field\nimport textstat\nfrom IPython.display import Markdown\nimport pprint as pp"
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": "# Check for the API key\nload_dotenv()\n\nif \"GOOGLE_API_KEY\" not in os.environ:\n    os.environ[\"GOOGLE_API_KEY\"] = getpass.getpass(\"Enter your Google AI API key: \")"
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": "ASSETS_DIR = Path(\".\")\nconfig_path = ASSETS_DIR / \"config.json\"\nwith open(config_path) as f:\n    CONFIG = json.load(f)\n\n# Load standalone schema files (config.json references them via $ref by path).\nwith open(ASSETS_DIR / \"input_schema.json\") as f:\n    INPUT_SCHEMA = json.load(f)\nwith open(ASSETS_DIR / \"output_schema.json\") as f:\n    OUTPUT_SCHEMA = json.load(f)\n\n# Load every prompt message declared in config (system, user, ...). Each\n# message has {role, source_path, sha256}. We verify each file's sha256\n# matches the declared hash -- drift tripwire #1, applied to every prompt\n# regardless of role. CI should promote a mismatch to a hard failure.\nPROMPT_MESSAGES = []  # list of (role, text) tuples, preserving config order\nfor msg_spec in CONFIG[\"steps\"][0][\"prompt\"][\"messages\"]:\n    role = msg_spec[\"role\"]\n    path = ASSETS_DIR / msg_spec[\"source_path\"]\n    text = path.read_text()\n    actual_sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()\n    declared_sha = msg_spec[\"sha256\"]\n    assert actual_sha == declared_sha, (\n        f\"prompt drift detected for role={role!r} ({msg_spec['source_path']}): \"\n        f\"declared {declared_sha[:12]}..., actual on disk {actual_sha[:12]}...\"\n    )\n    PROMPT_MESSAGES.append((role, text))\n\nprint(\n    f\"Loaded {CONFIG['evaluator']['id']} \"\n    f\"from {ASSETS_DIR.resolve()}\"\n)\nprint(f\"  model:       {CONFIG['steps'][0]['model']['name']}\")\nprint(f\"  temperature: {CONFIG['steps'][0]['generation']['temperature']}\")\nprint(f\"  prompts:\")\nfor msg_spec, (role, text) in zip(CONFIG[\"steps\"][0][\"prompt\"][\"messages\"], PROMPT_MESSAGES):\n    sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()[:12]\n    print(f\"    {role:>6}  {msg_spec['source_path']:<14} ({len(text):>5} chars, sha {sha})\")"
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": "# -------------------------------------------------------------------------\n# FK score helper (declared as a preprocessing step in CONFIG['preprocessing'])\n# -------------------------------------------------------------------------\n_FK_PRE = next(p for p in CONFIG[\"preprocessing\"] if p[\"id\"] == \"fk_score\")\n_FK_IMPL = _FK_PRE[\"implementation\"][\"python\"]\n_FK_LIB = _FK_IMPL[\"library\"]\n_FK_FN = _FK_IMPL[\"function\"]\n_FK_TRANSFORM = _FK_IMPL[\"post_transform\"]\n\nif _FK_LIB != \"textstat\":\n    raise ValueError(f\"unsupported fk library in config: {_FK_LIB!r}\")\n\n\ndef calculate_fk_score(text) -> float:\n    \"\"\"Compute Flesch-Kincaid Grade Level per CONFIG['preprocessing'].\"\"\"\n    fn = getattr(textstat, _FK_FN)\n    value = fn(text)\n    if _FK_TRANSFORM[\"type\"] == \"round\":\n        value = round(value, _FK_TRANSFORM[\"precision\"])\n    else:\n        raise ValueError(f\"unsupported post_transform type: {_FK_TRANSFORM['type']!r}\")\n    return value\n\n\n# -------------------------------------------------------------------------\n# Evaluator function: model / prompt / parser config all read from CONFIG\n# -------------------------------------------------------------------------\n_STEP = CONFIG[\"steps\"][0]  # single-step evaluator today. Extensible to multi-step evaluators.\n\ndef evaluate_text_complexity(text: str, grade_level: int):\n    \"\"\"\n    Evaluate the Organizational Structure-dimension complexity of a text using the canonical\n    config in evals/literacy/qualitative-text-complexity/organizational-structure/config.json + system.txt + user.txt.\n\n    Returns a dict with full I/O trace fields:\n      - rendered_prompt:  the actual list of messages sent to the model\n                          (input-side trace).\n      - raw_output:       the AIMessage object returned by the LLM\n                          (preserves response_metadata, usage_metadata).\n      - raw_text:         just the string content of the AIMessage.\n      - formatted_output: the parsed dict matching OUTPUT_SCHEMA.\n      - usage:            token-usage metadata if the provider returned it.\n\n    The LLM is invoked ONCE; include_raw=True returns both the raw AIMessage\n    and the parsed output without a second call.\n    \"\"\"\n    # 1. Structured output -- parser.kind == \"structured_output\" uses the model's\n    #    native output enforcement. OUTPUT_SCHEMA is loaded from output_schema.json,\n    #    the standalone source of truth. include_raw=True preserves the AIMessage\n    #    for tracing alongside the parsed result.\n    llm = ChatGoogleGenerativeAI(\n        model=_STEP[\"model\"][\"name\"],\n        temperature=_STEP[\"generation\"][\"temperature\"],\n    )\n    structured_llm = llm.with_structured_output(OUTPUT_SCHEMA, include_raw=True)\n\n    # 2. Prompt template -- every message's content was loaded from disk\n    #    and verified against config in the loader cell. We just feed the\n    #    (role, text) tuples straight into ChatPromptTemplate.\n    prompt_template = ChatPromptTemplate.from_messages(PROMPT_MESSAGES)\n\n    try:\n        # Step A: Calculate FK Score\n        fk_score = calculate_fk_score(text)\n        print(f\"Calculated Flesch-Kincaid Score: {fk_score}\")\n\n        inputs = {\"text\": text, \"grade_level\": grade_level, \"fk_score\": fk_score}\n\n        # Step B: Render the prompt up-front so we can return exactly what\n        #         was sent to the model (input-side trace).\n        rendered_messages = prompt_template.format_messages(**inputs)\n\n        # Step C: Single LLM call -> raw AIMessage + parsed output dict.\n        #         No second LLM call.\n        raw = structured_llm.invoke(rendered_messages)\n\n        if raw.get(\"parsing_error\"):\n            raise ValueError(f\"structured output parsing failed: {raw['parsing_error']}\")\n\n        # Step D: Return the full trace dict.\n        return {\n            \"rendered_prompt\": [m.model_dump() for m in rendered_messages],\n            \"raw_output\":       raw[\"raw\"],\n            \"raw_text\":         raw[\"raw\"].content,\n            \"formatted_output\": raw[\"parsed\"],\n            \"usage\":            getattr(raw[\"raw\"], \"usage_metadata\", None),\n        }\n    except Exception as e:\n        return f\"Error evaluating text: {e}\""
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": "sample_text = \"\"\"For centuries, the cod off Newfoundland seemed endless — boats returned so full that fishermen said you could walk across the water on their backs. Today the fishery is closed. The story of how that happened is not a simple one. New technology let trawlers catch more fish faster than ever before, scooping up entire schools in a single haul. But the fish were also vanishing for reasons the boats couldn't see: warming waters shifted the cod's feeding grounds, and the removal of so many large fish left populations too young to rebuild. Government scientists had warned of decline for years. Their estimates, it turned out, had been based on the catch reports of the very fleets with the most to lose from a shutdown. By the time the cod were counted accurately, there were almost none left to count.\"\"\"\n\nresult = evaluate_text_complexity(text=sample_text, grade_level=5)\npp.pprint(result[\"formatted_output\"] if isinstance(result, dict) else result)"
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": "# I/O trace breakdown -- this is what the SDK engineer will replicate in TS.\nprint(\"=\" * 60)\nprint(\"RENDERED PROMPT (input sent to the LLM)\")\nprint(\"=\" * 60)\npp.pprint(result[\"rendered_prompt\"] if isinstance(result, dict) else result)\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"RAW LLM TEXT (model's verbatim output)\")\nprint(\"=\" * 60)\nprint(result[\"raw_text\"] if isinstance(result, dict) else result)\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"PARSED OUTPUT (output_schema)\")\nprint(\"=\" * 60)\npp.pprint(result[\"formatted_output\"] if isinstance(result, dict) else result)\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"USAGE METADATA\")\nprint(\"=\" * 60)\npp.pprint(result[\"usage\"] if isinstance(result, dict) else result)"
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": "# -------------------------------------------------------------------------\n# Sniff-test runner: load fixtures.json and check predictions against expected\n# -------------------------------------------------------------------------\n# Fixtures live next to config.json + system.txt + user.txt and follow the\n# schema declared in CONFIG['fixtures']['schema']. Each case has:\n#   - id, description (optional)\n#   - input: {text, grade_level}                  -- runtime evaluator inputs\n#   - expected: {complexity_level}                -- ground-truth label from rubric\n#\n# Note: the fixture key 'complexity_level' maps to the runtime output's\n# 'complexity_score' field. We test that single value only -- the model's\n# free-text 'reasoning' field is non-deterministic across runs.\n\nfixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"path\"]\nwith open(fixtures_path) as f:\n    fixtures = json.load(f)\nprint(f\"Loaded {len(fixtures)} fixtures from {fixtures_path.name}\\n\")\n\n# Adjacency tolerance per CONFIG['fixtures']['tolerance'].\n# Derive rubric order from OUTPUT_SCHEMA.\n_RUBRIC_ORDER = OUTPUT_SCHEMA[\"properties\"][\"complexity_score\"][\"enum\"]\n_ALLOW_ADJ = bool(CONFIG[\"fixtures\"][\"tolerance\"].get(\"allow_adjacent_levels\", False))\n\ndef _score_outcome(predicted: str, expected: str):\n    \"\"\"Return ('exact' | 'adjacent' | 'fail', distance_or_None).\"\"\"\n    if predicted == expected:\n        return \"exact\", 0\n    if _ALLOW_ADJ and predicted in _RUBRIC_ORDER and expected in _RUBRIC_ORDER:\n        d = abs(_RUBRIC_ORDER.index(predicted) - _RUBRIC_ORDER.index(expected))\n        if d == 1:\n            return \"adjacent\", d\n    return \"fail\", None\n\n# Run each fixture, accumulate results\nresults = []\nfor fx in fixtures:\n    expected = fx[\"expected\"][\"complexity_score\"]\n    out = evaluate_text_complexity(\n        text=fx[\"input\"][\"text\"],\n        grade_level=fx[\"input\"][\"grade_level\"],\n    )\n    if isinstance(out, str):  # error path\n        results.append({\"id\": fx[\"id\"], \"status\": \"error\", \"predicted\": None, \"expected\": expected, \"error\": out})\n        continue\n    predicted = out[\"formatted_output\"][\"complexity_score\"]\n    status, _ = _score_outcome(predicted, expected)\n    results.append({\n        \"id\": fx[\"id\"], \"status\": status,\n        \"predicted\": predicted, \"expected\": expected,\n        \"description\": fx.get(\"description\", \"\"),\n    })\n\n# Per-case report\nprint(\"\\n\" + \"=\" * 78)\nprint(f\"{'ID':>5}  {'STATUS':<8}  {'PREDICTED':<22}  {'EXPECTED':<22}  DESCRIPTION\")\nprint(\"=\" * 78)\nfor r in results:\n    icon = {\"exact\": \"PASS\", \"adjacent\": \"PASS*\", \"fail\": \"FAIL\", \"error\": \"ERR\"}[r[\"status\"]]\n    print(f\"{r['id']:>5}  {icon:<8}  {(r['predicted'] or '-'):<22}  {r['expected']:<22}  {r.get('description','')[:25]}\")\n\n# Summary\nn_total = len(results)\nn_exact = sum(1 for r in results if r[\"status\"] == \"exact\")\nn_adj   = sum(1 for r in results if r[\"status\"] == \"adjacent\")\nn_fail  = sum(1 for r in results if r[\"status\"] == \"fail\")\nn_err   = sum(1 for r in results if r[\"status\"] == \"error\")\nprint(\"=\" * 78)\nprint(f\"Summary: {n_exact} exact, {n_adj} adjacent (tolerated), {n_fail} fail, {n_err} error  --  total {n_total}\")\nif _ALLOW_ADJ:\n    print(\"(Adjacency tolerance ON: predictions within +/-1 rubric step of the expected label count as PASS*.)\")"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}