Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions packages/core/compiler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -553,6 +553,48 @@ describe("CompiledStep.executionPlan carries parallel execution through compilat
});
});

// =============================================================================
// pre_output gate on_fail survives compilation (issue #72)
// =============================================================================

describe("pre_output gate carries on_fail through compilation", () => {
const actions: OnFailAction[] = ["retry", "escalate", "skip", "abort", "revise"];

// A step with no verification, so qualityGates[0] is the pre_output gate.
function specWithGate(onFail?: OnFailAction): LogicSpec {
return {
...makeSpec({ scored: { description: "Scored step" } }),
quality_gates: {
pre_output: [
{
name: "confidence-check",
check: "{{ output.score >= 0.8 }}",
message: "Score too low",
...(onFail !== undefined ? { on_fail: onFail } : {}),
},
],
},
};
}

it.each(actions)("forwards on_fail: %s onto the failing pre_output gate result", (onFail) => {
const result = compileStep(specWithGate(onFail), "scored", makeCtx());
expect(result.qualityGates).toHaveLength(1);
expect(result.qualityGates[0]!({ score: 0.5 })).toEqual({
passed: false,
message: "Score too low",
onFail,
});
});

it("omits onFail when the pre_output gate has no authored on_fail", () => {
const result = compileStep(specWithGate(), "scored", makeCtx());
const failed = result.qualityGates[0]!({ score: 0.5 });
expect(failed).toEqual({ passed: false, message: "Score too low" });
expect(failed).not.toHaveProperty("onFail");
});
});

// =============================================================================
// Output Format in systemPromptSegment
// =============================================================================
Expand Down
22 changes: 18 additions & 4 deletions packages/core/compiler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import type {
ExecutionPlan,
JsonSchemaObject,
LogicSpec,
OnFailAction,
QualityGateValidator,
Reasoning,
RetryConfig,
Expand Down Expand Up @@ -255,14 +256,27 @@ function compileSelfReflection(
* Compile a gate check expression into a QualityGateValidator function.
* Uses the expression engine's evaluate() to evaluate the check expression
* with the step output injected as `{ output }` in the expression context.
*
* When the gate carries an authored on_fail recovery action, it is forwarded
* onto the failing result so a runtime can recover what action a failed gate
* should trigger without reaching back into the un-compiled spec. Omitted when
* the author did not specify one (the source field is optional).
*/
function compileGateValidator(checkExpression: string, message?: string): QualityGateValidator {
function compileGateValidator(
checkExpression: string,
message?: string,
onFail?: OnFailAction,
): QualityGateValidator {
return (output: unknown) => {
const result = evaluate(checkExpression, { output });
if (result) {
return { passed: true };
}
return { passed: false, ...(message ? { message } : {}) };
return {
passed: false,
...(message ? { message } : {}),
...(onFail !== undefined ? { onFail } : {}),
};
};
}

Expand Down Expand Up @@ -398,7 +412,7 @@ export function compileStep(
}
if (spec.quality_gates?.pre_output) {
for (const gate of spec.quality_gates.pre_output) {
gates.push(compileGateValidator(gate.check, gate.message));
gates.push(compileGateValidator(gate.check, gate.message, gate.on_fail));
}
}
return gates;
Expand Down Expand Up @@ -463,7 +477,7 @@ export function compileWorkflow(spec: LogicSpec, context: WorkflowContext): Comp
const globalGates: QualityGateValidator[] = [];
if (spec.quality_gates?.pre_output) {
for (const gate of spec.quality_gates.pre_output) {
globalGates.push(compileGateValidator(gate.check, gate.message));
globalGates.push(compileGateValidator(gate.check, gate.message, gate.on_fail));
}
}

Expand Down
2 changes: 2 additions & 0 deletions packages/core/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -640,6 +640,8 @@ export interface QualityGateResult {
passed: boolean;
/** Human-readable reason surfaced when the check fails */
message?: string;
/** Recovery action authored on the gate, surfaced when the gate fails */
onFail?: OnFailAction;
}

/** Runtime quality gate validator function (v1.1 Compiler) */
Expand Down