diff --git a/examples/agentops_governance_events.py b/examples/agentops_governance_events.py new file mode 100644 index 0000000..5c9d128 --- /dev/null +++ b/examples/agentops_governance_events.py @@ -0,0 +1,103 @@ +"""Example: Export TealTiger governance decisions to AgentOps. + +Governance decisions appear in the AgentOps session timeline alongside +your agent's LLM calls, tool invocations, and errors. + +Requirements: + pip install tealtiger agentops + +Set environment variables: + AGENTOPS_API_KEY=your-key + OPENAI_API_KEY=sk-... +""" + +import agentops +from tealtiger.integrations.agentops import AgentOpsGovernanceReporter + +# Initialize AgentOps +agentops.init() + +# Create the governance reporter +reporter = AgentOpsGovernanceReporter() + +# --- Simulate governance decisions --- + +# Tool call allowed +reporter.report({ + "action": "ALLOW", + "correlation_id": "550e8400-e29b-41d4-a716-446655440000", + "agent_id": "research-bot", + "tool_slug": "GITHUB_GET_REPOS", + "toolkit_slug": "github", + "reason": "Tool in allowlist", + "reason_codes": ["POLICY_ALLOW"], + "risk_score": 0, + "evaluation_time_ms": 0.38, + "mode": "ENFORCE", + "cost_tracked": 0.001, + "cumulative_cost": 0.015, + "pii_detected": [], +}) + +# Tool call denied (appears as error in AgentOps timeline) +reporter.report({ + "action": "DENY", + "correlation_id": "660e8400-e29b-41d4-a716-446655440001", + "agent_id": "research-bot", + "tool_slug": "GMAIL_SEND_EMAIL", + "toolkit_slug": "gmail", + "reason": "Tool 'GMAIL_SEND_EMAIL' not in allowlist for agent 'research-bot'", + "reason_codes": ["TOOL_NOT_ALLOWED"], + "risk_score": 0.9, + "evaluation_time_ms": 0.52, + "mode": "ENFORCE", + "cost_tracked": 0.0, + "cumulative_cost": 0.015, + "pii_detected": [], +}) + +# PII detected (monitor mode — logged but not blocked) +reporter.report({ + "action": "MONITOR", + "correlation_id": "770e8400-e29b-41d4-a716-446655440002", + "agent_id": "research-bot", + "tool_slug": "SLACK_SEND_MESSAGE", + "toolkit_slug": "slack", + "reason": "PII detected (monitor mode)", + "reason_codes": ["PII_DETECTED"], + "risk_score": 0.6, + "evaluation_time_ms": 1.1, + "mode": "MONITOR", + "cost_tracked": 0.002, + "cumulative_cost": 0.017, + "pii_detected": [{"type": "email", "start": 12, "end": 30}], +}) + +# End session +agentops.end_session("Success") + +print(f"Reported {reporter.allow_count} allows, {reporter.deny_count} denials") +print("Check your AgentOps dashboard to see governance events in the session timeline.") +print() +print("In AgentOps you'll see:") +print(" - ALLOW: ActionEvent (governance:allow)") +print(" - DENY: ErrorEvent (governance:deny) — highlighted in red") +print(" - MONITOR: ActionEvent (governance:monitor)") + + +# --- Using with TealTiger observe() --- +# (Uncomment when running with real keys) + +# from tealtiger import observe +# from openai import OpenAI +# +# client = observe( +# OpenAI(), +# agent_id="my-agent", +# on_decision=reporter.report, +# ) +# +# response = client.chat.completions.create( +# model="gpt-4o-mini", +# messages=[{"role": "user", "content": "Hello!"}] +# ) diff --git a/src/tealtiger/integrations/__init__.py b/src/tealtiger/integrations/__init__.py index 7eeac76..80225a1 100644 --- a/src/tealtiger/integrations/__init__.py +++ b/src/tealtiger/integrations/__init__.py @@ -1,5 +1,6 @@ """TealTiger integrations with external observability and monitoring platforms.""" from tealtiger.integrations.langfuse import LangfuseGovernanceExporter +from tealtiger.integrations.agentops import AgentOpsGovernanceReporter -__all__ = ["LangfuseGovernanceExporter"] +__all__ = ["LangfuseGovernanceExporter", "AgentOpsGovernanceReporter"] diff --git a/src/tealtiger/integrations/agentops.py b/src/tealtiger/integrations/agentops.py new file mode 100644 index 0000000..a8501db --- /dev/null +++ b/src/tealtiger/integrations/agentops.py @@ -0,0 +1,158 @@ +"""TealTiger → AgentOps governance event exporter. + +Exports TealTiger governance decisions as AgentOps ActionEvents, enabling teams +to see governance enforcement in the AgentOps session timeline alongside their +agent telemetry. + +Usage: + import agentops + from tealtiger.integrations.agentops import AgentOpsGovernanceReporter + + agentops.init(api_key="your-key") + reporter = AgentOpsGovernanceReporter() + + # Use as on_decision callback + client = observe(OpenAI(), on_decision=reporter.report) + + # Or manually report a decision + reporter.report(decision) +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +try: + from agentops import ActionEvent, ErrorEvent + import agentops +except ImportError: + raise ImportError( + "agentops is required for this integration. " + "Install it with: pip install agentops" + ) + + +class AgentOpsGovernanceReporter: + """Export TealTiger governance decisions as AgentOps events. + + Each governance decision becomes an AgentOps ActionEvent (or ErrorEvent for + denials) visible in the session timeline. + + Decision mapping: + - ALLOW → ActionEvent with action_type="governance:allow" + - DENY → ErrorEvent with error_type="governance:deny" + - MONITOR → ActionEvent with action_type="governance:monitor" + - REFER → ActionEvent with action_type="governance:refer" + + Args: + session: An AgentOps session. If None, uses the default active session. + include_metadata: Whether to include full governance metadata in events. + """ + + def __init__( + self, + session=None, + include_metadata: bool = True, + ): + self._session = session + self._include_metadata = include_metadata + self._decisions: List[Dict[str, Any]] = [] + + def report(self, decision: Dict[str, Any], **kwargs) -> None: + """Report a governance decision as an AgentOps event. + + This method is designed to be used as the `on_decision` callback + for TealTiger's observe() or TealEngine. + + Args: + decision: A TealTiger GovernanceDecision dict containing at minimum: + - action: "ALLOW", "DENY", "MONITOR", or "REFER" + - correlation_id: UUID v4 for the decision + - Optional: reason, reason_codes, risk_score, evaluation_time_ms, + agent_id, tool_slug, pii_detected, cost_tracked, etc. + """ + action = decision.get("action", "ALLOW").upper() + self._decisions.append(decision) + + # Build event params + params = self._build_params(decision) + + if action == "DENY": + self._report_deny(decision, params) + else: + self._report_action(decision, params, action) + + def _build_params(self, decision: Dict[str, Any]) -> Dict[str, Any]: + """Build event parameters from a governance decision.""" + params: Dict[str, Any] = { + "governance_action": decision.get("action", "ALLOW"), + "correlation_id": decision.get("correlation_id", ""), + "agent_id": decision.get("agent_id", ""), + } + + if self._include_metadata: + params.update({ + "reason": decision.get("reason", ""), + "reason_codes": decision.get("reason_codes", []), + "risk_score": decision.get("risk_score", 0), + "evaluation_time_ms": decision.get("evaluation_time_ms", 0), + "mode": decision.get("mode", "OBSERVE"), + "cost_tracked": decision.get("cost_tracked", 0), + "cumulative_cost": decision.get("cumulative_cost", 0), + "pii_count": len(decision.get("pii_detected", [])), + }) + + if "tool_slug" in decision: + params["tool_slug"] = decision["tool_slug"] + if "toolkit_slug" in decision: + params["toolkit_slug"] = decision["toolkit_slug"] + if "policy_digest" in decision: + params["policy_digest"] = decision["policy_digest"] + + return params + + def _report_action( + self, decision: Dict[str, Any], params: Dict[str, Any], action: str + ) -> None: + """Report an ALLOW/MONITOR/REFER decision as an ActionEvent.""" + action_type = f"governance:{action.lower()}" + + event = ActionEvent( + action_type=action_type, + params=params, + returns=decision.get("reason", f"Governance {action}"), + ) + + session = self._session or agentops.get_session() + if session: + session.record(event) + + def _report_deny( + self, decision: Dict[str, Any], params: Dict[str, Any] + ) -> None: + """Report a DENY decision as an ErrorEvent.""" + reason = decision.get("reason", "Governance DENY") + + event = ErrorEvent( + error_type="governance:deny", + details=reason, + params=params, + ) + + session = self._session or agentops.get_session() + if session: + session.record(event) + + def get_decisions(self) -> List[Dict[str, Any]]: + """Return all recorded governance decisions.""" + return self._decisions.copy() + + @property + def deny_count(self) -> int: + """Count of DENY decisions reported.""" + return sum(1 for d in self._decisions if d.get("action", "").upper() == "DENY") + + @property + def allow_count(self) -> int: + """Count of ALLOW decisions reported.""" + return sum(1 for d in self._decisions if d.get("action", "").upper() == "ALLOW") diff --git a/tests/test_agentops_integration.py b/tests/test_agentops_integration.py new file mode 100644 index 0000000..75ae000 --- /dev/null +++ b/tests/test_agentops_integration.py @@ -0,0 +1,103 @@ +"""Tests for TealTiger → AgentOps governance event export.""" + +import pytest +from unittest.mock import MagicMock, patch + + +@pytest.fixture +def mock_agentops(): + """Mock agentops module.""" + mock = MagicMock() + mock_session = MagicMock() + mock.get_session.return_value = mock_session + return mock, mock_session + + +@pytest.fixture +def reporter(mock_agentops): + """Create an AgentOpsGovernanceReporter with mocked agentops.""" + mock, mock_session = mock_agentops + with patch.dict("sys.modules", { + "agentops": mock, + }): + # Re-import to pick up the mock + from tealtiger.integrations.agentops import AgentOpsGovernanceReporter + return AgentOpsGovernanceReporter(session=mock_session) + + +def test_allow_decision_creates_action_event(reporter, mock_agentops): + """ALLOW decisions should create ActionEvents.""" + _, mock_session = mock_agentops + + decision = { + "action": "ALLOW", + "correlation_id": "test-123", + "agent_id": "coder", + "tool_slug": "GITHUB_GET_REPOS", + "reason": "Policy allows", + "reason_codes": ["POLICY_ALLOW"], + "risk_score": 0, + "evaluation_time_ms": 0.42, + } + + reporter.report(decision) + + mock_session.record.assert_called_once() + assert reporter.allow_count == 1 + assert reporter.deny_count == 0 + + +def test_deny_decision_creates_error_event(reporter, mock_agentops): + """DENY decisions should create ErrorEvents.""" + _, mock_session = mock_agentops + + decision = { + "action": "DENY", + "correlation_id": "test-456", + "agent_id": "coder", + "tool_slug": "GMAIL_SEND_EMAIL", + "reason": "Tool not in allowlist", + "reason_codes": ["TOOL_NOT_ALLOWED"], + "risk_score": 0.9, + } + + reporter.report(decision) + + mock_session.record.assert_called_once() + assert reporter.deny_count == 1 + assert reporter.allow_count == 0 + + +def test_decisions_are_tracked(reporter, mock_agentops): + """All reported decisions should be stored.""" + reporter.report({"action": "ALLOW", "correlation_id": "1"}) + reporter.report({"action": "DENY", "correlation_id": "2"}) + reporter.report({"action": "MONITOR", "correlation_id": "3"}) + + decisions = reporter.get_decisions() + assert len(decisions) == 3 + assert decisions[0]["action"] == "ALLOW" + assert decisions[1]["action"] == "DENY" + assert decisions[2]["action"] == "MONITOR" + + +def test_metadata_included_in_params(reporter, mock_agentops): + """Event params should include governance metadata.""" + _, mock_session = mock_agentops + + decision = { + "action": "ALLOW", + "correlation_id": "meta-test", + "agent_id": "researcher", + "tool_slug": "SEARCH_DB", + "toolkit_slug": "database", + "risk_score": 0.3, + "evaluation_time_ms": 1.5, + "cost_tracked": 0.002, + "cumulative_cost": 0.15, + "pii_detected": [{"type": "email"}], + "policy_digest": "sha256:xyz", + } + + reporter.report(decision) + mock_session.record.assert_called_once()