diff --git a/examples/openai_function_calling_example.py b/examples/observers/openai_function_calling_example.py similarity index 85% rename from examples/openai_function_calling_example.py rename to examples/observers/openai_function_calling_example.py index c338a73..f157682 100644 --- a/examples/openai_function_calling_example.py +++ b/examples/observers/openai_function_calling_example.py @@ -1,12 +1,6 @@ from observers.observers import wrap_openai -from observers.stores import DatasetsStore from openai import OpenAI -store = DatasetsStore( - repo_name="gpt-4o-function-calling-traces", - every=5, # sync every 5 minutes -) - openai_client = OpenAI() tools = [ @@ -42,7 +36,7 @@ ] -client = wrap_openai(openai_client, store=store) +client = wrap_openai(openai_client) response = client.chat.completions.create( model="gpt-4o", diff --git a/examples/vision_example.py b/examples/observers/openai_vision_example.py similarity index 78% rename from examples/vision_example.py rename to examples/observers/openai_vision_example.py index 2b11b82..73129ec 100644 --- a/examples/vision_example.py +++ b/examples/observers/openai_vision_example.py @@ -1,14 +1,8 @@ from observers.observers import wrap_openai -from observers.stores import DatasetsStore from openai import OpenAI -store = DatasetsStore( - repo_name="gpt-4o-mini-vision-traces", - every=5, # sync every 5 minutes -) - openai_client = OpenAI() -client = wrap_openai(openai_client, store=store) +client = wrap_openai(openai_client) response = client.chat.completions.create( model="gpt-4o-mini", diff --git a/src/observers/observers/base.py b/src/observers/observers/base.py index aa630ca..33aa7a2 100644 --- a/src/observers/observers/base.py +++ b/src/observers/observers/base.py @@ -7,12 +7,6 @@ from argilla import Argilla -@dataclass -class Message: - role: Literal["system", "user", "assistant", "function"] - content: str - - @dataclass class Record(ABC): """ diff --git a/src/observers/observers/models/openai.py b/src/observers/observers/models/openai.py index fb22d16..050391b 100644 --- a/src/observers/observers/models/openai.py +++ b/src/observers/observers/models/openai.py @@ -3,7 +3,9 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union -from observers.observers.base import Message, Record +from openai.types.chat.chat_completion_message_param import ChatCompletionMessageParam + +from observers.observers.base import Record from observers.stores.duckdb import DuckDBStore if TYPE_CHECKING: @@ -22,7 +24,7 @@ class OpenAIResponseRecord(Record): model: str = None timestamp: str = field(default_factory=lambda: datetime.datetime.now().isoformat()) - messages: List[Message] = None + messages: List[ChatCompletionMessageParam] = None assistant_message: Optional[str] = None completion_tokens: Optional[int] = None prompt_tokens: Optional[int] = None @@ -82,7 +84,7 @@ def duckdb_schema(self): id VARCHAR PRIMARY KEY, model VARCHAR, timestamp TIMESTAMP, - messages STRUCT(role VARCHAR, content VARCHAR)[], + messages JSON, assistant_message TEXT, completion_tokens INTEGER, prompt_tokens INTEGER, @@ -177,7 +179,14 @@ def table_name(self): @property def json_fields(self): - return ["tool_calls", "function_call", "tags", "properties", "raw_response"] + return [ + "tool_calls", + "function_call", + "tags", + "properties", + "raw_response", + "messages", + ] @property def image_fields(self): diff --git a/tests/integration/observers/test_observers_examples.py b/tests/integration/observers/test_observers_examples.py index 5e0e8f2..09d8c56 100644 --- a/tests/integration/observers/test_observers_examples.py +++ b/tests/integration/observers/test_observers_examples.py @@ -8,6 +8,21 @@ ChatCompletionMessage, ) from openai.types.chat.chat_completion import Choice, CompletionUsage +from openai.types.chat.chat_completion_content_part_input_audio_param import ( + InputAudio, +) +from openai.types.chat.chat_completion_content_part_param import ( + ChatCompletionContentPartImageParam, + ChatCompletionContentPartInputAudioParam, + ChatCompletionContentPartTextParam, +) +from openai.types.chat.chat_completion_message_param import ( + ChatCompletionAssistantMessageParam, + ChatCompletionFunctionMessageParam, + ChatCompletionSystemMessageParam, + ChatCompletionToolMessageParam, + ChatCompletionUserMessageParam, +) def get_example_files(): @@ -29,6 +44,24 @@ def mock_clients(): def get_fake_return(): return ChatCompletion( id=str(uuid.uuid4()), + messages=[ + ChatCompletionSystemMessageParam(content=""), + ChatCompletionUserMessageParam( + content=ChatCompletionContentPartTextParam(text="test") + ), + ChatCompletionAssistantMessageParam(content=""), + ChatCompletionUserMessageParam( + content=ChatCompletionContentPartImageParam(image_url="image") + ), + ChatCompletionAssistantMessageParam(content=""), + ChatCompletionUserMessageParam( + content=ChatCompletionContentPartInputAudioParam( + input_audio=InputAudio(data="audio", format="wav") + ) + ), + ChatCompletionFunctionMessageParam(content=""), + ChatCompletionToolMessageParam(content=""), + ], choices=[ Choice( message=ChatCompletionMessage( @@ -37,7 +70,7 @@ def get_fake_return(): finish_reason="stop", index=0, logprobs=None, - ) + ), ], model="gpt-4o", usage=CompletionUsage(