From 3cb0664f12b2ed3918afe2e167bbe8ac62d54ecb Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 27 Nov 2024 09:34:02 +0100 Subject: [PATCH 1/2] fix: Add message conversion to JSON --- .../openai_function_calling_example.py | 8 +---- .../openai_vision_example.py} | 6 +--- src/observers/observers/base.py | 8 +---- src/observers/observers/models/openai.py | 17 ++++++--- .../observers/test_observers_examples.py | 35 ++++++++++++++++++- 5 files changed, 50 insertions(+), 24 deletions(-) rename examples/{ => observers}/openai_function_calling_example.py (85%) rename examples/{vision_example.py => observers/openai_vision_example.py} (88%) diff --git a/examples/openai_function_calling_example.py b/examples/observers/openai_function_calling_example.py similarity index 85% rename from examples/openai_function_calling_example.py rename to examples/observers/openai_function_calling_example.py index c338a73..f157682 100644 --- a/examples/openai_function_calling_example.py +++ b/examples/observers/openai_function_calling_example.py @@ -1,12 +1,6 @@ from observers.observers import wrap_openai -from observers.stores import DatasetsStore from openai import OpenAI -store = DatasetsStore( - repo_name="gpt-4o-function-calling-traces", - every=5, # sync every 5 minutes -) - openai_client = OpenAI() tools = [ @@ -42,7 +36,7 @@ ] -client = wrap_openai(openai_client, store=store) +client = wrap_openai(openai_client) response = client.chat.completions.create( model="gpt-4o", diff --git a/examples/vision_example.py b/examples/observers/openai_vision_example.py similarity index 88% rename from examples/vision_example.py rename to examples/observers/openai_vision_example.py index db62bec..3cb1271 100644 --- a/examples/vision_example.py +++ b/examples/observers/openai_vision_example.py @@ -2,11 +2,7 @@ from observers.stores import DatasetsStore from openai import OpenAI -store = DatasetsStore( - repo_name="gpt-4o-mini-vision-traces", - every=5, # sync every 5 messages -) - +store = DatasetsStore(repo_name="test-repo-json-messages") openai_client = OpenAI() client = wrap_openai(openai_client, store=store) diff --git a/src/observers/observers/base.py b/src/observers/observers/base.py index a85be3e..5dc6470 100644 --- a/src/observers/observers/base.py +++ b/src/observers/observers/base.py @@ -1,17 +1,11 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING if TYPE_CHECKING: from argilla import Argilla -@dataclass -class Message: - role: Literal["system", "user", "assistant", "function"] - content: str - - @dataclass class Record(ABC): """ diff --git a/src/observers/observers/models/openai.py b/src/observers/observers/models/openai.py index d9d74e1..ec86b4f 100644 --- a/src/observers/observers/models/openai.py +++ b/src/observers/observers/models/openai.py @@ -3,7 +3,9 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union -from observers.observers.base import Message, Record +from openai.types.chat.chat_completion_message_param import ChatCompletionMessageParam + +from observers.observers.base import Record from observers.stores.duckdb import DuckDBStore if TYPE_CHECKING: @@ -22,7 +24,7 @@ class OpenAIResponseRecord(Record): id: str = field(default_factory=lambda: str(uuid.uuid4())) model: str = None timestamp: str = field(default_factory=lambda: datetime.datetime.now().isoformat()) - messages: List[Message] = None + messages: List[ChatCompletionMessageParam] = None assistant_message: Optional[str] = None completion_tokens: Optional[int] = None prompt_tokens: Optional[int] = None @@ -65,7 +67,7 @@ def duckdb_schema(self): id VARCHAR PRIMARY KEY, model VARCHAR, timestamp TIMESTAMP, - messages STRUCT(role VARCHAR, content VARCHAR)[], + messages JSON, assistant_message TEXT, completion_tokens INTEGER, prompt_tokens INTEGER, @@ -160,7 +162,14 @@ def table_name(self): @property def json_fields(self): - return ["tool_calls", "function_call", "tags", "properties", "raw_response"] + return [ + "tool_calls", + "function_call", + "tags", + "properties", + "raw_response", + "messages", + ] def wrap_openai( diff --git a/tests/integration/observers/test_observers_examples.py b/tests/integration/observers/test_observers_examples.py index 5e0e8f2..09d8c56 100644 --- a/tests/integration/observers/test_observers_examples.py +++ b/tests/integration/observers/test_observers_examples.py @@ -8,6 +8,21 @@ ChatCompletionMessage, ) from openai.types.chat.chat_completion import Choice, CompletionUsage +from openai.types.chat.chat_completion_content_part_input_audio_param import ( + InputAudio, +) +from openai.types.chat.chat_completion_content_part_param import ( + ChatCompletionContentPartImageParam, + ChatCompletionContentPartInputAudioParam, + ChatCompletionContentPartTextParam, +) +from openai.types.chat.chat_completion_message_param import ( + ChatCompletionAssistantMessageParam, + ChatCompletionFunctionMessageParam, + ChatCompletionSystemMessageParam, + ChatCompletionToolMessageParam, + ChatCompletionUserMessageParam, +) def get_example_files(): @@ -29,6 +44,24 @@ def mock_clients(): def get_fake_return(): return ChatCompletion( id=str(uuid.uuid4()), + messages=[ + ChatCompletionSystemMessageParam(content=""), + ChatCompletionUserMessageParam( + content=ChatCompletionContentPartTextParam(text="test") + ), + ChatCompletionAssistantMessageParam(content=""), + ChatCompletionUserMessageParam( + content=ChatCompletionContentPartImageParam(image_url="image") + ), + ChatCompletionAssistantMessageParam(content=""), + ChatCompletionUserMessageParam( + content=ChatCompletionContentPartInputAudioParam( + input_audio=InputAudio(data="audio", format="wav") + ) + ), + ChatCompletionFunctionMessageParam(content=""), + ChatCompletionToolMessageParam(content=""), + ], choices=[ Choice( message=ChatCompletionMessage( @@ -37,7 +70,7 @@ def get_fake_return(): finish_reason="stop", index=0, logprobs=None, - ) + ), ], model="gpt-4o", usage=CompletionUsage( From 1741d927b3a822b0b71d1cb72b3d37f1a3760bd9 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 27 Nov 2024 09:37:12 +0100 Subject: [PATCH 2/2] tests: Remove dependency on DatasetsStore --- examples/observers/openai_vision_example.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/observers/openai_vision_example.py b/examples/observers/openai_vision_example.py index 3cb1271..73129ec 100644 --- a/examples/observers/openai_vision_example.py +++ b/examples/observers/openai_vision_example.py @@ -1,10 +1,8 @@ from observers.observers import wrap_openai -from observers.stores import DatasetsStore from openai import OpenAI -store = DatasetsStore(repo_name="test-repo-json-messages") openai_client = OpenAI() -client = wrap_openai(openai_client, store=store) +client = wrap_openai(openai_client) response = client.chat.completions.create( model="gpt-4o-mini",