diff --git a/evaluation/code_eval/coding/LiveCodeBench/lcb_runner/lm_styles.py b/evaluation/code_eval/coding/LiveCodeBench/lcb_runner/lm_styles.py index 38b6040..b56cefc 100644 --- a/evaluation/code_eval/coding/LiveCodeBench/lcb_runner/lm_styles.py +++ b/evaluation/code_eval/coding/LiveCodeBench/lcb_runner/lm_styles.py @@ -287,6 +287,14 @@ def to_dict(self) -> dict: datetime(2023, 4, 30), link="https://openai.com/index/spring-update", ), + ## Forge (OpenAI-compatible) + LanguageModel( + "OpenAI/gpt-4o-mini", + "Forge-OpenAI-GPT-4o-mini", + LMStyle.OpenAIChat, + datetime(2024, 7, 18), + link="https://forge.tensorblock.co", + ), ## O1-Mini and O1-Preview LanguageModel( "o1-preview-2024-09-12", diff --git a/evaluation/code_eval/coding/LiveCodeBench/lcb_runner/runner/oai_runner.py b/evaluation/code_eval/coding/LiveCodeBench/lcb_runner/runner/oai_runner.py index 50915e4..f34594d 100644 --- a/evaluation/code_eval/coding/LiveCodeBench/lcb_runner/runner/oai_runner.py +++ b/evaluation/code_eval/coding/LiveCodeBench/lcb_runner/runner/oai_runner.py @@ -12,12 +12,20 @@ class OpenAIRunner(BaseRunner): - client = OpenAI( - api_key=os.getenv("OPENAI_KEY"), - ) + @staticmethod + def _build_client(): + forge_api_key = os.getenv("FORGE_API_KEY") + if forge_api_key: + return OpenAI( + api_key=forge_api_key, + base_url=os.getenv("FORGE_API_BASE") + or "https://api.forge.tensorblock.co/v1", + ) + return OpenAI(api_key=os.getenv("OPENAI_KEY")) def __init__(self, args, model): super().__init__(args, model) + self.client = self._build_client() if model.model_style == LMStyle.OpenAIReasonPreview: self.client_kwargs: dict[str | str] = { "model": args.model, @@ -49,7 +57,7 @@ def _run_single(self, prompt: list[dict[str, str]]) -> list[str]: assert isinstance(prompt, list) try: - response = OpenAIRunner.client.chat.completions.create( + response = self.client.chat.completions.create( messages=prompt, **self.client_kwargs, )