ClientTools did invoke registered client functions but didn't work correctly. error occured!! #647
Description
Description
I'm developing a Twilio-based AI phone bot designed to handle appointment scheduling and cold transfers to a human agent. I've registered a client tool for cold transfers (call forwarding)—you can see this in the code snippet below. This client tool is also registered on the ElevenLabs Conversational AI Agent platform.
When I say, "Please connect me to a human agent," the client tool's forward_call function is triggered automatically, but that's when the error occurs.
Here are the error messages:
2025-10-16 08:32:33,261 - medical-voice-agent - INFO - Stored stream SID: MZ95e72472bf7sdfsdf3d7e171erg20575dd05
2025-10-16 08:32:33,262 - medical-voice-agent - INFO - Stored call SID: CA4914a8wqegd02b4c75f2asde9557a4
2025-10-16 08:32:33,820 - medical-voice-agent - INFO - Conversation started
2025-10-16 08:32:34,582 - medical-voice-agent - INFO - Agent: Hi, I can help you schedule meetings and manage your calendar. What can I help you with today?
2025-10-16 08:32:42,758 - medical-voice-agent - INFO - User: Please connect me human agent
2025-10-16 08:32:43,050 - medical-voice-agent - INFO - Agent: I...
Error sending user audio chunk: received 1008 (policy violation) Invalid message received; then sent 1008 (policy violation) Invalid message received
Error receiving message: received 1008 (policy violation) Invalid message received; then sent 1008 (policy violation) Invalid message received
how to fix this?
Code example
import asyncio
import json
import traceback
from fastapi import APIRouter, Request, WebSocket
from fastapi.responses import HTMLResponse
from twilio.twiml.voice_response import VoiceResponse, Connect
from twilio.rest import Client
from elevenlabs import ElevenLabs
from elevenlabs.conversational_ai.conversation import Conversation, ClientTools
from starlette.websockets import WebSocketDisconnect
from app.services.twilio_audio_interface import TwilioAudioInterface
from app.core.config import settings
from app.core.logger import logger
router = APIRouter()
def handle_agent_response(text: str):
logger.info(f"Agent: {text}")
def handle_user_transcript(text: str):
logger.info(f"User: {text}")
def create_forward_to_doctor(websocket: WebSocket):
"""Create a forward_to_doctor function with websocket context."""
async def forward_to_doctor():
logger.info("Forwarding to doctor")
try:
# Get the call SID from the websocket
call_sid = getattr(websocket, "call_sid", None)
if not call_sid:
logger.error("Error: No call SID available for transfer")
return
# TODO: Inform the caller
await websocket.send_json({
"event": "message",
"message": "We've detected a potential human service. Transferring you to a human. Please stay on the line."
})
# TODO: implement call forwarding
logger.info(f"Call transferred to doctor. Conference: {conference_name}")
websocket.close()
except Exception as e:
logger.info(f"Error transferring call to doctor: {str(e)}")
traceback.print_exc()
return forward_to_doctor
@router.post("/twilio/inbound_call")
async def handle_incoming_call(request: Request):
form_data = await request.form()
call_sid = form_data.get("CallSid", "Unknown")
from_number = form_data.get("From", "Unknown")
logger.info(f"Incoming call: CallSid={call_sid}, From={from_number}")
response = VoiceResponse()
connect = Connect()
connect.stream(url=f"wss://{request.url.hostname}/media-stream")
response.append(connect)
return HTMLResponse(content=str(response), media_type="application/xml")
@router.websocket("/media-stream")
async def handle_media_stream(websocket: WebSocket):
await websocket.accept()
logger.info("WebSocket connection opened")
audio_interface = TwilioAudioInterface(websocket)
eleven_labs_client = ElevenLabs(api_key=settings.ELEVENLABS_API_KEY)
# Get the current event loop
custom_loop = asyncio.get_running_loop()
# Create ClientTools with custom loop to prevent "different event loop" errors
client_tools = ClientTools(loop=custom_loop)
client_tools.register("forward_call", create_forward_to_doctor(websocket), is_async=True)
try:
websocket.stream_sid = None
websocket.call_sid = None
conversation = None
async for message in websocket.iter_text():
if not message:
continue
try:
data = json.loads(message)
# Store the stream SID when it's received in the start event
if data.get("event") == "start" and "start" in data:
websocket.stream_sid = data["start"].get("streamSid")
websocket.call_sid = data["start"].get("callSid")
logger.info(f"Stored stream SID: {websocket.stream_sid}")
logger.info(f"Stored call SID: {websocket.call_sid}")
# Initialize and start conversation AFTER audio interface has stream_sid
conversation = Conversation(
client=eleven_labs_client,
agent_id=settings.ELEVENLABS_AGENT_ID,
requires_auth=True, # Security > Enable authentication
audio_interface=audio_interface,
callback_agent_response=lambda text: handle_agent_response(text),
callback_user_transcript=lambda text: handle_user_transcript(text),
client_tools=client_tools
)
conversation.start_session()
logger.info("Conversation started")
# Process the Twilio message FIRST to set stream_sid in audio interface
await audio_interface.handle_twilio_message(data)
except Exception as e:
logger.error(f"Error processing message: {e}")
traceback.print_exc()
except WebSocketDisconnect:
logger.info("WebSocket disconnected")
except Exception:
logger.error("Error occurred in WebSocket handler:")
traceback.print_exc()
finally:
try:
if conversation:
conversation.end_session()
conversation.wait_for_session_end()
logger.info("Conversation ended")
except Exception:
logger.error("Error ending conversation session:")
traceback.print_exc()
Additional context
No response