class RealtimeConversation {
constructor(orgId, authToken, options = {}) {
this.orgId = orgId;
this.authToken = authToken;
this.ws = null;
this.keepAliveInterval = null;
this.audioQueue = [];
this.isPlaying = false;
// Configuration
this.options = {
responseFormat: options.responseFormat || 'voice',
vadEnabled: options.vadEnabled || false,
onMessage: options.onMessage || (() => {}),
onError: options.onError || console.error,
onClose: options.onClose || (() => {})
};
}
async connect(serviceId) {
const url = `wss://api.amigo.ai/v1/${this.orgId}/conversation/converse_realtime` +
`?response_format=${this.options.responseFormat}`;
this.ws = new WebSocket(url, [`bearer.authorization.amigo.ai.${this.authToken}`]);
return new Promise((resolve, reject) => {
this.ws.onopen = () => {
console.log('WebSocket connected');
// Start keep-alive
this.startKeepAlive();
// Initialize conversation
this.ws.send(JSON.stringify({
type: 'client.start-conversation',
service_id: serviceId,
service_version_set_name: 'release'
}));
};
this.ws.onmessage = (event) => {
const message = JSON.parse(event.data);
this.handleMessage(message);
if (message.type === 'server.conversation-created') {
resolve(message.conversation_id);
// Enable VAD if requested
if (this.options.vadEnabled) {
this.enableVAD();
}
}
};
this.ws.onerror = (error) => {
this.options.onError(error);
reject(error);
};
this.ws.onclose = (event) => {
this.cleanup();
this.options.onClose(event);
};
});
}
handleMessage(message) {
this.options.onMessage(message);
switch(message.type) {
case 'server.conversation-created':
console.log('Conversation started:', message.conversation_id);
break;
case 'server.new-message':
if (this.options.responseFormat === 'voice' && message.message) {
this.queueAudio(message.message);
}
break;
case 'server.interaction-complete':
console.log('Response complete');
break;
case 'server.vad-speech-started':
console.log('User speaking...');
this.pauseAudio();
break;
case 'server.vad-speech-ended':
console.log('User said:', message.transcript);
break;
}
}
// Keep connection alive
startKeepAlive() {
this.keepAliveInterval = setInterval(() => {
if (this.ws?.readyState === WebSocket.OPEN) {
this.ws.send(JSON.stringify({ type: 'client.extend-timeout' }));
}
}, 15000);
}
// Voice Activity Detection
async enableVAD() {
this.ws.send(JSON.stringify({
type: 'client.switch-vad-mode',
vad_mode_on: true
}));
// Start streaming microphone audio
await this.startAudioStream();
}
async startAudioStream() {
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
sampleRate: 16000,
echoCancellation: true,
noiseSuppression: true
}
});
const audioContext = new AudioContext({ sampleRate: 16000 });
const source = audioContext.createMediaStreamSource(stream);
const processor = audioContext.createScriptProcessor(4096, 1, 1);
let isFirstChunk = true;
processor.onaudioprocess = (e) => {
const pcmData = this.convertToPCM16(e.inputBuffer.getChannelData(0));
this.sendAudio(pcmData, isFirstChunk);
isFirstChunk = false;
};
source.connect(processor);
processor.connect(audioContext.destination);
}
// Send messages
sendText(text, messageType = 'user-message') {
if (this.ws?.readyState !== WebSocket.OPEN) {
throw new Error('WebSocket not connected');
}
this.ws.send(JSON.stringify({
type: 'client.new-text-message',
text: text,
message_type: messageType
}));
}
sendAudio(audioData, isFirstChunk = false) {
const message = {
type: 'client.new-audio-message',
audio: this.arrayBufferToBase64(audioData)
};
if (isFirstChunk) {
message.audio_config = {
type: 'pcm',
frame_rate: 16000,
sample_width: 2,
n_channels: 1
};
}
this.ws.send(JSON.stringify(message));
}
completeAudio() {
this.ws.send(JSON.stringify({
type: 'client.new-audio-message',
audio: null
}));
}
// Audio utilities
convertToPCM16(float32Array) {
const int16Array = new Int16Array(float32Array.length);
for (let i = 0; i < float32Array.length; i++) {
const s = Math.max(-1, Math.min(1, float32Array[i]));
int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
}
return int16Array.buffer;
}
arrayBufferToBase64(buffer) {
const bytes = new Uint8Array(buffer);
let binary = '';
bytes.forEach(b => binary += String.fromCharCode(b));
return btoa(binary);
}
base64ToArrayBuffer(base64) {
const binaryString = atob(base64);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
return bytes.buffer;
}
// Audio playback
queueAudio(base64Audio) {
const audioBuffer = this.base64ToArrayBuffer(base64Audio);
this.audioQueue.push(audioBuffer);
if (!this.isPlaying) {
this.playNextAudio();
}
}
async playNextAudio() {
if (this.audioQueue.length === 0) {
this.isPlaying = false;
return;
}
this.isPlaying = true;
const audioBuffer = this.audioQueue.shift();
// Play using Web Audio API
const audioContext = new AudioContext();
const source = audioContext.createBufferSource();
// Decode PCM data
const audioData = await audioContext.decodeAudioData(audioBuffer);
source.buffer = audioData;
source.connect(audioContext.destination);
source.onended = () => this.playNextAudio();
source.start();
}
pauseAudio() {
// Clear audio queue when interrupted
this.audioQueue = [];
this.isPlaying = false;
}
// Cleanup
async finish() {
if (this.options.vadEnabled) {
// Disable VAD first
this.ws.send(JSON.stringify({
type: 'client.switch-vad-mode',
vad_mode_on: false
}));
// Wait for confirmation
await new Promise(resolve => {
const handler = (event) => {
const msg = JSON.parse(event.data);
if (msg.type === 'server.vad-mode-switched') {
this.ws.removeEventListener('message', handler);
resolve();
}
};
this.ws.addEventListener('message', handler);
});
}
// Now finish conversation
this.ws.send(JSON.stringify({
type: 'client.finish-conversation'
}));
}
cleanup() {
if (this.keepAliveInterval) {
clearInterval(this.keepAliveInterval);
}
this.audioQueue = [];
this.isPlaying = false;
}
close() {
this.cleanup();
if (this.ws?.readyState === WebSocket.OPEN) {
this.ws.send(JSON.stringify({ type: 'client.close-connection' }));
this.ws.close();
}
}
}
// Usage Example
async function main() {
const client = new RealtimeConversation('your-org', 'your-auth-token', {
responseFormat: 'voice',
vadEnabled: true,
onMessage: (msg) => {
// Handle all messages
console.log('Message:', msg.type);
},
onError: (error) => {
console.error('Error:', error);
},
onClose: (event) => {
console.log('Closed:', event.code, event.reason);
}
});
try {
// Connect and start conversation
const conversationId = await client.connect('service-id');
console.log('Conversation ID:', conversationId);
// Send a text message
client.sendText('Hello, how can you help me?');
// Or manually send audio (if not using VAD)
// client.sendAudio(pcmAudioData, true);
// client.completeAudio();
// When done
// await client.finish();
} catch (error) {
console.error('Failed to connect:', error);
}
}
main();
