1 2const peerConnection = new RTCPeerConnection(); 3console.log("Peer connection created");
1 2// create an audio element 3const audioEl = document.querySelector(".placetoputaudioplayer"); 4const dd = document.createElement("audio"); 5dd.controls = true; 6audioEl.appendChild(dd); 7 8peerConnection.ontrack = (e) => { 9 // attach the incoming audio stream to the audio element 10 11 const mdata = new MediaStream([e.track]); 12 dd.srcObject = mdata; 13 dd.play(); 14};
1 2// aquire the user's microphone 3const ms = await navigator.mediaDevices.getUserMedia({ 4 audio: true 5}); 6 7// add the audio stream to the peer connection 8peerConnection.addTrack(ms.getAudioTracks()[0]); 9 10// add visual feedback for audio 11const audioContext = new AudioContext(); 12const analyser = audioContext.createAnalyser(); 13const source = audioContext.createMediaStreamSource(ms); 14source.connect(analyser); 15analyser.fftSize = 256; 16const bufferLength = analyser.frequencyBinCount; 17const dataArray = new Uint8Array(bufferLength); 18const canvas = document.createElement("canvas"); 19const audioEl = document.querySelector(".placetoputaudioplayer"); 20audioEl.appendChild(canvas); 21const canvasCtx = canvas.getContext("2d"); 22canvas.width = 100; 23canvas.height = 100; 24canvasCtx.clearRect(0, 0, canvas.width, canvas.height); 25 26function draw() { 27 requestAnimationFrame(draw); 28 analyser.getByteFrequencyData(dataArray); 29 canvasCtx.fillStyle = 'rgb(200, 200, 200)'; 30 canvasCtx.fillRect(0, 0, canvas.width, canvas.height); 31 canvasCtx.fillStyle = 'rgb(0, 0, 0)'; 32 const barWidth = (canvas.width / bufferLength) * 2.5; 33 let barHeight; 34 let x = 0; 35 for (let i = 0; i < bufferLength; i++) { 36 barHeight = dataArray[i]; 37 canvasCtx.fillRect(x, canvas.height - barHeight / 2, barWidth, barHeight / 2); 38 x += barWidth + 1; 39 } 40} 41draw(); 42
1 2const dataChannel = peerConnection.createDataChannel("oai-events"); 3 4var eventHandlers = {} 5 6dataChannel.onopen = () => { 7 console.log("Data channel opened"); 8} 9dataChannel.onerror = (e) => { 10 console.log("Data channel error:"+ JSON.stringify(e)); 11} 12dataChannel.onclose = () => { 13 console.log("Data channel closed"); 14} 15dataChannel.addEventListener("message", (e) => { 16 const data = JSON.parse(e.data); 17 const handler = eventHandlers[data.type]; 18 if (handler) { 19 handler(data); 20 } 21 else 22 { 23 // check typing for extra message types 24 } 25}); 26 27console.log("Data channel created"); 28
1 2 <div style="margin: 16px; display: flex"> 3 <div class="sessioninformation" style="display: flex; flex-direction: column; border: 1px solid #e0e0e0; border-radius: 8px; padding: 16px; text-align: left; font-size: 12px; width:50%"> 4 <h4>Session Information</h4> 5 <div class="sessioninfo"> 6 ... Session Data ... 7 </div> 8 </div> 9 <div class="messages" style="display: flex; flex-direction: column; border: 1px solid #e0e0e0; border-radius: 8px; padding: 16px; text-align: left; font-size: 12px; width:50%"> 10 <h4>Messages</h4> 11 <div class="messagecontent"> 12 ... Messages ... 13 </div> 14 </div> 15 </div> 16
1 2eventHandlers["session.created"] = (data) => { 3 // empty the session information div 4 document.querySelector(".sessioninfo").innerHTML = ""; 5 // for each key in the data object 6 for (const key in data.session) { 7 // create a div element 8 const div = document.createElement("div"); 9 // set the class of the div to the key 10 div.className = "sessioninfo-"+key; 11 // set the inner text to the key and value 12 div.innerText = key + ": " + data.session[key]; 13 // append the div to the session information div 14 document.querySelector(".sessioninfo").appendChild(div); 15 } 16} 17eventHandlers["conversation.created"] = (data) => { 18 console.log("Conversation created"); 19} 20
1 2 3eventHandlers["conversation.item.created"] = (data) => { 4 // create elements for the message 5 const message = data.item; 6 const div = document.createElement("div"); 7 div.className = "message"; 8 const audiocontent = message.content.filter((c) => c.type === "audio"); 9 const transcriptcontent = message.content.filter((c) => c.transcript); 10 const textcontent = message.content.filter((c) => c.type === "text" && !c.transcript); 11 12 const cdiv = document.createElement("div"); 13 cdiv.innerText = "Role: "+message.role; 14 div.appendChild(cdiv); 15 // add all audio content from message 16 for (const c of audiocontent) { 17 const audio = document.createElement("audio"); 18 audio.src = c.audio; 19 audio.controls = true; 20 div.appendChild(audio); 21 } 22 23 // add all transcript content from message 24 for (const c of transcriptcontent) { 25 const childdiv = document.createElement("div"); 26 childdiv.innerText = c.transcript; 27 div.appendChild(childdiv); 28 } 29 30 // add all text content from message 31 for (const c of textcontent) { 32 const childdiv = document.createElement("div"); 33 childdiv.innerText = c.text; 34 div.appendChild(childdiv); 35 } 36 37 document.querySelector(".messagecontent").appendChild(div); 38}
1 2 const offer = await peerConnection.createOffer(); 3 await peerConnection.setLocalDescription(offer); 4 console.log("Offer created"); 5
1 2const baseUrl = "https://api.featherless.ai/v1/realtime"; 3const response = await fetch(`${baseUrl}`, { 4 method: "POST", 5 headers: { 6 'Content-Type': 'application/json', 7 'Authorization': `Bearer null` 8 }, 9 body: JSON.stringify({ 10 model: 'recursal/QRWKV6-32B-Instruct-Preview-v0.1', 11 max_tokens: 100, 12 temperature: 0.5, 13 voice: 'kak/am_adam', 14 turn_detection: {type: "client_vad", threshold: 0.5}, 15 modalities: ["text", "audio"], 16 sdp: offer.sdp 17 }) 18 }) 19 20if (response.status !== 200) { 21 console.log("Error: " + await response.text()); 22 return; 23} 24const data = await response.text(); 25 26peerConnection.setRemoteDescription( 27 { 28 type: "answer", 29 sdp: data, 30 } 31 ) 32
1 2 <div> 3 <button 4 class="transcribebutton" 5 >Hold to Transcribe</button> 6 Transcription: 7 <div class="transcription"></div> 8 <button 9 class="responsebutton" 10 >Request response from AI</button> 11 Streamed Response: 12 <div class="streamedresponse"></div> 13 </div> 14
1 2// press to clear audio buffer on server, release to request transcription 3document.querySelector(".transcribebutton").addEventListener("mousedown", () => { 4 dataChannel.send(JSON.stringify({type: "input_audio_buffer.clear"})); 5 }) 6document.querySelector(".transcribebutton").addEventListener("mouseup", () => { 7 dataChannel.send(JSON.stringify({type: "input_audio_buffer.commit"})); 8 }) 9 10// response from server with finished transcription 11eventHandlers["conversation.item.input_audio_transcription.completed"] = (data) => { 12 document.querySelector(".transcription").innerText = data.transcript; 13} 14 15// lets user confirm buffer cleared 16eventHandlers["input_audio_buffer.cleared"] = (data) => { 17 document.querySelector(".transcription").innerText = "Buffer cleared"; 18} 19
1 2document.querySelector(".responsebutton").addEventListener("click", () => { 3 dataChannel.send(JSON.stringify({ 4 type: "response.create", 5 response: { 6 // you can override any of the session settings here 7 } 8 })); 9 }) 10eventHandlers["response.text.delta"] = (data) => { 11 document.querySelector(".streamedresponse").innerText = data.delta; 12 } 13