Enter your API key


Create a new RTCPeerConnection object.
1
2const peerConnection = new RTCPeerConnection();
3console.log("Peer connection created");

This will setup and display the incoming audio stream from the connection.
1
2// create an audio element
3const audioEl = document.querySelector(".placetoputaudioplayer");
4const dd = document.createElement("audio");
5dd.controls = true;
6audioEl.appendChild(dd);
7
8peerConnection.ontrack = (e) => {
9  // attach the incoming audio stream to the audio element
10
11  const mdata = new MediaStream([e.track]);
12  dd.srcObject = mdata;
13  dd.play();
14};

Get the user's audio stream and add it to the RTCPeerConnection.
1
2// aquire the user's microphone
3const ms = await navigator.mediaDevices.getUserMedia({
4  audio: true
5});
6
7// add the audio stream to the peer connection
8peerConnection.addTrack(ms.getAudioTracks()[0]);
9
10// add visual feedback for audio
11const audioContext = new AudioContext();
12const analyser = audioContext.createAnalyser();
13const source = audioContext.createMediaStreamSource(ms);
14source.connect(analyser);
15analyser.fftSize = 256;
16const bufferLength = analyser.frequencyBinCount;
17const dataArray = new Uint8Array(bufferLength);
18const canvas = document.createElement("canvas");
19const audioEl = document.querySelector(".placetoputaudioplayer");
20audioEl.appendChild(canvas);
21const canvasCtx = canvas.getContext("2d");
22canvas.width = 100;
23canvas.height = 100;
24canvasCtx.clearRect(0, 0, canvas.width, canvas.height);
25
26function draw() {
27  requestAnimationFrame(draw);
28  analyser.getByteFrequencyData(dataArray);
29  canvasCtx.fillStyle = 'rgb(200, 200, 200)';
30  canvasCtx.fillRect(0, 0, canvas.width, canvas.height);
31  canvasCtx.fillStyle = 'rgb(0, 0, 0)';
32  const barWidth = (canvas.width / bufferLength) * 2.5;
33  let barHeight;
34  let x = 0;
35  for (let i = 0; i < bufferLength; i++) {
36    barHeight = dataArray[i];
37    canvasCtx.fillRect(x, canvas.height - barHeight / 2, barWidth, barHeight / 2);
38    x += barWidth + 1;
39  }
40}
41draw();
42

Create a data channel for sending and receiving events.
1
2const dataChannel = peerConnection.createDataChannel("oai-events");
3
4var eventHandlers = {}
5
6dataChannel.onopen = () => {
7  console.log("Data channel opened");
8}
9dataChannel.onerror = (e) => {
10  console.log("Data channel error:"+ JSON.stringify(e));
11}
12dataChannel.onclose = () => {
13  console.log("Data channel closed");
14}
15dataChannel.addEventListener("message", (e) => {
16  const data = JSON.parse(e.data);
17  const handler = eventHandlers[data.type];
18  if (handler) {
19    handler(data);
20  }
21  else
22  {
23    // check typing for extra message types
24  }
25});
26
27console.log("Data channel created");
28

Create some visual feedback for the data coming from the server.
1
2      <div style="margin: 16px; display: flex">
3        <div class="sessioninformation" style="display: flex; flex-direction: column; border: 1px solid #e0e0e0; border-radius: 8px; padding: 16px;  text-align: left; font-size: 12px; width:50%">
4          <h4>Session Information</h4>
5          <div class="sessioninfo">
6          ... Session Data ...
7          </div>
8        </div>
9        <div class="messages" style="display: flex; flex-direction: column; border: 1px solid #e0e0e0; border-radius: 8px; padding: 16px; text-align: left; font-size: 12px; width:50%">
10          <h4>Messages</h4>
11          <div class="messagecontent">
12          ... Messages ...
13          </div>
14        </div>
15      </div>
16

Update the session information div with the data from the server.
1
2eventHandlers["session.created"] = (data) => {
3  // empty the session information div
4  document.querySelector(".sessioninfo").innerHTML = "";
5      // for each key in the data object
6  for (const key in data.session) {
7    // create a div element
8    const div = document.createElement("div");
9    // set the class of the div to the key
10    div.className = "sessioninfo-"+key;
11    // set the inner text to the key and value
12    div.innerText = key + ": " + data.session[key];
13    // append the div to the session information div
14    document.querySelector(".sessioninfo").appendChild(div);
15  }
16}
17eventHandlers["conversation.created"] = (data) => {
18  console.log("Conversation created");
19}
20

Handle messages from the server.
1
2
3eventHandlers["conversation.item.created"] = (data) => {
4  // create elements for the message
5  const message = data.item;
6  const div = document.createElement("div");
7  div.className = "message";
8  const audiocontent = message.content.filter((c) => c.type === "audio");
9  const transcriptcontent = message.content.filter((c) => c.transcript);
10  const textcontent = message.content.filter((c) => c.type === "text" && !c.transcript);
11
12  const cdiv = document.createElement("div");
13  cdiv.innerText = "Role: "+message.role;
14  div.appendChild(cdiv);
15  // add all audio content from message
16  for (const c of audiocontent) {
17    const audio = document.createElement("audio");
18    audio.src = c.audio;
19    audio.controls = true;
20    div.appendChild(audio);
21  }
22
23  // add all transcript content from message
24  for (const c of transcriptcontent) {
25    const childdiv = document.createElement("div");
26    childdiv.innerText = c.transcript;
27    div.appendChild(childdiv);
28  }
29
30  // add all text content from message
31  for (const c of textcontent) {
32    const childdiv = document.createElement("div");
33    childdiv.innerText = c.text;
34    div.appendChild(childdiv);
35  }
36
37  document.querySelector(".messagecontent").appendChild(div);
38}

Create an offer for the connection.
1
2  const offer = await peerConnection.createOffer();
3  await peerConnection.setLocalDescription(offer);
4  console.log("Offer created");
5

Send the offer to the Featherless API to establish the connection.
1
2const baseUrl = "https://api.featherless.ai/v1/realtime";
3const response = await fetch(`${baseUrl}`, {
4  method: "POST",
5  headers: {
6          'Content-Type': 'application/json',
7          'Authorization': `Bearer null`
8        },
9        body: JSON.stringify({
10          model: 'recursal/QRWKV6-32B-Instruct-Preview-v0.1',
11          max_tokens: 100,
12          temperature: 0.5,
13          voice: 'kak/am_adam',
14          turn_detection: {type: "client_vad", threshold: 0.5},
15          modalities: ["text", "audio"],
16          sdp: offer.sdp
17        })
18  })
19      
20if (response.status !== 200) {
21    console.log("Error: " + await response.text());
22    return;
23}
24const data = await response.text();
25
26peerConnection.setRemoteDescription(
27    {
28      type: "answer",
29      sdp: data,
30    }
31  )
32

Add some buttons to interact with the server.
1
2    <div>
3      <button
4        class="transcribebutton"
5      >Hold to Transcribe</button>
6      Transcription:
7      <div class="transcription"></div>
8      <button
9      class="responsebutton"
10      >Request response from AI</button>
11      Streamed Response:
12      <div class="streamedresponse"></div>
13    </div>
14

Add event listeners for transcribe button.
1
2// press to clear audio buffer on server, release to request transcription
3document.querySelector(".transcribebutton").addEventListener("mousedown", () => {
4  dataChannel.send(JSON.stringify({type: "input_audio_buffer.clear"}));
5  })
6document.querySelector(".transcribebutton").addEventListener("mouseup", () => {
7  dataChannel.send(JSON.stringify({type: "input_audio_buffer.commit"}));
8  })
9
10// response from server with finished transcription
11eventHandlers["conversation.item.input_audio_transcription.completed"] = (data) => {
12    document.querySelector(".transcription").innerText = data.transcript;
13}
14
15// lets user confirm buffer cleared
16eventHandlers["input_audio_buffer.cleared"] = (data) => {
17  document.querySelector(".transcription").innerText = "Buffer cleared";
18}
19

Add event listeners for response button.
1
2document.querySelector(".responsebutton").addEventListener("click", () => {
3    dataChannel.send(JSON.stringify({ 
4                type: "response.create",
5                response: {
6                  // you can override any of the session settings here
7                }
8              }));
9  })
10eventHandlers["response.text.delta"] = (data) => {
11    document.querySelector(".streamedresponse").innerText = data.delta;
12  }
13