Web Speech API Specification

To implement progressive enhancement for speech recognition, use SpeechRecognition.available(options) to check for a model that meets your required quality level, and SpeechRecognition.install(options) to attempt to download one if it is missing. If install() rejects, it typically means the device hardware (NPU/RAM) cannot support the requested quality level, and you should fall back to cloud-based recognition.

const meetConfig = {
  langs: ['en-US'],
  processLocally: true,
  quality: 'conversation' 
};

async function setupSpeech() {
  // 1. Check if a high-quality model is already available
  const availability = await SpeechRecognition.available(meetConfig);
  
  if (availability == 'available') {
    startOnDeviceSpeech();
    return;
  }

  // 2. If not, try to install one
  try {
    await SpeechRecognition.install(meetConfig);
    startOnDeviceSpeech();
  } catch (e) {
    // 3. Fallback: Device hardware is too weak or user denied download.
    // Fallback to Cloud to ensure transcript usability.
    console.log("High-quality on-device model unavailable. Using Cloud.");
    startCloudSpeech();
  }
}

This example demonstrates how to enable unspokenPunctuation alongside other standard settings like continuous and interimResults to capture natural, punctuated speech.

When unspokenPunctuation is enabled, the transcript in a final result will include inferred punctuation (e.g., "Hello there, how are you today?") instead of raw text ("hello there how are you today").

const recognition = new SpeechRecognition();

// Enable unspoken punctuation
recognition.unspokenPunctuation = true;

// Configure other settings
recognition.continuous = true;
recognition.interimResults = true;
recognition.lang = 'en-US';

recognition.onresult = (event) => {
  for (let i = event.resultIndex; i < event.results.length; ++i) {
    if (event.results[i].isFinal) {
      console.log('Final Result with Punctuation: ', event.results[i][0].transcript);
      // Example output: "Hello there, how are you today?"
    } else {
      console.log('Interim Result: ', event.results[i][0].transcript);
    }
  }
};

recognition.start();

This example demonstrates how to create SpeechRecognitionPhrase objects, assign them to a SpeechRecognition instance, and handle potential errors.

// A list of phrases relevant to our application's context.
const phraseData = [
  { phrase: 'Zoltan', boost: 3.0 },
  { phrase: 'Grog', boost: 2.0 },
];

// Create SpeechRecognitionPhrase objects.
const phraseObjects = phraseData.map(p => new SpeechRecognitionPhrase(p.phrase, p.boost));

const recognition = new SpeechRecognition();

// Assign the phrase objects to the recognition instance.
recognition.phrases = phraseObjects;

// We can also dynamically add/remove phrases.
recognition.phrases.push(new SpeechRecognitionPhrase('Xylia', 2.5));

// Some user agents (e.g. Chrome) might only support on-device contextual biasing.
recognition.processLocally = true;

recognition.onresult = (event) => {
  const transcript = event.results[0][0].transcript;
  console.log(`Result: ${transcript}`);
};

recognition.onerror = (event) => {
  if (event.error === 'phrases-not-supported') {
    console.warn('Contextual biasing is not supported by this browser/service.');
  }
};

// Start recognition when the user clicks a button.
document.getElementById('speak-button').onclick = () => {
  recognition.start();
};

You can use speechStartTime and speechEndTime to generate synchronized WebVTT captions and interactive transcript elements that allow users to jump to specific parts of a media playback.

1. Captions: Use the timestamps to instantiate a VTTCue and add it to a media track.
2. Click-to-Seek: Create DOM elements for each transcript segment. When clicked, set the mediaElement.currentTime to the result.speechStartTime to jump the playback to that exact moment.

const mediaElement = document.querySelector('video');
const track = mediaElement.addTextTrack('captions', 'English', 'en');
track.mode = 'showing';

const recognition = new SpeechRecognition();
recognition.continuous = true;
recognition.interimResults = false;

// Transcribe audio track from HTMLMediaElement
const audioStream = mediaElement.captureStream();
recognition.start(audioStream.getAudioTracks()[0]);

recognition.onresult = (event) => {
  for (let i = event.resultIndex; i < event.results.length; ++i) {
    const result = event.results[i];
    if (!result.isFinal) continue;

    const transcriptText = result[0].transcript;

    // 1. Create a synchronized WebVTT caption cue
    const cue = new VTTCue(result.speechStartTime, result.speechEndTime, transcriptText);
    track.addCue(cue);

    // 2. Build interactive transcript element ("Click-to-Seek")
    const transcriptItem = document.createElement('p');
    transcriptItem.textContent = `[${result.speechStartTime.toFixed(1)}s] ${transcriptText}`;
    transcriptItem.onclick = () => {
      mediaElement.currentTime = result.speechStartTime;
      mediaElement.play();
    };
    document.getElementById('transcript-container').appendChild(transcriptItem);
  }
};

For live video conferencing, you can use the timestamps to provide two distinct experiences:

1. Live Display: Render interim results immediately on screen to minimize perceived lag. Use result.isFinal to manage the visual state (e.g., styling interim vs. final text).
2. Recording Archive: When a result is finalized (isFinal), archive the text along with its speechStartTime and speechEndTime. This allows you to reconstruct frame-accurate subtitle tracks for recorded media during post-meeting replay.

const recognition = new SpeechRecognition();
recognition.continuous = true;
recognition.interimResults = true;

let hideTimeout = null;
const liveSubtitleOverlay = document.getElementById('live-captions');
const meetingTranscript = [];

recognition.onresult = (event) => {
  const result = event.results[event.resultIndex];
  const transcriptText = result[0].transcript;

  // Render live captions immediately on screen
  liveSubtitleOverlay.textContent = transcriptText;
  liveSubtitleOverlay.classList.toggle('interim', !result.isFinal);

  // Keep final captions on screen for a 3-second reading persistence window
  clearTimeout(hideTimeout);
  if (result.isFinal) {
    hideTimeout = setTimeout(() => {
      liveSubtitleOverlay.textContent = '';
    }, 3000);

    // Archive the finalized sentence with exact acoustic timestamps
    meetingTranscript.push({
      text: transcriptText,
      startTime: result.speechStartTime,
      endTime: result.speechEndTime
    });
  }
};

// Post-Meeting Replay: Synchronize caption cues with the recorded meeting video
function initializeRecordingCaptions(recordedVideoElement) {
  const captionTrack = recordedVideoElement.addTextTrack('captions', 'Meeting Captions');
  captionTrack.mode = 'showing';

  for (const entry of meetingTranscript) {
    captionTrack.addCue(new VTTCue(entry.startTime, entry.endTime, entry.text));
  }
}

The SpeechRecognitionResult interface can be extended with two new read-only attributes to provide the exact acoustic boundaries of a recognized speech segment. These timestamps are measured in seconds relative to the start of the audio stream ($t = 0.0\text{s}$), which is marked by the audiostart event.

• speechStartTime: The start timestamp of the speech segment in seconds.
• speechEndTime: The end timestamp of the speech segment in seconds.

Using seconds as a double ensures compatibility with the Web Audio API and HTML Media Elements, allowing for seamless synchronization without manual unit conversions.

partial interface SpeechRecognitionResult {
    // Start timestamp of the speech segment in seconds relative to the start of the audio stream (0.0s).
    readonly attribute double speechStartTime;

    // End timestamp of the speech segment in seconds relative to the start of the audio stream.
    readonly attribute double speechEndTime;
};