Implementing Real-time Speech-to-Text Transcription using AWS Transcribe with Next.js API Routes

[10000leaves]

I want to create an API for speech-to-text transcription using AWS SDK v3's AWS Transcribe with Next.js API Routes.

I tried to create the following API and hooks based on this reference code, but unfortunately, it's not working as expected. I would really appreciate any guidance.
https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/javascriptv3/example_code/cross-services/transcribe-streaming-app/src/libs/transcribeClient.js

Here's my current implementation that I'm struggling with.

package

• "@aws-sdk/client-transcribe-streaming": "^3.693.0"
• "microphone-stream": "^6.0.1"
• "immutability-helper": "^3.1.1"
• "buffer": "^6.0.3"

API

// src/app/api/transcribe/route.ts
import { NextRequest, NextResponse } from 'next/server';
import {
  TranscribeStreamingClient,
  StartStreamTranscriptionCommand,
  AudioStream,
} from '@aws-sdk/client-transcribe-streaming';

export const runtime = 'nodejs';

export async function POST(request: NextRequest) {
  let client: TranscribeStreamingClient | undefined;
  
  try {
    client = new TranscribeStreamingClient({
      region: process.env.NEXT_AWS_REGION,
      credentials: {
        accessKeyId: process.env.NEXT_AWS_TRANSCRIBE_ACCESS_KEY_ID!,
        secretAccessKey: process.env.NEXT_AWS_TRANSCRIBE_SECRET_ACCESS_KEY!,
      },
    });

    const { audioChunk } = await request.json();

    const audioStream: AsyncIterable<AudioStream> = {
      async *[Symbol.asyncIterator]() {
        yield {
          AudioEvent: {
            AudioChunk: Buffer.from(audioChunk)
          }
        };
      }
    };

    const command = new StartStreamTranscriptionCommand({
      LanguageCode: 'ja-JP',
      MediaEncoding: 'pcm',
      MediaSampleRateHertz: 44100,
      AudioStream: audioStream,
    });

    const response = await client.send(command);
    let transcriptResult = '';

    if (response.TranscriptResultStream) {
      for await (const event of response.TranscriptResultStream) {
        if (event.TranscriptEvent?.Transcript?.Results?.[0]) {
          const result = event.TranscriptEvent.Transcript.Results[0];
          if (result.Alternatives?.[0]?.Transcript) {
            transcriptResult = result.Alternatives[0].Transcript;
          }
        }
      }
    }

    return NextResponse.json({ transcript: transcriptResult });
  } catch (error) {
    console.error('Error in transcription:', error);
    return NextResponse.json(
      { error: 'Transcription failed', details: error },
      { status: 500 }
    );
  } finally {
    client?.destroy();
  }
}

hooks

// src/hooks/useTranscribe.ts
import MicrophoneStream from 'microphone-stream';
import { useState } from 'react';
import update from 'immutability-helper';
import { Buffer } from 'buffer';

const pcmEncodeChunk = (chunk: Buffer) => {
  const input = MicrophoneStream.toRaw(chunk);
  let offset = 0;
  const buffer = new ArrayBuffer(input.length * 2);
  const view = new DataView(buffer);
  for (let i = 0; i < input.length; i++, offset += 2) {
    const s = Math.max(-1, Math.min(1, input[i]));
    view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
  }
  return Buffer.from(buffer);
};

const useTranscribe = () => {
  const [micStream, setMicStream] = useState<MicrophoneStream | undefined>();
  const [recording, setRecording] = useState(false);
  const [error, setError] = useState(false);
  const [errorMessage, setErrorMessage] = useState<string>('');
  const [transcripts, setTranscripts] = useState<
    {
      isPartial: boolean;
      transcript: string;
    }[]
  >([]);

  const startStream = async (mic: MicrophoneStream) => {
    for await (const chunk of mic as unknown as Buffer[]) {
      try {
        const encodedChunk = pcmEncodeChunk(chunk);
        
        const response = await fetch('/api/transcribe', {
          method: 'POST',
          headers: {
            'Content-Type': 'application/json',
          },
          body: JSON.stringify({ 
            audioChunk: Array.from(encodedChunk)
          }),
        });

        if (!response.ok) {
          throw new Error('Transcription request failed');
        }

        const data = await response.json();
        
        if (data.transcript) {
          setTranscripts((prev) => {
            const index = prev.length - 1;
            if (prev.length === 0 || !prev[prev.length - 1].isPartial) {
              return update(prev, {
                $push: [{ isPartial: false, transcript: data.transcript }],
              });
            } else {
              return update(prev, {
                $splice: [[index, 1, { isPartial: false, transcript: data.transcript }]],
              });
            }
          });
        }
      } catch (e) {
        console.error('Error in transcription:', e);
        setError(true);
        setErrorMessage(e');
      }
    }
  };

  const startTranscription = async () => {
    const mic = new MicrophoneStream();
    try {
      setMicStream(mic);
      mic.setStream(
        await window.navigator.mediaDevices.getUserMedia({
          video: false,
          audio: {
            channelCount: 1,
            sampleRate: 44100,
          },
        })
      );
      
      setError(false);
      setErrorMessage('');
      setTranscripts([]);
      setRecording(true);
      await startStream(mic);
    } catch (e) {
      console.error('Error starting transcription:', e);
      setError(true);
      setErrorMessage(e);
    } finally {
      mic.stop();
      setRecording(false);
      setMicStream(undefined);
    }
  };

  const stopTranscription = () => {
    if (micStream) {
      micStream.stop();
      setRecording(false);
      setMicStream(undefined);
    }
  };

  const resetTranscripts = () => {
    setError(false);
    setErrorMessage('');
    setTranscripts([]);
  };

  return {
    startTranscription,
    stopTranscription,
    resetTranscripts,
    recording,
    transcripts,
    error,
    errorMessage,
  };
};

export default useTranscribe;

Could someone please help me understand what I'm doing wrong and how to implement this correctly?
I'm stuck and would really appreciate any assistance.