moonshive_vs_faster-whisper-tiny.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install gradio gtts pydub faster-whisper jiwer tensorflow matplotlib numpy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gradio as gr\n",
    "import moonshine\n",
    "import warnings\n",
    "import tempfile\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import jiwer\n",
    "import numpy as np\n",
    "from gtts import gTTS\n",
    "from pydub import AudioSegment\n",
    "from faster_whisper import WhisperModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs('charts', exist_ok=True)\n",
    "warnings.filterwarnings('ignore')\n",
    "model_name = \"tiny\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cumulative_moonshine_times = []\n",
    "cumulative_faster_whisper_times = []\n",
    "cumulative_moonshine_wers = []\n",
    "cumulative_faster_whisper_wers = []\n",
    "cumulative_percentage_differences = []\n",
    "cumulative_absolute_differences = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def google_text_to_speech(text: str, language: str = 'en') -> str:\n",
    "    \"\"\"Converts text to speech using Google Text To Speech and saves it as a .wav file.\n",
    "\n",
    "    Args:\n",
    "        text (str): The text to convert to speech.\n",
    "        language (str): The language code for the speech. Default is 'en' for English.\n",
    "\n",
    "    Returns:\n",
    "        str: The path to the saved .wav audio file.\n",
    "    \"\"\"\n",
    "    tts = gTTS(text=text, lang=language, slow=False)\n",
    "    with tempfile.NamedTemporaryFile(delete=False, suffix=\".wav\") as temp_audio_file:\n",
    "        tts.save(temp_audio_file.name)\n",
    "        return temp_audio_file.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transcribe_with_moonshine(audio_file: str) -> tuple[str, float]:\n",
    "    \"\"\"Transcribes audio using the Moonshine model.\n",
    "\n",
    "    Args:\n",
    "        audio_file (str): The path to the audio file to transcribe.\n",
    "\n",
    "    Returns:\n",
    "        tuple: A tuple containing the transcribed text and processing time.\n",
    "    \"\"\"\n",
    "    audio = AudioSegment.from_file(audio_file)\n",
    "    segment_duration = 30 * 1000  \n",
    "    transcriptions = []\n",
    "    start_time = time.time()\n",
    "\n",
    "    for i in range(0, len(audio), segment_duration):\n",
    "        segment = audio[i:i + segment_duration]\n",
    "        with tempfile.NamedTemporaryFile(delete=False, suffix=\".wav\") as temp_audio_file:\n",
    "            segment.export(temp_audio_file.name, format=\"wav\")\n",
    "            transcription = moonshine.transcribe(temp_audio_file.name)[0]\n",
    "            transcriptions.append(transcription)\n",
    "\n",
    "    processing_time = time.time() - start_time\n",
    "    return ' '.join(transcriptions), processing_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transcribe_with_faster_whisper(audio_file: str, model_name: str) -> tuple[str, float]:\n",
    "    \"\"\"Transcribes audio using the Faster-Whisper model.\n",
    "\n",
    "    Args:\n",
    "        audio_file (str): The path to the audio file to transcribe.\n",
    "        model_name (str): The model name to use for transcription.\n",
    "\n",
    "    Returns:\n",
    "        tuple: A tuple containing the transcribed text and processing time.\n",
    "    \"\"\"\n",
    "    model = WhisperModel(model_size_or_path=model_name, device=\"cpu\", compute_type=\"int8\")\n",
    "    start_time = time.time()\n",
    "    segments, _ = model.transcribe(audio_file, beam_size=5, language='en')\n",
    "    \n",
    "    transcription = ' '.join([segment.text for segment in segments])\n",
    "    processing_time = time.time() - start_time\n",
    "    \n",
    "    return transcription, processing_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_wer(reference: str, hypothesis: str) -> tuple[float, str]:\n",
    "    \"\"\"Calculates the Word Error Rate (WER) between reference and hypothesis transcriptions.\n",
    "\n",
    "    Args:\n",
    "        reference (str): The reference transcription.\n",
    "        hypothesis (str): The transcribed hypothesis.\n",
    "\n",
    "    Returns:\n",
    "        tuple: A tuple containing the WER and the normalized hypothesis transcription.\n",
    "    \"\"\"\n",
    "    transformation = jiwer.Compose([\n",
    "        jiwer.ExpandCommonEnglishContractions(),\n",
    "        jiwer.RemoveEmptyStrings(),\n",
    "        jiwer.ToLowerCase(),\n",
    "        jiwer.RemoveMultipleSpaces(),\n",
    "        jiwer.Strip(),\n",
    "        jiwer.RemovePunctuation()])\n",
    "    \n",
    "    transformed_reference = transformation(reference)\n",
    "    transformed_hypothesis = transformation(hypothesis)\n",
    "\n",
    "    return jiwer.wer(reference=transformed_reference, hypothesis=transformed_hypothesis), transformed_hypothesis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_comparison_chart(moonshine_time: float, faster_whisper_time: float, wer_moonshine: float, wer_whisper: float) -> str:\n",
    "    \"\"\"Creates a comparison chart for latency and WER.\n",
    "\n",
    "    Args:\n",
    "        moonshine_time (float): Processing time for Moonshine.\n",
    "        faster_whisper_time (float): Processing time for Faster-Whisper.\n",
    "        wer_moonshine (float): WER for Moonshine.\n",
    "        wer_whisper (float): WER for Faster-Whisper.\n",
    "\n",
    "    Returns:\n",
    "        str: The path to the saved comparison chart image.\n",
    "    \"\"\"\n",
    "    models = ['Moonshine', 'Faster-Whisper Tiny']\n",
    "    times = [moonshine_time, faster_whisper_time]\n",
    "    wers = [wer_moonshine, wer_whisper]\n",
    "\n",
    "    plt.figure(figsize=(12, 6))\n",
    "    \n",
    "    plt.subplot(1, 2, 1)\n",
    "    plt.bar(models, times, color=['red', 'yellow'])\n",
    "    plt.ylabel('Processing Time (seconds)')\n",
    "    plt.title('Latency Comparison')\n",
    "\n",
    "    plt.subplot(1, 2, 2)\n",
    "    plt.bar(models, wers, color=['orange', 'green'])\n",
    "    plt.ylabel('Word Error Rate (WER)')\n",
    "    plt.title('WER Comparison')\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('charts/latency_and_wer_comparison.png')\n",
    "    \n",
    "    return 'charts/latency_and_wer_comparison.png'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_cumulative_statistics(moonshine_time: float, faster_whisper_time: float, wer_moonshine: float, wer_whisper: float) -> None:\n",
    "    \"\"\"Updates cumulative statistics with the latest transcription results.\n",
    "\n",
    "    Args:\n",
    "        moonshine_time (float): Processing time for Moonshine.\n",
    "        faster_whisper_time (float): Processing time for Faster-Whisper.\n",
    "        wer_moonshine (float): WER for Moonshine.\n",
    "        wer_whisper (float): WER for Faster-Whisper.\n",
    "    \"\"\"\n",
    "    cumulative_moonshine_times.append(moonshine_time)\n",
    "    cumulative_faster_whisper_times.append(faster_whisper_time)\n",
    "    cumulative_moonshine_wers.append(wer_moonshine)\n",
    "    cumulative_faster_whisper_wers.append(wer_whisper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_cumulative_chart() -> str:\n",
    "    \"\"\"Creates a cumulative chart for average latency and WER.\n",
    "\n",
    "    Returns:\n",
    "        str: The path to the saved cumulative chart image.\n",
    "    \"\"\"\n",
    "    global cumulative_moonshine_times, cumulative_faster_whisper_times, cumulative_moonshine_wers, cumulative_faster_whisper_wers\n",
    "\n",
    "    models = ['Moonshine', 'Faster-Whisper Tiny']\n",
    "\n",
    "    avg_moonshine_time = np.mean(cumulative_moonshine_times) \n",
    "    avg_faster_whisper_time = np.mean(cumulative_faster_whisper_times) \n",
    "    avg_moonshine_wer = np.mean(cumulative_moonshine_wers)\n",
    "    avg_faster_whisper_wer = np.mean(cumulative_faster_whisper_wers) \n",
    "\n",
    "    plt.figure(figsize=(12, 6))\n",
    "    plt.subplot(1, 2, 1)\n",
    "    plt.bar(models, [avg_moonshine_time, avg_faster_whisper_time], color=['red', 'yellow'])\n",
    "    plt.ylabel('Average Processing Time (seconds)')\n",
    "    plt.title('Average Latency Over All Runs')\n",
    "\n",
    "    plt.subplot(1, 2, 2)\n",
    "    plt.bar(models, [avg_moonshine_wer, avg_faster_whisper_wer], color=['orange', 'green'])\n",
    "    plt.ylabel('Average WER')\n",
    "    plt.title('Average WER Over All Runs')\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('charts/cumulative_latency_and_wer_comparison.png')\n",
    "    \n",
    "    return 'charts/cumulative_latency_and_wer_comparison.png'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(text: str = None) -> tuple:\n",
    "    \"\"\"Main function to perform text-to-speech, transcription, WER calculation, and chart creation.\n",
    "\n",
    "    Args:\n",
    "        text (str, optional): Input text for conversion. If None, prompts for text input.\n",
    "\n",
    "    Returns:\n",
    "        tuple: A tuple containing the audio file path, normalized transcriptions, statistics, and chart file paths.\n",
    "    \"\"\"\n",
    "    if text is None:\n",
    "        return \"Please provide text input.\", None\n",
    "\n",
    "    audio_file = google_text_to_speech(text)\n",
    "\n",
    "    moonshine_transcription, moonshine_time = transcribe_with_moonshine(audio_file)\n",
    "    faster_whisper_transcription, faster_whisper_time = transcribe_with_faster_whisper(audio_file, \"tiny\")\n",
    "\n",
    "    wer_moonshine, normalized_moonshine_transcription = calculate_wer(text, moonshine_transcription)\n",
    "    wer_whisper, normalized_faster_whisper_transcription = calculate_wer(text, faster_whisper_transcription)\n",
    "\n",
    "    update_cumulative_statistics(moonshine_time, faster_whisper_time, wer_moonshine, wer_whisper)\n",
    "\n",
    "    absolute_difference = abs(moonshine_time - faster_whisper_time)\n",
    "    percentage_difference = (absolute_difference / min(moonshine_time, faster_whisper_time)) * 100 if min(moonshine_time, faster_whisper_time) > 0 else 0\n",
    "\n",
    "    cumulative_percentage_differences.append(percentage_difference)\n",
    "    cumulative_absolute_differences.append(absolute_difference)\n",
    "    \n",
    "    chart_file = create_comparison_chart(moonshine_time, faster_whisper_time, wer_moonshine, wer_whisper)\n",
    "    cumulative_chart_file = create_cumulative_chart()\n",
    "\n",
    "    statistics = (f\"Percentage Difference Latency: {percentage_difference/100:.2f}x\\n\"\n",
    "                  f\"Absolute Difference Latency: {absolute_difference:.2f} seconds\\n\"\n",
    "                  f\"\\n\"\n",
    "                  f\"Moonshine Latency: {moonshine_time:.2f} seconds\\n\"\n",
    "                  f\"Faster-Whisper Tiny Latency: {faster_whisper_time:.2f} seconds\\n\"\n",
    "                  f\"\\n\"\n",
    "                  f\"WER Moonshine: {wer_moonshine*100:.2f}%\\n\"\n",
    "                  f\"WER Faster-Whisper Tiny: {wer_whisper*100:.2f}%\")\n",
    "\n",
    "    total_runs = len(cumulative_moonshine_times)\n",
    "    cumulative_statistics = (f\"Total Runs: {total_runs}\\n\"\n",
    "                             f\"\\n\"\n",
    "                             f\"Average Percentage Difference Latency: {np.mean(cumulative_percentage_differences)/100:.2f}x\\n\"\n",
    "                             f\"Average Absolute Difference Latency: {np.mean(cumulative_absolute_differences):.2f} seconds\\n\"\n",
    "                             f\"\\n\"\n",
    "                             f\"Average Moonshine Latency: {np.mean(cumulative_moonshine_times):.2f} seconds\\n\"\n",
    "                             f\"Average Faster-Whisper Tiny Latency: {np.mean(cumulative_faster_whisper_times):.2f} seconds\\n\"\n",
    "                             f\"\\n\"\n",
    "                             f\"Average Moonshine WER: {np.mean(cumulative_moonshine_wers)*100:.2f}%\\n\"\n",
    "                             f\"Average Faster-Whisper Tiny WER: {np.mean(cumulative_faster_whisper_wers)*100:.2f}%\")\n",
    "\n",
    "    return (audio_file, normalized_moonshine_transcription, normalized_faster_whisper_transcription,\n",
    "            statistics, chart_file, cumulative_statistics, cumulative_chart_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iface = gr.Interface(\n",
    "    fn=main,\n",
    "    inputs=[\n",
    "        gr.Textbox(lines=5, placeholder=\"Enter text here (English only)\", label=\"📝 Text Input (English only)\")\n",
    "    ],\n",
    "    outputs=[\n",
    "    gr.Audio(label=\"🔊 Google Text to Speech Audio Output\"),\n",
    "    gr.Textbox(label=\"📝 Moonshine Transcription Output\"), \n",
    "    gr.Textbox(label=\"📝 Faster-Whisper Tiny Transcription Output\"),\n",
    "    gr.Textbox(label=\"📈 Statistics Output\"), \n",
    "    gr.Image(label=\"📊 Chart\"),\n",
    "    gr.Textbox(label=\"📈 Cumulative Statistics Output\"), \n",
    "    gr.Image(label=\"📊 Cumulative Chart\")\n",
    "    ],\n",
    "    title=\"Audio Transcription Benchmark: Moonshine vs Faster-Whisper Tiny\",\n",
    "    description = \"\"\"\n",
    "    Compare Latency and WER for Each Transcription Run: The comparator works by taking textual inputs that are converted into .wav audio files using Google Text To Speech 🔊. These audio files are then used as inputs for the two models, Moonshine 🗣️ and Whisper Tiny 💬. Each model will be evaluated and compared for individual runs and across all runs that the user wants to conduct, providing both single and cumulative statistics Charts 📊 regarding latency and Word Error Rate (WER) 📈. The WER is evaluated using the **jiwer** library, which assesses the accuracy of the transcription by comparing the reference input text with the generated transcription. The WER calculation is based on the formula:\n",
    "\n",
    "    WER = (S + D + I) / N\n",
    "\n",
    "    where \\( S \\) is the number of substitutions, \\( D \\) is the number of deletions, \\( I \\) is the number of insertions, and \\( N \\) is the total number of words in the reference text.\n",
    "\n",
    "    Before the evaluation, several normalization steps are applied to the transcription results to ensure consistency and accuracy ✨. These steps include expanding common English contractions to their full forms, removing any empty strings, converting all text to lowercase, eliminating multiple spaces, stripping leading and trailing whitespace, and removing punctuation to create a standardized basis for comparison and enhancing the reliability of the WER measurement.\n",
    "\n",
    "    Additionally, you can save the results (statistics and charts) of each run and cumulative results by pressing the \"Flag\" button. The results will be saved in the local `.gradio` directory, in subdirectories corresponding to the fields and chat IDs.\n",
    "    \"\"\"\n",
    "    )\n",
    "\n",
    "iface.launch(share=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}