\ No newline at end of file
diff --git a/notebooks/whisper-asr-genai/whisper-asr-genai.ipynb b/notebooks/whisper-asr-genai/whisper-asr-genai.ipynb
index f83573f4bcc..2e354bae169 100644
--- a/notebooks/whisper-asr-genai/whisper-asr-genai.ipynb
+++ b/notebooks/whisper-asr-genai/whisper-asr-genai.ipynb
@@ -22,7 +22,6 @@
"\n",
"\n",
"\n",
- "\n",
"### Installation Instructions\n",
"\n",
"This is a self-contained example that relies solely on its own code.\n",
@@ -33,8 +32,6 @@
"![](\"https://static.scarf.sh/a.png?x-pxid=5b5a4db0-7875-4bfb-bdbd-01698b5b1a77&file=notebooks/whisper-asr-genai/whisper-asr-genai.ipynb\")
\n",
"\n",
"\n",
- "\n",
- "\n",
"#### Table of contents:\n",
"\n",
"- [Prerequisites](#Prerequisites)\n",
@@ -64,7 +61,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 22,
"id": "bb9fc7f3-cea0-4adf-9ee6-4a3d15931db7",
"metadata": {
"ExecuteTime": {
@@ -128,7 +125,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
- "model_id": "c5035bf98145426fbc8f05edc8d7924a",
+ "model_id": "ab4d5dd6e7794d5184db6d9b49281717",
"version_major": 2,
"version_minor": 0
},
@@ -156,6 +153,10 @@
" \"openai/whisper-tiny\",\n",
" ],\n",
" \"English-only models\": [\n",
+ " \"distil-whisper/distil-large-v2\",\n",
+ " \"distil-whisper/distil-large-v3\",\n",
+ " \"distil-whisper/distil-medium.en\",\n",
+ " \"distil-whisper/distil-small.en\",\n",
" \"openai/whisper-medium.en\",\n",
" \"openai/whisper-small.en\",\n",
" \"openai/whisper-base.en\",\n",
@@ -175,22 +176,22 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 5,
"id": "9f42d9b8",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
- "model_id": "3209303207294a53b9b41ac90241982c",
+ "model_id": "fb48f63ba96c43388fa987c6c78f6e2c",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
- "Dropdown(description='Model:', index=6, options=('openai/whisper-large-v3', 'openai/whisper-large-v2', 'openai…"
+ "Dropdown(description='Model:', index=7, options=('openai/whisper-large-v3-turbo', 'openai/whisper-large-v3', '…"
]
},
- "execution_count": 3,
+ "execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
@@ -208,7 +209,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 6,
"id": "e5382431-497e-4688-b4ec-8958a92163e7",
"metadata": {
"ExecuteTime": {
@@ -248,7 +249,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 7,
"id": "620020a6",
"metadata": {},
"outputs": [
@@ -256,16 +257,16 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "'data/librispeech_asr_demo_validation_short.wav' already exists.\n"
+ "'data/courtroom.wav' already exists.\n"
]
},
{
"data": {
"text/plain": [
- "PosixPath('/home/labuser/work/notebook/openvino_notebooks/notebooks/whisper-asr-genai/data/librispeech_asr_demo_validation_short.wav')"
+ "PosixPath('/home/labuser/work/notebook/openvino_notebooks/notebooks/whisper-asr-genai/data/courtroom.wav')"
]
},
- "execution_count": 5,
+ "execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
@@ -273,11 +274,11 @@
"source": [
"from notebook_utils import download_file\n",
"\n",
- "en_example_short = Path(\"data\", \"librispeech_asr_demo_validation_short.wav\")\n",
+ "en_example_short = Path(\"data\", \"courtroom.wav\")\n",
"\n",
"# a wav sample\n",
"download_file(\n",
- " \"https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/librispeech_asr_demo_validation_0.wav\",\n",
+ " \"https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/courtroom.wav\",\n",
" en_example_short.name,\n",
" directory=en_example_short.parent,\n",
")"
@@ -285,7 +286,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 8,
"id": "61558185",
"metadata": {},
"outputs": [],
@@ -307,7 +308,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 9,
"id": "218836e9",
"metadata": {},
"outputs": [
@@ -316,7 +317,7 @@
"text/html": [
"\n",
" \n",
" "
@@ -340,7 +341,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Result: Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.\n"
+ "Result: Colonel Jessif, did you order the code rate? You don't have to answer that question. I'll answer the question. You want answers? I think I'm entitled. You want answers? I want the truth. You can't handle the truth.\n"
]
}
],
@@ -369,14 +370,14 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 10,
"id": "9c55f94d",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
- "model_id": "88e7bfa5930f4d4ab70361207e3011ef",
+ "model_id": "0faae0d4b763434f83d90306719f5767",
"version_major": 2,
"version_minor": 0
},
@@ -384,7 +385,7 @@
"Dropdown(description='Dataset language:', index=4, options=('japanese', 'dutch', 'french', 'spanish', 'italian…"
]
},
- "execution_count": 8,
+ "execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
@@ -408,7 +409,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 11,
"id": "515bec62",
"metadata": {},
"outputs": [],
@@ -424,7 +425,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 12,
"id": "bb3b3e3f",
"metadata": {},
"outputs": [
@@ -489,66 +490,10 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": null,
"id": "36f756d5",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, onnx, openvino\n"
- ]
- },
- {
- "data": {
- "text/markdown": [
- "**Export command:**"
- ],
- "text/plain": [
- "
diff --git a/notebooks/whisper-subtitles-generation/gradio_helper.py b/notebooks/whisper-subtitles-generation/gradio_helper.py
index 9edac14e3e2..4a658202fd5 100644
--- a/notebooks/whisper-subtitles-generation/gradio_helper.py
+++ b/notebooks/whisper-subtitles-generation/gradio_helper.py
@@ -1,12 +1,29 @@
from typing import Callable
+from pathlib import Path
import gradio as gr
-def make_demo(fn: Callable, quantized: bool):
+def make_demo(fn: Callable, quantized: bool, sample_path: Path):
demo = gr.Interface(
+ description=f"""
+
+
+ """,
fn=fn,
inputs=[
- gr.Textbox(label="YouTube URL"),
+ gr.Video(label="Video"),
gr.Radio(["Transcribe", "Translate"], value="Transcribe"),
gr.Checkbox(
value=quantized,
@@ -15,7 +32,7 @@ def make_demo(fn: Callable, quantized: bool):
),
],
outputs="video",
- examples=[["https://youtu.be/kgL5LBM-hFI", "Transcribe"]],
+ examples=[[sample_path, "Transcribe"]],
allow_flagging="never",
)
diff --git a/notebooks/whisper-subtitles-generation/utils.py b/notebooks/whisper-subtitles-generation/utils.py
deleted file mode 100644
index ff43b5b5eb0..00000000000
--- a/notebooks/whisper-subtitles-generation/utils.py
+++ /dev/null
@@ -1,323 +0,0 @@
-from collections import namedtuple
-from functools import partial
-import openvino as ov
-from pathlib import Path
-from typing import List, Optional, Union
-from math import floor, ceil
-
-import io
-from scipy.io import wavfile
-from moviepy.editor import VideoFileClip
-
-import numpy as np
-import torch
-
-from whisper.decoding import DecodingTask, Inference, DecodingOptions, DecodingResult
-
-
-class OpenVINOAudioEncoder(torch.nn.Module):
- """
- Helper for inference Whisper encoder model with OpenVINO
- """
-
- def __init__(self, core: ov.Core, model_path: Path, device="CPU"):
- super().__init__()
- self.model = core.read_model(model_path)
- self.compiled_model = core.compile_model(self.model, device)
- self.output_blob = self.compiled_model.output(0)
-
- def forward(self, mel: torch.Tensor):
- """
- Inference OpenVINO whisper encoder model.
-
- Parameters:
- mel: input audio fragment mel spectrogram.
- Returns:
- audio_features: torch tensor with encoded audio features.
- """
- return torch.from_numpy(self.compiled_model(mel)[self.output_blob])
-
-
-class OpenVINOTextDecoder(torch.nn.Module):
- """
- Helper for inference OpenVINO decoder model
- """
-
- def __init__(self, core: ov.Core, model_path: Path, device: str = "CPU"):
- super().__init__()
- self._core = core
- self.model = core.read_model(model_path)
- self._input_names = [inp.any_name for inp in self.model.inputs]
- self.compiled_model = core.compile_model(self.model, device)
- self.device = device
- self.blocks = []
-
- def init_past_inputs(self, feed_dict):
- """
- Initialize cache input for first step.
-
- Parameters:
- feed_dict: Dictonary with inputs for inference
- Returns:
- feed_dict: updated feed_dict
- """
- beam_size = feed_dict["x"].shape[0]
- audio_len = feed_dict["xa"].shape[2]
- previous_seq_len = 0
- for name in self._input_names:
- if name in ["x", "xa"]:
- continue
- feed_dict[name] = ov.Tensor(np.zeros((beam_size, previous_seq_len, audio_len), dtype=np.float32))
- return feed_dict
-
- def preprocess_kv_cache_inputs(self, feed_dict, kv_cache):
- """
- Transform kv_cache to inputs
-
- Parameters:
- feed_dict: dictionary with inputs for inference
- kv_cache: dictionary with cached attention hidden states from previous step
- Returns:
- feed_dict: updated feed dictionary with additional inputs
- """
- if not kv_cache:
- return self.init_past_inputs(feed_dict)
- for k, v in zip(self._input_names[2:], kv_cache):
- feed_dict[k] = ov.Tensor(v)
- return feed_dict
-
- def postprocess_outputs(self, outputs):
- """
- Transform model output to format expected by the pipeline
-
- Parameters:
- outputs: outputs: raw inference results.
- Returns:
- logits: decoder predicted token logits
- kv_cache: cached attention hidden states
- """
- logits = torch.from_numpy(outputs[0])
- kv_cache = list(outputs.values())[1:]
- return logits, kv_cache
-
- def forward(self, x: torch.Tensor, xa: torch.Tensor, kv_cache: Optional[dict] = None):
- """
- Inference decoder model.
-
- Parameters:
- x: torch.LongTensor, shape = (batch_size, <= n_ctx) the text tokens
- xa: torch.Tensor, shape = (batch_size, n_mels, n_audio_ctx)
- the encoded audio features to be attended on
- kv_cache: Dict[str, torch.Tensor], attention modules hidden states cache from previous steps
- Returns:
- logits: decoder predicted logits
- kv_cache: updated kv_cache with current step hidden states
- """
- feed_dict = {"x": ov.Tensor(x.numpy()), "xa": ov.Tensor(xa.numpy())}
- feed_dict = self.preprocess_kv_cache_inputs(feed_dict, kv_cache)
- res = self.compiled_model(feed_dict)
- return self.postprocess_outputs(res)
-
-
-class OpenVINOInference(Inference):
- """
- Wrapper for inference interface
- """
-
- def __init__(self, model: "Whisper", initial_token_length: int):
- self.model: "Whisper" = model
- self.initial_token_length = initial_token_length
- self.kv_cache = {}
-
- def logits(self, tokens: torch.Tensor, audio_features: torch.Tensor) -> torch.Tensor:
- """
- getting logits for given tokens sequence and audio features and save kv_cache
-
- Parameters:
- tokens: input tokens
- audio_features: input audio features
- Returns:
- logits: predicted by decoder logits
- """
- if tokens.shape[-1] > self.initial_token_length:
- # only need to use the last token except in the first forward pass
- tokens = tokens[:, -1:]
- logits, self.kv_cache = self.model.decoder(tokens, audio_features, kv_cache=self.kv_cache)
- return logits
-
- def cleanup_caching(self):
- """
- Reset kv_cache to initial state
- """
- self.kv_cache = {}
-
- def rearrange_kv_cache(self, source_indices):
- """
- Update hidden states cache for selected sequences
- Parameters:
- source_indicies: sequences indicies
- Returns:
- None
- """
- for module, tensor in self.kv_cache.items():
- # update the key/value cache to contain the selected sequences
- self.kv_cache[module] = tensor[source_indices].detach()
-
-
-class OpenVINODecodingTask(DecodingTask):
- """
- Class for decoding using OpenVINO
- """
-
- def __init__(self, model: "Whisper", options: DecodingOptions):
- super().__init__(model, options)
- self.inference = OpenVINOInference(model, len(self.initial_tokens))
-
-
-def patch_whisper_for_ov_inference(model):
- @torch.no_grad()
- def decode(
- model: "Whisper",
- mel: torch.Tensor,
- options: DecodingOptions = DecodingOptions(),
- ) -> Union[DecodingResult, List[DecodingResult]]:
- """
- Performs decoding of 30-second audio segment(s), provided as Mel spectrogram(s).
-
- Parameters
- ----------
- model: Whisper
- the Whisper model instance
-
- mel: torch.Tensor, shape = (80, 3000) or (*, 80, 3000)
- A tensor containing the Mel spectrogram(s)
-
- options: DecodingOptions
- A dataclass that contains all necessary options for decoding 30-second segments
-
- Returns
- -------
- result: Union[DecodingResult, List[DecodingResult]]
- The result(s) of decoding contained in `DecodingResult` dataclass instance(s)
- """
- single = mel.ndim == 2
- if single:
- mel = mel.unsqueeze(0)
-
- result = OpenVINODecodingTask(model, options).run(mel)
-
- if single:
- result = result[0]
-
- return result
-
- Parameter = namedtuple("Parameter", ["device"])
-
- def parameters():
- return iter([Parameter(torch.device("cpu"))])
-
- def logits(model, tokens: torch.Tensor, audio_features: torch.Tensor):
- """
- Override for logits extraction method
- Parameters:
- tokens: input tokens
- audio_features: input audio features
- Returns:
- logits: decoder predicted logits
- """
- return model.decoder(tokens, audio_features, None)[0]
-
- model.parameters = parameters
- model.decode = partial(decode, model)
- model.logits = partial(logits, model)
-
-
-def resample(audio, src_sample_rate, dst_sample_rate):
- """
- Resample audio to specific sample rate
-
- Parameters:
- audio: input audio signal
- src_sample_rate: source audio sample rate
- dst_sample_rate: destination audio sample rate
- Returns:
- resampled_audio: input audio signal resampled with dst_sample_rate
- """
- if src_sample_rate == dst_sample_rate:
- return audio
- duration = audio.shape[0] / src_sample_rate
- resampled_data = np.zeros(shape=(int(duration * dst_sample_rate)), dtype=np.float32)
- x_old = np.linspace(0, duration, audio.shape[0], dtype=np.float32)
- x_new = np.linspace(0, duration, resampled_data.shape[0], dtype=np.float32)
- resampled_audio = np.interp(x_new, x_old, audio)
- return resampled_audio.astype(np.float32)
-
-
-def audio_to_float(audio):
- """
- convert audio signal to floating point format
- """
- return audio.astype(np.float32) / np.iinfo(audio.dtype).max
-
-
-def get_audio(video_file):
- """
- Extract audio signal from a given video file, then convert it to float,
- then mono-channel format and resample it to the expected sample rate
-
- Parameters:
- video_file: path to input video file
- Returns:
- resampled_audio: mono-channel float audio signal with 16000 Hz sample rate
- extracted from video
- duration: duration of video fragment in seconds
- """
- input_video = VideoFileClip(str(video_file))
- duration = input_video.duration
- input_video.audio.write_audiofile(video_file.stem + ".wav", verbose=False, logger=None)
- input_audio_file = video_file.stem + ".wav"
- sample_rate, audio = wavfile.read(io.BytesIO(open(input_audio_file, "rb").read()))
- audio = audio_to_float(audio)
- if audio.ndim == 2:
- audio = audio.mean(axis=1)
-
- # The model expects mono-channel audio with a 16000 Hz sample rate, represented in floating point range. When the
- # audio from the input video does not meet these requirements, we will need to apply preprocessing.
- resampled_audio = resample(audio, sample_rate, 16000)
- return resampled_audio, duration
-
-
-def format_timestamp(seconds: float):
- """
- format time in srt-file expected format
- """
- assert seconds >= 0, "non-negative timestamp expected"
- milliseconds = round(seconds * 1000.0)
-
- hours = milliseconds // 3_600_000
- milliseconds -= hours * 3_600_000
-
- minutes = milliseconds // 60_000
- milliseconds -= minutes * 60_000
-
- seconds = milliseconds // 1_000
- milliseconds -= seconds * 1_000
-
- return (f"{hours}:" if hours > 0 else "00:") + f"{minutes:02d}:{seconds:02d},{milliseconds:03d}"
-
-
-def prepare_srt(transcription, filter_duration=None):
- """
- Format transcription into srt file format
- """
- segment_lines = []
- for segment in transcription["segments"]:
- if filter_duration is not None and (segment["start"] >= floor(filter_duration) or segment["end"] > ceil(filter_duration) + 1):
- break
- segment_lines.append(str(segment["id"] + 1) + "\n")
- time_start = format_timestamp(segment["start"])
- time_end = format_timestamp(segment["end"])
- time_str = f"{time_start} --> {time_end}\n"
- segment_lines.append(time_str)
- segment_lines.append(segment["text"] + "\n\n")
- return segment_lines
diff --git a/notebooks/whisper-subtitles-generation/whisper-subtitles-generation.ipynb b/notebooks/whisper-subtitles-generation/whisper-subtitles-generation.ipynb
index 0ed0c8ac67d..342e2bedce7 100644
--- a/notebooks/whisper-subtitles-generation/whisper-subtitles-generation.ipynb
+++ b/notebooks/whisper-subtitles-generation/whisper-subtitles-generation.ipynb
@@ -13,7 +13,7 @@
"\n",
"You can find more information about this model in the [research paper](https://cdn.openai.com/papers/whisper.pdf), [OpenAI blog](https://openai.com/blog/whisper/), [model card](https://github.com/openai/whisper/blob/main/model-card.md) and GitHub [repository](https://github.com/openai/whisper).\n",
"\n",
- "In this notebook, we will use Whisper with OpenVINO to generate subtitles in a sample video. Additionally, we will use [NNCF](https://github.com/openvinotoolkit/nncf) improving model performance by INT8 quantization.\n",
+ "In this notebook, we will use Whisper model with [OpenVINO Generate API](https://github.com/openvinotoolkit/openvino.genai) for [Whisper automatic speech recognition scenarios](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/python/whisper_speech_recognition/README.md) to generate subtitles in a sample video. Additionally, we will use [NNCF](https://github.com/openvinotoolkit/nncf) improving model performance by INT8 quantization.\n",
"Notebook contains the following steps:\n",
"1. Download the model.\n",
"2. Instantiate the PyTorch model pipeline.\n",
@@ -33,10 +33,6 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "\n",
- "\n",
- "\n",
- "\n",
"#### Table of contents:\n",
"\n",
"- [Prerequisites](#Prerequisites)\n",
@@ -80,18 +76,35 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 3,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
- "%pip install -q \"openvino>=2024.1.0\" \"nncf>=2.10.0\"\n",
- "%pip install -q \"python-ffmpeg<=1.0.16\" moviepy \"onnx!=1.16.2\" \"git+https://github.com/huggingface/optimum-intel.git\" \"torch>=2.1\" --extra-index-url https://download.pytorch.org/whl/cpu\n",
- "%pip install -q \"yt_dlp>=2024.8.6\" soundfile librosa jiwer\n",
+ "%pip install -q \"nncf>=2.13.0\"\n",
+ "%pip install -q --pre -U \"openvino\" \"openvino-tokenizers\" \"openvino-genai\" --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly\n",
+ "%pip install -q \"python-ffmpeg<=1.0.16\" \"ffmpeg\" \"moviepy\" \"onnx!=1.16.2\" \"git+https://github.com/huggingface/optimum-intel.git\" \"torch>=2.1\" --extra-index-url https://download.pytorch.org/whl/cpu\n",
+ "%pip install -q -U \"yt_dlp>=2024.8.6\" soundfile librosa jiwer\n",
"%pip install -q \"gradio>=4.19\""
]
},
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import requests\n",
+ "from pathlib import Path\n",
+ "\n",
+ "if not Path(\"notebook_utils.py\").exists():\n",
+ " r = requests.get(\n",
+ " url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py\",\n",
+ " )\n",
+ " open(\"notebook_utils.py\", \"w\").write(r.text)"
+ ]
+ },
{
"attachments": {},
"cell_type": "markdown",
@@ -123,12 +136,12 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
- "model_id": "6f1b0c699e444e7fa375cf5cc59d9a7c",
+ "model_id": "f12e1281744d4a769015824e4382c3c1",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
- "Dropdown(description='Model:', index=6, options=('openai/whisper-large-v3', 'openai/whisper-large-v2', 'openai…"
+ "Dropdown(description='Model:', index=7, options=('openai/whisper-large-v3-turbo', 'openai/whisper-large-v3', '…"
]
},
"execution_count": 2,
@@ -168,23 +181,16 @@
"### Convert model to OpenVINO Intermediate Representation (IR) format using Optimum-Intel.\n",
"[back to top ⬆️](#Table-of-contents:)\n",
"\n",
- "The Hugging Face Optimum API is a high-level API that enables us to convert and quantize models from the Hugging Face Transformers library to the OpenVINO™ IR format. For more details, refer to the [Hugging Face Optimum documentation](https://huggingface.co/docs/optimum/intel/inference).\n",
- "\n",
- "Optimum Intel can be used to load optimized models from the [Hugging Face Hub](https://huggingface.co/docs/optimum/intel/hf.co/models) and create pipelines to run an inference with OpenVINO Runtime using Hugging Face APIs. The Optimum Inference models are API compatible with Hugging Face Transformers models. This means we just need to replace the `AutoModelForXxx` class with the corresponding `OVModelForXxx` class.\n",
+ "Listed Whisper model are available for downloading via the [HuggingFace hub](https://huggingface.co/openai). We will use optimum-cli interface for exporting it into OpenVINO Intermediate Representation (IR) format.\n",
"\n",
- "Below is an example of the whisper-tiny model\n",
+ "Optimum CLI interface for converting models supports export to OpenVINO (supported starting optimum-intel 1.12 version).\n",
+ "General command format:\n",
"\n",
- "```diff\n",
- "-from transformers import AutoModelForSpeechSeq2Seq\n",
- "+from optimum.intel.openvino import OVModelForSpeechSeq2Seq\n",
- "from transformers import AutoTokenizer, pipeline\n",
- "\n",
- "model_id = \"openai/whisper-tiny\"\n",
- "-model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id)\n",
- "+model = OVModelForSpeechSeq2Seq.from_pretrained(model_id, export=True)\n",
+ "```bash\n",
+ "optimum-cli export openvino --model
+
+ + OpenVINO Generate API Whisper demo {'with quantized model.' if quantized else ''} +
+
+ If you use video more then 30 sec, please, note, max_length will be increased. You also could be update it useing generation_config.
+
+ \n", - "\n" - ], - "text/plain": [ - "\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f30e8952c50e4f6ca374dc6972b95fca", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n" - ], - "text/plain": [] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n" - ], - "text/plain": [ - "\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:nncf:12 ignored nodes were found by name in the NNCFGraph\n", - "INFO:nncf:16 ignored nodes were found by name in the NNCFGraph\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "520c7840eb5440859d2b9ba2123049a7", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n" - ], - "text/plain": [] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n" - ], - "text/plain": [ - "\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "5ddc5b2c750d4a4cbe67f0b8f7be4faf", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n" - ], - "text/plain": [] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n" - ], - "text/plain": [ - "\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "69846e3229834f7992738d50a33b354b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Quantizing decoder with past\n" - ] - }, - { - "data": { - "text/html": [ - "\n" - ], - "text/plain": [] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n" - ], - "text/plain": [ - "\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4c48f03790324ee99afdb4031429a09a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n" - ], - "text/plain": [] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n" - ], - "text/plain": [ - "\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:nncf:24 ignored nodes were found by name in the NNCFGraph\n", - "INFO:nncf:24 ignored nodes were found by name in the NNCFGraph\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "03249855c07f4b83bfb4289608bca05b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n" - ], - "text/plain": [] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n" - ], - "text/plain": [ - "\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "fea73d3c378a442bacb43bf1ab11b4ec", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n" - ], - "text/plain": [] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n" - ], - "text/plain": [ - "\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Compiling the encoder to AUTO ...\n", - "Compiling the decoder to AUTO ...\n", - "Compiling the decoder to AUTO ...\n" - ] - } - ], + "outputs": [], "source": [ "%%skip not $to_quantize.value\n", "\n", "import gc\n", "import shutil\n", "import nncf\n", - "from datasets import load_dataset\n", - "from tqdm.notebook import tqdm\n", - "\n", - "def extract_input_features(sample):\n", - " input_features = processor(\n", - " sample[\"audio\"][\"array\"],\n", - " sampling_rate=sample[\"audio\"][\"sampling_rate\"],\n", - " return_tensors=\"pt\",\n", - " ).input_features\n", - " return input_features\n", - "\n", + "import openvino as ov\n", "\n", "\n", - "CALIBRATION_DATASET_SIZE = 50\n", + "CALIBRATION_DATASET_SIZE = 30\n", "quantized_model_path = Path(f\"{model_dir}_quantized\")\n", "\n", "\n", "def quantize(ov_model: OVModelForSpeechSeq2Seq, calibration_dataset_size: int):\n", " if not quantized_model_path.exists():\n", - " encoder_calibration_data, decoder_calibration_data = collect_calibration_dataset(\n", - " ov_model, calibration_dataset_size\n", - " )\n", + " encoder_calibration_data, decoder_calibration_data = collect_calibration_dataset(ov_model, calibration_dataset_size)\n", " print(\"Quantizing encoder\")\n", " quantized_encoder = nncf.quantize(\n", " ov_model.encoder.model,\n", @@ -1146,7 +779,7 @@ " subset_size=len(encoder_calibration_data),\n", " model_type=nncf.ModelType.TRANSFORMER,\n", " # Smooth Quant algorithm reduces activation quantization error; optimal alpha value was obtained through grid search\n", - " advanced_parameters=nncf.AdvancedQuantizationParameters(smooth_quant_alpha=0.50)\n", + " advanced_parameters=nncf.AdvancedQuantizationParameters(smooth_quant_alpha=0.80),\n", " )\n", " ov.save_model(quantized_encoder, quantized_model_path / \"openvino_encoder_model.xml\")\n", " del quantized_encoder\n", @@ -1160,7 +793,7 @@ " subset_size=len(decoder_calibration_data),\n", " model_type=nncf.ModelType.TRANSFORMER,\n", " # Smooth Quant algorithm reduces activation quantization error; optimal alpha value was obtained through grid search\n", - " advanced_parameters=nncf.AdvancedQuantizationParameters(smooth_quant_alpha=0.96)\n", + " advanced_parameters=nncf.AdvancedQuantizationParameters(smooth_quant_alpha=0.96),\n", " )\n", " ov.save_model(quantized_decoder_with_past, quantized_model_path / \"openvino_decoder_with_past_model.xml\")\n", " del quantized_decoder_with_past\n", @@ -1173,14 +806,24 @@ " shutil.copy(model_path / \"generation_config.json\", quantized_model_path / \"generation_config.json\")\n", " shutil.copy(model_path / \"openvino_decoder_model.xml\", quantized_model_path / \"openvino_decoder_model.xml\")\n", " shutil.copy(model_path / \"openvino_decoder_model.bin\", quantized_model_path / \"openvino_decoder_model.bin\")\n", - "\n", - " quantized_ov_model = OVModelForSpeechSeq2Seq.from_pretrained(quantized_model_path, compile=False)\n", - " quantized_ov_model.to(device.value)\n", - " quantized_ov_model.compile()\n", - " return quantized_ov_model\n", - "\n", - "\n", - "ov_quantized_model = quantize(ov_model, CALIBRATION_DATASET_SIZE)" + " shutil.copy(model_path / \"openvino_tokenizer.xml\", quantized_model_path / \"openvino_tokenizer.xml\")\n", + " shutil.copy(model_path / \"openvino_tokenizer.bin\", quantized_model_path / \"openvino_tokenizer.bin\")\n", + " shutil.copy(model_path / \"openvino_detokenizer.xml\", quantized_model_path / \"openvino_detokenizer.xml\")\n", + " shutil.copy(model_path / \"openvino_detokenizer.bin\", quantized_model_path / \"openvino_detokenizer.bin\")\n", + " shutil.copy(model_path / \"tokenizer_config.json\", quantized_model_path / \"tokenizer_config.json\")\n", + " shutil.copy(model_path / \"tokenizer.json\", quantized_model_path / \"tokenizer.json\")\n", + " shutil.copy(model_path / \"vocab.json\", quantized_model_path / \"vocab.json\")\n", + " shutil.copy(model_path / \"preprocessor_config.json\", quantized_model_path / \"preprocessor_config.json\")\n", + " shutil.copy(model_path / \"special_tokens_map.json\", quantized_model_path / \"special_tokens_map.json\")\n", + " shutil.copy(model_path / \"normalizer.json\", quantized_model_path / \"normalizer.json\")\n", + " shutil.copy(model_path / \"merges.txt\", quantized_model_path / \"merges.txt\")\n", + " shutil.copy(model_path / \"added_tokens.json\", quantized_model_path / \"added_tokens.json\")\n", + "\n", + " quantized_ov_pipe = openvino_genai.WhisperPipeline(str(quantized_model_path), device=device.value)\n", + " return quantized_ov_pipe\n", + "\n", + "\n", + "quantized_ov_pipe = quantize(ov_model, CALIBRATION_DATASET_SIZE)" ] }, { @@ -1196,64 +839,13 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1\n", - "00:00:00,000 --> 00:00:05,000\n", - " What's that?\n", - "\n", - "2\n", - "00:00:05,000 --> 00:00:07,000\n", - " Oh, wow.\n", - "\n", - "3\n", - "00:00:09,000 --> 00:00:11,000\n", - " Hello humans.\n", - "\n", - "4\n", - "00:00:14,000 --> 00:00:15,000\n", - " Focus on me.\n", - "\n", - "5\n", - "00:00:15,000 --> 00:00:16,000\n", - " Focus on the guard.\n", - "\n", - "6\n", - "00:00:18,000 --> 00:00:20,000\n", - " Don't tell anyone what you're seen in here.\n", - "\n", - "7\n", - "00:00:22,000 --> 00:00:24,000\n", - " Have you seen what's in there?\n", - "\n", - "8\n", - "00:00:24,000 --> 00:00:25,000\n", - " They have intel.\n", - "\n", - "9\n", - "00:00:25,000 --> 00:00:27,000\n", - " This is where it all changes.\n", - "\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "if ov_quantized_model is not None:\n", - " int8_pipe = pipeline(\n", - " \"automatic-speech-recognition\",\n", - " model=ov_quantized_model,\n", - " chunk_length_s=30,\n", - " tokenizer=processor.tokenizer,\n", - " feature_extractor=processor.feature_extractor,\n", - " )\n", " inputs, duration = get_audio(output_file)\n", - " transcription = int8_pipe(inputs, generate_kwargs={\"task\": task.value}, return_timestamps=True)[\"chunks\"]\n", + " transcription = quantized_ov_pipe.generate(inputs[\"raw\"], task=task.value, return_timestamps=True).chunks\n", " srt_lines = prepare_srt(transcription, filter_duration=duration)\n", " print(\"\".join(srt_lines))\n", " widgets.Video.from_file(output_file, loop=False, width=800, height=800)" @@ -1269,20 +861,18 @@ "\n", "Finally, we compare original and quantized Whisper models from accuracy and performance stand-points.\n", "\n", - "To measure accuracy, we use `1 - WER` as a metric, where WER stands for Word Error Rate.\n", - "\n", - "When measuring inference time, we do it separately for encoder and decoder-with-past model forwards, and for the whole model inference too." + "To measure accuracy, we use `1 - WER` as a metric, where WER stands for Word Error Rate." ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a2822b0dcd584fd2aa28e01c607926d0", + "model_id": "d748cc7a3f394026907c0ef3780d58a8", "version_major": 2, "version_minor": 0 }, @@ -1296,7 +886,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3dc0e232e81c4de0ad82737f98f69d2a", + "model_id": "c2377a7e2dd840328913bc77e6f51ca9", "version_major": 2, "version_minor": 0 }, @@ -1311,11 +901,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Encoder performance speedup: 1.352\n", - "Decoder with past performance speedup: 1.342\n", - "Whole pipeline performance speedup: 1.350\n", - "Whisper transcription word accuracy. Original model: 81.67%. Quantized model: 83.67%.\n", - "Accuracy drop: -1.99%.\n" + "Whole pipeline performance speedup: 1.452\n", + "Whisper transcription word accuracy. Original model: 81.77%. Quantized model: 82.97%.\n", + "Accuracy drop: -1.20%.\n" ] } ], @@ -1326,69 +914,34 @@ "from contextlib import contextmanager\n", "from jiwer import wer, wer_standardize\n", "\n", - "\n", "TEST_DATASET_SIZE = 50\n", - "MEASURE_TIME = False\n", - "\n", - "@contextmanager\n", - "def time_measurement():\n", - " global MEASURE_TIME\n", - " try:\n", - " MEASURE_TIME = True\n", - " yield\n", - " finally:\n", - " MEASURE_TIME = False\n", - "\n", - "def time_fn(obj, fn_name, time_list):\n", - " original_fn = getattr(obj, fn_name)\n", - "\n", - " def wrapper(*args, **kwargs):\n", - " if not MEASURE_TIME:\n", - " return original_fn(*args, **kwargs)\n", - " start_time = time.perf_counter()\n", - " result = original_fn(*args, **kwargs)\n", - " end_time = time.perf_counter()\n", - " time_list.append(end_time - start_time)\n", - " return result\n", - "\n", - " setattr(obj, fn_name, wrapper)\n", "\n", "def calculate_transcription_time_and_accuracy(ov_model, test_samples):\n", - " encoder_infer_times = []\n", - " decoder_with_past_infer_times = []\n", " whole_infer_times = []\n", - " time_fn(ov_model, \"generate\", whole_infer_times)\n", - " time_fn(ov_model.encoder, \"forward\", encoder_infer_times)\n", - " time_fn(ov_model.decoder_with_past, \"forward\", decoder_with_past_infer_times)\n", "\n", " ground_truths = []\n", " predictions = []\n", " for data_item in tqdm(test_samples, desc=\"Measuring performance and accuracy\"):\n", - " input_features = extract_input_features(data_item)\n", - "\n", - " with time_measurement():\n", - " predicted_ids = ov_model.generate(input_features)\n", - " transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)\n", + " start_time = time.perf_counter()\n", + " transcription = ov_model.generate(data_item[\"audio\"][\"array\"], return_timestamps=True)\n", + " end_time = time.perf_counter()\n", + " whole_infer_times.append(end_time - start_time)\n", "\n", " ground_truths.append(data_item[\"text\"])\n", - " predictions.append(transcription[0])\n", + " predictions.append(transcription.texts[0])\n", "\n", " word_accuracy = (1 - wer(ground_truths, predictions, reference_transform=wer_standardize,\n", " hypothesis_transform=wer_standardize)) * 100\n", " mean_whole_infer_time = sum(whole_infer_times)\n", - " mean_encoder_infer_time = sum(encoder_infer_times)\n", - " mean_decoder_with_time_infer_time = sum(decoder_with_past_infer_times)\n", - " return word_accuracy, (mean_whole_infer_time, mean_encoder_infer_time, mean_decoder_with_time_infer_time)\n", + " return word_accuracy, mean_whole_infer_time\n", "\n", "test_dataset = load_dataset(\"openslr/librispeech_asr\", \"clean\", split=\"validation\", streaming=True, trust_remote_code=True)\n", "test_dataset = test_dataset.shuffle(seed=42).take(TEST_DATASET_SIZE)\n", "test_samples = [sample for sample in test_dataset]\n", "\n", - "accuracy_original, times_original = calculate_transcription_time_and_accuracy(ov_model, test_samples)\n", - "accuracy_quantized, times_quantized = calculate_transcription_time_and_accuracy(ov_quantized_model, test_samples)\n", - "print(f\"Encoder performance speedup: {times_original[1] / times_quantized[1]:.3f}\")\n", - "print(f\"Decoder with past performance speedup: {times_original[2] / times_quantized[2]:.3f}\")\n", - "print(f\"Whole pipeline performance speedup: {times_original[0] / times_quantized[0]:.3f}\")\n", + "accuracy_original, times_original = calculate_transcription_time_and_accuracy(ov_pipe, test_samples)\n", + "accuracy_quantized, times_quantized = calculate_transcription_time_and_accuracy(quantized_ov_pipe, test_samples)\n", + "print(f\"Whole pipeline performance speedup: {times_original / times_quantized:.3f}\")\n", "print(f\"Whisper transcription word accuracy. Original model: {accuracy_original:.2f}%. Quantized model: {accuracy_quantized:.2f}%.\")\n", "print(f\"Accuracy drop: {accuracy_original - accuracy_quantized:.2f}%.\")" ] @@ -1406,6 +959,7 @@ "cell_type": "code", "execution_count": null, "metadata": { + "scrolled": true, "test_replace": { " demo.launch(debug=True)": " demo.launch()", " demo.launch(share=True, debug=True)": " demo.launch(share=True)" @@ -1413,18 +967,26 @@ }, "outputs": [], "source": [ - "def transcribe(url, task, use_int8):\n", - " output_file = Path(\"downloaded_video.mp4\")\n", - " ydl_ops = {\"format\": \"best[ext=mp4]\", \"outtmpl\": output_file.as_posix()}\n", - " with yt_dlp.YoutubeDL(ydl_ops) as ydl:\n", - " ydl.download(link.value)\n", - " inputs, duration = get_audio(output_file)\n", - " m_pipe = int8_pipe if use_int8 else pipe\n", - " transcription = m_pipe(inputs, generate_kwargs={\"task\": task.lower()}, return_timestamps=True)[\"chunks\"]\n", + "def_config = ov_pipe.get_generation_config()\n", + "\n", + "\n", + "def transcribe(video_path, task, use_int8):\n", + " data_path = Path(video_path)\n", + " inputs, duration = get_audio(data_path)\n", + " m_pipe = quantized_ov_pipe if use_int8 else ov_pipe\n", + "\n", + " frame_num = len(inputs[\"raw\"]) / 16000\n", + " if frame_num > 30:\n", + " config = ov_pipe.get_generation_config()\n", + " chink_num = math.ceil(frame_num / 30)\n", + " config.max_length = chink_num * def_config.max_length\n", + " m_pipe.set_generation_config(config)\n", + "\n", + " transcription = m_pipe.generate(inputs[\"raw\"], task=task.lower(), return_timestamps=True).chunks\n", " srt_lines = prepare_srt(transcription, duration)\n", - " with output_file.with_suffix(\".srt\").open(\"w\") as f:\n", + " with data_path.with_suffix(\".srt\").open(\"w\") as f:\n", " f.writelines(srt_lines)\n", - " return [str(output_file), str(output_file.with_suffix(\".srt\"))]\n", + " return [str(data_path), str(data_path.with_suffix(\".srt\"))]\n", "\n", "\n", "if not Path(\"gradio_helper.py\").exists():\n", @@ -1433,7 +995,7 @@ "\n", "from gradio_helper import make_demo\n", "\n", - "demo = make_demo(fn=transcribe, quantized=ov_quantized_model is not None)\n", + "demo = make_demo(fn=transcribe, quantized=ov_quantized_model is not None, sample_path=output_file)\n", "\n", "try:\n", " demo.launch(debug=True)\n", @@ -1447,7 +1009,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -1461,7 +1023,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.10.12" }, "openvino_notebooks": { "imageUrl": "https://user-images.githubusercontent.com/29454499/204548693-1304ef33-c790-490d-8a8b-d5766acb6254.png",