{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "84150c46", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"3\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "9b4191fc", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-11-16 19:23:14 optimizers:67] Could not import distributed_fused_adam optimizer from Apex\n" ] } ], "source": [ "import contextlib\n", "\n", "import torch\n", "import nemo.collections.asr as nemo_asr\n", "\n", "from suno_utils.audio import Audio\n", "from suno_utils.utils.display import suppress_logging\n", "\n", "if (\n", " torch.cuda.is_available()\n", " and hasattr(torch.cuda, \"amp\")\n", " and hasattr(torch.cuda.amp, \"autocast\")\n", "):\n", " autocast = torch.cuda.amp.autocast\n", "else:\n", " @contextlib.contextmanager\n", " def autocast():\n", " yield\n", " \n", "SAMPLE_RATE = 16_000\n", "\n", "def load_model(model_name=\"stt_en_conformer_ctc_large\"):\n", " with suppress_logging():\n", " model = nemo_asr.models.EncDecCTCModelBPE.from_pretrained(model_name=model_name)\n", " model.preprocessor.featurizer.dither = 0.0\n", " model.preprocessor.featurizer.pad_to = 0\n", " model.eval()\n", " model.encoder.freeze()\n", " model.decoder.freeze()\n", " torch.cuda.synchronize()\n", " torch.cuda.empty_cache()\n", " return model\n", "\n", "def get_tokens_from_logits(logits, vocab):\n", " return [vocab[idx] for idx in logits.argmax(axis=1)]\n", "\n", "def get_text_from_tokens(tokens):\n", " squashed_tokens = []\n", " prev_token = None\n", " for token in tokens:\n", " if token != prev_token:\n", " squashed_tokens.append(token)\n", " prev_token = token\n", " text = \"\".join(squashed_tokens).replace(\"▁\", \" \")\n", " text = normalize_whitespace(text)\n", " return text\n", "\n", "def decode_logits(logits, vocab):\n", " tokens = get_tokens_from_logits(logits, vocab)\n", " text = get_text_from_tokens(tokens)\n", " return text\n", "\n", "def _collate_features(batch, pad_id=0):\n", " \"\"\"collate batch of audio sig, audio len\"\"\"\n", " packed_batch = list(zip(*batch))\n", " if len(packed_batch) == 2:\n", " _, audio_lengths = packed_batch\n", " else:\n", " raise ValueError(\"Expects 2 tensors in the batch!\")\n", " max_audio_len = 0\n", " has_audio = audio_lengths[0] is not None\n", " if not has_audio:\n", " return None, None\n", " max_audio_len = max(audio_lengths).item()\n", " audio_signal = []\n", " for sig, sig_len in batch:\n", " sig_len = sig_len.item()\n", " if sig_len < max_audio_len:\n", " pad = (0, int(max_audio_len - sig_len))\n", " sig = torch.nn.functional.pad(sig, pad)\n", " audio_signal.append(sig)\n", " audio_signal = torch.stack(audio_signal)\n", " audio_lengths = torch.stack(audio_lengths)\n", " return audio_signal, audio_lengths\n", "\n", "def _get_features(model, audio_signal, audio_lengths):\n", " processed_signal, processed_signal_length = model.preprocessor(\n", " input_signal=audio_signal, length=audio_lengths,\n", " )\n", " return processed_signal, processed_signal_length\n", "\n", "def _infer(model, processed_signal, processed_signal_length):\n", " logits, logits_len, labels = model.forward(\n", " processed_signal=processed_signal, processed_signal_length=processed_signal_length,\n", " )\n", " return logits, logits_len, labels\n", "\n", "MIN_PREDICT_DURATION_MS = 100 # this is to avoid model errors\n", "MAX_PREDICT_DURATION_MS = 35_100 # this is to avoid gpu oom\n", "\n", "MIN_PREDICT_ARRAY_LEN = int(MIN_PREDICT_DURATION_MS / 1_000 * SAMPLE_RATE)\n", "MAX_PREDICT_ARRAY_LEN = int(MAX_PREDICT_DURATION_MS / 1_000 * SAMPLE_RATE)\n", "\n", "def predict_logits(model, audio_arr_list):\n", " if len(audio_arr_list) == 0:\n", " return []\n", " # alert if contains an array that is too short or too long\n", " if any([\n", " (\n", " arr.shape[0] < MIN_PREDICT_ARRAY_LEN or \n", " arr.shape[0] > MAX_PREDICT_ARRAY_LEN\n", " ) for arr in audio_arr_list\n", " ]):\n", " raise ValueError(\"array size error\")\n", " with torch.inference_mode(), torch.no_grad(), autocast():\n", " arr_list_gpu = [torch.from_numpy(arr).to(model.device) for arr in audio_arr_list]\n", " arr_len_list_gpu = [torch.tensor(arr.shape[-1]).to(model.device) for arr in audio_arr_list]\n", " batch = list(zip(arr_list_gpu, arr_len_list_gpu))\n", " audio_signal, audio_lengths = _collate_features(batch)\n", " processed_signal, processed_signal_length = _get_features(model, audio_signal, audio_lengths)\n", " logits, logits_len, labels = _infer(model, processed_signal, processed_signal_length)\n", " logits_list = [l[:idx].detach().cpu().numpy().squeeze() for idx, l in zip(logits_len, logits)]\n", " # labels_list = [l[:idx].detach().cpu().numpy().squeeze() for idx, l in zip(logits_len, labels)]\n", " del (\n", " arr_list_gpu, arr_len_list_gpu, batch, audio_signal, audio_lengths, \n", " processed_signal, processed_signal_length, logits, logits_len, labels\n", " )\n", " torch.cuda.synchronize()\n", " return logits_list\n", "\n", "\n", "def predict_arrays(model, audio_arr_list, batch_size=16):\n", " vocab = {n: c for n, c in enumerate(list(model.decoder.vocabulary) + [\"\"])}\n", " out = []\n", " for arr_chunk in funcy.chunks(batch_size, audio_arr_list):\n", " # excempt short arrays\n", " to_pred_chunk = []\n", " original_idx_map = {}\n", " for n, arr in enumerate(arr_chunk):\n", " if len(arr) >= MIN_PREDICT_ARRAY_LEN:\n", " to_pred_chunk.append(arr)\n", " original_idx_map[n] = len(to_pred_chunk) - 1\n", " if len(to_pred_chunk) > 0:\n", " logits_list = predict_logits(model, to_pred_chunk)\n", " else:\n", " logits_list = []\n", " text_list = []\n", " for n in range(len(arr_chunk)):\n", " if n in original_idx_map:\n", " logits = logits_list[original_idx_map[n]]\n", " if len(logits) < 2:\n", " text = \"\"\n", " else:\n", " text = decode_logits(logits, vocab)\n", " text_list.append(text)\n", " else:\n", " text_list.append(\"\")\n", " out.extend(text_list)\n", " return out" ] }, { "cell_type": "code", "execution_count": 3, "id": "6fd38710", "metadata": {}, "outputs": [], "source": [ "model_citri = load_model(model_name=\"stt_en_citrinet_256\")\n", "model_conf = load_model(model_name=\"stt_en_conformer_ctc_large\")" ] }, { "cell_type": "code", "execution_count": null, "id": "f7154690", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a558b4f7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "29923e59", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 4, "id": "3da07249", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import funcy\n", "from suno_utils.utils.text import normalize_whitespace\n", "\n", "audio = Audio.from_file(\"/home/georg/data/sample_audio/russia.wav\")\n", "audio_arr = np.hstack([audio.array_float, audio.array_float])" ] }, { "cell_type": "code", "execution_count": 7, "id": "91158c90", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 1.92 s, sys: 310 ms, total: 2.23 s\n", "Wall time: 2.23 s\n" ] } ], "source": [ "%%time\n", "_ = predict_arrays(model_citri, [audio_arr for _ in range(100)], batch_size=16)" ] }, { "cell_type": "code", "execution_count": 1, "id": "6f83a95a", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-11-07 22:23:02 optimizers:67] Could not import distributed_fused_adam optimizer from Apex\n" ] } ], "source": [ "import torch\n", "import random\n", "import threading\n", "import funcy\n", "from suno_utils.audio import Audio\n", "from suno_utils.tasks.asr_v2 import predict_arrays#, load_model\n", "\n", "audio = Audio.from_file(\"/home/georg/data/sample_audio/russia.wav\")\n", "audio_arr = audio.array_float\n", "# model_1 = load_model(\"stt_en_conformer_ctc_small\")" ] }, { "cell_type": "code", "execution_count": 30, "id": "fcba9f6a", "metadata": {}, "outputs": [], "source": [ "audio_arr_list = [audio_arr for n in range(1000)]" ] }, { "cell_type": "code", "execution_count": 3, "id": "175607c9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 13.1 s, sys: 2.6 s, total: 15.7 s\n", "Wall time: 15.8 s\n" ] } ], "source": [ "%%time\n", "_ = predict_arrays(audio_arr_list, batch_size=16, model_name=\"stt_en_conformer_ctc_small\")" ] }, { "cell_type": "code", "execution_count": 4, "id": "e0b2f739", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 349 ms, sys: 47.1 ms, total: 396 ms\n", "Wall time: 178 ms\n" ] } ], "source": [ "%%time\n", "_ = predict_arrays(audio_arr_list, batch_size=16, model_name=\"stt_en_conformer_ctc_small\")" ] }, { "cell_type": "code", "execution_count": 35, "id": "b2efe411", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 3.6 s, sys: 126 ms, total: 3.73 s\n", "Wall time: 2.17 s\n" ] } ], "source": [ "%%time\n", "_ = predict_arrays(audio_arr_list, batch_size=16, model_name=\"stt_en_conformer_ctc_small\", n_gpus=2)\n", "torch.cuda.synchronize()" ] }, { "cell_type": "code", "execution_count": 33, "id": "9c36881f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 3.63 s, sys: 354 µs, total: 3.63 s\n", "Wall time: 3.63 s\n" ] } ], "source": [ "%%time\n", "_ = predict_arrays(audio_arr_list, batch_size=16, model_name=\"stt_en_conformer_ctc_small\", n_gpus=1)\n", "torch.cuda.synchronize()" ] }, { "cell_type": "code", "execution_count": 36, "id": "f8ffdc24", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1.672811059907834" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "3.63 / 2.17" ] }, { "cell_type": "code", "execution_count": null, "id": "fb1522d5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "27c71ef0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 1, "id": "e59c0cf6", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-11-08 10:49:47 optimizers:67] Could not import distributed_fused_adam optimizer from Apex\n" ] } ], "source": [ "import torch\n", "from torch.profiler import profile, record_function, ProfilerActivity\n", "from suno_utils.audio import Audio\n", "from suno_utils.tasks.asr_v2 import predict_arrays, preload_models\n", "# torch.backends.cudnn.enabled = False\n", "# torch.backends.cudnn.benchmark = False\n", "# torch.backends.cudnn.benchmark_limit = 1\n", "torch.backends.cudnn.benchmark = True\n", "preload_models(model_name=\"stt_en_conformer_ctc_small\", n_gpus=1)\n", "audio = Audio.from_file(\"/home/georg/data/sample_audio/russia.wav\")\n", "audio_arr = audio.array_float\n", "audio_arr_list = [audio_arr for n in range(100)]\n", "audio_arr_short_list = [audio_arr[:int(len(audio_arr) / 10)] for n in range(100)]\n", "torch.cuda.synchronize()" ] }, { "cell_type": "code", "execution_count": 2, "id": "ebfa73c5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 1.37 s, sys: 564 ms, total: 1.93 s\n", "Wall time: 2.05 s\n" ] } ], "source": [ "%%time\n", "_ = predict_arrays(audio_arr_list, batch_size=16, model_name=\"stt_en_conformer_ctc_small\", n_gpus=1)\n", "torch.cuda.synchronize()\n", "# Wall time: 2.35 s" ] }, { "cell_type": "code", "execution_count": 3, "id": "3855b98a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 3.76 s, sys: 0 ns, total: 3.76 s\n", "Wall time: 3.76 s\n" ] } ], "source": [ "%%time\n", "for _ in range(10):\n", " _ = predict_arrays(audio_arr_list, batch_size=16, model_name=\"stt_en_conformer_ctc_small\", n_gpus=1)\n", "torch.cuda.synchronize()\n", "# Wall time: 2.35 s" ] }, { "cell_type": "code", "execution_count": null, "id": "7ce07e52", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "53510d51", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "cde36174", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8b49180d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 5, "id": "57e1c508", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.mfa import align_transcript\n", "\n", "text_norm = \"my capstone design project is a mems based atomic force microscope one of the highest resolution types of instruments on the market today currently there's is a hundred thousand dollars ours is a thousand dollars it is kind of limitless on the market that it opens up\"" ] }, { "cell_type": "code", "execution_count": 12, "id": "670c3cb4", "metadata": {}, "outputs": [], "source": [ "out = align_transcript(\n", " audio, \n", " text_norm, \n", " remove_background=False,\n", ")" ] }, { "cell_type": "code", "execution_count": 13, "id": "75862b39", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'transcript': \"my capstone design project is a mems based atomic force microscope one of the highest resolution types of instruments on the market today currently there's is a hundred thousand dollars ours is a thousand dollars it is kind of limitless on the market that it opens up\",\n", " 'words': [{'case': 'not-found-in-transcript',\n", " 'end': 0.43,\n", " 'phones': [{'duration': 0.08, 'phone': 'ah_S'}],\n", " 'start': 0.35,\n", " 'word': 'uh'},\n", " {'alignedWord': 'my',\n", " 'case': 'success',\n", " 'end': 2.71,\n", " 'endOffset': 2,\n", " 'phones': [{'duration': 0.12, 'phone': 'm_B'},\n", " {'duration': 0.09, 'phone': 'ay_E'}],\n", " 'start': 2.5,\n", " 'startOffset': 0,\n", " 'word': 'my'},\n", " {'alignedWord': 'capstone',\n", " 'case': 'success',\n", " 'end': 3.32,\n", " 'endOffset': 11,\n", " 'phones': [{'duration': 0.09, 'phone': 'k_B'},\n", " {'duration': 0.09, 'phone': 'ae_I'},\n", " {'duration': 0.09, 'phone': 'p_I'},\n", " {'duration': 0.05, 'phone': 's_I'},\n", " {'duration': 0.08, 'phone': 't_I'},\n", " {'duration': 0.14, 'phone': 'ow_I'},\n", " {'duration': 0.07, 'phone': 'n_E'}],\n", " 'start': 2.71,\n", " 'startOffset': 3,\n", " 'word': 'capstone'},\n", " {'alignedWord': 'design',\n", " 'case': 'success',\n", " 'end': 3.6599999999999997,\n", " 'endOffset': 18,\n", " 'phones': [{'duration': 0.05, 'phone': 'd_B'},\n", " {'duration': 0.04, 'phone': 'ih_I'},\n", " {'duration': 0.09, 'phone': 'z_I'},\n", " {'duration': 0.09, 'phone': 'ay_I'},\n", " {'duration': 0.07, 'phone': 'n_E'}],\n", " 'start': 3.32,\n", " 'startOffset': 12,\n", " 'word': 'design'},\n", " {'alignedWord': 'project',\n", " 'case': 'success',\n", " 'end': 4.19,\n", " 'endOffset': 26,\n", " 'phones': [{'duration': 0.07, 'phone': 'p_B'},\n", " {'duration': 0.05, 'phone': 'r_I'},\n", " {'duration': 0.08, 'phone': 'aa_I'},\n", " {'duration': 0.11, 'phone': 'jh_I'},\n", " {'duration': 0.1, 'phone': 'eh_I'},\n", " {'duration': 0.08, 'phone': 'k_I'},\n", " {'duration': 0.04, 'phone': 't_E'}],\n", " 'start': 3.66,\n", " 'startOffset': 19,\n", " 'word': 'project'},\n", " {'alignedWord': 'is',\n", " 'case': 'success',\n", " 'end': 4.710000000000001,\n", " 'endOffset': 29,\n", " 'phones': [{'duration': 0.35, 'phone': 'ih_B'},\n", " {'duration': 0.13, 'phone': 'z_E'}],\n", " 'start': 4.23,\n", " 'startOffset': 27,\n", " 'word': 'is'},\n", " {'case': 'not-found-in-audio',\n", " 'endOffset': 31,\n", " 'startOffset': 30,\n", " 'word': 'a'},\n", " {'alignedWord': '',\n", " 'case': 'success',\n", " 'end': 5.16,\n", " 'endOffset': 36,\n", " 'phones': [{'duration': 0.25, 'phone': 'oov_S'}],\n", " 'start': 4.91,\n", " 'startOffset': 32,\n", " 'word': 'mems'},\n", " {'alignedWord': 'based',\n", " 'case': 'success',\n", " 'end': 5.52,\n", " 'endOffset': 42,\n", " 'phones': [{'duration': 0.14, 'phone': 'b_B'},\n", " {'duration': 0.11, 'phone': 'ey_I'},\n", " {'duration': 0.05, 'phone': 's_I'},\n", " {'duration': 0.05, 'phone': 't_E'}],\n", " 'start': 5.17,\n", " 'startOffset': 37,\n", " 'word': 'based'},\n", " {'alignedWord': 'atomic',\n", " 'case': 'success',\n", " 'end': 5.89,\n", " 'endOffset': 49,\n", " 'phones': [{'duration': 0.05, 'phone': 'ah_B'},\n", " {'duration': 0.08, 'phone': 't_I'},\n", " {'duration': 0.08, 'phone': 'aa_I'},\n", " {'duration': 0.03, 'phone': 'm_I'},\n", " {'duration': 0.08, 'phone': 'ih_I'},\n", " {'duration': 0.05, 'phone': 'k_E'}],\n", " 'start': 5.52,\n", " 'startOffset': 43,\n", " 'word': 'atomic'},\n", " {'alignedWord': 'force',\n", " 'case': 'success',\n", " 'end': 6.1,\n", " 'endOffset': 55,\n", " 'phones': [{'duration': 0.05, 'phone': 'f_B'},\n", " {'duration': 0.05, 'phone': 'ao_I'},\n", " {'duration': 0.05, 'phone': 'r_I'},\n", " {'duration': 0.06, 'phone': 's_E'}],\n", " 'start': 5.89,\n", " 'startOffset': 50,\n", " 'word': 'force'},\n", " {'alignedWord': 'microscope',\n", " 'case': 'success',\n", " 'end': 6.720000000000001,\n", " 'endOffset': 66,\n", " 'phones': [{'duration': 0.07, 'phone': 'm_B'},\n", " {'duration': 0.07, 'phone': 'ay_I'},\n", " {'duration': 0.06, 'phone': 'k_I'},\n", " {'duration': 0.05, 'phone': 'r_I'},\n", " {'duration': 0.03, 'phone': 'ah_I'},\n", " {'duration': 0.06, 'phone': 's_I'},\n", " {'duration': 0.06, 'phone': 'k_I'},\n", " {'duration': 0.14, 'phone': 'ow_I'},\n", " {'duration': 0.07, 'phone': 'p_E'}],\n", " 'start': 6.11,\n", " 'startOffset': 56,\n", " 'word': 'microscope'},\n", " {'alignedWord': 'one',\n", " 'case': 'success',\n", " 'end': 7.03,\n", " 'endOffset': 70,\n", " 'phones': [{'duration': 0.12, 'phone': 'w_B'},\n", " {'duration': 0.06, 'phone': 'ah_I'},\n", " {'duration': 0.05, 'phone': 'n_E'}],\n", " 'start': 6.8,\n", " 'startOffset': 67,\n", " 'word': 'one'},\n", " {'alignedWord': 'of',\n", " 'case': 'success',\n", " 'end': 7.09,\n", " 'endOffset': 73,\n", " 'phones': [{'duration': 0.01, 'phone': 'ah_B'},\n", " {'duration': 0.05, 'phone': 'v_E'}],\n", " 'start': 7.03,\n", " 'startOffset': 71,\n", " 'word': 'of'},\n", " {'alignedWord': 'the',\n", " 'case': 'success',\n", " 'end': 7.13,\n", " 'endOffset': 77,\n", " 'phones': [{'duration': 0.01, 'phone': 'dh_B'},\n", " {'duration': 0.03, 'phone': 'ah_E'}],\n", " 'start': 7.09,\n", " 'startOffset': 74,\n", " 'word': 'the'},\n", " {'alignedWord': 'highest',\n", " 'case': 'success',\n", " 'end': 7.3999999999999995,\n", " 'endOffset': 85,\n", " 'phones': [{'duration': 0.08, 'phone': 'hh_B'},\n", " {'duration': 0.08, 'phone': 'ay_I'},\n", " {'duration': 0.05, 'phone': 'ah_I'},\n", " {'duration': 0.01, 'phone': 's_I'},\n", " {'duration': 0.04, 'phone': 't_E'}],\n", " 'start': 7.14,\n", " 'startOffset': 78,\n", " 'word': 'highest'},\n", " {'alignedWord': 'resolution',\n", " 'case': 'success',\n", " 'end': 8.13,\n", " 'endOffset': 96,\n", " 'phones': [{'duration': 0.08, 'phone': 'r_B'},\n", " {'duration': 0.05, 'phone': 'eh_I'},\n", " {'duration': 0.02, 'phone': 'z_I'},\n", " {'duration': 0.01, 'phone': 'ah_I'},\n", " {'duration': 0.13, 'phone': 'l_I'},\n", " {'duration': 0.09, 'phone': 'uw_I'},\n", " {'duration': 0.1, 'phone': 'sh_I'},\n", " {'duration': 0.12, 'phone': 'ah_I'},\n", " {'duration': 0.13, 'phone': 'n_E'}],\n", " 'start': 7.4,\n", " 'startOffset': 86,\n", " 'word': 'resolution'},\n", " {'alignedWord': 'types',\n", " 'case': 'success',\n", " 'end': 8.579999,\n", " 'endOffset': 102,\n", " 'phones': [{'duration': 0.05, 'phone': 't_B'},\n", " {'duration': 0.07, 'phone': 'ay_I'},\n", " {'duration': 0.06, 'phone': 'p_I'},\n", " {'duration': 0.05, 'phone': 's_E'}],\n", " 'start': 8.349999,\n", " 'startOffset': 97,\n", " 'word': 'types'},\n", " {'alignedWord': 'of',\n", " 'case': 'success',\n", " 'end': 8.75,\n", " 'endOffset': 105,\n", " 'phones': [{'duration': 0.08, 'phone': 'ah_B'},\n", " {'duration': 0.09, 'phone': 'v_E'}],\n", " 'start': 8.58,\n", " 'startOffset': 103,\n", " 'word': 'of'},\n", " {'alignedWord': 'instruments',\n", " 'case': 'success',\n", " 'end': 9.32,\n", " 'endOffset': 117,\n", " 'phones': [{'duration': 0.06, 'phone': 'ih_B'},\n", " {'duration': 0.04, 'phone': 'n_I'},\n", " {'duration': 0.06, 'phone': 's_I'},\n", " {'duration': 0.04, 'phone': 't_I'},\n", " {'duration': 0.04, 'phone': 'r_I'},\n", " {'duration': 0.03, 'phone': 'ah_I'},\n", " {'duration': 0.06, 'phone': 'm_I'},\n", " {'duration': 0.06, 'phone': 'ah_I'},\n", " {'duration': 0.08, 'phone': 'n_I'},\n", " {'duration': 0.02, 'phone': 't_I'},\n", " {'duration': 0.08, 'phone': 's_E'}],\n", " 'start': 8.75,\n", " 'startOffset': 106,\n", " 'word': 'instruments'},\n", " {'alignedWord': 'on',\n", " 'case': 'success',\n", " 'end': 9.66,\n", " 'endOffset': 120,\n", " 'phones': [{'duration': 0.17, 'phone': 'ao_B'},\n", " {'duration': 0.05, 'phone': 'n_E'}],\n", " 'start': 9.44,\n", " 'startOffset': 118,\n", " 'word': 'on'},\n", " {'alignedWord': 'the',\n", " 'case': 'success',\n", " 'end': 9.74,\n", " 'endOffset': 124,\n", " 'phones': [{'duration': 0.03, 'phone': 'dh_B'},\n", " {'duration': 0.05, 'phone': 'ah_E'}],\n", " 'start': 9.66,\n", " 'startOffset': 121,\n", " 'word': 'the'},\n", " {'alignedWord': 'market',\n", " 'case': 'success',\n", " 'end': 10.03,\n", " 'endOffset': 131,\n", " 'phones': [{'duration': 0.03, 'phone': 'm_B'},\n", " {'duration': 0.05, 'phone': 'aa_I'},\n", " {'duration': 0.06, 'phone': 'r_I'},\n", " {'duration': 0.04, 'phone': 'k_I'},\n", " {'duration': 0.06, 'phone': 'ih_I'},\n", " {'duration': 0.05, 'phone': 't_E'}],\n", " 'start': 9.74,\n", " 'startOffset': 125,\n", " 'word': 'market'},\n", " {'alignedWord': 'today',\n", " 'case': 'success',\n", " 'end': 10.389999999999999,\n", " 'endOffset': 137,\n", " 'phones': [{'duration': 0.05, 'phone': 't_B'},\n", " {'duration': 0.06, 'phone': 'ah_I'},\n", " {'duration': 0.06, 'phone': 'd_I'},\n", " {'duration': 0.19, 'phone': 'ey_E'}],\n", " 'start': 10.03,\n", " 'startOffset': 132,\n", " 'word': 'today'},\n", " {'alignedWord': 'currently',\n", " 'case': 'success',\n", " 'end': 10.92,\n", " 'endOffset': 147,\n", " 'phones': [{'duration': 0.09, 'phone': 'k_B'},\n", " {'duration': 0.08, 'phone': 'er_I'},\n", " {'duration': 0.02, 'phone': 'ah_I'},\n", " {'duration': 0.01, 'phone': 'n_I'},\n", " {'duration': 0.05, 'phone': 't_I'},\n", " {'duration': 0.01, 'phone': 'l_I'},\n", " {'duration': 0.01, 'phone': 'iy_E'}],\n", " 'start': 10.65,\n", " 'startOffset': 138,\n", " 'word': 'currently'},\n", " {'case': 'not-found-in-audio',\n", " 'endOffset': 155,\n", " 'startOffset': 148,\n", " 'word': \"there's\"},\n", " {'case': 'not-found-in-audio',\n", " 'endOffset': 158,\n", " 'startOffset': 156,\n", " 'word': 'is'},\n", " {'case': 'not-found-in-audio',\n", " 'endOffset': 160,\n", " 'startOffset': 159,\n", " 'word': 'a'},\n", " {'alignedWord': 'hundred',\n", " 'case': 'success',\n", " 'end': 11.780000000000001,\n", " 'endOffset': 168,\n", " 'phones': [{'duration': 0.09, 'phone': 'hh_B'},\n", " {'duration': 0.01, 'phone': 'ah_I'},\n", " {'duration': 0.04, 'phone': 'n_I'},\n", " {'duration': 0.06, 'phone': 'er_I'},\n", " {'duration': 0.03, 'phone': 'd_E'}],\n", " 'start': 11.55,\n", " 'startOffset': 161,\n", " 'word': 'hundred'},\n", " {'alignedWord': 'thousand',\n", " 'case': 'success',\n", " 'end': 12.03,\n", " 'endOffset': 177,\n", " 'phones': [{'duration': 0.07, 'phone': 'th_B'},\n", " {'duration': 0.07, 'phone': 'aw_I'},\n", " {'duration': 0.06, 'phone': 'z_I'},\n", " {'duration': 0.03, 'phone': 'ah_I'},\n", " {'duration': 0.01, 'phone': 'n_I'},\n", " {'duration': 0.01, 'phone': 'd_E'}],\n", " 'start': 11.78,\n", " 'startOffset': 169,\n", " 'word': 'thousand'},\n", " {'alignedWord': 'dollars',\n", " 'case': 'success',\n", " 'end': 12.45,\n", " 'endOffset': 185,\n", " 'phones': [{'duration': 0.07, 'phone': 'd_B'},\n", " {'duration': 0.06, 'phone': 'aa_I'},\n", " {'duration': 0.11, 'phone': 'l_I'},\n", " {'duration': 0.1, 'phone': 'er_I'},\n", " {'duration': 0.08, 'phone': 'z_E'}],\n", " 'start': 12.03,\n", " 'startOffset': 178,\n", " 'word': 'dollars'},\n", " {'alignedWord': 'ours',\n", " 'case': 'success',\n", " 'end': 12.889999,\n", " 'endOffset': 190,\n", " 'phones': [{'duration': 0.12, 'phone': 'aa_B'},\n", " {'duration': 0.1, 'phone': 'r_I'},\n", " {'duration': 0.07, 'phone': 'z_E'}],\n", " 'start': 12.599999,\n", " 'startOffset': 186,\n", " 'word': 'ours'},\n", " {'alignedWord': 'is',\n", " 'case': 'success',\n", " 'end': 13.029999,\n", " 'endOffset': 193,\n", " 'phones': [{'duration': 0.06, 'phone': 'ih_B'},\n", " {'duration': 0.08, 'phone': 'z_E'}],\n", " 'start': 12.889999,\n", " 'startOffset': 191,\n", " 'word': 'is'},\n", " {'alignedWord': 'a',\n", " 'case': 'success',\n", " 'end': 13.04,\n", " 'endOffset': 195,\n", " 'phones': [{'duration': 0.01, 'phone': 'ah_S'}],\n", " 'start': 13.03,\n", " 'startOffset': 194,\n", " 'word': 'a'},\n", " {'alignedWord': 'thousand',\n", " 'case': 'success',\n", " 'end': 13.37,\n", " 'endOffset': 204,\n", " 'phones': [{'duration': 0.09, 'phone': 'th_B'},\n", " {'duration': 0.11, 'phone': 'aw_I'},\n", " {'duration': 0.05, 'phone': 'z_I'},\n", " {'duration': 0.05, 'phone': 'ah_I'},\n", " {'duration': 0.03, 'phone': 'n_E'}],\n", " 'start': 13.04,\n", " 'startOffset': 196,\n", " 'word': 'thousand'},\n", " {'alignedWord': 'dollars',\n", " 'case': 'success',\n", " 'end': 13.799999999999999,\n", " 'endOffset': 212,\n", " 'phones': [{'duration': 0.03, 'phone': 'd_B'},\n", " {'duration': 0.08, 'phone': 'aa_I'},\n", " {'duration': 0.1, 'phone': 'l_I'},\n", " {'duration': 0.12, 'phone': 'er_I'},\n", " {'duration': 0.1, 'phone': 'z_E'}],\n", " 'start': 13.37,\n", " 'startOffset': 205,\n", " 'word': 'dollars'},\n", " {'alignedWord': 'it',\n", " 'case': 'success',\n", " 'end': 14.19,\n", " 'endOffset': 215,\n", " 'phones': [{'duration': 0.01, 'phone': 'ih_B'},\n", " {'duration': 0.03, 'phone': 't_E'}],\n", " 'start': 14.149999999999999,\n", " 'startOffset': 213,\n", " 'word': 'it'},\n", " {'alignedWord': 'is',\n", " 'case': 'success',\n", " 'end': 14.249999999999998,\n", " 'endOffset': 218,\n", " 'phones': [{'duration': 0.01, 'phone': 'ih_B'},\n", " {'duration': 0.05, 'phone': 'z_E'}],\n", " 'start': 14.19,\n", " 'startOffset': 216,\n", " 'word': 'is'},\n", " {'alignedWord': 'kind',\n", " 'case': 'success',\n", " 'end': 14.419999999999998,\n", " 'endOffset': 223,\n", " 'phones': [{'duration': 0.06, 'phone': 'k_B'},\n", " {'duration': 0.05, 'phone': 'ay_I'},\n", " {'duration': 0.01, 'phone': 'n_I'},\n", " {'duration': 0.05, 'phone': 'd_E'}],\n", " 'start': 14.249999999999998,\n", " 'startOffset': 219,\n", " 'word': 'kind'},\n", " {'alignedWord': 'of',\n", " 'case': 'success',\n", " 'end': 14.45,\n", " 'endOffset': 226,\n", " 'phones': [{'duration': 0.01, 'phone': 'ah_B'},\n", " {'duration': 0.02, 'phone': 'v_E'}],\n", " 'start': 14.419999999999998,\n", " 'startOffset': 224,\n", " 'word': 'of'},\n", " {'alignedWord': 'limitless',\n", " 'case': 'success',\n", " 'end': 15.18,\n", " 'endOffset': 236,\n", " 'phones': [{'duration': 0.11, 'phone': 'l_B'},\n", " {'duration': 0.04, 'phone': 'ih_I'},\n", " {'duration': 0.05, 'phone': 'm_I'},\n", " {'duration': 0.06, 'phone': 'ah_I'},\n", " {'duration': 0.05, 'phone': 't_I'},\n", " {'duration': 0.08, 'phone': 'l_I'},\n", " {'duration': 0.09, 'phone': 'ah_I'},\n", " {'duration': 0.08, 'phone': 's_E'}],\n", " 'start': 14.62,\n", " 'startOffset': 227,\n", " 'word': 'limitless'},\n", " {'alignedWord': 'on',\n", " 'case': 'success',\n", " 'end': 15.55,\n", " 'endOffset': 239,\n", " 'phones': [{'duration': 0.14, 'phone': 'ao_B'},\n", " {'duration': 0.17, 'phone': 'n_E'}],\n", " 'start': 15.24,\n", " 'startOffset': 237,\n", " 'word': 'on'},\n", " {'alignedWord': 'the',\n", " 'case': 'success',\n", " 'end': 15.72,\n", " 'endOffset': 243,\n", " 'phones': [{'duration': 0.1, 'phone': 'dh_B'},\n", " {'duration': 0.05, 'phone': 'ah_E'}],\n", " 'start': 15.57,\n", " 'startOffset': 240,\n", " 'word': 'the'},\n", " {'alignedWord': 'market',\n", " 'case': 'success',\n", " 'end': 16.129998999999998,\n", " 'endOffset': 250,\n", " 'phones': [{'duration': 0.05, 'phone': 'm_B'},\n", " {'duration': 0.07, 'phone': 'aa_I'},\n", " {'duration': 0.06, 'phone': 'r_I'},\n", " {'duration': 0.08, 'phone': 'k_I'},\n", " {'duration': 0.08, 'phone': 'ih_I'},\n", " {'duration': 0.07, 'phone': 't_E'}],\n", " 'start': 15.719999,\n", " 'startOffset': 244,\n", " 'word': 'market'},\n", " {'alignedWord': 'that',\n", " 'case': 'success',\n", " 'end': 16.269999000000002,\n", " 'endOffset': 255,\n", " 'phones': [{'duration': 0.02, 'phone': 'dh_B'},\n", " {'duration': 0.06, 'phone': 'ae_I'},\n", " {'duration': 0.06, 'phone': 't_E'}],\n", " 'start': 16.129999,\n", " 'startOffset': 251,\n", " 'word': 'that'},\n", " {'alignedWord': 'it',\n", " 'case': 'success',\n", " 'end': 16.46,\n", " 'endOffset': 258,\n", " 'phones': [{'duration': 0.07, 'phone': 'ih_B'},\n", " {'duration': 0.12, 'phone': 't_E'}],\n", " 'start': 16.27,\n", " 'startOffset': 256,\n", " 'word': 'it'},\n", " {'alignedWord': 'opens',\n", " 'case': 'success',\n", " 'end': 16.86,\n", " 'endOffset': 264,\n", " 'phones': [{'duration': 0.08, 'phone': 'ow_B'},\n", " {'duration': 0.06, 'phone': 'p_I'},\n", " {'duration': 0.05, 'phone': 'ah_I'},\n", " {'duration': 0.06, 'phone': 'n_I'},\n", " {'duration': 0.03, 'phone': 'z_E'}],\n", " 'start': 16.58,\n", " 'startOffset': 259,\n", " 'word': 'opens'},\n", " {'alignedWord': 'up',\n", " 'case': 'success',\n", " 'end': 17.059998999999998,\n", " 'endOffset': 267,\n", " 'phones': [{'duration': 0.12, 'phone': 'ah_B'},\n", " {'duration': 0.08, 'phone': 'p_E'}],\n", " 'start': 16.859999,\n", " 'startOffset': 265,\n", " 'word': 'up'}]}" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out" ] }, { "cell_type": "code", "execution_count": null, "id": "d5e8497e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 15, "id": "df1b3b98", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio\n", "audio = Audio.from_file(\"/home/georg/notebooks/test.wav\")" ] }, { "cell_type": "code", "execution_count": 4, "id": "4889512a", "metadata": {}, "outputs": [], "source": [ "out = align_text(\n", " [\"/home/georg/notebooks/test.wav\"],\n", " [text_norm],\n", " \"/home/georg/anaconda3/etc/profile.d/conda.sh\",\n", " \"mfa\",\n", " num_cores=1,\n", " dictionary_name='english_mfa',\n", " acoustic_model_name='english_mfa',\n", ")" ] }, { "cell_type": "code", "execution_count": 2, "id": "9c781946", "metadata": {}, "outputs": [], "source": [ "import whisper\n", "model = whisper.load_model(\"medium.en\")" ] }, { "cell_type": "code", "execution_count": 9, "id": "37428c23", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "the projects get bigger and more ambitious. My capstone design project is a MEMS based atomic force microscope, one of the highest resolution types of instruments on the market today. Current AFMs are $100,000 and ours is $1,000. It's kind of limitless on the market that it opens up. Our project is bread and butter.\n" ] } ], "source": [ "fp = \"/home/georg/notebooks/test.wav\"\n", "audio = whisper.load_audio(fp)\n", "audio = whisper.pad_or_trim(audio)\n", "mels = whisper.log_mel_spectrogram(audio).to(model.device)\n", "options = whisper.DecodingOptions(language=\"en\", without_timestamps=True)\n", "result = model.decode(mels, options)\n", "print(result.text)" ] }, { "cell_type": "code", "execution_count": null, "id": "9820f568", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c201cd21", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4ab7f3ca", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f8534785", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f3d34c08", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "93942f72", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }