{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "51b05211",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "175716c4",
   "metadata": {},
   "source": [
    "## HF implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e610c806",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from suno_utils.audio import Audio\n",
    "from transformers import WhisperForConditionalGeneration, WhisperProcessor\n",
    "\n",
    "model = WhisperForConditionalGeneration.from_pretrained(\"openai/whisper-small.en\").to(\"cuda\")\n",
    "processor = WhisperProcessor.from_pretrained(\"openai/whisper-small.en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ef8e770b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "President Joe Biden, backed by the full symbolic power of the Western Alliance, is locked in a showdown with Russian President Vladimir Putin, who is using Ukraine as a hostage to try to force the US to renegotiate the settled outcome of the Cold War.\n",
      "CPU times: user 1.93 s, sys: 34.4 ms, total: 1.96 s\n",
      "Wall time: 621 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "audio_arr = Audio.from_file(\"sample_audio/russia.wav\").array_float\n",
    "with torch.no_grad():\n",
    "    input_features = processor(audio_arr, return_tensors=\"pt\", sampling_rate=16_000).input_features.to(\"cuda\")\n",
    "    generated_ids = model.generate(\n",
    "        inputs=input_features,\n",
    "        num_beams=3,\n",
    "        max_length=100,\n",
    "    )\n",
    "transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()\n",
    "del input_features, generated_ids\n",
    "print(transcription)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f1fdd3d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e67764e8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "d9ca8720",
   "metadata": {},
   "source": [
    "## Official repo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1b58ddbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import whisper\n",
    "whisper_model = whisper.load_model(\"small.en\") # tiny, base, small, medium, large"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0f250b0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " President Joe Biden, backed by the full symbolic power of the Western Alliance, is locked in a showdown with Russian President Vladimir Putin, who is using Ukraine as a hostage to try to force the US to renegotiate the settled outcome of the Cold War.\n"
     ]
    }
   ],
   "source": [
    "result = whisper_model.transcribe(\"sample_audio/russia.wav\")\n",
    "print(result[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "59df8f43",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load audio and pad/trim it to fit 30 seconds\n",
    "audio = whisper.load_audio(\"sample_audio/russia.wav\")\n",
    "audio2 = whisper.pad_or_trim(audio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "df0adbd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make log-Mel spectrogram and move to the same device as the model\n",
    "mel = whisper.log_mel_spectrogram(audio2).to(whisper_model.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c7249bc3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "President Joe Biden, backed by the full symbolic power of the Western Alliance, is locked in a showdown with Russian President Vladimir Putin, who is using Ukraine as a hostage to try to force the US to renegotiate the settled outcome of the Cold War.\n"
     ]
    }
   ],
   "source": [
    "# decode the audio\n",
    "options = whisper.DecodingOptions(\n",
    "    language=\"en\",\n",
    "    temperature=0.0,\n",
    "    sample_len= None,\n",
    "    best_of=None,\n",
    "    beam_size= None,\n",
    "    patience= None,\n",
    "    length_penalty=None,\n",
    ")\n",
    "result = whisper.decode(whisper_model, mel, options)\n",
    "print(result.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "64b90bfd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecodingResult(audio_features=tensor([[-0.4705, -0.2708,  0.0569,  ...,  0.4895, -0.0644,  1.0781],\n",
       "        [-0.3047,  0.6772,  0.2039,  ..., -0.0676, -0.8306,  0.0182],\n",
       "        [-0.1855,  0.5420,  0.2739,  ..., -0.2070, -0.8755,  0.5791],\n",
       "        ...,\n",
       "        [-0.0310, -0.0341, -0.0315,  ..., -0.0137, -0.0023, -0.0140],\n",
       "        [-1.9961, -0.5112,  0.5254,  ...,  0.0758, -1.5605,  0.9565],\n",
       "        [-1.5820, -0.4546,  0.6714,  ...,  0.5259, -0.4839,  0.2169]],\n",
       "       device='cuda:0', dtype=torch.float16), language='en', language_probs=None, tokens=[50363, 1992, 5689, 21010, 11, 9763, 416, 262, 1336, 18975, 1176, 286, 262, 4885, 10302, 11, 318, 8970, 287, 50673, 50673, 257, 33338, 351, 3394, 1992, 14840, 8144, 11, 508, 318, 1262, 7049, 355, 257, 23229, 284, 50929, 50929, 1949, 284, 2700, 262, 1294, 284, 48312, 378, 262, 10282, 8055, 286, 262, 10250, 1810, 13, 51141], text='President Joe Biden, backed by the full symbolic power of the Western Alliance, is locked in a showdown with Russian President Vladimir Putin, who is using Ukraine as a hostage to try to force the US to renegotiate the settled outcome of the Cold War.', avg_logprob=-0.12054604814763654, no_speech_prob=0.015752272680401802, temperature=0.0, compression_ratio=1.4101123595505618)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5877355",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f01997bd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a89f17f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81ad874d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f3d6a0f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5384455",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7dd87fc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4a88008",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "4e4e84c9",
   "metadata": {},
   "source": [
    "## Playground"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7d4c955f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/openai/whisper/blob/main/notebooks/LibriSpeech.ipynb\n",
    "# https://github.com/openai/whisper/blob/main/notebooks/Multilingual_ASR.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b692caaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import whisper\n",
    "\n",
    "# model_sm = whisper.load_model(\"base\")\n",
    "# model = whisper.load_model(\"medium\")\n",
    "# model_lg = whisper.load_model(\"large\")\n",
    "\n",
    "# model_sm = whisper.load_model(\"base.en\")\n",
    "model = whisper.load_model(\"medium.en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1144a0b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import whisper\n",
    "\n",
    "# model = whisper.load_model(\"base\")\n",
    "\n",
    "# # load audio and pad/trim it to fit 30 seconds\n",
    "# audio = whisper.load_audio(\"audio.mp3\")\n",
    "# audio = whisper.pad_or_trim(audio)\n",
    "\n",
    "# # make log-Mel spectrogram and move to the same device as the model\n",
    "# mel = whisper.log_mel_spectrogram(audio).to(model.device)\n",
    "\n",
    "# # detect the spoken language\n",
    "# _, probs = model.detect_language(mel)\n",
    "# print(f\"Detected language: {max(probs, key=probs.get)}\")\n",
    "\n",
    "# # decode the audio\n",
    "# options = whisper.DecodingOptions()  # language=\"en\", without_timestamps=True\n",
    "# result = whisper.decode(model, mel, options)\n",
    "\n",
    "# # print the recognized text\n",
    "# print(result.text)\n",
    "\n",
    "# options = dict(language=language, beam_size=5, best_of=5)\n",
    "# transcribe_options = dict(task=\"transcribe\", **options)\n",
    "# translate_options = dict(task=\"translate\", **options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e4748ab7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mr. Jones is a 28 year old gentleman with a history of ischemic cardiomyopathy, EF of 20%, hypertension, hyperlipidemia, status post, orthotopic heart transplant in 2016, coming in today with substernal chest pain and shortness of breath. Period.'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fp = \"sample_data/audio_16k/med_1.wav\"\n",
    "# fp = \"sample_data/audio_16k/med_2.wav\"\n",
    "# fp = \"sample_data/audio_16k/med_3.wav\"\n",
    "\n",
    "audio = whisper.load_audio(fp)\n",
    "audio = whisper.pad_or_trim(audio)\n",
    "mels = whisper.log_mel_spectrogram(audio).to(model.device)\n",
    "options = whisper.DecodingOptions(language=\"en\", without_timestamps=True)\n",
    "result = model.decode(mels, options)\n",
    "print(result.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dfc43f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = \"sample_data/audio_16k/med_1.wav\"\n",
    "# fp = \"sample_data/audio_16k/med_2.wav\"\n",
    "# fp = \"sample_data/audio_16k/med_3.wav\"\n",
    "\n",
    "result = model.transcribe(fp)\n",
    "print(result[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1cfb858",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a8267e2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b8be3a53",
   "metadata": {},
   "outputs": [],
   "source": [
    "audio = Audio.from_file(\"german_rap.wav\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "30e57808",
   "metadata": {},
   "outputs": [],
   "source": [
    "audio.get_slice(0, 30).to_wav(\"tmp/german_rap_p1.wav\")\n",
    "audio.get_slice(30, 60).to_wav(\"tmp/german_rap_p2.wav\")\n",
    "audio.get_slice(60, 90).to_wav(\"tmp/german_rap_p3.wav\")\n",
    "audio.get_slice(90, 120).to_wav(\"tmp/german_rap_p4.wav\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c115b19",
   "metadata": {},
   "outputs": [],
   "source": [
    "https://www.youtube.com/watch?v=9Ew461CAlmQ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "50311d5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Es wird Zeit, weil er lo lo lo lo lo ist, meine Mermis aus dem Scho, Scho, Scho, Scho, Scho, Scho, Gudea, mit mein'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'n'\n",
      "Geh ich hoch, aber mehr mi flow war auch fußballer Los baller, wo sind deine Jungs wo? Kommen nur mit volum 6, 2 Kicke die Reime und rülpe die Scheine Geh mal du lieber zur Seite Digga ich komme alleine und halte dich wie kelle ich alleine alleine Du prissa, du prissa, du redest zu viel Ich scheiße die shot digga bon appetit Mein vorname je meh mi speed card, redouille de meine city habiba du geh weg Auf dem E-carogen will die Million Geh zur Seite digga ich will alles so Für dich ist merro seine Liga viel zu hoch Und bald bin ich in den chat für die Habala\n",
      "Ich hab's euch gesagt alle Ich hab schon alles nur durch Handy-Videos gefickt man Ihr habt lang genug gewartet Aber jetzt bin ich da Und ich werd alles auseinandernehmen Ihr habt grad ein original Melody flow gehört Aber jetzt wird's krass sein Jetzt kriegt ihr den krassesten Part man\n",
      "Los, QDH, woo!\n"
     ]
    }
   ],
   "source": [
    "for n in range(1, 5):\n",
    "    fp = f\"tmp/german_rap_p{n}.wav\"\n",
    "    audio = whisper.load_audio(fp)\n",
    "    audio = whisper.pad_or_trim(audio)\n",
    "    mels = whisper.log_mel_spectrogram(audio).to(model.device)\n",
    "    options = whisper.DecodingOptions(language=\"de\", without_timestamps=True)\n",
    "    result = model.decode(mels, options)\n",
    "    print(result.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d32242f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "[Songtext zu „Baller los“]\n",
    "\n",
    "[Intro]\n",
    "Es wird Zeit (Baller' lo-lo-lo-lo-los)\n",
    "Meine Mermis (Aufm Scho-Scho-Scho-Scho-Schoß)\n",
    "QDH (Wird bald groß, groß, groß)\n",
    "Mero, ey, ey, ey (Mermi-Flow)\n",
    "\n",
    "[Part 1]\n",
    "Baller' meine Packs, die Texte für die Neider (Ja)\n",
    "Jacke mit Pelz, sie glänzt, vom Designer (Ja)\n",
    "Wieder ma' der Beste, Beste, bin geiler (Ja)\n",
    "Digga, ich bang', du denkst, es war einfach (Hah, hah)\n",
    "Kommen alle Weiber, wie 'ne Leiter geh' ich hoch, aber\n",
    "Mermi-Flow war auch Fußballer\n",
    "Los, baller! Wo sind deine Jungs, wo?\n",
    "Komme nur mit Volo Sechs-Zwo (Brrra)\n",
    "Kicke die Reime und rippe die Scheine (Ja)\n",
    "Geh ma' du lieber zur Seite (Wouh)\n",
    "Digga, ich komme alleine\n",
    "Und halte dich wie Kelesh an einer Leine\n",
    "Du Pisser, du Pisser, du redest zu viel\n",
    "Ich esse dich auf, Digga, bon appétit\n",
    "Mein Vorname G, Mermi-Speed, quatre-deux-huit\n",
    "Meine City, ya habib, also geh weg (Pah-pah-pah-pah, wouh)\n",
    "[Pre-Hook]\n",
    "Auf dem Weg nach oben, will die Million (Skrrt, skrrt)\n",
    "Geh zur Seite, Digga, ich will alles hol'n (Wouh)\n",
    "Für dich ist Mero seine Liga viel zu hoch (Pah)\n",
    "Und bald bin ich in den Charts, QDH (QDH, baller' los, ah-ah, brrra)\n",
    "\n",
    "[Hook]\n",
    "Baller' lo-lo-lo-lo-los\n",
    "'ne Wumme aufm Scho-Scho-Scho-Scho-Schoß\n",
    "Und mein Name wird bald groß, groß, groß\n",
    "Bruder, Rüsselsheim, Schüsse fall'n wie gewohnt (Brra)\n",
    "\n",
    "[Interlude]\n",
    "Heh, ja, Mann!\n",
    "Ich hab's euch gesagt, Aller\n",
    "Ich hab' schon alles nur durch Handyvideos gefickt, Mann!\n",
    "Ihr habt lang genug gewartet, aber jetzt bin ich da\n",
    "Und ich werd' alles auseinandernehm'n! (Baller' lo-lo-lo-lo-los)\n",
    "Heh, ihr habt grad ein'n original Mermi-Flow gehört (Aufm Scho-Scho-Scho-Scho-Schoß)\n",
    "Aber jetzt wird's krasser! (Wird bald groß, groß, groß)\n",
    "Jetzt kriegt ihr den krassesten Part, Mann!\n",
    "Gökhan Abi, baller los, QDH, wouh!\n",
    "\n",
    "[Part 2]\n",
    "Bam, bam, wie 'ne Pumpgun bin ich dran, Mann\n",
    "Asoziale Flows von einem Aslan (QDH)\n",
    "Bitte lass mal deine Gangsterfilme\n",
    "Weil ich dich nur am lästern finde, ey (Pah, pah, pah, pah)\n",
    "QDH, Motherfuck, bana bak, kriegst Uppercuts\n",
    "Komme mit den Bratans aus Lazarat (Pah, pah, pah, pah)\n",
    "Baller' wieder unnormale Flows, deine Hoes gucken zu\n",
    "Weil der Mero Birra fickt wieder Shows\n",
    "Ich zeig' (Brra) dir die Gegend, weil\n",
    "Tijara-Para sind Regeln hier in meiner Stadt (Wouh, wouh)\n",
    "Ya hero ya mero, will alles\n",
    "Denn alles ist Para und Para macht satt (Pah-pah, pah-pah, pah-pah)\n",
    "Enes schießt auf Enemys und jeder sieht\n",
    "Wer Welle schiebt, fällt immer tief\n",
    "Meine Schelle zieht, ja, das kennen sie (Brra, wouh)\n",
    "\n",
    "\n",
    "[Pre-Hook]\n",
    "Auf dem Weg nach oben, will die Million (Skrrt, skrrt)\n",
    "Geh zur Seite, Digga, ich will alles hol'n (Wouh)\n",
    "Für dich ist Mero seine Liga viel zu hoch (Pah)\n",
    "Und bald bin ich in den Charts, QDH (QDH, baller' los, ah-ah, brrra)\n",
    "\n",
    "[Hook]\n",
    "Baller' lo-lo-lo-lo-los\n",
    "'ne Wumme aufm Scho-Scho-Scho-Scho-Schoß\n",
    "Und mein Name wird bald groß, groß, groß\n",
    "Bruder, Rüsselsheim, Schüsse fall'n wie gewohnt (Brrra)\n",
    "\n",
    "[Bridge]\n",
    "La-la-la-la-lau\n",
    "La-la-la-la-la-la-lau\n",
    "La-la-la-la-lau\n",
    "QDH, ja, ja, ja, ja, ja\n",
    "\n",
    "[Outro]\n",
    "Es wird Zeit (Baller' lo-lo-lo-lo-los)\n",
    "Meine Mermis (Aufm Scho-Scho-Scho-Scho-Schoß)\n",
    "QDH (Wird bald groß, groß, groß)\n",
    "(Ja-ja-ja, ja-ja-ja)\n",
    "\n",
    "\n",
    "Message supply\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "249c34b5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afd47fb6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f1cf5dd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ea10bc5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5bcb0d3a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}