{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "7503fbeb", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:13.108291Z", "start_time": "2024-04-24T02:00:13.106831Z" }, "execution": { "iopub.execute_input": "2024-10-17T19:58:11.186292Z", "iopub.status.busy": "2024-10-17T19:58:11.185680Z", "iopub.status.idle": "2024-10-17T19:58:11.190593Z", "shell.execute_reply": "2024-10-17T19:58:11.190160Z", "shell.execute_reply.started": "2024-10-17T19:58:11.186273Z" } }, "outputs": [], "source": [ "import os\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "c6fcdd37", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:14.496223Z", "start_time": "2024-04-24T02:00:13.109247Z" }, "execution": { "iopub.execute_input": "2024-10-17T19:58:12.111028Z", "iopub.status.busy": "2024-10-17T19:58:12.110882Z", "iopub.status.idle": "2024-10-17T19:58:14.121405Z", "shell.execute_reply": "2024-10-17T19:58:14.120759Z", "shell.execute_reply.started": "2024-10-17T19:58:12.111014Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2.6.0.dev20241015+cu124\n", "2.5.0.dev20241015+cu124\n", "cuda\n" ] } ], "source": [ "import torch\n", "import torchaudio\n", "import re\n", "import json\n", "import tqdm\n", "import numpy as np\n", "from Bio import Align\n", "from suno_utils.audio import Audio\n", "from suno_utils.tasks.hoot import (\n", " encode_filepaths,\n", " encode,\n", " clean_text,\n", " encode_and_align,\n", " _get_alignable_tokens,\n", " _assign_word_timings,\n", " get_aligned_lyrics,\n", " ctc_align,\n", " load_model_list,\n", " get_word_timing_from_audio_and_lyrics,\n", " _merge_into_lines,\n", " decode_logits,\n", " load_model,\n", ")\n", "from suno_utils.utils.s3 import read_from_s3\n", "import ctc_segmentation\n", "from suno_utils.utils.metrics import get_cer\n", "\n", "\n", "print(torch.__version__)\n", "print(torchaudio.__version__)\n", "\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "print(device)\n", "\n", "%reload_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 3, "id": "0ccd88e3", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:14.560456Z", "start_time": "2024-04-24T02:00:14.514064Z" }, "execution": { "iopub.execute_input": "2024-10-17T19:58:17.010533Z", "iopub.status.busy": "2024-10-17T19:58:17.010083Z", "iopub.status.idle": "2024-10-17T19:58:17.022448Z", "shell.execute_reply": "2024-10-17T19:58:17.021934Z", "shell.execute_reply.started": "2024-10-17T19:58:17.010516Z" } }, "outputs": [], "source": [ "def _reload_models_if_needed():\n", " model_list = load_model_list(\n", " # checkpoint_filepath=os.path.join(\"/home/tony/Data/checkpoints/hoot/2023-11-17_15-08-57\", \"28k_ckpt.pt\"),\n", " # checkpoint_filepath=os.path.join(\"/app/suno/models\", \"hoot_v3.pt\"),\n", " # checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-01-15_03-43-37/25k_ckpt.pt\",\n", " # tokenizer_filepath=\"/app/suno/models/hoot_v3_tokenizer.model\",\n", " # checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-01-18_05-50-04/200k_ckpt.pt\",\n", " # tokenizer_filepath=\"/app/suno/models/hoot_v3_tokenizer.model\",\n", " # checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-02-07_22-07-27/50k_ckpt.pt\",\n", " # tokenizer_filepath=\"/app/suno/data/dpo/models/tokenizer_60k.json\",\n", " # checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-03-08_04-17-25/90k_ckpt.pt\",\n", " # checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-03-11_12-05-57/55k_ckpt.pt\",\n", " # checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-03-10_12-08-54/20k_ckpt.pt\",\n", " # checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-03-14_20-52-13/200k_ckpt.pt\",\n", " # checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-03-19_04-47-27/250k_ckpt.pt\",\n", " # tokenizer_filepath=\"/home/tony/Work/tony/hoot/tokenizers/v5/tokenizer_spe_bpe_v20488/tokenizer.model\",\n", " # checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-03-31_23-32-54/250k_ckpt.pt\",\n", " # tokenizer_filepath=\"/home/tony/Work/tony/FineTuning_chirp_v4/tmp/tokenizer_60k_20250307.json\",\n", " # checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-03-31_23-27-08/250k_ckpt.pt\",\n", " # tokenizer_filepath=\"/home/tony/Work/tony/hoot/tokenizers/v5/tokenizer_spe_bpe_v20488/tokenizer.model\",\n", " # checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-06-11_14-48-32/100k_ckpt.pt\",\n", " checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-06-20_04-42-57/35k_ckpt.pt\",\n", " tokenizer_filepath=\"/home/tony/Work/tony/hoot/tokenizers/v5/tokenizer_spe_bpe_v20481/tokenizer.model\",\n", " n_gpus=None,\n", " )\n", " model = model_list[0][\"model\"]\n", " tokenizer = model_list[0][\"tokenizer\"]\n", " return model_list, model, tokenizer" ] }, { "cell_type": "code", "execution_count": 4, "id": "eeccfd1d", "metadata": {}, "outputs": [], "source": [ "# example_audio = Audio.from_file(\"/app/suno/data/audios/discogs/Id2SWW7wpHU.mp3\")\n", "# example_audio = Audio.from_file(\"/home/m4burns/example.wav\")\n", "example_audio = Audio.from_file(\"/home/m4burns/Edit Test Track.mp3\")\n", "# example_audio = Audio.from_file(\"/home/tony/Work/tony/audios/walking_down.mp3\")\n", "# example_audio = Audio.from_s3(\n", "# \"s3://suno-data/shared/nfdg/W3JRaGV42Rg/W3JRaGV42Rg/audio.webm\"\n", "# )" ] }, { "cell_type": "code", "execution_count": 5, "id": "4a785920", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "134.64" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "example_audio.duration_s" ] }, { "cell_type": "code", "execution_count": 6, "id": "b9d964f7", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "example_audio.play()" ] }, { "cell_type": "code", "execution_count": 7, "id": "ee2e4799", "metadata": {}, "outputs": [], "source": [ "example_audio.to_hq_mp3(\"../audios/marc_example.mp3\")\n", "example_audio = Audio.from_file(\"../audios/marc_example.mp3\") # , n_channels=2)" ] }, { "cell_type": "code", "execution_count": 8, "id": "7f286004", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[\"and a and a counting all i love to count numbers so big such a high amount increments of one all day that is why i'm here to say and a yeah let's hear it\"]\n", "[\"and a and a counting all i love to count numbers so big such a high amount increments of one all day that is why i'm here to say and a yeah let's hear it\"]\n" ] } ], "source": [ "import numpy as np\n", "from suno_utils.tasks.hoot import (\n", " get_word_timing_from_audio_and_lyrics,\n", " encode,\n", " models,\n", ")\n", "\n", "model_list, model, tokenizer = _reload_models_if_needed()\n", "# good, works\n", "print(encode([example_audio]))\n", "# -> [\"paint drippin city blocks graffiti dreams talking loud of concrete symphonies play dope rhythm in the chaos life's antidote\"]\n", "\n", "# bad, fails\n", "print(encode([example_audio], batch_size=1))\n", "# -> ['ppin citytehonies']" ] }, { "cell_type": "code", "execution_count": 9, "id": "122235d7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[verse] hello hello! [ chorus] 你好 你是猪.\n" ] } ], "source": [ "test_token_array = tokenizer.encode(\"[Verse] Hello hello! [ chorus] 你好 你是猪.\")\n", "print(tokenizer.decode(test_token_array))" ] }, { "cell_type": "code", "execution_count": 10, "id": "d71f083f", "metadata": {}, "outputs": [], "source": [ "prior_text = \"\"\"\n", "[verse]\n", "one, two, three\n", "and a four, five, six\n", "seven, eight, nine\n", "and a ten, eleven, twelve\n", "\n", "thirteen fourteen\n", "and a fifteen sixteen\n", "seventeen eighteen\n", "and a nineteen twenty\n", "\n", "[chorus]\n", "counting oh i love to count\n", "numbers so big such a high amount\n", "increments of one all day\n", "that is why i'm here to say\n", "\n", "[verse]\n", "twenty-one, twenty-two, twenty-three \n", "and a twenty-four, twenty-five, twenty-six \n", "twenty-seven, twenty-eight, twenty-nine \n", "and a thirty, thirty-one, thirty-two\n", "\n", "[bridge]\n", "thirty-three, thirty-four, thirty-five \n", "and a thirty-six, thirty-seven, thirty-eight \n", "thirty-nine, forty, forty-one \n", "and a forty-two, forty-three, forty-four\n", "\n", "[chorus]\n", "counting oh i love to count\n", "numbers so big such a high amount\n", "increments of one all day\n", "that is why i'm here to say\n", "\n", "[outro]\n", "Zero, zero, zero, mmm mmm, zero \n", "Zero, zero, zero, aw yeah, zero \n", "Zero, zero, oh oh zero, woo!\n", "Zero, zero, zero, let's hear it, zero!\"\"\"" ] }, { "cell_type": "code", "execution_count": 11, "id": "ca54621e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "with prior --> and a and a counting all i love to count numbers so big such a high amount increments of one all day that is why i'm here to say and a yeah let's hear it\n", "without prior --> and a and a counting all i love to count numbers so big such a high amount increments of one all day that is why i'm here to say and a yeah let's hear it\n" ] } ], "source": [ "hoot_without_prior = encode(example_audio, batch_size=1)\n", "hoot_with_prior = encode(example_audio, prior_texts=[prior_text])\n", "print(\"with prior -->\", hoot_with_prior)\n", "print(\"without prior -->\", hoot_without_prior)" ] }, { "cell_type": "code", "execution_count": 12, "id": "7025df58", "metadata": {}, "outputs": [], "source": [ "def new_infer_with_audio(input_audio, lyrics):\n", " # try to load the tokens directly\n", " y = tokenizer.encode(lyrics)\n", " x = torch.from_numpy(input_audio.convert(16_000, 2, 1).array_float).to(\"cuda\")\n", " # print(x.shape, y.shape)\n", " # clip to max space we have for logits\n", " y = torch.from_numpy(y[: int(round(x.shape[-1] / 16_000 * 10))])\n", " # print(y.shape)\n", " x_len = torch.tensor([x.shape[-1]], device=\"cuda\")\n", " y = y.to(\"cuda\")\n", " x = torch.stack([x])\n", " # x_len = torch.stack(x_len)\n", " y = torch.stack([y])\n", " # print(x.shape, x_len.shape, y.shape, x_len)\n", " with torch.no_grad():\n", " logits, loss = model(x, x_len, text_input=y, return_decoded=True)\n", " # print(loss, logits.shape)\n", " logits = logits.to(\"cpu\").numpy()\n", " logits = logits[0]\n", " # print(logits.shape)\n", " out = tokenizer.decode_logits(logits, prior_text=lyrics, return_timing_indices=True)\n", " print(out[0])\n", " out = tokenizer.decode(out[0])\n", " return out" ] }, { "cell_type": "code", "execution_count": 13, "id": "47069d78", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[267, 53, 4, 2269, 4271, 53, 4, 1411, 8105, 5717, 5717, 53, 4, 11572, 5717, 2597, 5717, 4683, 7908, 53, 4, 4747, 4463, 5641, 168, 2, 165, 34, 1786, 8238, 114, 755, 2463, 4, 765, 8958, 651, 3833, 18, 5746, 98, 267, 116, 364, 91, 119, 483, 2, 11823, 11813, 403, 34, 274, 4463, 4463, 4463, 53, 4, 4463, 4463, 4463, 4463, 4683, 4463, 53, 4, 8105, 8105, 8105, 8105, 8105, 8105, 53, 4, 8105, 8105, 8105, 8105, 53, 4, 9125, 5628, 3543, 3543, 3543, 3543, 5628, 5628, 5628, 168, 175, 5628, 5628, 168, 168, 5628, 2528, 5628, 242, 11823, 11807, 797, 55, 5628]\n" ] }, { "data": { "text/plain": [ "\"one and a four seven and a ele thirtyteenteen and a fifteen sixteen eighteen and a nin twenty counting oh i love to count numbers so big such a high amount increments of one all day that is why i'm here to say twenty twenty twenty and a twenty twenty twenty twenty eight twenty and a thirty thirty thirty thirty thirty thirty and a thirty thirty thirty thirty and a forty zero mmm mmm mmm mmm zero zero zero oh yeah zero zero oh oh zero woo zero let's hear it zero\"" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "new_infer_with_audio(example_audio, prior_text)" ] }, { "cell_type": "code", "execution_count": 14, "id": "57d6c6b1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[267, 883]\n", "0 new --> one two\n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[]\n", "10 new --> \n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[11572, 5717, 5717, 4683, 7908, 53, 4, 4747, 4463, 5641, 116, 2, 165, 34, 1786, 1786]\n", "20 new --> fifteenteen eighteen and a nin twenty counting all i love to count count\n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[8958, 8238, 114, 755, 2463, 4, 765, 8958, 3833, 5746, 98, 267, 116, 364]\n", "30 new --> amou numbers so big such a high amou incments of one all day\n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[91, 119, 483, 2, 11823, 11813, 403, 34, 274, 4463, 883, 1992]\n", "40 new --> that is why i'm here to say twenty two three\n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[]\n", "50 new --> \n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[]\n", "60 new --> \n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[]\n", "70 new --> \n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[1992, 2269, 2269]\n", "80 new --> three four four\n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[168]\n", "90 new --> oh\n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[]\n", "100 new --> \n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[]\n", "110 new --> \n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[]\n", "120 new --> \n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[]\n", "130 new --> \n", "old --> \n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "test_audio_chunk_duration = 10\n", "for i in range(0, int(example_audio.duration_s), test_audio_chunk_duration):\n", " local_example_audio = example_audio.get_segment(\n", " from_s=i, to_s=i + test_audio_chunk_duration\n", " )\n", " text = new_infer_with_audio(local_example_audio, prior_text)\n", " print(i, \"new -->\", text)\n", " hoot_with_prior = encode(local_example_audio, prior_texts=[\"prior_text\"])\n", " print(\"old -->\", hoot_with_prior)\n", " local_example_audio.play()" ] }, { "cell_type": "code", "execution_count": 15, "id": "e1166400", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "with prior --> oh yeah let's hear it\n" ] } ], "source": [ "hoot_with_prior = encode(\n", " example_audio.get_segment(from_s=60, to_s=120), prior_texts=[prior_text]\n", ")\n", "print(\"with prior -->\", hoot_with_prior)" ] }, { "cell_type": "code", "execution_count": 16, "id": "9acaa8ea", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "20481" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenizer.n_vocab" ] }, { "cell_type": "code", "execution_count": 17, "id": "b02d8287", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "你好 你是猪\n" ] } ], "source": [ "test_token_array = tokenizer.encode(\"你好 你是猪\")\n", "print(tokenizer.decode(test_token_array))" ] }, { "cell_type": "code", "execution_count": 18, "id": "307315fb", "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[18], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "BREAK" ] }, { "cell_type": "code", "execution_count": 19, "id": "a3bc0604", "metadata": {}, "outputs": [], "source": [ "example_audio = Audio.from_file(\"/app/suno/data/audios/discogs/C_q3IlgGwFw.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "id": "a6467cb5", "metadata": {}, "outputs": [], "source": [ "example_audio.duration_s" ] }, { "cell_type": "code", "execution_count": 21, "id": "5a86c6b4", "metadata": {}, "outputs": [], "source": [ "from suno_utils.tasks.hoot import LOGIT_DURATION_S" ] }, { "cell_type": "code", "execution_count": null, "id": "b26e1cd4", "metadata": {}, "outputs": [], "source": [ "216.144 / LOGIT_DURATION_S" ] }, { "cell_type": "code", "execution_count": 20, "id": "e78d880a", "metadata": {}, "outputs": [], "source": [ "out = encode(example_audio, return_logits=True)" ] }, { "cell_type": "code", "execution_count": 21, "id": "336c540d", "metadata": {}, "outputs": [], "source": [ "prior_text = \"\"\"\n", "Pusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on aika suudelmain\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on tunne voimakkain\\n\\nKun kansa tuppisuuna naamat mököttäin\\nTungoksessa hiipii töistä kotiinpäin\\nSilloin toisiamme niin kuin kiusallamme\\nMuiden nähden suudellaan\\n\\nJa mitä nyrpeämmän tyypin näämmekin\\nSuutelemaan juuri siihen jäämmekin\\nSyrjään väisty emme, naurain suutelemme\\nHuomaamatta ruuhkaakaan\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on aika suudelman\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on tunne voimakkain\\n\\nKun töihin hapannaamat ryntää aamuisin\\nOikein hyvin pyyhkii meillä silloinkin\\nKun on tapanamme että aloitamme\\nSuukoin päivän jokaisen\\n\\nKun tiukat merkonomit pitkään katselee\\nJotkut pikkusielut heistä vilkaisee\\nPaheksuen meitä, ärsytämme heitä\\nVielä kerran suudellen\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on aika suudelman\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on tunne voimakkain\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on aika suudelmain\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": 25, "id": "88eae81f", "metadata": {}, "outputs": [], "source": [ "_ = load_model(\n", " checkpoint_filepath=\"/home/tony/Data/checkpoints/hoot/2025-02-13_05-07-20/15k_ckpt.pt\",\n", " tokenizer_filepath=\"/home/tony/Work/tony/hoot/tokenizers/v5/tokenizer_spe_bpe_v20481/tokenizer.model\",\n", ")" ] }, { "cell_type": "code", "execution_count": 24, "id": "909957a6", "metadata": {}, "outputs": [], "source": [ "true_text_norm = clean_text(prior_text)\n", "decoded_preds = decode_logits(out, prior_text.lower())\n", "# print(\"pre compute cer for test_id\", test_id)\n", "cer_val = round(get_cer(true_text_norm, decoded_preds), 3)\n", "# print(\"pre align for test_id\", test_id)\n", "# print(\"decoded_preds, \", decoded_preds )\n", "# print(\"basic_cleaned_lyrics\", basic_cleaned_lyrics)\n", "word_timings = ctc_align(out.astype(np.float16), true_text_norm)" ] }, { "cell_type": "code", "execution_count": null, "id": "b2210c1f", "metadata": {}, "outputs": [], "source": [ "cer_val" ] }, { "cell_type": "code", "execution_count": null, "id": "c5c7e136", "metadata": {}, "outputs": [], "source": [ "BREAK" ] }, { "cell_type": "code", "execution_count": null, "id": "8e9ae0d9", "metadata": {}, "outputs": [], "source": [ "# good, works\n", "print(encode(example_audio))\n", "# -> [\"paint drippin city blocks graffiti dreams talking loud of concrete symphonies play dope rhythm in the chaos life's antidote\"]\n", "\n", "# bad, fails\n", "print(encode(example_audio, batch_size=1))\n", "# -> ['ppin citytehonies']" ] }, { "cell_type": "markdown", "id": "6cff3425", "metadata": { "ExecuteTime": { "end_time": "2023-12-18T17:47:51.681132Z", "start_time": "2023-12-18T17:47:51.662114Z" } }, "source": [ "# Test alignment" ] }, { "cell_type": "code", "execution_count": 4, "id": "d4ff6ed6", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:17.115000Z", "start_time": "2024-04-24T02:00:14.561957Z" }, "execution": { "iopub.execute_input": "2024-10-17T19:58:20.694141Z", "iopub.status.busy": "2024-10-17T19:58:20.693646Z", "iopub.status.idle": "2024-10-17T19:58:21.112545Z", "shell.execute_reply": "2024-10-17T19:58:21.111794Z", "shell.execute_reply.started": "2024-10-17T19:58:20.694125Z" } }, "outputs": [], "source": [ "model_list, model, tokenizer = _reload_models_if_needed()" ] }, { "cell_type": "code", "execution_count": 5, "id": "e884d407", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:18.124130Z", "start_time": "2024-04-24T02:00:17.116306Z" } }, "outputs": [], "source": [ "input_audio = Audio.from_file(\"../audios/walking_down.mp3\", n_channels=2)\n", "test_input_audio = input_audio # .get_segment(from_s=180, to_s=200.1)\n", "# test_input_audio = test_input_audio.append(test_input_audio)\n", "# test_input_audio.play(compress=False)\n", "\n", "# gen_id = \"1cee79db-a0ee-44e4-9646-d312cd620997\" # chinese\n", "# mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.mp3\"\n", "# print(mp3_filepath)\n", "# test_input_audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "# test_input_audio.play()" ] }, { "cell_type": "code", "execution_count": 6, "id": "47304ff7", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:18.138195Z", "start_time": "2024-04-24T02:00:18.125329Z" } }, "outputs": [], "source": [ "prior_texts = \"\"\"\n", "[intro]\n", "\n", "[verse]\n", "Walking down the street, feeling so alive\n", "Got my head in the clouds, got a gleam in my eye\n", "Every step I take, it's like a brand new start\n", "No matter where I'm going, I'll always find my part\n", "(oh-oh-oh)\n", "\n", "[chorus]\n", "Life is like a high-wire act, we're dancing in the sky\n", "No need to worry, no need to ask why\n", "With a little bit of courage, we can chase our dreams\n", "No matter what comes our way, we'll always be a team\n", "(we're unstoppable, yeah)\n", "\n", "[outro]\n", "\"\"\"\n", "if test_input_audio.duration_s > 180:\n", " prior_texts = \"\"\"\n", " [verse 1]\n", " 滚滚长江东逝水\n", " 浪花淘尽英雄\n", "\n", " [verse 2]\n", " 是非成败转头空\n", " 青山依旧在\n", " 几度夕阳红\n", "\n", " [bridge]\n", " 白发渔樵江渚上\n", " 惯看秋月春风\n", "\n", " [chorus]\n", " 一壶浊酒喜相逢\n", " 古今多少事\n", " 都付笑谈中\n", "\n", " [Refrain]\n", " 一壶浊酒喜相逢\n", " 古今多少事\n", " 都付笑谈中\n", "\n", " [end]\n", " \"\"\"" ] }, { "cell_type": "code", "execution_count": 7, "id": "fdfdc537", "metadata": {}, "outputs": [], "source": [ "# test_input_audio = Audio.from_file(\"/app/suno/data/audios/deezer/3peun0nwqlk.mp3\")\n", "# prior_texts = \"\"\"\n", "# ओये हुस्न की बुलबुल\n", "# ओये हिंद की बुलबुल\n", "# ओये कहा को जाती गाल सुना ना\n", "# सुर्ख जोड़ा जो पह्न कर मेरा दिलदार छल्ला\n", "# हार्टाफा शोर उठा लूट लिया लूट लिया\n", "# अरे सुर्ख जोड़ा जो पह्न कर मेरा दिलदार छल्ला\n", "# हार्टाफा शोर उठा लूट लिया लूट लिया\n", "# मेरा दिल जान वाई वाई\n", "# तुझ पे कुर्बान वाई वाई\n", "# मेरा दिल जान वाई वाई\n", "# तुझ पे कुर्बान वाई वाई\n", "# मेरा दिल जान वाई वाई\n", "# तुझ पे कुर्बान वाई वाई उहू\n", "# सुर्ख जोड़ा जो पह्न कर मेरा दिलदार छल्ला\n", "# हार्टाफा शोर उठा लूट लिया लूट लिया\n", "# जाने मॅन नूवर हैं तू\n", "# बड़ी माशूर हैं ती\n", "# जाने मॅन नूवर हैं तू\n", "# बड़ी माशूर हैं ती\n", "# इश्क़ में चूर हैं हम\n", "# इश्क़ में चूर हैं तू\n", "# इश्क़ में चूर हैं हम\n", "# इश्क़ में चूर हैं तू\n", "# हुस्न आड़ा हुआ जिस दूं गुले गुलजार चला\n", "# हार्टाफा शोर उठा लूट लिया लूट लिया\n", "# सुर्ख जोड़ा जो पह्न कर मेरा दिलदार छल्ला\n", "# हार्टाफा शोर उठा लूट लिया लूट लिया\n", "# हुंसे हिलाते हैं सनम इन पहाड़ो के जिगर\n", "# ये समंदर घूर के हुममें ऐसा असर\n", "# ये समंदर घूर के हुममें ऐसा असर\n", "# अजीब क़यामत सी हुई जब की मेरा यार चला\n", "# हार्टाफा शोर उठा लूट लिया लूट लिया\n", "# सुर्ख जोड़ा जो पह्न कर मेरा दिलदार छल्ला\n", "# हार्टाफा शोर उठा लूट लिया लूट लिया\n", "# मेरा दिल जान वाई वाई\n", "# तुझ पे कुर्बान वाई वाई\n", "# मेरा दिल जान वाई वाई\n", "# तुझ पे कुर्बान वाई वाई\n", "# मेरा दिल जान वाई वाई\n", "# तुझ पे कुर्बान वाई वाई उहू\n", "# सुर्ख जोड़ा जो पह्न कर मेरा दिलदार छल्ला\n", "# हार्टाफा शोर उठा लूट लिया लूट लिया\n", "# \"\"\"" ] }, { "cell_type": "code", "execution_count": 8, "id": "32e06906", "metadata": {}, "outputs": [], "source": [ "# test_input_audio = Audio.from_s3(\n", "# \"s3://suno-data/shared/nfdg/W3JRaGV42Rg/W3JRaGV42Rg/audio.webm\"\n", "# )\n", "# prior_texts = \"\"\"\n", "# Pusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on aika suudelmain\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on tunne voimakkain\\n\\nKun kansa tuppisuuna naamat mököttäin\\nTungoksessa hiipii töistä kotiinpäin\\nSilloin toisiamme niin kuin kiusallamme\\nMuiden nähden suudellaan\\n\\nJa mitä nyrpeämmän tyypin näämmekin\\nSuutelemaan juuri siihen jäämmekin\\nSyrjään väisty emme, naurain suutelemme\\nHuomaamatta ruuhkaakaan\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on aika suudelman\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on tunne voimakkain\\n\\nKun töihin hapannaamat ryntää aamuisin\\nOikein hyvin pyyhkii meillä silloinkin\\nKun on tapanamme että aloitamme\\nSuukoin päivän jokaisen\\n\\nKun tiukat merkonomit pitkään katselee\\nJotkut pikkusielut heistä vilkaisee\\nPaheksuen meitä, ärsytämme heitä\\nVielä kerran suudellen\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on aika suudelman\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on tunne voimakkain\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on aika suudelmain\n", "# \"\"\"" ] }, { "cell_type": "code", "execution_count": 9, "id": "73aedfb5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "with prior --> walking down the street feeling so alive got my head in the clouds got a gleam in my eye every step i take it's like a brand new start no matter where i'm going i'll always find my part life is like a highw act we're dancing in the sky no need to worry no need to ask why with a little bit of courage we can chase our dreams no matter what comes our way we'll always be a team we're unstoppable yeah\n", "without prior --> walking down the street feeling so alive got my head in the clouds got a gleam in my eye every step i take it's like a brand new start no matter where i'm going i'll always find my part life is like a hard wire ra we're dancing in the sky no need to worry no need to ask why with a little bit of courage we can chase our dreams no matter what comes our way we'll always be a team we're unstoppable yeah\n" ] } ], "source": [ "# get the hooted output -- with and without prior\n", "hoot_without_prior = encode(test_input_audio)\n", "hoot_with_prior = encode(test_input_audio, prior_texts=[prior_texts])\n", "print(\"with prior -->\", hoot_with_prior)\n", "print(\"without prior -->\", hoot_without_prior)" ] }, { "cell_type": "code", "execution_count": 10, "id": "b5abd487", "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'new_infer_with_audio' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mnew_infer_with_audio\u001b[49m(test_input_audio, prior_texts)\n", "\u001b[0;31mNameError\u001b[0m: name 'new_infer_with_audio' is not defined" ] } ], "source": [ "new_infer_with_audio(test_input_audio, prior_texts)" ] }, { "cell_type": "code", "execution_count": null, "id": "2013de74", "metadata": {}, "outputs": [], "source": [ "model_list, model, tokenizer = _reload_models_if_needed()" ] }, { "cell_type": "code", "execution_count": 11, "id": "97bf81bd", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:19.950252Z", "start_time": "2024-04-24T02:00:19.858997Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Walking down the street, feeling so alive\n", "Got my head in the clouds, got a gleam in my eye\n", "Every step I take, it's like a brand new start\n", "No matter where I'm going, I'll always find my part\n", "(oh-oh-oh)\n", "\n", "[chorus]\n", "Life is like a high-wire act, we're dancing in the sky\n", "No need to worry, no need to ask why\n", "With a little bit of courage, we can chase our dreams\n", "No matter what comes our way, we'll always be a team\n", "(we're\n" ] } ], "source": [ "# this is the continue function test\n", "aligned_lyrics = encode_and_align(\n", " test_input_audio,\n", " prior_texts=prior_texts,\n", ")\n", "print(aligned_lyrics)" ] }, { "cell_type": "code", "execution_count": 12, "id": "159144dc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Merging lyrics:start_idx--> 0, end_idx--> 178\n", "\n", "[intro]\n", "\n", "[verse]\n", "Walking down the street, feeling so alive\n", "Got my head in the clouds, got a gleam in my eye\n", "Every step I take, it's like a brand new start\n", "No matter where I'm going, I'll always find my part\n", "(oh-oh-oh)\n", "\n", "[chorus]\n", "Life is like a high-wire act, we're dancing in the sky\n", "No need to worry, no need to ask why\n", "With a little bit of courage, we can chase our dreams\n", "No matter what comes our way, we'll always be a team\n", "(we're unstoppable, yeah)\n", "\n", "[outro]\n", "\n", "\n" ] } ], "source": [ "aligned_lyrics = encode_and_align(\n", " test_input_audio,\n", " prior_texts=prior_texts,\n", " do_multi_align=True,\n", ")\n", "print(aligned_lyrics)" ] }, { "cell_type": "code", "execution_count": 13, "id": "a3713615", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0\n", "Walking down the street, feeling so alive\n", "Got my head in the clouds, got a gleam in my eye\n", "Every step I take, it's like a brand new start\n", "No matter where I'm going, I'll always find my part\n", "(oh-oh-oh)\n", "\n", "[chorus]\n", "Life is like a\n", "25\n", "Life is like a high-wire act, we're dancing in the sky\n", "No need to worry, no need to ask why\n", "With a little bit of courage, we can chase our dreams\n", "No matter what comes our way, we'll always be a team\n", "(we're\n", "50\n", "FAILED alignments -- alignment length 0. Fallback to hoot.\n", "\n" ] } ], "source": [ "chunk_duration = 25\n", "estimated_lyrics = []\n", "for start_s in range(0, int(test_input_audio.duration_s), chunk_duration):\n", " print(start_s)\n", " overlap_duration = 5\n", " aligned_lyrics = encode_and_align(\n", " test_input_audio.get_segment(\n", " from_s=max(0, start_s - overlap_duration),\n", " to_s=start_s + chunk_duration + overlap_duration,\n", " ),\n", " prior_texts=prior_texts,\n", " )\n", " print(aligned_lyrics)" ] }, { "cell_type": "code", "execution_count": 14, "id": "1cabba3b", "metadata": {}, "outputs": [], "source": [ "model_list, model, tokenizer = _reload_models_if_needed()" ] }, { "cell_type": "code", "execution_count": 15, "id": "45c8ebaf", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.073120Z", "start_time": "2024-04-24T02:00:19.951452Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Merging lyrics:start_idx--> 0, end_idx--> 178\n", "{'word': '\\n'}\n", "{'word': '[intro]\\n\\n'}\n", "{'word': '[verse]\\nWalking ', 'success': True, 'start_s': 11.25, 'end_s': 11.45, 'p_align': 0.809}\n", "{'word': 'down ', 'success': True, 'start_s': 11.45, 'end_s': 11.81, 'p_align': 0.992}\n", "{'word': 'the ', 'success': True, 'start_s': 11.81, 'end_s': 12.09, 'p_align': 0.977}\n", "{'word': 'street, ', 'success': True, 'start_s': 12.09, 'end_s': 12.45, 'p_align': 0.704}\n", "{'word': 'feeling ', 'success': True, 'start_s': 12.45, 'end_s': 12.96, 'p_align': 0.768}\n", "{'word': 'so ', 'success': True, 'start_s': 12.96, 'end_s': 13.56, 'p_align': 0.995}\n", "{'word': 'alive\\n', 'success': True, 'start_s': 13.56, 'end_s': 14.28, 'p_align': 0.992}\n", "{'word': 'Got ', 'success': True, 'start_s': 14.28, 'end_s': 14.84, 'p_align': 0.988}\n", "{'word': 'my ', 'success': True, 'start_s': 14.84, 'end_s': 15.12, 'p_align': 0.995}\n", "{'word': 'head ', 'success': True, 'start_s': 15.12, 'end_s': 15.44, 'p_align': 0.993}\n", "{'word': 'in ', 'success': True, 'start_s': 15.44, 'end_s': 15.76, 'p_align': 0.981}\n", "{'word': 'the ', 'success': True, 'start_s': 15.76, 'end_s': 16.04, 'p_align': 0.947}\n", "{'word': 'clouds, ', 'success': True, 'start_s': 16.04, 'end_s': 16.48, 'p_align': 0.986}\n", "{'word': 'got ', 'success': True, 'start_s': 16.48, 'end_s': 16.88, 'p_align': 0.965}\n", "{'word': 'a ', 'success': True, 'start_s': 16.88, 'end_s': 17.23, 'p_align': 0.944}\n", "{'word': 'gleam in ', 'success': True, 'start_s': 17.23, 'end_s': 17.55, 'p_align': 0.982}\n", "{'word': 'my ', 'success': True, 'start_s': 17.55, 'end_s': 17.75, 'p_align': 0.984}\n", "{'word': 'eye\\n', 'success': True, 'start_s': 17.75, 'end_s': 18.31, 'p_align': 0.745}\n", "{'word': 'Every ', 'success': True, 'start_s': 18.31, 'end_s': 18.95, 'p_align': 0.981}\n", "{'word': 'step ', 'success': True, 'start_s': 18.95, 'end_s': 19.35, 'p_align': 0.992}\n", "{'word': 'I ', 'success': True, 'start_s': 19.35, 'end_s': 19.71, 'p_align': 0.992}\n", "{'word': 'take, ', 'success': True, 'start_s': 19.71, 'end_s': 20.27, 'p_align': 0.994}\n", "{'word': \"it's like \", 'success': True, 'start_s': 20.27, 'end_s': 20.78, 'p_align': 0.984}\n", "{'word': 'a ', 'success': True, 'start_s': 20.78, 'end_s': 21.02, 'p_align': 0.974}\n", "{'word': 'brand ', 'success': True, 'start_s': 21.02, 'end_s': 21.3, 'p_align': 0.987}\n", "{'word': 'new ', 'success': True, 'start_s': 21.3, 'end_s': 21.66, 'p_align': 0.993}\n", "{'word': 'start\\n', 'success': True, 'start_s': 21.66, 'end_s': 22.1, 'p_align': 0.984}\n", "{'word': 'No ', 'success': True, 'start_s': 22.1, 'end_s': 22.42, 'p_align': 0.982}\n", "{'word': 'matter ', 'success': True, 'start_s': 22.42, 'end_s': 22.7, 'p_align': 0.963}\n", "{'word': 'where ', 'success': True, 'start_s': 22.7, 'end_s': 23.34, 'p_align': 0.879}\n", "{'word': \"I'm going, \", 'success': True, 'start_s': 23.34, 'end_s': 24.21, 'p_align': 0.922}\n", "{'word': \"I'll always \", 'success': True, 'start_s': 24.21, 'end_s': 24.85, 'p_align': 0.695}\n", "{'word': 'find ', 'success': True, 'start_s': 24.85, 'end_s': 25.05, 'p_align': 0.955}\n", "{'word': 'my ', 'success': True, 'start_s': 25.05, 'end_s': 25.25, 'p_align': 0.97}\n", "{'word': 'part\\n(', 'success': True, 'start_s': 25.25, 'end_s': 25.37, 'p_align': 0.52}\n", "{'word': 'oh-oh-oh)\\n\\n'}\n", "{'word': '[chorus]\\nLife ', 'success': True, 'start_s': 29.28, 'end_s': 29.44, 'p_align': 0.984}\n", "{'word': 'is ', 'success': True, 'start_s': 29.44, 'end_s': 29.88, 'p_align': 0.973}\n", "{'word': 'like ', 'success': True, 'start_s': 29.88, 'end_s': 30.44, 'p_align': 0.992}\n", "{'word': 'a ', 'success': True, 'start_s': 30.44, 'end_s': 31.32, 'p_align': 0.99}\n", "{'word': 'high-wire act, ', 'success': True, 'start_s': 31.59, 'end_s': 32.63, 'p_align': 0.019}\n", "{'word': \"we're dancing \", 'success': True, 'start_s': 32.63, 'end_s': 33.35, 'p_align': 0.792}\n", "{'word': 'in ', 'success': True, 'start_s': 33.35, 'end_s': 33.79, 'p_align': 0.988}\n", "{'word': 'the ', 'success': True, 'start_s': 33.79, 'end_s': 34.07, 'p_align': 0.992}\n", "{'word': 'sky\\n', 'success': True, 'start_s': 34.07, 'end_s': 34.77, 'p_align': 0.989}\n", "{'word': 'No ', 'success': True, 'start_s': 34.83, 'end_s': 35.5, 'p_align': 0.744}\n", "{'word': 'need ', 'success': True, 'start_s': 35.5, 'end_s': 35.82, 'p_align': 0.982}\n", "{'word': 'to ', 'success': True, 'start_s': 35.82, 'end_s': 36.1, 'p_align': 0.987}\n", "{'word': 'worry, ', 'success': True, 'start_s': 36.1, 'end_s': 36.46, 'p_align': 0.995}\n", "{'word': 'no ', 'success': True, 'start_s': 36.46, 'end_s': 36.86, 'p_align': 0.954}\n", "{'word': 'need ', 'success': True, 'start_s': 36.86, 'end_s': 37.22, 'p_align': 0.987}\n", "{'word': 'to ', 'success': True, 'start_s': 37.22, 'end_s': 37.58, 'p_align': 0.988}\n", "{'word': 'ask ', 'success': True, 'start_s': 37.58, 'end_s': 37.94, 'p_align': 0.995}\n", "{'word': 'why\\n', 'success': True, 'start_s': 37.94, 'end_s': 38.38, 'p_align': 0.995}\n", "{'word': 'With ', 'success': True, 'start_s': 38.38, 'end_s': 38.78, 'p_align': 0.992}\n", "{'word': 'a ', 'success': True, 'start_s': 38.78, 'end_s': 39.06, 'p_align': 0.985}\n", "{'word': 'little ', 'success': True, 'start_s': 39.06, 'end_s': 39.41, 'p_align': 0.987}\n", "{'word': 'bit ', 'success': True, 'start_s': 39.41, 'end_s': 39.77, 'p_align': 0.992}\n", "{'word': 'of ', 'success': True, 'start_s': 39.77, 'end_s': 40.05, 'p_align': 0.986}\n", "{'word': 'courage, ', 'success': True, 'start_s': 40.05, 'end_s': 40.45, 'p_align': 0.993}\n", "{'word': 'we ', 'success': True, 'start_s': 40.45, 'end_s': 40.77, 'p_align': 0.992}\n", "{'word': 'can ', 'success': True, 'start_s': 40.77, 'end_s': 41.05, 'p_align': 0.987}\n", "{'word': 'chase ', 'success': True, 'start_s': 41.05, 'end_s': 41.45, 'p_align': 0.981}\n", "{'word': 'our ', 'success': True, 'start_s': 41.45, 'end_s': 41.89, 'p_align': 0.991}\n", "{'word': 'dreams\\n', 'success': True, 'start_s': 41.89, 'end_s': 42.53, 'p_align': 0.97}\n", "{'word': 'No ', 'success': True, 'start_s': 42.53, 'end_s': 43.05, 'p_align': 0.984}\n", "{'word': 'matter ', 'success': True, 'start_s': 43.05, 'end_s': 43.4, 'p_align': 0.995}\n", "{'word': 'what ', 'success': True, 'start_s': 43.4, 'end_s': 43.88, 'p_align': 0.995}\n", "{'word': 'comes ', 'success': True, 'start_s': 43.88, 'end_s': 44.4, 'p_align': 0.98}\n", "{'word': 'our ', 'success': True, 'start_s': 44.4, 'end_s': 44.84, 'p_align': 0.991}\n", "{'word': 'way, ', 'success': True, 'start_s': 44.84, 'end_s': 45.36, 'p_align': 0.993}\n", "{'word': \"we'll always \", 'success': True, 'start_s': 45.36, 'end_s': 45.96, 'p_align': 0.99}\n", "{'word': 'be ', 'success': True, 'start_s': 45.96, 'end_s': 46.32, 'p_align': 0.993}\n", "{'word': 'a ', 'success': True, 'start_s': 46.32, 'end_s': 46.56, 'p_align': 0.985}\n", "{'word': 'team\\n(', 'success': True, 'start_s': 46.56, 'end_s': 46.68, 'p_align': 0.996}\n", "{'word': \"we're \"}\n", "{'word': 'unstoppable, '}\n", "{'word': 'yeah)\\n\\n'}\n", "{'word': '[outro]\\n\\n'}\n", "{'hoot_lyrics': \"walking down the street feeling so alive got my head in the clouds got a gleam in my eye every step i take it's like a brand new start no matter where i'm going i'll always find my part life is like a high wire act we're dancing in the sky no need to worry no need to ask why with a little bit of courage we can chase our dreams no matter what comes our way we'll always be a team we're\"}\n", "{'hoot_cer': 0.015789473684210527}\n", "done with basics\n" ] } ], "source": [ "padded_words_timed = get_word_timing_from_audio_and_lyrics(\n", " test_input_audio, prior_texts, do_multi_align=True, multi_chunk_size=376\n", ")\n", "for word_d in padded_words_timed:\n", " print(word_d)\n", "print(\"done with basics\")" ] }, { "cell_type": "code", "execution_count": null, "id": "5cdf03f0", "metadata": {}, "outputs": [], "source": [ "padded_words_timed = get_word_timing_from_audio_and_lyrics(\n", " test_input_audio.get_segment(from_s=0, to_s=30.1), prior_texts\n", ")\n", "print(padded_words_timed)" ] }, { "cell_type": "code", "execution_count": null, "id": "ad022825", "metadata": {}, "outputs": [], "source": [ "from suno_utils.tasks.hoot import LOGIT_DURATION_S\n", "\n", "start_s = 0\n", "test_input_segment = (\n", " test_input_audio # .get_segment(from_s=start_s, to_s=start_s + 10.1)\n", ")\n", "hoot_with_prior_logits = encode(\n", " test_input_segment, prior_texts=[prior_texts], return_logits=True\n", ")\n", "true_text_norm = clean_text(prior_texts)\n", "decoded_preds, timing_indices = tokenizer.decode_logits(\n", " hoot_with_prior_logits, prior_text=prior_texts, return_timing_indices=True\n", ")\n", "start_timing_indices = [\n", " round(start_s + i * LOGIT_DURATION_S, 2) for (i, j) in timing_indices\n", "]\n", "end_timing_indices = [\n", " round(start_s + j * LOGIT_DURATION_S, 2) for (i, j) in timing_indices\n", "]\n", "decoded_preds = [tokenizer.decode([i]) for i in decoded_preds]\n", "for word, start_timing, end_timing in zip(\n", " decoded_preds, start_timing_indices, end_timing_indices\n", "):\n", " print(word, start_timing, end_timing)\n", "with open(\"/home/tony/Work/walking_timing.json\", \"w\") as fp:\n", " json.dump({\"decoded_preds\": decoded_preds, \"timing_indices\": timing_indices}, fp)\n", "# print(\"cer_val\", cer_val)" ] }, { "cell_type": "code", "execution_count": null, "id": "95307a88", "metadata": {}, "outputs": [], "source": [ "BREAK" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "c2a4ea3c", "metadata": {}, "source": [ "## Try ctc" ] }, { "cell_type": "code", "execution_count": 13, "id": "807bea43", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.223968Z", "start_time": "2024-04-24T02:00:20.149679Z" } }, "outputs": [], "source": [ "frame_duration_s = 30 / 376" ] }, { "cell_type": "code", "execution_count": 14, "id": "670b31fd", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.356125Z", "start_time": "2024-04-24T02:00:20.225503Z" } }, "outputs": [], "source": [ "logits = encode(test_input_audio, return_logits=True)\n", "if not isinstance(logits, torch.Tensor):\n", " logits = torch.from_numpy(logits)\n", "if round(logits[0].sum().item(), 1) == 1.0:\n", " probs = logits\n", "elif round(torch.exp(logits[0]).sum().item(), 1) == 1.0:\n", " probs = torch.exp(logits)\n", "else:\n", " probs = torch.nn.functional.softmax(logits, dim=-1)\n", "probs = probs.numpy()" ] }, { "cell_type": "code", "execution_count": 15, "id": "e7475b36", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.367487Z", "start_time": "2024-04-24T02:00:20.357353Z" } }, "outputs": [], "source": [ "true_text = prior_texts" ] }, { "cell_type": "code", "execution_count": 16, "id": "bad4dfa0", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.477564Z", "start_time": "2024-04-24T02:00:20.368729Z" } }, "outputs": [], "source": [ "model_list = load_model_list()\n", "tokenizer = model_list[0][\"tokenizer\"]\n", "assert len(tokenizer.vocab) + 1 == probs.shape[1]\n", "inv_vocab = {v: k for k, v in tokenizer.vocab.items()}\n", "unk_id = tokenizer.vocab[\"\"]\n", "# add dummy char cause beginning is messed up\n", "word_transcripts = [\"a\"] + true_text.split()\n", "tokens = []\n", "for word_transcript in word_transcripts:\n", " tok_ids = tokenizer.encode(word_transcript)\n", " tokens.append(tok_ids[tok_ids != unk_id])\n", "char_list = [inv_vocab[i] for i in range(len(inv_vocab))]\n", "config = ctc_segmentation.CtcSegmentationParameters(\n", " char_list=char_list,\n", " blank_transition_cost_zero=True,\n", " replace_spaces_with_blanks=True,\n", " index_duration=frame_duration_s,\n", " backtrack_from_max_t=True,\n", " skip_prob=0.01,\n", ")" ] }, { "cell_type": "code", "execution_count": 17, "id": "4d59cb75", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.545299Z", "start_time": "2024-04-24T02:00:20.478695Z" } }, "outputs": [], "source": [ "ground_truth_mat, utt_begin_indices = ctc_segmentation.prepare_token_list(\n", " config, tokens\n", ")\n", "# print(ground_truth_mat, utt_begin_indices)\n", "timings, char_probs, state_list = ctc_segmentation.ctc_segmentation(\n", " config, probs, ground_truth_mat\n", ")" ] }, { "cell_type": "code", "execution_count": 18, "id": "9b57228a", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.596403Z", "start_time": "2024-04-24T02:00:20.547115Z" } }, "outputs": [], "source": [ "likely_probs = probs.argmax(axis=1).tolist()\n", "likely_probs = [i for i in likely_probs if i != 20480]" ] }, { "cell_type": "code", "execution_count": 19, "id": "7b767cc1", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.644876Z", "start_time": "2024-04-24T02:00:20.597506Z" } }, "outputs": [], "source": [ "gap_char = 0\n", "ground_truth_list = tokenizer.encode(true_text).tolist()" ] }, { "cell_type": "code", "execution_count": 20, "id": "f3796ea5", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.701253Z", "start_time": "2024-04-24T02:00:20.646313Z" } }, "outputs": [], "source": [ "aligner = Align.PairwiseAligner()\n", "aligner.mode = \"global\"\n", "aligner.match_score = 2\n", "aligner.mismatch_score = -1\n", "aligner.open_gap_score = -0.1\n", "aligner.extend_gap_score = -0.5\n", "aligner.target_end_gap_score = 0.0\n", "aligner.query_end_gap_score = 0.0\n", "aligner.alphabet = list(range(0, tokenizer.n_vocab + 1))" ] }, { "cell_type": "code", "execution_count": 21, "id": "cf0ee205", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.763042Z", "start_time": "2024-04-24T02:00:20.702893Z" } }, "outputs": [], "source": [ "# from math import log\n", "# def gap_function(x, y): # x is gap position in seq, y is gap length\n", "# if y == 0: # No gap\n", "# return 0\n", "# elif y == 1: # Gap open penalty\n", "# return -2\n", "# return -(2 + y / 4.0 + log(y) / 2.0)\n", "# alignment = pairwise2.align.globalmc(\n", "# ground_truth_list,\n", "# likely_probs,\n", "# 5, -4, gap_function, gap_function,\n", "# gap_char=[gap_char],\n", "# penalize_end_gaps=False,\n", "# penalize_extend_when_opening=False,\n", "# )[0]" ] }, { "cell_type": "code", "execution_count": null, "id": "a0372e5e", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.832987Z", "start_time": "2024-04-24T02:00:20.764043Z" } }, "outputs": [], "source": [ "alignment = aligner.align(ground_truth_list, likely_probs)[0]\n", "# # trim the two ends\n", "# seq_a, seq_b = alignment.seqA, alignment.seqB\n", "# seq_a = seq_a[::-1]\n", "# seq_b = seq_b[::-1]\n", "# while seq_a and seq_b and seq_b[-1] == gap_char:\n", "# seq_b.pop()\n", "# seq_a.pop()\n", "# seq_a = seq_a[::-1]\n", "# seq_b = seq_b[::-1]\n", "# while seq_a and seq_b and seq_b[-1] == gap_char:\n", "# seq_b.pop()\n", "# seq_a.pop()\n", "seq_a, seq_b = alignment.aligned\n", "seq_a = ground_truth_list[seq_a[0][0] : seq_a[-1][-1] + 1]\n", "real_truth = \" \".join(tokenizer.decode(int(i)) for i in seq_a)\n", "print(real_truth)" ] }, { "cell_type": "code", "execution_count": null, "id": "50b16445", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.890477Z", "start_time": "2024-04-24T02:00:20.834063Z" } }, "outputs": [], "source": [ "print(\n", " \"timings\",\n", " timings.shape,\n", ")\n", "print(\"char_probs\", char_probs.shape, char_probs[-0], char_probs[-1])\n", "print(\"word_transcripts\", len(word_transcripts))\n", "print(\"state_list\", len(state_list))\n", "print(\"utt_begin_indices\", len(utt_begin_indices))\n", "print(\"ground_truth_mat\", ground_truth_mat.shape)\n", "print(\"char prob\", char_probs.shape)\n", "print(tokenizer.n_vocab)\n", "print(tokenizer.decode(20480 - 2))\n", "print(tokenizer.encode(\"ah ah\"))\n", "print(tokenizer.encode(\"ah🛳 ah\"))" ] }, { "cell_type": "code", "execution_count": 24, "id": "0f4cd710", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:20.954095Z", "start_time": "2024-04-24T02:00:20.891514Z" } }, "outputs": [], "source": [ "# while char_probs.shape[0] > 0 and char_probs[-1] < 0.001:\n", "# char_probs = char_probs[:-1]\n", "# timings = timings[:-1]\n", "# utt_begin_indices = utt_begin_indices[:-1]\n", "# word_transcripts.pop()" ] }, { "cell_type": "code", "execution_count": null, "id": "37600574", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:21.016063Z", "start_time": "2024-04-24T02:00:20.955908Z" } }, "outputs": [], "source": [ "print(\"done with some basic ctc\")" ] }, { "cell_type": "markdown", "id": "53a8f788", "metadata": { "ExecuteTime": { "end_time": "2023-11-13T19:11:14.875248Z", "start_time": "2023-11-13T19:11:14.873671Z" } }, "source": [ "# Start from some test data" ] }, { "cell_type": "code", "execution_count": null, "id": "570dbaf1", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:33.594147Z", "start_time": "2024-04-24T02:00:21.017085Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/metas.json\", \"r\") as fp:\n", "# with open(\"/home/tony/Data/Hoot/ytm_metas.json\", \"r\") as fp:\n", "with open(\"/home/tony/Data/Hoot/deezer_metas.json\", \"r\") as fp:\n", " metas = json.load(fp)\n", "print(len(metas))\n", "metas_map = {m[\"id\"]: m for m in metas}\n", "original_id_metas_map = {m[\"original_id\"]: m for m in metas}" ] }, { "cell_type": "code", "execution_count": null, "id": "04ae3a08", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:33.610536Z", "start_time": "2024-04-24T02:00:33.597934Z" } }, "outputs": [], "source": [ "# train_info = []\n", "# with open(\n", "# \"/home/tony/Data/Hoot/multi_large_vocab_filtered_train_manifest.json\", \"r\"\n", "# ) as fp:\n", "# for l in tqdm.tqdm(fp):\n", "# my_l = json.loads(l)\n", "# if (\n", "# my_l[\"lang\"] == \"zh\"\n", "# and my_l[\"audio_filepath\"] == \"/app/suno/data/hoot/LL50Fu4UvG0.wav\"\n", "# ):\n", "# train_info.append(json.loads(l))" ] }, { "cell_type": "code", "execution_count": null, "id": "9e8350de", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:33.871366Z", "start_time": "2024-04-24T02:00:33.612179Z" } }, "outputs": [], "source": [ "train_info = []\n", "hooted_info = []\n", "for i, meta in enumerate(metas_map.values()):\n", " if meta[\"id\"] == \"sSA7CmtRkdQ\":\n", " # if meta[\"id\"] == \"74b2ba74-e58b-4750-a255-30c6fcdaf1e6\":\n", " # if meta[\"genius_slug\"] == \"Jay-chou-chrysanthemum-terrace-lyrics\":\n", " # if meta[\"genius_slug\"] == \"Alonzo-tout-va-bien-lyrics\":\n", " # if meta[\"genius_slug\"] == \"Lubalin-due-time-lyrics\":\n", " # if meta[\"genius_slug\"] == \"Taylor-swift-anti-hero-lyrics\":\n", " # if meta[\"genius_slug\"].startswith(\"Taylor-swift\"):\n", " # print(meta)\n", " train_info.append(meta)\n", " # print(i)\n", " with open(\n", " f\"/home/tony/Data/Hoot/alignments/outputs_3_deezer/batch_{(i // 100) * 100}.json\",\n", " # f\"/home/tony/Data/Hoot/alignments/outputs_3/batch_{(i // 100) * 100}.json\",\n", " \"r\",\n", " ) as fp:\n", " hooted_outputs = json.load(fp)\n", " for hooted_output in hooted_outputs:\n", " if hooted_output[\"id\"] == meta[\"id\"]:\n", " hooted_info.append(hooted_output)\n", "print(len(train_info), len(hooted_info))\n", "# print(train_info, hooted_info)" ] }, { "cell_type": "code", "execution_count": null, "id": "59429a37", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:33.885518Z", "start_time": "2024-04-24T02:00:33.873245Z" } }, "outputs": [], "source": [ "# # # for debug a single one\n", "# train_info = []\n", "# train_info.append(metas[925186])\n", "# train_info[0]" ] }, { "cell_type": "markdown", "id": "a3d112c7", "metadata": {}, "source": [ "# Redo hoot" ] }, { "cell_type": "code", "execution_count": null, "id": "bbd3e51e", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:33.951359Z", "start_time": "2024-04-24T02:00:33.887026Z" } }, "outputs": [], "source": [ "align_metas = []" ] }, { "cell_type": "code", "execution_count": null, "id": "380e3695", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:34.012892Z", "start_time": "2024-04-24T02:00:33.952919Z" } }, "outputs": [], "source": [ "_ = _reload_models_if_needed()" ] }, { "cell_type": "code", "execution_count": null, "id": "9ba66c36", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:38.719934Z", "start_time": "2024-04-24T02:00:34.014588Z" } }, "outputs": [], "source": [ "# for test_id in [10]:\n", "for test_id in tqdm.tqdm(range(len(train_info))):\n", " meta_id = train_info[test_id][\"id\"]\n", " out = encode_filepaths([metas_map[meta_id][\"audio_filepath\"]], return_logits=True)\n", " basic_cleaned_lyrics = (\n", " metas_map[meta_id][\"lyrics\"]\n", " if \"lyrics\" in metas_map[meta_id]\n", " else metas_map[meta_id][\"text\"]\n", " )\n", " decoded_preds = tokenizer.decode_logits(out[0], prior_text=basic_cleaned_lyrics)\n", " true_text_norm = clean_text(basic_cleaned_lyrics)\n", " cer_val = round(get_cer(true_text_norm, decoded_preds), 3)\n", " word_timings = ctc_align(out[0].astype(np.float16), basic_cleaned_lyrics)\n", " print(cer_val, decoded_preds)\n", " align_metas.append(\n", " {\n", " \"id\": meta_id,\n", " \"duration_s\": metas_map[meta_id][\"duration_s\"],\n", " \"alignment\": word_timings,\n", " \"cer_val\": cer_val,\n", " \"lyrics\": basic_cleaned_lyrics,\n", " }\n", " )" ] }, { "cell_type": "code", "execution_count": null, "id": "051fa022", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:38.737522Z", "start_time": "2024-04-24T02:00:38.721918Z" } }, "outputs": [], "source": [ "metas_map[meta_id]" ] }, { "cell_type": "code", "execution_count": null, "id": "19d8c87d", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:38.798090Z", "start_time": "2024-04-24T02:00:38.738506Z" } }, "outputs": [], "source": [ "aligned_lyrics_map = get_aligned_lyrics(align_metas)" ] }, { "cell_type": "code", "execution_count": null, "id": "577170f0", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:38.852006Z", "start_time": "2024-04-24T02:00:38.799574Z" } }, "outputs": [], "source": [ "# check we get the oraligned_lyrics_mapiginal text out\n", "meta_id = list(aligned_lyrics_map.keys())[0]\n", "original_id, lyrics = (\n", " metas_map[meta_id][\"original_id\"],\n", " basic_cleaned_lyrics,\n", ")\n", "assert \"\".join(w[\"text\"] for w in list(aligned_lyrics_map.values())[0]) == lyrics" ] }, { "cell_type": "code", "execution_count": null, "id": "ef49fc1c", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:38.909311Z", "start_time": "2024-04-24T02:00:38.853566Z" } }, "outputs": [], "source": [ "print(\"\".join(w[\"text\"] for w in list(aligned_lyrics_map.values())[0]))" ] }, { "cell_type": "code", "execution_count": null, "id": "e0d1e588", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:38.965222Z", "start_time": "2024-04-24T02:00:38.911006Z" } }, "outputs": [], "source": [ "print(lyrics)" ] }, { "cell_type": "code", "execution_count": null, "id": "f0ed65a8", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.214308Z", "start_time": "2024-04-24T02:00:38.966246Z" } }, "outputs": [], "source": [ "audio = Audio.from_file(os.path.join(\"/app/suno/data/hoot\", f\"{original_id}.wav\"))\n", "for s in aligned_lyrics_map[meta_id]:\n", " print(\n", " s[\"text\"],\n", " s[\"start_s\"],\n", " s[\"end_s\"],\n", " s[\"end_s\"] - s[\"start_s\"],\n", " s[\"vocal_start_s\"],\n", " )\n", " audio.get_segment(s[\"start_s\"], s[\"end_s\"]).play()\n", " print(\"-\" * 10)" ] }, { "cell_type": "markdown", "id": "9d7224f3", "metadata": {}, "source": [ "# Hooted lyrics alignment" ] }, { "cell_type": "code", "execution_count": null, "id": "ed1a160e", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.215997Z", "start_time": "2024-04-24T02:00:39.215987Z" } }, "outputs": [], "source": [ "# if debug, may need this:\n", "# model_list, model, tokenizer = _reload_models_if_needed()" ] }, { "cell_type": "code", "execution_count": null, "id": "8dbc0f18", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:01:45.582069Z", "start_time": "2024-04-24T02:01:45.053008Z" } }, "outputs": [], "source": [ "test_input_id = \"f249512f-67f6-4f83-8af3-954a7951fdae\"\n", "# audio = Audio.from_s3(f\"s3://suno-data-uploads/studio/uploads/{test_input_id}.mp3\")\n", "audio = Audio.from_file(\"../audios/walking_down.mp3\")\n", "print(audio.duration_s)" ] }, { "cell_type": "code", "execution_count": null, "id": "c48fa719", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:01:47.041238Z", "start_time": "2024-04-24T02:01:47.028757Z" } }, "outputs": [], "source": [ "prior_texts = \"\"\"\n", "[intro]\n", "\n", "[verse]\n", "\n", "Walking down the street, feeling so alive\n", "Got my head in the clouds, got a gleam in my eye\n", "Every step I take, it's like a brand new start\n", "No matter where I'm going, I'll always find my part\n", "(oh-oh-oh)\n", "\n", "[chorus]\n", "\n", "Life is like a high-wire act, we're dancing in the sky\n", "No need to worry, no need to ask why\n", "With a little bit of courage, we can chase our dreams\n", "No matter what comes our way, we'll always be a team\n", "(we're unstoppable, yeah)\n", "\n", "[outro]\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "eabecb44", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:01:53.468861Z", "start_time": "2024-04-24T02:01:53.456622Z" } }, "outputs": [], "source": [ "# prior_texts = \" [outro] \\n [outro] \\n [outro] \\n\" + prior_texts + \" [outro] \\n [outro] \\n [outro] \\n\"" ] }, { "cell_type": "code", "execution_count": null, "id": "b0bf5547", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:01:54.158047Z", "start_time": "2024-04-24T02:01:53.799535Z" } }, "outputs": [], "source": [ "padded_words_timed = get_word_timing_from_audio_and_lyrics(audio, prior_texts)" ] }, { "cell_type": "code", "execution_count": null, "id": "d132b7d1", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:02:40.531772Z", "start_time": "2024-04-24T02:02:40.516775Z" } }, "outputs": [], "source": [ "print(padded_words_timed)" ] }, { "cell_type": "code", "execution_count": null, "id": "4ee614fe", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:01:54.249149Z", "start_time": "2024-04-24T02:01:54.159954Z" } }, "outputs": [], "source": [ "annotated_lines = _merge_into_lines(padded_words_timed)\n", "print(annotated_lines)\n", "assert \"\".join(k[\"text\"] for k in annotated_lines) == prior_texts" ] }, { "cell_type": "code", "execution_count": null, "id": "2542dbff", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.220381Z", "start_time": "2024-04-24T02:00:39.220373Z" } }, "outputs": [], "source": [ "print(padded_words_timed)" ] }, { "cell_type": "code", "execution_count": null, "id": "95a8f592", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.221038Z", "start_time": "2024-04-24T02:00:39.221030Z" } }, "outputs": [], "source": [ "# try chinese\n", "print(metas_map[meta_id])\n", "audio = Audio.from_s3(f\"{metas_map[meta_id]['audio_filepath']}\")\n", "prior_texts = metas_map[meta_id][\"lyrics\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "57936afa", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.221628Z", "start_time": "2024-04-24T02:00:39.221620Z" } }, "outputs": [], "source": [ "padded_words_timed = get_word_timing_from_audio_and_lyrics(audio, prior_texts)" ] }, { "cell_type": "code", "execution_count": null, "id": "0d99c763", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.222457Z", "start_time": "2024-04-24T02:00:39.222449Z" } }, "outputs": [], "source": [ "print(padded_words_timed)" ] }, { "cell_type": "markdown", "id": "7799768a", "metadata": {}, "source": [ "# Direct loop" ] }, { "cell_type": "code", "execution_count": null, "id": "36e317df", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.223142Z", "start_time": "2024-04-24T02:00:39.223134Z" } }, "outputs": [], "source": [ "# hooted_info = []\n", "# with open(\n", "# f\"/home/tony/Data/Hoot/alignments/outputs_2nd_2/batch_73200.json\", \"r\"\n", "# ) as fp:\n", "# hooted_outputs = json.load(fp)\n", "# for hooted_output in hooted_outputs[2:]:\n", "# hooted_info.append(hooted_output)\n", "# break\n", "# if hooted_info:\n", "# aligned_lyrics_map = get_aligned_lyrics(\n", "# hooted_info[:1], silent=True, return_only_valid=True\n", "# )" ] }, { "cell_type": "code", "execution_count": null, "id": "461f7325", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.223864Z", "start_time": "2024-04-24T02:00:39.223856Z" } }, "outputs": [], "source": [ "meta_id = list(aligned_lyrics_map.keys())[0]\n", "original_id, lyrics = (\n", " metas_map[meta_id][\"original_id\"],\n", " metas_map[meta_id][\"lyrics\"],\n", ")\n", "audio = Audio.from_file(os.path.join(\"/app/suno/data/hoot\", f\"{original_id}.wav\"))\n", "# print(lyrics)\n", "# audio.play()\n", "print(\"check aligned clips\")\n", "# for s in aligned_lyrics_map[meta_id]:\n", "# print(\n", "# s[\"text\"],\n", "# s[\"start_s\"],\n", "# s[\"end_s\"],\n", "# s[\"end_s\"] - s[\"start_s\"],\n", "# s[\"vocal_start_s\"],\n", "# )\n", "# audio.get_segment(s[\"start_s\"], s[\"end_s\"]).play()\n", "# print(\"-\" * 10)" ] }, { "cell_type": "markdown", "id": "d4171686", "metadata": {}, "source": [ "# alignment jobs" ] }, { "cell_type": "code", "execution_count": null, "id": "cf886804", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.224444Z", "start_time": "2024-04-24T02:00:39.224437Z" } }, "outputs": [], "source": [ "# python hoot_alignment.py --start_index 1200000 --end_index 1300000" ] }, { "cell_type": "code", "execution_count": null, "id": "8443b212", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.225217Z", "start_time": "2024-04-24T02:00:39.225209Z" } }, "outputs": [], "source": [ "# python hoot_alignment.py --start_index 1050000 --end_index 1100000" ] }, { "cell_type": "code", "execution_count": null, "id": "ec2cd5ba", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.225891Z", "start_time": "2024-04-24T02:00:39.225883Z" } }, "outputs": [], "source": [ "# 1836168 / 100" ] }, { "cell_type": "code", "execution_count": null, "id": "99f9e4ae", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.226581Z", "start_time": "2024-04-24T02:00:39.226573Z" } }, "outputs": [], "source": [ "# python hoot_alignment.py --start_index 0 --end_index 1900000" ] }, { "cell_type": "code", "execution_count": null, "id": "714da821", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.227168Z", "start_time": "2024-04-24T02:00:39.227161Z" } }, "outputs": [], "source": [ "# python hoot_alignment.py --start_index 290000 --end_index 300000" ] }, { "cell_type": "code", "execution_count": null, "id": "bc9f09d3", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.227879Z", "start_time": "2024-04-24T02:00:39.227872Z" } }, "outputs": [], "source": [ "# step = 200000\n", "# for i in range(0, 1836168, step):\n", "# print(f\"python hoot_alignment.py --start_index {i} --end_index {i + step}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "7781ac60", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.228612Z", "start_time": "2024-04-24T02:00:39.228604Z" } }, "outputs": [], "source": [ "import re" ] }, { "cell_type": "code", "execution_count": null, "id": "150dfc59", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.229294Z", "start_time": "2024-04-24T02:00:39.229286Z" } }, "outputs": [], "source": [ "print(re.match(\".\\w\", \"[intro : tag]\"))\n", "print(re.match(\".\\w\", \"je's\"))\n", "print(re.match(\".\\w+\", \"]\"))\n", "print(re.match(\".\\w+\", \"\\n\"))\n", "print(re.split(\"\\w\", \"\\n\\n\"))\n", "print(re.findall(r\"\\n\\n\", \"是谁在阁楼上冰冷的绝望\\n\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "f6825ffe", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.230003Z", "start_time": "2024-04-24T02:00:39.229996Z" } }, "outputs": [], "source": [ "a = list(range(20))" ] }, { "cell_type": "code", "execution_count": null, "id": "ac4a4319", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.230597Z", "start_time": "2024-04-24T02:00:39.230589Z" } }, "outputs": [], "source": [ "a[:-5]" ] }, { "cell_type": "code", "execution_count": null, "id": "8fb90a31", "metadata": { "ExecuteTime": { "end_time": "2024-04-24T02:00:39.231372Z", "start_time": "2024-04-24T02:00:39.231364Z" } }, "outputs": [], "source": [ "a[-5:]" ] }, { "cell_type": "code", "execution_count": null, "id": "d74847c8", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env_dev", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }