{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "0e09c52f-da2a-4a21-b659-b58383cba6f4",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:45:35.597375Z",
"iopub.status.busy": "2024-12-06T13:45:35.597241Z",
"iopub.status.idle": "2024-12-06T13:45:35.601693Z",
"shell.execute_reply": "2024-12-06T13:45:35.601285Z",
"shell.execute_reply.started": "2024-12-06T13:45:35.597358Z"
}
},
"outputs": [],
"source": [
"# !nvidia-smi\n",
"# !echo $HOSTNAME\n",
"\n",
"import os\n",
"\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"3\""
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "03070210",
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"from suno_utils.audio import Audio\n",
"from suno_utils.diffusion.generation import generate_chunk, DiffusionGenerationConfig\n",
"from suno_utils.tasks.upsample_engine import UpsampleEngine, Request\n",
"from suno_utils.diffusion import generation as diffusion_gen\n",
"import time\n",
"\n",
"import json\n",
"import numpy as np\n",
"from suno_utils.utils.s3 import read_from_s3\n",
"\n",
"\n",
"np.random.seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "c046f7b4",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:45:35.602274Z",
"iopub.status.busy": "2024-12-06T13:45:35.602144Z",
"iopub.status.idle": "2024-12-06T13:46:12.711927Z",
"shell.execute_reply": "2024-12-06T13:46:12.711104Z",
"shell.execute_reply.started": "2024-12-06T13:45:35.602261Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"loading semantic model...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/tony/Work/glockenspiel/suno_utils/suno_utils/tasks/mert_25.py:148: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
" sd = _torch_load_p(checkpoint_filepath)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"loading codec model...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/tony/Work/glockenspiel/suno_utils/suno_utils/tasks/dac_vae_100hz_peaq.py:73: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
" sd = load_f(checkpoint_filepath)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"loading diffusion model...\n",
"loading weights...\n",
"converting model to precision torch.bfloat16...\n",
"dit_model loaded onto cuda:0\n",
"loading tokenizer...\n",
"done!\n"
]
}
],
"source": [
"diffusion_gen.preload_models(\n",
" tokenizer_filepath=\"/app/suno/data/dpo/models/tokenizer_60k.json\",\n",
" semantic_model_filepath=\"/app/suno/data/dpo/models/mert_25.pt\",\n",
" semantic_clusters_filepath=\"/app/suno/data/dpo/models/mert_25_2x4k.npy\",\n",
" codec_filepath=\"/home/georg/notebooks/gpu_nb/tmp/25hz_vae_peaq_kl_0.005.pth\",\n",
" compile=True, # can't compile with flash v2\n",
" dit_model_filepath=\"/app/suno/checkpoints/2024-12-05_03-42-43_s4538/step_5000_infer.pt\",\n",
")\n",
"duration_s = 30"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "cf696991",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:46:14.634044Z",
"iopub.status.busy": "2024-12-06T13:46:14.633608Z",
"iopub.status.idle": "2024-12-06T13:46:17.829185Z",
"shell.execute_reply": "2024-12-06T13:46:17.828454Z",
"shell.execute_reply.started": "2024-12-06T13:46:14.634026Z"
},
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"s3://suno-data-uploads/studio/uploads/ae137e1f-7562-48f6-a11c-b125e645282c.npz\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# gen_id = \"99bffa17-7e59-47b4-a048-5528cbda05d5\" # sister\n",
"# gen_id = \"081d73c4-7805-4212-9c80-8db1137ca3c4\" # friends\n",
"# gen_id = \"562f762d-6ced-4080-9af1-910ee3d0a5dc\" # something real\n",
"# gen_id = \"23c15c62-494d-422d-8a60-8b0454044322\" # rubber duck\n",
"# gen_id = \"4b140a9e-964b-422c-85b5-5861ad1a9d38\" # once\n",
"# gen_id = \"7b214347-fa38-4e9b-96f4-f7ec65adea45\" # rock n roll\n",
"# gen_id = \"a5e2198a-f352-4abb-9a24-7f81b143ded3\" # stone\n",
"# gen_id = \"7f5e7819-b7a1-449d-a76c-9c38950c4274\" # canada\n",
"# gen_id = \"27135940-957f-4152-9e40-72e5c10a46bd\" # drunk\n",
"# gen_id = \"caf1d7ae-ce81-4c75-b737-fd41e47a38ba\" # linger\n",
"# gen_id = \"ac5aa29a-f18f-406f-a904-c47493f1beec\" # linger vox\n",
"# gen_id = \"b04719c3-012a-4a4a-9b02-3cb2875d1744\" # courtney\n",
"# gen_id = \"7407c7b3-8aa2-4f0a-82e8-fa19933f6e94\" # opera\n",
"# gen_id = \"5b9624ef-b416-4095-a5f3-4945f09bba86\" # butterflies\n",
"# gen_id = \"e757aa19-5c33-4945-aaa1-1519b7890f0b\" # bubblegum\n",
"# gen_id = \"7f774078-1672-4858-a37f-acad373c5a84\" # golden\n",
"# gen_id = \"1cee79db-a0ee-44e4-9646-d312cd620997\" # chinese\n",
"# gen_id = \"3b1830d7-5553-4785-be0b-ffeadf7756d8\" # debug\n",
"# gen_id_audio = gen_id\n",
"# gen_id = \"6845dc51-71b1-491c-b0be-1f6c691a2f42\"\n",
"\n",
"gen_id = \"ae137e1f-7562-48f6-a11c-b125e645282c\"\n",
"gen_id_audio = \"cb105e06-6e20-4acc-9d1c-e7c9b36b24e2\"\n",
"# gen_id_audio = gen_id\n",
"\n",
"s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n",
"mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id_audio}.mp3\"\n",
"print(s3_filepath)\n",
"data = read_from_s3(s3_filepath, read_f=np.load)\n",
"audio = Audio.from_s3(mp3_filepath, n_channels=2)\n",
"\n",
"if \"v3.0_raw\" in data:\n",
" codes = data[\"v3.0_raw\"]\n",
"elif \"v3.5_raw\" in data:\n",
" codes = data[\"v3.5_raw\"]\n",
"elif \"v4.0_raw\" in data:\n",
" codes = data[\"v4.0_raw\"]\n",
"else:\n",
" raise ValueError(\"No codes found\")\n",
"\n",
"text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n",
"aligned_lyrics = json.loads(text_data)\n",
"# print(aligned_lyrics)\n",
"# lyrics = \"\".join(w[\"word\"] for w in aligned_lyrics if \"word\" in w)\n",
"lyrics = \"walking down the streets feeling so alive i've got my head in the clouds got a gleam in my eye every step i take it's like a brand new start no matter where i'm going i'll always find my part life is like a hard wire act we're dancing in the sky no need to worry no need to ask why with a little bit of courage we can chase our dreams no matter what comes our way we'll always be a team we're unstoppable yeah\"\n",
"audio.normalize_volume().play()\n",
"sem_codes = torch.tensor(codes[:, 0]).to(torch.long)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "9b6e8953",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:46:17.830104Z",
"iopub.status.busy": "2024-12-06T13:46:17.829941Z",
"iopub.status.idle": "2024-12-06T13:46:17.833988Z",
"shell.execute_reply": "2024-12-06T13:46:17.833408Z",
"shell.execute_reply.started": "2024-12-06T13:46:17.830089Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[0, 1, 2]\n"
]
}
],
"source": [
"# seeds = np.random.randint(0, 2**32 - 1, 3)\n",
"seeds = [0, 1, 2]\n",
"print(seeds)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "497e54f5",
"metadata": {},
"outputs": [],
"source": [
"steps = 12\n",
"text_cfg_coef = 2.0\n",
"ctx_cfg_coef = 1.0"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "a9529e00",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\u001b[A\u001b[A"
]
}
],
"source": [
"upsample_engine = UpsampleEngine(min_chunk_size=25 * 30)\n",
"upsample_engine.start()"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "7c9c5fca",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Adding upsample request test_seed_0\n",
"Warning: 1/1 jobs are ready to process\n",
"Shortest waiting buffer: -0.02s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\u001b[A\u001b[A"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: 1/1 jobs are ready to process\n",
"Shortest waiting buffer: 29.04s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\u001b[A\u001b[A"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Adding upsample request test_seed_1\n",
"Warning: 1/2 jobs are ready to process\n",
"Shortest waiting buffer: -0.02s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\u001b[A\u001b[A"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: 1/2 jobs are ready to process\n",
"Shortest waiting buffer: 29.06s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\u001b[A\u001b[A"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Adding upsample request test_seed_2\n",
"Warning: 1/3 jobs are ready to process\n",
"Shortest waiting buffer: -0.01s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\u001b[A\u001b[A"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: 1/3 jobs are ready to process\n",
"Shortest waiting buffer: 29.07s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\u001b[A\u001b[A"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"for seed in seeds:\n",
" diffusion_generation_config = diffusion_gen.DiffusionGenerationConfig(\n",
" lyrics=lyrics or \"\",\n",
" tags=\"orchestral, orchestra, classical, epic movie\",\n",
" text_cfg_coef=text_cfg_coef,\n",
" ctx_cfg_coef=ctx_cfg_coef,\n",
" steps=steps,\n",
" seed=seed,\n",
" )\n",
" job = upsample_engine.add_request(\n",
" Request(\n",
" f\"test_seed_{seed}\",\n",
" diffusion_generation_config,\n",
" sem_codes,\n",
" input_tokens_finished=True,\n",
" )\n",
" )\n",
" audios = []\n",
" for audio in job.audio_generator():\n",
" audios.append(audio)\n",
" upsampled_audio = Audio.concatenate(audios)\n",
" upsampled_audio.play()"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "b7c11f0d",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
}
],
"source": [
"upsample_engine_short = UpsampleEngine(min_chunk_size=25 * 15)\n",
"upsample_engine_short.start()"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "274a2d1f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Adding upsample request test_seed_0\n",
"Warning: 1/1 jobs are ready to process\n",
"Shortest waiting buffer: -0.03s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: 1/1 jobs are ready to process\n",
"Shortest waiting buffer: 14.06s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: 1/1 jobs are ready to process\n",
"Shortest waiting buffer: 28.37s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: 1/1 jobs are ready to process\n",
"Shortest waiting buffer: 42.69s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Adding upsample request test_seed_1\n",
"Warning: 1/2 jobs are ready to process\n",
"Shortest waiting buffer: -0.00s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: 1/2 jobs are ready to process\n",
"Shortest waiting buffer: 14.12s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: 1/2 jobs are ready to process\n",
"Shortest waiting buffer: 28.44s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: 1/2 jobs are ready to process\n",
"Shortest waiting buffer: 42.76s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Adding upsample request test_seed_2\n",
"Warning: 1/3 jobs are ready to process\n",
"Shortest waiting buffer: -0.03s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: 1/3 jobs are ready to process\n",
"Shortest waiting buffer: 14.09s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: 1/3 jobs are ready to process\n",
"Shortest waiting buffer: 28.41s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: 1/3 jobs are ready to process\n",
"Shortest waiting buffer: 42.73s\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[A\u001b[A\u001b[A\u001b[A"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"for seed in seeds:\n",
" diffusion_generation_config = diffusion_gen.DiffusionGenerationConfig(\n",
" lyrics=lyrics or \"\",\n",
" tags=\"orchestral, orchestra, classical, epic movie\",\n",
" text_cfg_coef=text_cfg_coef,\n",
" ctx_cfg_coef=ctx_cfg_coef,\n",
" steps=steps,\n",
" seed=seed,\n",
" )\n",
" job = upsample_engine_short.add_request(\n",
" Request(\n",
" f\"test_seed_{seed}\",\n",
" diffusion_generation_config,\n",
" sem_codes,\n",
" input_tokens_finished=True,\n",
" )\n",
" )\n",
" audios = []\n",
" for audio in job.audio_generator():\n",
" audios.append(audio)\n",
" upsampled_audio = Audio.concatenate(audios)\n",
" upsampled_audio.play()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "f6d3d5f3-fcfa-4b95-af01-98e6c01d8f86",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:46:26.466716Z",
"iopub.status.busy": "2024-12-06T13:46:26.466571Z",
"iopub.status.idle": "2024-12-06T13:46:28.276299Z",
"shell.execute_reply": "2024-12-06T13:46:28.275815Z",
"shell.execute_reply.started": "2024-12-06T13:46:26.466700Z"
}
},
"outputs": [],
"source": [
"# for i in [0, 1, 2]:\n",
"# ref_audio = Audio.from_file(\n",
"# f\"/home/tony/Work/tony/audios/walking_down_diff_ft_1_ref_{i}.wav\"\n",
"# )\n",
"# ref_audio.play()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "1e2f7d0b-37d5-485e-a153-a53e5e7b4c01",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:46:28.276988Z",
"iopub.status.busy": "2024-12-06T13:46:28.276841Z",
"iopub.status.idle": "2024-12-06T13:46:32.259367Z",
"shell.execute_reply": "2024-12-06T13:46:32.258825Z",
"shell.execute_reply.started": "2024-12-06T13:46:28.276974Z"
},
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"s3://suno-data-uploads/studio/uploads/3b1830d7-5553-4785-be0b-ffeadf7756d8.npz\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"gen_id = \"3b1830d7-5553-4785-be0b-ffeadf7756d8\" # debug\n",
"gen_id_audio = gen_id\n",
"\n",
"s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n",
"mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id_audio}.mp3\"\n",
"print(s3_filepath)\n",
"data = read_from_s3(s3_filepath, read_f=np.load)\n",
"\n",
"audio = Audio.from_s3(mp3_filepath, n_channels=2)\n",
"\n",
"if \"v3.0_raw\" in data:\n",
" codes = data[\"v3.0_raw\"]\n",
"elif \"v3.5_raw\" in data:\n",
" codes = data[\"v3.5_raw\"]\n",
"elif \"v4.0_raw\" in data:\n",
" codes = data[\"v4.0_raw\"]\n",
"else:\n",
" raise ValueError(\"No codes found\")\n",
"\n",
"text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n",
"aligned_lyrics = json.loads(text_data)\n",
"# print(aligned_lyrics)\n",
"lyrics = \"\".join(w[\"word\"] for w in aligned_lyrics if \"word\" in w)\n",
"audio.normalize_volume().play()"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "00e3d45c",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:46:32.260073Z",
"iopub.status.busy": "2024-12-06T13:46:32.259928Z",
"iopub.status.idle": "2024-12-06T13:46:49.404722Z",
"shell.execute_reply": "2024-12-06T13:46:49.404235Z",
"shell.execute_reply.started": "2024-12-06T13:46:32.260059Z"
}
},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'generate' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[13], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m audios \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m n \u001b[38;5;129;01min\u001b[39;00m seeds:\n\u001b[0;32m----> 4\u001b[0m pred_audio \u001b[38;5;241m=\u001b[39m \u001b[43mgenerate\u001b[49m(\n\u001b[1;32m 5\u001b[0m sem_codes,\n\u001b[1;32m 6\u001b[0m lyrics\u001b[38;5;241m=\u001b[39mlyrics,\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m# history_lyrics=history_text,\u001b[39;00m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;66;03m# tags='shoegaze, ethereal, art, ego',\u001b[39;00m\n\u001b[1;32m 9\u001b[0m tags\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m80s, new wave, female power\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 10\u001b[0m text_cfg_coef\u001b[38;5;241m=\u001b[39mtext_cfg_coef,\n\u001b[1;32m 11\u001b[0m ctx_cfg_coef\u001b[38;5;241m=\u001b[39mctx_cfg_coef,\n\u001b[1;32m 12\u001b[0m steps\u001b[38;5;241m=\u001b[39msteps,\n\u001b[1;32m 13\u001b[0m seed\u001b[38;5;241m=\u001b[39mn,\n\u001b[1;32m 14\u001b[0m )\u001b[38;5;241m.\u001b[39mnormalize_volume()\n\u001b[1;32m 15\u001b[0m audios\u001b[38;5;241m.\u001b[39mappend(pred_audio)\n\u001b[1;32m 16\u001b[0m pred_audio\u001b[38;5;241m.\u001b[39mplay()\n",
"\u001b[0;31mNameError\u001b[0m: name 'generate' is not defined"
]
}
],
"source": [
"sem_codes = torch.tensor(codes[:, 0]).to(torch.long)\n",
"audios = []\n",
"for n in seeds:\n",
" pred_audio = generate(\n",
" sem_codes,\n",
" lyrics=lyrics,\n",
" # history_lyrics=history_text,\n",
" # tags='shoegaze, ethereal, art, ego',\n",
" tags=\"80s, new wave, female power\",\n",
" text_cfg_coef=text_cfg_coef,\n",
" ctx_cfg_coef=ctx_cfg_coef,\n",
" steps=steps,\n",
" seed=n,\n",
" ).normalize_volume()\n",
" audios.append(pred_audio)\n",
" pred_audio.play()\n",
"# break"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "77725223",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:46:49.405457Z",
"iopub.status.busy": "2024-12-06T13:46:49.405310Z",
"iopub.status.idle": "2024-12-06T13:46:49.408034Z",
"shell.execute_reply": "2024-12-06T13:46:49.407627Z",
"shell.execute_reply.started": "2024-12-06T13:46:49.405442Z"
}
},
"outputs": [],
"source": [
"# chunk_window_size = 3000\n",
"\n",
"# for n_chunk in range(8):\n",
"# idx_pad = 5\n",
"# chunk_start_s = n_chunk * chunk_window_size / models[\"dit_model\"].io_hz\n",
"# chunk_end_s = (n_chunk + 1) * chunk_window_size / models[\"dit_model\"].io_hz\n",
"# print(chunk_start_s, chunk_end_s)\n",
"\n",
"# chunk_start_indices = [\n",
"# idx\n",
"# for idx, m in enumerate(aligned_lyrics)\n",
"# if \"start_s\" in m and m[\"start_s\"] >= chunk_start_s\n",
"# ]\n",
"# chunk_end_indices = [\n",
"# idx for idx, m in enumerate(aligned_lyrics) if \"end_s\" in m and m[\"end_s\"] <= chunk_end_s\n",
"# ]\n",
"\n",
"\n",
"# if len(chunk_start_indices) > 0 and len(chunk_end_indices) > 0:\n",
"# chunk_start_idx = max(0, chunk_start_indices[0] - idx_pad)\n",
"# chunk_end_idx = min(len(aligned_lyrics), chunk_end_indices[-1] + idx_pad)\n",
"# print(chunk_start_idx, chunk_end_idx)\n",
"# lyrics_chunk = \"\".join(\n",
"# [m[\"word\"] for m in aligned_lyrics[chunk_start_idx:chunk_end_idx]]\n",
"# )\n",
"# #lyrics_chunk = simplify_whitespace(lyrics_chunk, retain_newlines=True)\n",
"# else:\n",
"# lyrics_chunk = \"\"\n",
"# print(lyrics_chunk)"
]
},
{
"cell_type": "markdown",
"id": "441ef260",
"metadata": {},
"source": [
"# Testbench"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "2115a8e4",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:46:49.408629Z",
"iopub.status.busy": "2024-12-06T13:46:49.408500Z",
"iopub.status.idle": "2024-12-06T13:46:49.448779Z",
"shell.execute_reply": "2024-12-06T13:46:49.448381Z",
"shell.execute_reply.started": "2024-12-06T13:46:49.408617Z"
}
},
"outputs": [],
"source": [
"# import json\n",
"# import numpy as np\n",
"# from suno_utils.utils.s3 import read_from_s3\n",
"# import shutil\n",
"# output_dir = \"outputs/custom_infer_stable_audio-10102024\"\n",
"# if os.path.exists(output_dir):\n",
"# shutil.rmtree(output_dir)\n",
"# os.makedirs(output_dir, exist_ok=True)\n",
"\n",
"# gen_ids = [\n",
"# (\"99bffa17-7e59-47b4-a048-5528cbda05d5\", \"country\"), # sister\n",
"# (\"081d73c4-7805-4212-9c80-8db1137ca3c4\", \"rock\"), # friends\n",
"# (\"562f762d-6ced-4080-9af1-910ee3d0a5dc\", \"rock\"), # something real\n",
"# (\"23c15c62-494d-422d-8a60-8b0454044322\", \"shanty\"), # rubber duck\n",
"# (\"4b140a9e-964b-422c-85b5-5861ad1a9d38\", \"indie rock\"), # once\n",
"# (\"7b214347-fa38-4e9b-96f4-f7ec65adea45\", \"rock n roll\"), # rock n roll\n",
"# (\"7f5e7819-b7a1-449d-a76c-9c38950c4274\", \"pop\"), # canada\n",
"# (\"27135940-957f-4152-9e40-72e5c10a46bd\", \"pop\"), # drunk\n",
"# (\"caf1d7ae-ce81-4c75-b737-fd41e47a38ba\", \"pop\"), # linger\n",
"# (\"ac5aa29a-f18f-406f-a904-c47493f1beec\", \"pop\"), # linger vox\n",
"# (\"b04719c3-012a-4a4a-9b02-3cb2875d1744\", \"pop\"), # courtney\n",
"# (\"7407c7b3-8aa2-4f0a-82e8-fa19933f6e94\", \"opera\"), # opera\n",
"# (\"5b9624ef-b416-4095-a5f3-4945f09bba86\", \"math rock\"), # butterflies\n",
"# ]\n",
"\n",
"# for gen_id, tags in gen_ids:\n",
"# mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.mp3\"\n",
"# print(tags, mp3_filepath)\n",
"# audio = Audio.from_s3(mp3_filepath, n_channels=2)\n",
"# audio.normalize_volume()\n",
"# text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n",
"# aligned_lyrics = json.loads(text_data)\n",
"\n",
"# steps = 32\n",
"# text_cfg_coef = 4.0\n",
"# ctx_cfg_coef = 2.0\n",
"# seeds = np.random.randint(0, 2**32 - 1, 3)\n",
"\n",
"# for n in seeds:\n",
"# pred_audio = generate(\n",
"# audio,\n",
"# aligned_lyrics=aligned_lyrics,\n",
"# tags=tags,\n",
"# text_cfg_coef=text_cfg_coef,\n",
"# ctx_cfg_coef=ctx_cfg_coef,\n",
"# steps=steps,\n",
"# seed=n,\n",
"# ).normalize_volume()\n",
"\n",
"# # save audio to disk\n",
"# pred_audio.write_mp3(f\"{output_dir}/pred_{gen_id}_{n}.mp3\")"
]
},
{
"cell_type": "markdown",
"id": "bb494e8a",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "2ce62b93",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:46:49.449339Z",
"iopub.status.busy": "2024-12-06T13:46:49.449209Z",
"iopub.status.idle": "2024-12-06T13:46:49.486047Z",
"shell.execute_reply": "2024-12-06T13:46:49.485602Z",
"shell.execute_reply.started": "2024-12-06T13:46:49.449326Z"
}
},
"outputs": [],
"source": [
"# import json\n",
"# with open(\"walking_aligned_lyrics.json\") as f:\n",
"# aligned_lyrics = json.load(f)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "75675270",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:46:49.486667Z",
"iopub.status.busy": "2024-12-06T13:46:49.486530Z",
"iopub.status.idle": "2024-12-06T13:46:49.522254Z",
"shell.execute_reply": "2024-12-06T13:46:49.521571Z",
"shell.execute_reply.started": "2024-12-06T13:46:49.486654Z"
}
},
"outputs": [],
"source": [
"# # TODO: pad with silence for integer multiple of sliding\n",
"# if models[\"dit_model\"].ctx_len is not None:\n",
"# window_s = int(round((models[\"dit_model\"].block_size - models[\"dit_model\"].ctx_len) / models[\"dit_model\"].io_hz))\n",
"# if audio.duration_s % window_s > 0.1:\n",
"# audio = Audio.concatenate([\n",
"# audio,\n",
"# Audio.from_silence(window_s - audio.duration_s % window_s + 0.01, sample_rate=audio.sample_rate, n_channels=audio.n_channels)\n",
"# ])"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "e2d70822",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:46:49.523062Z",
"iopub.status.busy": "2024-12-06T13:46:49.522907Z",
"iopub.status.idle": "2024-12-06T13:46:49.557699Z",
"shell.execute_reply": "2024-12-06T13:46:49.557215Z",
"shell.execute_reply.started": "2024-12-06T13:46:49.523047Z"
}
},
"outputs": [],
"source": [
"# # get latents for preconditioning\n",
"# import torch\n",
"# from suno_utils.tasks.dac_vae_100hz_peaq import encode as codec_encode\n",
"# # latent_audio = audio\n",
"# latent_audio = Audio.from_file(\"test.mp3\", sample_rate=48_000, n_channels=2).get_segment(from_s=offset_s, to_s=offset_s+duration_s)\n",
"# init_latents = torch.from_numpy(codec_encode(latent_audio).T[None]) * 2.5\n",
"# latent_audio.play()"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "9c946bc7",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:46:49.558519Z",
"iopub.status.busy": "2024-12-06T13:46:49.558376Z",
"iopub.status.idle": "2024-12-06T13:46:49.594652Z",
"shell.execute_reply": "2024-12-06T13:46:49.594168Z",
"shell.execute_reply.started": "2024-12-06T13:46:49.558506Z"
}
},
"outputs": [],
"source": [
"# import torch\n",
"# models = _retrieve_models()\n",
"# extra_args = torch.load(\"extra_args.pt\", weights_only=True)\n",
"# torch.manual_seed(0)\n",
"# x = torch.randn([1, 128, 100*30], device=\"cuda\")*50\n",
"# with torch.no_grad():\n",
"# out = models[\"dit_model\"].forward_inference(\n",
"# x,\n",
"# x.new_ones([x.shape[0]])*50,\n",
"# cross_attn_cond=extra_args[\"cross_attn_cond\"],\n",
"# empty_cross_attn_cond=extra_args[\"empty_cross_attn_cond\"],\n",
"# cfg_scale=extra_args[\"cfg_scale\"]\n",
"# )\n",
"# print(out.shape)\n",
"# out"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "818cf6a9",
"metadata": {
"execution": {
"iopub.execute_input": "2024-12-06T13:46:49.595362Z",
"iopub.status.busy": "2024-12-06T13:46:49.595229Z",
"iopub.status.idle": "2024-12-06T13:46:49.632236Z",
"shell.execute_reply": "2024-12-06T13:46:49.631749Z",
"shell.execute_reply.started": "2024-12-06T13:46:49.595350Z"
}
},
"outputs": [],
"source": [
"# PAG\n",
"# https://arxiv.org/pdf/2403.17377\n",
"# https://huggingface.co/hyoungwoncho/sd_perturbed_attention_guidance/blob/main/pipeline.py\n",
"# https://github.com/KU-CVLAB/Perturbed-Attention-Guidance?tab=readme-ov-file"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "abe70d19",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "56142e0c",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "7d56f346",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "b12de9ec",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "acda5fe8",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.15"
}
},
"nbformat": 4,
"nbformat_minor": 5
}