{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"| NVIDIA-SMI 560.35.03 Driver Version: 560.35.03 CUDA Version: 12.6 |\n",
"| 0 NVIDIA H100 80GB HBM3 Off | 00000000:0F:00.0 Off | 0 |\n"
]
}
],
"source": [
"!nvidia-smi | grep NVIDIA | head -2"
]
},
{
"cell_type": "code",
"execution_count": 63,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/minz/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
" warnings.warn(\n",
"/home/minz/glockenspiel/suno_utils/suno_utils/models/musicfm/modeling_MusicFM.py:305: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
" S = torch.load(model_path)[\"state_dict\"]\n",
"/home/minz/neon/ditto-training/ditto_v2/models/ditto.py:35: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
" S = torch.load(model_path)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"model loaded!\n"
]
}
],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"5\"\n",
"os.environ[\"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION\"] = \"python\"\n",
"os.environ['SLURM_PROCID'] = os.environ.get('SLURM_PROCID', '0')\n",
"\n",
"import sys\n",
"sys.path.append(\"/home/minz/neon/ditto-training\")\n",
"\n",
"import glob\n",
"import random\n",
"import torch\n",
"import numpy as np\n",
"from suno_utils.audio import Audio\n",
"from ditto_v2.models.ditto import Ditto\n",
"\n",
"# get model\n",
"ditto = Ditto(latent_dim=128, model_path=\"/home/minz/logs/ditto_v2_local_8gpu_cont/ditto_v2_epoch_57.pt\", is_flash=False)\n",
"ditto = ditto.eval()\n",
"ditto = ditto.to(\"cuda:0\")"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
"genius_filelist = glob.glob(\"/app/suno/data/v2_audio/genius_hq/*.wav\")\n",
"random.seed(134)\n",
"random.shuffle(genius_filelist)\n",
"self_sim_emb = np.load(\"/home/minz/temp/genius_self_sim_emb.npy\")\n",
"artist_sim_emb = np.load(\"/home/minz/temp/genius_artist_sim_emb.npy\")\n",
"album_sim_emb = np.load(\"/home/minz/temp/genius_album_sim_emb.npy\")\n",
"genre_sim_emb = np.load(\"/home/minz/temp/genius_genre_sim_emb.npy\")\n",
"self_vox_sim_emb = np.load(\"/home/minz/temp/genius_self_vox_sim_emb.npy\")\n",
"artist_vox_sim_emb = np.load(\"/home/minz/temp/genius_artist_vox_sim_emb.npy\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This demo is using 10k songs from GeniusHQ data. "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Similarity model"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"There are three different similarity models. I recommend artist-similarity model, but try them all!"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"seed\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 1\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 2\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 3\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 4\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 5\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# audio similarity (self similarity)\n",
"index = 1412 # <- Put any number lower than 10,000\n",
"similarity = self_sim_emb[index] @ self_sim_emb.T\n",
"sorted_indices = np.argsort(similarity)[::-1]\n",
"\n",
"print(\"seed\")\n",
"wav = Audio.from_file(genius_filelist[index])\n",
"wav.play()\n",
"for i in range(1, 6):\n",
" print(f\"similar {i}\")\n",
" wav = Audio.from_file(genius_filelist[sorted_indices[i]])\n",
" wav.play()"
]
},
{
"cell_type": "code",
"execution_count": 45,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"seed\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 1\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 2\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 3\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 4\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 5\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# audio similarity (artist similarity)\n",
"index = 5209 # <- Put any number lower than 10,000\n",
"similarity = artist_sim_emb[index] @ artist_sim_emb.T\n",
"sorted_indices = np.argsort(similarity)[::-1]\n",
"\n",
"print(\"seed\")\n",
"wav = Audio.from_file(genius_filelist[index])\n",
"wav.play()\n",
"for i in range(1, 6):\n",
" print(f\"similar {i}\")\n",
" wav = Audio.from_file(genius_filelist[sorted_indices[i]])\n",
" wav.play()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"seed\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 1\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 2\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 3\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 4\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 5\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# audio similarity (album similarity)\n",
"index = 1412 # <- Put any number lower than 10,000\n",
"similarity = album_sim_emb[index] @ album_sim_emb.T\n",
"sorted_indices = np.argsort(similarity)[::-1]\n",
"\n",
"print(\"seed\")\n",
"wav = Audio.from_file(genius_filelist[index])\n",
"wav.play()\n",
"for i in range(1, 6):\n",
" print(f\"similar {i}\")\n",
" wav = Audio.from_file(genius_filelist[sorted_indices[i]])\n",
" wav.play()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Vocal similarity"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"There are two vocal similarity models. I think this is not ready yet for product. It needs some more tuning."
]
},
{
"cell_type": "code",
"execution_count": 58,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"seed\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 1\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 2\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 3\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 4\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 5\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# vox similarity (self similarity)\n",
"index = 9323 # <- Put any number lower than 10,000\n",
"similarity = self_vox_sim_emb[index] @ self_vox_sim_emb.T\n",
"\n",
"sorted_indices = np.argsort(similarity)[::-1]\n",
"\n",
"print(\"seed\")\n",
"wav = Audio.from_file(genius_filelist[index])\n",
"wav.play()\n",
"for i in range(1, 6):\n",
" print(f\"similar {i}\")\n",
" wav = Audio.from_file(genius_filelist[sorted_indices[i]])\n",
" wav.play()"
]
},
{
"cell_type": "code",
"execution_count": 89,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"seed\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 1\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 2\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 3\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 4\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 5\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# vox similarity (artist similarity)\n",
"index = 7451 # <- Put any number lower than 10,000\n",
"similarity = artist_vox_sim_emb[index] @ artist_vox_sim_emb.T\n",
"\n",
"sorted_indices = np.argsort(similarity)[::-1]\n",
"\n",
"print(\"seed\")\n",
"wav = Audio.from_file(genius_filelist[index])\n",
"wav.play()\n",
"for i in range(1, 6):\n",
" print(f\"similar {i}\")\n",
" wav = Audio.from_file(genius_filelist[sorted_indices[i]])\n",
" wav.play()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Retrieval model"
]
},
{
"cell_type": "code",
"execution_count": 105,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"brazilian funk\n",
"similar 0\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 1\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"similar 2\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# text to audio\n",
"text = \"brazilian funk\" # <- Put any genre here\n",
"\n",
"text_emb = ditto.text_to_latent(text)[0].detach().cpu().numpy()\n",
"similarity = text_emb @ genre_sim_emb.T\n",
"sorted_indices = np.argsort(similarity)[::-1]\n",
"\n",
"print(text)\n",
"for i in range(3):\n",
" print(f\"similar {i}\")\n",
" wav = Audio.from_file(genius_filelist[sorted_indices[i]])\n",
" wav.play()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "suno_env",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.14"
}
},
"nbformat": 4,
"nbformat_minor": 2
}