{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "a0d83737",
"metadata": {},
"outputs": [],
"source": [
"# !nvidia-smi"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "fed7c348",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\""
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "88c3355d",
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"sys.path.insert(0, \"/home/georg/code/neon/diffusion-infer/\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "b59c8785",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"loading semantic model...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/georg/code/glockenspiel/suno_utils/suno_utils/tasks/mert_25.py:148: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
" sd = _torch_load_p(checkpoint_filepath)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"loading codec model...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/georg/code/glockenspiel/suno_utils/suno_utils/tasks/dac_vae_100hz_peaq.py:72: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
" sd = load_f(checkpoint_filepath)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"loading diffusion model...\n",
"using ema\n",
"loading tokenizer...\n",
"done!\n"
]
}
],
"source": [
"from suno_utils.audio import Audio\n",
"from generation import preload_models, generate, _retrieve_models\n",
"CKPT_DIR = \"/home/christian/code/neon/stable-audio-tools/harmonai_train/\"\n",
"_ = preload_models(\n",
" tokenizer_filepath=\"/home/georg/notebooks/gpu_nb/tmp/tokenizer_60k.json\",\n",
" semantic_model_filepath=\"/home/georg/notebooks/gpu_nb/tmp/mert_25.pt\",\n",
" semantic_clusters_filepath=\"/home/georg/notebooks/gpu_nb/tmp/mert_25_2x4k.npy\",\n",
" \n",
" ### 25hz ###\n",
"# codec_filepath=\"/home/georg/notebooks/gpu_nb/tmp/25hz_vae_peaq_kl_0.005.pth\",\n",
"\n",
" # 2min\n",
"# dit_model_filepath=os.path.join(CKPT_DIR, \"wz8fc0h9/checkpoints/last.ckpt\"),\n",
"# dit_config_filepath=\"/home/georg/notebooks/gpu_nb/tmp/1b_2m.json\",\n",
" \n",
" # 6min\n",
"# dit_model_filepath=os.path.join(CKPT_DIR, \"g7etrlgw/checkpoints/last.ckpt\"),\n",
"# dit_config_filepath=\"/home/georg/notebooks/gpu_nb/tmp/1b_6m.json\",\n",
" \n",
" ### 100hz ###\n",
" codec_filepath=\"/home/georg/notebooks/gpu_nb/tmp/100hz_vae_peaq_kl_0.005.pth\",\n",
"\n",
" # 30s\n",
"# dit_model_filepath=os.path.join(CKPT_DIR, \"v24pfvep/checkpoints/last.ckpt\"),\n",
"# dit_config_filepath=\"/home/georg/notebooks/gpu_nb/tmp/1b_30s.json\",\n",
" \n",
" # 30s (overlap)\n",
" dit_model_filepath=os.path.join(CKPT_DIR, \"vky8nwhb/checkpoints/last.ckpt\"),\n",
" dit_config_filepath=\"/home/georg/notebooks/gpu_nb/tmp/1b_30s_ctx.json\",\n",
" \n",
" # 30s (new)\n",
"# dit_model_filepath=os.path.join(CKPT_DIR, \"yhgs7ibp/checkpoints/last.ckpt\"),\n",
"# dit_config_filepath=\"/home/georg/notebooks/gpu_nb/tmp/1b_30s_2.json\",\n",
"\n",
" # 2mins\n",
"# dit_model_filepath=os.path.join(CKPT_DIR, \"3fhnp5mz/checkpoints/last.ckpt\"),\n",
"# dit_config_filepath=\"/home/georg/notebooks/gpu_nb/tmp/1b_2m_100hz_2.json\",\n",
"\n",
")\n",
"models = _retrieve_models()\n",
"if models[\"dit_model\"].ctx_len is not None:\n",
" model_duration_s = 6 * 60\n",
"else:\n",
" model_duration_s = models[\"dit_model\"].block_size // models[\"dit_model\"].io_hz\n",
"duration_s = 2*60 if model_duration_s >= 2*60 else 30"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "032b57fe",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"offset_s = 0\n",
"\n",
"audio_fp = \"/home/georg/notebooks/samples/walking_down.mp3\"\n",
"tags = \"Country\"\n",
"if duration_s <= 30.1:\n",
" lyrics = \"Walking down the street\\nFeeling so alive\\nGot my head in the clouds\\nGot a gleam in my eye\\nEvery step that i take\\nIs like a brand new start\\nNo matter where I'm going\\nI'll always find my part\\n\\nOooo\\n\\nLife is like a high\"\n",
"else:\n",
" lyrics = \"Walking down the street\\nFeeling so alive\\nGot my head in the clouds\\nGot a gleam in my eye\\nEvery step that i take\\nIs like a brand new start\\nNo matter where I'm going\\nI'll always find my part\\n\\nOooo\\n\\nLife is like a high wire act\\nwe're dancing in the sky\\nIt's a worry no need to ask why\\nWith a little bit or courage\\nWe can chase our dreams\\nNo matter what comes away\\nWe'll always be a team\\n\\nDon't stop oooo, yeah\"\n",
"\n",
"# audio_fp = \"/home/georg/notebooks/samples/norah.wav\"\n",
"# tags = \"jazz, pop, female vocal\"\n",
"# lyrics = \"[Verse 1]\\nI waited 'til I saw the sun\\nI don't know why I didn't come\\nI left you by the house of fun\\nI don't know why I didn't come\\nI don't know why I didn't come\\n\\n[Verse 2]\\nWhen I saw the break of day\\nI wished that I could fly away\\nInstead of kneeling in the sand\\nCatching teardrops in my hand\\n\\n[Refrain]\\nMy heart is drenched in wine\\nBut you'll be on my mind forever\\n\\n[Verse 3]\\nOut across the endless sea\\nI will die in ecstasy\\nBut I'll be a bag of bones\\nDriving down the road alone\\n\\n[Refrain]\\nMy heart is drenched in wine\\nBut you'll be on my mind\"\n",
"\n",
"# audio_fp = \"/home/georg/notebooks/samples/turkey_time.mp3\"\n",
"# tags = \"Country\"\n",
"# lyrics = \"[Verse]\\nGather 'round now, it's turkey time\\nCatchin' up with all your kin and dimes\\nFeastin' on that golden bird, so fine (so fine)\\nPass the gravy, pour some wine\\n\\n[Chorus]\\nTurkey time, it's a country rhyme\\nGrab your plate, fill it up, take your time\\nAnd when the sun starts settin', we'll gather 'round\\nSingin' songs, sippin' cider, dancin' all night sound (oooh)\\nTurkey time, it's a country rhyme\\n\\n[Verse]\\nWell, it's that time of year, with family near\\nGatherin' 'round the table, filled with cheer\\nTurkey's roasting, gravy's flowin'\\nMashed potatoes piled high, it's Thanksiving time\\n\\n[Chorus]\\nTurkey time, it's a country rhyme\\nGrab your plate, fill it up, take your time\\nAnd when the sun starts settin', we'll gather 'round\\nSingin' songs, sippin' cider, dancin' all night sound (oooh)\\nTurkey time, it's a country rhyme\\n\\n[Bridge] \\nTurkey time is here again\\nTurkey time is here again\\nTurkey time is here again\\nTurkey tiiiiime!\"\n",
"\n",
"\n",
"# audio_fp = \"/home/georg/notebooks/samples/mistletoe.mp3\"\n",
"# tags = \"Country\"\n",
"# lyrics = \"[intro]\\nooooh\\n(what is life, what is love?)\\n[Verse]\\nAnother year has come and gone, and I'm not where I long to be\\nSnowfall on the city streets, but it ain't Christmas to me\"\n",
"\n",
"# audio_fp = \"/home/georg/notebooks/samples/halo.wav\"\n",
"# tags = \"R&B, Ballad, Pop\"\n",
"# if duration_s <= 30.1:\n",
"# offset_s = 20\n",
"# lyrics = \"[Verse 1]\\nRemember those walls I built?\\nWell, baby, they're tumblin' down\\nAnd they didn't even put up a fight\\nThey didn't even make a sound\\nI found a way to let you in\\nBut I never really had a doubt\\nStandin' in the light of your halo\\nI got my angel now\\n\\n[Pre-Chorus]\\nIt's like\"\n",
"# else:\n",
"# lyrics = \"[Verse 1]\\nRemember those walls I built?\\nWell, baby, they're tumblin' down\\nAnd they didn't even put up a fight\\nThey didn't even make a sound\\nI found a way to let you in\\nBut I never really had a doubt\\nStandin' in the light of your halo\\nI got my angel now\\n\\n[Pre-Chorus]\\nIt's like I've been awakened\\nEvery rule, I had you breakin'\\nIt's the risk that I'm takin'\\nI ain't ever gonna shut you out\\n\\n[Chorus]\\nEverywhere I'm lookin' now\\nI'm surrounded by your embrace\\nBaby, I can see your halo\\nYou know you're my savin' grace\\nYou're everything I need and more\\nIt's written all over your face\\nBaby, I can feel your halo\\nPray it won't fade away\\n\\n[Post-Chorus]\\nI can feel your halo, halo, halo\\nI can see your halo, halo, halo\\nI can feel your halo, halo, halo\\nI can see your halo, halo, halo, ooh\\n\\n[Verse 2]\\nHit me like a ray of sun\\nBurnin' through my darkest night\\nYou're the only one that I want\\nThink I'm addicted to your light\\nI swore I'd never fall again\\nBut this don't even feel like fallin'\\nGravity can't begin\\nTo pull me back to\"\n",
"\n",
"audio = Audio.from_file(audio_fp, n_channels=2).get_segment(from_s=offset_s, to_s=offset_s+duration_s+0.01)\n",
"# TODO: pad with silence for integer multiple of sliding\n",
"if models[\"dit_model\"].ctx_len is not None:\n",
" window_s = int(round((models[\"dit_model\"].block_size - models[\"dit_model\"].ctx_len) / models[\"dit_model\"].io_hz))\n",
" if audio.duration_s % window_s > 0.1:\n",
" audio = Audio.concatenate([\n",
" audio, \n",
" Audio.from_silence(window_s - audio.duration_s % window_s + 0.01, sample_rate=audio.sample_rate, n_channels=audio.n_channels)\n",
" ])\n",
"audio.normalize_volume().play()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "f2f7e714",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|████████████████████████████████████████████████████████████████████████| 3/3 [00:59<00:00, 19.80s/it]\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|████████████████████████████████████████████████████████████████████████| 3/3 [00:59<00:00, 19.98s/it]\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|████████████████████████████████████████████████████████████████████████| 3/3 [00:59<00:00, 19.99s/it]\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"steps = 250\n",
"cfg_coef = 4\n",
"\n",
"for n in range(3):\n",
" pred_audio = generate(\n",
" audio, \n",
" lyrics=lyrics, \n",
" tags=tags, \n",
" cfg_coef=cfg_coef, \n",
" steps=steps,\n",
" seed=n,\n",
" )\n",
" pred_audio.normalize_volume().play()\n",
"# break"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "d0d30bc8",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|████████████████████████████████████████████████████████████████████████| 6/6 [02:00<00:00, 20.16s/it]\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|████████████████████████████████████████████████████████████████████████| 6/6 [02:00<00:00, 20.01s/it]\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|████████████████████████████████████████████████████████████████████████| 6/6 [02:00<00:00, 20.04s/it]\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "aff184ff",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "11ef466e",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "ec8ba45d",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "bb494e8a",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9c946bc7",
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"models = _retrieve_models()\n",
"extra_args = torch.load(\"extra_args.pt\", weights_only=True)\n",
"torch.manual_seed(0)\n",
"with torch.no_grad():\n",
" x = torch.randn([1, 128, 100*30], device=\"cuda\") * 50\n",
" out = models[\"dit_model\"].forward(\n",
" x,\n",
" x.new_ones([x.shape[0]])*50,\n",
" cross_attn_cond=extra_args[\"cross_attn_cond\"],\n",
" empty_cross_attn_cond=extra_args[\"empty_cross_attn_cond\"],\n",
" cfg_scale=extra_args[\"cfg_scale\"]\n",
" )\n",
"out.shape"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "818cf6a9",
"metadata": {},
"outputs": [],
"source": [
"# PAG\n",
"# https://arxiv.org/pdf/2403.17377\n",
"# https://huggingface.co/hyoungwoncho/sd_perturbed_attention_guidance/blob/main/pipeline.py\n",
"# https://github.com/KU-CVLAB/Perturbed-Attention-Guidance?tab=readme-ov-file"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "480d3a32",
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"s = {\n",
" \"io_hz\": 100,\n",
" \"io_channels\": 128,\n",
" \"embed_dim\": 1536,\n",
" \"depth\": 24,\n",
" \"n_heads\": 24,\n",
" \"qk_norm\": True,\n",
" \"block_size\": 3000,\n",
" \"cond_type\": \"adaLN\",\n",
" \"cond_semantic_n_vocab\": 4001,\n",
" \"cond_semantic_len\": 750,\n",
" \"cond_text_n_vocab\": 60005,\n",
" \"cond_text_len\": 2560,\n",
"}\n",
"with open(\"/home/georg/notebooks/gpu_nb/tmp/1b_30s.json\", \"w\") as f:\n",
" json.dump(s, f)\n",
"s = {\n",
" \"io_hz\": 25,\n",
" \"io_channels\": 128,\n",
" \"embed_dim\": 1536,\n",
" \"depth\": 24,\n",
" \"n_heads\": 24,\n",
" \"qk_norm\": True,\n",
" \"block_size\": 3000,\n",
" \"cond_type\": \"adaLN\",\n",
" \"cond_semantic_n_vocab\": 4001,\n",
" \"cond_semantic_len\": 3000,\n",
" \"cond_text_n_vocab\": 60001,\n",
" \"cond_text_len\": 2560,\n",
"}\n",
"with open(\"/home/georg/notebooks/gpu_nb/tmp/1b_2m.json\", \"w\") as f:\n",
" json.dump(s, f)\n",
"s = {\n",
" \"io_hz\": 25,\n",
" \"io_channels\": 128,\n",
" \"embed_dim\": 1536,\n",
" \"depth\": 24,\n",
" \"n_heads\": 24,\n",
" \"qk_norm\": True,\n",
" \"block_size\": 3000,\n",
" \"cond_type\": \"adaLN\",\n",
" \"cond_semantic_n_vocab\": 4001,\n",
" \"cond_semantic_len\": 3000,\n",
" \"cond_text_n_vocab\": 60001,\n",
" \"cond_text_len\": 2560,\n",
" \"cond_phoneme_n_vocab\": 84,\n",
" \"cond_phoneme_len\": 5120,\n",
"}\n",
"with open(\"/home/georg/notebooks/gpu_nb/tmp/1b_2m_phoneme.json\", \"w\") as f:\n",
" json.dump(s, f)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "18f4dc97",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "abe70d19",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "cf43452f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "56142e0c",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "acda5fe8",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.14"
}
},
"nbformat": 4,
"nbformat_minor": 5
}