{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "a0d83737", "metadata": {}, "outputs": [], "source": [ "# !nvidia-smi\n", "# !echo $HOSTNAME" ] }, { "cell_type": "code", "execution_count": 2, "id": "fed7c348", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"6\"\n", "import torch\n", "import numpy as np\n", "from suno_utils.audio import Audio\n" ] }, { "cell_type": "code", "execution_count": null, "id": "74d60c8a", "metadata": {}, "outputs": [], "source": [ "import torch\n", "from suno_utils.gpt.generation import GenerationConfig\n", "from suno_utils.gpt.engine import Engine\n", "from suno_utils.gpt.generation_engine import make_request\n", "\n", "N_BATCH = 2\n", "\n", "engine = Engine(\n", "# \"/app/suno/checkpoints/2025-01-26_01-35-19/last_ckpt_infer.pt\", # skip\n", " #\"/app/suno/checkpoints/2025-01-27_22-26-09/last_ckpt_infer.pt\", # base long\n", "# \"/app/suno/checkpoints/2025-01-18_18-03-13/last_ckpt_infer.pt\", # base\n", "# \"/app/suno/checkpoints/2024-12-05_22-01-59/last_ckpt_infer.pt\", # (old chef)\n", " #\"/app/suno/checkpoints/2025-01-28_12-59-39/last_ckpt_infer.pt\", # base long\n", " #\"/app/suno/checkpoints/2025-01-31_02-57-52/last_c#kpt_infer.pt\", # first attempt 50k finetune\n", " #\"/app/suno/checkpoints/2025-02-02_19-46-10/step_80000_infer.pt\", # second attempt 80k finetune probably overfit\n", " #\"/app/suno/checkpoints/2025-02-04_01-33-55/step_83000_infer.pt\", # third attempt 83k finetune\n", " #\"/app/suno/checkpoints/2025-01-31_03-14-13/last_ckpt_infer.pt\", # base model from gk\n", " #\"/app/suno/checkpoints/2025-02-02_21-04-52/last_ckpt_infer.pt\", # 80% converged base model from gk\n", " #\"/app/suno/checkpoints/2025-02-04_21-04-31/80k_ckpt_infer.pt\", # 6B 80% converged base model from gk\n", " #\"/app/suno/checkpoints/2025-02-06_11-22-04/step_2000_infer.pt\", #6b finetune\n", " #\"/app/suno/checkpoints/2025-02-10_16-52-41/step_19000_infer.pt\", #6b finetune v2\n", " #\"/app/suno/checkpoints/2025-02-12_11-24-30/last_ckpt_infer.pt\", #6b finetune v3\n", " #\"/app/suno/checkpoints/2025-02-13_00-23-25/last_ckpt_infer.pt\", #6b finetune v4\n", " #\"/app/suno/checkpoints/2025-02-13_11-27-29/last_ckpt_infer.pt\", #6b finetune v5\n", " #\"/app/suno/checkpoints/2025-02-24_01-06-45/last_ckpt_infer.pt\", #6b finetune v6\n", " #\"/app/suno/checkpoints/2025-04-12_03-10-00/last_ckpt_infer.pt\", #6b finetune v8 (infill covers)\n", " #\"/app/suno/checkpoints/2025-04-15_19-56-08/last_ckpt_infer.pt\", #6b finetune v11 \n", " #\"/app/suno/checkpoints/2025-04-17_02-00-09/last_ckpt_infer.pt\", #6b finetune v12 \n", " #\"/app/suno/checkpoints/2025-04-18_18-52-13/last_ckpt_infer.pt\", #6b finetune v14\n", " #\"/app/suno/checkpoints/2025-05-01_15-20-49/last_ckpt_infer.pt\", #6b finetune new tags\n", " #\"/app/suno/checkpoints/2025-05-07_17-43-07/last_ckpt_infer.pt\", #6b ft test no persona/cover filter\n", " #\"/app/suno/checkpoints/2025-05-08_19-44-00/last_ckpt_infer.pt\", #6b ft test with persona/cover filter\n", " \"/app/suno/checkpoints/2025-05-12_13-17-50/last_ckpt_infer.pt\", #6b ft test with persona/cover filter\n", " \"s3://suno-data/georg/models/tokenizers/tokenizer_60k.json\",\n", " max_sequences=4*N_BATCH,\n", " compile=False,\n", ")\n", "cfg = engine.model.config\n", "\n", "ckpt_name = \"2025-02-06_11-22-04\"" ] }, { "cell_type": "code", "execution_count": null, "id": "a6a9f401", "metadata": {}, "outputs": [], "source": [ "from suno_utils.diffusion.generation import (\n", " preload_dit_model,\n", " preload_tokenizer,\n", " TOKENIZER_FILEPATH,\n", " SEMANTIC_MODEL_FILEPATH,\n", " SEMANTIC_CLUSTERS_FILEPATH,\n", " _retrieve_models,\n", ")\n", "\n", "from suno_utils.tasks.mert_25 import (\n", " preload_models as preload_semantic_models,\n", " encode as encode_semantic,\n", ")\n", "\n", "from suno_utils.tasks.dac_vae_fixed_25hz import (\n", " preload_models as preload_codec_models,\n", " decode as codec_decode,\n", " decode_stream_to_full_audio,\n", ")\n", "\n", "from suno_utils.diffusion import generation as diffusion_gen\n", "#from suno_utils.tasks.upsample_engine import UpsampleEngine, Request, Job\n", "from suno_utils.tasks.upsample_engine_old import UpsampleEngine, Request, Job\n", "\n", " # load diffusion model\n", "num_gpus = torch.cuda.device_count()\n", "cuda_device = torch.cuda.current_device()\n", "print(f\"Found {num_gpus} GPUs. Using GPU {cuda_device}.\")\n", "\n", "# dit models\n", "#dit_model_filepath = \"/app/suno/checkpoints/2025-04-12_05-22-06_s1150/step_6000_ckpt.pt\"\n", "#dit_model_filepath = \"/app/suno/checkpoints/2025-02-17_16-54-01_s7787/last_ckpt_infer.pt\" # base model\n", "#dit_model_filepath = \"/app/suno/checkpoints/2025-04-11_14-14-12_s4538/last_ckpt_infer.pt\" # audio tag finetune\n", "#dit_model_filepath = \"/app/suno/checkpoints/2025-04-10_01-05-22_s3784/last_ckpt_infer.pt\" # shared context\n", "dit_model_filepath = \"/app/suno/modal/models/tony/tmp/diff/v45_2b_step_2mil_ft_8k_infill_apr21_t1_18_cs0.pt\"\n", "\n", "# marc ckpts testing\n", "#dit_model_filepath = \"/app/suno/data/dpo/models/diff_v2_2b_2mil_ft_v0.pt\"\n", "#dit_model_filepath = \"/app/suno/checkpoints/2025-04-10_04-13-39_s12/step_9000_infer.pt\"\n", "#dit_model_filepath = \"/app/suno/checkpoints/2025-04-16_19-50-48_s4352/last_ckpt_infer.pt\"\n", "\n", "# other models\n", "tokenizer_filepath = \"s3://suno-data/georg/models/tokenizers/tokenizer_60k.json\"\n", "semantic_model_filepath = \"s3://suno-data/georg/models/semantic/mert_25.pt\"\n", "semantic_clusters_filepath = (\n", " \"s3://suno-data/georg/models/semantic/mert_25_2x4k.npy\"\n", ")\n", "codec_filepath = \"s3://suno-data/minz/models/dac_vae_tuned_25hz.pth\"\n", "CODEC_SCALE_FACTOR = 0.4\n", "SCALE_CTX_VECTOR = True\n", "\n", "_ = diffusion_gen.preload_dit_model(\n", " dit_model_filepath=dit_model_filepath,\n", " use_ema_if_exists=True,\n", " compile=True,\n", " weights_precision=torch.bfloat16,\n", ")\n", "_ = preload_tokenizer(tokenizer_filepath)\n", "_ = preload_semantic_models(semantic_model_filepath, semantic_clusters_filepath)\n", "_ = preload_codec_models(codec_filepath)\n", "\n", "diffusion_engine = UpsampleEngine(min_chunk_size=25 * 30, vae_version=\"v_vae_25_tuned_2\")" ] }, { "cell_type": "code", "execution_count": 19, "id": "5ec28b37", "metadata": {}, "outputs": [], "source": [ "# # [Verse 1: female vocals, soft, beautiful]\n", "# # [Pre-Chorus: male, aggressive, shouting]\n", "# # [Chorus: female & male, together, harmonies]\n", "\n", "# text = \"\"\"\n", "# [Verse 1]\n", "# You are my fire\n", "# The one desire\n", "# Believe when I say\n", "# I want it that way\n", "\n", "# [Verse 2]\n", "# But we are two worlds apart\n", "# Can't reach to your heart\n", "# When you say\n", "# That I want it that way\n", "\n", "# [Chorus]\n", "# Tell me why\n", "# Ain't nothin' but a heartache\n", "# Tell me why\n", "# Ain't nothing but a mistake\n", "# Tell me why\n", "# I never wanna hear you say\n", "# I want it that way\n", "\n", "# [Verse 3]\n", "# Am I your fire?\n", "# Your one desire\n", "# Yes, I know it's too late\n", "# But I want it that way\n", "\n", "# [Chorus]\n", "# Tell me why\n", "# Ain't nothin' but a heartache\n", "# Tell me why\n", "# Ain't nothin' but a mistake\n", "# Tell me why\n", "# I never wanna hear you say\n", "# I want it that way\n", "\n", "# [Bridge]\n", "# Now I can see that we've fallen apart\n", "# From the way that it used to be, yeah\n", "# No matter the distance, I want you to know\n", "# That deep down inside of me\n", "\n", "# [Verse 4]\n", "# You are my fire\n", "# The one desire\n", "# You are (You are, you are, you are)\n", "# Don't wanna hear you say\n", "\n", "# [Break]\n", "# Ain't nothin' but a heartache (Hey, yeah)\n", "# Ain't nothin' but a mistake (Don't wanna hear you say)\n", "# I never wanna hear you say (Oh, yeah)\n", "# I want it that way\n", "# [Chorus: Brian, All, Nick, AJ]\n", "# Tell me why\n", "# Ain't nothin' but a heartache\n", "# Tell me why\n", "# Ain't nothin' but a mistake\n", "# Tell me why\n", "# I never wanna hear you say (Don't wanna hear you say)\n", "# I want it that way\n", "# Tell me why\n", "# Ain't nothin' but a heartache\n", "# Ain't nothin' but a mistake\n", "# Tell me why\n", "# I never wanna hear you say (Never wanna hear you say, yeah)\n", "# I want it that way\n", "\n", "# [Outro]\n", "# 'Cause I want it that way\n", "# \"\"\"\n", "\n", "text = \"\"\"\n", "[Verse 1: female vocals]\n", "You are somebody that I don't know\n", "But you're takin' shots at me like it's Patrón\n", "And I'm just like, damn, it's 7 AM\n", "Say it in the street, that's a knock-out\n", "But you say it in a Tweet, that's a cop-out\n", "And I'm just like, \"Hey, are you okay?\"\n", "\n", "[Pre-Chorus: male vocals]\n", "And I ain't tryna mess with your self-expression\n", "But I've learned a lesson that stressin' and obsessin' 'bout somebody else is no fun\n", "And snakes and stones never broke my bones\n", "\n", "[Chorus: female vocals, male vocals]\n", "So oh-oh, oh-oh, oh-oh, oh-oh, oh-oh\n", "You need to calm down, you're being too loud\n", "And I'm just like oh-oh, oh-oh, oh-oh, oh-oh, oh-oh (Oh)\n", "You need to just stop, like can you just not step on my gown?\n", "You need to calm down\n", "\n", "[Verse 2: female vocals]\n", "You are somebody that we don't know\n", "But you're comin' at my friends like a missile\n", "Why are you mad when you could be GLAAD? (You could be GLAAD)\n", "Sunshine on the street at the parade\n", "But you would rather be in the dark ages\n", "Makin' that sign must've taken all night\n", "\n", "[Pre-Chorus: male vocals]\n", "You just need to take several seats and then try to restore the peace\n", "And control your urges to scream about all the people you hate\n", "'Cause shade never made anybody less gay\n", "\n", "[Chorus: female vocals, male vocals]\n", "So oh-oh, oh-oh, oh-oh, oh-oh, oh-oh\n", "You need to calm down, you're being too loud\n", "And I'm just like oh-oh, oh-oh, oh-oh, oh-oh, oh-oh (Oh)\n", "You need to just stop, like can you just not step on his gown?\n", "You need to calm down\n", "\n", "[Bridge: male vocals]\n", "And we see you over there on the internet\n", "Comparing all the girls who are killing it\n", "But we figured you out\n", "We all know now we all got crowns\n", "You need to calm down\n", "\n", "[Chorus: female vocals, male vocals]\n", "Oh-oh, oh-oh, oh-oh, oh-oh, oh-oh\n", "You need to calm down (You need to calm down)\n", "You're being too loud (You're being too loud)\n", "And I'm just like oh-oh, oh-oh, oh-oh, oh-oh, oh-oh (Oh)\n", "You need to just stop (Can you stop?)\n", "Like can you just not step on our gowns?\n", "You need to calm down\n", "\"\"\"\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "text = \"\"\"\n", "[verse]\n", "Embrace the random\n", "Avoid the void\n", "Stuck in the sky\n", "Like a smiley asteroid\n", "See you on the other side\n", "Clouds knocking on my door\n", "Dream to sleep tonight\n", "What am I flying for?\n", "(four, four, four, four)\n", "\n", "[chorus]\n", "Hey\n", "I'm on holiday\n", "From the ground\n", "Up in the clouds\n", "Hey\n", "I'm on holiday\n", "From the ground\n", "Up in the clouds\n", "\n", "[instrumental break]\n", "\n", "[bridge]\n", "Stars in my wet eyes\n", "(Droplets of condensation)\n", "Taste the water vapor\n", "(Just like a conversation)\n", "\n", "[chorus]\n", "Hey\n", "I'm on holiday\n", "From the ground\n", "Up in the clouds\n", "Hey\n", "I'm on holiday\n", "From the ground\n", "Up in the clouds\n", "\n", "[bridge]\n", "Stars in my wet eyes\n", "(Droplets of condensation)\n", "Taste the water vapor\n", "(Just like a conversation)\n", "\n", "[chorus]\n", "Hey\n", "I'm on holiday\n", "From the ground\n", "Up in the clouds\n", "Hey\n", "I'm on holiday\n", "From the ground\n", "Up in the clouds\n", "\"\"\"\n", "\n", "\n", "text = \"\"\"\n", "[Verse 1]\n", "Sexy to someone is all I really want\n", "Sometimes sexy to someone is all I really want\n", "Nothin' more, nothin' less, I've thought, walkin' through the park\n", "Sexy to someone, I think about it all\n", "Checkin' out of the hotel or moments at a bar\n", "Ask if I'm in a movie, no, I didn't get the part\n", "\n", "[Chorus]\n", "Sexy to somebody, it would help me out\n", "Oh, I need a reason to get out of the house\n", "And it's just a little thing I can't live without\n", "\n", "[Verse 2]\n", "Sexy is something I see in everything\n", "Honey stickin' to your hands, sugar on the rim\n", "Nothing more, nothing less, I've thought, take it all to heart\n", "I want afterglowing, and when I call a car\n", "Send me eyes with the knowing that I could pull it off\n", "Ask if I'm doing TV, no, I didn't get the job\n", "\n", "[Chorus]\n", "Sexy to somebody, it would help me out\n", "Oh, I need a reason to get out of the house\n", "And it's just a little thing I can't live without\n", "\n", "[Bridge]\n", "I want to be sexy to someone (Is it too much to ask?)\n", "I want to be sexy to someone (Then what's holding you back?)\n", "I want to be sexy to someone (It's not too much to ask)\n", "Sexy to someone\n", "\n", "[Chorus]\n", "Sexy to somebody, it would help me out\n", "Oh, I need a reason to get out of the house\n", "And it's just a little thing I can't live without\n", "Sexy to somebody, it would help me out\n", "Oh, I need a reason to get out of the house\n", "And it's just a little thing I can't live without\n", "\"\"\"\n", "\n", "\n", "\n", "\n", "\n", "\n", "text = \"\"\"\n", "Almost Heaven, West Virginia\n", "Blue Ridge Mountains, Shenandoah River\n", "Life is old there, older than the trees\n", "Younger than the mountains, growin' like a breeze\n", "Country roads, take me home\n", "To the place I belong\n", "West Virginia, mountain mama\n", "Take me home, country roads\n", "All my memories gather 'round her\n", "Miner's lady, stranger to blue water\n", "Dark and dusty, painted on the sky\n", "Misty taste of moonshine, teardrop in my eye\n", "Country roads, take me home\n", "To the place I belong\n", "West Virginia, mountain mama\n", "Take me home, country roads\n", "I hear her voice in the mornin' hour, she calls me\n", "The radio reminds me of my home far away\n", "Drivin' down the road, I get a feelin'\n", "That I should've been home yesterday, yesterday\n", "Country roads, take me home\n", "To the place I belong\n", "West Virginia, mountain mama\n", "Take me home, country roads\n", "Country roads, take me home\n", "To the place I belong\n", "West Virginia, mountain mama\n", "Take me home, country roads\n", "Take me home, (down) country roads\n", "Take me home, (down) country roads\n", "\"\"\"\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "text = \"\"\"\n", "君の長い髪を揺らす 風は遥か南の島へ\n", "歌うように 踊るように 優しさをなびかせていたね\n", "Hey hey hey, breeze on the seaside\n", "忘れかけてた あの日の君は 7月色の風に乗って\n", "夏の匂い 君への思い 切ないほどに漂わせて\n", "Hey hey hey, breeze on the seaside\n", "僕にもう一度 与えてほしい\n", "燃え上がる夏の 暑い日差しを\n", "Summer breeze 君と summer breeze 僕の\n", "出会いの夏は暑い浜辺\n", "Radio が唄う 恋の melody\n", "口ずさむ君 まるで shining sun\n", "Hey hey hey, breeze on the seaside\n", "月の光 浮かべた海と 君しかいない 夜の浜辺\n", "冷たい砂に 隠した心 握りしめれば 今もつらい\n", "Hey hey hey, breeze on the seaside\n", "Hey hey hey, breeze on the seaside\n", "君にもう一度 伝えてみたい\n", "行き過ぎる夏の 寂しさだけを\n", "僕にもう一度 与えてほしい\n", "燃え上がる夏の 暑い日差しを\n", "Hey hey hey\n", "Hey hey hey\n", "\n", "\"\"\"\n", "\n", "\n", "\n", "text = \"\"\"\n", "Permission is hereby granted, \n", "free of charge, \n", "to any person obtaining a copy of this software \n", "and associated documentation files \n", "(the “Software”), \n", "to deal in the Software without restriction, \n", "including without limitation the rights to use, \n", "copy, modify, merge, publish, distribute, sublicense, \n", "and/or sell copies of the Software, \n", "and to permit persons to whom the Software \n", "is furnished to do so, subject to the following conditions:\n", "The above copyright notice and this permission notice shall \n", "be included in all copies or substantial portions of the Software.\n", "\"\"\"\n", "\n", "text = \"\"\"\n", "CAT CAT CAT CAT CAT CAT CAT CAT \n", "CAT CAT CAT CAT CAT CAT CAT CAT CAT \n", "CAT CAT CAT CAT CAT CAT CAT CAT CAT \n", "CAT CAT CAT CAT CAT CAT CAT CAT \n", "\n", "CAT CAT CAT CAT CAT CAT CAT CAT \n", "CAT CAT CAT CAT CAT CAT CAT CAT CAT \n", "CAT CAT CAT CAT CAT CAT CAT CAT CAT \n", "CAT CAT CAT CAT CAT CAT CAT CAT \n", "\"\"\"\n", "\n", "\n", "\n", "\n", "text = \"\"\"\n", "[Verse 1]\n", "Oh\n", "What kind of bird am I?\n", "I'm not the kind that flies\n", "Just walks around and chirps\n", "I'm an auk!\n", "\n", " \n", "[Verse 2]\n", "Oh\n", "What kind of bird am I?\n", "I'm not the type that flies\n", "Just walks around and chirps\n", "I chirp all night and day\n", "I'm chirping anyway\n", "I'm chirpin' when I breathe\n", "Chirpin' underneath\n", "I'm an auk!\n", "\n", "[Verse 1]\n", "Oh\n", "What kind of bird am I?\n", "I'm not the kind that flies\n", "Just walks around and chirps\n", "I'm an auk!\n", "\n", "I'm an auk!\n", "\n", "I'm an auk!\n", "\"\"\"\n", "\n", "text = \"\"\"\n", "Take all the time you need\n", "Take all the time you need\n", "Welcome your enemies\n", "\n", "Take all the time you need\n", "You’re buried underneath\n", "All the rivers, rocks and trees\n", "I’m water when you need\n", "sunshine, sunshine\n", "\n", "Hey, you’re steel\n", "Cause nothing real is touched\n", "You are enough\n", "Today\n", "You are enough\n", "\n", "Take all the time you need\n", "To color and believe\n", "\n", "I’m drifting off to sleep\n", "Some Uber shifts this week\n", "But I’m here and listening\n", "From moons away\n", "\n", "Hey, you’re steel\n", "Cuz nothing real is touched\n", "You are enough\n", "Today\n", "You are enough\n", "\n", "Without the sun’s exposure\n", "My insides turn to steel\n", "But out the door is sunlight\n", "Sunlight, oh so real\n", "But out the door is sunlight\n", "Oh sunlight, is real\n", "\n", "Oh, sunshine\n", "Oh, sunshine\n", "\"\"\"\n", "\n", "text = \"\"\"\n", "So right, so clean\n", "But does it mean anything, no no\n", "So right, so clean\n", "Does it mean anything, no no\n", "It sounds like something that I saw in a movie\n", "It looks like something that I've seen on TV\n", "Oh perfect thing\n", "I wish I could believe in your dream\n", "I see signs of life in alleys and corners\n", "I smell death on a crowded street\n", "I feel the last old building burnin'\n", "I've got nowhere left to put my feet\n", "'Cause it's so right, so clean\n", "Does it mean anything, no no\n", "And it's so right, so clean\n", "It doesn't mean anything, no no\n", "'Cause you never mind the ones who got left behind\n", "You never mind the ones who got left behind\n", "You never mind the ones who got left behind\n", "You never mind the ones who got left behind\n", "'Cause it's so right, so clean\n", "It doesn't mean anything, no no\n", "It's so right, so clean\n", "But does it mean anything, no no\n", "I wish I could believe in your dream\n", "I wish I could believe in your dream\n", "I wish I could believe in your dream\n", "I wish I could believe in your dream\n", "I wish I could believe in your dream (so right, so clean)\n", "I wish I could believe in your dream (so right, so clean)\n", "I wish I could believe in your dream (so right, so clean)\n", "I wish I could believe in your dream (so right, so clean)\n", "\"\"\"\n", "\n", "text = \"\"\"\n", "[Verse 1]\n", "You make disaster while you sail in black waters\n", "Go off and save the mess you made of black waters\n", "It wasn't safe to be honest\n", "Grow and shave all the offset\n", "\n", "[Chorus]\n", "And you warn to save time\n", "Before you wait, why?\n", "But you nod to save time\n", "And belong to safe tries\n", "\n", "[Verse 2]\n", "You waste your mind to settle down black waters\n", "You face the side of better times in black waters\n", "Weather changes your mind\n", "Bet you'd call me inside\n", "\n", "[Chorus]\n", "And you warn to save time\n", "Before you wait, why?\n", "But you nod to save time\n", "Belong to safe tries\n", "\"\"\"\n", "\n", "# tags = \"grunge alternative rock\"\n", "# tags = \"guitar violin drums\"\n", "#tags = \"A melancholic country song featuring prominent pedal steel guitar.\"\n", "\n", "#tags = \"Indie Rock, Post-Hardcore, Emo, Slacker Rock, Power Pop, Noise Pop\"\n", "#tags = \"yatch rock, live recording, crowd, live performance\"\n", "#tags = \"Pop, high energy, riser, drop, catchy, edm, 2020s\"\n", "#tags = \"Electronic, Alternative Pop, French pop, Dreamy, Trap, grammy winner, billboard hot 100, chart topper, 2020\"\n", "#tags = \"trap\"\n", "#tags = \"Pop, female vocalist, 2020s\"\n", "#tags = \"Spanish Classical Music, Baroque Music, Modern Classical, Concerto, Western Classical Music, Latin American Classical Music\"\n", "#tags = \"2020, grammy winner, Singer-Songwriter, Pop Rock, Rock, Contemporary R&B, groovy, guitar riff, chatchy\"\n", "#tags = \"Alternative Rock, Power Pop, Emo-Pop, 2007, heavy guitars, studio\"\n", "#tags = \"Midwest Emo, horn, 1999, Chicago School, Math Rock, Post-Rock, Slowcore, Math Pop, Emo-pop\"\n", "#tags = \"Midwest Emo, Pop Punk, 2012, Emo-Pop, Alternative Rock, 2010s\"\n", "#tags = \"2014, Pop Punk, Emo-Pop, Midwest Emo, Emo, Emo Revival, introspective, melancholic, energetic\"\n", "#tags = \"Gregorian Chant, Hymns, Christian, ethereal, female vocalist, medieval, atmospheric, spiritual, melodic, choral, peaceful, hypnotic, hymn, meditative, minimalistic, soothing, religious, soft, longing, dense, melancholic, mysterious, repetitive, acoustic\"\n", "# tags = \"Two-track instrumental featuring ambient nature sounds and rhythmic clanging, followed by sinister synthesizer music and buzzing electric circuit.\"\n", "#tags = \"Country, southern accent, deep south, raspy vocals\"\n", "#tags = \"Gospel, church, choir, holy, cathedral, reverb, r&b\"\n", "#tags = \"Electronic, Alternative Pop, French pop, french accent, Dreamy, Trap, grammy winning, billboard hot 100, chart topper, 2020\"\n", "#tags = \"Contemporary R&B, Pop Rap, Alternative R&B, Trap, nocturnal, female vocalist, aquatic, alcohol, male vocalist, sexual\"\n", "#tags = \"Pop Soul, Neo-Soul, Jazz Pop, Brill Building, Twist, drugs, female vocalist, rebellious, alcohol, rhythmic, depressive, introspective, passionate, sarcastic, melodic, hedonistic, anthemic\"\n", "#tags = \"power ballad, female vocals, male vocals, pop, rap, hip-hop, emotive, powerful, haunting\"\n", "#tags = \"Melodic Techno, Future Rave, Melodic House, rhythmic, atmospheric\"\n", "#tags = \"Dance-Pop, Electro House, Festival Progressive House, Electropop, Dutch House, passionate\"\n", "# tags = \"passionate, raspy vocals, powerful delivery, female, pop\"\n", "# tags = \"Pop, female vocals, orchestra\"\n", "#tags = \"Pop, female vocals, r&b, grammy winner, 2020s, billboard hot 100 2020\"\n", "#tags = \"Pop Rap, Trap, Emo Rap, Alt-Pop, Emo-Pop, Beat, 808, Heavy\"\n", "#tags = \"drums\"\n", "#tags = \"Hypnagogic Pop, Neo-Psychedelia, Alternative R&B, Bedroom Pop, Sophisti-Pop, Indietronica\"\n", "#tags = \"Soft Rock, Sophisti-Pop, Pop Soul, Chamber Pop, Psychedelic Pop, Bedroom Pop, mellow, sexual, sensual, soothing, female vocalist,\"\n", "# tags = \"female vocals; This track start off quiet and slow featuring string instruments including a violin. It then builds into an energetic chorus with drums and trumpets.\"\n", "#tags = \"Jazz Pop, Vocal Jazz, Lounge, Folk Pop, Piano Blues, Smooth Jazz, grammy winner\"\n", "#tags = \"Noise Pop, Slacker Rock, Shoegaze, Indie Rock, Heavy Guitar\"\n", "#tags = \"Jazz Fusion, Hard Bop, Cool Jazz, Modal Jazz, Post-Bop, Jazz-Funk\"\n", "#tags = \"City Pop, Jazz Fusion, 1980, Synth Funk, summer, happy, warm, ethereal, tropical, melodic, male vocalist, calm, uplifting, nocturnal, natural\"\n", "#tags = \"1987, Microtonal Classical, Minimalism, microtonal, minimalistic, hypnotic, meditative, instrumental, avant-garde, improvisation, complex, spiritual\"\n", "#tags = \"Math Pop, electric guitar solo, Flamenco nuevo, technical, complex, futuristic, uncommon time signatures\"\n", "#tags = \"Smooth Jazz, Jazz Fusion, Jazz-Rock, Jazz-Funk, 1980\"\n", "#tags = \"edm\"\n", "#tags = \"psychedelic, psychedelic rock, psychedelic pop\"\n", "#tags = \"lofi hiphop chill piano film studio ghibli, japanese\"\n", "#tags = \"Emotive male vocals with a slightly raspy tone, melodic electric guitar riffs layered with strummed acoustic guitar, steady mid-tempo drum beat, warm bassline anchoring the rhythm, touches of keyboard/synth providing atmospheric depth, dynamic build during the chorus with increased instrumentation, subtle use of reverb, overall polished pop rock production typical of early 2000s Indonesian music, lyrics expressing a sense of irony and heartbreak.\"\n", "\n", "#tags = \"microtonal, instrumental, avant-garde, complex\"\n", "#tags = \"Trap, Rap, Emo Rap, Rage, Cloud Rap, Rage, Synth, Glitch, Beat, 808\"\n", "#tags = \"Krushclub, Glitch, Cloud Rap, Southern Hip Hop, Jersey Club Rap, Jersey Club, Rage\"\n", "#tags = \"Stoner Rock, Stoner Metal, Palm Desert Scene, Heavy Psych, Psychedelic Rock\"\n", "#tags = \"Pop, very compressed, very bright\"\n", "#tags = \"Rock, rolled-off, dull, very dynamic, warm, very warm\"\n", "#tags = \"Rock\"\n", "#text = \"[Instrumental]\"\n", "#tags = \"Piano, Classical Period, Sonata, sonata\"\n", "\n", "#tags = \"Emo Pop, Alt Pop, Dark Bubblegum Pop, Yami Kawaii, J-Pop, Darkwave, Female Vocals, Haunting Synths, passionte female vocals, melancholic\"\n", "#tags = \"j-rock, electric guitar, electro, synthwave, female vocals, glitch core, glitch\"\n", "# text = \"\"\n", "# tags = \"orchestra, symphony\"\n", "# tags = \"orchestra, powerful trailer music, epic, dark, violin, female vocalizations\"\n", "# tags = \"The piece that opens with a soft, flute-led melody gently awakening the senses, conjuring the warmth of a sunrise over tranquil hills. The orchestra gradually swells with smooth strings and light percussion, evoking a blossoming sense of optimism and renewed energy. A graceful interplay of woodwinds and strings concludes on a delicate, uplifting note that captures the fresh promise of morning.\"\n", "# tags = \"A stirring, festive march led by bold brass fanfares and triumphant rhythms that command attention right from the start. Let snappy snare drums and buoyant woodwinds propel the music forward with an infectious energy. The piece culminates in a grand, celebratory finale that invites enthusiastic clapping and cheering.\"\n", "#tags = \"Hypnotic, atmospheric, emotional, liquid dnb, UK garage, ambient, loop-based, vocal chops, minimal, ethereal, club-ready, warm synths, lush pads, repetitive vocals, soft build, euphoric drop\"\n", "#tags = \"Gregorian chant\"\n", "\n", "#tags = \"billboard hot 100 2019, indie rock, grammy best production, grammy best song, grammy best album, grammy best artist, grammy best record, grammy best song, grammy best album, grammy best artist, grammy best record\"\n", "tags = \"\"\n", "\n", "import numpy as np\n", "n_skip_semantic = 1\n", "#n_skip_semantic = 4\n", "old_gconf = True\n", "\n", "if old_gconf:\n", " gconf = GenerationConfig(\n", " text=text,\n", " text_tags=tags,\n", " cfg_coef=1.1,\n", " cfg_coef_tags=1.1,\n", " cfg_coef_max_steps=None,\n", " cfg_coef_tags_max_steps=None,\n", " n_repeat_tags=1,\n", " n_skip_semantic=n_skip_semantic,\n", " temp_semantic=0.92,\n", " n_batch=1,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=int(1*60/n_skip_semantic),\n", " random_seed=np.random.randint(0, 1000000),\n", " )\n", "else:\n", " gconf = GenerationConfig(\n", " text=text,\n", " text_tags=tags,\n", " cfg_coef=1.2,\n", " cfg_coef_tags=1.5,\n", " cfg_coef_max_steps=None,\n", " cfg_coef_tags_max_steps=None,\n", " n_repeat_tags=1,\n", " n_repeat_neg_tags=1,\n", " n_skip_semantic=n_skip_semantic,\n", " #text_start_control_tags=\"{min_duration:60}\",\n", " cfg_coef_neg_tags=0.0,\n", " text_neg_tags=None,\n", " temp_semantic=0.92,\n", " #top_k_semantic=None,\n", " #top_p_semantic=None,\n", " #min_p_semantic=0.001,\n", " n_batch=1,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " #use_alternate_cfg=True,\n", " max_gen_duration_s=int(2*60/n_skip_semantic),\n", " random_seed=np.random.randint(0, 1000000),\n", " )" ] }, { "cell_type": "code", "execution_count": null, "id": "3133ec0f", "metadata": {}, "outputs": [], "source": [ "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "out_gpt = []\n", "for n, job in enumerate(jobs):\n", " stream = engine.token_generator(job)\n", " arr = torch.stack(list(stream))[:,1]\n", " if arr[-1] == 4000:\n", " arr = arr[:-1]\n", " print(f\"{round(arr.shape[-1]/25*n_skip_semantic)}s for track {n}\")\n", " # do stuff incase skip\n", " arr2 = torch.zeros(arr.shape[0]*n_skip_semantic, dtype=arr.dtype) + cfg.semantic_pad_token\n", " arr2[::n_skip_semantic] = arr\n", " # add\n", " out_gpt.append(arr2)" ] }, { "cell_type": "code", "execution_count": null, "id": "19788eb0", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "for in_sem_arr in out_gpt:\n", " for seed in [np.random.randint(0, 1000000) for _ in range(2)]:\n", " p = 0.75\n", " mask = np.random.binomial(1, p, size=in_sem_arr.shape[0])\n", " print(mask)\n", " in_sem_arr_pad = in_sem_arr.clone()\n", " in_sem_arr_pad[mask == 1] = cfg.semantic_pad_token\n", " all_pad = torch.ones(750).long() * cfg.semantic_pad_token\n", " \n", " diffusion_seed = 100\n", " diffusion_steps = 16\n", " diffusion_text_cfg_coef = 2.5\n", " noise_ctx_level = 0.75\n", " noise_ctx_pad_len = 0\n", " # diffusion_ctx_cfg_coef = 1.0\n", "\n", " gen_cfg = diffusion_gen.DiffusionGenerationConfig(\n", " steps=diffusion_steps,\n", " lyrics=text,\n", " tags=tags,\n", " text_cfg_coef=diffusion_text_cfg_coef,\n", " codec_scale_factor=CODEC_SCALE_FACTOR,\n", " scale_ctx_vector=SCALE_CTX_VECTOR,\n", " noise_ctx_level=noise_ctx_level,\n", " noise_ctx_pad_len=noise_ctx_pad_len,\n", " drop_semantic_tokens=False,\n", " seed=np.random.randint(0, 1000000),\n", " #rho=1.0,\n", " )\n", "\n", " request = Request(\n", " id=\"dummy\",\n", " generation_config=gen_cfg,\n", " tokens=in_sem_arr,\n", " input_tokens_finished=True,\n", " )\n", "\n", " result = diffusion_engine.run_request(request)\n", " concat_audio = Audio.concatenate(result.generated_audios)\n", " concat_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "d8e4db07", "metadata": {}, "outputs": [], "source": [ " # test training data semantic as a reference point\n", "from suno_utils.utils.text import read_jsonl\n", "\n", "VAE_DIM = 128\n", "VAE_N_MEMMAP_TOKENS = 750\n", "\n", "SEMANTIC_N_CODEBOOKS = 1\n", "SEMANTIC_N_MEMMAP_TOKENS = 750\n", "\n", "#base_dir = \"/app/suno/data/diffusion_mix/dac_vae_fixed_25hz\"\n", "#base_dir = \"/app/suno/data/diffusion_mix/dac_vae_tuned_25hz\"\n", "base_dir = \"/app/suno/data/diffusion_v5/v0\"\n", "#base_dir = \"/app/suno/data/diffusion_mix/vae_25hz_30s\"\n", "#base_dir = \"/mnt/localdisk/cjs_shards/\"\n", "metas = read_jsonl(f\"{base_dir}/metas_val_aligned_v3.jsonl\", progress=False)\n", "vae_memmap_filepath = f\"{base_dir}/data_vae_val.bin\"\n", "semantic_memmap_filepath = f\"{base_dir}/data_semantic_val.bin\"\n", "\n", "# load memmaps\n", "vae_memmap = np.memmap(vae_memmap_filepath, dtype=np.float16, mode=\"r\")\n", "semantic_memmap = np.memmap(semantic_memmap_filepath, dtype=np.uint16, mode=\"r\")\n", "\n", "# reshape memmaps\n", "vae_data = vae_memmap.reshape(-1, VAE_N_MEMMAP_TOKENS, VAE_DIM)\n", "semantic_data = semantic_memmap.reshape(-1, SEMANTIC_N_MEMMAP_TOKENS, SEMANTIC_N_CODEBOOKS)\n", "\n", "print(vae_data.shape, semantic_data.shape, len(metas))" ] }, { "cell_type": "code", "execution_count": null, "id": "68452692", "metadata": {}, "outputs": [], "source": [ "idx = 4987\n", "#idx = 5003\n", "\n", "print(metas[idx])\n", "print(metas[idx][\"text_aligned\"])\n", "audio = codec_decode(vae_data[idx])\n", "audio.play()\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "10ecf455", "metadata": {}, "outputs": [], "source": [ "from suno_utils.tasks.dac_vae_100hz_peaq import decode as decode_vae\n", "from suno_utils.tasks.dac_vae_100hz_peaq import preload_models as preload_vae\n", "checkpoint_filepath = \"s3://suno-data/christian/25hz_vae_peaq_kl_0.005.pth\"\n", "\n", "from suno_utils.tasks.dac_vae_25hz_fixed import decode as decode_vae\n", "from suno_utils.tasks.dac_vae_25hz_fixed import preload_models as preload_vae\n", "checkpoint_filepath = \"s3://suno-data/minz/\n", "\n", "_ = preload_vae(checkpoint_filepath)\n" ] }, { "cell_type": "code", "execution_count": 56, "id": "be7ba92e", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio" ] }, { "cell_type": "code", "execution_count": null, "id": "4b4b62af", "metadata": {}, "outputs": [], "source": [ "idx = 1500\n", "in_sem_arr = torch.from_numpy(semantic_data[idx][:,0]).long()\n", "print(in_sem_arr.shape)\n", "\n", "meta = metas[idx]\n", "s3_filepath = meta[\"s3_filepath\"]\n", "# download to \"audio.mp3\"\n", "os.system(f\"aws s3 cp {s3_filepath} audio.mp3\")\n", "\n", "audio = Audio.from_file(\"audio.mp3\", n_channels=2).normalize_volume()\n", "audio.play()\n", "\n", "# codec decode\n", "in_vae_arr = torch.from_numpy(vae_data[idx].astype(np.float32))\n", "print(in_vae_arr.shape)\n", "audio = decode_vae(in_vae_arr)\n", "audio.play()\n", "\n", "\n", "for seed in [np.random.randint(0, 1000000) for _ in range(1)]:\n", " generate(\n", " in_sem_arr,\n", " lyrics=text,\n", " tags=tags, \n", " text_cfg_coef=1.5,\n", " steps=32,\n", " seed=np.random.randint(0, 1000000),\n", " downscale_ctx_vector=True,\n", " ).play()\n" ] }, { "cell_type": "markdown", "id": "c1bd1317", "metadata": {}, "source": [ "# Evaluation generation" ] }, { "cell_type": "code", "execution_count": null, "id": "350c7f31", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import os\n", "# load prompts from prod and then generate\n", "\n", "prompt_set_name = \"interesting_clips_exp_20250129_full\"\n", "pkl_filepath = \"/home/tony/Data/Preference/13b_v32/interesting_clips_exp_20250129_full.pkl\"\n", "df = pd.read_pickle(pkl_filepath)\n", "df[\"id_x\"] = df[\"id\"]\n", "\n", "df = df.reset_index(drop=True)\n", "#print(df[\"model_name\"].value_counts())\n", "\n", "# only take from chirp-v4-h-s-32\n", "df = df[df[\"model_name\"] == \"chirp-v4-h-s-32\"]\n", "# sample N rows at random \n", "N = 100\n", "df = df.sample(N)\n", "print(df[\"model_name\"].value_counts())\n", "\n", "# loop for eval\n", "eval_name = f\"{ckpt_name}_{prompt_set_name}\"\n", "out_dir = f\"/home/christian/benchmark/gpt45/{eval_name}/\"\n", "print(out_dir)\n", "os.makedirs(out_dir, exist_ok=True)\n", "\n", "# create a csv file with the info " ] }, { "cell_type": "code", "execution_count": null, "id": "92f37b93", "metadata": {}, "outputs": [], "source": [ "# construct prompts\n", "prompts = []\n", "for i, row in df.iterrows():\n", " seed = np.random.randint(0, 1000000)\n", " prompt = {\n", " \"tags\" : row.metadata[\"tags\"],\n", " \"text\" : row.prompt_text,\n", " \"s3_id\" : row.id_x,\n", " \"audio_filepath\" : os.path.join(out_dir, f\"{row.id_x}_{seed}.mp3\"),\n", " \"seed\" : seed\n", " }\n", " prompts.append(prompt)\n", "print(len(prompts))\n", "\n", "# create a new dataframe with the prompts\n", "df_prompts = pd.DataFrame(prompts)\n", "df_prompts.to_csv(f\"{out_dir}/metadata.csv\", index=False)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "803e433e", "metadata": {}, "outputs": [], "source": [ "df_prompts.head()" ] }, { "cell_type": "code", "execution_count": 8, "id": "a77eb977", "metadata": {}, "outputs": [], "source": [ "# load prompts from bench \n", "import pandas as pd\n", "import os\n", "prompt_dir = \"/home/christian/code/christian/musicBench/prompts\"\n", "csv_path = os.path.join(prompt_dir, \"02_singing_prompts.csv\")\n", "df = pd.read_csv(csv_path)\n", "\n", "df.head()" ] }, { "cell_type": "code", "execution_count": 9, "id": "f64ec057", "metadata": {}, "outputs": [], "source": [ "from tqdm import tqdm\n", "import numpy as np\n", "\n", "ckpt_name = \"2025-02-02_21-04-52\"\n", "output_dir = f\"/home/christian/code/christian/musicBench/outputs/{ckpt_name}\"\n", "os.makedirs(output_dir, exist_ok=True)\n", "\n", "for i, row in tqdm(df.iterrows(), total=len(df)):\n", " tags = row[\"tags\"]\n", " text = row[\"text\"]\n", " s3_id = row[\"id\"]\n", " audio_filepath = os.path.join(output_dir, f\"{s3_id}.mp3\")\n", "\n", " n_skip_semantic = 1\n", " gconf = GenerationConfig(\n", " text=text,\n", " text_tags=tags,\n", " cfg_coef=1.2,\n", " cfg_coef_tags=2.5,\n", " cfg_coef_max_steps=None,\n", " cfg_coef_tags_max_steps=None,\n", " n_repeat_tags=3,\n", " n_skip_semantic=n_skip_semantic,\n", " text_start_control_tags=\"{min_duration:120}\",\n", " cfg_coef_neg_tags=-1,\n", " text_neg_tags=\"repetitive, loop\",\n", " n_batch=1,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=int(2*60/n_skip_semantic),\n", " random_seed=2,\n", " )\n", "\n", " requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", " ]\n", " jobs = engine.run_request(requests, tqdm_enabled=True)\n", " out_gpt = []\n", " # run gpt\n", " for n, job in enumerate(jobs):\n", " stream = engine.token_generator(job)\n", " arr = torch.stack(list(stream))[:,1]\n", " if arr[-1] == 4000:\n", " arr = arr[:-1]\n", " print(f\"{round(arr.shape[-1]/25*n_skip_semantic)}s for track {n}\")\n", " # do stuff incase skip\n", " arr2 = torch.zeros(arr.shape[0]*n_skip_semantic, dtype=arr.dtype) + cfg.semantic_pad_token\n", " arr2[::n_skip_semantic] = arr\n", " # add\n", " out_gpt.append(arr2)\n", " # run diffusion\n", " #for in_sem_arr in out_gpt:\n", " in_sem_arr = out_gpt[0] # only take first one\n", " for seed in [np.random.randint(0, 1000000) for _ in range(1)]:\n", " audio = generate(\n", " in_sem_arr,\n", " lyrics=text,\n", " tags=tags, \n", " text_cfg_coef=1.5,\n", " steps=16,\n", " seed=np.random.randint(0, 1000000),\n", " downscale_ctx_vector=False,\n", " )\n", " try:\n", " # save audio\n", " audio.write_hq_mp3(audio_filepath)\n", " except Exception as e:\n", " print(e)\n", " print(f\"failed to save {ckpt_name}_{seed}_{tags}.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "id": "5cb96fc3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "43513761", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "fbe4f87d", "metadata": {}, "source": [ "### Over/Underpaint" ] }, { "cell_type": "code", "execution_count": null, "id": "6566827e", "metadata": {}, "outputs": [], "source": [ "import random\n", "from suno_utils.audio import Audio\n", "from generation import encode_semantic\n", "from suno_utils.tasks.demucs import split_vocals" ] }, { "cell_type": "code", "execution_count": null, "id": "aaf94996", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "from suno_utils.utils.text import read_jsonl\n", "mm = np.memmap(\"/app/suno/data/chirp_v5/v1/data_val.bin\", dtype=np.uint16, mode=\"r\")\n", "metas = read_jsonl(\"/app/suno/data/chirp_v5/v1/metas_val.jsonl\")\n", "overpaint_metas = [m for m in metas if m[\"dataset\"] == \"musdb_stems_overpaint\" and m[\"type\"] == \"instrumental\"]\n", "underpaint_metas = [m for m in metas if m[\"dataset\"] == \"musdb_stems_underpaint\" and m[\"type\"] == \"vocals\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "f98dcf7e", "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "[Verse 1]\n", "Remember those walls I built?\n", "Well, baby, they're tumblin' down\n", "And they didn't even put up a fight\n", "They didn't even make a sound\n", "I found a way to let you in\n", "But I never really had a doubt\n", "Standin' in the light of your halo\n", "I got my angel now\n", "\n", "[Pre-Chorus]\n", "It's like I've been awakened\n", "Every rule, I had you breakin'\n", "It's the risk that I'm takin'\n", "I ain't ever gonna shut you out\n", "\n", "[Chorus]\n", "Everywhere I'm lookin' now\n", "I'm surrounded by your embrace\n", "Baby, I can see your halo\n", "You know you're my savin' grace\n", "You're everything I need and more\n", "It's written all over your face\n", "Baby, I can feel your halo\n", "Pray it won't fade away\n", "\n", "[Post-Chorus]\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo, ooh\n", "\n", "[Verse 2]\n", "Hit me like a ray of sun\n", "Burnin' through my darkest night\n", "You're the only one that I want\n", "Think I'm addicted to your light\n", "I swore I'd never fall again\n", "But this don't even feel like fallin'\n", "Gravity can't begin\n", "To pull me back to the ground again\n", "\n", "[Pre-Chorus]\n", "It's like I've been awakened\n", "Every rule, I had you breakin'\n", "The risk that I'm takin'\n", "I'm never gonna shut you out\n", "\n", "[Chorus]\n", "Everywhere I'm lookin' now\n", "I'm surrounded by your embrace\n", "Baby, I can see your halo\n", "You know you're my savin' grace\n", "You're everything I need and more\n", "It's written all over your face\n", "Baby, I can feel your halo\n", "Pray it won't fade away\n", "\n", "[Post-Chorus]\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo, ooh\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo, ooh\n", "\n", "[Bridge]\n", "Halo, ooh\n", "Halo, ooh\n", "Ooh\n", "\n", "[Chorus]\n", "Everywhere I'm lookin' now\n", "I'm surrounded by your embrace\n", "Baby, I can see your halo\n", "You know you're my savin' grace\n", "You're everything I need and more\n", "It's written all over your face\n", "Baby, I can feel your halo\n", "Pray it won't fade away\n", "\n", "[Post-Chorus]\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo, ooh\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo, ooh\n", "\"\"\"" ] }, { "cell_type": "markdown", "id": "fa3cd8c8", "metadata": {}, "source": [ "#### Underpaint" ] }, { "cell_type": "code", "execution_count": null, "id": "28bf850c", "metadata": {}, "outputs": [], "source": [ "a = Audio.from_file(\"../samples/halo.wav\", sample_rate=44_100, n_channels=2)\n", "# # a = Audio.from_file(\"../samples/calm_down.mp3\", sample_rate=44_100, n_channels=2)\n", "\n", "a_vocals, a_other = split_vocals(a.convert(44_100, 2, 2))\n", "vocals_arr = encode_semantic(a_vocals.convert(44_100, 2, 2).normalize_volume())[:,:1]\n", "# instrumental_arr = encode_semantic(a_other.convert(44_100, 2, 2).normalize_volume())[:,:1]" ] }, { "cell_type": "code", "execution_count": null, "id": "10e6d9b1", "metadata": {}, "outputs": [], "source": [ "# m = random.choice(underpaint_metas)\n", "# vocals_arr = mm[m[\"offset_idx\"]:m[\"offset_idx\"]+m[\"n_tokens\"]][:60*4][None].copy()" ] }, { "cell_type": "code", "execution_count": null, "id": "1023d4b8", "metadata": {}, "outputs": [], "source": [ "tags = \"Pop\"" ] }, { "cell_type": "code", "execution_count": null, "id": "4fa2fa71", "metadata": {}, "outputs": [], "source": [ "gconf = GenerationConfig(\n", " text=text,\n", " text_tags=tags,\n", " underpaint_arr=vocals_arr,\n", " cfg_coef=1.0,\n", " cfg_coef_tags=1.0,\n", " n_repeat_tags=1,\n", " n_batch=1,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=2*60,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "020e0ac5", "metadata": { "scrolled": true }, "outputs": [], "source": [ "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "out_gpt = []\n", "for n, job in enumerate(jobs):\n", " stream = engine.token_generator(job)\n", " arr = torch.stack(list(stream))[:,1]\n", " if arr[-1] == 4000:\n", " arr = arr[:-1]\n", " print(f\"{round(arr.shape[-1]/25)}s for track {n}\")\n", " out_gpt.append(arr)" ] }, { "cell_type": "code", "execution_count": null, "id": "cf653ddf", "metadata": {}, "outputs": [], "source": [ "for in_sem_arr in out_gpt:\n", " generate(\n", " in_sem_arr,\n", " lyrics=text,\n", " tags=tags, \n", " text_cfg_coef=1.0,\n", " steps=16,\n", " seed=0,\n", " ).play()" ] }, { "cell_type": "markdown", "id": "7f687e4c", "metadata": {}, "source": [ "#### Overpaint" ] }, { "cell_type": "code", "execution_count": null, "id": "a779ace0", "metadata": {}, "outputs": [], "source": [ "a = Audio.from_file(\"../samples/halo.wav\", sample_rate=44_100, n_channels=2)\n", "# a = Audio.from_file(\"../samples/calm_down.mp3\", sample_rate=44_100, n_channels=2)\n", "\n", "a_vocals, a_other = split_vocals(a.convert(44_100, 2, 2))\n", "vocals_arr = encode_semantic(a_vocals.convert(44_100, 2, 2).normalize_volume())[:,:1]\n", "instrumental_arr = encode_semantic(a_other.convert(44_100, 2, 2).normalize_volume())[:,:1]" ] }, { "cell_type": "code", "execution_count": null, "id": "233a72dc", "metadata": {}, "outputs": [], "source": [ "# a_vocals = Audio.from_file(\"../samples/martin_vocals_up.mp3\", sample_rate=44_100, n_channels=2)\n", "# vocals_arr = encode_semantic(a_vocals.convert(44_100, 2, 2).normalize_volume())[:,:1]\n", "\n", "# a_instrumental = Audio.from_file(\"../samples/martin_instrumental.m4a\", sample_rate=44_100, n_channels=2)\n", "# instrumental_arr = encode_semantic(a_instrumental.convert(44_100, 2, 2).normalize_volume())[:,:1]\n", "\n", "# m = random.choice(overpaint_metas)\n", "# instrumental_arr = mm[m[\"offset_idx\"]:m[\"offset_idx\"]+m[\"n_tokens\"]][:60*4][None].copy()" ] }, { "cell_type": "code", "execution_count": null, "id": "1fc2a900", "metadata": {}, "outputs": [], "source": [ "gconf = GenerationConfig(\n", " text=text,\n", "# text_tags=\"pop, sentimental, folk\",\n", " overpaint_arr=instrumental_arr,\n", " cfg_coef=1.2,\n", "# cfg_coef_tags=2.0,\n", " n_repeat_tags=1,\n", " n_batch=1,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=2*60,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "d4462f43", "metadata": {}, "outputs": [], "source": [ "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "out_gpt = []\n", "for n, job in enumerate(jobs):\n", " stream = engine.token_generator(job)\n", " arr = torch.stack(list(stream))[:,1]\n", " if arr[-1] == 4000:\n", " arr = arr[:-1]\n", " print(f\"{round(arr.shape[-1]/25)}s for track {n}\")\n", " out_gpt.append(arr)" ] }, { "cell_type": "code", "execution_count": null, "id": "7c4e154e", "metadata": {}, "outputs": [], "source": [ "for in_sem_arr in out_gpt:\n", " generate(\n", " in_sem_arr,\n", " lyrics=text,\n", "# tags=tags, \n", " text_cfg_coef=1.0,\n", " steps=16,\n", " seed=0,\n", " ).play()" ] }, { "cell_type": "code", "execution_count": null, "id": "4b1d7b0b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6267ad78", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "abc1d47c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c61e6914", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "c95fb777", "metadata": {}, "source": [ "### Cover" ] }, { "cell_type": "code", "execution_count": null, "id": "f21c1f65", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio\n", "from generation import encode_semantic\n", "from suno_utils.tasks.demucs import split_vocals\n", "\n", "# audio_filepath = \"../samples/martin2.m4a\"\n", "audio_filepath = \"../samples/halo.wav\"\n", "# audio_filepath = \"../samples/corn_chase.mp3\"\n", "# audio_filepath = \"../samples/martin_vocals.m4a\"\n", "\n", "a = Audio.from_file(audio_filepath, sample_rate=44_100, n_channels=2)\n", "cover_arr = encode_semantic(a.normalize_volume())[:,:1]\n", "a.play()" ] }, { "cell_type": "code", "execution_count": 9, "id": "75677dcc", "metadata": {}, "outputs": [], "source": [ "# text = \"\"\"\n", "# [Verse]\n", "# Raindrops falling away\n", "# Dancing in the grey\n", "# Clouds are here to stay\n", "# Love on a rainy day\n", "\n", "# [Verse 2]\n", "# Puddles on the ground\n", "# Feet splashing 'round\n", "# You and me we found\n", "# Paradise in the sound\n", "\n", "# [Chorus]\n", "# Love on a rainy day\n", "# Washing doubts away\n", "# Hold me and let's sway\n", "# It's our perfect cliche\n", "\n", "# Love on a rainy day\n", "# Washing doubts away\n", "# Hold me and let's sway\n", "# It's our perfect cliche\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# Well, you only need the light when it's burning low\n", "# Only miss the sun when it starts to snow\n", "# Only know you love her when you let her go\n", "# Only know you've been high when you're feeling low\n", "# Only hate the road when you're missing home\n", "# Only know you love her when you let her go\n", "\n", "# And you let her go\n", "# \"\"\"\n", "\n", "text = \"\"\"\n", "[Verse 1]\n", "Remember those walls I built?\n", "Well, baby, they're tumblin' down\n", "And they didn't even put up a fight\n", "They didn't even make a sound\n", "I found a way to let you in\n", "But I never really had a doubt\n", "Standin' in the light of your halo\n", "I got my angel now\n", "\n", "[Pre-Chorus]\n", "It's like I've been awakened\n", "Every rule, I had you breakin'\n", "It's the risk that I'm takin'\n", "I ain't ever gonna shut you out\n", "\n", "[Chorus]\n", "Everywhere I'm lookin' now\n", "I'm surrounded by your embrace\n", "Baby, I can see your halo\n", "You know you're my savin' grace\n", "You're everything I need and more\n", "It's written all over your face\n", "Baby, I can feel your halo\n", "Pray it won't fade away\n", "\n", "[Post-Chorus]\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo, ooh\n", "\n", "[Verse 2]\n", "Hit me like a ray of sun\n", "Burnin' through my darkest night\n", "You're the only one that I want\n", "Think I'm addicted to your light\n", "I swore I'd never fall again\n", "But this don't even feel like fallin'\n", "Gravity can't begin\n", "To pull me back to the ground again\n", "\n", "[Pre-Chorus]\n", "It's like I've been awakened\n", "Every rule, I had you breakin'\n", "The risk that I'm takin'\n", "I'm never gonna shut you out\n", "\n", "[Chorus]\n", "Everywhere I'm lookin' now\n", "I'm surrounded by your embrace\n", "Baby, I can see your halo\n", "You know you're my savin' grace\n", "You're everything I need and more\n", "It's written all over your face\n", "Baby, I can feel your halo\n", "Pray it won't fade away\n", "\n", "[Post-Chorus]\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo, ooh\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo, ooh\n", "\n", "[Bridge]\n", "Halo, ooh\n", "Halo, ooh\n", "Ooh\n", "\n", "[Chorus]\n", "Everywhere I'm lookin' now\n", "I'm surrounded by your embrace\n", "Baby, I can see your halo\n", "You know you're my savin' grace\n", "You're everything I need and more\n", "It's written all over your face\n", "Baby, I can feel your halo\n", "Pray it won't fade away\n", "\n", "[Post-Chorus]\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo, ooh\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo\n", "I can feel your halo, halo, halo\n", "I can see your halo, halo, halo, ooh\n", "\"\"\"\n", "\n", "tags = \"folk music, haunting, sad, female vocals, emotive, guitar\"\n", "# tags = \"R&B, Pop, Contemporary R&B, Ballad, male vocals\"\n", "\n", "# text = \"\"\n", "# tags = \"Big Room Trance\"" ] }, { "cell_type": "code", "execution_count": 10, "id": "99745ae8", "metadata": {}, "outputs": [], "source": [ "gconf = GenerationConfig(\n", " text=text,\n", " text_tags=tags,\n", " cover_arr=cover_arr,\n", " cfg_coef=1.2,\n", " cfg_coef_tags=2.5,\n", " n_repeat_tags=1,\n", " n_batch=1,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=2*60,\n", " random_seed=0,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "c2a70da1", "metadata": {}, "outputs": [], "source": [ "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "out_gpt = []\n", "for n, job in enumerate(jobs):\n", " stream = engine.token_generator(job)\n", " arr = torch.stack(list(stream))[:,1]\n", " if arr[-1] == 4000:\n", " arr = arr[:-1]\n", " print(f\"{round(arr.shape[-1]/25)}s for track {n}\")\n", " out_gpt.append(arr)" ] }, { "cell_type": "code", "execution_count": null, "id": "d44152cc", "metadata": {}, "outputs": [], "source": [ "for in_sem_arr in out_gpt:\n", " generate(\n", " in_sem_arr,\n", " lyrics=text,\n", " tags=tags, \n", " text_cfg_coef=1.0,\n", " steps=16,\n", " seed=0,\n", " ).play()" ] }, { "cell_type": "code", "execution_count": null, "id": "3452380a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "be956547", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "19570e8d", "metadata": {}, "source": [ "### Visualize prompt" ] }, { "cell_type": "code", "execution_count": null, "id": "6e5ff457", "metadata": {}, "outputs": [], "source": [ "from suno_utils.gpt.prompt import Prompt\n", "in_arr = requests[0].streams[0].prompt\n", "prompt = Prompt(\"\", engine.model.config)\n", "prompt.visualize(in_arr, compress=True)\n", "prompt.visualize(in_arr, compress=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "d042392a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "699bda31", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "312a749f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b7f5cce1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "03a473d7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bbb3d211", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "addcbe70", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b290ea0b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "242acd6e", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 5 }