{ "cells": [ { "cell_type": "markdown", "id": "title", "metadata": {}, "source": [ "# Sample2Song Inference Testing Suite\n", "**Date:** August 21, 2025 \n", "**Purpose:** Comprehensive testing of sample2song functionality with various conditioning approaches \n", "**Based on:** chicken_inference_suite_vocal_infill_base.ipynb \n", "\n", "## Test Areas:\n", "1. **Multi-Sample Testing:** 1-10 samples simultaneously\n", "2. **Style Diversity:** Different sample genres and characteristics \n", "3. **Producer Tag Testing:** Standard vs custom producer tags\n", "4. **Timing Accuracy:** Sample timing and placement precision\n", "5. **Pretrain Coverage:** Sample2song functionality in base models" ] }, { "cell_type": "markdown", "id": "61bffb0c", "metadata": {}, "source": [ "# setup" ] }, { "cell_type": "code", "execution_count": null, "id": "setup", "metadata": {}, "outputs": [], "source": [ "import os\n", "import random\n", "import numpy as np\n", "import torch\n", "from pathlib import Path\n", "import json\n", "import time\n", "from datetime import datetime\n", "\n", "# Set GPU\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n", "\n", "# For reproducibility\n", "torch.manual_seed(42)\n", "random.seed(42)\n", "np.random.seed(42)\n", "\n", "print(f\"Sample2Song Testing Started: {datetime.now()}\")" ] }, { "cell_type": "markdown", "id": "model_load", "metadata": {}, "source": [ "## Model Loading\n", "Load the sample2song trained model and supporting components" ] }, { "cell_type": "code", "execution_count": null, "id": "load_model", "metadata": {}, "outputs": [], "source": [ "from suno_utils.gpt.generation import load_model, GPT\n", "from suno_utils.utils.s3 import download_s3_file_if_needed\n", "from suno_utils.audio import Audio\n", "\n", "N_BATCH = 2\n", "\n", "# gpt_model_paths = {\n", "# \"pretrain\": \"/volume/models/vibert/2025-08-17_15-35-37_v5_pretrain.pt\",\n", "# \"dpo\": \"/volume/models/vibert/2025-08-24_08-31-23_v5_dpo.pt\",\n", "# }\n", "\n", "gpt_model_paths = {\n", " \"pretrain\": \"/app2/suno/checkpoints/2025-08-17_15-35-37/last_ckpt_infer.pt\",\n", " \"dpo\": \"/app2/suno/checkpoints/2025-08-24_08-31-23/last_ckpt_infer.pt\",\n", "}\n", "\n", "# Model paths - update with latest sample2song model\n", "gpt_model_path = gpt_model_paths[\"dpo\"]\n", "tokenizer_path = \"s3://suno-data/georg/models/tokenizers/tokenizer_60k.json\"\n", "\n", "print(\"Loading GPT model...\")\n", "model_container = load_model(\n", " ckpt_path=download_s3_file_if_needed(gpt_model_path),\n", " tokenizer_path=download_s3_file_if_needed(tokenizer_path),\n", " # device=\"cuda:1\",\n", ")\n", "model: GPT = model_container[\"model\"]\n", "assert isinstance(model, GPT)\n", "cfg = model.config\n", "print(f\"Model loaded: {cfg.n_layer} layers, {cfg.n_head} heads, {cfg.n_embd} dimensions\")\n", "\n", "# Load diffusion and codec models\n", "from suno_utils.diffusion import generation as diffusion_gen\n", "from suno_utils.tasks.upsample_engine import UpsampleEngine, Request\n", "from suno_utils.tasks.dac_vae_fixed_25hz import preload_models as preload_codec_models, decode\n", "\n", "print(\"Loading diffusion model...\")\n", "dit_model_filepath = \"/app/suno/checkpoints/2025-02-17_16-54-01_s7787/last_ckpt.pt\" # base diffusion\n", "diffusion_gen.preload_models(\n", " dit_model_filepath=dit_model_filepath,\n", " # device=\"cuda:1\",\n", ")\n", "\n", "print(\"Loading codec model...\")\n", "preload_codec_models(\"s3://suno-data/minz/models/dac_vae_tuned_25hz.pth\")\n", "\n", "diffusion_engine = UpsampleEngine(compile=False)\n", "print(\"✅ All models loaded successfully\")" ] }, { "cell_type": "markdown", "id": "24a02bb0", "metadata": {}, "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bct_setup", "metadata": {}, "outputs": [], "source": [ "# BCT (Block-based Conditioning Transformer) setup for sample2song\n", "from suno_utils.gpt.bct.bct_generation_simple import BCTGenerationConfig, BlockSequence, generate_block\n", "from suno_utils.gpt.bct.bct import Block, BlockType, TensorDict\n", "\n", "# Define block types\n", "TextBlockType = BlockType(\n", " name=\"text\",\n", " is_causal=True,\n", ")\n", "\n", "CausalSemanticBlockType = BlockType(\n", " name=\"semantic\",\n", " is_causal=True,\n", ")\n", "\n", "SampleBlockType = BlockType(\n", " name=\"sample\", \n", " is_causal=True,\n", ")\n", "\n", "VoxBlockType = BlockType(\n", " name=\"vox\",\n", " is_causal=True,\n", ")\n", "\n", "CoverBlockType = BlockType(\n", " name=\"cover\",\n", " is_causal=True,\n", ")\n", "\n", "print(\"✅ BCT framework initialized\")" ] }, { "cell_type": "markdown", "id": "helpers", "metadata": {}, "source": [ "## Helper Functions\n", "Core utilities for sample processing and inference" ] }, { "cell_type": "code", "execution_count": null, "id": "audio_helpers", "metadata": {}, "outputs": [], "source": [ "# Audio processing helpers\n", "from suno_utils.diffusion.generation import encode_semantic\n", "from suno_utils.tasks.mert_25 import preload_models as preload_semantic_models_\n", "from suno_utils.tasks.demucs import split_vocals\n", "\n", "# Load semantic clustering model\n", "cluster_model = preload_semantic_models_(\n", " checkpoint_filepath=\"s3://suno-data/georg/models/semantic/mert_25.pt\",\n", " centroids_filepath=\"s3://suno-data/georg/models/semantic/mert_25_2x4k.npy\",\n", ")[\"cluster_model\"].cpu()\n", "\n", "def make_semantics(audio, pool_factor=25):\n", " \"\"\"Convert audio to semantic tokens\"\"\"\n", " arr = encode_semantic(audio.convert(44_100, 2, 2).normalize_volume(), do_clustering=False)\n", " if arr.shape[0] % pool_factor != 0:\n", " arr = arr[: -(arr.shape[0] % pool_factor)]\n", " pooled_arr = arr.reshape(-1, pool_factor, 768).mean(1)\n", " arr = cluster_model.encode(torch.from_numpy(arr)[None, :, :])[0, :, 0].tolist()\n", " pooled_arr = cluster_model.encode(torch.from_numpy(pooled_arr)[None, :, :])[0, :, 0].tolist()\n", " return pooled_arr, arr\n", "\n", "def run_diffusion(codes, lyrics=\"\", cfg_scale=2.0):\n", " \"\"\"Convert semantic codes to audio via diffusion\"\"\"\n", " assert codes.min() >= 0 and codes.max() < model.config.semantic_pad_token\n", " gen_cfg = diffusion_gen.DiffusionGenerationConfig(\n", " lyrics=lyrics,\n", " text_cfg_coef=cfg_scale,\n", " ctx_cfg_coef=1.0,\n", " steps=12,\n", " codec_scale_factor=0.4,\n", " scale_ctx_vector=True,\n", " )\n", " request = Request(\n", " id=\"sample2song\",\n", " generation_config=gen_cfg,\n", " tokens=codes.cpu(),\n", " input_tokens_finished=True,\n", " )\n", " result = diffusion_engine.run_request(request)\n", " vae_latents = torch.concat(result.vae_latents)\n", " audio = decode(vae_latents)\n", " return audio\n", "\n", "print(\"✅ Audio processing helpers loaded\")\n", "\n", "def run_inference(blocks, max_steps=25 * 60, time_inputs=None, text_cfg_boost=0.0, return_raw=False):\n", " if time_inputs is None:\n", " time_inputs = list(range(max_steps))\n", "\n", " if text_cfg_boost == 0.0 or blocks[0].spec.name != \"text\":\n", " prompts = [(1, blocks)]\n", " else:\n", " no_text_blocks = BlockSequence(blocks[1:])\n", " prompts = [(1 + text_cfg_boost, blocks), (-text_cfg_boost, no_text_blocks)]\n", "\n", " gconf = BCTGenerationConfig(\n", " prompts,\n", " max_autoregressive_steps=max_steps,\n", " eos_token=cfg.semantic_pad_token,\n", " # eos_token=None,\n", " temperature=0.9,\n", " # time_inputs=time_inputs,\n", " compile=False,\n", " )\n", "\n", " block = generate_block(model, gconf)\n", " all_sem_codes = block.inputs[\"semantic_input\"][0, 0, 1 : len(block)]\n", " if return_raw:\n", " return all_sem_codes\n", "\n", " return run_diffusion(all_sem_codes)\n", "\n", "\n", "# Core inference functions\n", "def make_text_block(text: str):\n", " \"\"\"Create text conditioning block\"\"\"\n", " text_tokens = model_container[\"tokenizer\"].encode(text) + [cfg.text_infer_token]\n", " return Block(\n", " TextBlockType,\n", " inputs=TensorDict({\n", " \"text_input\": torch.tensor(text_tokens).reshape(1, 1, -1),\n", " }),\n", " )\n", "\n", "def make_sample_block(audio_samples, sample_times=None, sample_sources=None):\n", " \"\"\"Create sample conditioning block for multiple audio samples\"\"\"\n", " sample_tokens = []\n", " \n", " for i, sample_audio in enumerate(audio_samples):\n", " # Convert sample to semantic tokens\n", " if isinstance(sample_audio, str): # if path\n", " sample_audio = Audio.from_file(sample_audio)\n", " \n", " _, sample_semantic = make_semantics(sample_audio)\n", " \n", " # Add sample metadata tokens\n", " sample_tokens.append(cfg.audio_sample_token) # sample start token\n", " \n", " # Add timing if provided\n", " if sample_times and len(sample_times) > i:\n", " # Encode timing information (simplified)\n", " timing_token = min(int(sample_times[i] * 25), cfg.semantic_pad_token - 1)\n", " sample_tokens.append(timing_token)\n", " \n", " # Add sample semantic tokens\n", " sample_tokens.extend(sample_semantic[:500]) # limit length\n", " \n", " # Add source type if provided \n", " if sample_sources and len(sample_sources) > i:\n", " source = sample_sources[i]\n", " if \"vocal\" in source.lower():\n", " sample_tokens.append(cfg.semantic_vocals_token)\n", " elif \"drum\" in source.lower():\n", " sample_tokens.append(cfg.semantic_drums_token)\n", " \n", " return Block(\n", " SampleBlockType,\n", " inputs=TensorDict({\n", " \"sample_input\": torch.tensor(sample_tokens).reshape(1, 1, -1),\n", " }),\n", " )\n", "\n", "def run_sample2song_inference(text, audio_samples, sample_times=None, sample_sources=None, \n", " max_steps=1500, text_cfg_boost=0.3, return_raw=False):\n", " \"\"\"Main sample2song inference function\"\"\"\n", " \n", " # Create blocks\n", " text_block = make_text_block(text)\n", " sample_block = make_sample_block(audio_samples, sample_times, sample_sources)\n", " \n", " sem_block = Block(\n", " CausalSemanticBlockType,\n", " inputs=TensorDict({\n", " \"semantic_input\": torch.full((1, 1, 1), cfg.semantic_infer_token),\n", " }),\n", " )\n", " \n", " # Setup blocks with/without text CFG\n", " blocks = BlockSequence([text_block, sample_block, sem_block])\n", " \n", " if text_cfg_boost == 0.0:\n", " prompts = [(1, blocks)]\n", " else:\n", " no_text_blocks = BlockSequence([sample_block, sem_block])\n", " prompts = [(1 + text_cfg_boost, blocks), (-text_cfg_boost, no_text_blocks)]\n", " \n", " # Generate\n", " gconf = BCTGenerationConfig(\n", " prompts,\n", " max_autoregressive_steps=max_steps,\n", " eos_token=cfg.semantic_pad_token,\n", " temperature=0.9,\n", " compile=False,\n", " )\n", " \n", " block = generate_block(model, gconf)\n", " all_sem_codes = block.inputs[\"semantic_input\"][0, 0, 1 : len(block)]\n", " \n", " if return_raw:\n", " return all_sem_codes\n", " \n", " return run_diffusion(all_sem_codes, lyrics=text)\n", "\n", "print(\"✅ Inference helpers loaded\")" ] }, { "cell_type": "markdown", "id": "4bdbf1de", "metadata": {}, "source": [ "## Copy from Victor" ] }, { "cell_type": "code", "execution_count": null, "id": "2c00f3d2", "metadata": {}, "outputs": [], "source": [ "lyrics = \"\"\" \n", "{start_offset:125}\n", "\n", "\n", "[verse]\n", "[5.0]oh, my love[8.0]\n", "[8.1]My friend you know[15.0]\n", "[15.1]it's been a while[20.0]\n", "[20.1]Without thinking of you[25.0]\n", "[25.1]but the thought makes me smile[30.0]\n", "\n", "[chorus]\n", "[40.0]I'm so tired of wanting\n", "[40.1]wanting more than this[45.0]\n", "[45.1]i know it but what am i to do[50.0]\n", "[50.1]i need some space to breathe,[55.0]\n", "[55.1]so give me some room[60.0]\n", "\n", "[versef]\n", "[80.0]oh, my love\n", "you have a heart of stone\n", "cause since i've come home\n", "i've never felt so alone\n", "but the thought makes me smile\n", "\"\"\"\n", "# lyrics = \"[gangster rap]\\n\\n[safe mumble mode]\"\n", "text = lyrics\n", "\n", "def make_text_block(text: str):\n", " text_tokens = model_container[\"tokenizer\"].encode(text) + [cfg.text_infer_token]\n", " return Block(\n", " TextBlockType,\n", " inputs=TensorDict(\n", " {\n", " \"text_input\": torch.tensor(text_tokens).reshape(1, 1, -1),\n", " }\n", " ),\n", " )\n", "\n", "text_block = make_text_block(text)\n", "\n", "sem_block = Block(\n", " CausalSemanticBlockType,\n", " inputs=TensorDict(\n", " {\n", " \"semantic_input\": torch.full((1, 1, 1), cfg.semantic_infer_token),\n", " }\n", " ),\n", ")\n", "\n", "no_text_blocks = BlockSequence([sem_block])\n", "blocks = BlockSequence([text_block, sem_block])\n", "\n", "T = 25 * 60\n", "\n", "codes = run_inference(\n", " BlockSequence([text_block, sem_block]), max_steps=1000, return_raw=True, text_cfg_boost=0.3\n", ")\n", "run_diffusion(codes, lyrics=lyrics).play()" ] }, { "cell_type": "markdown", "id": "463a1e8a", "metadata": {}, "source": [ "# sample + stem gen" ] }, { "cell_type": "markdown", "id": "1ee0c962", "metadata": {}, "source": [ "## get a metal guitar sample" ] }, { "cell_type": "code", "execution_count": null, "id": "ba7ca2fc", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio\n", "\n", "a = Audio.from_file(\"/home/georg/notebooks/samples/halo.wav\", sample_rate=44_100, n_channels=2)\n", "a_vocals, a_other = split_vocals(a.convert(44_100, 2, 2))" ] }, { "cell_type": "code", "execution_count": null, "id": "ce4f2f18", "metadata": {}, "outputs": [], "source": [ "a_other_segment = a_other.stereo()\n", "pooled_halo_arr, halo_arr = make_semantics(a_other_segment)\n", "print(len(pooled_halo_arr), len(halo_arr))\n", "stem_list = [cfg.semantic_stem_token] + halo_arr" ] }, { "cell_type": "code", "execution_count": null, "id": "554141eb", "metadata": {}, "outputs": [], "source": [ "a_other_segment.resample(48000).play()" ] }, { "cell_type": "code", "execution_count": null, "id": "405772f2", "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "{add metal guitar;activity>80%}\n", "\"\"\"\n", "text_block = make_text_block(text)\n", "stem_block = Block(\n", " spec=CausalSemanticBlockType,\n", " inputs={\n", " \"semantic_input\": torch.tensor(stem_list).reshape(1, 1, -1),\n", " },\n", ")\n", "\n", "sem_block = Block(\n", " CausalSemanticBlockType,\n", " inputs=TensorDict(\n", " {\n", " \"semantic_input\": torch.full((1, 1, 1), cfg.semantic_infer_token),\n", " }\n", " ),\n", ")\n", "\n", "out = run_inference(\n", " BlockSequence([text_block, stem_block, sem_block]), max_steps=6000, text_cfg_boost=0.5\n", ")\n", "\n", "mix = (out + a_other_segment.resample(48000)).normalize_volume()\n", "out.play()\n", "mix.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "21adc74a", "metadata": {}, "outputs": [], "source": [ "out.get_slice(from_s=46, to_s=55).play()" ] }, { "cell_type": "code", "execution_count": null, "id": "6fa98520", "metadata": {}, "outputs": [], "source": [ "out.get_slice(from_s=46, to_s=55).write_mp3(\"/home/vibert/logs/2025-08-26/metal_guitar.mp3\")" ] }, { "cell_type": "markdown", "id": "c58c3290", "metadata": {}, "source": [ "## use sample + stem gen" ] }, { "cell_type": "code", "execution_count": null, "id": "b47b4b06", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio\n", "\n", "metal_guitar = \"/home/vibert/logs/2025-08-26/metal_guitar.mp3\"\n", "\n", "metal_guitar_sample = Audio.from_file(metal_guitar)\n", "\n", "metal_guitar_sample.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "8649d485", "metadata": {}, "outputs": [], "source": [ "_, metal_guitar_semantic = make_semantics(metal_guitar_sample)\n", "\n", "\n", "metal_guitar_sample_block = Block(\n", " SampleBlockType,\n", " inputs=TensorDict({\n", " \"semantic_input\": torch.tensor([cfg.semantic_sample_token] + metal_guitar_semantic[:250]).reshape(1, 1, -1),\n", " }),\n", ")" ] }, { "cell_type": "markdown", "id": "ac7fed26", "metadata": {}, "source": [ "## without CFG" ] }, { "cell_type": "code", "execution_count": null, "id": "10a325e1", "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "{add metal guitar;activity>80%;audio_sample_stem_0;audio_sample_time_0:46}\n", "\"\"\"\n", "text_block = make_text_block(text)\n", "stem_block = Block(\n", " spec=CausalSemanticBlockType,\n", " inputs={\n", " \"semantic_input\": torch.tensor(stem_list).reshape(1, 1, -1),\n", " },\n", ")\n", "\n", "sem_block = Block(\n", " CausalSemanticBlockType,\n", " inputs=TensorDict(\n", " {\n", " \"semantic_input\": torch.full((1, 1, 1), cfg.semantic_infer_token),\n", " }\n", " ),\n", ")\n", "\n", "sample2stem_out = run_inference(\n", " BlockSequence([text_block, metal_guitar_sample_block, stem_block, sem_block]), max_steps=6000, text_cfg_boost=0.5\n", ")\n", "\n", "sample2stem_mix = (sample2stem_out + a_other_segment.resample(48000)).normalize_volume()\n", "sample2stem_out.play()\n", "sample2stem_mix.play()" ] }, { "cell_type": "markdown", "id": "9f7bfd01", "metadata": {}, "source": [ "## with CFG" ] }, { "cell_type": "code", "execution_count": null, "id": "46abdd65", "metadata": {}, "outputs": [], "source": [ "text = \"\"\"\n", "{add metal guitar;activity>80%;audio_sample_stem_0;audio_sample_time_0:46}\n", "\"\"\"\n", "text_block = make_text_block(text)\n", "stem_block = Block(\n", " spec=CausalSemanticBlockType,\n", " inputs={\n", " \"semantic_input\": torch.tensor(stem_list).reshape(1, 1, -1),\n", " },\n", ")\n", "\n", "sem_block = Block(\n", " CausalSemanticBlockType,\n", " inputs=TensorDict(\n", " {\n", " \"semantic_input\": torch.full((1, 1, 1), cfg.semantic_infer_token),\n", " }\n", " ),\n", ")\n", "\n", "text_cfg_boost = 0.3\n", "audio_sample_cfg_boost = 0.5\n", "max_steps = 1000\n", "\n", "blocks = BlockSequence([text_block, metal_guitar_sample_block, stem_block, sem_block])\n", "no_text_blocks = BlockSequence([metal_guitar_sample_block, stem_block, sem_block])\n", "no_sample_blocks = BlockSequence([text_block, stem_block, sem_block])\n", "prompts = [\n", " (1 + text_cfg_boost + audio_sample_cfg_boost, blocks),\n", " (-text_cfg_boost, no_text_blocks),\n", " (-audio_sample_cfg_boost, no_sample_blocks),\n", "]\n", "gconf = BCTGenerationConfig(\n", " prompts,\n", " max_autoregressive_steps=6000,\n", " eos_token=cfg.semantic_pad_token,\n", " # eos_token=None,\n", " temperature=0.9,\n", " # time_inputs=time_inputs,\n", " compile=False,\n", ")\n", "\n", "block = generate_block(model, gconf)\n", "all_sem_codes = block.inputs[\"semantic_input\"][0, 0, 1 : len(block)]\n", "\n", "sample2stem_cfg_out = run_diffusion(all_sem_codes)\n", "\n", "mix = (sample2stem_cfg_out + a_other_segment.resample(48000)).normalize_volume()\n", "sample2stem_cfg_out.play()\n", "mix.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "da4945b7", "metadata": {}, "outputs": [], "source": [ "sample2stem_cfg_out\n" ] }, { "cell_type": "code", "execution_count": null, "id": "b8414d71", "metadata": {}, "outputs": [], "source": [ "loud_sample2stem_cfg_out, gain = sample2stem_cfg_out.apply_gain(2)\n", "loud_mix = (loud_sample2stem_cfg_out + a_other_segment.resample(48000)).normalize_volume()\n", "loud_mix.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "38b36ec9", "metadata": {}, "outputs": [], "source": [ "sample2stem_cfg_out.write_mp3(\"/home/vibert/logs/2025-08-26/sample2stem_cfg_out.mp3\")\n", "loud_mix.write_mp3(\"/home/vibert/logs/2025-08-26/sample2stem_cfg_mix.mp3\")" ] }, { "cell_type": "code", "execution_count": null, "id": "352aa6a4", "metadata": {}, "outputs": [], "source": [ "sample2stem_mix.write_mp3(\"/home/vibert/logs/2025-08-26/sample2stem_mix_2.mp3\")" ] }, { "cell_type": "markdown", "id": "d6cfc6fd", "metadata": {}, "source": [ "# Sample2song fart" ] }, { "cell_type": "code", "execution_count": null, "id": "fd586ca3", "metadata": {}, "outputs": [], "source": [ "text = \"\"\" \n", "{start_offset:0;audio_sample_stem_0;audio_sample_time_0:0}\n", "\n", "[hip hop]\n", "\n", "[verse]\n", "oh, my love\n", "My friend you know\n", "it's been a while\n", "Without thinking of you\n", "but the thought makes me smile\n", "\n", "[chorus]\n", "I'm so tired of wanting\n", "wanting more than this\n", "i know it but what am i to do\n", "i need some space to breathe,\n", "so give me some room\n", "\n", "[verse]\n", "oh, my love\n", "you have a heart of stone\n", "cause since i've come home\n", "i've never felt so alone\n", "but the thought makes me smile\n", "\"\"\"\n", "text_block = make_text_block(text)\n", "fart_sample_fp = \"/home/vibert/data/sample2song/fart.mp3\"\n", "fart_sample_audio = Audio.from_file(fart_sample_fp)\n", "_, fart_semantic = make_semantics(fart_sample_audio)\n", "fart_sample_block = Block(\n", " SampleBlockType,\n", " inputs=TensorDict({\n", " \"semantic_input\": torch.tensor([cfg.semantic_sample_token] + fart_semantic[:100]).reshape(1, 1, -1),\n", " }),\n", ")\n", "sem_block = Block(\n", " CausalSemanticBlockType,\n", " inputs=TensorDict({\n", " \"semantic_input\": torch.full((1, 1, 1), cfg.semantic_infer_token),\n", " }),\n", ")\n", "blocks = BlockSequence([text_block, fart_sample_block, sem_block])\n", "out = run_inference(blocks, max_steps=1000, text_cfg_boost=0.8)\n", "fart_sample_audio.play()\n", "out.play()" ] }, { "cell_type": "markdown", "id": "c6a094cf", "metadata": {}, "source": [ "# Sample2song fart (CFG enhanced)" ] }, { "cell_type": "code", "execution_count": null, "id": "cbfdcdd2", "metadata": {}, "outputs": [], "source": [ "text = \"\"\" \n", "{start_offset:0;audio_sample_stem_0;audio_sample_time_0:2}\n", "\n", "[experimental hip hop]\n", "\n", "[verse]\n", "oh, my love\n", "My friend you know\n", "it's been a while\n", "Without thinking of you\n", "but the thought makes me smile\n", "\n", "[chorus]\n", "I'm so tired of wanting\n", "wanting more than this\n", "i know it but what am i to do\n", "i need some space to breathe,\n", "so give me some room\n", "\n", "[verse]\n", "oh, my love\n", "you have a heart of stone\n", "cause since i've come home\n", "i've never felt so alone\n", "but the thought makes me smile\n", "\"\"\"\n", "text_block = make_text_block(text)\n", "fart_sample_fp = \"/home/vibert/data/sample2song/fart.mp3\"\n", "fart_sample_audio = Audio.from_file(fart_sample_fp)\n", "_, fart_semantic = make_semantics(fart_sample_audio)\n", "fart_sample_block = Block(\n", " SampleBlockType,\n", " inputs=TensorDict({\n", " \"semantic_input\": torch.tensor([cfg.semantic_sample_token] + fart_semantic[:50]).reshape(1, 1, -1),\n", " }),\n", ")\n", "sem_block = Block(\n", " CausalSemanticBlockType,\n", " inputs=TensorDict({\n", " \"semantic_input\": torch.full((1, 1, 1), cfg.semantic_infer_token),\n", " }),\n", ")\n", "blocks = BlockSequence([text_block, fart_sample_block, sem_block])\n", "\n", "\n", "text_cfg_boost = 0.2\n", "audio_sample_cfg_boost = 0.8\n", "max_steps = 1000\n", "\n", "# Refactored to allow 3 entries in prompts when both boosts are active\n", "\n", "text_is_active = text_cfg_boost != 0.0 and blocks[0].spec.name == \"text\"\n", "sample_is_active = audio_sample_cfg_boost != 0.0 and blocks[1].spec.name == \"sample\"\n", "\n", "if not text_is_active and not sample_is_active:\n", " prompts = [(1, blocks)]\n", "elif text_is_active and not sample_is_active:\n", " no_text_blocks = BlockSequence(blocks[1:])\n", " prompts = [(1 + text_cfg_boost, blocks), (-text_cfg_boost, no_text_blocks)]\n", "elif not text_is_active and sample_is_active:\n", " no_sample_blocks = BlockSequence([blocks[0]] + blocks[2:])\n", " prompts = [(1 + audio_sample_cfg_boost, blocks), (-audio_sample_cfg_boost, no_sample_blocks)]\n", "else:\n", " # Both boosts are active: 3 entries\n", " no_text_blocks = BlockSequence([fart_sample_block, sem_block])\n", " no_sample_blocks = BlockSequence([text_block, sem_block])\n", " prompts = [\n", " (1 + text_cfg_boost + audio_sample_cfg_boost, blocks),\n", " (-text_cfg_boost, no_text_blocks),\n", " (-audio_sample_cfg_boost, no_sample_blocks),\n", " ]\n", "\n", "gconf = BCTGenerationConfig(\n", " prompts,\n", " max_autoregressive_steps=max_steps,\n", " eos_token=cfg.semantic_pad_token,\n", " # eos_token=None,\n", " temperature=0.9,\n", " # time_inputs=time_inputs,\n", " compile=False,\n", ")\n", "\n", "block = generate_block(model, gconf)\n", "all_sem_codes = block.inputs[\"semantic_input\"][0, 0, 1 : len(block)]\n", "\n", "fart_cfg_out = run_diffusion(all_sem_codes)\n", "fart_cfg_out.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "b3cf6124", "metadata": {}, "outputs": [], "source": [ "fart_sample_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "f909f57a", "metadata": {}, "outputs": [], "source": [ "# {start_offset:0;audio_sample_stem_0;audio_sample_time_0:3}\n", "# {start_offset:0;audio_sample_stem_0}\n", "text = \"\"\" \n", "{start_offset:0;audio_sample_stem_0;audio_sample_time_0:1}\n", "\n", "[punk]\n", "\n", "[verse]\n", "oh, my love\n", "My friend like to fart (fart noise)\n", "it's been a while\n", "Without thinking of you\n", "but the thought makes me smile\n", "\n", "[chorus]\n", "I'm so tired of wanting\n", "wanting more than this\n", "i know it but what am i to do\n", "i need some space to breathe,\n", "so give me some room\n", "\n", "[verse]\n", "oh, my love\n", "you have a heart of stone\n", "cause since i've come home\n", "i've never felt so alone\n", "but the thought makes me smile\n", "\"\"\"\n", "text_block = make_text_block(text)\n", "fart_sample_fp = \"/home/vibert/data/sample2song/fart.mp3\"\n", "fart_sample_audio = Audio.from_file(fart_sample_fp)\n", "_, fart_semantic = make_semantics(fart_sample_audio)\n", "fart_sample_block = Block(\n", " SampleBlockType,\n", " inputs=TensorDict({\n", " \"semantic_input\": torch.tensor([cfg.semantic_sample_token] + fart_semantic[:50]).reshape(1, 1, -1),\n", " }),\n", ")\n", "sem_block = Block(\n", " CausalSemanticBlockType,\n", " inputs=TensorDict({\n", " \"semantic_input\": torch.full((1, 1, 1), cfg.semantic_infer_token),\n", " }),\n", ")\n", "blocks = BlockSequence([text_block, fart_sample_block, sem_block])\n", "\n", "\n", "text_cfg_boost = 0.3\n", "audio_sample_cfg_boost = 0.7\n", "max_steps = 1000\n", "\n", "# Refactored to allow 3 entries in prompts when both boosts are active\n", "\n", "text_is_active = text_cfg_boost != 0.0 and blocks[0].spec.name == \"text\"\n", "sample_is_active = audio_sample_cfg_boost != 0.0 and blocks[1].spec.name == \"sample\"\n", "\n", "if not text_is_active and not sample_is_active:\n", " prompts = [(1, blocks)]\n", "elif text_is_active and not sample_is_active:\n", " no_text_blocks = BlockSequence(blocks[1:])\n", " prompts = [(1 + text_cfg_boost, blocks), (-text_cfg_boost, no_text_blocks)]\n", "elif not text_is_active and sample_is_active:\n", " no_sample_blocks = BlockSequence([blocks[0]] + blocks[2:])\n", " prompts = [(1 + audio_sample_cfg_boost, blocks), (-audio_sample_cfg_boost, no_sample_blocks)]\n", "else:\n", " # Both boosts are active: 3 entries\n", " no_text_blocks = BlockSequence([fart_sample_block, sem_block])\n", " no_sample_blocks = BlockSequence([text_block, sem_block])\n", " prompts = [\n", " (1 + text_cfg_boost + audio_sample_cfg_boost, blocks),\n", " (-text_cfg_boost, no_text_blocks),\n", " (-audio_sample_cfg_boost, no_sample_blocks),\n", " ]\n", "\n", "gconf = BCTGenerationConfig(\n", " prompts,\n", " max_autoregressive_steps=max_steps,\n", " eos_token=cfg.semantic_pad_token,\n", " # eos_token=None,\n", " temperature=0.9,\n", " # time_inputs=time_inputs,\n", " compile=False,\n", ")\n", "\n", "block = generate_block(model, gconf)\n", "all_sem_codes = block.inputs[\"semantic_input\"][0, 0, 1 : len(block)]\n", "\n", "fart_cfg_out = run_diffusion(all_sem_codes)\n", "fart_cfg_out.play()" ] }, { "cell_type": "markdown", "id": "72acc8d1", "metadata": {}, "source": [ "# Misc" ] }, { "cell_type": "markdown", "id": "fb1067b3", "metadata": {}, "source": [ "## Test stem gen" ] }, { "cell_type": "code", "execution_count": null, "id": "55b284a2", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio\n", "\n", "a = Audio.from_file(\"/home/georg/notebooks/samples/halo.wav\", sample_rate=44_100, n_channels=2)\n", "a_vocals, a_other = split_vocals(a.convert(44_100, 2, 2))" ] }, { "cell_type": "code", "execution_count": null, "id": "28fa36ef", "metadata": {}, "outputs": [], "source": [ "from suno_utils.tasks.mert_25 import preload_models as preload_semantic_models_\n", "from suno_utils.diffusion.generation import encode_semantic\n", "from suno_utils.audio import Audio\n", "\n", "a_other_segment = a_other.stereo()\n", "\n", "# a = Audio.from_file(\"/home/georg/notebooks/samples/halo.wav\", sample_rate=44_100, n_channels=2)\n", "pooled_halo_arr, halo_arr = make_semantics(a_other_segment)\n", "print(len(pooled_halo_arr), len(halo_arr))\n", "stem_list = [cfg.semantic_stem_token] + halo_arr" ] }, { "cell_type": "code", "execution_count": null, "id": "6da70199", "metadata": {}, "outputs": [], "source": [ "a_other_segment.resample(48000).play()" ] }, { "cell_type": "code", "execution_count": null, "id": "97a941a6", "metadata": {}, "outputs": [], "source": [ "start_s = 40\n", "end_s = start_s + 10\n", "pre_audio = Audio.from_silence(start_s, sample_rate=44_100)\n", "post_audio = Audio.from_silence(a_other_segment.duration_s - end_s, sample_rate=44_100)\n", "\n", "pooled_history_arr, history_arr = make_semantics(pre_audio)\n", "pooled_future_arr, future_arr = make_semantics(post_audio)\n", "\n", "\n", "pre_list = [cfg.semantic_history_token] + history_arr\n", "post_list = [cfg.semantic_future_token] + future_arr\n", "pre_audio.play()\n", "post_audio.play()\n", "\n", "pre_block = Block(\n", " spec=CausalSemanticBlockType,\n", " inputs=TensorDict(\n", " {\n", " \"semantic_input\": torch.tensor(pre_list).reshape(1, 1, -1),\n", " }\n", " ),\n", ")\n", "post_block = Block(\n", " spec=CausalSemanticBlockType,\n", " inputs=TensorDict(\n", " {\n", " \"semantic_input\": torch.tensor(post_list).reshape(1, 1, -1),\n", " }\n", " ),\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "175a1c02", "metadata": {}, "outputs": [], "source": [ "# extract vocals\n", "text = \"\"\"\n", "{add Vocals}\n", "\n", "[pop]\n", "\n", "[verse]\n", "oh, my love\n", "My friend you know\n", "it's been a while\n", "Without thinking of you\n", "but the thought makes me smile\n", "\n", "[chorus]\n", "I'm so tired of wanting\n", "wanting more than this\n", "i know it but what am i to do\n", "i need some space to breathe,\n", "so give me some room\n", "\n", "[verse]\n", "oh, my love\n", "you have a heart of stone\n", "cause since i've come home\n", "i've never felt so alone\n", "but the thought makes me smile\n", "\"\"\"\n", "text_block = make_text_block(text)\n", "extract_text_block = make_text_block(\"{add Violin;activity>80%}\")\n", "stem_block = Block(\n", " spec=CausalSemanticBlockType,\n", " inputs={\n", " \"semantic_input\": torch.tensor(stem_list).reshape(1, 1, -1),\n", " },\n", ")\n", "# stem_blocks = BlockSequence([text_block, stem_block, pre_block, post_block, sem_block])\n", "# vocals = run_inference(stem_blocks, max_steps=T, text_cfg_boost=0.0)\n", "# vocals.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "c9a50883", "metadata": {}, "outputs": [], "source": [ "stem_blocks = BlockSequence([text_block, stem_block, pre_block, post_block, sem_block])\n", "vocals = run_inference(stem_blocks, max_steps=T, text_cfg_boost=0.0)\n", "vocals.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "0c9b46bb", "metadata": {}, "outputs": [], "source": [ "out = run_inference(\n", " BlockSequence([text_block, stem_block, sem_block]), max_steps=6000, text_cfg_boost=0.5\n", ")\n", "out.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "eafb80e1", "metadata": {}, "outputs": [], "source": [ "mix = (out + a_other_segment.resample(48000)).normalize_volume()\n", "mix.write_mp3(\"stems_test.mp3\")\n", "mix.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "037f08c8", "metadata": {}, "outputs": [], "source": [ "pwd" ] } ], "metadata": { "kernelspec": { "display_name": "suno_3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" } }, "nbformat": 4, "nbformat_minor": 5 }