{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "0e09c52f-da2a-4a21-b659-b58383cba6f4", "metadata": {}, "outputs": [], "source": [ "# !nvidia-smi\n", "# !echo $HOSTNAME\n", "\n", "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"6\"\n", "\n", "import sys\n", "#sys.path.insert(0, \"/home/georg/code/neon/sunoDiff/\")\n", "sys.path.insert(0, \"/home/tony/Work/neon/sunoDiff/\")" ] }, { "cell_type": "code", "execution_count": null, "id": "c046f7b4", "metadata": {}, "outputs": [], "source": [ "import torch\n", "from suno_utils.audio import Audio\n", "from generation import preload_models, generate, _retrieve_models\n", "CKPT_DIR = \"/home/christian/code/neon/stable-audio-tools/harmonai_train/\"\n", "_ = preload_models(\n", " tokenizer_filepath=\"/app/suno/data/dpo/models/tokenizer_60k.json\",\n", " semantic_model_filepath=\"/app/suno/data/dpo/models/mert_25.pt\",\n", " semantic_clusters_filepath=\"/app/suno/data/dpo/models/mert_25_2x4k.npy\",\n", " weights_precision=torch.bfloat16,\n", " compile=True,\n", " ### 25hz ###\n", " # codec_filepath=\"/home/georg/notebooks/gpu_nb/tmp/25hz_vae_peaq_kl_0.005.pth\",\n", " codec_filepath=\"/app/suno/data/dpo/models/dac_vae_tuned_25hz.pth\",\n", " ### 100hz ###\n", " # codec_filepath=\"/home/georg/notebooks/gpu_nb/tmp/100hz_vae_peaq_kl_0.005.pth\",\n", " \n", " # 30s (overlap newest)\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-10-17_04-15-19_s4176/last_ckpt_infer.pt\",\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-10-11_16-53-57_s8050/last_ckpt_infer.pt\", # default\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-10-27_00-05-50_s2159/last_ckpt_infer.pt\", # artificial data dpo\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-10-26_04-39-23_s121/last_ckpt_infer.pt\", # default check point\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-11-25_03-10-14_s4910/last_ckpt_infer.pt\", # quality model\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-11-23_14-22-16_s994/last_ckpt_infer.pt\", # \n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-11-30_09-50-23_s5271/step_3000_infer.pt\", # t1\n", " # dit_model_filepath=\"/app/suno/checkpoints/2024-12-10_19-53-58_s7792/step_3000_infer.pt\", # t3\n", " # dit_model_filepath=\"/app/suno/checkpoints/2025-02-20_21-37-57_s6971/step_3000_infer.pt\", # t5\n", " # dit_model_filepath=\"/app/suno/checkpoints/2025-03-04_05-30-37_s7677/step_30000_infer.pt\", # step_5000_infer.pt\",\n", " # dit_model_filepath=\"/app/suno/data/dpo/models/diff_v2_2b_2mil_ft_v0.pt\", # reference diff v2\n", " # dit_model_filepath=\"/app/suno/data/dpo/models/diff_v2_2b_2mil_ft_infill_20250421_v2.pt\", # t16\n", " dit_model_filepath=\"/app2/suno/checkpoints/2025-09-08_02-48-27_s3202/last_ckpt_infer.pt\",\n", " codec_scale_factor=0.4\n", ")\n", "models = _retrieve_models()\n", "model_duration_s = 30\n", "if models[\"dit_model\"].ctx_len is not None:\n", " model_duration_s = 6 * 60\n", "else:\n", " model_duration_s = models[\"dit_model\"].block_size // models[\"dit_model\"].io_hz\n", "duration_s = 2*60 if model_duration_s >= 2*60 else 30" ] }, { "cell_type": "code", "execution_count": null, "id": "cf696991", "metadata": { "scrolled": true }, "outputs": [], "source": [ "import json\n", "import numpy as np\n", "from suno_utils.utils.s3 import read_from_s3\n", "\n", "#gen_id = \"99bffa17-7e59-47b4-a048-5528cbda05d5\" # sister\n", "#gen_id = \"081d73c4-7805-4212-9c80-8db1137ca3c4\" # friends\n", "#gen_id = \"562f762d-6ced-4080-9af1-910ee3d0a5dc\" # something real\n", "#gen_id = \"23c15c62-494d-422d-8a60-8b0454044322\" # rubber duck\n", "#gen_id = \"4b140a9e-964b-422c-85b5-5861ad1a9d38\" # once\n", "#gen_id = \"7b214347-fa38-4e9b-96f4-f7ec65adea45\" # rock n roll\n", "# gen_id = \"a5e2198a-f352-4abb-9a24-7f81b143ded3\" # stone\n", "#gen_id = \"7f5e7819-b7a1-449d-a76c-9c38950c4274\" # canada\n", "#gen_id = \"27135940-957f-4152-9e40-72e5c10a46bd\" # drunk\n", "#gen_id = \"caf1d7ae-ce81-4c75-b737-fd41e47a38ba\" # linger\n", "#gen_id = \"ac5aa29a-f18f-406f-a904-c47493f1beec\" # linger vox\n", "#gen_id = \"b04719c3-012a-4a4a-9b02-3cb2875d1744\" # courtney\n", "#gen_id = \"7407c7b3-8aa2-4f0a-82e8-fa19933f6e94\" # opera\n", "#gen_id = \"5b9624ef-b416-4095-a5f3-4945f09bba86\" # butterflies\n", "#gen_id = \"e757aa19-5c33-4945-aaa1-1519b7890f0b\" # bubblegum\n", "#gen_id = \"7f774078-1672-4858-a37f-acad373c5a84\" # golden\n", "# gen_id = \"1cee79db-a0ee-44e4-9646-d312cd620997\" # chinese\n", "# gen_id = \"3b1830d7-5553-4785-be0b-ffeadf7756d8\" # debug\n", "# gen_id_audio = gen_id\n", "# gen_id = \"6845dc51-71b1-491c-b0be-1f6c691a2f42\"\n", "\n", "gen_id = \"ae137e1f-7562-48f6-a11c-b125e645282c\"\n", "gen_id_audio = \"cb105e06-6e20-4acc-9d1c-e7c9b36b24e2\"\n", "# gen_id_audio = gen_id\n", "\n", "s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n", "mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id_audio}.mp3\"\n", "print(s3_filepath)\n", "data = read_from_s3(s3_filepath, read_f=np.load)\n", "\n", "audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "\n", "if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", "elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", "elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", "else:\n", " raise ValueError(\"No codes found\")\n", "\n", "text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n", "aligned_lyrics = json.loads(text_data)\n", "# print(aligned_lyrics)\n", "# lyrics = \"\".join(w[\"word\"] for w in aligned_lyrics if \"word\" in w)\n", "lyrics = \"walking down the streets feeling so alive i've got my head in the clouds got a gleam in my eye every step i take it's like a brand new start no matter where i'm going i'll always find my part life is like a hard wire act we're dancing in the sky no need to worry no need to ask why with a little bit of courage we can chase our dreams no matter what comes our way we'll always be a team we're unstoppable yeah\"\n", "audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "id": "9b6e8953", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "np.random.seed(42)\n", "# seeds = np.random.randint(0, 2**32 - 1, 3)\n", "seeds = list(range(10))\n", "print(seeds)\n", "sem_codes = torch.tensor(codes[:, 0]).to(torch.long)" ] }, { "cell_type": "code", "execution_count": null, "id": "c0c34cd3", "metadata": {}, "outputs": [], "source": [ "steps = 10\n", "text_cfg_coef = 2.0\n", "ctx_cfg_coef = 1.0\n", "\n", "audios = []\n", "loudness = []\n", "for n in seeds:\n", " pred_audio = generate(\n", " sem_codes, \n", " lyrics=lyrics,\n", " # history_lyrics=history_text,\n", " # tags='shoegaze, ethereal, art, ego',\n", " tags='orchestral, orchestra, classical, epic movie',\n", " text_cfg_coef=text_cfg_coef, \n", " ctx_cfg_coef=ctx_cfg_coef,\n", " steps=steps,\n", " seed=n,\n", " semantic_skip_factor=1,\n", " downscale_ctx_vector=False,\n", " noise_ctx_vector=0.5,\n", " noise_ctx_vector_pad_size=25,\n", " ) #.normalize_volume()\n", " audios.append(pred_audio)\n", " pred_audio.play()\n", " loudness.append(pred_audio.loudness)\n", "# break" ] }, { "cell_type": "code", "execution_count": null, "id": "23820e6a-fd0c-4987-913e-18503a1ba898", "metadata": {}, "outputs": [], "source": [ "# 2024-12-05_03-42-43_s4538 -- t2\n", "# -14.392693014792687\n", "# 2025-01-26_02-13-54_s2704 -- t4\n", "# -14.534069602355155 3000 ckpt -- no cfg -14.754058349827892\n", "# 2025-03-20_17-24-50_s5157 -- new t7?\n", "# -10.780650125828082? 3000 ckpt -- no cfg -10.910366948410893\n", "# -10.697783175589027 6000 ckpt -- no cfg -10.806917115002616\n", "# -11.053680987364716 9000 ckpt\n", "# -10.806871747859812 18000 ckpt\n", "# -10.630577732025055 30000 ckpt\n", "# 2025-03-23_17-12-48_s8305\n", "# -11.550114522297708 3000 ckpt\n", "# -11.565421655789049 9000 ckpt\n", "print(f\"all {loudness}, \\n mean: {round(np.mean(loudness), 3)}, \\n median: {round(np.median(loudness), 3)}, \\n std: {round(np.std(loudness), 3)}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "3397e39c-4945-4499-a741-def247902db2", "metadata": {}, "outputs": [], "source": [ "BREAK" ] }, { "cell_type": "code", "execution_count": null, "id": "f04089cf-cfe6-4b70-bfa2-0a95c02d7e4a", "metadata": {}, "outputs": [], "source": [ "# for i in ([0, 1, 2]):\n", "# ref_audio = Audio.from_file(f\"/home/tony/Work/tony/audios/walking_down_diff_ft_1_ref_{i}.wav\")\n", "# ref_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "e47c4f72-1e8d-41d6-801b-f01d23c0e6e5", "metadata": {}, "outputs": [], "source": [ "seeds = list(range(3))" ] }, { "cell_type": "code", "execution_count": null, "id": "577de81a-4cea-45a2-af1f-2f83e1fcf86f", "metadata": {}, "outputs": [], "source": [ "gen_id = \"3b1830d7-5553-4785-be0b-ffeadf7756d8\" # debug\n", "gen_id_audio = gen_id\n", "\n", "s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n", "mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id_audio}.mp3\"\n", "print(s3_filepath)\n", "data = read_from_s3(s3_filepath, read_f=np.load)\n", "\n", "audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "\n", "if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", "elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", "elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", "else:\n", " raise ValueError(\"No codes found\")\n", "\n", "text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n", "aligned_lyrics = json.loads(text_data)\n", "# print(aligned_lyrics)\n", "lyrics = \"\".join(w[\"word\"] for w in aligned_lyrics if \"word\" in w)\n", "audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "id": "00e3d45c", "metadata": {}, "outputs": [], "source": [ "sem_codes = torch.tensor(codes[:, 0]).to(torch.long)\n", "audios = []\n", "for n in seeds:\n", " pred_audio = generate(\n", " sem_codes, \n", " lyrics=lyrics,\n", " # history_lyrics=history_text,\n", " # tags='shoegaze, ethereal, art, ego',\n", " tags='80s, new wave, female power',\n", " text_cfg_coef=text_cfg_coef, \n", " ctx_cfg_coef=ctx_cfg_coef,\n", " steps=steps,\n", " seed=n,\n", " semantic_skip_factor=1,\n", " downscale_ctx_vector=False,\n", " noise_ctx_vector=0.5,\n", " noise_ctx_vector_pad_size=25,\n", " ).normalize_volume()\n", " audios.append(pred_audio)\n", " pred_audio.play()\n", "# break" ] }, { "cell_type": "code", "execution_count": null, "id": "e4801d38-b3ae-451d-8f29-354bfe03129f", "metadata": {}, "outputs": [], "source": [ "gen_id = \"ebb4ce1d-8f79-4a12-8aa8-858bcc65b44a\" # debug\n", "gen_id_audio = gen_id\n", "\n", "s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n", "mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id_audio}.mp3\"\n", "print(s3_filepath)\n", "data = read_from_s3(s3_filepath, read_f=np.load)\n", "\n", "audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "\n", "if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", "elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", "elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", "else:\n", " raise ValueError(\"No codes found\")\n", "\n", "text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n", "aligned_lyrics = json.loads(text_data)\n", "# print(aligned_lyrics)\n", "lyrics = \"\".join(w[\"word\"] for w in aligned_lyrics if \"word\" in w)\n", "audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "id": "4291cffb-7ac4-475e-a164-65e2fe69e23a", "metadata": {}, "outputs": [], "source": [ "sem_codes = torch.tensor(codes[:, 0]).to(torch.long)\n", "audios = []\n", "for n in seeds:\n", " pred_audio = generate(\n", " sem_codes, \n", " lyrics=lyrics,\n", " # history_lyrics=history_text,\n", " # tags='shoegaze, ethereal, art, ego',\n", " tags='pop',\n", " text_cfg_coef=text_cfg_coef, \n", " ctx_cfg_coef=ctx_cfg_coef,\n", " steps=steps,\n", " seed=n,\n", " semantic_skip_factor=1,\n", " downscale_ctx_vector=False,\n", " noise_ctx_vector=0.5,\n", " noise_ctx_vector_pad_size=25,\n", " ).normalize_volume()\n", " audios.append(pred_audio)\n", " pred_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "88c27c31-1f29-4b1a-ba6b-c09bd5d2dd99", "metadata": {}, "outputs": [], "source": [ "gen_id = \"ce168306-b6af-4120-b938-9c2ada58af81\" # debug\n", "gen_id_audio = gen_id\n", "\n", "s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.npz\"\n", "mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id_audio}.mp3\"\n", "print(s3_filepath)\n", "data = read_from_s3(s3_filepath, read_f=np.load)\n", "\n", "audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "\n", "if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", "elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", "elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", "else:\n", " raise ValueError(\"No codes found\")\n", "\n", "text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n", "aligned_lyrics = json.loads(text_data)\n", "# print(aligned_lyrics)\n", "lyrics = \"\".join(w[\"word\"] for w in aligned_lyrics if \"word\" in w)\n", "audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": null, "id": "caf2a194-f022-43e7-9a00-060c9a9ccbd3", "metadata": {}, "outputs": [], "source": [ "sem_codes = torch.tensor(codes[:, 0]).to(torch.long)\n", "audios = []\n", "for n in seeds:\n", " pred_audio = generate(\n", " sem_codes, \n", " lyrics=lyrics,\n", " # history_lyrics=history_text,\n", " # tags='shoegaze, ethereal, art, ego',\n", " tags='pop',\n", " text_cfg_coef=text_cfg_coef, \n", " ctx_cfg_coef=ctx_cfg_coef,\n", " steps=steps,\n", " seed=n,\n", " semantic_skip_factor=1,\n", " downscale_ctx_vector=False,\n", " noise_ctx_vector=0.5,\n", " noise_ctx_vector_pad_size=25,\n", " ).normalize_volume()\n", " audios.append(pred_audio)\n", " pred_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "77725223", "metadata": {}, "outputs": [], "source": [ "# chunk_window_size = 3000\n", "\n", "# for n_chunk in range(8):\n", "# idx_pad = 5\n", "# chunk_start_s = n_chunk * chunk_window_size / models[\"dit_model\"].io_hz\n", "# chunk_end_s = (n_chunk + 1) * chunk_window_size / models[\"dit_model\"].io_hz\n", "# print(chunk_start_s, chunk_end_s)\n", "\n", "# chunk_start_indices = [\n", "# idx\n", "# for idx, m in enumerate(aligned_lyrics)\n", "# if \"start_s\" in m and m[\"start_s\"] >= chunk_start_s\n", "# ]\n", "# chunk_end_indices = [\n", "# idx for idx, m in enumerate(aligned_lyrics) if \"end_s\" in m and m[\"end_s\"] <= chunk_end_s\n", "# ]\n", "\n", "\n", "# if len(chunk_start_indices) > 0 and len(chunk_end_indices) > 0:\n", "# chunk_start_idx = max(0, chunk_start_indices[0] - idx_pad)\n", "# chunk_end_idx = min(len(aligned_lyrics), chunk_end_indices[-1] + idx_pad)\n", "# print(chunk_start_idx, chunk_end_idx)\n", "# lyrics_chunk = \"\".join(\n", "# [m[\"word\"] for m in aligned_lyrics[chunk_start_idx:chunk_end_idx]]\n", "# )\n", "# #lyrics_chunk = simplify_whitespace(lyrics_chunk, retain_newlines=True)\n", "# else:\n", "# lyrics_chunk = \"\"\n", "# print(lyrics_chunk)" ] }, { "cell_type": "markdown", "id": "441ef260", "metadata": {}, "source": [ "# Testbench" ] }, { "cell_type": "code", "execution_count": null, "id": "2115a8e4", "metadata": {}, "outputs": [], "source": [ "# import json\n", "# import numpy as np\n", "# from suno_utils.utils.s3 import read_from_s3\n", "# import shutil\n", "# output_dir = \"outputs/custom_infer_stable_audio-10102024\"\n", "# if os.path.exists(output_dir):\n", "# shutil.rmtree(output_dir)\n", "# os.makedirs(output_dir, exist_ok=True)\n", "\n", "# gen_ids = [\n", "# (\"99bffa17-7e59-47b4-a048-5528cbda05d5\", \"country\"), # sister\n", "# (\"081d73c4-7805-4212-9c80-8db1137ca3c4\", \"rock\"), # friends\n", "# (\"562f762d-6ced-4080-9af1-910ee3d0a5dc\", \"rock\"), # something real\n", "# (\"23c15c62-494d-422d-8a60-8b0454044322\", \"shanty\"), # rubber duck\n", "# (\"4b140a9e-964b-422c-85b5-5861ad1a9d38\", \"indie rock\"), # once\n", "# (\"7b214347-fa38-4e9b-96f4-f7ec65adea45\", \"rock n roll\"), # rock n roll\n", "# (\"7f5e7819-b7a1-449d-a76c-9c38950c4274\", \"pop\"), # canada\n", "# (\"27135940-957f-4152-9e40-72e5c10a46bd\", \"pop\"), # drunk\n", "# (\"caf1d7ae-ce81-4c75-b737-fd41e47a38ba\", \"pop\"), # linger\n", "# (\"ac5aa29a-f18f-406f-a904-c47493f1beec\", \"pop\"), # linger vox\n", "# (\"b04719c3-012a-4a4a-9b02-3cb2875d1744\", \"pop\"), # courtney\n", "# (\"7407c7b3-8aa2-4f0a-82e8-fa19933f6e94\", \"opera\"), # opera\n", "# (\"5b9624ef-b416-4095-a5f3-4945f09bba86\", \"math rock\"), # butterflies\n", "# ]\n", "\n", "# for gen_id, tags in gen_ids:\n", "# mp3_filepath = f\"s3://suno-data-uploads/studio/uploads/{gen_id}.mp3\"\n", "# print(tags, mp3_filepath)\n", "# audio = Audio.from_s3(mp3_filepath, n_channels=2)\n", "# audio.normalize_volume()\n", "# text_data = read_from_s3(f\"s3://suno-data-uploads/studio/uploads/{gen_id}_hoot.json\")\n", "# aligned_lyrics = json.loads(text_data)\n", "\n", "# steps = 32\n", "# text_cfg_coef = 4.0\n", "# ctx_cfg_coef = 2.0\n", "# seeds = np.random.randint(0, 2**32 - 1, 3)\n", "\n", "# for n in seeds:\n", "# pred_audio = generate(\n", "# audio, \n", "# aligned_lyrics=aligned_lyrics,\n", "# tags=tags, \n", "# text_cfg_coef=text_cfg_coef, \n", "# ctx_cfg_coef=ctx_cfg_coef,\n", "# steps=steps,\n", "# seed=n,\n", "# ).normalize_volume()\n", "\n", "# # save audio to disk\n", "# pred_audio.write_mp3(f\"{output_dir}/pred_{gen_id}_{n}.mp3\")" ] }, { "cell_type": "markdown", "id": "bb494e8a", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "2ce62b93", "metadata": {}, "outputs": [], "source": [ "# import json\n", "# with open(\"walking_aligned_lyrics.json\") as f:\n", "# aligned_lyrics = json.load(f)" ] }, { "cell_type": "code", "execution_count": null, "id": "75675270", "metadata": {}, "outputs": [], "source": [ "# # TODO: pad with silence for integer multiple of sliding\n", "# if models[\"dit_model\"].ctx_len is not None:\n", "# window_s = int(round((models[\"dit_model\"].block_size - models[\"dit_model\"].ctx_len) / models[\"dit_model\"].io_hz))\n", "# if audio.duration_s % window_s > 0.1:\n", "# audio = Audio.concatenate([\n", "# audio, \n", "# Audio.from_silence(window_s - audio.duration_s % window_s + 0.01, sample_rate=audio.sample_rate, n_channels=audio.n_channels)\n", "# ])" ] }, { "cell_type": "code", "execution_count": null, "id": "e2d70822", "metadata": {}, "outputs": [], "source": [ "# # get latents for preconditioning\n", "# import torch\n", "# from suno_utils.tasks.dac_vae_100hz_peaq import encode as codec_encode\n", "# # latent_audio = audio\n", "# latent_audio = Audio.from_file(\"test.mp3\", sample_rate=48_000, n_channels=2).get_segment(from_s=offset_s, to_s=offset_s+duration_s)\n", "# init_latents = torch.from_numpy(codec_encode(latent_audio).T[None]) * 2.5\n", "# latent_audio.play()" ] }, { "cell_type": "code", "execution_count": null, "id": "9c946bc7", "metadata": {}, "outputs": [], "source": [ "# import torch\n", "# models = _retrieve_models()\n", "# extra_args = torch.load(\"extra_args.pt\", weights_only=True)\n", "# torch.manual_seed(0)\n", "# x = torch.randn([1, 128, 100*30], device=\"cuda\")*50\n", "# with torch.no_grad():\n", "# out = models[\"dit_model\"].forward_inference(\n", "# x,\n", "# x.new_ones([x.shape[0]])*50,\n", "# cross_attn_cond=extra_args[\"cross_attn_cond\"],\n", "# empty_cross_attn_cond=extra_args[\"empty_cross_attn_cond\"],\n", "# cfg_scale=extra_args[\"cfg_scale\"]\n", "# )\n", "# print(out.shape)\n", "# out" ] }, { "cell_type": "code", "execution_count": null, "id": "818cf6a9", "metadata": {}, "outputs": [], "source": [ "# PAG\n", "# https://arxiv.org/pdf/2403.17377\n", "# https://huggingface.co/hyoungwoncho/sd_perturbed_attention_guidance/blob/main/pipeline.py\n", "# https://github.com/KU-CVLAB/Perturbed-Attention-Guidance?tab=readme-ov-file" ] }, { "cell_type": "code", "execution_count": null, "id": "abe70d19", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "56142e0c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7d56f346", "metadata": {}, "outputs": [], "source": [ "from generation import get_sigmas_polyexponential\n", "import math" ] }, { "cell_type": "code", "execution_count": null, "id": "ab1ba298-77db-4df9-a489-9697cc45fa33", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b12de9ec", "metadata": {}, "outputs": [], "source": [ "\n", "\n", "def sigma_to_t(sigma):\n", " return sigma.atan() / math.pi * 2" ] }, { "cell_type": "code", "execution_count": null, "id": "acda5fe8", "metadata": {}, "outputs": [], "source": [ "\n", "n_steps = 12\n", "sigmas = get_sigmas_polyexponential(n_steps, 0.5, 50, 1.0).to(torch.bfloat16)\n", "print([round(sigma_to_t(x).item(), 5) for x in sigmas])" ] }, { "cell_type": "code", "execution_count": null, "id": "b9bb1df7-157e-4754-b324-fd246c89ef00", "metadata": {}, "outputs": [], "source": [ "n_steps = 6\n", "sigmas = get_sigmas_polyexponential(n_steps, 0.5, 50, 1.0).to(torch.bfloat16)\n", "print([round(sigma_to_t(x).item(), 5) for x in sigmas])" ] }, { "cell_type": "code", "execution_count": null, "id": "e395745b-a704-44ce-b471-ac795d3b0f55", "metadata": {}, "outputs": [], "source": [ "n_steps = 8\n", "sigmas = get_sigmas_polyexponential(n_steps, 0.5, 50, 1.0).to(torch.bfloat16)\n", "print([round(sigma_to_t(x).item(), 5) for x in sigmas])" ] }, { "cell_type": "code", "execution_count": null, "id": "5dfafd26-b2f6-4d82-9d09-bcbead570fe4", "metadata": {}, "outputs": [], "source": [ "torch.randint(0, 6, (2,))" ] }, { "cell_type": "code", "execution_count": null, "id": "54f3d31a-ee9e-4bca-aa67-c7f14334878f", "metadata": {}, "outputs": [], "source": [ "torch.tensor([0.98828, 0.96875, 0.92188, 0.80469, 0.57031, 0.29492, 0.0]).to(torch.bfloat16)" ] }, { "cell_type": "code", "execution_count": null, "id": "b77a9385-4416-4d7d-848d-b579dd47eeda", "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "plt.plot([round(sigma_to_t(x).item(), 5) for x in sigmas], \"o-\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "id": "37f7e71f-c158-4dda-97c2-b702c1a1aef3", "metadata": {}, "outputs": [], "source": [ "rng = torch.quasirandom.SobolEngine(1, scramble=True, seed=0)" ] }, { "cell_type": "code", "execution_count": null, "id": "df997f1c-b142-4702-a4bb-288ad3e6184c", "metadata": {}, "outputs": [], "source": [ "rng.draw(2)[:, 0]" ] }, { "cell_type": "code", "execution_count": null, "id": "b11030ab-e054-499f-8f5a-88a7f693aad8", "metadata": {}, "outputs": [], "source": [ "import random\n", "torch.tensor(random.choices([0, 1, 2, 3], k=2))" ] }, { "cell_type": "code", "execution_count": null, "id": "cca0fb12-8cd7-419e-9a21-b13954577803", "metadata": {}, "outputs": [], "source": [ "t_steps_lookup = {1:[1], 2:[1, 2]}" ] }, { "cell_type": "code", "execution_count": null, "id": "d8076a9c-921f-41fb-9837-9f93770be68e", "metadata": {}, "outputs": [], "source": [ "random_steps = random.choice(list(t_steps_lookup.keys()))\n", "print(random_steps)\n", "t = torch.tensor(random.choices(t_steps_lookup[random_steps], k=2))\n", "t" ] }, { "cell_type": "code", "execution_count": null, "id": "47330994-67a5-4aac-a404-ecee64d7847a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "846ed640-188d-4a26-9b13-6441a466e49c", "metadata": {}, "outputs": [], "source": [ "\n", "from transformers import PreTrainedTokenizerFast\n", "tokenizer = PreTrainedTokenizerFast(\n", " tokenizer_file=\"/app/suno/data/dpo/models/tokenizer_60k.json\",\n", " unk_token=\"[UNK]\",\n", " pad_token=\"[PAD]\",\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "4a661f55-ca5a-408c-95fc-03a005b13a62", "metadata": {}, "outputs": [], "source": [ "input_text = \"{P: 0} [pop]\"" ] }, { "cell_type": "code", "execution_count": null, "id": "72b5b8c9-3536-4566-8ae6-f8b0a35eaea5", "metadata": {}, "outputs": [], "source": [ "tokenizer.encode_plus(\"{P: 0} [pop]\")" ] }, { "cell_type": "code", "execution_count": null, "id": "aa5e0e0d-4e29-4d87-8fa4-f93ec1048455", "metadata": {}, "outputs": [], "source": [ "tokenizer.encode_plus(\"{P: 1} [pop]\")" ] }, { "cell_type": "code", "execution_count": null, "id": "a22c4bc4-f23d-4dff-b5cf-055165facbbd", "metadata": {}, "outputs": [], "source": [ "for i in range(20):\n", " print(i, tokenizer.encode_plus(\"[P: \" + str(i) + \"] [pop]\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "a8ce4d2e-28d4-4704-b015-d27c444a3de8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0dca079e-160a-46f0-8089-e580c4ed9303", "metadata": {}, "outputs": [], "source": [ "a = list(range(10))" ] }, { "cell_type": "code", "execution_count": null, "id": "cb9c36a0-9e16-4175-8564-14d14186276b", "metadata": {}, "outputs": [], "source": [ "a[-1:]" ] }, { "cell_type": "code", "execution_count": null, "id": "067fd988-cbe9-49fb-9285-4be27d0c5fdd", "metadata": {}, "outputs": [], "source": [ "a[-0:]" ] }, { "cell_type": "code", "execution_count": null, "id": "94a3b77a-9c47-4509-ad0c-9fadfd160d64", "metadata": {}, "outputs": [], "source": [ "a[10:]" ] }, { "cell_type": "code", "execution_count": null, "id": "60ae89f3-ae9d-4ebb-9e51-1fbcb1fb015c", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" } }, "nbformat": 4, "nbformat_minor": 5 }