{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:17:19.057335Z", "iopub.status.busy": "2024-07-19T20:17:19.056997Z", "iopub.status.idle": "2024-07-19T20:17:19.994224Z", "shell.execute_reply": "2024-07-19T20:17:19.993754Z", "shell.execute_reply.started": "2024-07-19T20:17:19.057315Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "| NVIDIA-SMI 535.161.07 Driver Version: 535.161.07 CUDA Version: 12.2 |\n", "| 0 NVIDIA H100 80GB HBM3 On | 00000000:0F:00.0 Off | 0 |\n" ] } ], "source": [ "!nvidia-smi | grep NVIDIA | head -2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:17:19.995184Z", "iopub.status.busy": "2024-07-19T20:17:19.995025Z", "iopub.status.idle": "2024-07-19T20:17:20.105624Z", "shell.execute_reply": "2024-07-19T20:17:20.105088Z", "shell.execute_reply.started": "2024-07-19T20:17:19.995167Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "compute-h100-ord-node-13\n" ] } ], "source": [ "!echo $HOSTNAME" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:17:20.106510Z", "iopub.status.busy": "2024-07-19T20:17:20.106356Z", "iopub.status.idle": "2024-07-19T20:17:20.108776Z", "shell.execute_reply": "2024-07-19T20:17:20.108389Z", "shell.execute_reply.started": "2024-07-19T20:17:20.106494Z" } }, "outputs": [], "source": [ "# !aws s3 cp s3://suno-data/georg/tmp/model.pt /app/suno/tmp/2b.pt\n", "# !aws s3 cp s3://suno-data/georg/tmp/model_c25.pt /app/suno/tmp/2b_c25.pt\n", "# !aws s3 cp s3://suno-data/georg/tmp/model_s250c1.pt /app/suno/tmp/2b_s250c1.pt\n", "# !aws s3 cp s3://suno-data/georg/tmp/model_s250c5.pt /app/suno/tmp/2b_s250c5.pt\n", "# !aws s3 cp s3://suno-data/georg/tmp/model_s250c25.pt /app/suno/tmp/2b_s250c25.pt\n", "\n", "# !aws s3 cp s3://suno-data/georg/tmp/model_noSem_s250c25.pt /app/suno/tmp/2b_noSem_s250c25.pt" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:17:20.109465Z", "iopub.status.busy": "2024-07-19T20:17:20.109338Z", "iopub.status.idle": "2024-07-19T20:17:20.146192Z", "shell.execute_reply": "2024-07-19T20:17:20.145824Z", "shell.execute_reply.started": "2024-07-19T20:17:20.109451Z" } }, "outputs": [], "source": [ "# ## fix 13b\n", "# from collections import OrderedDict\n", "# import os\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"\n", "# import torch\n", "# sd = torch.load(\"/app/suno/checkpoints/2024-05-09_14-19-24/last_ckpt_infer.pt\")\n", "# sd[\"model_args\"][\"embed_scale_factor\"] = 1.0\n", "# sd[\"model_args\"][\"activation_f\"] = \"gelu\"\n", "# sd[\"model_args\"][\"use_qk_norm\"] = False\n", "# del sd[\"model_args\"][\"attention_type\"]\n", "# sd[\"model\"] = OrderedDict({k: v.to(torch.bfloat16) for k, v in sd[\"model\"].items()})\n", "# torch.save(sd, \"/app/suno/checkpoints/2024-05-09_14-19-24/last_ckpt_infer__fix.pt\")" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:17:20.147529Z", "iopub.status.busy": "2024-07-19T20:17:20.147263Z", "iopub.status.idle": "2024-07-19T20:17:20.182578Z", "shell.execute_reply": "2024-07-19T20:17:20.182206Z", "shell.execute_reply.started": "2024-07-19T20:17:20.147512Z" } }, "outputs": [], "source": [ "# !aws s3 cp /app/suno/checkpoints/2024-05-09_14-19-24/last_ckpt_infer__fix.pt s3://suno-data/georg/tmp/model_13b.pt" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:17:20.183234Z", "iopub.status.busy": "2024-07-19T20:17:20.183105Z", "iopub.status.idle": "2024-07-19T20:17:20.217819Z", "shell.execute_reply": "2024-07-19T20:17:20.217449Z", "shell.execute_reply.started": "2024-07-19T20:17:20.183220Z" } }, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"6\"" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:17:20.218484Z", "iopub.status.busy": "2024-07-19T20:17:20.218354Z", "iopub.status.idle": "2024-07-19T20:17:27.905282Z", "shell.execute_reply": "2024-07-19T20:17:27.904689Z", "shell.execute_reply.started": "2024-07-19T20:17:20.218470Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "WILL USE FLASH ATTN: True\n", "WARNING: feature_extractor_cqt requires the libray 'nnAudio'\n", "Failed to import Bio.\n" ] } ], "source": [ "from suno_utils.gpt.chirp_v2_5 import (\n", " preload_codec_models,\n", " _get_model_if_needed,\n", " GenerationConfig,\n", " codec_decode_stream_to_full_audio,\n", ")\n", "from suno_utils.gpt.generation_engine import (\n", " make_prompt,\n", " align_codes,\n", " make_request,\n", ")\n", "from suno_utils.gpt.engine import Engine, Request, Job\n", "from suno_utils.gpt.prompt import Prompt\n", "\n", "N_BATCH = 2\n", "MAX_STREAMS = N_BATCH * 4" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:17:27.907672Z", "iopub.status.busy": "2024-07-19T20:17:27.907536Z", "iopub.status.idle": "2024-07-19T20:19:13.546888Z", "shell.execute_reply": "2024-07-19T20:19:13.546196Z", "shell.execute_reply.started": "2024-07-19T20:17:27.907656Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "model loaded: /app/suno/data/dpo/models/model_13b_2h.pt\n", "model loaded: 11153.7M params\n", "GPTConfig(n_layer=40, n_head=40, n_kv_head=2, d_head=128, block_size=8832, bias=False, text_vocab_size=60032, text_codebook_size=60001, text_pad_token=60001, semantic_vocab_size=4032, semantic_codebook_size=4000, semantic_n_codebooks=1, semantic_pad_token=4000, semantic_infer_token=4001, semantic_mask_token=4002, semantic_cover_token=4003, semantic_artist_token=4004, semantic_future_token=4005, semantic_rate_hz=25, semantic_shift_factor=50, coarse_vocab_size=2112, coarse_codebook_size=2048, coarse_n_codebooks=12, coarse_pad_token=2048, coarse_infer_token=2049, coarse_rate_hz=25, coarse_shift_factor=5, t_text=2560, t_audio=6272, t_memmap=6016, use_sin_pos_emb=False, use_rotary_pos_emb=True, rope_theta=500000, attention_type='torch', use_qk_norm=True, activation_f='silu', embed_scale_factor=10.0, attention_sliding_window_size=-1, global_every_n_layers=1)\n" ] } ], "source": [ "_ = preload_codec_models(\"/app/suno/models/chirp_v2/dac_2c_25x12.pt\")\n", "gpt_ckpt_path = _get_model_if_needed(\n", "# \"s3://suno-data/georg/checkpoints/chirp_v2_5/7b_ft.pt\"\n", "# \"/app/suno/checkpoints/2024-05-09_14-19-24/last_ckpt_infer__fix.pt\" # 13b\n", "# \"/app/suno/data/dpo/models/model_30b_150k.pt\" # 30b\n", "# \"/app/suno/checkpoints/2024-07-04_20-47-47/last_ckpt_infer.pt\" # 30b 2h\n", "# \"/app/suno/checkpoints/2024-07-04_20-47-47/last_ckpt_infer.pt\" # 30b 2h\n", "# \"/app/suno/checkpoints/2024-07-09_21-16-29/last_ckpt_infer.pt\" # 13b 2h ft dpo v3\n", "\"/app/suno/data/dpo/models/model_13b_2h.pt\"\n", ")\n", "engine = Engine(\n", " gpt_ckpt_path,\n", " \"/app/suno/models/chirp_v2/tokenizer_60k.json\",\n", " max_sequences=MAX_STREAMS,\n", " compile=False,\n", ")\n", "model = engine.model\n", "cfg = model.config\n", "tokenizer = engine.tokenizer\n", "\n", "print(cfg)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Basic preds" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:19:13.548641Z", "iopub.status.busy": "2024-07-19T20:19:13.548359Z", "iopub.status.idle": "2024-07-19T20:19:13.552188Z", "shell.execute_reply": "2024-07-19T20:19:13.551674Z", "shell.execute_reply.started": "2024-07-19T20:19:13.548623Z" } }, "outputs": [], "source": [ "text = \"\"\"\n", "[Verse 1]\n", "Almost Heaven, West Virginia\n", "Blue Ridge Mountains, Shenandoah River\n", "Life is old there, older than the trees\n", "Younger than the mountains, growing like a breeze\n", "\n", "[Chorus]\n", "Country roads, take me home\n", "To the place I belong\n", "West Virginia, mountain mama\n", "Take me home, country roads\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", "# text_tags=\"dark epic trailer music, violin, vocalizations, female\",\n", "# text_tags=\"pop, power ballad, female, violin\",\n", " text_tags=\"bluegrass, female\",\n", "# text_tags=\"sea shanty, sea shanty, sea shanty\",\n", " \n", " cfg_coef=1.3,\n", " cfg_coef_tags=2,\n", " n_repeat_tags=3,\n", "\n", " # text_start_control_tags=\"{start;start:0;vocals:intro}\",\n", " # text_end_control_tags=\"{end}\",\n", " cfg_coef_neg_tags=-2,\n", " text_neg_tags=\"repetitive, loop\",\n", " \n", "# temp_semantic=0.85,\n", "# temp_coarse=0.9,\n", "# top_k_semantic=None,\n", "# top_k_coarse=None,\n", "# top_p_semantic=0.95,\n", "# top_p_coarse=0.98,\n", " \n", " n_batch=1,\n", " min_eos_p=0.1,\n", "# allow_eos=False,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=60,\n", ")\n", "\n", "# request = make_request(\"0\", gconf, cfg, tokenizer)\n", "# in_arr = request.streams[0].prompt\n", "# prompt = Prompt(gconf, cfg, tokenizer)\n", "# prompt.visualize(in_arr)\n", "# for n in range(4):\n", "# print(\"--\", n, \"--\")\n", "# print(tokenizer.decode(request.streams[n].prompt[0][:-1].tolist()))\n", "# print()" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:19:13.553031Z", "iopub.status.busy": "2024-07-19T20:19:13.552881Z", "iopub.status.idle": "2024-07-19T20:19:41.762696Z", "shell.execute_reply": "2024-07-19T20:19:41.762105Z", "shell.execute_reply.started": "2024-07-19T20:19:13.553015Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Adding 2 requests\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine add request: 0%| | 0/2 [00:00\n", " \n", " Your browser does not support the audio element.\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "# print(requests[0])\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "\n", "for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:50:45.633300Z", "iopub.status.busy": "2024-07-10T02:50:45.633133Z", "iopub.status.idle": "2024-07-10T02:51:11.999859Z", "shell.execute_reply": "2024-07-10T02:51:11.999207Z", "shell.execute_reply.started": "2024-07-10T02:50:45.633282Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Adding 2 requests\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine add request: 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 15.96it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Adding request 0 with 4 streams\n", "add_request releasing lock for request 0\n", "Adding request 1 with 4 streams\n", "add_request releasing lock for request 1\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 1065it [00:18, 58.66it/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 0 found EOS at step 1054\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 1167it [00:19, 58.76it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 0 completed at step 1159\n", "Job 0 completed\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 1287it [00:21, 59.01it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 1 found EOS at step 1277\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 1382it [00:23, 58.82it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 1 completed at step 1382\n", "Job 1 completed\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "# print(requests[0])\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "\n", "for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:51:12.000735Z", "iopub.status.busy": "2024-07-10T02:51:12.000580Z", "iopub.status.idle": "2024-07-10T02:51:12.004123Z", "shell.execute_reply": "2024-07-10T02:51:12.003701Z", "shell.execute_reply.started": "2024-07-10T02:51:12.000718Z" } }, "outputs": [], "source": [ "text = \"\"\"\n", "[Intro]\n", "I can feel the heat risin'\n", "Everything is on fire\n", "Today's a painful re-\n", "Minder of why\n", "It can only get brighter\n", "The further you put it behind ya\n", "But right now I'm on the\n", "Inside lookin' out, ‘cause—\n", "\n", "[Chorus]\n", "I'm standin' in the flames\n", "And it’s a beautiful kind of pain\n", "Settin' fire to yesterday\n", "To find the light, find the light, find the light\n", "Standin' in the flames\n", "And it’s a beautiful kind of pain\n", "Settin' fire to yesterday\n", "Find the light, find the light, find the light\n", "\n", "[Verse 1]\n", "Yesterday was the tornado warning, today's like the morning after\n", "Your world is torn in half, you wake in its wake\n", "To start the mourning process and rebuilding, you're still a work in progress\n", "Today's a whole new chapter, it's like an enormous asthma\n", "Thunderstorm has passed ya, you weathered it and poked its\n", "Eye out with the thornbush that ya used to smell the roses\n", "Stopped to inhale, can't even tell your nose is stuffed\n", "So focused on the bright side then you floor the gas pedal\n", "And hit the corner faster, more assertive, never looking back\n", "May hit the curb, but every day's a new learning curve as ya\n", "Steer through life, sometimes you might not wanna swerve but you have to\n", "To avert a disaster, lucky no permanent damage\n", "‘Cause they hurt you so ba\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " text_tags=\"Rap, In English, USA, Midwest Rap, Detroit Rap, Alternative, Hip-Hop\",\n", "\n", " cfg_coef=1.3,\n", " cfg_coef_tags=2,\n", " n_repeat_tags=3,\n", "\n", "# temp_semantic=0.8,\n", "# temp_coarse=1,\n", "# top_k_semantic=1000,\n", "# top_k_coarse=100,\n", "# top_p_semantic=None,\n", "# top_p_coarse=None,\n", " \n", "# text_start_control_tags=\"{start;start:0;vocals:intro}\",\n", "# text_end_control_tags=\"{end}\",\n", "# cfg_coef_neg_tags=-2,\n", "# text_neg_tags=\"repetitive, loop\",\n", "\n", " n_batch=1,\n", " min_eos_p=0.1,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", " max_gen_duration_s=60*2,\n", ")" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:51:12.004821Z", "iopub.status.busy": "2024-07-10T02:51:12.004684Z", "iopub.status.idle": "2024-07-10T02:51:53.643527Z", "shell.execute_reply": "2024-07-10T02:51:53.642885Z", "shell.execute_reply.started": "2024-07-10T02:51:12.004806Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Adding 2 requests\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine add request: 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 15.55it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Adding request 0 with 3 streams\n", "add_request releasing lock for request 0\n", "Adding request 1 with 3 streams\n", "add_request releasing lock for request 1\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 1644it [00:28, 58.51it/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 1 found EOS at step 1636\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 1752it [00:29, 58.86it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 1 completed at step 1741\n", "Job 1 completed\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 2088it [00:35, 58.94it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 0 found EOS at step 2078\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 2183it [00:37, 58.55it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 0 completed at step 2183\n", "Job 0 completed\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "requests = [\n", " make_request(f\"{i}\", gconf, engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "# print(requests[0])\n", "jobs = engine.run_request(requests, tqdm_enabled=True)\n", "\n", "for job in jobs:\n", " stream = engine.token_generator(job)\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Custom Input Array" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:19:41.763797Z", "iopub.status.busy": "2024-07-19T20:19:41.763518Z", "iopub.status.idle": "2024-07-19T20:19:43.287831Z", "shell.execute_reply": "2024-07-19T20:19:43.287308Z", "shell.execute_reply.started": "2024-07-19T20:19:41.763778Z" } }, "outputs": [], "source": [ "import numpy as np\n", "from suno_utils.audio import Audio\n", "from suno_utils.gpt.chirp_v2_5 import (\n", " codec_encode, codec_decode, preload_semantic_models, semantic_encode\n", ")\n", "_ = preload_semantic_models(\n", " checkpoint_filepath=\"/app/suno/models/chirp_v2/mert_25.pt\",\n", " centroids_filepath=\"/app/suno/models/chirp_v2/mert_25_2x4k.npy\",\n", " device=\"cpu\",\n", ")\n", "\n", "# # MusicFM\n", "# from suno_utils.tasks.musicfm_v2 import (\n", "# preload_models as preload_semantic_models, \n", "# encode as semantic_encode\n", "# )\n", "# _ = preload_semantic_models(\n", "# centroids_filepath=\"s3://suno-data/minz/models/musicfm_concat_centroids_2s_8192.npy\",\n", "# )\n", "\n", "def process_audio(audio, cfg):\n", " audio = audio.normalize_volume(-16)\n", " sem_arr = semantic_encode(audio, device=\"cpu\")\n", " coarse_arr = codec_encode(audio)\n", " n_frames = min(sem_arr.shape[0], coarse_arr.shape[0])\n", " sem_arr = sem_arr[:n_frames, : cfg.semantic_n_codebooks]\n", " coarse_arr = coarse_arr[:n_frames, : cfg.coarse_n_codebooks]\n", "\n", " a_arr = np.concatenate([sem_arr, coarse_arr], axis=-1)\n", " return a_arr\n", "\n", "def load_audio(fp):\n", " return Audio.from_file(fp, n_channels=2, sample_rate=48_000, byte_width=2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Continue" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:51:56.584614Z", "iopub.status.busy": "2024-07-10T02:51:56.584349Z", "iopub.status.idle": "2024-07-10T02:51:57.498647Z", "shell.execute_reply": "2024-07-10T02:51:57.498154Z", "shell.execute_reply.started": "2024-07-10T02:51:56.584596Z" } }, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "audio = Audio.from_file(\"audios/50cent.wav\", n_channels=2, sample_rate=48_000, byte_width=2) # 50 cent\n", "audio = audio.get_segment(from_s=24.99, to_s=45)\n", "# audio = Audio.from_file(\"tmp/eminem_doctor.wav\", n_channels=2, sample_rate=48_000, byte_width=2) # eminem doctor\n", "history_text = \"\"\"\n", "Welcome to the candy shop\n", "\n", "Yeah, uh-huh\n", "So seductive\n", "\n", "I take you to the candy shop\n", "I let you lick the lollipop\n", "Go 'head, girl, don't you stop\n", "\"\"\"\n", "text = \"\"\"\n", "Almost Heaven, West Virginia\n", "Blue Ridge Mountains, Shenandoah River\n", "Life is old there, older than the trees\n", "Younger than the mountains, growing like a breeze\n", "\n", "Country roads, take me home\n", "To the place I belong\n", "West Virginia, mountain mama\n", "Take me home, country roads\n", "\"\"\"\n", "in_history_arr = process_audio(audio, cfg)\n", "audio.play()" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:51:57.499494Z", "iopub.status.busy": "2024-07-10T02:51:57.499341Z", "iopub.status.idle": "2024-07-10T02:51:57.501756Z", "shell.execute_reply": "2024-07-10T02:51:57.501356Z", "shell.execute_reply.started": "2024-07-10T02:51:57.499477Z" } }, "outputs": [], "source": [ "# codec_decode(codec_encode(audio)).play()" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:51:57.502454Z", "iopub.status.busy": "2024-07-10T02:51:57.502310Z", "iopub.status.idle": "2024-07-10T02:52:18.365175Z", "shell.execute_reply": "2024-07-10T02:52:18.364503Z", "shell.execute_reply.started": "2024-07-10T02:51:57.502438Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Adding 2 requests\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine add request: 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 15.99it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Adding request 0 with 2 streams\n", "add_request releasing lock for request 0\n", "Adding request 1 with 2 streams\n", "add_request releasing lock for request 1\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 1105it [00:18, 60.06it/s] \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 0 completed\n", "Job 1 completed\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "gconf = GenerationConfig(\n", " text=text,\n", "# text_tags=\"bluegrass female\",\n", " history_arr=in_history_arr,\n", " history_text=history_text,\n", " n_batch=1,\n", " cfg_coef=1.2,\n", " cfg_coef_tags=0,\n", " max_gen_duration_s=40,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", "\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Oracle" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:52:18.366266Z", "iopub.status.busy": "2024-07-10T02:52:18.366084Z", "iopub.status.idle": "2024-07-10T02:52:18.368958Z", "shell.execute_reply": "2024-07-10T02:52:18.368456Z", "shell.execute_reply.started": "2024-07-10T02:52:18.366247Z" } }, "outputs": [], "source": [ "# audio = load_audio(\"tmp/50cent.wav\").get_segment(from_s=24.99, to_s=45)\n", "# text = \"\"\"\n", "# Welcome to the candy shop\n", "\n", "# Yeah, uh-huh\n", "# So seductive\n", "\n", "# I take you to the candy shop\n", "# I let you lick the lollipop\n", "# Go 'head, girl, don't you stop\n", "# \"\"\"\n", "# in_oracle_arr = process_audio(audio, cfg)\n", "# audio.play()" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:52:18.369836Z", "iopub.status.busy": "2024-07-10T02:52:18.369692Z", "iopub.status.idle": "2024-07-10T02:52:19.331738Z", "shell.execute_reply": "2024-07-10T02:52:19.331073Z", "shell.execute_reply.started": "2024-07-10T02:52:18.369820Z" } }, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "audio = load_audio(\"audios/halo.wav\").get_segment(from_s=49.99, to_s=70)\n", "text = \"\"\"\n", "It's like I've been awakened\n", "Every rule, I had you breakin'\n", "The risk that I'm takin'\n", "I'm never gonna shut you out\n", "\n", "Everywhere I'm lookin' now\n", "I'm surrounded by your embrace\n", "Baby, I can see your halo\n", "\"\"\"\n", "in_oracle_arr = process_audio(audio, cfg)\n", "# in_oracle_arr[:,:1] = cfg.semantic_mask_token\n", "audio.normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:52:19.332799Z", "iopub.status.busy": "2024-07-10T02:52:19.332624Z", "iopub.status.idle": "2024-07-10T02:52:19.335468Z", "shell.execute_reply": "2024-07-10T02:52:19.334951Z", "shell.execute_reply.started": "2024-07-10T02:52:19.332780Z" } }, "outputs": [], "source": [ "# codec_decode(codec_encode(audio)[:,:6]).normalize_volume().play()" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:52:19.338547Z", "iopub.status.busy": "2024-07-10T02:52:19.338088Z", "iopub.status.idle": "2024-07-10T02:52:30.623172Z", "shell.execute_reply": "2024-07-10T02:52:30.622616Z", "shell.execute_reply.started": "2024-07-10T02:52:19.338526Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Adding 2 requests\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine add request: 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 57.05it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Adding request 0 with 1 streams\n", "add_request releasing lock for request 0\n", "Adding request 1 with 1 streams\n", "add_request releasing lock for request 1\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 518it [00:08, 60.26it/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 0 found EOS at step 509\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 605it [00:09, 60.61it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 0 completed\n", "Job 1 completed\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "gconf = GenerationConfig(\n", " text=text,\n", "# text_tags=\"bluegrass female\",\n", " oracle_arr=in_oracle_arr[:,:1],\n", " n_batch=1,\n", " cfg_coef=1, #.3,\n", " cfg_coef_tags=0,\n", " cfg_coef_neg_tags=0,\n", " max_gen_duration_s=20,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", "\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Covers" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:19:43.288875Z", "iopub.status.busy": "2024-07-19T20:19:43.288611Z", "iopub.status.idle": "2024-07-19T20:19:44.875857Z", "shell.execute_reply": "2024-07-19T20:19:44.875116Z", "shell.execute_reply.started": "2024-07-19T20:19:43.288857Z" } }, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# audio = Audio.from_file(\"tmp/georg_country.mp3\", n_channels=2).get_segment(to_s=60.01)\n", "# audio = Audio.from_file(\"tmp/georg_bday.mp3\", n_channels=2).get_segment(to_s=60.01)\n", "audio = load_audio(\"audios/MurphysLawSchool_short_super.mp3\")\n", "audio.play()\n", "in_cover_arr = process_audio(audio, cfg)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T20:19:44.876906Z", "iopub.status.busy": "2024-07-19T20:19:44.876732Z", "iopub.status.idle": "2024-07-19T20:20:15.224007Z", "shell.execute_reply": "2024-07-19T20:20:15.223354Z", "shell.execute_reply.started": "2024-07-19T20:19:44.876889Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Adding 2 requests\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine add request: 0%| | 0/2 [00:00\n", " \n", " Your browser does not support the audio element.\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# text = \"\"\"\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# Happy birthday dear Margu\n", "# Happy birthday to you\n", "\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# Happy birthday dear Keenan\n", "# Happy birthday to you\n", "\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# Happy birthday dear Mikey\n", "# Happy birthday to you\n", "\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# Happy birthday dear Suno\n", "# Happy birthday to you\n", "\n", "# Happy birthday to you\n", "# Happy birthday to you\n", "# I don't know what I'm gonna do next\n", "# Happy birthday to you\n", "# \"\"\"\n", "\n", "# text = \"\"\"\n", "# [Verse 1]\n", "# Almost Heaven, West Virginia\n", "# Blue Ridge Mountains, Shenandoah River\n", "# Life is old there, older than the trees\n", "# Younger than the mountains, growing like a breeze\n", "\n", "# [Chorus]\n", "# Country roads, take me home\n", "# To the place I belong\n", "# West Virginia, mountain mama\n", "# Take me home, country roads\n", "\n", "# [Verse 2]\n", "# All my memories gather 'round her\n", "# Miner's lady, stranger to blue water\n", "# Dark and dusty, painted on the sky\n", "# Misty taste of moonshine, teardrop in my eye\n", "# \"\"\"\n", "\n", "text = \"\"\"\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " text_tags=\"orchestral\",\n", "# text_tags=\"pop, power ballad, female\",\n", " cover_arr=in_cover_arr,\n", " n_batch=1,\n", " cfg_coef=1.5,\n", " cfg_coef_tags=2,\n", " max_gen_duration_s=60,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "assert(N_BATCH * 4 <= 16)\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", "\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 54, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T18:52:52.730945Z", "iopub.status.busy": "2024-07-19T18:52:52.730526Z", "iopub.status.idle": "2024-07-19T18:52:52.942744Z", "shell.execute_reply": "2024-07-19T18:52:52.942054Z", "shell.execute_reply.started": "2024-07-19T18:52:52.730923Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/tony/Work/tony\n" ] } ], "source": [ "!pwd" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Artist" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:52:57.004741Z", "iopub.status.busy": "2024-07-10T02:52:57.004579Z", "iopub.status.idle": "2024-07-10T02:53:00.988952Z", "shell.execute_reply": "2024-07-10T02:53:00.988321Z", "shell.execute_reply.started": "2024-07-10T02:52:57.004723Z" } }, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "audio = Audio.from_file(\"audios/georg_country.mp3\", n_channels=2).get_segment(to_s=5.01)\n", "# audio = load_audio(\"tmp/halo.wav\").get_segment(from_s=49.99, to_s=70)\n", "audio.play()\n", "in_artist_arr = process_audio(audio, cfg)" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:53:00.989934Z", "iopub.status.busy": "2024-07-10T02:53:00.989763Z", "iopub.status.idle": "2024-07-10T02:53:21.556592Z", "shell.execute_reply": "2024-07-10T02:53:21.556084Z", "shell.execute_reply.started": "2024-07-10T02:53:00.989916Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Adding 2 requests\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine add request: 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 15.26it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Adding request 0 with 3 streams\n", "add_request releasing lock for request 0\n", "Adding request 1 with 3 streams\n", "add_request releasing lock for request 1\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 1105it [00:18, 60.03it/s] \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 0 completed\n", "Job 1 completed\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# text = \"\"\"\n", "# [Verse 1]\n", "# Almost Heaven, West Virginia\n", "# Blue Ridge Mountains, Shenandoah River\n", "# Life is old there, older than the trees\n", "# Younger than the mountains, growing like a breeze\n", "\n", "# [Chorus]\n", "# Country roads, take me home\n", "# To the place I belong\n", "# West Virginia, mountain mama\n", "# Take me home, country roads\n", "\n", "# [Verse 2]\n", "# All my memories gather 'round her\n", "# Miner's lady, stranger to blue water\n", "# Dark and dusty, painted on the sky\n", "# Misty taste of moonshine, teardrop in my eye\n", "# \"\"\"\n", "\n", "text = \"\"\"\n", "[Verse 1]\n", "Remember those walls I built?\n", "Well, baby, they're tumblin' down\n", "And they didn't even put up a fight\n", "They didn't even make a sound\n", "\n", "[Chorus]\n", "Everywhere I'm lookin' now\n", "I'm surrounded by your embrace\n", "Baby, I can see your halo\n", "You know you're my savin' grace\n", "\n", "[Verse 2]\n", "Hit me like a ray of sun\n", "Burnin' through my darkest night\n", "You're the only one that I want\n", "Think I'm addicted to your light\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " text_tags=\"bluegrass\",\n", " artist_arr=in_artist_arr,\n", " n_batch=1,\n", " cfg_coef=1.5,\n", " cfg_coef_tags=2,\n", " max_gen_duration_s=40,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "assert(N_BATCH * 4 <= 16)\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", "\n", " audio = codec_decode_stream_to_full_audio(align_codes(stream, cfg))\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Infill" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "execution": { "iopub.execute_input": "2024-07-18T23:23:22.692567Z", "iopub.status.busy": "2024-07-18T23:23:22.692226Z", "iopub.status.idle": "2024-07-18T23:23:23.813153Z", "shell.execute_reply": "2024-07-18T23:23:23.812457Z", "shell.execute_reply.started": "2024-07-18T23:23:22.692546Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "download: s3://suno-data-uploads/studio/uploads/378b4142-4a00-4913-90a5-dc988bd44d96.npz to ./378b4142-4a00-4913-90a5-dc988bd44d96.npz\n" ] } ], "source": [ "!aws s3 cp s3://suno-data-uploads/studio/uploads/378b4142-4a00-4913-90a5-dc988bd44d96.npz ." ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "execution": { "iopub.execute_input": "2024-07-18T23:24:04.002204Z", "iopub.status.busy": "2024-07-18T23:24:04.002035Z", "iopub.status.idle": "2024-07-18T23:24:04.004901Z", "shell.execute_reply": "2024-07-18T23:24:04.004460Z", "shell.execute_reply.started": "2024-07-18T23:24:04.002186Z" } }, "outputs": [], "source": [ "test_arr = np.load(\"378b4142-4a00-4913-90a5-dc988bd44d96.npz\")" ] }, { "cell_type": "code", "execution_count": 49, "metadata": { "execution": { "iopub.execute_input": "2024-07-18T23:26:01.104593Z", "iopub.status.busy": "2024-07-18T23:26:01.104021Z", "iopub.status.idle": "2024-07-18T23:26:01.107218Z", "shell.execute_reply": "2024-07-18T23:26:01.106764Z", "shell.execute_reply.started": "2024-07-18T23:26:01.104572Z" } }, "outputs": [], "source": [ "in_infill_arr = test_arr[\"v3.5_raw\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 50, "metadata": { "execution": { "iopub.execute_input": "2024-07-18T23:26:04.591513Z", "iopub.status.busy": "2024-07-18T23:26:04.591347Z", "iopub.status.idle": "2024-07-18T23:26:04.594100Z", "shell.execute_reply": "2024-07-18T23:26:04.593676Z", "shell.execute_reply.started": "2024-07-18T23:26:04.591496Z" } }, "outputs": [], "source": [ "# audio = Audio.from_file(\"tmp/georg_country.mp3\", n_channels=2).get_segment(to_s=5.01)\n", "# audio = load_audio(\"audios/country_road.mp3\").get_segment(from_s=0, to_s=20)\n", "# audio.play()\n", "# in_infill_arr = process_audio(audio, cfg)\n", "\n", "in_history_arr = in_infill_arr[:25*12,:].copy()\n", "in_future_arr = in_infill_arr[25*22:,:].copy()" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "execution": { "iopub.execute_input": "2024-07-18T23:26:05.499173Z", "iopub.status.busy": "2024-07-18T23:26:05.499018Z", "iopub.status.idle": "2024-07-18T23:26:05.502038Z", "shell.execute_reply": "2024-07-18T23:26:05.501632Z", "shell.execute_reply.started": "2024-07-18T23:26:05.499158Z" } }, "outputs": [ { "data": { "text/plain": [ "(300, 13)" ] }, "execution_count": 51, "metadata": {}, "output_type": "execute_result" } ], "source": [ "in_history_arr.shape" ] }, { "cell_type": "code", "execution_count": 52, "metadata": { "execution": { "iopub.execute_input": "2024-07-18T23:26:07.351566Z", "iopub.status.busy": "2024-07-18T23:26:07.351413Z", "iopub.status.idle": "2024-07-18T23:26:07.353766Z", "shell.execute_reply": "2024-07-18T23:26:07.353345Z", "shell.execute_reply.started": "2024-07-18T23:26:07.351550Z" } }, "outputs": [], "source": [ "import torch" ] }, { "cell_type": "code", "execution_count": 53, "metadata": { "execution": { "iopub.execute_input": "2024-07-18T23:26:07.524564Z", "iopub.status.busy": "2024-07-18T23:26:07.524419Z", "iopub.status.idle": "2024-07-18T23:26:15.831963Z", "shell.execute_reply": "2024-07-18T23:26:15.831357Z", "shell.execute_reply.started": "2024-07-18T23:26:07.524548Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Adding 2 requests\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine add request: 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 11.38it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Adding request 0 with 2 streams\n", "add_request releasing lock for request 0\n", "Adding request 1 with 2 streams\n", "add_request releasing lock for request 1\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 257it [00:04, 58.38it/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 0 found EOS at step 250\n", "Job 1 found EOS at step 250\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Engine step: 355it [00:06, 58.17it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Job 0 completed at step 355\n", "Job 0 completed\n", "Job 1 completed at step 355\n", "Job 1 completed\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "text = \"\"\"\n", "[Verse 1]\n", "Almost v4, West Virginia\n", "Blue Ridge Mountains, Shenandoah River\n", "Life is old there, older than the victor\n", "Younger than the mountains, growing like a breeze\n", "\n", "[Chorus]\n", "Country roads, take me home\n", "To the place I belong\n", "West Virginia, mountain mama\n", "Take me home, country roads\n", "\"\"\"\n", "\n", "gconf = GenerationConfig(\n", " text=text,\n", " history_arr=in_history_arr,\n", " future_arr=in_future_arr,\n", " n_batch=1,\n", " cfg_coef=1.3,\n", " cfg_coef_tags=0,\n", " max_gen_duration_s=15,\n", " min_text_offset=0,\n", " eos_pad_duration_s=0,\n", ")\n", "\n", "request = make_request(\"0\", gconf, cfg, tokenizer)\n", "in_arr = request.streams[0].prompt\n", "\n", "prompt = Prompt(gconf, cfg, tokenizer)\n", "\n", "assert(N_BATCH * 4 <= 16)\n", "requests = [\n", " make_request(f\"{i}\", gconf.modify(n_batch=1), engine.model.config, engine.tokenizer)\n", " for i in range(N_BATCH)\n", "]\n", "for job in engine.run_request(requests, tqdm_enabled=True):\n", " stream = engine.token_generator(job)\n", " audio = codec_decode(np.concatenate([\n", " in_history_arr[:,-12:],\n", " torch.stack(list(align_codes(stream, cfg))).detach().cpu().numpy()[:,-12:],\n", " in_future_arr[:,-12:],\n", " ], axis=0))\n", " audio.play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:53:38.632532Z", "iopub.status.busy": "2024-07-10T02:53:38.632367Z", "iopub.status.idle": "2024-07-10T02:53:38.634872Z", "shell.execute_reply": "2024-07-10T02:53:38.634471Z", "shell.execute_reply.started": "2024-07-10T02:53:38.632514Z" } }, "outputs": [], "source": [ "# import numpy as np\n", "# in_arr = requests[0].streams[0].prompt\n", "# generated = np.stack([t.cpu().numpy() for t in job.generated_tokens]).T\n", "# print(generated.shape)\n", "# arr = np.concatenate([in_arr, generated], axis=1)\n", "# prompt.visualize(arr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 30, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:53:38.635582Z", "iopub.status.busy": "2024-07-10T02:53:38.635436Z", "iopub.status.idle": "2024-07-10T02:53:38.677208Z", "shell.execute_reply": "2024-07-10T02:53:38.676807Z", "shell.execute_reply.started": "2024-07-10T02:53:38.635566Z" } }, "outputs": [], "source": [ "import json" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:53:38.677891Z", "iopub.status.busy": "2024-07-10T02:53:38.677761Z", "iopub.status.idle": "2024-07-10T02:53:38.714311Z", "shell.execute_reply": "2024-07-10T02:53:38.713886Z", "shell.execute_reply.started": "2024-07-10T02:53:38.677877Z" } }, "outputs": [], "source": [ "def read_jsonl(filepath, parse_idx_set=None):\n", " data = []\n", " with open(filepath) as f:\n", " line_idx = 0\n", " for line in f:\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " if parse_idx_set is not None and line_idx not in parse_idx_set:\n", " data.append(None)\n", " line_idx += 1\n", " continue\n", " m = json.loads(line)\n", " data.append(m)\n", " line_idx += 1\n", " return data" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:53:38.715035Z", "iopub.status.busy": "2024-07-10T02:53:38.714886Z", "iopub.status.idle": "2024-07-10T02:53:38.771434Z", "shell.execute_reply": "2024-07-10T02:53:38.770979Z", "shell.execute_reply.started": "2024-07-10T02:53:38.715019Z" } }, "outputs": [], "source": [ "eval_loss = read_jsonl(\"/app/suno/data/dpo/2b_before_recode_v0/eval_val_loss/loss.jsonl\")" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:53:38.772163Z", "iopub.status.busy": "2024-07-10T02:53:38.772018Z", "iopub.status.idle": "2024-07-10T02:53:38.789565Z", "shell.execute_reply": "2024-07-10T02:53:38.789161Z", "shell.execute_reply.started": "2024-07-10T02:53:38.772147Z" } }, "outputs": [ { "data": { "text/plain": [ "1568" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(eval_loss)" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:53:38.790265Z", "iopub.status.busy": "2024-07-10T02:53:38.790129Z", "iopub.status.idle": "2024-07-10T02:53:38.822565Z", "shell.execute_reply": "2024-07-10T02:53:38.822164Z", "shell.execute_reply.started": "2024-07-10T02:53:38.790250Z" } }, "outputs": [ { "data": { "text/plain": [ "{'idx': 451,\n", " 'loss_sem': 1.77344,\n", " 'loss_sem_10s': 2.875,\n", " 'loss_sem_no_text': 1.74219,\n", " 'loss_sem_no_text_10s': 2.4375,\n", " 'loss_coarse': 4.46615,\n", " 'loss_coarse_10s': 3.68359,\n", " 'loss_coarse_no_text': 4.47135,\n", " 'loss_coarse_no_text_10s': 3.75651,\n", " 'loss_coarse_0': 2.42188,\n", " 'loss_coarse_1': 3.54688,\n", " 'loss_coarse_2': 4.46875,\n", " 'loss_coarse_3': 4.625,\n", " 'loss_coarse_4': 4.59375,\n", " 'loss_coarse_5': 4.71875,\n", " 'loss_coarse_6': 4.8125,\n", " 'loss_coarse_7': 4.84375,\n", " 'loss_coarse_8': 4.90625,\n", " 'loss_coarse_9': 4.9375,\n", " 'loss_coarse_10': 4.90625,\n", " 'loss_coarse_11': 4.8125}" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "eval_loss[1193]" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:53:38.823263Z", "iopub.status.busy": "2024-07-10T02:53:38.823125Z", "iopub.status.idle": "2024-07-10T02:53:38.858801Z", "shell.execute_reply": "2024-07-10T02:53:38.858399Z", "shell.execute_reply.started": "2024-07-10T02:53:38.823248Z" } }, "outputs": [ { "data": { "text/plain": [ "[{'idx': 311,\n", " 'loss_sem': 2.875,\n", " 'loss_sem_10s': 5.3125,\n", " 'loss_sem_no_text': 2.60938,\n", " 'loss_sem_no_text_10s': 3.0625,\n", " 'loss_coarse': 5.34375,\n", " 'loss_coarse_10s': 4.15365,\n", " 'loss_coarse_no_text': 5.28385,\n", " 'loss_coarse_no_text_10s': 3.96745,\n", " 'loss_coarse_0': 4.0,\n", " 'loss_coarse_1': 5.09375,\n", " 'loss_coarse_2': 5.75,\n", " 'loss_coarse_3': 5.84375,\n", " 'loss_coarse_4': 5.75,\n", " 'loss_coarse_5': 5.53125,\n", " 'loss_coarse_6': 5.625,\n", " 'loss_coarse_7': 5.375,\n", " 'loss_coarse_8': 5.3125,\n", " 'loss_coarse_9': 5.3125,\n", " 'loss_coarse_10': 5.375,\n", " 'loss_coarse_11': 5.15625}]" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "[x for x in eval_loss if x[\"idx\"] == 311]" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:53:38.859515Z", "iopub.status.busy": "2024-07-10T02:53:38.859360Z", "iopub.status.idle": "2024-07-10T02:53:38.896727Z", "shell.execute_reply": "2024-07-10T02:53:38.896316Z", "shell.execute_reply.started": "2024-07-10T02:53:38.859499Z" } }, "outputs": [], "source": [ "from collections import Counter\n", "c = Counter()\n", "for i in eval_loss:\n", " c[i[\"idx\"]] += 1" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "execution": { "iopub.execute_input": "2024-07-10T02:53:38.897428Z", "iopub.status.busy": "2024-07-10T02:53:38.897286Z", "iopub.status.idle": "2024-07-10T02:53:38.933958Z", "shell.execute_reply": "2024-07-10T02:53:38.933545Z", "shell.execute_reply.started": "2024-07-10T02:53:38.897414Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1568\n" ] } ], "source": [ "print(len(c))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" } }, "nbformat": 4, "nbformat_minor": 4 }