{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "d04cf52f",
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"from matplotlib import pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "d31d8850",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\""
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "91147ff2",
"metadata": {},
"outputs": [],
"source": [
"import tqdm\n",
"import math\n",
"import torch\n",
"import random\n",
"import funcy\n",
"import copy\n",
"import gc\n",
"import re\n",
"import json\n",
"import tempfile\n",
"import collections\n",
"import pandas as pd\n",
"import numpy as np\n",
"from joblib import Parallel, delayed\n",
"\n",
"from suno_utils.audio import Audio\n",
"from suno_utils.utils.text import write_jsonl, read_jsonl, write_json, read_json, normalize_whitespace\n",
"from suno_utils.utils.s3 import read_from_s3, check_s3_file_exists, open_from_s3"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3d60b5ec",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n",
"from suno_utils.tasks.dac_2c_12cb import decode as codec_decode\n",
"_ = preload_codec_models(\"/app/suno/models/dac_2c_25x12.pt\")\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "3a57db60",
"metadata": {},
"outputs": [],
"source": [
"test_metas = read_jsonl(\"/mnt/localdisk/data/metas_val.jsonl\")\n",
"test_info = read_json(\"/mnt/localdisk/data/info_val.json\")\n",
"test_data = np.memmap(\"/mnt/localdisk/data/data_val.bin\", dtype=np.uint16, mode=\"r\").reshape(-1, 3008, 13)\n",
"\n",
"# test_metas = read_jsonl(\"/mnt/localdisk/data/metas_tr.jsonl\")\n",
"# test_info = read_json(\"/mnt/localdisk/data/info_tr.json\")\n",
"# test_data = np.memmap(\"/mnt/localdisk/data/data_tr.bin\", dtype=np.uint16, mode=\"r\").reshape(-1, 3008, 13)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "5254988f",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'idx': 0,\n",
" 'loss_sem': 3.1875,\n",
" 'loss_sem_10s': 3.10938,\n",
" 'loss_sem_no_text': 3.1875,\n",
" 'loss_sem_no_text_10s': 3.10938,\n",
" 'loss_coarse': 5.61328,\n",
" 'loss_coarse_10s': 3.66667,\n",
" 'loss_coarse_no_text': 5.61328,\n",
" 'loss_coarse_no_text_10s': 3.66667}"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"loss_data = read_jsonl(\"/app/suno/checkpoints/2023-12-04_22-16-18/loss.jsonl\") # val\n",
"# loss_data = read_jsonl(\"/app/suno/checkpoints/2023-12-04_22-37-16/loss.jsonl\") # tr\n",
"loss_data[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d96f7999",
"metadata": {},
"outputs": [],
"source": [
"# look at lowest 1 percentile of loss\n",
"# look at highest 1 percentile of loss"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "08b49af0",
"metadata": {},
"outputs": [],
"source": [
"# easy loss_sem_10s - basic background beat or super short\n",
"# hard loss_sem_10s - complex music"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ae80419e",
"metadata": {},
"outputs": [],
"source": [
"# filter to at least 20s duration\n",
"# filter to 1-5 pctl of semantic\n",
"# filter to 1-5 pctl of coarse\n",
"\n",
"# TODO: filter text difference incorrect"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "6f658347",
"metadata": {},
"outputs": [],
"source": [
"lower_bound = np.quantile([m[\"loss_sem_10s\"] for m in loss_data], 0.01)\n",
"upper_bound = np.quantile([m[\"loss_sem_10s\"] for m in loss_data], 0.99)\n",
"lower_idx_list = [m[\"idx\"] for m in loss_data if m[\"loss_sem_10s\"] <= lower_bound]\n",
"upper_idx_list = [m[\"idx\"] for m in loss_data if m[\"loss_sem_10s\"] >= upper_bound]"
]
},
{
"cell_type": "code",
"execution_count": 58,
"id": "24abbd10",
"metadata": {},
"outputs": [],
"source": [
"lower_bound = np.quantile([m[\"loss_coarse_10s\"] for m in loss_data], 0.01)\n",
"upper_bound = np.quantile([m[\"loss_coarse_10s\"] for m in loss_data], 0.99)\n",
"lower_idx_list = [m[\"idx\"] for m in loss_data if m[\"loss_coarse_10s\"] <= lower_bound]\n",
"upper_idx_list = [m[\"idx\"] for m in loss_data if m[\"loss_coarse_10s\"] >= upper_bound]"
]
},
{
"cell_type": "code",
"execution_count": 72,
"id": "ffa2da16",
"metadata": {},
"outputs": [],
"source": [
"# block incorrect text conditioning\n",
"upper_bound = np.quantile([\n",
" (m[\"loss_sem\"] - m[\"loss_sem_no_text\"]) / max(0.01, m[\"loss_sem\"], m[\"loss_sem_no_text\"]) \n",
" for m in loss_data\n",
"], 0.99)\n",
"upper_idx_list = [\n",
" m[\"idx\"] \n",
" for m in loss_data \n",
" if (m[\"loss_sem\"] - m[\"loss_sem_no_text\"]) / max(0.01, m[\"loss_sem\"], m[\"loss_sem_no_text\"]) >= upper_bound\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 76,
"id": "1c3c4273",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"194"
]
},
"execution_count": 76,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"idx = upper_idx_list[1]\n",
"idx"
]
},
{
"cell_type": "code",
"execution_count": 77,
"id": "86d0b339",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'dataset': 'youtube_music',\n",
" 'id': 'Z5g_elQ5SWY',\n",
" 'start_s': 120.32,\n",
" 'end_s': 147.12,\n",
" 'original_duration_s': 147.12,\n",
" 'tags': ['country & americana', 'feel good'],\n",
" 'tags_private': ['country & americana',\n",
" 'Life By the Drop',\n",
" 'Stevie Ray Vaughan & Double Trouble',\n",
" 'The Real Deal: Greatest Hits Volume 2',\n",
" 'feel good'],\n",
" 'views': 124133}"
]
},
"execution_count": 77,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"test_metas[idx]"
]
},
{
"cell_type": "code",
"execution_count": 78,
"id": "c3b7af1d",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"arr = test_data[idx,1:].copy().astype(np.int16)[:,1:]\n",
"pad_idx_arr = np.where(arr == 4096)[0]\n",
"if len(pad_idx_arr) > 0:\n",
" arr = arr[:pad_idx_arr[0],:]\n",
"codec_decode(arr[:10*25,:]).play()"
]
},
{
"cell_type": "markdown",
"id": "3fa2aed3",
"metadata": {},
"source": [
"## (TODO): apply all these filters on a per-dataset basis"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3c85264a",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "c5f00cfd",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "8497b40d",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "fd670d81",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "44083930",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "a9e13a64",
"metadata": {},
"source": [
"### Playground"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ae139759",
"metadata": {},
"outputs": [],
"source": [
"blocked_loss_idx_set = set()\n",
"\n",
"# block hard semantic and super easy\n",
"lower_bound = np.quantile([m[\"loss_sem_10s\"] for m in loss_data if m[\"idx\"] in dataset_idx_set], 0.01)\n",
"upper_bound = np.quantile([m[\"loss_sem_10s\"] for m in loss_data if m[\"idx\"] in dataset_idx_set], 0.95)\n",
"for m in loss_data:\n",
" if m[\"loss_sem_10s\"] < lower_bound or m[\"loss_sem_10s\"] > upper_bound:\n",
" blocked_loss_idx_set.add(m[\"idx\"])\n",
"\n",
"# block hard coarse and super easy\n",
"lower_bound = np.quantile([m[\"loss_coarse_10s\"] for m in loss_data if m[\"idx\"] in dataset_idx_set], 0.01)\n",
"upper_bound = np.quantile([m[\"loss_coarse_10s\"] for m in loss_data if m[\"idx\"] in dataset_idx_set], 0.95)\n",
"for m in loss_data:\n",
" if m[\"loss_coarse_10s\"] < lower_bound or m[\"loss_coarse_10s\"] > upper_bound:\n",
" blocked_loss_idx_set.add(m[\"idx\"])\n",
" \n",
"# block incorrect text\n",
"upper_bound = np.quantile(\n",
" [\n",
" (m[\"loss_sem\"] - m[\"loss_sem_no_text\"]) / max(0.01, m[\"loss_sem\"], m[\"loss_sem_no_text\"]) \n",
" for m in loss_data if m[\"idx\"] in dataset_idx_set\n",
" ], \n",
" 0.95\n",
")\n",
"for m in loss_data:\n",
" if (m[\"loss_sem\"] - m[\"loss_sem_no_text\"]) / max(0.01, m[\"loss_sem\"], m[\"loss_sem_no_text\"]) > upper_bound:\n",
" blocked_loss_idx_set.add(m[\"idx\"])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "480c8bad",
"metadata": {},
"outputs": [],
"source": [
"# TODO: investigate biggest outliers and define cutoffs"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "20dc19cc",
"metadata": {},
"outputs": [],
"source": [
"dataset_idx_set = set()\n",
"for idx, m in enumerate(test_metas):\n",
" if m[\"dataset\"] not in (\"genius_hq_lyrics\", \"pond5_music\"):\n",
" continue\n",
" if m[\"end_s\"] - m[\"start_s\"] < 20:\n",
" continue\n",
" dataset_idx_set.add(idx)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "654feec4",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "d0d2dd01",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "814638e1",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "5f20e852",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "97a43c75",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'dataset': 'youtube_music',\n",
" 'id': 'gGrh8fEXFak',\n",
" 'start_s': 0.0,\n",
" 'end_s': 66.16,\n",
" 'original_duration_s': 227.72,\n",
" 'tags_private': ['Ivan 2 Filoz', 'United Kingdom 2', 'Descansa Tranquilo'],\n",
" 'views': 3741}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"test_metas[123]"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "7fb4839f",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"dict_keys(['youtube_music_lyrics', 'youtube_music', 'youtube_music_lyrics_foreign', 'genius_hq_lyrics', 'genius_hq_lyrics_foreign', 'genius_hq', 'freesound', 'imslp', 'jamendo', 'pond5_music', 'deezer_lyrics_foreign', 'deezer', 'deezer_lyrics', 'ytm_tagged', 'musescore_lyrics'])"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"test_info.keys()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "2c2e1b2d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"dict_keys(['idx_list'])"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"test_info[\"youtube_music_lyrics\"][\"idx_list\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3b8585bb",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "160f4588",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "bad774e1",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}