{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "d31d8850", "metadata": { "execution": { "iopub.execute_input": "2024-08-08T04:00:52.839678Z", "iopub.status.busy": "2024-08-08T04:00:52.839529Z", "iopub.status.idle": "2024-08-08T04:00:52.844396Z", "shell.execute_reply": "2024-08-08T04:00:52.843545Z", "shell.execute_reply.started": "2024-08-08T04:00:52.839656Z" } }, "outputs": [], "source": [ "import os\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "91147ff2", "metadata": { "execution": { "iopub.execute_input": "2024-08-08T04:00:52.845304Z", "iopub.status.busy": "2024-08-08T04:00:52.845151Z", "iopub.status.idle": "2024-08-08T04:00:55.661461Z", "shell.execute_reply": "2024-08-08T04:00:55.660938Z", "shell.execute_reply.started": "2024-08-08T04:00:52.845288Z" } }, "outputs": [], "source": [ "import collections\n", "import copy\n", "import gc\n", "import json\n", "import math\n", "import random\n", "import re\n", "import tempfile\n", "\n", "import funcy\n", "import numpy as np\n", "import pandas as pd\n", "import torch\n", "import tqdm\n", "from joblib import Parallel, delayed\n", "from suno_utils.audio import Audio\n", "from suno_utils.utils.s3 import check_s3_file_exists, open_from_s3, read_from_s3\n", "from suno_utils.utils.text import (\n", " normalize_whitespace,\n", " read_json,\n", " read_jsonl,\n", " write_json,\n", " write_jsonl,\n", ")\n", "%matplotlib inline\n", "from matplotlib import pyplot as plt" ] }, { "cell_type": "code", "execution_count": 3, "id": "3d60b5ec", "metadata": { "execution": { "iopub.execute_input": "2024-08-08T04:00:55.662326Z", "iopub.status.busy": "2024-08-08T04:00:55.662127Z", "iopub.status.idle": "2024-08-08T04:01:00.443542Z", "shell.execute_reply": "2024-08-08T04:01:00.442940Z", "shell.execute_reply.started": "2024-08-08T04:00:55.662309Z" } }, "outputs": [], "source": [ "from suno_utils.tasks.dac_2c_12cb import decode as codec_decode\n", "from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "\n", "_ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\")" ] }, { "cell_type": "code", "execution_count": 4, "id": "3a57db60", "metadata": { "execution": { "iopub.execute_input": "2024-08-08T04:01:00.444563Z", "iopub.status.busy": "2024-08-08T04:01:00.444301Z", "iopub.status.idle": "2024-08-08T04:11:56.137886Z", "shell.execute_reply": "2024-08-08T04:11:56.137154Z", "shell.execute_reply.started": "2024-08-08T04:01:00.444545Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "IOStream.flush timed out\n" ] } ], "source": [ "# test_metas = read_jsonl(\"/app/suno/data/chirp_v4/multi/metas_val.jsonl\")\n", "# test_info = read_json(\"/app/suno/data/chirp_v4/multi/info_val.json\")\n", "# test_data = np.memmap(\"/app/suno/data/chirp_v4/multi/data_val.bin\", dtype=np.uint16, mode=\"r\").reshape(-1, 6016, 13)\n", "\n", "# takes 11 mins to load the train\n", "test_metas = read_jsonl(\"/app/suno/data/chirp_v4/multi/metas_tr.jsonl\")\n", "test_info = read_json(\"/app/suno/data/chirp_v4/multi/info_tr.json\")\n", "test_data = np.memmap(\n", " \"/app/suno/data/chirp_v4/multi/data_tr.bin\", dtype=np.uint16, mode=\"r\"\n", ").reshape(-1, 6016, 13)\n", "\n", "assert len(test_data) == len(test_metas)\n", "assert test_data[:100, :, 0].min() >= 0\n", "assert test_data[:100, :, 0].max() <= 4000\n", "assert test_data[:100, :, 1:].min() >= 0\n", "assert test_data[:100, :, 1:].max() <= 2048" ] }, { "cell_type": "code", "execution_count": 5, "id": "2d70c4f3-9f67-4149-b05c-1bad703ac6af", "metadata": { "execution": { "iopub.execute_input": "2024-08-08T04:11:56.143608Z", "iopub.status.busy": "2024-08-08T04:11:56.143465Z", "iopub.status.idle": "2024-08-08T04:11:56.146464Z", "shell.execute_reply": "2024-08-08T04:11:56.146011Z", "shell.execute_reply.started": "2024-08-08T04:11:56.143591Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(90972737, 6016, 13)\n" ] } ], "source": [ "print(test_data.shape)" ] }, { "cell_type": "code", "execution_count": 6, "id": "3fcf1e4c-2921-4229-88aa-6af96c57da25", "metadata": { "execution": { "iopub.execute_input": "2024-08-08T04:11:56.148247Z", "iopub.status.busy": "2024-08-08T04:11:56.148110Z", "iopub.status.idle": "2024-08-08T04:11:56.189464Z", "shell.execute_reply": "2024-08-08T04:11:56.189050Z", "shell.execute_reply.started": "2024-08-08T04:11:56.148231Z" } }, "outputs": [ { "data": { "text/plain": [ "{'dataset': 'youtube_music_lyrics',\n", " 'task': 'default',\n", " 'id': 'H6iYuHu0K7g',\n", " 'start_s': 0.0,\n", " 'end_s': 230.0,\n", " 'original_duration_s': 230.0,\n", " 'vocal_start_s': None,\n", " 'vocal_end_s': None,\n", " 'tags': ['country & americana', 'feel good'],\n", " 'text': \"Now they tell me the streets of heaven\\nAre paved with diamonds & gold\\nI may not get there but if I do\\nI believe I gonna take off my shoes\\n\\nGonna walk around heaven\\nGonna walk around heaven\\nBarefooted singing God's praise\\nGonna walk around heaven\\nGonna walk around heaven\\nWhere there ain't no end of days\\n\\nAin't a secret if you know me\\nI been no account most of my life\\nBut I been converted and now I got the spirit\\nSo there's a chance I gonna see this paradise\\n\\nWhen I get to heaven preachers tell me\\nI get a halo some wings and a harp\\nThat's well and good\\nBut what I do hear\\nIs Sister Rosetta Tharpe\",\n", " 'text_lang': 'en',\n", " 'n_tokens': 5750}" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_metas[0]" ] }, { "cell_type": "code", "execution_count": null, "id": "4709d06a-4555-4b77-b5c5-7c044bd3db05", "metadata": { "execution": { "execution_failed": "2024-08-06T13:51:25.105Z" } }, "outputs": [], "source": [ "unique_meta_ids = set()\n", "for test_meta in test_metas:\n", " unique_meta_ids.add(test_meta[\"id\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "6f9636f0-36e0-4a18-8d3f-0aaca65de8d8", "metadata": { "execution": { "execution_failed": "2024-08-06T13:51:25.105Z" } }, "outputs": [], "source": [ "len(test_metas), len(unique_meta_ids)" ] }, { "cell_type": "code", "execution_count": null, "id": "469ed6b9-e825-4d0b-b186-f55861784e2a", "metadata": { "execution": { "execution_failed": "2024-08-06T13:51:25.105Z" } }, "outputs": [], "source": [ "!ls /app/suno/data/dpo/chirp_v4_multi/eval_tr_loss/" ] }, { "cell_type": "code", "execution_count": null, "id": "f41fda9d-ca95-4a89-9272-6e44c17b4d59", "metadata": { "execution": { "iopub.execute_input": "2024-06-27T15:28:02.795250Z", "iopub.status.busy": "2024-06-27T15:28:02.794806Z", "iopub.status.idle": "2024-06-27T15:35:25.074195Z", "shell.execute_reply": "2024-06-27T15:35:25.073531Z", "shell.execute_reply.started": "2024-06-27T15:28:02.795225Z" } }, "outputs": [], "source": [ "# 7 min to load half of it\n", "loss_data = read_jsonl(\n", " \"/app/suno/data/dpo/chirp_v4_multi/eval_tr_loss/loss_570.jsonl\"\n", ") # val\n", "print(len(loss_data))\n", "loss_df = pd.DataFrame(loss_data)\n", "\n", "# seen_idx = set()\n", "# for row in loss_data:\n", "# seen_idx.add(row[\"idx\"])\n", "# print(len(seen_idx))\n", "# seen_idx = sorted(list(seen_idx))\n", "# with open(\"/app/suno/data/dpo/chirp_v4_multi/eval_tr_loss/processed_idx.json\", \"w\") as fp:\n", "# json.dump(seen_idx, fp)\n", "# with open(\n", "# \"/app/suno/data/dpo/chirp_v4_multi/eval_tr_loss/processed_idx.json\", \"r\"\n", "# ) as fp:\n", "# prev_seen = json.load(fp)" ] }, { "cell_type": "code", "execution_count": null, "id": "d7dff12c-1e80-4f75-9882-5d769e4ea895", "metadata": { "execution": { "iopub.execute_input": "2024-06-27T17:02:26.551901Z", "iopub.status.busy": "2024-06-27T17:02:26.551543Z", "iopub.status.idle": "2024-06-27T17:11:27.649475Z", "shell.execute_reply": "2024-06-27T17:11:27.648700Z", "shell.execute_reply.started": "2024-06-27T17:02:26.551880Z" } }, "outputs": [], "source": [ "loss_data_2 = read_jsonl(\n", " \"/app/suno/data/dpo/chirp_v4_multi/eval_tr_loss/loss.jsonl\"\n", ") # val\n", "print(len(loss_data_2))\n", "loss_df_2 = pd.DataFrame(loss_data_2)" ] }, { "cell_type": "code", "execution_count": null, "id": "7ed02147-0346-4649-a06b-47757d98f127", "metadata": { "execution": { "iopub.execute_input": "2024-06-27T20:11:31.293567Z", "iopub.status.busy": "2024-06-27T20:11:31.293145Z", "iopub.status.idle": "2024-06-27T20:11:38.311587Z", "shell.execute_reply": "2024-06-27T20:11:38.310824Z", "shell.execute_reply.started": "2024-06-27T20:11:31.293543Z" } }, "outputs": [], "source": [ "total_loss_df = pd.concat([loss_df, loss_df_2])" ] }, { "cell_type": "code", "execution_count": null, "id": "f44c3418-9716-4381-b2af-bd05b57298a5", "metadata": { "execution": { "iopub.execute_input": "2024-06-27T20:11:40.102630Z", "iopub.status.busy": "2024-06-27T20:11:40.102118Z", "iopub.status.idle": "2024-06-27T20:11:40.106328Z", "shell.execute_reply": "2024-06-27T20:11:40.105800Z", "shell.execute_reply.started": "2024-06-27T20:11:40.102607Z" } }, "outputs": [], "source": [ "total_loss_df.columns" ] }, { "cell_type": "code", "execution_count": null, "id": "918ef541-5200-4698-b2fa-a6fe5233f7e6", "metadata": { "execution": { "iopub.execute_input": "2024-06-27T20:12:29.444826Z", "iopub.status.busy": "2024-06-27T20:12:29.444432Z", "iopub.status.idle": "2024-06-27T20:13:24.886869Z", "shell.execute_reply": "2024-06-27T20:13:24.886093Z", "shell.execute_reply.started": "2024-06-27T20:12:29.444804Z" } }, "outputs": [], "source": [ "print(total_loss_df.shape, total_loss_df[\"idx\"].nunique())\n", "loss_df = total_loss_df.copy()" ] }, { "cell_type": "code", "execution_count": null, "id": "ae2fa1f6-5719-49e7-bb16-4b80b9cb6279", "metadata": { "execution": { "iopub.execute_input": "2024-06-27T20:14:05.241200Z", "iopub.status.busy": "2024-06-27T20:14:05.240869Z", "iopub.status.idle": "2024-06-27T20:14:08.475997Z", "shell.execute_reply": "2024-06-27T20:14:08.475257Z", "shell.execute_reply.started": "2024-06-27T20:14:05.241178Z" } }, "outputs": [], "source": [ "loss_df[\"sem_text_diff\"] = (loss_df[\"loss_sem\"] - loss_df[\"loss_sem_no_text\"]) / loss_df[\"loss_sem\"]\n", "loss_df[\"coarse_text_diff\"] = (loss_df[\"loss_coarse\"] - loss_df[\"loss_coarse_no_text\"]) / loss_df[\"loss_coarse\"]\n", "loss_df[\"sem_10s_text_diff\"] = (loss_df[\"loss_sem_10s\"] - loss_df[\"loss_sem_no_text_10s\"]) / loss_df[\"loss_sem_10s\"]\n", "loss_df[\"coarse_10s_text_diff\"] = (loss_df[\"loss_coarse_10s\"] - loss_df[\"loss_coarse_no_text_10s\"]) / loss_df[\"loss_coarse_10s\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "1d97d735-a229-43a1-9e5d-769ab76814e6", "metadata": { "execution": { "iopub.execute_input": "2024-06-27T20:14:09.813093Z", "iopub.status.busy": "2024-06-27T20:14:09.812895Z", "iopub.status.idle": "2024-06-27T20:14:09.816856Z", "shell.execute_reply": "2024-06-27T20:14:09.816352Z", "shell.execute_reply.started": "2024-06-27T20:14:09.813076Z" } }, "outputs": [], "source": [ "loss_columns = [i for i in loss_df.columns if i != \"idx\"]\n", "len(loss_columns), len(loss_df.columns)" ] }, { "cell_type": "code", "execution_count": null, "id": "7951e042-b88f-4d8c-8457-07607f470e8e", "metadata": { "execution": { "iopub.execute_input": "2024-06-27T20:14:11.037536Z", "iopub.status.busy": "2024-06-27T20:14:11.037161Z", "iopub.status.idle": "2024-06-27T20:15:16.786657Z", "shell.execute_reply": "2024-06-27T20:15:16.785906Z", "shell.execute_reply.started": "2024-06-27T20:14:11.037512Z" } }, "outputs": [], "source": [ "# Calculate the 5th and 95th percentiles for each column\n", "lower_bound = loss_df[loss_columns].quantile(0.01)\n", "upper_bound = loss_df[loss_columns].quantile(0.99)" ] }, { "cell_type": "code", "execution_count": null, "id": "1a6d7fb9-87f7-4f5c-b84e-0417ac127e5e", "metadata": { "execution": { "iopub.execute_input": "2024-06-27T20:31:40.846194Z", "iopub.status.busy": "2024-06-27T20:31:40.845536Z", "iopub.status.idle": "2024-06-27T20:41:24.288853Z", "shell.execute_reply": "2024-06-27T20:41:24.288050Z", "shell.execute_reply.started": "2024-06-27T20:31:40.846171Z" } }, "outputs": [], "source": [ "# Apply the filter to keep values within the 5th to 95th percentile range\n", "loss_df_filtered = loss_df.copy().reset_index(drop=True)\n", "loss_df_filtered[loss_columns] = loss_df_filtered[loss_columns].apply(lambda x: x[(x >= lower_bound[x.name]) & (x <= upper_bound[x.name])])\n", "\n", "# Drop rows that have NaN values resulting from the filtering\n", "loss_df_filtered = loss_df_filtered.dropna()\n", "\n", "# Display the filtered DataFrame\n", "print(loss_df_filtered.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "c0165640-260c-4c08-9e5b-82ca8ea91220", "metadata": { "execution": { "iopub.execute_input": "2024-06-27T20:41:24.290221Z", "iopub.status.busy": "2024-06-27T20:41:24.290024Z", "iopub.status.idle": "2024-06-27T20:41:24.294011Z", "shell.execute_reply": "2024-06-27T20:41:24.293512Z", "shell.execute_reply.started": "2024-06-27T20:41:24.290203Z" } }, "outputs": [], "source": [ "loss_df_filtered.shape[0] / loss_df.shape[0]" ] }, { "cell_type": "code", "execution_count": null, "id": "9be5e861-dc08-4f02-975c-d86391b24c75", "metadata": { "execution": { "iopub.execute_input": "2024-07-01T02:48:36.530677Z", "iopub.status.busy": "2024-07-01T02:48:36.530262Z", "iopub.status.idle": "2024-07-01T02:48:44.432530Z", "shell.execute_reply": "2024-07-01T02:48:44.431992Z", "shell.execute_reply.started": "2024-07-01T02:48:36.530654Z" } }, "outputs": [], "source": [ "loss_df_filtered[\"idx\"].nunique()" ] }, { "cell_type": "code", "execution_count": null, "id": "b6ad44ce-114a-4584-a8b6-b18870ae8612", "metadata": { "execution": { "iopub.execute_input": "2024-07-01T03:05:59.761232Z", "iopub.status.busy": "2024-07-01T03:05:59.760940Z", "iopub.status.idle": "2024-07-01T03:06:25.705145Z", "shell.execute_reply": "2024-07-01T03:06:25.704535Z", "shell.execute_reply.started": "2024-07-01T03:05:59.761212Z" } }, "outputs": [], "source": [ "bad_loss_idxs = set(loss_df[\"idx\"]) - set(loss_df_filtered[\"idx\"])\n", "print(len(bad_loss_idxs))" ] }, { "cell_type": "code", "execution_count": null, "id": "82936992-c9b8-4fa9-b6a1-7dca73499129", "metadata": { "execution": { "iopub.execute_input": "2024-07-01T03:09:23.732533Z", "iopub.status.busy": "2024-07-01T03:09:23.732379Z", "iopub.status.idle": "2024-07-01T03:09:35.802692Z", "shell.execute_reply": "2024-07-01T03:09:35.802089Z", "shell.execute_reply.started": "2024-07-01T03:09:23.732514Z" } }, "outputs": [], "source": [ "bad_test_metas = [meta for i, meta in enumerate(test_metas) if i in bad_loss_idxs]\n", "print(len(bad_test_metas))" ] }, { "cell_type": "code", "execution_count": null, "id": "e8c0a2de-0ebd-4b06-ba37-6545eb136bfc", "metadata": { "execution": { "iopub.execute_input": "2024-07-01T03:13:10.628374Z", "iopub.status.busy": "2024-07-01T03:13:10.628179Z", "iopub.status.idle": "2024-07-01T03:13:18.000820Z", "shell.execute_reply": "2024-07-01T03:13:18.000061Z", "shell.execute_reply.started": "2024-07-01T03:13:10.628355Z" } }, "outputs": [], "source": [ "from collections import defaultdict\n", "bad_test_metas_dataset_id = defaultdict(set)\n", "for meta in bad_test_metas:\n", " bad_test_metas_dataset_id[meta[\"dataset\"]].add(meta[\"id\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "ea915487-5db8-4659-9a8e-1d3118ff1f28", "metadata": { "execution": { "iopub.execute_input": "2024-07-01T03:15:49.285768Z", "iopub.status.busy": "2024-07-01T03:15:49.285375Z", "iopub.status.idle": "2024-07-01T03:15:49.289665Z", "shell.execute_reply": "2024-07-01T03:15:49.289155Z", "shell.execute_reply.started": "2024-07-01T03:15:49.285747Z" } }, "outputs": [], "source": [ "for k, v in bad_test_metas_dataset_id.items():\n", " try:\n", " all_size = len(test_info[k][\"idx_list\"])\n", " except:\n", " all_size = 100\n", " print(k, len(v), all_size, len(v) / all_size)" ] }, { "cell_type": "code", "execution_count": null, "id": "f8bce324-1771-47de-a30e-6f01d8df46fa", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0ebacc3e-aaf7-470f-aefc-059b29f12ccd", "metadata": { "execution": { "iopub.execute_input": "2024-06-21T04:37:05.900278Z", "iopub.status.busy": "2024-06-21T04:37:05.900130Z", "iopub.status.idle": "2024-06-21T04:37:05.903873Z", "shell.execute_reply": "2024-06-21T04:37:05.903426Z", "shell.execute_reply.started": "2024-06-21T04:37:05.900262Z" } }, "outputs": [], "source": [ "# seen_ids = set([m[\"idx\"] for m in loss_data])" ] }, { "cell_type": "code", "execution_count": null, "id": "d96f7999", "metadata": { "execution": { "iopub.execute_input": "2024-06-21T16:48:40.167099Z", "iopub.status.busy": "2024-06-21T16:48:40.166786Z", "iopub.status.idle": "2024-06-21T16:48:40.208528Z", "shell.execute_reply": "2024-06-21T16:48:40.208028Z", "shell.execute_reply.started": "2024-06-21T16:48:40.167080Z" } }, "outputs": [], "source": [ "# look at lowest 1 percentile of loss\n", "# look at highest 1 percentile of loss" ] }, { "cell_type": "code", "execution_count": null, "id": "08b49af0", "metadata": { "execution": { "iopub.execute_input": "2024-06-21T16:48:40.209870Z", "iopub.status.busy": "2024-06-21T16:48:40.209550Z", "iopub.status.idle": "2024-06-21T16:48:40.247792Z", "shell.execute_reply": "2024-06-21T16:48:40.247310Z", "shell.execute_reply.started": "2024-06-21T16:48:40.209851Z" } }, "outputs": [], "source": [ "# easy loss_sem_10s - basic background beat or super short\n", "# hard loss_sem_10s - complex music" ] }, { "cell_type": "code", "execution_count": null, "id": "ae80419e", "metadata": { "execution": { "iopub.execute_input": "2024-06-21T16:48:40.248555Z", "iopub.status.busy": "2024-06-21T16:48:40.248419Z", "iopub.status.idle": "2024-06-21T16:48:40.287767Z", "shell.execute_reply": "2024-06-21T16:48:40.287275Z", "shell.execute_reply.started": "2024-06-21T16:48:40.248541Z" } }, "outputs": [], "source": [ "# filter to at least 20s duration\n", "# filter to 1-5 pctl of semantic\n", "# filter to 1-5 pctl of coarse\n", "\n", "# TODO: filter text difference incorrect" ] }, { "cell_type": "code", "execution_count": null, "id": "6f658347", "metadata": { "execution": { "iopub.execute_input": "2024-06-21T16:48:43.932237Z", "iopub.status.busy": "2024-06-21T16:48:43.931937Z", "iopub.status.idle": "2024-06-21T16:48:48.521218Z", "shell.execute_reply": "2024-06-21T16:48:48.520510Z", "shell.execute_reply.started": "2024-06-21T16:48:43.932219Z" } }, "outputs": [], "source": [ "lower_bound = np.quantile([m[\"loss_sem_10s\"] for m in loss_data], 0.01)\n", "upper_bound = np.quantile([m[\"loss_sem_10s\"] for m in loss_data], 0.99)\n", "lower_idx_list = [m[\"idx\"] for m in loss_data if m[\"loss_sem_10s\"] <= lower_bound]\n", "upper_idx_list = [m[\"idx\"] for m in loss_data if m[\"loss_sem_10s\"] >= upper_bound]" ] }, { "cell_type": "code", "execution_count": null, "id": "24abbd10", "metadata": { "execution": { "iopub.execute_input": "2024-06-21T16:48:48.522789Z", "iopub.status.busy": "2024-06-21T16:48:48.522323Z", "iopub.status.idle": "2024-06-21T16:48:53.113592Z", "shell.execute_reply": "2024-06-21T16:48:53.112917Z", "shell.execute_reply.started": "2024-06-21T16:48:48.522769Z" } }, "outputs": [], "source": [ "lower_bound = np.quantile([m[\"loss_coarse_10s\"] for m in loss_data], 0.01)\n", "upper_bound = np.quantile([m[\"loss_coarse_10s\"] for m in loss_data], 0.99)\n", "lower_idx_list = [m[\"idx\"] for m in loss_data if m[\"loss_coarse_10s\"] <= lower_bound]\n", "upper_idx_list = [m[\"idx\"] for m in loss_data if m[\"loss_coarse_10s\"] >= upper_bound]" ] }, { "cell_type": "code", "execution_count": null, "id": "ffa2da16", "metadata": { "execution": { "iopub.execute_input": "2024-06-21T16:48:53.114537Z", "iopub.status.busy": "2024-06-21T16:48:53.114375Z", "iopub.status.idle": "2024-06-21T16:48:56.657145Z", "shell.execute_reply": "2024-06-21T16:48:56.656451Z", "shell.execute_reply.started": "2024-06-21T16:48:53.114520Z" } }, "outputs": [], "source": [ "# block incorrect text conditioning\n", "upper_bound = np.quantile(\n", " [\n", " (m[\"loss_sem\"] - m[\"loss_sem_no_text\"])\n", " / max(0.01, m[\"loss_sem\"], m[\"loss_sem_no_text\"])\n", " for m in loss_data\n", " ],\n", " 0.99,\n", ")\n", "upper_idx_list = [\n", " m[\"idx\"]\n", " for m in loss_data\n", " if (m[\"loss_sem\"] - m[\"loss_sem_no_text\"])\n", " / max(0.01, m[\"loss_sem\"], m[\"loss_sem_no_text\"])\n", " >= upper_bound\n", "]" ] }, { "cell_type": "code", "execution_count": null, "id": "26582cc1-41af-4228-93fb-20d695f08aa5", "metadata": { "execution": { "iopub.execute_input": "2024-06-21T16:48:56.658859Z", "iopub.status.busy": "2024-06-21T16:48:56.658376Z", "iopub.status.idle": "2024-06-21T16:48:56.661583Z", "shell.execute_reply": "2024-06-21T16:48:56.661032Z", "shell.execute_reply.started": "2024-06-21T16:48:56.658838Z" } }, "outputs": [], "source": [ "print(len(upper_idx_list))" ] }, { "cell_type": "code", "execution_count": null, "id": "1c3c4273", "metadata": { "execution": { "iopub.execute_input": "2024-06-21T16:53:18.312418Z", "iopub.status.busy": "2024-06-21T16:53:18.312134Z", "iopub.status.idle": "2024-06-21T16:53:18.315594Z", "shell.execute_reply": "2024-06-21T16:53:18.315181Z", "shell.execute_reply.started": "2024-06-21T16:53:18.312399Z" } }, "outputs": [], "source": [ "idx = upper_idx_list[0]\n", "idx" ] }, { "cell_type": "code", "execution_count": null, "id": "86d0b339", "metadata": { "execution": { "iopub.execute_input": "2024-06-21T16:53:19.646714Z", "iopub.status.busy": "2024-06-21T16:53:19.646558Z", "iopub.status.idle": "2024-06-21T16:53:19.649816Z", "shell.execute_reply": "2024-06-21T16:53:19.649419Z", "shell.execute_reply.started": "2024-06-21T16:53:19.646699Z" } }, "outputs": [], "source": [ "test_metas[idx]" ] }, { "cell_type": "code", "execution_count": null, "id": "c3b7af1d", "metadata": { "execution": { "iopub.execute_input": "2024-06-21T21:03:23.931612Z", "iopub.status.busy": "2024-06-21T21:03:23.931084Z", "iopub.status.idle": "2024-06-21T21:04:10.753621Z", "shell.execute_reply": "2024-06-21T21:04:10.753046Z", "shell.execute_reply.started": "2024-06-21T21:03:23.931590Z" } }, "outputs": [], "source": [ "arr = test_data[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "pad_idx_arr = np.where(arr == 4096)[0]\n", "if len(pad_idx_arr) > 0:\n", " arr = arr[: pad_idx_arr[0], :]\n", "print(arr.shape)\n", "codec_decode(arr).play()" ] }, { "cell_type": "markdown", "id": "3fa2aed3", "metadata": {}, "source": [ "## (TODO): apply all these filters on a per-dataset basis" ] }, { "cell_type": "code", "execution_count": null, "id": "3c85264a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c5f00cfd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8497b40d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fd670d81", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "44083930", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "a9e13a64", "metadata": {}, "source": [ "### Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "ae139759", "metadata": {}, "outputs": [], "source": [ "blocked_loss_idx_set = set()\n", "\n", "# block hard semantic and super easy\n", "lower_bound = np.quantile(\n", " [m[\"loss_sem_10s\"] for m in loss_data if m[\"idx\"] in dataset_idx_set], 0.01\n", ")\n", "upper_bound = np.quantile(\n", " [m[\"loss_sem_10s\"] for m in loss_data if m[\"idx\"] in dataset_idx_set], 0.95\n", ")\n", "for m in loss_data:\n", " if m[\"loss_sem_10s\"] < lower_bound or m[\"loss_sem_10s\"] > upper_bound:\n", " blocked_loss_idx_set.add(m[\"idx\"])\n", "\n", "# block hard coarse and super easy\n", "lower_bound = np.quantile(\n", " [m[\"loss_coarse_10s\"] for m in loss_data if m[\"idx\"] in dataset_idx_set], 0.01\n", ")\n", "upper_bound = np.quantile(\n", " [m[\"loss_coarse_10s\"] for m in loss_data if m[\"idx\"] in dataset_idx_set], 0.95\n", ")\n", "for m in loss_data:\n", " if m[\"loss_coarse_10s\"] < lower_bound or m[\"loss_coarse_10s\"] > upper_bound:\n", " blocked_loss_idx_set.add(m[\"idx\"])\n", "\n", "# block incorrect text\n", "upper_bound = np.quantile(\n", " [\n", " (m[\"loss_sem\"] - m[\"loss_sem_no_text\"])\n", " / max(0.01, m[\"loss_sem\"], m[\"loss_sem_no_text\"])\n", " for m in loss_data\n", " if m[\"idx\"] in dataset_idx_set\n", " ],\n", " 0.95,\n", ")\n", "for m in loss_data:\n", " if (m[\"loss_sem\"] - m[\"loss_sem_no_text\"]) / max(\n", " 0.01, m[\"loss_sem\"], m[\"loss_sem_no_text\"]\n", " ) > upper_bound:\n", " blocked_loss_idx_set.add(m[\"idx\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "480c8bad", "metadata": {}, "outputs": [], "source": [ "# TODO: investigate biggest outliers and define cutoffs" ] }, { "cell_type": "code", "execution_count": null, "id": "20dc19cc", "metadata": {}, "outputs": [], "source": [ "dataset_idx_set = set()\n", "for idx, m in enumerate(test_metas):\n", " if m[\"dataset\"] not in (\"genius_hq_lyrics\", \"pond5_music\"):\n", " continue\n", " if m[\"end_s\"] - m[\"start_s\"] < 20:\n", " continue\n", " dataset_idx_set.add(idx)" ] }, { "cell_type": "code", "execution_count": null, "id": "654feec4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d0d2dd01", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "814638e1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "5f20e852", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "124a380a-9be0-402b-aec8-f26b9ea9d21d", "metadata": { "execution": { "iopub.execute_input": "2024-07-09T19:18:00.452135Z", "iopub.status.busy": "2024-07-09T19:18:00.451695Z", "iopub.status.idle": "2024-07-09T19:18:00.496105Z", "shell.execute_reply": "2024-07-09T19:18:00.495452Z", "shell.execute_reply.started": "2024-07-09T19:18:00.452113Z" } }, "outputs": [], "source": [ "loss_2b_data = read_jsonl(\n", " \"/app/suno/data/dpo/chirp_v4_multi/eval_2b_raw_bt28_val_2node/loss.jsonl\"\n", ") # val\n", "print(len(loss_2b_data))\n", "loss_2b_df = pd.DataFrame(loss_2b_data[1:], columns=loss_2b_data[0])" ] }, { "cell_type": "code", "execution_count": null, "id": "da3c310b-4669-45ea-bb3f-4418ac5ec1c1", "metadata": { "execution": { "iopub.execute_input": "2024-07-09T19:18:01.161291Z", "iopub.status.busy": "2024-07-09T19:18:01.160830Z", "iopub.status.idle": "2024-07-09T19:18:01.165099Z", "shell.execute_reply": "2024-07-09T19:18:01.164572Z", "shell.execute_reply.started": "2024-07-09T19:18:01.161271Z" } }, "outputs": [], "source": [ "loss_2b_df[\"idx\"].nunique()" ] }, { "cell_type": "code", "execution_count": null, "id": "67d052e0-292b-4005-bb9d-cb55ec6f1abc", "metadata": { "execution": { "iopub.execute_input": "2024-07-09T19:18:01.693079Z", "iopub.status.busy": "2024-07-09T19:18:01.692910Z", "iopub.status.idle": "2024-07-09T19:18:01.695477Z", "shell.execute_reply": "2024-07-09T19:18:01.694973Z", "shell.execute_reply.started": "2024-07-09T19:18:01.693062Z" } }, "outputs": [], "source": [ "import seaborn as sns" ] }, { "cell_type": "code", "execution_count": null, "id": "a7323007-f770-4354-a6af-28d1ff64cc42", "metadata": { "execution": { "iopub.execute_input": "2024-07-09T19:18:11.760537Z", "iopub.status.busy": "2024-07-09T19:18:11.759863Z", "iopub.status.idle": "2024-07-09T19:18:11.762933Z", "shell.execute_reply": "2024-07-09T19:18:11.762389Z", "shell.execute_reply.started": "2024-07-09T19:18:11.760515Z" } }, "outputs": [], "source": [ "# sns.pairplot(loss_2b_df[[\"loss_sem\", \"loss_sem_no_text\", \"loss_coarse\", \"loss_coarse_no_text\"]], kind=\"kde\")" ] }, { "cell_type": "code", "execution_count": null, "id": "8c41449b-499a-417a-ad1b-0764d2893984", "metadata": { "execution": { "iopub.execute_input": "2024-07-09T19:18:51.562950Z", "iopub.status.busy": "2024-07-09T19:18:51.562602Z", "iopub.status.idle": "2024-07-09T19:18:51.568898Z", "shell.execute_reply": "2024-07-09T19:18:51.568315Z", "shell.execute_reply.started": "2024-07-09T19:18:51.562928Z" } }, "outputs": [], "source": [ "loss_2b_df[\"sem_text_diff\"] = (loss_2b_df[\"loss_sem\"] - loss_2b_df[\"loss_sem_no_text\"]) / loss_2b_df[\"loss_sem\"]\n", "loss_2b_df[\"coarse_text_diff\"] = (loss_2b_df[\"loss_coarse\"] - loss_2b_df[\"loss_coarse_no_text\"]) / loss_2b_df[\"loss_coarse\"]\n", "loss_2b_df[\"sem_10s_text_diff\"] = (loss_2b_df[\"loss_sem_10s\"] - loss_2b_df[\"loss_sem_no_text_10s\"]) / loss_2b_df[\"loss_sem_10s\"]\n", "loss_2b_df[\"coarse_10s_text_diff\"] = (loss_2b_df[\"loss_coarse_10s\"] - loss_2b_df[\"loss_coarse_no_text_10s\"]) / loss_2b_df[\"loss_coarse_10s\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "8f412957-f9de-418a-b401-ae54a11e6257", "metadata": { "execution": { "iopub.execute_input": "2024-07-09T19:20:13.096182Z", "iopub.status.busy": "2024-07-09T19:20:13.095612Z", "iopub.status.idle": "2024-07-09T19:20:13.274051Z", "shell.execute_reply": "2024-07-09T19:20:13.273501Z", "shell.execute_reply.started": "2024-07-09T19:20:13.096158Z" } }, "outputs": [], "source": [ "loss_2b_df[\"coarse_10s_text_diff\"].hist(bins=np.linspace(-2, 2, 100))" ] }, { "cell_type": "code", "execution_count": null, "id": "41ce3c95-f22b-4c7b-b3b3-5f684fb59be9", "metadata": { "execution": { "iopub.execute_input": "2024-07-06T13:52:50.058203Z", "iopub.status.busy": "2024-07-06T13:52:50.057845Z", "iopub.status.idle": "2024-07-06T13:52:50.061790Z", "shell.execute_reply": "2024-07-06T13:52:50.061314Z", "shell.execute_reply.started": "2024-07-06T13:52:50.058182Z" } }, "outputs": [], "source": [ "loss_2b_df.columns" ] }, { "cell_type": "code", "execution_count": null, "id": "7c20dbc7-29cd-40b0-b29d-37e183ac5aed", "metadata": { "execution": { "iopub.execute_input": "2024-07-06T23:10:27.415571Z", "iopub.status.busy": "2024-07-06T23:10:27.415262Z", "iopub.status.idle": "2024-07-06T23:10:27.419079Z", "shell.execute_reply": "2024-07-06T23:10:27.418662Z", "shell.execute_reply.started": "2024-07-06T23:10:27.415552Z" } }, "outputs": [], "source": [ "loss_val_data[:5]" ] }, { "cell_type": "code", "execution_count": null, "id": "55957bcb-f6e3-4f43-a962-8e1a228e0055", "metadata": { "execution": { "iopub.execute_input": "2024-07-07T21:04:07.162568Z", "iopub.status.busy": "2024-07-07T21:04:07.162223Z", "iopub.status.idle": "2024-07-07T21:09:41.000272Z", "shell.execute_reply": "2024-07-07T21:09:40.999574Z", "shell.execute_reply.started": "2024-07-07T21:04:07.162547Z" } }, "outputs": [], "source": [ "loss_val_data = read_jsonl(\n", " \"/app/suno/data/dpo/chirp_v4_multi/eval_2b_raw_bt28_tr/loss.jsonl\"\n", ") # val\n", "print(len(loss_val_data))\n", "loss_val_df = pd.DataFrame(loss_val_data)" ] }, { "cell_type": "code", "execution_count": null, "id": "8a3e3746-2af7-4987-ae56-e21187790a1f", "metadata": { "execution": { "iopub.execute_input": "2024-07-07T21:10:55.706520Z", "iopub.status.busy": "2024-07-07T21:10:55.706348Z", "iopub.status.idle": "2024-07-07T21:12:26.645614Z", "shell.execute_reply": "2024-07-07T21:12:26.644837Z", "shell.execute_reply.started": "2024-07-07T21:10:55.706501Z" } }, "outputs": [], "source": [ "loss_val_df = pd.DataFrame(loss_val_data[1:], columns=loss_val_data[0])" ] }, { "cell_type": "code", "execution_count": null, "id": "4f9fcbe2-d050-423d-ad0b-0fb56ef522fe", "metadata": { "execution": { "iopub.execute_input": "2024-07-07T21:24:17.494653Z", "iopub.status.busy": "2024-07-07T21:24:17.494256Z", "iopub.status.idle": "2024-07-07T21:24:17.627057Z", "shell.execute_reply": "2024-07-07T21:24:17.626479Z", "shell.execute_reply.started": "2024-07-07T21:24:17.494625Z" } }, "outputs": [], "source": [ "loss_val_df[\"idx\"].nunique()" ] }, { "cell_type": "code", "execution_count": null, "id": "e3c077a2-4700-4eba-a564-5ef9cd908ee9", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" } }, "nbformat": 4, "nbformat_minor": 5 }