{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "execution": { "iopub.execute_input": "2025-06-13T21:17:16.427625Z", "iopub.status.busy": "2025-06-13T21:17:16.427488Z", "iopub.status.idle": "2025-06-13T21:17:16.441695Z", "shell.execute_reply": "2025-06-13T21:17:16.441269Z", "shell.execute_reply.started": "2025-06-13T21:17:16.427604Z" } }, "outputs": [], "source": [ "# setup autoload\n", "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.040680Z", "start_time": "2024-05-16T13:58:19.777010Z" }, "execution": { "iopub.execute_input": "2025-06-13T21:17:16.442253Z", "iopub.status.busy": "2025-06-13T21:17:16.442122Z", "iopub.status.idle": "2025-06-13T21:17:18.956926Z", "shell.execute_reply": "2025-06-13T21:17:18.956344Z", "shell.execute_reply.started": "2025-06-13T21:17:16.442239Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The autoreload extension is already loaded. To reload it, use:\n", " %reload_ext autoreload\n" ] } ], "source": [ "import ast\n", "import os\n", "import shutil\n", "import sys\n", "from collections import defaultdict\n", "\n", "import json\n", "import numpy as np\n", "import pandas as pd\n", "\n", "sys.path.append(\"/home/tony/Work/tony/Preference\")\n", "from preference_data_preparation_auk import *\n", "from preference_helper import *\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n", "from tqdm import tqdm\n", "\n", "pd.set_option(\"display.max_rows\", 500)\n", "pd.set_option(\"display.max_columns\", 500)\n", "pd.set_option(\"display.width\", 1000)\n", "\n", "# setup autoload\n", "%load_ext autoreload\n", "%autoreload 2\n", "\n", "\n", "def custom_parse(x):\n", " try:\n", " return json.loads(x)\n", " except:\n", " return {}" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.082172Z", "start_time": "2024-05-16T13:58:21.041926Z" }, "execution": { "iopub.execute_input": "2025-06-13T21:17:18.957697Z", "iopub.status.busy": "2025-06-13T21:17:18.957491Z", "iopub.status.idle": "2025-06-13T21:17:19.002961Z", "shell.execute_reply": "2025-06-13T21:17:19.002487Z", "shell.execute_reply.started": "2025-06-13T21:17:18.957680Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "N_TOKENS_AUDIO 12000\n" ] } ], "source": [ "OUT_DATA_DIR = \"/app2/suno/data/dpo/auk_t1_v29\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "shutil.copyfile(\n", " \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", " os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", ")\n", "NPZ_DIR = \"/app2/suno/data/dpo/auk_t1_npz\"\n", "N_TOKENS_AUDIO = 25 * 8 * 60\n", "print(\"N_TOKENS_AUDIO\", N_TOKENS_AUDIO)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2025-06-13T21:17:19.003568Z", "iopub.status.busy": "2025-06-13T21:17:19.003421Z", "iopub.status.idle": "2025-06-13T21:30:59.153994Z", "shell.execute_reply": "2025-06-13T21:30:59.153326Z", "shell.execute_reply.started": "2025-06-13T21:17:19.003553Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Loading pickle files: 0%| | 0/2 [00:00 cutoff_date].shape)\n", "# df = df[(df[\"created_at\"] > cutoff_date)].copy()\n", "# print(\"after date cut\", df.shape)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "execution": { "iopub.execute_input": "2025-06-13T21:31:00.043746Z", "iopub.status.busy": "2025-06-13T21:31:00.043615Z", "iopub.status.idle": "2025-06-13T21:31:12.935475Z", "shell.execute_reply": "2025-06-13T21:31:12.934868Z", "shell.execute_reply.started": "2025-06-13T21:31:00.043733Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "after dropna (7568046, 87)\n" ] } ], "source": [ "df = df.dropna(axis=1, how=\"all\")\n", "print(\"after dropna\", df.shape)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.199480Z", "start_time": "2024-05-16T13:58:53.963687Z" }, "execution": { "iopub.execute_input": "2025-06-13T21:31:12.936257Z", "iopub.status.busy": "2025-06-13T21:31:12.936095Z", "iopub.status.idle": "2025-06-13T21:34:51.796703Z", "shell.execute_reply": "2025-06-13T21:34:51.795979Z", "shell.execute_reply.started": "2025-06-13T21:31:12.936241Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "12184904\n", "12184904\n", "pre-downloaded df (7568046, 87)\n", "downloaded df (7568042, 87)\n" ] } ], "source": [ "converted_paths = os.listdir(NPZ_DIR)\n", "print(len(converted_paths))\n", "\n", "converted_paths = set([f.replace(\".npz\", \"\") for f in converted_paths])\n", "print(len(converted_paths))\n", "\n", "print(\"pre-downloaded df\", df.shape)\n", "df[df[\"s3_id\"].isin(converted_paths)].shape\n", "df = df[df[\"s3_id\"].isin(converted_paths)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.467253Z", "start_time": "2024-05-16T13:58:56.207647Z" }, "execution": { "iopub.execute_input": "2025-06-13T21:34:51.797602Z", "iopub.status.busy": "2025-06-13T21:34:51.797434Z", "iopub.status.idle": "2025-06-13T21:34:52.911332Z", "shell.execute_reply": "2025-06-13T21:34:52.910763Z", "shell.execute_reply.started": "2025-06-13T21:34:51.797586Z" } }, "outputs": [ { "data": { "text/plain": [ "task\n", " 5096440\n", "cover 1237182\n", "artist_consistency 642082\n", "artist_cover 233824\n", "upload_extend 169176\n", "extend 158088\n", "artist_extend 31250\n", "Name: count, dtype: int64" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"task\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.592883Z", "start_time": "2024-05-16T13:58:56.470781Z" }, "execution": { "iopub.execute_input": "2025-06-13T21:34:52.912185Z", "iopub.status.busy": "2025-06-13T21:34:52.912019Z", "iopub.status.idle": "2025-06-13T21:34:57.530065Z", "shell.execute_reply": "2025-06-13T21:34:57.529383Z", "shell.execute_reply.started": "2025-06-13T21:34:52.912170Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "preference model_name \n", "False chirp-auk-t1 3784019\n", "True chirp-auk-t1 3784023\n", "Name: count, dtype: int64\n", "before filter on model name (7568042, 87)\n", "after filter on model name (7568042, 87)\n", "is_public\n", "False 7198158\n", "True 369884\n", "Name: count, dtype: int64\n" ] } ], "source": [ "## for 13b this is easy for now\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "print(\"before filter on model name\", df.shape)\n", "df = df[df[\"model_name\"].isin([\"chirp-auk-t1\"])]\n", "# df = df[df[\"model_name\"].isin([\"chirp-v3p5-engine-t-6\"])]\n", "print(\"after filter on model name\", df.shape)\n", "print(df[\"is_public\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.909539Z", "start_time": "2024-05-16T13:58:56.595736Z" }, "execution": { "iopub.execute_input": "2025-06-13T21:34:57.530913Z", "iopub.status.busy": "2025-06-13T21:34:57.530751Z", "iopub.status.idle": "2025-06-13T21:35:12.485742Z", "shell.execute_reply": "2025-06-13T21:35:12.485048Z", "shell.execute_reply.started": "2025-06-13T21:34:57.530896Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "before filter on request id pairs (7568042, 87)\n", "after filter on request id pairs (7568038, 87)\n", "preference model_name \n", "False chirp-auk-t1 3784019\n", "True chirp-auk-t1 3784019\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(\"before filter on request id pairs\", df.shape)\n", "df = df[\n", " df[\"request_id\"].isin(\n", " df[\"request_id\"].value_counts().index[df[\"request_id\"].value_counts() == 2]\n", " )\n", "]\n", "print(\"after filter on request id pairs\", df.shape)\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "assert df.shape[0] == df[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "execution": { "iopub.execute_input": "2025-06-13T21:35:12.486583Z", "iopub.status.busy": "2025-06-13T21:35:12.486416Z", "iopub.status.idle": "2025-06-13T22:01:06.610405Z", "shell.execute_reply": "2025-06-13T22:01:06.609710Z", "shell.execute_reply.started": "2025-06-13T21:35:12.486567Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "unique_requests 3784019\n", "before removing duplicates (7568038, 189)\n", "after removing duplicates (7568038, 182)\n" ] } ], "source": [ "# Let's use the old selection for now -- for quality assurance\n", "# expand the metadata columns -- this takes forever...~ 6 mins\n", "# test_slice = df[\"metadata\"].apply(lambda x: ast.literal_eval(str(x)))\n", "# test_slice = df[\"metadata\"].apply(lambda x: custom_parse(x))\n", "test_slice = df[\"metadata\"] # .apply(lambda x: custom_parse(x))\n", "test_slice_series = test_slice.apply(pd.Series)\n", "df = pd.concat([df, test_slice_series], axis=1, join=\"inner\")\n", "print(\"unique_requests\", df[\"request_id\"].nunique())\n", "# remove the duplicates\n", "print(\"before removing duplicates\", df.shape)\n", "df = df.loc[:, ~df.columns.duplicated()].copy()\n", "print(\"after removing duplicates\", df.shape)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "execution": { "iopub.execute_input": "2025-06-13T22:01:06.655679Z", "iopub.status.busy": "2025-06-13T22:01:06.655436Z", "iopub.status.idle": "2025-06-13T22:01:33.346281Z", "shell.execute_reply": "2025-06-13T22:01:33.345718Z", "shell.execute_reply.started": "2025-06-13T22:01:06.655661Z" } }, "outputs": [ { "data": { "text/plain": [ "pos_diff_preference\n", "1.0 2593211\n", "2.0 1190808\n", "Name: count, dtype: int64" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df.sort_values(by=[\"request_id\", \"preference\", \"diff_preference\"])\n", "df[\"pos_diff_preference\"] = df[\"diff_preference\"].diff()\n", "# df[\"cer_diff_preference\"] = df[\"cer\"].diff()\n", "df[df[\"preference\"]][\"pos_diff_preference\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "execution": { "iopub.execute_input": "2025-06-13T22:01:33.347129Z", "iopub.status.busy": "2025-06-13T22:01:33.346970Z", "iopub.status.idle": "2025-06-13T22:01:40.859475Z", "shell.execute_reply": "2025-06-13T22:01:40.858792Z", "shell.execute_reply.started": "2025-06-13T22:01:33.347114Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive param_experiment\n", "n_tag_2 90407\n", "cfg_steps_240 89922\n", "n_tag_1 89597\n", "temp_s_95 89132\n", "tag_cfg_1 89044\n", "cfg_steps_60 88718\n", "temp_s_85 88496\n", "temp_s_80 87250\n", "cfg_steps_10 86043\n", "tag_cfg_3 85280\n", "mask_control_slider 14080\n", "Name: count, dtype: int64\n" ] } ], "source": [ "try:\n", " print(\"positive\", df[df[\"preference\"]][\"param_experiment\"].value_counts())\n", "except:\n", " pass" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "execution": { "iopub.execute_input": "2025-06-13T22:01:40.860345Z", "iopub.status.busy": "2025-06-13T22:01:40.860181Z", "iopub.status.idle": "2025-06-13T22:03:44.472330Z", "shell.execute_reply": "2025-06-13T22:03:44.471746Z", "shell.execute_reply.started": "2025-06-13T22:01:40.860329Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found 204963 duplicated prompts 102485 unique requests\n", "Found 49454 request_ids with duplicate prompts but not highest play counts in their group\n", "['de68121d-03f0-4e92-96a4-dbfc192103dc', '1e32db78-9c98-4995-8492-a5e98aaf65ac', '730936a6-f6bb-4f43-a54d-d698b88b8cb6', 'fd2c071e-3d91-4149-aa31-28256a78ce9b', 'ff2db895-04ac-4bdf-a73d-ccd4fb68a53b', 'b69720e0-d8b1-4209-a075-c009631fe29e', '1e5b8877-d64f-4982-abe9-83f3963a496d', '5a4ac700-4a0c-4d42-a991-1ca2df3886e0', '4950055f-b604-4197-bd52-92deaaa61f1b', '056306a8-55ac-402f-b1b4-d2f55744e6d5']\n", "Before dedup user gen requests 7568038\n", "After dedup user gen requests 7568038\n" ] } ], "source": [ "# Find duplicated prompts with count > 2\n", "duplicate_entries = df.groupby(\n", " [\"user_id\", \"prompt_text\", \"tags\", \"task\", \"edited_clip_id\"]\n", ").filter(lambda x: len(x) > 2)\n", "print(\n", " \"Found\",\n", " len(duplicate_entries),\n", " \"duplicated prompts\",\n", " len(duplicate_entries[\"request_id\"].unique()),\n", " \"unique requests\",\n", ")\n", "\n", "# Group by user_id, prompt_text, and tags to find duplicate prompt groups\n", "prompt_groups = duplicate_entries.groupby(\n", " [\"user_id\", \"prompt_text\", \"tags\", \"task\", \"edited_clip_id\"]\n", ")\n", "\n", "# For each prompt group, find the request_id with the highest total reaction_play_count\n", "low_play_count_request_ids = []\n", "for prompt_key, prompt_group in prompt_groups:\n", " # Get the sum of reaction_play_count for each request_id in this group\n", " request_play_counts = prompt_group.groupby(\"request_id\")[\n", " \"reaction_play_count\"\n", " ].sum()\n", "\n", " # Find the max play count in this group\n", " max_play_count = request_play_counts.max()\n", "\n", " # Add request_ids that don't have the max play count to our filter list\n", " lower_play_count_request_ids = request_play_counts[\n", " request_play_counts < max_play_count\n", " ].index.tolist()\n", " low_play_count_request_ids.extend(lower_play_count_request_ids)\n", "\n", "# Display the filtered request IDs\n", "print(\n", " f\"Found {len(low_play_count_request_ids)} request_ids with duplicate prompts but not highest play counts in their group\"\n", ")\n", "print(\n", " low_play_count_request_ids[:10]\n", " if len(low_play_count_request_ids) > 10\n", " else low_play_count_request_ids\n", ")\n", "print(\"Before dedup user gen requests\", df.shape[0])\n", "# df = df[~df[\"request_id\"].isin(low_play_count_request_ids)]\n", "print(\"After dedup user gen requests\", df.shape[0])" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:40.799375Z", "start_time": "2024-05-16T13:59:36.394236Z" }, "execution": { "iopub.execute_input": "2025-06-13T22:03:44.473116Z", "iopub.status.busy": "2025-06-13T22:03:44.472956Z", "iopub.status.idle": "2025-06-13T22:04:45.731043Z", "shell.execute_reply": "2025-06-13T22:04:45.730439Z", "shell.execute_reply.started": "2025-06-13T22:03:44.473099Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "133054\n", "good_continue_at\n", "True 7549900\n", "False 18138\n", "Name: count, dtype: int64\n", "\n", " Check some basics... \n", " preference\n", "False 3784019\n", "True 3784019\n", "Name: count, dtype: int64 model_name\n", "chirp-auk-t1 7568038\n", "Name: count, dtype: int64 preference model_name \n", "False chirp-auk-t1 3784019\n", "True chirp-auk-t1 3784019\n", "Name: count, dtype: int64\n", "task\n", " 5096436\n", "cover 1237182\n", "artist_consistency 642082\n", "artist_cover 233824\n", "upload_extend 169176\n", "extend 158088\n", "artist_extend 31250\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df[\"id\"] = df[\"str_id\"]\n", "# get the original duration of the clips, if they are concacted\n", "df[\"original_duration_s\"] = df[\"total_start_s\"] + df[\"duration\"]\n", "# classify the continue at behavoirs by the duration choice\n", "audio_prompt_id_to_continue_at = {}\n", "\n", "for _, row in df[~df[\"continued_parent\"].isna()].iterrows():\n", " audio_prompt_id = row[\"continued_parent\"]\n", " if audio_prompt_id not in audio_prompt_id_to_continue_at:\n", " audio_prompt_id_to_continue_at[audio_prompt_id] = row[\"continue_at\"]\n", " else:\n", " # pick the max\n", " audio_prompt_id = max(\n", " audio_prompt_id_to_continue_at[audio_prompt_id], row[\"continue_at\"]\n", " )\n", "print(len(audio_prompt_id_to_continue_at))\n", "df[\"has_continue_and_start_continue_at\"] = df[\"id\"].apply(\n", " lambda x: audio_prompt_id_to_continue_at.get(x)\n", ")\n", "# we want continue at to be at most of the clip...\n", "df[\"good_continue_at\"] = (\n", " (df[\"has_continue_and_start_continue_at\"] / df[\"duration\"]) > 0.9\n", ") | df[\"has_continue_and_start_continue_at\"].isna()\n", "print(df[\"good_continue_at\"].value_counts())\n", "\n", "\n", "print(\n", " \"\\n Check some basics... \\n\",\n", " df[\"preference\"].value_counts(),\n", " df[\"model_name\"].value_counts(),\n", " df.groupby([\"preference\"])[\"model_name\"].value_counts(),\n", ")\n", "\n", "df = df.sort_values(by=[\"request_id\", \"preference\"])\n", "df[\"duration_rel_diff\"] = df[\"duration\"].diff()\n", "df[\"play_rel_diff\"] = df[\"reaction_play_count\"].diff()\n", "print(df[\"task\"].value_counts())\n", "\n", "# df[\"post_infill_duration\"] = (\n", "# df[\"duration\"]\n", "# + df[\"infill_context_end_s\"]\n", "# - df[\"infill_context_start_s\"]\n", "# - df[\"include_future_s\"]\n", "# - df[\"include_history_s\"]\n", "# - df[\"infill_dur_s\"]\n", "# )" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.035167Z", "start_time": "2024-05-16T13:59:40.801098Z" }, "execution": { "iopub.execute_input": "2025-06-13T22:04:45.734337Z", "iopub.status.busy": "2025-06-13T22:04:45.734064Z", "iopub.status.idle": "2025-06-13T22:05:28.481679Z", "shell.execute_reply": "2025-06-13T22:05:28.481057Z", "shell.execute_reply.started": "2025-06-13T22:04:45.734318Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "after duration 0.9995634271392401\n", "after infill duration 1.0\n", "neg_filter_reaction_play_count 1.0\n", "neg_filter_upvote_count 0.9887\n", "neg_filter_norm_play_frac 1.0\n", "neg_filter_continues 0.9999\n", "----------------\n", "pos_filter_continues 0.9952\n", "pos_filter_reaction_play_count 1.0\n", "pos_filter_relative_play_count 0.9764\n", "pos_filter_cer_diff_preference 1.0\n", "pos_filter_bad_flags 0.9998\n", "after filter on play counts 0.9881\n", "after filter on higher quality 0.3122\n", "----------------\n", "negative 3739101 positive 1107945\n", "----------------\n", "total pair requests 3784019 --> selected pair requests 1095809 frac 0.290 --> total intitial users 327270\n" ] } ], "source": [ "normal_pos_play_count = 3\n", "# this is lower, cause a concat is probably already ensuring that it is good\n", "concat_pos_play_count = 1\n", "# this is a filter on the concated clip\n", "concat_total_play_count = 3\n", "\n", "all_fitlers = (df[\"duration\"] >= 10) & (df[\"duration\"] <= 480)\n", "print(\"after duration\", all_fitlers.sum() / df.shape[0])\n", "infill_duration_filter = ~df[\"task\"].isin(\n", " [\n", " \"infill\",\n", " \"infill_intro\",\n", " \"infill_outro\",\n", " ]\n", ") # | (df[\"post_infill_duration\"] <= 239)\n", "print(\"after infill duration\", infill_duration_filter.sum() / df.shape[0])\n", "# negative fitlers\n", "total_negative = df[~df[\"preference\"]].shape[0]\n", "neg_filter_reaction_play_count = (~df[\"preference\"]) & (df[\"reaction_play_count\"] >= 1)\n", "print(\n", " \"neg_filter_reaction_play_count\",\n", " round(neg_filter_reaction_play_count.sum() / total_negative, 4),\n", ")\n", "neg_filter_upvote_count = (~df[\"preference\"]) & (df[\"upvote_count\"] == 0)\n", "print(\n", " \"neg_filter_upvote_count\",\n", " round(neg_filter_upvote_count.sum() / total_negative, 4),\n", ")\n", "neg_filter_norm_play_frac = (~df[\"preference\"]) & (df[\"norm_play_frac\"] <= 3.1)\n", "print(\n", " \"neg_filter_norm_play_frac\",\n", " round(neg_filter_norm_play_frac.sum() / total_negative, 4),\n", ")\n", "neg_filter_continues = (~df[\"preference\"]) & (\n", " df[\"has_continue_and_start_continue_at\"].isna()\n", ")\n", "print(\n", " \"neg_filter_continues\",\n", " round(neg_filter_continues.sum() / total_negative, 4),\n", ")\n", "\n", "neg_filter_selection_mask = (\n", " all_fitlers\n", " & infill_duration_filter\n", " & neg_filter_reaction_play_count\n", " & neg_filter_upvote_count\n", " & neg_filter_norm_play_frac\n", " & neg_filter_continues\n", ")\n", "\n", "print(\"----------------\")\n", "total_positive = df[df[\"preference\"]].shape[0]\n", "assert total_positive == total_negative\n", "pos_filter_continues = (df[\"preference\"]) & (df[\"good_continue_at\"])\n", "print(\"pos_filter_continues\", round(pos_filter_continues.sum() / total_positive, 4))\n", "pos_filter_reaction_play_count = (df[\"preference\"]) & (df[\"reaction_play_count\"] >= 1)\n", "print(\n", " \"pos_filter_reaction_play_count\",\n", " round(pos_filter_reaction_play_count.sum() / total_positive, 4),\n", ")\n", "pos_filter_relative_play_count = (df[\"preference\"]) & (df[\"play_rel_diff\"] >= 0)\n", "print(\n", " \"pos_filter_relative_play_count\",\n", " round(pos_filter_relative_play_count.sum() / total_positive, 4),\n", ")\n", "pos_filter_cer_diff_preference = (\n", " df[\n", " \"preference\"\n", " ] # & (df[\"pos_diff_preference\"] == 2) # & (df[\"cer_diff_preference\"] < 0.5) & (df[\"cer\"] < 0.99)\n", ")\n", "print(\n", " \"pos_filter_cer_diff_preference\",\n", " round(pos_filter_cer_diff_preference.sum() / total_positive, 4),\n", ")\n", "pos_filter_bad_flags = (\n", " (df[\"preference\"]) & (df[\"flag_count\"] == 0) & (df[\"dislike_count\"] == 0)\n", ")\n", "print(\n", " \"pos_filter_bad_flags\",\n", " round(pos_filter_bad_flags.sum() / total_positive, 4),\n", ")\n", "pos_filter_play_counts = (df[\"preference\"]) & (\n", " (\n", " (df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= concat_pos_play_count)\n", " & (df[\"concat_play_counts\"] >= concat_total_play_count)\n", " )\n", " | (\n", " (~df[\"part_of_concat\"]) & (df[\"reaction_play_count\"] >= normal_pos_play_count)\n", " # & (df[\"norm_play_frac\"] >= 2.1) # this is a bit of a luxury cut...\n", " )\n", " | (df[\"task\"].isin([\"infill\", \"infill_intro\", \"infill_outro\"]))\n", ")\n", "print(\n", " \"after filter on play counts\",\n", " round(pos_filter_play_counts.sum() / total_positive, 4),\n", ")\n", "high_quality_tasks_filter = (\n", " (\n", " df[\"task\"].isin(\n", " [\n", " \"cover\",\n", " \"upload_extend\",\n", " \"cover_extend\",\n", " \"artist_cover\",\n", " \"extend\",\n", " \"artist_consistency\",\n", " \"artist_extend\",\n", " \"\",\n", " ]\n", " )\n", " )\n", " & (\n", " (df[\"upvote_count\"] >= 1) # (df[\"upvote_count\"] >= 1)\n", " | (df[\"reaction_play_count\"] >= 5)\n", " | (df[\"concat_play_counts\"] >= 5)\n", " )\n", " & (\n", " (df[\"part_of_concat\"])\n", " | (\n", " (~df[\"part_of_concat\"])\n", " & (df[\"norm_play_frac\"] >= 5.1) # this is a bit of a luxury cut...\n", " & (\n", " df[\"norm_play_frac\"] >= df[\"reaction_play_count\"] / 3\n", " ) # play duration is not low on average\n", " )\n", " )\n", ")\n", "medium_quality_tasks_filter = (\n", " df[\"task\"].isin(\n", " [\n", " \"infill\",\n", " \"infill_intro\",\n", " \"infill_outro\",\n", " ]\n", " )\n", ") & ( # let more infill through only in this case...\n", " (\n", " df[\"upvote_count\"] >= 0\n", " ) # (df[\"upvote_count\"] >= 1) (df[\"pos_diff_preference\"] == 2)\n", " | (df[\"reaction_play_count\"] >= 1)\n", " | (df[\"concat_play_counts\"] >= 1)\n", ")\n", "pos_filter_higher_quality = (df[\"preference\"]) & (\n", " high_quality_tasks_filter | medium_quality_tasks_filter\n", ")\n", "print(\n", " \"after filter on higher quality\",\n", " round(pos_filter_higher_quality.sum() / total_positive, 4),\n", ")\n", "\n", "user_gen_filter = (\n", " df[\"user_n_clips\"] >= 20\n", ") # user needs to have genereated at least 100 over the time period\n", "\n", "pos_duration_filter = (df[\"task\"] != \"\") | (\n", " (df[\"duration\"] < 8 * 60 - 10)\n", " & ((df[\"duration_rel_diff\"] / df[\"duration\"]) < 0.50)\n", " & (df[\"duration_rel_diff\"] < 60 * 2)\n", ")\n", "\n", "print(\"----------------\")\n", "pos_filter_selectin_mask = (\n", " (df[\"preference\"]) # get basics aligned\n", " & all_fitlers\n", " & infill_duration_filter\n", " & pos_filter_continues\n", " & pos_filter_reaction_play_count\n", " & pos_filter_relative_play_count\n", " & pos_filter_cer_diff_preference\n", " & pos_filter_bad_flags\n", " & pos_filter_play_counts\n", " & pos_filter_higher_quality\n", " & user_gen_filter\n", " & pos_duration_filter\n", ")\n", "print(\n", " \"negative\",\n", " sum(neg_filter_selection_mask),\n", " \"positive\",\n", " sum(pos_filter_selectin_mask),\n", ")\n", "\n", "neg_filter_requests = df[neg_filter_selection_mask][\"request_id\"].unique()\n", "pos_filter_requests = df[pos_filter_selectin_mask][\"request_id\"].unique()\n", "# looking for very strong signal here:\n", "# listen to the positive/negative more than once\n", "# disliked one of the clips\n", "unique_requests = set(pos_filter_requests).intersection(neg_filter_requests)\n", "print(\"----------------\")\n", "print(\n", " \"total pair requests\",\n", " df[\"request_id\"].nunique(),\n", " \" --> selected pair requests\",\n", " len(unique_requests),\n", " f\"frac {len(unique_requests) / df['request_id'].nunique():.3f}\",\n", " \" --> total intitial users\",\n", " df[\"user_id\"].nunique(),\n", ")" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.250737Z", "start_time": "2024-05-16T13:59:41.036434Z" }, "execution": { "iopub.execute_input": "2025-06-13T22:05:28.482476Z", "iopub.status.busy": "2025-06-13T22:05:28.482317Z", "iopub.status.idle": "2025-06-13T22:05:42.711910Z", "shell.execute_reply": "2025-06-13T22:05:42.711292Z", "shell.execute_reply.started": "2025-06-13T22:05:28.482459Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "auk_t1_v29 requests 1095809 clips 2191618 total khrs 112.431; N gpus for 1000 iters 136.976; 4 gpus for x iters 34244.031; n unique users 193067 n pro users 179367\n" ] } ], "source": [ "df_slice = df[df[\"request_id\"].isin(set(unique_requests))].copy()\n", "print(\n", " f\"{os.path.basename(OUT_DATA_DIR)} requests\",\n", " df_slice[\"request_id\"].nunique(),\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"4 gpus for x iters {df_slice.shape[0] / 8 / 2 / 4:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", " f\"n pro users {df_slice[df_slice['is_pro_user']]['user_id'].nunique()}\",\n", ")\n", "# auk_mix_t1_v2 requests 102002 clips 204004 total khrs 9.191; N gpus for 1000 iters 12.750; 4 gpus for x iters 3187.562; n unique users 36408 n pro users 34038\n", "# auk_t1_v1 requests 9179 clips 18358 total khrs 0.854; N gpus for 1000 iters 1.147; 4 gpus for x iters 286.844; n unique users 6288 n pro users 6275\n", "# auk_t1_v2 requests 40903 clips 81806 total khrs 3.864; N gpus for 1000 iters 5.113; 4 gpus for x iters 1278.219; n unique users 21079 n pro users 20966\n", "# auk_t1_v3 requests 102015 clips 204030 total khrs 9.703; N gpus for 1000 iters 12.752; 4 gpus for x iters 3187.969; n unique users 42837 n pro users 42462\n", "# auk_t1_v4 requests 211452 clips 422904 total khrs 20.216; N gpus for 1000 iters 26.431; 4 gpus for x iters 6607.875; n unique users 71182 n pro users 70059\n", "# auk_t1_v9 requests 547743 clips 1095486 total khrs 56.567; N gpus for 1000 iters 68.468; 4 gpus for x iters 17116.969; n unique users 129354 n pro users 125126\n", "# auk_t1_v13 requests 627646 clips 1255292 total khrs 64.530; N gpus for 1000 iters 78.456; 4 gpus for x iters 19613.938; n unique users 139873 n pro users 134738\n", "# auk_t1_v17 requests 494902 clips 989804 total khrs 51.445; N gpus for 1000 iters 61.863; 4 gpus for x iters 15465.688; n unique users 114240 n pro users 109708\n", "# auk_t1_v19 requests 740881 clips 1481762 total khrs 76.133; N gpus for 1000 iters 92.610; 4 gpus for x iters 23152.531; n unique users 154570 n pro users 147261\n", "# auk_t1_v24 requests 717745 clips 1435490 total khrs 74.515; N gpus for 1000 iters 89.718; 4 gpus for x iters 22429.531; n unique users 139344 n pro users 130740\n", "# auk_t1_v29 requests 1097586 clips 2195172 total khrs 112.614; N gpus for 1000 iters 137.198; 4 gpus for x iters 34299.562; n unique users 193138 n pro users 179429" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.277006Z", "start_time": "2024-05-16T13:59:41.252105Z" }, "execution": { "iopub.execute_input": "2025-06-13T22:05:42.712708Z", "iopub.status.busy": "2025-06-13T22:05:42.712542Z", "iopub.status.idle": "2025-06-13T22:05:43.735170Z", "shell.execute_reply": "2025-06-13T22:05:43.734560Z", "shell.execute_reply.started": "2025-06-13T22:05:42.712691Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive in playlist (322170, 188)\n", "task\n", " 1519336\n", "cover 289244\n", "artist_consistency 191794\n", "extend 75714\n", "artist_cover 61594\n", "upload_extend 38014\n", "artist_extend 15922\n", "Name: count, dtype: int64\n" ] } ], "source": [ "test_mask = (df_slice[\"preference\"]) & (\n", " (df_slice[\"is_in_playlist\"]) | (df_slice[\"concat_in_playlist\"])\n", ")\n", "print(\"positive in playlist\", df_slice[test_mask].shape)\n", "print(df_slice[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "execution": { "iopub.execute_input": "2025-06-13T22:05:43.735949Z", "iopub.status.busy": "2025-06-13T22:05:43.735791Z", "iopub.status.idle": "2025-06-13T22:05:43.764212Z", "shell.execute_reply": "2025-06-13T22:05:43.763719Z", "shell.execute_reply.started": "2025-06-13T22:05:43.735932Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "is_public\n", "False 2009877\n", "True 181741\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(df_slice[\"is_public\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "execution": { "iopub.execute_input": "2025-06-13T22:05:43.764862Z", "iopub.status.busy": "2025-06-13T22:05:43.764716Z", "iopub.status.idle": "2025-06-13T22:05:44.158604Z", "shell.execute_reply": "2025-06-13T22:05:44.157956Z", "shell.execute_reply.started": "2025-06-13T22:05:43.764843Z" } }, "outputs": [], "source": [ "df_slice[\"npz_path\"] = df_slice[\"s3_id\"].map(lambda x: f\"{NPZ_DIR}/{x}.npz\")" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "execution": { "iopub.execute_input": "2025-06-13T22:05:44.159392Z", "iopub.status.busy": "2025-06-13T22:05:44.159226Z", "iopub.status.idle": "2025-06-13T22:06:24.571012Z", "shell.execute_reply": "2025-06-13T22:06:24.570393Z", "shell.execute_reply.started": "2025-06-13T22:05:44.159374Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1507280\n", "(2191618, 189)\n", "(703108, 189)\n" ] } ], "source": [ "tr_metas_t1_v7 = read_jsonl(\n", " os.path.join(\"/app2/suno/data/dpo/auk_t1_v6\", f\"meta_tr.jsonl\")\n", ")\n", "tr_metas_t1_v6 = read_jsonl(\n", " os.path.join(\"/app2/suno/data/dpo/auk_t1_v7\", f\"meta_tr.jsonl\")\n", ")\n", "tr_metas_t1_v19 = read_jsonl(\n", " os.path.join(\"/app2/suno/data/dpo/auk_t1_v19\", f\"meta_tr.jsonl\")\n", ")\n", "known_train_ids = set()\n", "for prev_tr_meta in tr_metas_t1_v7:\n", " known_train_ids.add(prev_tr_meta[\"id\"])\n", "for prev_tr_meta in tr_metas_t1_v6:\n", " known_train_ids.add(prev_tr_meta[\"id\"])\n", "for prev_tr_meta in tr_metas_t1_v19:\n", " known_train_ids.add(prev_tr_meta[\"id\"])\n", "print(len(known_train_ids))\n", "print(df_slice.shape)\n", "df_slice = df_slice[~df_slice[\"id\"].isin(known_train_ids)].copy()\n", "print(df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "execution": { "iopub.execute_input": "2025-06-13T22:06:24.571799Z", "iopub.status.busy": "2025-06-13T22:06:24.571639Z", "iopub.status.idle": "2025-06-13T22:06:25.386235Z", "shell.execute_reply": "2025-06-13T22:06:25.385671Z", "shell.execute_reply.started": "2025-06-13T22:06:24.571782Z" } }, "outputs": [], "source": [ "# df_slice[\"created_at\"] = pd.to_datetime(df_slice[\"created_at\"], utc=True)\n", "# cutoff_date = pd.to_datetime(\"2025-05-17\", utc=True)\n", "# print(df_slice.shape, df_slice[df_slice[\"created_at\"] >= cutoff_date].shape)\n", "# df_slice = df_slice[(df_slice[\"created_at\"] >= cutoff_date)].copy()\n", "# print(\"after date cut\", df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "execution": { "iopub.execute_input": "2025-06-13T22:06:25.387004Z", "iopub.status.busy": "2025-06-13T22:06:25.386852Z", "iopub.status.idle": "2025-06-13T22:06:25.401927Z", "shell.execute_reply": "2025-06-13T22:06:25.401459Z", "shell.execute_reply.started": "2025-06-13T22:06:25.386987Z" } }, "outputs": [], "source": [ "# df_slice = df_slice[(df_slice[\"task\"] != \"cover\")].copy()" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T14:00:20.866354Z", "start_time": "2024-05-16T14:00:12.443344Z" }, "execution": { "iopub.execute_input": "2025-06-13T22:06:25.402686Z", "iopub.status.busy": "2025-06-13T22:06:25.402546Z", "iopub.status.idle": "2025-06-13T22:06:25.923166Z", "shell.execute_reply": "2025-06-13T22:06:25.922547Z", "shell.execute_reply.started": "2025-06-13T22:06:25.402671Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(703108, 189)\n", "task\n", " 488854\n", "cover 104652\n", "artist_consistency 55946\n", "extend 21564\n", "artist_cover 19160\n", "upload_extend 9256\n", "artist_extend 3676\n", "Name: count, dtype: int64\n" ] }, { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[29], line 6\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mvalue_counts())\n\u001b[0;32m----> 6\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "# df_slice.to_pickle(\n", "# \"/home/tony/Data/Preference/auk_t1/interesting_clips_auk_t1_20250613_v29_slice.pkl\"\n", "# )\n", "print(df_slice.shape)\n", "print(df_slice[\"task\"].value_counts())\n", "BREAK" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932296Z", "start_time": "2024-05-16T13:59:41.932287Z" }, "execution": { "iopub.execute_input": "2025-06-14T00:13:31.726271Z", "iopub.status.busy": "2025-06-14T00:13:31.725862Z", "iopub.status.idle": "2025-06-14T00:13:32.206764Z", "shell.execute_reply": "2025-06-14T00:13:32.206156Z", "shell.execute_reply.started": "2025-06-14T00:13:31.726247Z" } }, "outputs": [], "source": [ "# don't have continue at\n", "df_slice[\"request_id\"] = df_slice[\"request_id\"].astype(str)\n", "# df_slice[\"npz_path\"] = df_slice[\"npz_path\"].apply(lambda x: str(x).replace(\"_npz\", \"_npz/\"))\n", "# df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique(), df_slice[\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-06-14T00:13:32.207790Z", "iopub.status.busy": "2025-06-14T00:13:32.207619Z", "iopub.status.idle": "2025-06-14T00:13:36.037051Z", "shell.execute_reply": "2025-06-14T00:13:36.036424Z", "shell.execute_reply.started": "2025-06-14T00:13:32.207772Z" } }, "outputs": [], "source": [ "print(df_slice.shape)\n", "df_slice = df_slice[df_slice[\"request_id\"].apply(lambda x: len(x) > 3)]\n", "print(df_slice.shape)\n", "# df_slice = df_slice[df_slice[\"is_pro_user\"]].copy()\n", "print(df_slice.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932966Z", "start_time": "2024-05-16T13:59:41.932957Z" }, "execution": { "iopub.execute_input": "2025-06-14T00:13:36.037851Z", "iopub.status.busy": "2025-06-14T00:13:36.037689Z", "iopub.status.idle": "2025-06-14T00:13:36.243884Z", "shell.execute_reply": "2025-06-14T00:13:36.243256Z", "shell.execute_reply.started": "2025-06-14T00:13:36.037833Z" } }, "outputs": [], "source": [ "final_filtered_requests = df_slice[\"request_id\"].astype(str).unique()\n", "# final_filtered_requests = df_slice[df_slice[\"is_pro_user\"]][\"request_id\"].astype(str).unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934277Z", "start_time": "2024-05-16T13:59:41.934268Z" }, "execution": { "iopub.execute_input": "2025-06-14T00:13:36.245208Z", "iopub.status.busy": "2025-06-14T00:13:36.245016Z", "iopub.status.idle": "2025-06-14T00:13:42.331401Z", "shell.execute_reply": "2025-06-14T00:13:42.330765Z", "shell.execute_reply.started": "2025-06-14T00:13:36.245189Z" } }, "outputs": [], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df # .reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df # .reset_index()\n", "train_df = train_df.reset_index(drop=True)\n", "val_df = val_df.reset_index(drop=True)\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.935620Z", "start_time": "2024-05-16T13:59:41.935613Z" }, "execution": { "iopub.execute_input": "2025-06-14T00:13:42.332200Z", "iopub.status.busy": "2025-06-14T00:13:42.332036Z", "iopub.status.idle": "2025-06-14T00:14:08.612184Z", "shell.execute_reply": "2025-06-14T00:14:08.611610Z", "shell.execute_reply.started": "2025-06-14T00:13:42.332182Z" } }, "outputs": [], "source": [ "total_duration = 0\n", "for i, row in tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " total_duration += row[\"duration\"]\n", " except Exception as E:\n", " print(i, row)\n", " print(E)\n", " raise ValueError()\n", "\n", "print(\n", " f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1000} nodes, {train_df.shape[0] / 8 / 8 / 16} iters\"\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936268Z", "start_time": "2024-05-16T13:59:41.936260Z" }, "execution": { "iopub.execute_input": "2025-06-14T00:14:08.612943Z", "iopub.status.busy": "2025-06-14T00:14:08.612784Z", "iopub.status.idle": "2025-06-14T00:15:51.623684Z", "shell.execute_reply": "2025-06-14T00:15:51.623139Z", "shell.execute_reply.started": "2025-06-14T00:14:08.612927Z" } }, "outputs": [], "source": [ "make_dataset(\n", " val_df, OUT_DATA_DIR, is_val=True, npz_dir=NPZ_DIR, t_data_memmap=N_TOKENS_AUDIO\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-06-14T00:15:51.624440Z", "iopub.status.busy": "2025-06-14T00:15:51.624285Z", "iopub.status.idle": "2025-06-14T00:15:52.009395Z", "shell.execute_reply": "2025-06-14T00:15:52.008918Z", "shell.execute_reply.started": "2025-06-14T00:15:51.624423Z" } }, "outputs": [], "source": [ "# test_npz = np.load(\"/app/suno/data/dpo/30b_npz/26d19085-18da-4701-af43-122684543891.npz\")\n", "# for k in test_npz.keys():\n", "# print(k)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936964Z", "start_time": "2024-05-16T13:59:41.936957Z" }, "execution": { "iopub.execute_input": "2025-06-14T00:15:52.010094Z", "iopub.status.busy": "2025-06-14T00:15:52.009946Z" } }, "outputs": [], "source": [ "make_dataset(\n", " train_df, OUT_DATA_DIR, is_val=False, npz_dir=NPZ_DIR, t_data_memmap=N_TOKENS_AUDIO\n", ")" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.937879Z", "start_time": "2024-05-16T13:59:41.937870Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, N_TOKENS_AUDIO, 1)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "task_counts = Counter()\n", "for test_meta in test_metas:\n", " task_counts[test_meta.get(\"task\")] += 1\n", "print(task_counts)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.938629Z", "start_time": "2024-05-16T13:59:41.938621Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as codec_encode,\n", "# decode_stream_to_full_audio as codec_decode,\n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# decode as decode\n", "# )\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "# assert len(test_metas) == len(mm)\n", "# idx_list = list(range(len(test_metas)))\n", "# # random.shuffle(idx_list)\n", "# # idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939205Z", "start_time": "2024-05-16T13:59:41.939198Z" } }, "outputs": [], "source": [ "# import random\n", "# idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "# assert \"original_duration_s\" in test_metas[idx]\n", "# # positive index should be shifted by 1\n", "# pos_idx = idx + 1\n", "# print(\n", "# \"tags:\",\n", "# test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", "# test_metas[idx].get(\"tags\"),\n", "# )\n", "# arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[: pad_idx_arr[0], :]\n", "# pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pos_pad_idx_arr) > 0:\n", "# pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "# a = decode(arr)\n", "# print(\"\\n negative example \\n\", test_metas[idx])\n", "# a.play(compress=False)\n", "# pos_a = decode(pos_arr)\n", "# print(\"\\n positive example \\n\", test_metas[pos_idx])\n", "# pos_a.play(compress=False)\n", "# print(\n", "# \"text:\",\n", "# test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", "# test_metas[idx].get(\"text\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939977Z", "start_time": "2024-05-16T13:59:41.939969Z" } }, "outputs": [], "source": [ "# val_df[val_df[\"tags\"] == 'a vibrant blend of experimental jazz fusion, drum-and-bass and swagger fuzzed-out guitars']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.940610Z", "start_time": "2024-05-16T13:59:41.940603Z" } }, "outputs": [], "source": [ "# from collections import Counter\n", "# c = Counter()\n", "# for _, row in df_slice.iterrows():\n", "# # print(row[\"metadata\"])\n", "# for k in ast.literal_eval(row[\"metadata\"]).keys():\n", "# c[k] += 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941167Z", "start_time": "2024-05-16T13:59:41.941159Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941801Z", "start_time": "2024-05-16T13:59:41.941793Z" } }, "outputs": [], "source": [ "def validation_on_metas(input_metas):\n", " total_bad = 0\n", " total_good = 0\n", " for idx in range(len(input_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if input_metas[idx].get(\"tags\") != input_metas[pos_idx].get(\"tags\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", " print(total_good, total_bad)\n", " return\n", "\n", "\n", "validation_on_metas(test_metas)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.942520Z", "start_time": "2024-05-16T13:59:41.942511Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.943072Z", "start_time": "2024-05-16T13:59:41.943065Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)\n", "# make sure they are offset by 1 and exactly 1\n", "for i, j in zip(n_neg_tr, n_pos_tr):\n", " assert i == j - 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.944246Z", "start_time": "2024-05-16T13:59:41.944237Z" } }, "outputs": [], "source": [ "total_iters = len(n_neg_tr) + len(n_pos_tr)\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945249Z", "start_time": "2024-05-16T13:59:41.945241Z" } }, "outputs": [], "source": [ "print(\"1 epoch per batch 4, total\", total_iters / 8 / 2 / 4)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# import time\n", "# time.sleep(60 * 60 * 2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945972Z", "start_time": "2024-05-16T13:59:41.945964Z" } }, "outputs": [], "source": [ "!cd /home/tony/Work/tony/slurm/bluejay && sbatch sbatch_ipo_bluejay" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import shutil\n", "\n", "# Basic file copy\n", "shutil.copy(\n", " \"/home/tony/Work/tony/Preference/make_dataset_auk_t1.ipynb\",\n", " os.path.join(OUT_DATA_DIR, \"make_dataset.ipynb\"),\n", ")\n", "print(\"Cache kept!\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# some gymathtics loading prev data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.946562Z", "start_time": "2024-05-16T13:59:41.946555Z" } }, "outputs": [], "source": [ "# prev_v3_data = \"/app/suno/data/dpo/7v_v20_full/\"\n", "\n", "# test_val_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_val.jsonl\"))\n", "# test_tr_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_tr.jsonl\"))\n", "\n", "# all_ids = set()\n", "# for meta in test_val_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# for meta in test_tr_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# print(len(all_ids), len(test_val_metas) + len(test_tr_metas))\n", "\n", "# all_ids = list(all_ids)\n", "# with open(\"/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id.json\", \"w\") as fp:\n", "# json.dump(all_ids, fp)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# x_data = train_df[train_df[\"preference\"]][\"similarity\"]\n", "# y_data = train_df[~train_df[\"preference\"]][\"similarity\"]\n", "# from matplotlib.colors import LogNorm\n", "\n", "# fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 10))\n", "\n", "# # 2D Histogram\n", "# h = ax1.hist2d(\n", "# x_data,\n", "# y_data,\n", "# bins=(50, 50),\n", "# cmap=\"coolwarm\",\n", "# range=[[0, 1], [0, 1]],\n", "# norm=LogNorm(),\n", "# )\n", "\n", "# ax1.set_xlabel(\"Semantic Distance (Preferred)\")\n", "# ax1.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "# ax1.set_title(\n", "# \"2D Histogram of Semantic Distances: Preferred vs Non-Preferred (Log Scale)\"\n", "# )\n", "\n", "# cbar1 = plt.colorbar(h[3], ax=ax1)\n", "# cbar1.set_label(\"Number of Request IDs (Log Scale)\")\n", "\n", "# # Scatter plot\n", "# ax2.scatter(x_data, y_data, alpha=0.1, s=1)\n", "# ax2.set_xlabel(\"Semantic Distance (Preferred)\")\n", "# ax2.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "# ax2.set_title(\"Scatter Plot of Semantic Distances: Preferred vs Non-Preferred\")\n", "# ax2.set_xlim(0, 1)\n", "# ax2.set_ylim(0, 1)\n", "\n", "# plt.tight_layout()\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# train_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_tr.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "train_info.keys()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env_dev", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }