{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T18:57:46.227249Z", "iopub.status.busy": "2025-11-23T18:57:46.227110Z", "iopub.status.idle": "2025-11-23T18:57:46.241055Z", "shell.execute_reply": "2025-11-23T18:57:46.240637Z", "shell.execute_reply.started": "2025-11-23T18:57:46.227232Z" } }, "outputs": [], "source": [ "# setup autoload\n", "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.040680Z", "start_time": "2024-05-16T13:58:19.777010Z" }, "execution": { "iopub.execute_input": "2025-11-23T18:57:46.241660Z", "iopub.status.busy": "2025-11-23T18:57:46.241532Z", "iopub.status.idle": "2025-11-23T18:57:48.860978Z", "shell.execute_reply": "2025-11-23T18:57:48.860441Z", "shell.execute_reply.started": "2025-11-23T18:57:46.241647Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The autoreload extension is already loaded. To reload it, use:\n", " %reload_ext autoreload\n" ] } ], "source": [ "import ast\n", "import os\n", "import shutil\n", "import sys\n", "from collections import defaultdict\n", "\n", "import json\n", "import numpy as np\n", "import pandas as pd\n", "from preference_data_preparation_auk import *\n", "from preference_helper import *\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n", "from tqdm import tqdm\n", "\n", "pd.set_option(\"display.max_rows\", 500)\n", "pd.set_option(\"display.max_columns\", 500)\n", "pd.set_option(\"display.width\", 1000)\n", "\n", "# setup autoload\n", "%load_ext autoreload\n", "%autoreload 2\n", "\n", "\n", "def custom_parse(x):\n", " try:\n", " return json.loads(x)\n", " except:\n", " return {}" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.082172Z", "start_time": "2024-05-16T13:58:21.041926Z" }, "execution": { "iopub.execute_input": "2025-11-23T18:57:48.861760Z", "iopub.status.busy": "2025-11-23T18:57:48.861547Z", "iopub.status.idle": "2025-11-23T18:57:48.930943Z", "shell.execute_reply": "2025-11-23T18:57:48.930474Z", "shell.execute_reply.started": "2025-11-23T18:57:48.861745Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "N_TOKENS_AUDIO 12000\n" ] } ], "source": [ "OUT_DATA_DIR = \"/app2/suno/data/dpo/crow_t1_v62\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "shutil.copyfile(\n", " \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", " os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", ")\n", "NPZ_DIR = \"/app2/suno/data/dpo/crow_t1_npz\"\n", "N_TOKENS_AUDIO = 25 * 8 * 60\n", "print(\"N_TOKENS_AUDIO\", N_TOKENS_AUDIO)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T18:57:48.931556Z", "iopub.status.busy": "2025-11-23T18:57:48.931415Z", "iopub.status.idle": "2025-11-23T19:02:08.583018Z", "shell.execute_reply": "2025-11-23T19:02:08.582440Z", "shell.execute_reply.started": "2025-11-23T18:57:48.931542Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "before dropna (9414156, 91)\n", "after dropna (9414156, 86)\n" ] } ], "source": [ "df = pd.read_pickle(\n", " \"/home/tony/Data/Preference/crow_t1/interesting_clips_crow_t1_20251025.pkl\"\n", ")\n", "print(\"before dropna\", df.shape)\n", "df = df.dropna(axis=1, how=\"all\")\n", "print(\"after dropna\", df.shape)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.199480Z", "start_time": "2024-05-16T13:58:53.963687Z" }, "execution": { "iopub.execute_input": "2025-11-23T19:02:08.583771Z", "iopub.status.busy": "2025-11-23T19:02:08.583618Z", "iopub.status.idle": "2025-11-23T19:41:00.087633Z", "shell.execute_reply": "2025-11-23T19:41:00.086829Z", "shell.execute_reply.started": "2025-11-23T19:02:08.583756Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "21402125\n", "21402125\n", "pre-downloaded df (9414156, 86)\n", "downloaded df (9414154, 86)\n" ] } ], "source": [ "converted_paths = os.listdir(NPZ_DIR)\n", "print(len(converted_paths))\n", "\n", "converted_paths = set([f.replace(\".npz\", \"\") for f in converted_paths])\n", "print(len(converted_paths))\n", "\n", "print(\"pre-downloaded df\", df.shape)\n", "df[df[\"s3_id\"].isin(converted_paths)].shape\n", "df = df[df[\"s3_id\"].isin(converted_paths)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.467253Z", "start_time": "2024-05-16T13:58:56.207647Z" }, "execution": { "iopub.execute_input": "2025-11-23T19:41:00.088555Z", "iopub.status.busy": "2025-11-23T19:41:00.088381Z", "iopub.status.idle": "2025-11-23T19:41:00.529266Z", "shell.execute_reply": "2025-11-23T19:41:00.528781Z", "shell.execute_reply.started": "2025-11-23T19:41:00.088538Z" } }, "outputs": [ { "data": { "text/plain": [ "task\n", " 5701671\n", "cover 2376190\n", "artist_consistency 573350\n", "artist_cover 335386\n", "extend 124510\n", "playlist_condition 93188\n", "overpainting 53848\n", "upload_extend 43790\n", "underpainting 39118\n", "infill 33138\n", "artist_extend 24392\n", "stem_condition 12825\n", "stem_condition_infill 2748\n", "Name: count, dtype: int64" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"task\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.592883Z", "start_time": "2024-05-16T13:58:56.470781Z" }, "execution": { "iopub.execute_input": "2025-11-23T19:41:00.530009Z", "iopub.status.busy": "2025-11-23T19:41:00.529858Z", "iopub.status.idle": "2025-11-23T19:41:06.759382Z", "shell.execute_reply": "2025-11-23T19:41:06.758776Z", "shell.execute_reply.started": "2025-11-23T19:41:00.529995Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "preference model_name \n", "False chirp-crow-t1 4699290\n", " chirp-crow-t2 7786\n", "True chirp-crow-t1 4699291\n", " chirp-crow-t2 7787\n", "Name: count, dtype: int64\n", "before filter on model name (9414154, 86)\n", "after filter on model name (9414154, 86)\n", "is_public\n", "False 8945981\n", "True 468173\n", "Name: count, dtype: int64\n" ] } ], "source": [ "## for 13b this is easy for now\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "print(\"before filter on model name\", df.shape)\n", "df = df[df[\"model_name\"].isin([\"chirp-crow-t1\", \"chirp-crow-t2\"])]\n", "# df = df[df[\"model_name\"].isin([\"chirp-v3p5-engine-t-6\"])]\n", "print(\"after filter on model name\", df.shape)\n", "print(df[\"is_public\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.909539Z", "start_time": "2024-05-16T13:58:56.595736Z" }, "execution": { "iopub.execute_input": "2025-11-23T19:41:06.760192Z", "iopub.status.busy": "2025-11-23T19:41:06.760032Z", "iopub.status.idle": "2025-11-23T19:41:28.764697Z", "shell.execute_reply": "2025-11-23T19:41:28.763905Z", "shell.execute_reply.started": "2025-11-23T19:41:06.760176Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "before filter on request id pairs (9414154, 86)\n", "after filter on request id pairs (9414152, 86)\n", "preference model_name \n", "False chirp-crow-t1 4699290\n", " chirp-crow-t2 7786\n", "True chirp-crow-t1 4699290\n", " chirp-crow-t2 7786\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(\"before filter on request id pairs\", df.shape)\n", "df = df[\n", " df[\"request_id\"].isin(\n", " df[\"request_id\"].value_counts().index[df[\"request_id\"].value_counts() == 2]\n", " )\n", "]\n", "print(\"after filter on request id pairs\", df.shape)\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "assert df.shape[0] == df[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T19:41:28.765739Z", "iopub.status.busy": "2025-11-23T19:41:28.765537Z", "iopub.status.idle": "2025-11-23T20:14:57.653776Z", "shell.execute_reply": "2025-11-23T20:14:57.653039Z", "shell.execute_reply.started": "2025-11-23T19:41:28.765721Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "unique_requests 4707076\n", "before removing duplicates (9414152, 211)\n", "after removing duplicates (9414152, 204)\n" ] } ], "source": [ "# Let's use the old selection for now -- for quality assurance\n", "# expand the metadata columns -- this takes forever...~ 6 mins\n", "# test_slice = df[\"metadata\"].apply(lambda x: ast.literal_eval(str(x)))\n", "# test_slice = df[\"metadata\"].apply(lambda x: custom_parse(x))\n", "test_slice = df[\"metadata\"] # .apply(lambda x: custom_parse(x))\n", "test_slice_series = test_slice.apply(pd.Series)\n", "df = pd.concat([df, test_slice_series], axis=1, join=\"inner\")\n", "print(\"unique_requests\", df[\"request_id\"].nunique())\n", "# remove the duplicates\n", "print(\"before removing duplicates\", df.shape)\n", "df = df.loc[:, ~df.columns.duplicated()].copy()\n", "print(\"after removing duplicates\", df.shape)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T20:14:57.654718Z", "iopub.status.busy": "2025-11-23T20:14:57.654552Z", "iopub.status.idle": "2025-11-23T20:15:37.091001Z", "shell.execute_reply": "2025-11-23T20:15:37.090360Z", "shell.execute_reply.started": "2025-11-23T20:14:57.654701Z" } }, "outputs": [ { "data": { "text/plain": [ "pos_diff_preference\n", "1.0 3214089\n", "2.0 1492987\n", "Name: count, dtype: int64" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df.sort_values(by=[\"request_id\", \"preference\", \"diff_preference\"])\n", "df[\"pos_diff_preference\"] = df[\"diff_preference\"].diff()\n", "# df[\"cer_diff_preference\"] = df[\"cer\"].diff()\n", "df[df[\"preference\"]][\"pos_diff_preference\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T20:15:37.091904Z", "iopub.status.busy": "2025-11-23T20:15:37.091737Z", "iopub.status.idle": "2025-11-23T20:15:47.991279Z", "shell.execute_reply": "2025-11-23T20:15:47.990539Z", "shell.execute_reply.started": "2025-11-23T20:15:37.091887Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive param_experiment\n", "mask_control_slider 270096\n", "cfg_steps_10 230480\n", "text_cfg_15 214997\n", "text_cfg_12 213137\n", "cfg_steps_30 156409\n", "max_tag_len_1024 103295\n", "cfg_steps_480 80040\n", "def_audio_text_cfg_05 33577\n", "def_audio_cfg_05 33238\n", "tag_cfg_05 13568\n", "tag_cfg_20 8801\n", "temp_s_85 7928\n", "temp_s_95 7917\n", "temp_s_100 7908\n", "temp_s_80 7834\n", "diff_text_cfg_25 5132\n", "diff_text_cfg_15 4912\n", "n_tag_3 4299\n", "text_cfg_20 1034\n", "Name: count, dtype: int64\n" ] } ], "source": [ "try:\n", " print(\"positive\", df[df[\"preference\"]][\"param_experiment\"].value_counts())\n", "except:\n", " pass" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T20:15:47.992242Z", "iopub.status.busy": "2025-11-23T20:15:47.992071Z", "iopub.status.idle": "2025-11-23T20:18:56.720238Z", "shell.execute_reply": "2025-11-23T20:18:56.719646Z", "shell.execute_reply.started": "2025-11-23T20:15:47.992225Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found 541996 duplicated prompts 270998 unique requests\n", "Found 142909 request_ids with duplicate prompts but not highest play counts in their group\n", "['1b4804f4-5ae7-4771-b155-7565ee5aef84', '2fc0a3c9-5b91-4c26-aad2-53046d62a65e', 'e1bed84f-9d4e-40d9-aa23-78be7f355f1c', '0adde2d1-5cee-4877-a59f-9ce311e46132', '0bfc985d-734f-41bb-8670-187e1d7daa23', '132feac1-f6c9-4034-a7fe-574c4e070e0b', '16e0cf10-93f1-4eb5-a521-3ec4d9a25449', '337e27d0-9729-40fa-a060-b3040012f49b', '5e1c79f3-48cc-43c3-a57d-2f43861935eb', '75d79410-0c00-4758-ba93-83140f31e0a6']\n", "Before dedup user gen requests 9414152\n", "After dedup user gen requests 9414152\n" ] } ], "source": [ "# Find duplicated prompts with count > 2\n", "duplicate_entries = df.groupby(\n", " [\"user_id\", \"prompt_text\", \"tags\", \"task\", \"edited_clip_id\"]\n", ").filter(lambda x: len(x) > 2)\n", "print(\n", " \"Found\",\n", " len(duplicate_entries),\n", " \"duplicated prompts\",\n", " len(duplicate_entries[\"request_id\"].unique()),\n", " \"unique requests\",\n", ")\n", "\n", "# Group by user_id, prompt_text, and tags to find duplicate prompt groups\n", "prompt_groups = duplicate_entries.groupby(\n", " [\"user_id\", \"prompt_text\", \"tags\", \"task\", \"edited_clip_id\"]\n", ")\n", "\n", "# For each prompt group, find the request_id with the highest total reaction_play_count\n", "low_play_count_request_ids = []\n", "for prompt_key, prompt_group in prompt_groups:\n", " # Get the sum of reaction_play_count for each request_id in this group\n", " request_play_counts = prompt_group.groupby(\"request_id\")[\n", " \"reaction_play_count\"\n", " ].sum()\n", "\n", " # Find the max play count in this group\n", " max_play_count = request_play_counts.max()\n", "\n", " # Add request_ids that don't have the max play count to our filter list\n", " lower_play_count_request_ids = request_play_counts[\n", " request_play_counts < max_play_count\n", " ].index.tolist()\n", " low_play_count_request_ids.extend(lower_play_count_request_ids)\n", "\n", "# Display the filtered request IDs\n", "print(\n", " f\"Found {len(low_play_count_request_ids)} request_ids with duplicate prompts but not highest play counts in their group\"\n", ")\n", "print(\n", " low_play_count_request_ids[:10]\n", " if len(low_play_count_request_ids) > 10\n", " else low_play_count_request_ids\n", ")\n", "print(\"Before dedup user gen requests\", df.shape[0])\n", "# df = df[~df[\"request_id\"].isin(low_play_count_request_ids)]\n", "print(\"After dedup user gen requests\", df.shape[0])" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:40.799375Z", "start_time": "2024-05-16T13:59:36.394236Z" }, "execution": { "iopub.execute_input": "2025-11-23T20:18:56.721006Z", "iopub.status.busy": "2025-11-23T20:18:56.720848Z", "iopub.status.idle": "2025-11-23T20:20:04.568584Z", "shell.execute_reply": "2025-11-23T20:20:04.567894Z", "shell.execute_reply.started": "2025-11-23T20:18:56.720990Z" }, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "93987\n", "good_continue_at\n", "True 9399275\n", "False 14877\n", "Name: count, dtype: int64\n", "\n", " Check some basics... \n", " preference\n", "False 4707076\n", "True 4707076\n", "Name: count, dtype: int64 model_name\n", "chirp-crow-t1 9398580\n", "chirp-crow-t2 15572\n", "Name: count, dtype: int64 preference model_name \n", "False chirp-crow-t1 4699290\n", " chirp-crow-t2 7786\n", "True chirp-crow-t1 4699290\n", " chirp-crow-t2 7786\n", "Name: count, dtype: int64\n", "task\n", " 5701670\n", "cover 2376190\n", "artist_consistency 573350\n", "artist_cover 335386\n", "extend 124510\n", "playlist_condition 93188\n", "overpainting 53848\n", "upload_extend 43790\n", "underpainting 39118\n", "infill 33138\n", "artist_extend 24392\n", "stem_condition 12824\n", "stem_condition_infill 2748\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df[\"id\"] = df[\"str_id\"]\n", "# get the original duration of the clips, if they are concacted\n", "df[\"original_duration_s\"] = df[\"total_start_s\"] + df[\"duration\"]\n", "# classify the continue at behavoirs by the duration choice\n", "audio_prompt_id_to_continue_at = {}\n", "\n", "for _, row in df[~df[\"continued_parent\"].isna()].iterrows():\n", " audio_prompt_id = row[\"continued_parent\"]\n", " if audio_prompt_id not in audio_prompt_id_to_continue_at:\n", " audio_prompt_id_to_continue_at[audio_prompt_id] = row[\"continue_at\"]\n", " else:\n", " # pick the max\n", " audio_prompt_id = max(\n", " audio_prompt_id_to_continue_at[audio_prompt_id], row[\"continue_at\"]\n", " )\n", "print(len(audio_prompt_id_to_continue_at))\n", "df[\"has_continue_and_start_continue_at\"] = df[\"id\"].apply(\n", " lambda x: audio_prompt_id_to_continue_at.get(x)\n", ")\n", "# we want continue at to be at most of the clip...\n", "df[\"good_continue_at\"] = (\n", " (df[\"has_continue_and_start_continue_at\"] / df[\"duration\"]) > 0.9\n", ") | df[\"has_continue_and_start_continue_at\"].isna()\n", "print(df[\"good_continue_at\"].value_counts())\n", "\n", "\n", "print(\n", " \"\\n Check some basics... \\n\",\n", " df[\"preference\"].value_counts(),\n", " df[\"model_name\"].value_counts(),\n", " df.groupby([\"preference\"])[\"model_name\"].value_counts(),\n", ")\n", "\n", "df = df.sort_values(by=[\"request_id\", \"preference\"])\n", "df[\"duration_rel_diff\"] = df[\"duration\"].diff()\n", "df[\"play_rel_diff\"] = df[\"reaction_play_count\"].diff()\n", "print(df[\"task\"].value_counts())\n", "\n", "df[\"post_infill_duration\"] = (\n", " df[\"duration\"]\n", " + df[\"infill_context_end_s\"]\n", " - df[\"infill_context_start_s\"]\n", " - df[\"include_future_s\"]\n", " - df[\"include_history_s\"]\n", " - df[\"infill_dur_s\"]\n", ")" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T20:20:04.569574Z", "iopub.status.busy": "2025-11-23T20:20:04.569295Z", "iopub.status.idle": "2025-11-23T20:20:25.768366Z", "shell.execute_reply": "2025-11-23T20:20:25.767691Z", "shell.execute_reply.started": "2025-11-23T20:20:04.569558Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive preference_score\n", "1 2469675\n", "2 1417021\n", "0 504459\n", "3 295110\n", "4 19584\n", "5 1227\n", "Name: count, dtype: int64\n", "negative preference_score\n", "0 4613333\n", "1 86005\n", "2 7259\n", "3 466\n", "4 12\n", "5 1\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df[\"preference_score\"] = (\n", " df[\"upvoted\"].astype(int)\n", " + df[\"has_action\"].astype(int)\n", " + df[\"part_of_concat\"].astype(int)\n", " + df[\"is_in_playlist\"].astype(int)\n", " + (df[\"n_edits\"] >= 10).astype(int)\n", ")\n", "print(\"positive\", df[df[\"preference\"]][\"preference_score\"].value_counts())\n", "print(\"negative\", df[~df[\"preference\"]][\"preference_score\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.035167Z", "start_time": "2024-05-16T13:59:40.801098Z" }, "execution": { "iopub.execute_input": "2025-11-23T22:04:56.370615Z", "iopub.status.busy": "2025-11-23T22:04:56.370120Z", "iopub.status.idle": "2025-11-23T22:05:53.799857Z", "shell.execute_reply": "2025-11-23T22:05:53.799223Z", "shell.execute_reply.started": "2025-11-23T22:04:56.370595Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "after duration 0.9852874693334036\n", "after infill duration 0.9999501813864913\n", "neg_filter_reaction_play_count 1.0\n", "neg_filter_upvote_count 0.9907\n", "neg_filter_norm_play_frac 1.0\n", "neg_filter_continues 1.0\n", "----------------\n", "pos_filter_continues 0.9969\n", "pos_filter_reaction_play_count 1.0\n", "pos_filter_relative_play_count 0.9774\n", "pos_filter_cer_diff_preference 1.0\n", "pos_filter_bad_flags 0.9999\n", "after filter on play counts 0.9874\n", "after filter on higher quality 0.3557\n", "----------------\n", "negative 4594110 positive 1139615\n", "----------------\n", "total pair requests 4707076 --> selected pair requests 1128871 frac 0.240 --> total intitial users 563055\n" ] } ], "source": [ "normal_pos_play_count = 3\n", "# this is lower, cause a concat is probably already ensuring that it is good\n", "concat_pos_play_count = 1\n", "# this is a filter on the concated clip\n", "concat_total_play_count = 3\n", "\n", "all_fitlers = (df[\"duration\"] >= 10) & (df[\"duration\"] <= 480) & (df[\"clip_type\"] != \"preview\")\n", "print(\"after duration\", all_fitlers.sum() / df.shape[0])\n", "infill_duration_filter = ~df[\"task\"].isin(\n", " [\n", " \"infill\",\n", " \"infill_intro\",\n", " \"infill_outro\",\n", " ]\n", ") | (df[\"post_infill_duration\"] <= 239)\n", "print(\"after infill duration\", infill_duration_filter.sum() / df.shape[0])\n", "# negative fitlers\n", "total_negative = df[~df[\"preference\"]].shape[0]\n", "neg_filter_reaction_play_count = (~df[\"preference\"]) & (df[\"reaction_play_count\"] >= 1)\n", "print(\n", " \"neg_filter_reaction_play_count\",\n", " round(neg_filter_reaction_play_count.sum() / total_negative, 4),\n", ")\n", "neg_filter_upvote_count = (~df[\"preference\"]) & (df[\"upvote_count\"] == 0)\n", "print(\n", " \"neg_filter_upvote_count\",\n", " round(neg_filter_upvote_count.sum() / total_negative, 4),\n", ")\n", "neg_filter_norm_play_frac = (~df[\"preference\"]) & (df[\"norm_play_frac\"] <= 3.1)\n", "print(\n", " \"neg_filter_norm_play_frac\",\n", " round(neg_filter_norm_play_frac.sum() / total_negative, 4),\n", ")\n", "neg_filter_continues = (~df[\"preference\"]) & (\n", " df[\"has_continue_and_start_continue_at\"].isna()\n", ")\n", "print(\n", " \"neg_filter_continues\",\n", " round(neg_filter_continues.sum() / total_negative, 4),\n", ")\n", "\n", "neg_filter_selection_mask = (\n", " all_fitlers\n", " & infill_duration_filter\n", " & neg_filter_reaction_play_count\n", " & neg_filter_upvote_count\n", " & neg_filter_norm_play_frac\n", " & neg_filter_continues\n", ")\n", "\n", "print(\"----------------\")\n", "total_positive = df[df[\"preference\"]].shape[0]\n", "assert total_positive == total_negative\n", "pos_filter_continues = (df[\"preference\"]) & (df[\"good_continue_at\"])\n", "print(\"pos_filter_continues\", round(pos_filter_continues.sum() / total_positive, 4))\n", "pos_filter_reaction_play_count = (df[\"preference\"]) & (df[\"reaction_play_count\"] >= 1)\n", "print(\n", " \"pos_filter_reaction_play_count\",\n", " round(pos_filter_reaction_play_count.sum() / total_positive, 4),\n", ")\n", "pos_filter_relative_play_count = (df[\"preference\"]) & (df[\"play_rel_diff\"] >= 0)\n", "print(\n", " \"pos_filter_relative_play_count\",\n", " round(pos_filter_relative_play_count.sum() / total_positive, 4),\n", ")\n", "pos_filter_cer_diff_preference = (\n", " df[\n", " \"preference\"\n", " ] # & (df[\"pos_diff_preference\"] == 2) # & (df[\"cer_diff_preference\"] < 0.5) & (df[\"cer\"] < 0.99)\n", ")\n", "print(\n", " \"pos_filter_cer_diff_preference\",\n", " round(pos_filter_cer_diff_preference.sum() / total_positive, 4),\n", ")\n", "pos_filter_bad_flags = (\n", " (df[\"preference\"]) & (df[\"flag_count\"] == 0) & (df[\"dislike_count\"] == 0)\n", ")\n", "print(\n", " \"pos_filter_bad_flags\",\n", " round(pos_filter_bad_flags.sum() / total_positive, 4),\n", ")\n", "pos_filter_play_counts = (df[\"preference\"]) & (\n", " (\n", " (df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= concat_pos_play_count)\n", " & (df[\"concat_play_counts\"] >= concat_total_play_count)\n", " )\n", " | (\n", " (~df[\"part_of_concat\"]) & (df[\"reaction_play_count\"] >= normal_pos_play_count)\n", " # & (df[\"norm_play_frac\"] >= 2.1) # this is a bit of a luxury cut...\n", " )\n", " | (df[\"task\"].isin([\"infill\", \"infill_intro\", \"infill_outro\"]))\n", ")\n", "print(\n", " \"after filter on play counts\",\n", " round(pos_filter_play_counts.sum() / total_positive, 4),\n", ")\n", "high_quality_tasks_filter = (\n", " (\n", " df[\"task\"].isin(\n", " [\n", " \"cover\",\n", " \"upload_extend\",\n", " \"cover_extend\",\n", " \"artist_cover\",\n", " \"extend\",\n", " \"artist_consistency\",\n", " \"artist_extend\",\n", " \"playlist_condition\",\n", " \"overpainting\",\n", " \"underpainting\",\n", " \"\",\n", " ]\n", " )\n", " )\n", " & (\n", " (df[\"upvote_count\"] >= 1) # (df[\"upvote_count\"] >= 1)\n", " | (df[\"reaction_play_count\"] >= 5)\n", " | (df[\"concat_play_counts\"] >= 5)\n", " )\n", " & (\n", " (df[\"part_of_concat\"])\n", " | (\n", " (~df[\"part_of_concat\"])\n", " & (df[\"norm_play_frac\"] >= 5.1) # this is a bit of a luxury cut...\n", " & (\n", " df[\"norm_play_frac\"] >= df[\"reaction_play_count\"] / 3\n", " ) # play duration is not low on average\n", " )\n", " )\n", ")\n", "medium_quality_tasks_filter = (\n", " df[\"task\"].isin(\n", " [\n", " \"infill\",\n", " \"infill_intro\",\n", " \"infill_outro\",\n", " \"stem_condition\",\n", " ]\n", " )\n", ") & ( # let more infill through only in this case...\n", " (\n", " df[\"upvote_count\"] >= 1\n", " ) # (df[\"upvote_count\"] >= 1) (df[\"pos_diff_preference\"] == 2)\n", " | (df[\"reaction_play_count\"] >= 1)\n", " | (df[\"concat_play_counts\"] >= 1)\n", ")\n", "pos_filter_higher_quality = (df[\"preference\"]) & (\n", " high_quality_tasks_filter | medium_quality_tasks_filter\n", ")\n", "print(\n", " \"after filter on higher quality\",\n", " round(pos_filter_higher_quality.sum() / total_positive, 4),\n", ")\n", "\n", "user_gen_filter = (\n", " df[\"user_n_clips\"] >= 100\n", ") # user needs to have genereated at least 100 over the time period\n", "\n", "print(\"----------------\")\n", "pos_filter_selectin_mask = (\n", " (df[\"preference\"]) # get basics aligned\n", " & all_fitlers\n", " & infill_duration_filter\n", " & pos_filter_continues\n", " & pos_filter_reaction_play_count\n", " & pos_filter_relative_play_count\n", " & pos_filter_cer_diff_preference\n", " & pos_filter_bad_flags\n", " & pos_filter_play_counts\n", " & pos_filter_higher_quality\n", " & user_gen_filter\n", ")\n", "print(\n", " \"negative\",\n", " sum(neg_filter_selection_mask),\n", " \"positive\",\n", " sum(pos_filter_selectin_mask),\n", ")\n", "\n", "neg_filter_requests = df[neg_filter_selection_mask][\"request_id\"].unique()\n", "pos_filter_requests = df[pos_filter_selectin_mask][\"request_id\"].unique()\n", "# looking for very strong signal here:\n", "# listen to the positive/negative more than once\n", "# disliked one of the clips\n", "unique_requests = set(pos_filter_requests).intersection(neg_filter_requests)\n", "print(\"----------------\")\n", "print(\n", " \"total pair requests\",\n", " df[\"request_id\"].nunique(),\n", " \" --> selected pair requests\",\n", " len(unique_requests),\n", " f\"frac {len(unique_requests) / df['request_id'].nunique():.3f}\",\n", " \" --> total intitial users\",\n", " df[\"user_id\"].nunique(),\n", ")" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.250737Z", "start_time": "2024-05-16T13:59:41.036434Z" }, "execution": { "iopub.execute_input": "2025-11-23T22:06:14.121360Z", "iopub.status.busy": "2025-11-23T22:06:14.121185Z", "iopub.status.idle": "2025-11-23T22:06:36.879928Z", "shell.execute_reply": "2025-11-23T22:06:36.879305Z", "shell.execute_reply.started": "2025-11-23T22:06:14.121344Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "task\n", " 1326144\n", "cover 536748\n", "artist_consistency 155882\n", "artist_cover 86858\n", "extend 56870\n", "infill 24502\n", "playlist_condition 22820\n", "overpainting 11138\n", "artist_extend 11092\n", "stem_condition 10006\n", "upload_extend 9050\n", "underpainting 6632\n", "Name: count, dtype: int64\n", "crow_t1_v61 requests 1128871 clips 2257742 total khrs 113.348; N gpus for 1000 iters 141.109; 4 gpus for x iters 35277.219; n unique users 198445 n pro users 192068\n" ] } ], "source": [ "df_slice = df[df[\"request_id\"].isin(set(unique_requests))].copy()\n", "print(df_slice[\"task\"].value_counts())\n", "print(\n", " f\"{os.path.basename(OUT_DATA_DIR)} requests\",\n", " df_slice[\"request_id\"].nunique(),\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"4 gpus for x iters {df_slice.shape[0] / 8 / 2 / 4:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", " f\"n pro users {df_slice[df_slice['is_pro_user']]['user_id'].nunique()}\",\n", ")\n", "# auk_mix_t1_v2 requests 102002 clips 204004 total khrs 9.191; N gpus for 1000 iters 12.750; 4 gpus for x iters 3187.562; n unique users 36408 n pro users 34038\n", "# auk_t1_v1 requests 9179 clips 18358 total khrs 0.854; N gpus for 1000 iters 1.147; 4 gpus for x iters 286.844; n unique users 6288 n pro users 6275\n", "# auk_t1_v2 requests 40903 clips 81806 total khrs 3.864; N gpus for 1000 iters 5.113; 4 gpus for x iters 1278.219; n unique users 21079 n pro users 20966\n", "# auk_t1_v3 requests 102015 clips 204030 total khrs 9.703; N gpus for 1000 iters 12.752; 4 gpus for x iters 3187.969; n unique users 42837 n pro users 42462\n", "# auk_t1_v4 requests 211452 clips 422904 total khrs 20.216; N gpus for 1000 iters 26.431; 4 gpus for x iters 6607.875; n unique users 71182 n pro users 70059\n", "# auk_t1_v9 requests 547743 clips 1095486 total khrs 56.567; N gpus for 1000 iters 68.468; 4 gpus for x iters 17116.969; n unique users 129354 n pro users 125126\n", "# auk_t1_v13 requests 627646 clips 1255292 total khrs 64.530; N gpus for 1000 iters 78.456; 4 gpus for x iters 19613.938; n unique users 139873 n pro users 134738\n", "# auk_t1_v17 requests 494902 clips 989804 total khrs 51.445; N gpus for 1000 iters 61.863; 4 gpus for x iters 15465.688; n unique users 114240 n pro users 109708\n", "# auk_t1_v19 requests 740881 clips 1481762 total khrs 76.133; N gpus for 1000 iters 92.610; 4 gpus for x iters 23152.531; n unique users 154570 n pro users 147261\n", "# auk_t1_v24 requests 717745 clips 1435490 total khrs 74.515; N gpus for 1000 iters 89.718; 4 gpus for x iters 22429.531; n unique users 139344 n pro users 130740\n", "# auk_t1_v29 requests 1097586 clips 2195172 total khrs 112.614; N gpus for 1000 iters 137.198; 4 gpus for x iters 34299.562; n unique users 193138 n pro users 179429\n", "# auk_t1_v30 requests 1224422 clips 2448844 total khrs 125.330; N gpus for 1000 iters 153.053; 4 gpus for x iters 38263.188; n unique users 203138 n pro users 186758\n", "# auk_t1_v31 requests 389265 clips 778530 total khrs 40.761; N gpus for 1000 iters 48.658; 4 gpus for x iters 12164.531; n unique users 85198 n pro users 78445\n", "# auk_t1_v33 requests 1285261 clips 2570522 total khrs 131.585; N gpus for 1000 iters 160.658; 4 gpus for x iters 40164.406; n unique users 208932 n pro users 191380\n", "# auk_t1_v33 requests 1086639 clips 2173278 total khrs 110.641; N gpus for 1000 iters 135.830; 4 gpus for x iters 33957.469; n unique users 197293 n pro users 180968 -- play dur from /3 to /2\n", "# auk_t1_v33 requests 806877 clips 1613754 total khrs 81.964; N gpus for 1000 iters 100.860; 4 gpus for x iters 25214.906; n unique users 156679 n pro users 144253 -- filter to web\n", "# auk_t1_v37 requests 1339132 clips 2678264 total khrs 137.055; N gpus for 1000 iters 167.392; 4 gpus for x iters 41847.875; n unique users 214313 n pro users 196815\n", "# auk_t1_v38 requests 979451 clips 1958902 total khrs 102.038; N gpus for 1000 iters 122.431; 4 gpus for x iters 30607.844; n unique users 170773 n pro users 156737\n", "# auk_t1_v43 requests 184775 clips 369550 total khrs 19.089; N gpus for 1000 iters 23.097; 4 gpus for x iters 5774.219; n unique users 60652 n pro users 59126\n", "# auk_t1_v45 requests 1176216 clips 2352432 total khrs 122.102; N gpus for 1000 iters 147.027; 4 gpus for x iters 36756.750; n unique users 176751 n pro users 163097\n", "# auk_t1_v48 requests 294407 clips 588814 total khrs 29.992; N gpus for 1000 iters 36.801; 4 gpus for x iters 9200.219; n unique users 80991 n pro users 78825\n", "# bluejay_t1_v1 requests 47405 clips 94810 total khrs 5.515; N gpus for 1000 iters 5.926; 4 gpus for x iters 1481.406; n unique users 24344 n pro users 24194\n", "# bluejay_t1_v2 requests 101265 clips 202530 total khrs 11.753; N gpus for 1000 iters 12.658; 4 gpus for x iters 3164.531; n unique users 44574 n pro users 44024\n", "# bluejay_t1_v3 requests 144765 clips 289530 total khrs 16.756; N gpus for 1000 iters 18.096; 4 gpus for x iters 4523.906; n unique users 58421 n pro users 57449\n", "# bluejay_t1_v5 requests 179289 clips 358578 total khrs 20.764; N gpus for 1000 iters 22.411; 4 gpus for x iters 5602.781; n unique users 67850 n pro users 66572\n", "# bluejay_t1_v7 requests 282305 clips 564610 total khrs 32.692; N gpus for 1000 iters 35.288; 4 gpus for x iters 8822.031; n unique users 93023 n pro users 90936\n", "# bluejay_t1_v9 requests 540570 clips 1081140 total khrs 62.455; N gpus for 1000 iters 67.571; 4 gpus for x iters 16892.812; n unique users 145720 n pro users 142027\n", "# bluejay_t1_v11 requests 184433 clips 368866 total khrs 20.955; N gpus for 1000 iters 23.054; 4 gpus for x iters 5763.531; n unique users 52149 n pro users 51402\n", "# bluejay_t1_v12 requests 293821 clips 587642 total khrs 33.820; N gpus for 1000 iters 36.728; 4 gpus for x iters 9181.906; n unique users 98138 n pro users 95838\n", "# bluejay_t1_v13 requests 355678 clips 711356 total khrs 40.908; N gpus for 1000 iters 44.460; 4 gpus for x iters 11114.938; n unique users 111544 n pro users 108497\n", "# bluejay_t1_v17 requests 415411 clips 830822 total khrs 48.043; N gpus for 1000 iters 51.926; 4 gpus for x iters 12981.594; n unique users 124572 n pro users 120683\n", "# bluejay_t1_v24 requests 486884 clips 973768 total khrs 56.292; N gpus for 1000 iters 60.861; 4 gpus for x iters 15215.125; n unique users 138727 n pro users 133829\n", "# bluejay_t1_v26 requests 336220 clips 672440 total khrs 38.800; N gpus for 1000 iters 42.028; 4 gpus for x iters 10506.875; n unique users 80832 n pro users 78854\n", "# bluejay_t1_v28 requests 683709 clips 1367418 total khrs 78.507; N gpus for 1000 iters 85.464; 4 gpus for x iters 21365.906; n unique users 172776 n pro users 164584\n", "# bluejay_t1_v28 requests 662650 clips 1325300 total khrs 76.539; N gpus for 1000 iters 82.831; 4 gpus for x iters 20707.812; n unique users 170192 n pro users 162222\n", "# bluejay_t1_v31 requests 855246 clips 1710492 total khrs 97.846; N gpus for 1000 iters 106.906; 4 gpus for x iters 26726.438; n unique users 198438 n pro users 186068\n", "# bluejay_t1_v31 requests 829339 clips 1658678 total khrs 95.430; N gpus for 1000 iters 103.667; 4 gpus for x iters 25916.844; n unique users 195575 n pro users 183523\n", "# bluejay_t1_v31 requests 932058 clips 1864116 total khrs 107.136; N gpus for 1000 iters 116.507; 4 gpus for x iters 29126.812; n unique users 203738 n pro users 190639\n", "# bluejay_t1_v41 requests 1122075 clips 2244150 total khrs 128.936; N gpus for 1000 iters 140.259; 4 gpus for x iters 35064.844; n unique users 232511 n pro users 214225\n", "# bluejay_t1_v42 requests 1234273 clips 2468546 total khrs 140.767; N gpus for 1000 iters 154.284; 4 gpus for x iters 38571.031; n unique users 246967 n pro users 226020\n", "# crow_t1_v1 requests 131955 clips 263910 total khrs 12.518; N gpus for 1000 iters 16.494; 4 gpus for x iters 4123.594; n unique users 61682 n pro users 56836\n", "# crow_t1_v3 requests 272342 clips 544684 total khrs 26.003; N gpus for 1000 iters 34.043; 4 gpus for x iters 8510.688; n unique users 106845 n pro users 96483\n", "# crow_t1_v6 requests 369774 clips 739548 total khrs 35.356; N gpus for 1000 iters 46.222; 4 gpus for x iters 11555.438; n unique users 134281 n pro users 119919\n", "# crow_t1_v13 requests 558885 clips 1117770 total khrs 53.496; N gpus for 1000 iters 69.861; 4 gpus for x iters 17465.156; n unique users 182267 n pro users 160132\n", "# crow_t1_v17 requests 792420 clips 1584840 total khrs 77.740; N gpus for 1000 iters 99.052; 4 gpus for x iters 24763.125; n unique users 212279 n pro users 199173\n", "# crow_t1_v21 requests 1078035 clips 2156070 total khrs 106.094; N gpus for 1000 iters 134.754; 4 gpus for x iters 33688.594; n unique users 263938 n pro users 245247\n", "# crow_t1_v31 requests 1569448 clips 3138896 total khrs 155.285; N gpus for 1000 iters 196.181; 4 gpus for x iters 49045.250; n unique users 343002 n pro users 310320\n", "# crow_t1_v31 requests 1128871 clips 2257742 total khrs 113.348; N gpus for 1000 iters 141.109; 4 gpus for x iters 35277.219; n unique users 198445 n pro users 192068\n", "# crow_t1_vxx requests 1275697 clips 2551394 total khrs 128.287; N gpus for 1000 iters 159.462; 4 gpus for x iters 39865.531; n unique users 216731 n pro users 207174\n", "# crow_t1_v41 requests 719413 clips 1438826 total khrs 73.029; N gpus for 1000 iters 89.927; 4 gpus for x iters 22481.656; n unique users 147416 n pro users 142879\n", "# crow_t1_v51 requests 1376601 clips 2753202 total khrs 138.547; N gpus for 1000 iters 172.075; 4 gpus for x iters 43018.781; n unique users 228819 n pro users 217450\n", "# crow_t1_v61 requests 1406632 clips 2813264 total khrs 140.377; N gpus for 1000 iters 175.829; 4 gpus for x iters 43957.250; n unique users 251116 n pro users 235715" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T22:06:39.821524Z", "iopub.status.busy": "2025-11-23T22:06:39.821134Z", "iopub.status.idle": "2025-11-23T22:06:39.844067Z", "shell.execute_reply": "2025-11-23T22:06:39.843580Z", "shell.execute_reply.started": "2025-11-23T22:06:39.821504Z" } }, "outputs": [], "source": [ "# import time\n", "# time.sleep(3600)" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T22:06:40.974389Z", "iopub.status.busy": "2025-11-23T22:06:40.974088Z", "iopub.status.idle": "2025-11-23T22:09:57.892342Z", "shell.execute_reply": "2025-11-23T22:09:57.891715Z", "shell.execute_reply.started": "2025-11-23T22:06:40.974370Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Total hoot cer scores: 4527525\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 116889/116889 [03:05<00:00, 628.80it/s]\n" ] } ], "source": [ "# Load existing hoot CER cache\n", "with open(\"/home/tony/Data/Preference/crow_t1/hoot_cer.json\", \"r\") as file:\n", " clip_id_to_cer = json.load(file)\n", "print(\"Total hoot cer scores:\", len(clip_id_to_cer))\n", "# clip_id_to_cer = {}\n", "JSON_DIR = \"/app2/suno/data/dpo/crow_t1_json/\"\n", "\n", "# Extract unique s3_ids from df_slice that are not already in the cache\n", "clip_ids = set(df_slice[\"s3_id\"]) - set(clip_id_to_cer.keys())\n", "\n", "for clip_id in tqdm(clip_ids):\n", " hoot_json_path = os.path.join(JSON_DIR, f\"{clip_id}_hoot.json\")\n", " if not os.path.exists(hoot_json_path):\n", " clip_id_to_cer[clip_id] = 1.0\n", " continue\n", " with open(hoot_json_path, \"r\") as f:\n", " data = json.load(f)\n", " for data_dict in data:\n", " if \"hoot_cer\" in data_dict:\n", " clip_id_to_cer[clip_id] = data_dict[\"hoot_cer\"]\n", " break\n", "\n", "# Update the hoot CER cache\n", "with open(\"/home/tony/Data/Preference/crow_t1/hoot_cer.json\", \"w\") as file:\n", " json.dump(clip_id_to_cer, file)" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T22:09:57.893408Z", "iopub.status.busy": "2025-11-23T22:09:57.893246Z", "iopub.status.idle": "2025-11-23T22:11:05.806061Z", "shell.execute_reply": "2025-11-23T22:11:05.805290Z", "shell.execute_reply.started": "2025-11-23T22:09:57.893393Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "count 1.128871e+06\n", "mean -5.329892e-03\n", "std 9.014233e-02\n", "min -1.000000e+00\n", "10% -6.735751e-02\n", "50% 0.000000e+00\n", "90% 5.494505e-02\n", "95% 9.166667e-02\n", "96% 1.056460e-01\n", "97% 1.256378e-01\n", "98% 1.573648e-01\n", "99% 2.285714e-01\n", "99.5% 3.507780e-01\n", "99.9% 7.272727e-01\n", "max 1.000000e+00\n", "Name: cer_diff, dtype: float64\n", "crow_t1_v61 requests 1121682 clips 2243364 total khrs 112.764; N gpus for 1000 iters 140.210; 4 gpus for x iters 35052.562; n unique users 197954 n pro users 191593\n" ] } ], "source": [ "# # add the cer to the df\n", "df_slice[\"cer\"] = df_slice[\"s3_id\"].map(clip_id_to_cer)\n", "df_slice = df_slice.fillna({\"cer\": 1})\n", "df_slice[\"cer_diff\"] = df_slice[\"cer\"].diff()\n", "df_slice = df_slice.fillna({\"cer_diff\": 0})\n", "print(df_slice[df_slice[\"preference\"]][\"cer_diff\"].describe(percentiles=[0.1, 0.5, 0.9, 0.95, 0.96, 0.97, 0.98, 0.99, 0.995, 0.999]))\n", "\n", "# just trimming off tail is fine and safe. Used to be 0.2. Multiple rounds now so probably 0.3 is still fine.\n", "cer_mask = (df_slice[\"preference\"]) & (df_slice[\"cer_diff\"] < 0.3)\n", "pos_cer_filter_requests = df_slice[cer_mask][\"request_id\"].unique()\n", "df_slice = df_slice[df_slice[\"request_id\"].isin(set(pos_cer_filter_requests))].copy()\n", "print(\n", " f\"{os.path.basename(OUT_DATA_DIR)} requests\",\n", " df_slice[\"request_id\"].nunique(),\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"4 gpus for x iters {df_slice.shape[0] / 8 / 2 / 4:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", " f\"n pro users {df_slice[df_slice['is_pro_user']]['user_id'].nunique()}\",\n", ")" ] }, { "cell_type": "code", "execution_count": 44, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T22:11:05.807136Z", "iopub.status.busy": "2025-11-23T22:11:05.806833Z", "iopub.status.idle": "2025-11-23T22:11:06.423522Z", "shell.execute_reply": "2025-11-23T22:11:06.422871Z", "shell.execute_reply.started": "2025-11-23T22:11:05.807119Z" } }, "outputs": [ { "data": { "text/plain": [ "source\n", "web 1678078\n", "android 294882\n", "ios 270404\n", "Name: count, dtype: int64" ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_slice[\"source\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.277006Z", "start_time": "2024-05-16T13:59:41.252105Z" }, "execution": { "iopub.execute_input": "2025-11-23T22:11:06.425241Z", "iopub.status.busy": "2025-11-23T22:11:06.424725Z", "iopub.status.idle": "2025-11-23T22:11:07.744754Z", "shell.execute_reply": "2025-11-23T22:11:07.744017Z", "shell.execute_reply.started": "2025-11-23T22:11:06.425222Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive in playlist (341595, 214)\n", "task\n", " 1318624\n", "cover 533524\n", "artist_consistency 155106\n", "artist_cover 86268\n", "extend 55894\n", "infill 23954\n", "playlist_condition 22658\n", "overpainting 10958\n", "artist_extend 10904\n", "stem_condition 9968\n", "upload_extend 8926\n", "underpainting 6580\n", "Name: count, dtype: int64\n" ] } ], "source": [ "test_mask = (df_slice[\"preference\"]) & (\n", " (df_slice[\"is_in_playlist\"]) | (df_slice[\"concat_in_playlist\"])\n", ")\n", "print(\"positive in playlist\", df_slice[test_mask].shape)\n", "print(df_slice[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T22:11:07.745659Z", "iopub.status.busy": "2025-11-23T22:11:07.745494Z", "iopub.status.idle": "2025-11-23T22:11:07.775706Z", "shell.execute_reply": "2025-11-23T22:11:07.775058Z", "shell.execute_reply.started": "2025-11-23T22:11:07.745643Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "is_public\n", "False 2065434\n", "True 177930\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(df_slice[\"is_public\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T22:11:07.776645Z", "iopub.status.busy": "2025-11-23T22:11:07.776328Z", "iopub.status.idle": "2025-11-23T22:11:08.507467Z", "shell.execute_reply": "2025-11-23T22:11:08.506778Z", "shell.execute_reply.started": "2025-11-23T22:11:07.776629Z" } }, "outputs": [], "source": [ "df_slice[\"npz_path\"] = df_slice[\"s3_id\"].map(lambda x: f\"{NPZ_DIR}/{x}.npz\")" ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T22:11:08.508349Z", "iopub.status.busy": "2025-11-23T22:11:08.508188Z", "iopub.status.idle": "2025-11-23T22:11:25.976526Z", "shell.execute_reply": "2025-11-23T22:11:25.975732Z", "shell.execute_reply.started": "2025-11-23T22:11:08.508334Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(2243364, 215)\n" ] } ], "source": [ "df_total = df_slice.copy()\n", "print(df_total.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## SOME SNOWFLAKE LYRICS SHIT YOU DON\"T WNAT TO KNOW" ] }, { "cell_type": "code", "execution_count": 49, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T22:11:25.977426Z", "iopub.status.busy": "2025-11-23T22:11:25.977257Z", "iopub.status.idle": "2025-11-23T22:11:26.578913Z", "shell.execute_reply": "2025-11-23T22:11:26.578386Z", "shell.execute_reply.started": "2025-11-23T22:11:25.977411Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2025-11-23 22:11:26,002 - INFO - Snowflake Connector for Python Version: 3.12.3, Python Version: 3.10.15, Platform: Linux-5.15.0-122-generic-x86_64-with-glibc2.35\n", "2025-11-23 22:11:26,003 - INFO - Connecting to GLOBAL Snowflake domain\n", "2025-11-23 22:11:26,003 - INFO - This connection is in OCSP Fail Open Mode. TLS Certificates would be checked for validity and revocation status. Any other Certificate Revocation related exceptions or OCSP Responder failures would be disregarded in favor of connectivity.\n", "2025-11-23 22:11:26,399 - INFO - Snowpark Session information: \n", "\"version\" : 1.23.0,\n", "\"python.version\" : 3.10.15,\n", "\"python.connector.version\" : 3.12.3,\n", "\"python.connector.session.id\" : 1572757016091398,\n", "\"os.name\" : Linux\n", "\n", "2025-11-23 22:11:26,399 - INFO - New root object was created for \n", "2025-11-23 22:11:26,400 - INFO - performing a HTTP GET call to /api/v2/session/parameters/effective\n", "2025-11-23 22:11:26,575 - INFO - telemetry client created for , telemetry enabled: True\n", "2025-11-23 22:11:26,576 - INFO - Snowflake Core version: 0.11.0, on Python 3.10.15, on platform: Linux-5.15.0-122-generic-x86_64-with-glibc2.35\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "PROD\n" ] } ], "source": [ "home_dir = os.path.expanduser(\"~\")\n", "snow_password_path = os.path.join(home_dir, \".aws\", \"snow_pw.txt\")\n", "if os.path.exists(snow_password_path):\n", " # !pip install snowflake\n", " from snowflake.core import Root\n", " from snowflake.snowpark import Session\n", "\n", " with open(snow_password_path, \"r\") as fp:\n", " fp_lines = fp.readlines()\n", " snow_password = fp_lines[0].strip()\n", " snow_username = fp_lines[1].strip()\n", "\n", " CONNECTION_PARAMETERS = {\n", " \"account\": \"fu90569.us-east-2.aws\",\n", " \"user\": snow_username,\n", " \"private_key_file\": \"/home/tony/.aws/rsa_key.p8\",\n", " \"role\": \"ACCOUNTADMIN\",\n", " \"database\": \"SUNO_PROD\",\n", " \"warehouse\": \"SUNO_PROD_LARGE\",\n", " \"schema\": \"PROD\",\n", " }\n", "\n", "if not os.path.exists(snow_password_path):\n", " raise Exception(\"you are not authorized to access snowflake -- please setup\")\n", "\n", "snow_session = Session.builder.configs(CONNECTION_PARAMETERS).create()\n", "\n", "snow_root = Root(snow_session)\n", "snow_schema = snow_root.databases[\"SUNO_PROD\"].schemas[\"PROD\"]\n", "print(snow_schema.name)" ] }, { "cell_type": "code", "execution_count": 50, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T22:11:26.579642Z", "iopub.status.busy": "2025-11-23T22:11:26.579492Z", "iopub.status.idle": "2025-11-23T22:14:01.969300Z", "shell.execute_reply": "2025-11-23T22:14:01.968509Z", "shell.execute_reply.started": "2025-11-23T22:11:26.579627Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 0/1 [00:00= start_cutoff_date)\n", "print(df_total.shape, df_total[date_mask].shape)\n", "df_slice = df_total[date_mask].copy()\n", "print(\"after date cut\", df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 56, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T22:19:53.417572Z", "iopub.status.busy": "2025-11-23T22:19:53.417403Z", "iopub.status.idle": "2025-11-23T22:19:53.438981Z", "shell.execute_reply": "2025-11-23T22:19:53.438518Z", "shell.execute_reply.started": "2025-11-23T22:19:53.417557Z" } }, "outputs": [], "source": [ "# df_slice = df_total.copy()" ] }, { "cell_type": "code", "execution_count": 57, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T22:19:53.439661Z", "iopub.status.busy": "2025-11-23T22:19:53.439511Z", "iopub.status.idle": "2025-11-23T22:19:53.457644Z", "shell.execute_reply": "2025-11-23T22:19:53.457211Z", "shell.execute_reply.started": "2025-11-23T22:19:53.439646Z" } }, "outputs": [], "source": [ "# from tqdm import tqdm\n", "\n", "# list_of_past_data = [\n", "# \"/app2/suno/data/dpo/bluejay_t1_v42\",\n", "# \"/app2/suno/data/dpo/bluejay_t1_v43\",\n", "# \"/app2/suno/data/dpo/bluejay_t1_v44\",\n", "# \"/app2/suno/data/dpo/bluejay_t1_v45\",\n", "# \"/app2/suno/data/dpo/bluejay_t1_v46\",\n", "# \"/app2/suno/data/dpo/bluejay_t1_v47\",\n", "# \"/app2/suno/data/dpo/bluejay_t1_v48\",\n", "# \"/app2/suno/data/dpo/bluejay_t1_v59\",\n", "# \"/app2/suno/data/dpo/bluejay_t1_v60\",\n", "# \"/app2/suno/data/dpo/bluejay_t1_v61\",\n", "# ]\n", "\n", "# # Collect all known train IDs from previous meta_tr.jsonl files, showing progress and set growth\n", "# known_train_ids = set()\n", "# for data_dir in tqdm(list_of_past_data, desc=\"Collecting known train IDs\"):\n", "# metas = read_jsonl(os.path.join(data_dir, \"meta_tr.jsonl\"))\n", "# before = len(known_train_ids)\n", "# known_train_ids.update(meta[\"id\"] for meta in metas)\n", "# after = len(known_train_ids)\n", "# tqdm.write(f\"Added {after - before} new IDs from {data_dir} (total: {after})\")\n", "\n", "# print(f\"Total known train IDs: {len(known_train_ids)}\")\n", "# print(f\"Original df_slice shape: {df_slice.shape}\")\n", "# df_slice = df_slice[~df_slice[\"id\"].isin(known_train_ids)].copy()\n", "# print(f\"Filtered df_slice shape: {df_slice.shape}\")" ] }, { "cell_type": "code", "execution_count": 58, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T22:19:53.458246Z", "iopub.status.busy": "2025-11-23T22:19:53.458113Z", "iopub.status.idle": "2025-11-23T22:19:53.473322Z", "shell.execute_reply": "2025-11-23T22:19:53.472893Z", "shell.execute_reply.started": "2025-11-23T22:19:53.458233Z" } }, "outputs": [], "source": [ "# from collections import Counter, defaultdict\n", "# from typing import List, Dict, Set\n", "\n", "# # Read all meta files\n", "# meta_files: List[str] = [\n", "# \"/app/suno/data/dpo/30b_t1_v23/meta_tr.jsonl\",\n", "# \"/app/suno/data/dpo/30b_t6_v35/meta_tr.jsonl\",\n", "# \"/app/suno/data/dpo/13b_s32_v34/meta_tr.jsonl\",\n", "# \"/app/suno/data/dpo/auk_mix_t1_v6/meta_tr.jsonl\",\n", "# \"/app/suno/data/dpo/auk_mix_t1_v14/meta_tr.jsonl\",\n", "# \"/app2/suno/data/dpo/auk_t1_v6/meta_tr.jsonl\",\n", "# \"/app2/suno/data/dpo/auk_t1_v7/meta_tr.jsonl\",\n", "# \"/app2/suno/data/dpo/auk_t1_v19/meta_tr.jsonl\",\n", "# \"/app2/suno/data/dpo/auk_t1_v29/meta_tr.jsonl\",\n", "# ]\n", "\n", "# user_id_counter: Counter[str] = Counter()\n", "# for meta_file in tqdm(meta_files):\n", "# metas = read_jsonl(meta_file)\n", "# # Count each user_id once per dataset\n", "# user_ids: Set[str] = {meta[\"user_id\"] for meta in metas if \"user_id\" in meta}\n", "# user_id_counter.update(user_ids)\n", "\n", "# # Map from count to set of user_ids\n", "# count_to_users: Dict[int, Set[str]] = defaultdict(set)\n", "# for user_id, count in user_id_counter.items():\n", "# count_to_users[count].add(user_id)\n", "\n", "# for count in sorted(count_to_users):\n", "# users = count_to_users[count]\n", "# print(f\"Count {count}: {len(users)} users\")\n", "\n", "# # with open(\"/home/tony/Data/top_user_8.json\", \"w\") as fp:\n", "# # json.dump(list(count_to_users[8]) , fp)" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T21:04:47.369513Z", "iopub.status.busy": "2025-11-23T21:04:47.369363Z", "iopub.status.idle": "2025-11-23T21:04:47.384615Z", "shell.execute_reply": "2025-11-23T21:04:47.384098Z", "shell.execute_reply.started": "2025-11-23T21:04:47.369499Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/top_user/mask_control_low.json\", \"r\") as fp:\n", "# very_good_users = json.load(fp)\n", "# # very_good_users = count_to_users[5]\n", "# very_good_users_mask = df_total[\"user_id\"].isin(very_good_users)\n", "# print(df_total[very_good_users_mask].shape, df_total.shape)\n", "# print(df_total[very_good_users_mask][\"user_id\"].nunique(), df_total[\"user_id\"].nunique())\n", "# # df_total[very_good_users_mask][\"task\"].value_counts()\n", "# df_slice = df_total[very_good_users_mask].copy()" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "execution": { "iopub.execute_input": "2025-11-23T21:04:47.385557Z", "iopub.status.busy": "2025-11-23T21:04:47.385186Z", "iopub.status.idle": "2025-11-23T21:04:47.400724Z", "shell.execute_reply": "2025-11-23T21:04:47.400206Z", "shell.execute_reply.started": "2025-11-23T21:04:47.385543Z" } }, "outputs": [], "source": [ "# print(\n", "# \"Before filtering by user_id and task\",\n", "# df_slice.shape[0],\n", "# \"user_id unique:\",\n", "# df_slice[\"user_id\"].nunique(),\n", "# )\n", "\n", "# # Create a copy to avoid fragmentation warning\n", "# df_slice = df_slice.copy()\n", "\n", "# # Calculate score for each row: reaction_play_count + 5 if preference is True, else 0\n", "# score_values = (\n", "# df_slice[\"reaction_play_count\"] + (5 * df_slice[\"upvote_count\"].astype(int))\n", "# ) * df_slice[\"preference\"].astype(int)\n", "\n", "# # Use pd.concat to add the score column efficiently\n", "# df_slice = pd.concat(\n", "# [df_slice, pd.DataFrame({\"score\": score_values}, index=df_slice.index)], axis=1\n", "# )\n", "\n", "# # Group by user_id and task, then for each group find the request_id with highest score\n", "# best_request_ids = []\n", "# for (user_id, task), group in tqdm(\n", "# df_slice.groupby([\"user_id\", \"task\"]), desc=\"Processing user_id and task groups\"\n", "# ):\n", "# # Get the request_id with the highest score in this group\n", "# best_request_id = group.loc[group[\"score\"].idxmax(), \"request_id\"]\n", "# best_request_ids.append(best_request_id)\n", "\n", "# # Filter df_slice to keep only the best request_ids for each user_id, task combination\n", "# df_slice = df_slice[df_slice[\"request_id\"].isin(best_request_ids)].copy()\n", "\n", "# print(\n", "# \"After filtering by user_id and task\",\n", "# df_slice.shape[0],\n", "# \"user_id unique:\",\n", "# df_slice[\"user_id\"].nunique(),\n", "# )" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T14:00:20.866354Z", "start_time": "2024-05-16T14:00:12.443344Z" }, "execution": { "iopub.execute_input": "2025-11-23T21:04:47.401628Z", "iopub.status.busy": "2025-11-23T21:04:47.401479Z", "iopub.status.idle": "2025-11-23T21:04:47.765329Z", "shell.execute_reply": "2025-11-23T21:04:47.764650Z", "shell.execute_reply.started": "2025-11-23T21:04:47.401615Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(197098, 216)\n", "task\n", " 114700\n", "cover 48540\n", "artist_consistency 14806\n", "artist_cover 7856\n", "extend 4610\n", "playlist_condition 2134\n", "overpainting 1158\n", "upload_extend 934\n", "artist_extend 884\n", "infill 792\n", "underpainting 530\n", "stem_condition 154\n", "Name: count, dtype: int64\n" ] }, { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[37], line 6\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mvalue_counts())\n\u001b[0;32m----> 6\u001b[0m \u001b[43mBREAK\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m# (269282, 199)\u001b[39;00m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "# df_slice.to_pickle(\n", "# \"/home/tony/Data/Preference/30b_v6/interesting_clips_v4_h_t_6_20250426_full_long_slice.pkl\"\n", "# )\n", "print(df_slice.shape)\n", "print(df_slice[\"task\"].value_counts())\n", "BREAK\n", "# (269282, 199)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.765756Z", "iopub.status.idle": "2025-11-23T21:04:47.765936Z", "shell.execute_reply": "2025-11-23T21:04:47.765852Z", "shell.execute_reply.started": "2025-11-23T21:04:47.765843Z" } }, "outputs": [], "source": [ "print(df_slice.shape)\n", "df_slice = df_slice[df_slice[\"request_id\"].apply(lambda x: len(x) > 3)]\n", "print(df_slice.shape)\n", "# df_slice = df_slice[df_slice[\"is_pro_user\"]].copy()\n", "print(df_slice.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932296Z", "start_time": "2024-05-16T13:59:41.932287Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.766741Z", "iopub.status.idle": "2025-11-23T21:04:47.766902Z", "shell.execute_reply": "2025-11-23T21:04:47.766829Z", "shell.execute_reply.started": "2025-11-23T21:04:47.766822Z" } }, "outputs": [], "source": [ "# don't have continue at\n", "df_slice[\"request_id\"] = df_slice[\"request_id\"].astype(str)\n", "# df_slice[\"npz_path\"] = df_slice[\"npz_path\"].apply(lambda x: str(x).replace(\"_npz\", \"_npz/\"))\n", "# df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique(), df_slice[\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932966Z", "start_time": "2024-05-16T13:59:41.932957Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.767382Z", "iopub.status.idle": "2025-11-23T21:04:47.767557Z", "shell.execute_reply": "2025-11-23T21:04:47.767480Z", "shell.execute_reply.started": "2025-11-23T21:04:47.767472Z" } }, "outputs": [], "source": [ "final_filtered_requests = df_slice[\"request_id\"].astype(str).unique()\n", "# final_filtered_requests = df_slice[df_slice[\"is_pro_user\"]][\"request_id\"].astype(str).unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934277Z", "start_time": "2024-05-16T13:59:41.934268Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.767866Z", "iopub.status.idle": "2025-11-23T21:04:47.768014Z", "shell.execute_reply": "2025-11-23T21:04:47.767945Z", "shell.execute_reply.started": "2025-11-23T21:04:47.767937Z" } }, "outputs": [], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df # .reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df # .reset_index()\n", "train_df = train_df.reset_index(drop=True)\n", "val_df = val_df.reset_index(drop=True)\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.935620Z", "start_time": "2024-05-16T13:59:41.935613Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.768475Z", "iopub.status.idle": "2025-11-23T21:04:47.768623Z", "shell.execute_reply": "2025-11-23T21:04:47.768555Z", "shell.execute_reply.started": "2025-11-23T21:04:47.768549Z" } }, "outputs": [], "source": [ "total_duration = 0\n", "for i, row in tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " total_duration += row[\"duration\"]\n", " except Exception as E:\n", " print(i, row)\n", " print(E)\n", " raise ValueError()\n", "\n", "print(\n", " f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1000} nodes, {train_df.shape[0] / 8 / 6 / 16} iters\"\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936268Z", "start_time": "2024-05-16T13:59:41.936260Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.768905Z", "iopub.status.idle": "2025-11-23T21:04:47.769052Z", "shell.execute_reply": "2025-11-23T21:04:47.768980Z", "shell.execute_reply.started": "2025-11-23T21:04:47.768973Z" } }, "outputs": [], "source": [ "make_dataset(\n", " val_df, OUT_DATA_DIR, is_val=True, npz_dir=NPZ_DIR, t_data_memmap=N_TOKENS_AUDIO\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.769584Z", "iopub.status.idle": "2025-11-23T21:04:47.769728Z", "shell.execute_reply": "2025-11-23T21:04:47.769661Z", "shell.execute_reply.started": "2025-11-23T21:04:47.769655Z" } }, "outputs": [], "source": [ "# test_npz = np.load(\"/app/suno/data/dpo/30b_npz/26d19085-18da-4701-af43-122684543891.npz\")\n", "# for k in test_npz.keys():\n", "# print(k)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936964Z", "start_time": "2024-05-16T13:59:41.936957Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.770045Z", "iopub.status.idle": "2025-11-23T21:04:47.770189Z", "shell.execute_reply": "2025-11-23T21:04:47.770122Z", "shell.execute_reply.started": "2025-11-23T21:04:47.770115Z" } }, "outputs": [], "source": [ "make_dataset(\n", " train_df, OUT_DATA_DIR, is_val=False, npz_dir=NPZ_DIR, t_data_memmap=N_TOKENS_AUDIO\n", ")" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.937879Z", "start_time": "2024-05-16T13:59:41.937870Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.770674Z", "iopub.status.idle": "2025-11-23T21:04:47.770834Z", "shell.execute_reply": "2025-11-23T21:04:47.770760Z", "shell.execute_reply.started": "2025-11-23T21:04:47.770753Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, N_TOKENS_AUDIO, 1)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.771249Z", "iopub.status.idle": "2025-11-23T21:04:47.771397Z", "shell.execute_reply": "2025-11-23T21:04:47.771328Z", "shell.execute_reply.started": "2025-11-23T21:04:47.771321Z" } }, "outputs": [], "source": [ "task_counts = Counter()\n", "for test_meta in test_metas:\n", " task_counts[test_meta.get(\"task\")] += 1\n", "print(task_counts)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.938629Z", "start_time": "2024-05-16T13:59:41.938621Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.771850Z", "iopub.status.idle": "2025-11-23T21:04:47.771994Z", "shell.execute_reply": "2025-11-23T21:04:47.771928Z", "shell.execute_reply.started": "2025-11-23T21:04:47.771920Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as codec_encode,\n", "# decode_stream_to_full_audio as codec_decode,\n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# decode as decode\n", "# )\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "# assert len(test_metas) == len(mm)\n", "# idx_list = list(range(len(test_metas)))\n", "# # random.shuffle(idx_list)\n", "# # idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939205Z", "start_time": "2024-05-16T13:59:41.939198Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.772299Z", "iopub.status.idle": "2025-11-23T21:04:47.772435Z", "shell.execute_reply": "2025-11-23T21:04:47.772370Z", "shell.execute_reply.started": "2025-11-23T21:04:47.772364Z" } }, "outputs": [], "source": [ "# import random\n", "# idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "# assert \"original_duration_s\" in test_metas[idx]\n", "# # positive index should be shifted by 1\n", "# pos_idx = idx + 1\n", "# print(\n", "# \"tags:\",\n", "# test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", "# test_metas[idx].get(\"tags\"),\n", "# )\n", "# arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[: pad_idx_arr[0], :]\n", "# pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pos_pad_idx_arr) > 0:\n", "# pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "# a = decode(arr)\n", "# print(\"\\n negative example \\n\", test_metas[idx])\n", "# a.play(compress=False)\n", "# pos_a = decode(pos_arr)\n", "# print(\"\\n positive example \\n\", test_metas[pos_idx])\n", "# pos_a.play(compress=False)\n", "# print(\n", "# \"text:\",\n", "# test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", "# test_metas[idx].get(\"text\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939977Z", "start_time": "2024-05-16T13:59:41.939969Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.772977Z", "iopub.status.idle": "2025-11-23T21:04:47.773132Z", "shell.execute_reply": "2025-11-23T21:04:47.773058Z", "shell.execute_reply.started": "2025-11-23T21:04:47.773051Z" } }, "outputs": [], "source": [ "# val_df[val_df[\"tags\"] == 'a vibrant blend of experimental jazz fusion, drum-and-bass and swagger fuzzed-out guitars']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.940610Z", "start_time": "2024-05-16T13:59:41.940603Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.773578Z", "iopub.status.idle": "2025-11-23T21:04:47.773726Z", "shell.execute_reply": "2025-11-23T21:04:47.773657Z", "shell.execute_reply.started": "2025-11-23T21:04:47.773650Z" } }, "outputs": [], "source": [ "# from collections import Counter\n", "# c = Counter()\n", "# for _, row in df_slice.iterrows():\n", "# # print(row[\"metadata\"])\n", "# for k in ast.literal_eval(row[\"metadata\"]).keys():\n", "# c[k] += 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941167Z", "start_time": "2024-05-16T13:59:41.941159Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.774019Z", "iopub.status.idle": "2025-11-23T21:04:47.774160Z", "shell.execute_reply": "2025-11-23T21:04:47.774094Z", "shell.execute_reply.started": "2025-11-23T21:04:47.774087Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941801Z", "start_time": "2024-05-16T13:59:41.941793Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.774563Z", "iopub.status.idle": "2025-11-23T21:04:47.774702Z", "shell.execute_reply": "2025-11-23T21:04:47.774637Z", "shell.execute_reply.started": "2025-11-23T21:04:47.774630Z" } }, "outputs": [], "source": [ "def validation_on_metas(input_metas):\n", " total_bad = 0\n", " total_good = 0\n", " for idx in range(len(input_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if input_metas[idx].get(\"tags\") != input_metas[pos_idx].get(\"tags\"):\n", " print(\n", " input_metas[idx].get(\"id\"),\n", " input_metas[idx].get(\"tags\"),\n", " input_metas[pos_idx].get(\"id\"),\n", " input_metas[pos_idx].get(\"tags\"),\n", " )\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " elif input_metas[idx].get(\"text\") != input_metas[pos_idx].get(\"text\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", " print(total_good, total_bad)\n", " return\n", "\n", "\n", "validation_on_metas(test_metas)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.942520Z", "start_time": "2024-05-16T13:59:41.942511Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.775134Z", "iopub.status.idle": "2025-11-23T21:04:47.775277Z", "shell.execute_reply": "2025-11-23T21:04:47.775210Z", "shell.execute_reply.started": "2025-11-23T21:04:47.775203Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.943072Z", "start_time": "2024-05-16T13:59:41.943065Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.775604Z", "iopub.status.idle": "2025-11-23T21:04:47.775751Z", "shell.execute_reply": "2025-11-23T21:04:47.775683Z", "shell.execute_reply.started": "2025-11-23T21:04:47.775676Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)\n", "# make sure they are offset by 1 and exactly 1\n", "for i, j in zip(n_neg_tr, n_pos_tr):\n", " assert i == j - 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.944246Z", "start_time": "2024-05-16T13:59:41.944237Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.776022Z", "iopub.status.idle": "2025-11-23T21:04:47.776164Z", "shell.execute_reply": "2025-11-23T21:04:47.776097Z", "shell.execute_reply.started": "2025-11-23T21:04:47.776090Z" } }, "outputs": [], "source": [ "total_iters = len(n_neg_tr) + len(n_pos_tr)\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.776578Z", "iopub.status.idle": "2025-11-23T21:04:47.776722Z", "shell.execute_reply": "2025-11-23T21:04:47.776655Z", "shell.execute_reply.started": "2025-11-23T21:04:47.776649Z" } }, "outputs": [], "source": [ "metas_tr = read_jsonl(os.path.join(OUT_DATA_DIR, \"meta_tr.jsonl\"))\n", "validation_on_metas(metas_tr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.777188Z", "iopub.status.idle": "2025-11-23T21:04:47.777328Z", "shell.execute_reply": "2025-11-23T21:04:47.777262Z", "shell.execute_reply.started": "2025-11-23T21:04:47.777255Z" } }, "outputs": [], "source": [ "# new_metas_tr = []\n", "# for index, l in enumerate(metas_tr):\n", "# if index % 2 == 1:\n", "# last_l = new_metas_tr[-1]\n", "# if l[\"tags\"] != last_l[\"tags\"]:\n", "# print(l[\"tags\"], last_l[\"tags\"])\n", "# l[\"tags\"] = last_l[\"tags\"]\n", "# new_metas_tr.append(l)\n", "# validation_on_metas(new_metas_tr)\n", "# write_jsonl(new_metas_tr, os.path.join(OUT_DATA_DIR, \"meta_tr.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945249Z", "start_time": "2024-05-16T13:59:41.945241Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.777629Z", "iopub.status.idle": "2025-11-23T21:04:47.777763Z", "shell.execute_reply": "2025-11-23T21:04:47.777699Z", "shell.execute_reply.started": "2025-11-23T21:04:47.777693Z" } }, "outputs": [], "source": [ "print(\"1 epoch per batch 4, total\", total_iters / 8 / 8 / 4)\n", "print(\"1 epoch per batch 6, total\", total_iters / 16 / 8 / 6)\n", "print(\"1 epoch per batch 8, total\", total_iters / 16 / 8 / 8)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.778030Z", "iopub.status.idle": "2025-11-23T21:04:47.778302Z", "shell.execute_reply": "2025-11-23T21:04:47.778101Z", "shell.execute_reply.started": "2025-11-23T21:04:47.778094Z" } }, "outputs": [], "source": [ "# import time\n", "# time.sleep(60 * 60 * 1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.778830Z", "iopub.status.idle": "2025-11-23T21:04:47.778978Z", "shell.execute_reply": "2025-11-23T21:04:47.778909Z", "shell.execute_reply.started": "2025-11-23T21:04:47.778903Z" } }, "outputs": [], "source": [ "# df[\"control_tags\"].value_counts()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945972Z", "start_time": "2024-05-16T13:59:41.945964Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.779279Z", "iopub.status.idle": "2025-11-23T21:04:47.779415Z", "shell.execute_reply": "2025-11-23T21:04:47.779351Z", "shell.execute_reply.started": "2025-11-23T21:04:47.779345Z" } }, "outputs": [], "source": [ "# !cd /home/tony/Work/tony/slurm/crow && sbatch sbatch_ipo_crow" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.779785Z", "iopub.status.idle": "2025-11-23T21:04:47.779921Z", "shell.execute_reply": "2025-11-23T21:04:47.779857Z", "shell.execute_reply.started": "2025-11-23T21:04:47.779851Z" } }, "outputs": [], "source": [ "import shutil\n", "\n", "# Basic file copy\n", "shutil.copy(\n", " \"/home/tony/Work/tony/Preference/make_dataset_crow_t1_dup.ipynb\",\n", " os.path.join(OUT_DATA_DIR, \"make_dataset.ipynb\"),\n", ")\n", "print(\"Cache kept!\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.780166Z", "iopub.status.idle": "2025-11-23T21:04:47.780301Z", "shell.execute_reply": "2025-11-23T21:04:47.780238Z", "shell.execute_reply.started": "2025-11-23T21:04:47.780231Z" } }, "outputs": [], "source": [ "BREAK" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# some gymathtics loading prev data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.946562Z", "start_time": "2024-05-16T13:59:41.946555Z" }, "execution": { "iopub.status.busy": "2025-11-23T21:04:47.780760Z", "iopub.status.idle": "2025-11-23T21:04:47.780909Z", "shell.execute_reply": "2025-11-23T21:04:47.780839Z", "shell.execute_reply.started": "2025-11-23T21:04:47.780832Z" } }, "outputs": [], "source": [ "# prev_v3_data = \"/app/suno/data/dpo/7v_v20_full/\"\n", "\n", "# test_val_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_val.jsonl\"))\n", "# test_tr_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_tr.jsonl\"))\n", "\n", "# all_ids = set()\n", "# for meta in test_val_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# for meta in test_tr_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# print(len(all_ids), len(test_val_metas) + len(test_tr_metas))\n", "\n", "# all_ids = list(all_ids)\n", "# with open(\"/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id.json\", \"w\") as fp:\n", "# json.dump(all_ids, fp)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.781237Z", "iopub.status.idle": "2025-11-23T21:04:47.781378Z", "shell.execute_reply": "2025-11-23T21:04:47.781312Z", "shell.execute_reply.started": "2025-11-23T21:04:47.781305Z" } }, "outputs": [], "source": [ "# x_data = train_df[train_df[\"preference\"]][\"similarity\"]\n", "# y_data = train_df[~train_df[\"preference\"]][\"similarity\"]\n", "# from matplotlib.colors import LogNorm\n", "\n", "# fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 10))\n", "\n", "# # 2D Histogram\n", "# h = ax1.hist2d(\n", "# x_data,\n", "# y_data,\n", "# bins=(50, 50),\n", "# cmap=\"coolwarm\",\n", "# range=[[0, 1], [0, 1]],\n", "# norm=LogNorm(),\n", "# )\n", "\n", "# ax1.set_xlabel(\"Semantic Distance (Preferred)\")\n", "# ax1.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "# ax1.set_title(\n", "# \"2D Histogram of Semantic Distances: Preferred vs Non-Preferred (Log Scale)\"\n", "# )\n", "\n", "# cbar1 = plt.colorbar(h[3], ax=ax1)\n", "# cbar1.set_label(\"Number of Request IDs (Log Scale)\")\n", "\n", "# # Scatter plot\n", "# ax2.scatter(x_data, y_data, alpha=0.1, s=1)\n", "# ax2.set_xlabel(\"Semantic Distance (Preferred)\")\n", "# ax2.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "# ax2.set_title(\"Scatter Plot of Semantic Distances: Preferred vs Non-Preferred\")\n", "# ax2.set_xlim(0, 1)\n", "# ax2.set_ylim(0, 1)\n", "\n", "# plt.tight_layout()\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.781861Z", "iopub.status.idle": "2025-11-23T21:04:47.782005Z", "shell.execute_reply": "2025-11-23T21:04:47.781936Z", "shell.execute_reply.started": "2025-11-23T21:04:47.781929Z" } }, "outputs": [], "source": [ "# train_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_tr.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.782364Z", "iopub.status.idle": "2025-11-23T21:04:47.782502Z", "shell.execute_reply": "2025-11-23T21:04:47.782436Z", "shell.execute_reply.started": "2025-11-23T21:04:47.782429Z" } }, "outputs": [], "source": [ "# train_info.keys()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.782911Z", "iopub.status.idle": "2025-11-23T21:04:47.783049Z", "shell.execute_reply": "2025-11-23T21:04:47.782983Z", "shell.execute_reply.started": "2025-11-23T21:04:47.782977Z" } }, "outputs": [], "source": [ "# import torch\n", "\n", "# a = torch.tensor([6.2500e-04, 3.9062e-05, 2.3462e-03])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.783420Z", "iopub.status.idle": "2025-11-23T21:04:47.783562Z", "shell.execute_reply": "2025-11-23T21:04:47.783494Z", "shell.execute_reply.started": "2025-11-23T21:04:47.783488Z" } }, "outputs": [], "source": [ "# a.mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.784002Z", "iopub.status.idle": "2025-11-23T21:04:47.784145Z", "shell.execute_reply": "2025-11-23T21:04:47.784077Z", "shell.execute_reply.started": "2025-11-23T21:04:47.784070Z" } }, "outputs": [], "source": [ "# import time\n", "# time.sleep(3600 * 3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.784602Z", "iopub.status.idle": "2025-11-23T21:04:47.784742Z", "shell.execute_reply": "2025-11-23T21:04:47.784675Z", "shell.execute_reply.started": "2025-11-23T21:04:47.784669Z" } }, "outputs": [], "source": [ "# !cd /home/tony/Work/tony/slurm/diffusion && sbatch run_diffusion_infill.sh" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.785065Z", "iopub.status.idle": "2025-11-23T21:04:47.785201Z", "shell.execute_reply": "2025-11-23T21:04:47.785138Z", "shell.execute_reply.started": "2025-11-23T21:04:47.785131Z" } }, "outputs": [], "source": [ "def get_control_slider(metadata):\n", " if \"param_experiment\" in metadata:\n", " exp = metadata.get(\"param_experiment\", \"\")\n", " if exp:\n", " if exp == \"mask_control_slider\":\n", " if not metadata.get(\"control_sliders\", None):\n", " return False\n", " else:\n", " return True\n", " return None\n", "\n", "df[\"mask_control\"] = df.apply(\n", " lambda row: get_control_slider(row[\"metadata\"]), axis=1\n", ")\n", "masked_request_id = df[~df[\"mask_control\"].isna()][\"request_id\"].unique()\n", "subset_df = df[df[\"request_id\"].isin(masked_request_id)].copy()\n", "print(\"check subset shape\", df.shape, subset_df.shape)\n", "# control masked\n", "user_pref_pct = (\n", " subset_df[subset_df[\"mask_control\"] == True].groupby(\"user_id\")[\"preference\"]\n", " .apply(lambda x: x.mean())\n", ")\n", "print(user_pref_pct.head())\n", "# Plot a histogram of the user preference percentages\n", "plt.figure(figsize=(6, 4))\n", "plt.hist(user_pref_pct, bins=np.linspace(0, 1, 100), edgecolor=\"black\")\n", "plt.xlabel(\"Preference % for mask_control\")\n", "plt.ylabel(\"Number of Users\")\n", "plt.title(\"Histogram of User Preference for 'mask_control'\")\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.785546Z", "iopub.status.idle": "2025-11-23T21:04:47.785700Z", "shell.execute_reply": "2025-11-23T21:04:47.785629Z", "shell.execute_reply.started": "2025-11-23T21:04:47.785622Z" } }, "outputs": [], "source": [ "user_counts = subset_df.groupby(\"user_id\")[\"preference\"].count()\n", "eligible_users = user_counts[user_counts >= 4].index\n", "len(eligible_users)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.786157Z", "iopub.status.idle": "2025-11-23T21:04:47.786306Z", "shell.execute_reply": "2025-11-23T21:04:47.786237Z", "shell.execute_reply.started": "2025-11-23T21:04:47.786230Z" } }, "outputs": [], "source": [ "# control masked\n", "eligible_user_pref_pct = (\n", " subset_df[subset_df[\"user_id\"].isin(eligible_users) & (subset_df[\"mask_control\"] == True)].groupby(\"user_id\")[\"preference\"]\n", " .apply(lambda x: x.mean())\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.786625Z", "iopub.status.idle": "2025-11-23T21:04:47.786768Z", "shell.execute_reply": "2025-11-23T21:04:47.786698Z", "shell.execute_reply.started": "2025-11-23T21:04:47.786692Z" } }, "outputs": [], "source": [ "# Plot a histogram of the user preference percentages\n", "plt.figure(figsize=(6, 4))\n", "plt.hist(eligible_user_pref_pct, bins=np.linspace(0, 1, 20), edgecolor=\"black\")\n", "plt.xlabel(\"Preference % for mask_control\")\n", "plt.ylabel(\"Number of Users\")\n", "plt.title(\"Histogram of User Preference for 'mask_control'\")\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.787251Z", "iopub.status.idle": "2025-11-23T21:04:47.787405Z", "shell.execute_reply": "2025-11-23T21:04:47.787336Z", "shell.execute_reply.started": "2025-11-23T21:04:47.787328Z" } }, "outputs": [], "source": [ "eligible_user_pref_pct[172918] # kakermix" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## sliders" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.787825Z", "iopub.status.idle": "2025-11-23T21:04:47.787972Z", "shell.execute_reply": "2025-11-23T21:04:47.787902Z", "shell.execute_reply.started": "2025-11-23T21:04:47.787895Z" } }, "outputs": [], "source": [ "from typing import Any\n", "import pandas as pd\n", "\n", "def map_control_sliders_to_df(df: pd.DataFrame) -> pd.DataFrame:\n", " \"\"\"Extracts 'style_weight' and 'weirdness_constraint' from the 'control_sliders' dict in the 'metadata' column\n", " and adds them as new columns to the DataFrame.\n", "\n", " Args:\n", " df (pd.DataFrame): DataFrame with a 'metadata' column containing a 'control_sliders' dict.\n", "\n", " Returns:\n", " pd.DataFrame: DataFrame with added 'style_weight' and 'weirdness_constraint' columns.\n", "\n", " Raises:\n", " KeyError: If 'control_sliders', 'style_weight', or 'weirdness_constraint' are missing in any row.\n", " TypeError: If the extracted values are not floats.\n", "\n", " Example:\n", " >>> import pandas as pd\n", " >>> data = [{'metadata': {'control_sliders': {'style_weight': 0.89, 'weirdness_constraint': 0.8}}}]\n", " >>> df = pd.DataFrame(data)\n", " >>> df = map_control_sliders_to_df(df)\n", " >>> df[['style_weight', 'weirdness_constraint']].iloc[0].tolist()\n", " [0.89, 0.8]\n", " \"\"\"\n", " # Vectorized extraction for performance\n", " sliders = df[\"metadata\"].map(lambda m: m.get(\"control_sliders\", {}))\n", " style_weight = sliders.map(lambda s: s.get(\"style_weight\", None))\n", " weirdness_constraint = sliders.map(lambda s: s.get(\"weirdness_constraint\", None))\n", " audio_weight = sliders.map(lambda s: s.get(\"audio_weight\", None))\n", "\n", " df[\"style_weight\"] = style_weight\n", " df[\"weirdness_constraint\"] = weirdness_constraint\n", " df[\"audio_weight\"] = audio_weight\n", " return df\n", "\n", "def print_percentiles(\n", " data: np.ndarray,\n", " percentiles: list[float] = [5, 10, 20, 25, 50, 75, 80, 90, 95]\n", ") -> None:\n", " \"\"\"Prints specified percentiles of the data.\n", "\n", " Args:\n", " data (np.ndarray): Array of values to compute percentiles for.\n", " percentiles (list[float], optional): List of percentiles to print. Defaults to [5, 10, 20, 25, 50, 75, 80, 90, 95].\n", "\n", " Example:\n", " >>> print_percentiles(np.array([1, 2, 3, 4, 5]))\n", " \"\"\"\n", " data = data[~data.isna()]\n", " results = np.percentile(data, percentiles)\n", " print(\"Percentiles:\")\n", " for p, v in zip(percentiles, results):\n", " print(f\" {p:>3}%: {v:.4f}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.788396Z", "iopub.status.idle": "2025-11-23T21:04:47.788542Z", "shell.execute_reply": "2025-11-23T21:04:47.788473Z", "shell.execute_reply.started": "2025-11-23T21:04:47.788466Z" } }, "outputs": [], "source": [ "df_total = map_control_sliders_to_df(df_total)\n", "df_total[[\"style_weight\",\"weirdness_constraint\",\"audio_weight\"]].describe()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.788946Z", "iopub.status.idle": "2025-11-23T21:04:47.789093Z", "shell.execute_reply": "2025-11-23T21:04:47.789025Z", "shell.execute_reply.started": "2025-11-23T21:04:47.789018Z" } }, "outputs": [], "source": [ "plt.figure(figsize=(8, 5))\n", "df_total[\"style_weight\"].hist(bins=np.linspace(0, 1, 100), color=\"skyblue\", edgecolor=\"black\")\n", "print_percentiles(df_total[\"style_weight\"])\n", "plt.title(\"Distribution of Style Weight\")\n", "plt.xlabel(\"Style Weight\")\n", "plt.ylabel(\"Counts\")\n", "plt.grid(True, linestyle=\"--\", alpha=0.6)\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.789486Z", "iopub.status.idle": "2025-11-23T21:04:47.789628Z", "shell.execute_reply": "2025-11-23T21:04:47.789560Z", "shell.execute_reply.started": "2025-11-23T21:04:47.789553Z" } }, "outputs": [], "source": [ "plt.figure(figsize=(8, 5))\n", "df_total[\"weirdness_constraint\"].hist(bins=np.linspace(0, 1, 100), color=\"skyblue\", edgecolor=\"black\")\n", "print_percentiles(df_total[\"weirdness_constraint\"])\n", "plt.title(\"Distribution of Weirdness\")\n", "plt.xlabel(\"weirdness_constraint\")\n", "plt.ylabel(\"Counts\")\n", "plt.grid(True, linestyle=\"--\", alpha=0.6)\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.790255Z", "iopub.status.idle": "2025-11-23T21:04:47.790413Z", "shell.execute_reply": "2025-11-23T21:04:47.790344Z", "shell.execute_reply.started": "2025-11-23T21:04:47.790337Z" } }, "outputs": [], "source": [ "plt.figure(figsize=(8, 5))\n", "df_total[\"audio_weight\"].hist(bins=np.linspace(0, 1, 100), color=\"skyblue\", edgecolor=\"black\")\n", "print_percentiles(df_total[\"audio_weight\"])\n", "plt.title(\"Distribution of audio_weight\")\n", "plt.xlabel(\"audio_weight\")\n", "plt.ylabel(\"Counts\")\n", "plt.grid(True, linestyle=\"--\", alpha=0.6)\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Fetch other parameters" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.790832Z", "iopub.status.idle": "2025-11-23T21:04:47.790981Z", "shell.execute_reply": "2025-11-23T21:04:47.790911Z", "shell.execute_reply.started": "2025-11-23T21:04:47.790904Z" } }, "outputs": [], "source": [ "# df_total[df_total[\"task\"] == \"stem_condition\"][[\"s3_id\", \"preference\", \"request_id\", \"control_tags\"]].head(n=100)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.791383Z", "iopub.status.idle": "2025-11-23T21:04:47.791526Z", "shell.execute_reply": "2025-11-23T21:04:47.791458Z", "shell.execute_reply.started": "2025-11-23T21:04:47.791452Z" } }, "outputs": [], "source": [ "len(metas_tr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.791780Z", "iopub.status.idle": "2025-11-23T21:04:47.791915Z", "shell.execute_reply": "2025-11-23T21:04:47.791850Z", "shell.execute_reply.started": "2025-11-23T21:04:47.791844Z" } }, "outputs": [], "source": [ "c = Counter()\n", "for data_meta in metas_tr:\n", " c[data_meta.get(\"control_tags\", \"\") if isinstance(data_meta.get(\"control_tags\", \"\"), str) else \"\"] += 1\n", "print(c)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.792195Z", "iopub.status.idle": "2025-11-23T21:04:47.792331Z", "shell.execute_reply": "2025-11-23T21:04:47.792266Z", "shell.execute_reply.started": "2025-11-23T21:04:47.792260Z" } }, "outputs": [], "source": [ "# with open(\"/app2/suno/data/dpo/crow_t1_v3/dodo_t23_2025-08-28_01-30-09_cached_loss.json\", \"r\") as fp:\n", "# old_loss = json.load(fp)\n", "# with open(\"/app2/suno/data/dpo/crow_t1_v3/dodo_t23_2025-08-28_01-30-09_redo3_cached_loss.json\", \"r\") as fp:\n", "# new_loss = json.load(fp)\n", "\n", "# sem_losses = []\n", "# for curr_id, o_v in old_loss[\"train\"].items():\n", "# n_v = new_loss[\"train\"][curr_id]\n", "# for k in o_v:\n", "# assert o_v[k] == n_v[k]\n", "# if k == \"semantic_0\":\n", "# if o_v[\"orig_idx\"] % 2 == 1:\n", "# sem_losses.append(o_v[k])\n", "# # if o_v[k] > 4 and o_v[\"orig_idx\"] % 2 == 1:\n", "# # print(o_v, o_v[\"orig_idx\"] % 3360, o_v[\"orig_idx\"] % 3360 // 4)\n", "# sem_losses.sort()\n", "# plt.hist(sem_losses, bins=np.linspace(0, 10, 100))\n", "# plt.yscale(\"log\")\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.792760Z", "iopub.status.idle": "2025-11-23T21:04:47.792909Z", "shell.execute_reply": "2025-11-23T21:04:47.792840Z", "shell.execute_reply.started": "2025-11-23T21:04:47.792833Z" } }, "outputs": [], "source": [ "# df[df[\"is_public\"]].to_pickle(\"/home/tony/Data/Preference/crow_t1/interesting_clips_crow_t1_20251020_public_only.pkl\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.793359Z", "iopub.status.idle": "2025-11-23T21:04:47.793504Z", "shell.execute_reply": "2025-11-23T21:04:47.793437Z", "shell.execute_reply.started": "2025-11-23T21:04:47.793431Z" } }, "outputs": [], "source": [ "df[df[\"is_public\"] & (df[\"tags\"].str.len() < 1)]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.793785Z", "iopub.status.idle": "2025-11-23T21:04:47.793923Z", "shell.execute_reply": "2025-11-23T21:04:47.793859Z", "shell.execute_reply.started": "2025-11-23T21:04:47.793852Z" } }, "outputs": [], "source": [ "from suno_analytics.preference_data_selection import (\n", " plot_clip_distribution\n", ")\n", "plot_clip_distribution(df)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.794354Z", "iopub.status.idle": "2025-11-23T21:04:47.794496Z", "shell.execute_reply": "2025-11-23T21:04:47.794429Z", "shell.execute_reply.started": "2025-11-23T21:04:47.794422Z" } }, "outputs": [], "source": [ "plot_clip_distribution(df_total)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.795036Z", "iopub.status.idle": "2025-11-23T21:04:47.795188Z", "shell.execute_reply": "2025-11-23T21:04:47.795117Z", "shell.execute_reply.started": "2025-11-23T21:04:47.795110Z" } }, "outputs": [], "source": [ "plot_clip_distribution(df_total[df_total[\"source\"] == \"ios\"])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-23T21:04:47.795605Z", "iopub.status.idle": "2025-11-23T21:04:47.795750Z", "shell.execute_reply": "2025-11-23T21:04:47.795680Z", "shell.execute_reply.started": "2025-11-23T21:04:47.795673Z" } }, "outputs": [], "source": [ "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "\n", "# Assuming df is your dataframe with 'created_at' column\n", "# df = pd.read_csv('your_data.csv')\n", "\n", "# Convert created_at to datetime if not already\n", "df['created_at'] = pd.to_datetime(df['created_at'])\n", "\n", "# Extract day and hour\n", "df['date'] = df['created_at'].dt.date\n", "df['hour'] = df['created_at'].dt.hour\n", "\n", "# Define your cut_frac function here\n", "def compute_cut_frac(group_df):\n", " \"\"\"\n", " Define your cut fraction logic here.\n", " \n", " Examples:\n", " - Fraction filtered out: (group_df['filtered'] == True).mean()\n", " - Fraction above threshold: (group_df['margin'] > 0.5).mean()\n", " - Fraction in bottom quantile: (group_df['score'] < group_df['score'].quantile(0.2)).mean()\n", " \"\"\"\n", " # CUSTOMIZE THIS:\n", " # Example: fraction of rows where margin > threshold\n", " threshold = 0.5\n", " cut_frac = (group_df[\"preference\"] & (group_df[\"norm_play_frac\"] > 5.1) \n", " # & (group_df[\"norm_play_frac\"] >= group_df[\"reaction_play_count\"] / 3)\n", " ).mean()\n", " \n", " return cut_frac\n", "\n", "# Group by date and hour\n", "daily_hourly_stats = df.groupby(['date', 'hour']).apply(compute_cut_frac).reset_index()\n", "daily_hourly_stats.columns = ['date', 'hour', 'cut_frac']\n", "\n", "# Also compute count per day-hour\n", "daily_hourly_counts = df.groupby(['date', 'hour']).size().reset_index(name='count')\n", "daily_hourly_stats = daily_hourly_stats.merge(daily_hourly_counts, on=['date', 'hour'])\n", "\n", "# Create plots\n", "fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(14, 10))\n", "\n", "# Plot 1: Line plot with one line per day\n", "for date in sorted(df['date'].unique()):\n", " date_data = daily_hourly_stats[daily_hourly_stats['date'] == date]\n", " ax1.plot(date_data['hour'], date_data['cut_frac'], \n", " marker='o', linewidth=2, markersize=4, label=str(date), alpha=0.7)\n", "\n", "ax1.set_xlabel('Hour of Day', fontsize=12)\n", "ax1.set_ylabel('Cut Fraction', fontsize=12)\n", "ax1.set_title('Cut Fraction by Day and Hour', fontsize=14, fontweight='bold')\n", "ax1.grid(True, alpha=0.3)\n", "ax1.set_xlim(-0.5, 23.5)\n", "ax1.set_xticks(range(0, 24, 2))\n", "ax1.legend(bbox_to_anchor=(1.05, 1), loc='upper left', title='Date')\n", "\n", "# Plot 2: Heatmap\n", "pivot_table = daily_hourly_stats.pivot(index='date', columns='hour', values='cut_frac')\n", "im = ax2.imshow(pivot_table, aspect='auto', cmap='viridis', interpolation='nearest')\n", "\n", "ax2.set_xlabel('Hour of Day', fontsize=12)\n", "ax2.set_ylabel('Date', fontsize=12)\n", "ax2.set_title('Cut Fraction Heatmap', fontsize=14, fontweight='bold')\n", "ax2.set_xticks(range(24))\n", "ax2.set_xticklabels(range(24))\n", "ax2.set_yticks(range(len(pivot_table)))\n", "ax2.set_yticklabels([str(d) for d in pivot_table.index])\n", "\n", "# Add colorbar\n", "cbar = plt.colorbar(im, ax=ax2)\n", "cbar.set_label('Cut Fraction', fontsize=12)\n", "\n", "fig.tight_layout()\n", "plt.show()\n", "\n", "# Print summary stats\n", "print(f\"\\nSummary Statistics:\")\n", "print(f\"Overall cut_frac: {daily_hourly_stats['cut_frac'].mean():.3f}\")\n", "print(f\"Min cut_frac: {daily_hourly_stats['cut_frac'].min():.3f}\")\n", "print(f\"Max cut_frac: {daily_hourly_stats['cut_frac'].max():.3f}\")\n", "print(f\"Total samples: {daily_hourly_stats['count'].sum()}\")\n", "print(f\"\\nPer-day averages:\")\n", "day_avg = daily_hourly_stats.groupby('date')['cut_frac'].mean()\n", "for date, avg in day_avg.items():\n", " print(f\" {date}: {avg:.3f}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }