{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:17:39.531841Z", "start_time": "2024-03-04T01:17:38.668920Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_1918232/2161522484.py:1: DeprecationWarning: \n", "Pyarrow will become a required dependency of pandas in the next major release of pandas (pandas 3.0),\n", "(to allow more performant data types, such as the Arrow string type, and better interoperability with other libraries)\n", "but was not found to be installed on your system.\n", "If this would cause problems for you,\n", "please provide us feedback at https://github.com/pandas-dev/pandas/issues/54466\n", " \n", " import pandas as pd\n" ] } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "import os\n", "from tqdm import tqdm\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "import sys\n", "from collections import defaultdict\n", "from suno_utils.utils.text import (\n", " write_jsonl,\n", " read_jsonl,\n", " write_json,\n", " read_json,\n", ")\n", "import ast\n", "\n", "sys.path.insert(0, \"/home/tony/Work/glockenspiel/sunoGPT/scripts/\")\n", "\n", "from data_preparation_7b import *\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:17:39.535702Z", "start_time": "2024-03-04T01:17:39.533180Z" } }, "outputs": [], "source": [ "# NOTE THESE ARE 3b numbers\n", "TEXT_CODEBOOK_SIZE = 60_001\n", "TEXT_PAD_TOKEN = TEXT_CODEBOOK_SIZE\n", "TEXT_VOCAB_SIZE = 60_032\n", "\n", "SEMANTIC_CODEBOOK_SIZE = 4000\n", "SEMANTIC_N_CODEBOOKS = 1\n", "SEMANTIC_PAD_TOKEN = SEMANTIC_CODEBOOK_SIZE\n", "SEMANTIC_INFER_TOKEN = SEMANTIC_CODEBOOK_SIZE + 1\n", "SEMANTIC_VOCAB_SIZE = 4032\n", "SEMANTIC_RATE_HZ = 25\n", "SEMANTIC_SHIFT_FACTOR = 50\n", "\n", "COARSE_CODEBOOK_SIZE = 4096\n", "COARSE_N_CODEBOOKS = 8\n", "COARSE_PAD_TOKEN = COARSE_CODEBOOK_SIZE\n", "COARSE_INFER_TOKEN = COARSE_CODEBOOK_SIZE + 1\n", "COARSE_VOCAB_SIZE = 4160\n", "\n", "BLOCK_SIZE = 4288\n", "N_TOKENS_TEXT = 1152\n", "N_TOKENS_AUDIO = 3008 # max 120s of audio\n", "N_PAD_TOKENS_AUDIO = 3008\n", "\n", "SEMANTIC_EMBED_DIR = \"mert_25_2x4k\"\n", "CODEC_EMBED_DIR = \"dac_2c_25_8\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:17:39.597556Z", "start_time": "2024-03-04T01:17:39.537312Z" } }, "outputs": [], "source": [ "OUT_DATA_DIR = \"/app/suno/data/dpo/3v_v0/\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "NPZ_DIR = \"/app/suno/data/dpo/npz\"" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:17:46.110598Z", "start_time": "2024-03-04T01:17:39.599576Z" } }, "outputs": [ { "data": { "text/plain": [ "(376932, 37)" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = pd.read_csv(\"/home/tony/Data/Preference/7b_v0/interesting_clips.csv\")\n", "df.shape\n", "# v0: (68746, 31)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:18:24.351677Z", "start_time": "2024-03-04T01:17:46.111729Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_1918232/3778571731.py:1: DtypeWarning: Columns (3,10,13,16,17,19,25,28,29,30,31,32,33,34) have mixed types. Specify dtype option on import or set low_memory=False.\n", " df = pd.read_csv(\"/home/tony/Data/Preference/v4/interesting_clips.csv\")\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "(1926120, 35)\n" ] } ], "source": [ "df = pd.read_csv(\"/home/tony/Data/Preference/v4/interesting_clips.csv\")\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:20:18.029733Z", "start_time": "2024-03-04T01:18:24.352869Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3519009\n", "pre-downloaded df (1926120, 35)\n", "downloaded df (1926091, 35)\n" ] } ], "source": [ "converted_paths_v2 = os.listdir(\"/app/suno/data/dpo/npz\")\n", "converted_paths_v2 = set([f.replace(\".npz\", \"\") for f in converted_paths_v2])\n", "print(len(converted_paths_v2))\n", "print(\"pre-downloaded df\", df.shape)\n", "df = df[df[\"s3_id\"].isin(converted_paths_v2)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:20:18.376159Z", "start_time": "2024-03-04T01:20:18.030960Z" } }, "outputs": [ { "data": { "text/plain": [ "is_7b\n", "False 1926091\n", "Name: count, dtype: int64" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"is_7b\"] = df[\"model_name\"].str.contains(\"v3\")\n", "df[\"is_7b\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:20:20.393151Z", "start_time": "2024-03-04T01:20:18.377331Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(1926091, 36)\n", "(1926090, 36)\n" ] } ], "source": [ "print(df.shape)\n", "df = df[df['request_id'].isin(df['request_id'].value_counts().index[df['request_id'].value_counts() == 2])]\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:20:20.721744Z", "start_time": "2024-03-04T01:20:20.394457Z" } }, "outputs": [], "source": [ "assert df.shape[0] == df[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:20:21.087617Z", "start_time": "2024-03-04T01:20:20.723425Z" } }, "outputs": [ { "data": { "text/plain": [ "play_count\n", "3.0 489274\n", "4.0 295994\n", "2.0 261822\n", "1.0 206298\n", "5.0 186592\n", " ... \n", "9041.0 1\n", "357.0 1\n", "417.0 1\n", "396.0 1\n", "406.0 1\n", "Name: count, Length: 365, dtype: int64" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"play_count\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:27:23.987523Z", "start_time": "2024-03-04T01:20:21.088805Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "963045\n" ] } ], "source": [ "# expand the metadata columns\n", "test_slice = df[\"metadata\"].apply(lambda x: pd.Series(ast.literal_eval(x)))\n", "df = pd.concat([df, test_slice], axis=1, join=\"inner\")\n", "print(df[\"request_id\"].nunique())" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:27:25.731555Z", "start_time": "2024-03-04T01:27:23.997512Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "963043\n" ] } ], "source": [ "# double check we removed the gpt prompted ones for now\n", "df = df[df[\"has_gpt_prompt\"] == False]\n", "print(df[\"request_id\"].nunique())" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:27:57.834997Z", "start_time": "2024-03-04T01:27:25.748145Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "297371\n", "good_continue_at\n", "True 1926082\n", "False 4\n", "Name: count, dtype: int64\n" ] } ], "source": [ "audio_prompt_id_to_continue_at = {}\n", "for _, row in df[~df[\"audio_prompt_id\"].isna()].iterrows():\n", " audio_prompt_id = row[\"audio_prompt_id\"]\n", " if audio_prompt_id not in audio_prompt_id_to_continue_at:\n", " audio_prompt_id_to_continue_at[audio_prompt_id] = row[\"continue_at\"]\n", " else:\n", " # pick the max\n", " audio_prompt_id = max(\n", " audio_prompt_id_to_continue_at[audio_prompt_id], row[\"continue_at\"]\n", " )\n", "print(len(audio_prompt_id_to_continue_at))\n", "df[\"has_continue_and_start_continue_at\"] = df[\"id\"].apply(\n", " lambda x: audio_prompt_id_to_continue_at.get(x)\n", ")\n", "# we want continue at to be at least half of the clip...\n", "df[\"good_continue_at\"] = (\n", " (df[\"has_continue_and_start_continue_at\"] / df[\"duration\"]) > 0.85\n", ") | df[\"has_continue_and_start_continue_at\"].isna()\n", "print(df[\"good_continue_at\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:27:57.916122Z", "start_time": "2024-03-04T01:27:57.836181Z" } }, "outputs": [ { "data": { "text/plain": [ "(preference\n", " False 963043\n", " True 963043\n", " Name: count, dtype: int64,\n", " is_7b\n", " False 1926086\n", " Name: count, dtype: int64)" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"preference\"].value_counts(), df[\"is_7b\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:27:58.070800Z", "start_time": "2024-03-04T01:27:57.917230Z" } }, "outputs": [ { "data": { "text/plain": [ "model_name\n", "chirp-v2-xxl-alpha 1908156\n", "chirp-v2-engine-v12 12084\n", "chirp-v2-engine-v8 4116\n", "chirp-v2-engine-v0 1092\n", "chirp-v2-engine-jingle 628\n", "chirp-v2-engine-v13 6\n", "chirp-v2-engine-dev 4\n", "Name: count, dtype: int64" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"model_name\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:28:00.060070Z", "start_time": "2024-03-04T01:27:58.071821Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "negative 728473 positive 133450\n", "total 963043 selected 111277 0.11554728085869478\n" ] } ], "source": [ "neg_filter_selection_mask = (\n", " (df[\"preference\"] == False)\n", " & (df[\"play_count\"] >= 1)\n", " & (df[\"duration\"] > 5)\n", " & (df[\"duration\"] <= 120)\n", " & (\n", " ((df[\"is_7b\"] == False) & (df[\"play_count\"] >= 2) & (df[\"duration\"] >= 10))\n", " | ((df[\"is_7b\"] == True) & (df[\"play_count\"] >= 1))\n", " )\n", ")\n", "pos_filter_selectin_mask = (df[\"preference\"] == True) & (\n", " (\n", " ((df[\"play_count\"] >= 3) & (df[\"is_7b\"] == True))\n", " | ((df[\"play_count\"] >= 10) & (df[\"is_7b\"] == False))\n", " )\n", " & (df[\"duration\"] > 5)\n", " & (df[\"duration\"] <= 120)\n", " & (df[\"good_continue_at\"])\n", ")\n", "print(\n", " \"negative\",\n", " sum(neg_filter_selection_mask),\n", " \"positive\",\n", " sum(pos_filter_selectin_mask),\n", ")\n", "\n", "neg_filter_requests = df[neg_filter_selection_mask][\"request_id\"].unique()\n", "pos_filter_requests = df[pos_filter_selectin_mask][\"request_id\"].unique()\n", "# looking for very strong signal here:\n", "# listen to the positive/negative more than once\n", "# disliked one of the clips\n", "unique_requests = set(pos_filter_requests).intersection(neg_filter_requests)\n", "print(\n", " \"total\",\n", " df[\"request_id\"].nunique(),\n", " \"selected\",\n", " len(unique_requests),\n", " len(unique_requests) / df[\"request_id\"].nunique(),\n", ")" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:28:00.729690Z", "start_time": "2024-03-04T01:28:00.061245Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "111277\n" ] } ], "source": [ "df_slice = df[df[\"request_id\"].isin(set(unique_requests))].copy()\n", "print(df_slice[\"request_id\"].nunique())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T02:56:01.136104Z", "start_time": "2024-03-04T02:56:01.134362Z" } }, "outputs": [], "source": [ "# I don't like this but what options do I have...\n", "BAD_IDs = set([\"62756885-1f8d-4ec1-9bc0-d4f7431ebcdf\"])" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T02:56:01.806222Z", "start_time": "2024-03-04T02:56:01.630774Z" } }, "outputs": [ { "data": { "text/plain": [ "93630" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# don't have continue at\n", "df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T03:50:47.552143Z", "start_time": "2024-03-04T03:50:47.549697Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "12\n" ] } ], "source": [ "print(len(BAD_IDs))" ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T03:50:52.218299Z", "start_time": "2024-03-04T03:50:52.183489Z" } }, "outputs": [], "source": [ "bad_requests = df_slice[df_slice[\"id\"].isin(BAD_IDs)][\"request_id\"].unique()" ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T03:50:53.391953Z", "start_time": "2024-03-04T03:50:53.101712Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "111265\n" ] } ], "source": [ "final_filtered_requests = df_slice[~df_slice[\"request_id\"].isin(bad_requests)][\"request_id\"].unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T03:50:55.243278Z", "start_time": "2024-03-04T03:50:54.162473Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "110152 1113\n", "(220304, 63) (2226, 63)\n" ] } ], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df.reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df.reset_index()\n", "\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "code", "execution_count": 49, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T03:50:55.259152Z", "start_time": "2024-03-04T03:50:55.244769Z" } }, "outputs": [], "source": [ "def reshift(arr):\n", " sem_start_idx = 0\n", " sem_end_idx = len(arr) - 1\n", " semantic_arr = arr[:, :SEMANTIC_N_CODEBOOKS]\n", " coarse_arr = arr[:, SEMANTIC_N_CODEBOOKS:]\n", "\n", " coarse_start_idx = int(round(sem_start_idx * COARSE_RATE_HZ / SEMANTIC_RATE_HZ))\n", " coarse_end_idx = int(round(sem_end_idx * COARSE_RATE_HZ / SEMANTIC_RATE_HZ))\n", " assert sem_end_idx >= 0 and coarse_start_idx >= 0\n", " assert not (sem_end_idx > len(semantic_arr) or coarse_end_idx > len(coarse_arr))\n", "\n", " # get array segments\n", " arr_s = semantic_arr[sem_start_idx:sem_end_idx, :].copy()\n", " arr_c = coarse_arr[coarse_start_idx:coarse_end_idx, :].copy()\n", " assert arr_s.max() <= SEMANTIC_PAD_TOKEN\n", " assert arr_c.max() <= COARSE_PAD_TOKEN\n", " assert len(arr_s) == len(arr_c)\n", " # concat and stack\n", " if len(arr_c) < N_TOKENS_AUDIO:\n", " arr_c = np.pad(\n", " arr_c,\n", " ((0, N_TOKENS_AUDIO - len(arr_c)), (0, 0)),\n", " constant_values=COARSE_PAD_TOKEN,\n", " mode=\"constant\",\n", " )\n", " arr_s = np.pad(\n", " arr_s,\n", " ((0, N_TOKENS_AUDIO - len(arr_s)), (0, 0)),\n", " constant_values=SEMANTIC_PAD_TOKEN,\n", " mode=\"constant\",\n", " )\n", " arr = np.concatenate([arr_s, arr_c], axis=-1)\n", " arr = arr.astype(np.uint16)\n", " assert arr.shape == (N_TOKENS_AUDIO, SEMANTIC_N_CODEBOOKS + COARSE_N_CODEBOOKS)\n", " return arr\n", "\n", "\n", "def make_dataset(df, is_val=False):\n", " dset_type = \"val\" if is_val else \"tr\"\n", " out_mmap_path = os.path.join(OUT_DATA_DIR, f\"data_{dset_type}.bin\")\n", " out_metas_path = os.path.join(OUT_DATA_DIR, f\"meta_{dset_type}.jsonl\")\n", " out_info_filepath = os.path.join(OUT_DATA_DIR, f\"info_{dset_type}.json\")\n", "\n", " # gather the data\n", " _ = np.memmap(out_mmap_path, dtype=np.uint16, mode=\"w+\", shape=(1,))\n", " n_offs = 0\n", " tot_duration_dict = defaultdict(int)\n", " datasets_info = defaultdict(dict)\n", " n = 0\n", " for i, row in tqdm.tqdm(df.iterrows(), total=len(df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " # make mmap -- two different paths\n", " local_path = (\n", " f\"{NPZ_DIR}/{row['s3_id']}.npz\"\n", " if row[\"is_7b\"]\n", " else f\"/app/suno/data/dpo/npz/{row['s3_id']}.npz\"\n", " )\n", " if not os.path.exists(local_path):\n", " # print(row, local_path)\n", " raise ValueError()\n", " # print(local_path)\n", " # print( np.load(local_path))\n", " try:\n", " arr = (\n", " np.load(local_path)[\"v3.0_raw\"]\n", " if row[\"is_7b\"]\n", " else np.load(local_path)[\"v1_raw\"]\n", " )\n", " except Exception as e:\n", " print(local_path)\n", " raise e\n", " assert arr.shape[0] <= 3000\n", " assert arr.shape[1] == 9\n", " arr_duration = arr.shape[0] / 25\n", " # print(arr.shape)\n", " try:\n", " arr = reshift(arr)\n", " except Exception as e:\n", " print(local_path)\n", " BAD_IDs.add(os.path.basename(local_path).replace(\".npz\", \"\"))\n", " continue\n", " # print(\"after shift and pad\", arr.shape)\n", " arr = arr.reshape(\n", " -1,\n", " )\n", " # print(arr.shape)\n", " out_mm = np.memmap(\n", " out_mmap_path,\n", " dtype=np.uint16,\n", " mode=\"r+\",\n", " shape=(n_offs + arr.size,),\n", " )\n", " out_mm[n_offs : n_offs + arr.size] = arr\n", " # print(f\"offset is: {n_offs}\")\n", " # break\n", " # write it once\n", " out_mm.flush()\n", " del out_mm\n", " add_metas = []\n", " add_meta = {\n", " \"dataset\": f\"perference_{int(row['preference'])}\",\n", " \"id\": row[\"s3_id\"], # this is the row s3_id\n", " \"start_s\": 0, # row[\"total_start_s\"] if row[\"total_start_s\"] else 0,\n", " \"end_s\": None,\n", " # (\n", " # None if not row[\"total_clip_s\"] else row[\"total_clip_s\"]\n", " # ), # for full clips we do know it has an edding, other wise, we don't know\n", " \"original_duration_s\": arr_duration,\n", " \"vocal_start_s\": None, # these are unfortunately missing for now\n", " \"vocal_end_s\": None, # these are unfortunately missing for now\n", " \"tags\": [\n", " row[\"tags\"] if row[\"tags\"] else \"\"\n", " ], # tags is a list, do you know :)\n", " \"text\": row[\"prompt\"] if row[\"prompt\"] else \"\",\n", " }\n", " add_metas.append(add_meta)\n", " tot_duration_dict[row[\"preference\"]] += arr_duration\n", " write_jsonl(\n", " add_metas,\n", " os.path.join(out_metas_path),\n", " do_append=bool(n_offs != 0),\n", " )\n", " if \"idx_list\" not in datasets_info[add_meta[\"dataset\"]]:\n", " datasets_info[add_meta[\"dataset\"]][\"idx_list\"] = [n]\n", " else:\n", " datasets_info[add_meta[\"dataset\"]][\"idx_list\"].append(n)\n", " n += 1\n", " n_offs += arr.size\n", "\n", " write_json(datasets_info, out_info_filepath)\n", " print(f\"Total {n} clips\")\n", " for k, v in tot_duration_dict.items():\n", " print(f\"{round(v / 60 / 60):,} hours of {k}\")\n", " print(f\"Done\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": 50, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T03:50:56.091331Z", "start_time": "2024-03-04T03:50:56.089378Z" } }, "outputs": [], "source": [ "# val_df[[\"request_id\", \"metadata\", \"updated_at\", \"user_id\", \"preference\"]].head()" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T03:51:06.563505Z", "start_time": "2024-03-04T03:50:56.669344Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 220304/220304 [00:09<00:00, 22274.68it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "3,820 hours of 220304 clips, 9.179333333333334 nodes\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "total_duration = 0\n", "for i, row in tqdm.tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " except:\n", " print(i, row)\n", " total_duration += row[\"duration\"]\n", "print(f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1500} nodes\")" ] }, { "cell_type": "code", "execution_count": 52, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T03:51:31.671072Z", "start_time": "2024-03-04T03:51:06.564929Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 2226/2226 [00:24<00:00, 89.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 2226 clips\n", "19 hours of False\n", "19 hours of True\n", "Done\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "make_dataset(val_df, is_val=True)" ] }, { "cell_type": "code", "execution_count": 53, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T04:33:21.422613Z", "start_time": "2024-03-04T03:51:31.672485Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 220304/220304 [41:38<00:00, 88.19it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 220304 clips\n", "1,911 hours of False\n", "1,909 hours of True\n", "Done\n" ] } ], "source": [ "make_dataset(train_df, is_val=False)" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:38:38.771989Z", "start_time": "2024-03-04T01:38:38.771979Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, 3008, 9)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000\n", "assert mm[:100, :, 1:].min() >= 0\n", "assert mm[:100, :, 1:].max() <= 4096" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:38:38.772780Z", "start_time": "2024-03-04T01:38:38.772771Z" } }, "outputs": [], "source": [ "# randomly listen to some stuff\n", "from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "from suno_utils.tasks.dac_2c_12cb import (\n", " encode as codec_encode,\n", " decode_stream_to_full_audio as codec_decode,\n", " EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", ")\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\"\n", "_ = preload_codec_models(\"/app/suno/tony/v3/dac_2c_25x12.pt\", device=\"cuda\")\n", "assert len(test_metas) == len(mm)\n", "idx_list = list(range(len(test_metas)))\n", "# random.shuffle(idx_list)\n", "# idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "print(len(mm))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:38:38.773642Z", "start_time": "2024-03-04T01:38:38.773634Z" } }, "outputs": [], "source": [ "idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "assert \"original_duration_s\" in test_metas[idx]\n", "# positive index should be shifted by 1\n", "pos_idx = idx + 1\n", "print(\n", " \"tags:\",\n", " test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", " test_metas[idx].get(\"tags\"),\n", ")\n", "arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "if len(pad_idx_arr) > 0:\n", " arr = arr[: pad_idx_arr[0], :]\n", "pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "if len(pos_pad_idx_arr) > 0:\n", " pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "a = codec_decode(arr)\n", "print(\"negative example\")\n", "a.play(compress=False)\n", "pos_a = codec_decode(pos_arr)\n", "print(\"positive example\")\n", "pos_a.play(compress=False)\n", "print(\n", " \"text:\",\n", " test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", " test_metas[idx].get(\"text\"),\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:38:38.774377Z", "start_time": "2024-03-04T01:38:38.774368Z" } }, "outputs": [], "source": [ "from collections import Counter\n", "c = Counter()\n", "for _, row in df_slice.iterrows():\n", " # print(row[\"metadata\"])\n", " for k in ast.literal_eval(row[\"metadata\"]).keys():\n", " c[k] += 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:38:38.775129Z", "start_time": "2024-03-04T01:38:38.775121Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:38:38.775808Z", "start_time": "2024-03-04T01:38:38.775799Z" } }, "outputs": [], "source": [ "total_bad = 0\n", "total_good = 0\n", "for idx in range(len(test_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if test_metas[idx].get(\"tags\") != test_metas[pos_idx].get(\"tags\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", "print(total_good, total_bad)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:38:38.776349Z", "start_time": "2024-03-04T01:38:38.776341Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:38:38.777186Z", "start_time": "2024-03-04T01:38:38.777177Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:38:38.777769Z", "start_time": "2024-03-04T01:38:38.777761Z" } }, "outputs": [], "source": [ "total_iters = (len(n_neg_tr) + len(n_pos_tr))\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-03-04T01:38:38.778445Z", "start_time": "2024-03-04T01:38:38.778437Z" } }, "outputs": [], "source": [ "print(\"1 epoch per batch 4, total\", total_iters / 8 / 4 / 2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 2 }