{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T10:36:16.214454Z", "iopub.status.busy": "2025-07-01T10:36:16.214332Z", "iopub.status.idle": "2025-07-01T10:36:16.227399Z", "shell.execute_reply": "2025-07-01T10:36:16.226988Z", "shell.execute_reply.started": "2025-07-01T10:36:16.214436Z" } }, "outputs": [], "source": [ "# setup autoload\n", "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.040680Z", "start_time": "2024-05-16T13:58:19.777010Z" }, "execution": { "iopub.execute_input": "2025-07-01T10:36:16.228331Z", "iopub.status.busy": "2025-07-01T10:36:16.228188Z", "iopub.status.idle": "2025-07-01T10:36:18.632871Z", "shell.execute_reply": "2025-07-01T10:36:18.632332Z", "shell.execute_reply.started": "2025-07-01T10:36:16.228316Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The autoreload extension is already loaded. To reload it, use:\n", " %reload_ext autoreload\n" ] } ], "source": [ "import ast\n", "import os\n", "import shutil\n", "import sys\n", "from collections import defaultdict\n", "\n", "import json\n", "import numpy as np\n", "import pandas as pd\n", "from preference_data_preparation_4min_30b_task import *\n", "from preference_helper import *\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n", "from tqdm import tqdm\n", "\n", "pd.set_option(\"display.max_rows\", 500)\n", "pd.set_option(\"display.max_columns\", 500)\n", "pd.set_option(\"display.width\", 1000)\n", "\n", "# setup autoload\n", "%load_ext autoreload\n", "%autoreload 2\n", "\n", "\n", "def custom_parse(x):\n", " try:\n", " return json.loads(x)\n", " except:\n", " return {}" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.082172Z", "start_time": "2024-05-16T13:58:21.041926Z" }, "execution": { "iopub.execute_input": "2025-07-01T10:36:18.633719Z", "iopub.status.busy": "2025-07-01T10:36:18.633428Z", "iopub.status.idle": "2025-07-01T10:36:18.719934Z", "shell.execute_reply": "2025-07-01T10:36:18.719480Z", "shell.execute_reply.started": "2025-07-01T10:36:18.633704Z" } }, "outputs": [], "source": [ "OUT_DATA_DIR = \"/app2/suno/data/dpo/bluejay_mix_t2\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "shutil.copyfile(\n", " \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", " os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", ")\n", "NPZ_DIR = \"/app/suno/data/dpo/30b_npz\"\n", "# NOTE FOR 30b we increase this from 6016 up\n", "N_TOKENS_AUDIO = 6016" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:53.962528Z", "start_time": "2024-05-16T13:58:21.105919Z" }, "execution": { "iopub.execute_input": "2025-06-03T18:43:23.246328Z", "iopub.status.busy": "2025-06-03T18:43:23.246152Z", "iopub.status.idle": "2025-06-03T18:43:37.586617Z", "shell.execute_reply": "2025-06-03T18:43:37.586255Z", "shell.execute_reply.started": "2025-06-03T18:43:23.246316Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Preference data shape (931852, 90)\n" ] } ], "source": [ "df = pd.read_pickle(\n", " \"/home/tony/Data/Preference/30b_v6/interesting_clips_v4_h_t_6_20250501_full_long.pkl\"\n", ")\n", "print(\"Preference data shape\", df.shape)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:43:37.587228Z", "iopub.status.busy": "2025-06-03T18:43:37.587012Z", "iopub.status.idle": "2025-06-03T18:43:39.597769Z", "shell.execute_reply": "2025-06-03T18:43:39.597399Z", "shell.execute_reply.started": "2025-06-03T18:43:37.587215Z" } }, "outputs": [], "source": [ "# df[\"created_at\"] = pd.to_datetime(df[\"created_at\"], utc=True)\n", "# cutoff_date = pd.to_datetime(\"2025-04-12\", utc=True)\n", "# # cutoff_date = pd.to_datetime(\"2025-04-17\", utc=True)\n", "# print(df.shape, df[df[\"created_at\"] >= cutoff_date].shape)\n", "# df = df[(df[\"created_at\"] >= cutoff_date)].copy()\n", "# print(\"after date cut\", df.shape)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:43:39.602984Z", "iopub.status.busy": "2025-06-03T18:43:39.602745Z", "iopub.status.idle": "2025-06-03T18:43:40.429557Z", "shell.execute_reply": "2025-06-03T18:43:40.429193Z", "shell.execute_reply.started": "2025-06-03T18:43:39.602970Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "after dropna (931852, 85)\n" ] } ], "source": [ "df = df.dropna(axis=1, how=\"all\")\n", "print(\"after dropna\", df.shape)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.199480Z", "start_time": "2024-05-16T13:58:53.963687Z" }, "execution": { "iopub.execute_input": "2025-06-03T18:43:40.430543Z", "iopub.status.busy": "2025-06-03T18:43:40.430301Z", "iopub.status.idle": "2025-06-03T18:49:02.664583Z", "shell.execute_reply": "2025-06-03T18:49:02.664215Z", "shell.execute_reply.started": "2025-06-03T18:43:40.430530Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "6442462\n", "6442462\n", "pre-downloaded df (931852, 85)\n", "downloaded df (931851, 85)\n" ] } ], "source": [ "converted_paths = os.listdir(NPZ_DIR)\n", "print(len(converted_paths))\n", "\n", "converted_paths = set([f.replace(\".npz\", \"\") for f in converted_paths])\n", "print(len(converted_paths))\n", "\n", "print(\"pre-downloaded df\", df.shape)\n", "df[df[\"s3_id\"].isin(converted_paths)].shape\n", "df = df[df[\"s3_id\"].isin(converted_paths)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.467253Z", "start_time": "2024-05-16T13:58:56.207647Z" }, "execution": { "iopub.execute_input": "2025-06-03T18:49:02.665136Z", "iopub.status.busy": "2025-06-03T18:49:02.664973Z", "iopub.status.idle": "2025-06-03T18:49:03.822609Z", "shell.execute_reply": "2025-06-03T18:49:03.822260Z", "shell.execute_reply.started": "2025-06-03T18:49:02.665123Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "is_30b\n", "True 931851\n", "Name: count, dtype: int64\n" ] }, { "data": { "text/plain": [ "task\n", "cover 502287\n", "artist_consistency 377994\n", "infill 51546\n", " 20\n", "artist_extend 4\n", "Name: count, dtype: int64" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"is_30b\"] = df[\"model_name\"].str.contains(\"-t\")\n", "print(df[\"is_30b\"].value_counts())\n", "df[\"task\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:49:03.823240Z", "iopub.status.busy": "2025-06-03T18:49:03.823001Z", "iopub.status.idle": "2025-06-03T18:49:03.836953Z", "shell.execute_reply": "2025-06-03T18:49:03.836653Z", "shell.execute_reply.started": "2025-06-03T18:49:03.823228Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(931851, 86)\n" ] } ], "source": [ "# drop extend for now\n", "# reason is -- cause they are likely caused by extend from 13b\n", "# we don't want contamination\n", "# df = df[df[\"task\"] != \"extend\"].copy()\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.592883Z", "start_time": "2024-05-16T13:58:56.470781Z" }, "execution": { "iopub.execute_input": "2025-06-03T18:49:03.837446Z", "iopub.status.busy": "2025-06-03T18:49:03.837271Z", "iopub.status.idle": "2025-06-03T18:49:03.996198Z", "shell.execute_reply": "2025-06-03T18:49:03.995838Z", "shell.execute_reply.started": "2025-06-03T18:49:03.837434Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "preference model_name \n", "False chirp-v4-h-t-6 465926\n", "True chirp-v4-h-t-6 465925\n", "Name: count, dtype: int64\n", "before filter on model name (931851, 86)\n", "after filter on model name (931851, 86)\n" ] } ], "source": [ "## for 13b this is easy for now\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "print(\"before filter on model name\", df.shape)\n", "df = df[df[\"model_name\"].isin([\"chirp-v4-h-t-6\", \"chirp-v3p5-engine-t-6\"])]\n", "# df = df[df[\"model_name\"].isin([\"chirp-v3p5-engine-t-6\"])]\n", "print(\"after filter on model name\", df.shape)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:49:03.996770Z", "iopub.status.busy": "2025-06-03T18:49:03.996573Z", "iopub.status.idle": "2025-06-03T18:49:04.012270Z", "shell.execute_reply": "2025-06-03T18:49:04.011995Z", "shell.execute_reply.started": "2025-06-03T18:49:03.996758Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "is_public\n", "False 908417\n", "True 23434\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(df[\"is_public\"].value_counts())\n", "# remove public for now cause fucking users\n", "# df = df[~df[\"is_public\"]]" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.909539Z", "start_time": "2024-05-16T13:58:56.595736Z" }, "execution": { "iopub.execute_input": "2025-06-03T18:49:04.012734Z", "iopub.status.busy": "2025-06-03T18:49:04.012579Z", "iopub.status.idle": "2025-06-03T18:49:04.624021Z", "shell.execute_reply": "2025-06-03T18:49:04.623628Z", "shell.execute_reply.started": "2025-06-03T18:49:04.012723Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "before filter on request id pairs (931851, 86)\n", "after filter on request id pairs (931850, 86)\n", "preference model_name \n", "False chirp-v4-h-t-6 465925\n", "True chirp-v4-h-t-6 465925\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(\"before filter on request id pairs\", df.shape)\n", "df = df[\n", " df[\"request_id\"].isin(\n", " df[\"request_id\"].value_counts().index[df[\"request_id\"].value_counts() == 2]\n", " )\n", "]\n", "print(\"after filter on request id pairs\", df.shape)\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "assert df.shape[0] == df[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:49:04.625796Z", "iopub.status.busy": "2025-06-03T18:49:04.625607Z", "iopub.status.idle": "2025-06-03T18:50:15.511358Z", "shell.execute_reply": "2025-06-03T18:50:15.510996Z", "shell.execute_reply.started": "2025-06-03T18:49:04.625783Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "unique_requests 465925\n", "before removing duplicates (931850, 165)\n", "after removing duplicates (931850, 158)\n" ] } ], "source": [ "# Let's use the old selection for now -- for quality assurance\n", "# expand the metadata columns -- this takes forever...~ 6 mins\n", "# test_slice = df[\"metadata\"].apply(lambda x: ast.literal_eval(str(x)))\n", "# test_slice = df[\"metadata\"].apply(lambda x: custom_parse(x))\n", "test_slice = df[\"metadata\"] # .apply(lambda x: custom_parse(x))\n", "test_slice_series = test_slice.apply(pd.Series)\n", "df = pd.concat([df, test_slice_series], axis=1, join=\"inner\")\n", "print(\"unique_requests\", df[\"request_id\"].nunique())\n", "# remove the duplicates\n", "print(\"before removing duplicates\", df.shape)\n", "df = df.loc[:, ~df.columns.duplicated()].copy()\n", "print(\"after removing duplicates\", df.shape)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:15.512060Z", "iopub.status.busy": "2025-06-03T18:50:15.511749Z", "iopub.status.idle": "2025-06-03T18:50:16.319307Z", "shell.execute_reply": "2025-06-03T18:50:16.318970Z", "shell.execute_reply.started": "2025-06-03T18:50:15.512047Z" } }, "outputs": [ { "data": { "text/plain": [ "task\n", "cover 502286\n", "artist_consistency 377994\n", "infill 51546\n", " 20\n", "artist_extend 4\n", "Name: count, dtype: int64" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"task\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:16.319885Z", "iopub.status.busy": "2025-06-03T18:50:16.319685Z", "iopub.status.idle": "2025-06-03T18:50:17.187823Z", "shell.execute_reply": "2025-06-03T18:50:17.187497Z", "shell.execute_reply.started": "2025-06-03T18:50:16.319873Z" } }, "outputs": [ { "data": { "text/plain": [ "pos_diff_preference\n", "1.0 324077\n", "2.0 141848\n", "Name: count, dtype: int64" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df.sort_values(by=[\"request_id\", \"preference\", \"diff_preference\"])\n", "df[\"pos_diff_preference\"] = df[\"diff_preference\"].diff()\n", "# df[\"cer_diff_preference\"] = df[\"cer\"].diff()\n", "df[df[\"preference\"]][\"pos_diff_preference\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:17.188415Z", "iopub.status.busy": "2025-06-03T18:50:17.188181Z", "iopub.status.idle": "2025-06-03T18:50:17.489079Z", "shell.execute_reply": "2025-06-03T18:50:17.488721Z", "shell.execute_reply.started": "2025-06-03T18:50:17.188402Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive param_experiment\n", "text_1 11556\n", "temp_s_80 11381\n", "temp_s_70 10863\n", "min_p_0001 363\n", "min_p_002 354\n", "step_8 191\n", "text_3 170\n", "step_12 168\n", "Name: count, dtype: int64\n" ] } ], "source": [ "try:\n", " print(\"positive\", df[df[\"preference\"]][\"param_experiment\"].value_counts())\n", "except:\n", " pass" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:17.489680Z", "iopub.status.busy": "2025-06-03T18:50:17.489479Z", "iopub.status.idle": "2025-06-03T18:50:30.989567Z", "shell.execute_reply": "2025-06-03T18:50:30.989209Z", "shell.execute_reply.started": "2025-06-03T18:50:17.489667Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found 79903 duplicated prompts 39952 unique requests\n", "Found 19464 request_ids with duplicate prompts but not highest play counts in their group\n", "['c8390ab2-e5be-4d69-b5bf-4e9404088949', 'e12dafc1-45e3-4abd-9a58-d5e4293ae33b', '6f5b4451-853e-4196-818c-345d9fa0f684', '69e8eef7-4326-4796-b9f7-833143e3afde', 'b75e653f-eb32-409e-a5ce-1ef39f1fa1c1', 'fa5aba0d-4424-438b-9f87-ba5d0ef503cb', 'e3959990-3e05-4c7b-9792-8d57b0290b99', '115a0270-2735-43a2-80d4-05b371926471', '14e7fadb-f34c-4f26-9bda-c378283f529a', '47de7b19-69c3-4a23-967f-658950a9df5a']\n", "Before dedup user gen requests 931850\n", "After dedup user gen requests 931850\n" ] } ], "source": [ "# Find duplicated prompts with count > 2\n", "duplicate_entries = df.groupby(\n", " [\"user_id\", \"prompt_text\", \"tags\", \"task\", \"edited_clip_id\"]\n", ").filter(lambda x: len(x) > 2)\n", "print(\n", " \"Found\",\n", " len(duplicate_entries),\n", " \"duplicated prompts\",\n", " len(duplicate_entries[\"request_id\"].unique()),\n", " \"unique requests\",\n", ")\n", "\n", "# Group by user_id, prompt_text, and tags to find duplicate prompt groups\n", "prompt_groups = duplicate_entries.groupby(\n", " [\"user_id\", \"prompt_text\", \"tags\", \"task\", \"edited_clip_id\"]\n", ")\n", "\n", "# For each prompt group, find the request_id with the highest total reaction_play_count\n", "low_play_count_request_ids = []\n", "for prompt_key, prompt_group in prompt_groups:\n", " # Get the sum of reaction_play_count for each request_id in this group\n", " request_play_counts = prompt_group.groupby(\"request_id\")[\n", " \"reaction_play_count\"\n", " ].sum()\n", "\n", " # Find the max play count in this group\n", " max_play_count = request_play_counts.max()\n", "\n", " # Add request_ids that don't have the max play count to our filter list\n", " lower_play_count_request_ids = request_play_counts[\n", " request_play_counts < max_play_count\n", " ].index.tolist()\n", " low_play_count_request_ids.extend(lower_play_count_request_ids)\n", "\n", "# Display the filtered request IDs\n", "print(\n", " f\"Found {len(low_play_count_request_ids)} request_ids with duplicate prompts but not highest play counts in their group\"\n", ")\n", "print(\n", " low_play_count_request_ids[:10]\n", " if len(low_play_count_request_ids) > 10\n", " else low_play_count_request_ids\n", ")\n", "print(\"Before dedup user gen requests\", df.shape[0])\n", "# df = df[~df[\"request_id\"].isin(low_play_count_request_ids)]\n", "print(\"After dedup user gen requests\", df.shape[0])" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:40.799375Z", "start_time": "2024-05-16T13:59:36.394236Z" }, "execution": { "iopub.execute_input": "2025-06-03T18:50:30.990093Z", "iopub.status.busy": "2025-06-03T18:50:30.989949Z", "iopub.status.idle": "2025-06-03T18:50:33.107667Z", "shell.execute_reply": "2025-06-03T18:50:33.107316Z", "shell.execute_reply.started": "2025-06-03T18:50:30.990080Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "22935\n", "good_continue_at\n", "True 931850\n", "Name: count, dtype: int64\n", "\n", " Check some basics... \n", " preference\n", "False 465925\n", "True 465925\n", "Name: count, dtype: int64 is_30b\n", "True 931850\n", "Name: count, dtype: int64 model_name\n", "chirp-v4-h-t-6 931850\n", "Name: count, dtype: int64 preference model_name \n", "False chirp-v4-h-t-6 465925\n", "True chirp-v4-h-t-6 465925\n", "Name: count, dtype: int64\n", "task\n", "cover 502286\n", "artist_consistency 377994\n", "infill 51546\n", " 20\n", "artist_extend 4\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df[\"id\"] = df[\"str_id\"]\n", "# get the original duration of the clips, if they are concacted\n", "df[\"original_duration_s\"] = df[\"total_start_s\"] + df[\"duration\"]\n", "# classify the continue at behavoirs by the duration choice\n", "audio_prompt_id_to_continue_at = {}\n", "\n", "for _, row in df[~df[\"continued_parent\"].isna()].iterrows():\n", " audio_prompt_id = row[\"continued_parent\"]\n", " if audio_prompt_id not in audio_prompt_id_to_continue_at:\n", " audio_prompt_id_to_continue_at[audio_prompt_id] = row[\"continue_at\"]\n", " else:\n", " # pick the max\n", " audio_prompt_id = max(\n", " audio_prompt_id_to_continue_at[audio_prompt_id], row[\"continue_at\"]\n", " )\n", "print(len(audio_prompt_id_to_continue_at))\n", "df[\"has_continue_and_start_continue_at\"] = df[\"id\"].apply(\n", " lambda x: audio_prompt_id_to_continue_at.get(x)\n", ")\n", "# we want continue at to be at most of the clip...\n", "df[\"good_continue_at\"] = (\n", " (df[\"has_continue_and_start_continue_at\"] / df[\"duration\"]) > 0.9\n", ") | df[\"has_continue_and_start_continue_at\"].isna()\n", "print(df[\"good_continue_at\"].value_counts())\n", "\n", "\n", "print(\n", " \"\\n Check some basics... \\n\",\n", " df[\"preference\"].value_counts(),\n", " df[\"is_30b\"].value_counts(),\n", " df[\"model_name\"].value_counts(),\n", " df.groupby([\"preference\"])[\"model_name\"].value_counts(),\n", ")\n", "\n", "df = df.sort_values(by=[\"request_id\", \"preference\"])\n", "df[\"duration_rel_diff\"] = df[\"duration\"].diff()\n", "df[\"play_rel_diff\"] = df[\"reaction_play_count\"].diff()\n", "print(df[\"task\"].value_counts())\n", "\n", "df[\"post_infill_duration\"] = (\n", " df[\"duration\"]\n", " + df[\"infill_context_end_s\"]\n", " - df[\"infill_context_start_s\"]\n", " - df[\"include_future_s\"]\n", " - df[\"include_history_s\"]\n", " - df[\"infill_dur_s\"]\n", ")" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:33.108248Z", "iopub.status.busy": "2025-06-03T18:50:33.108048Z", "iopub.status.idle": "2025-06-03T18:50:33.276257Z", "shell.execute_reply": "2025-06-03T18:50:33.275933Z", "shell.execute_reply.started": "2025-06-03T18:50:33.108235Z" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvEUlEQVR4nO3df3TUVX7/8dd88gMSkkBIgsjCKiUmoBES1JNNHJbVyrpF3XMi7UKLq1iO4CrCUVagFIUgmwQED6J2oUpRFJq1grr+oFvdisUTEGrJslEEIq7GZm0yQzAhgfyaz/cPvhkzMZBMfszMzTwf5+SczGfuzOe+547yyr2fHw7btm0BAAAYxAp2BwAAAPxFgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxokMdgf6m9tdp0DdLMHhkJKS4gO6z2ALx5ol6qbugS8ca5aoOxTqbutLVwZ8gLFtBXwwgrHPYAvHmiXqDjfhWHc41ixRtwlYQgIAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgnAF/N2qYxbIcsiyH97HHY8vjMeTWqACAgCHAIGRYlkOJibGyrG8nBj0ej2pqGggxAAAfBBiEjPOzL5berjihU41nNXxQjKaPuUKW5SDAAAB8EGAQck41nlXVuYZgdwMAEMI4iBcAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4fgWYp556Sunp6T4/P/nJT7zPNzY2Kj8/X9nZ2crKytIDDzwgl8vl8x6VlZWaN2+eJk2apJycHK1du1YtLS0+bT788EPl5eUpIyND06ZN0+7du3tRIgAAGGgi/X3BFVdcoW3btnkfR0REeH8vKCjQ+++/r40bNyo+Pl6PPfaYFixYoOLiYklSa2ur5s+fr+TkZBUXF6uqqkpLly5VVFSUHnroIUlSRUWF5s+fr1mzZmn9+vXav3+/VqxYoZSUFE2ZMqW39QIAgAHA7wATERGhlJSU72yvq6vTrl27tH79euXk5Eg6H2imT5+u0tJSZWZm6oMPPlB5ebm2bdum5ORkTZgwQYsWLdL69eu1YMECRUdHq7i4WKNHj9ayZcskSePGjdNHH32k559/ngADAAAk9SDAfPHFF3I6nRo0aJAyMzO1ePFijRo1SmVlZWpublZubq637bhx4zRq1ChvgCktLVVaWpqSk5O9bZxOp1atWqXy8nJdeeWVKi0t9Qag9m0KCgp6VKDD0aOX9WpfgdxnsAWq5lD7TMNxrCXqDqe6w7FmibpDoe7u9sGvADNx4kQVFhZq7Nixqq6u1jPPPKPZs2frjTfekMvlUlRUlBISEnxek5SUpOrqakmSy+XyCS+SvI+7anPmzBmdO3dOgwcP9qfLSkqK96t9XwjGPoOtP2tOTBzSb+/dW+E41hJ1h5NwrFmibhP4FWCmTp3q/X38+PGaNGmSbrjhBu3Zs8fvYBEobnedbDsw+3I4zg9+IPcZbH1Zc0SE1WlYqampV2urp3dv3sfCcawl6g6nusOxZom6Q6Hutr50xe8lpPYSEhJ0+eWX68svv1Rubq6am5tVW1vrMwvjdru9x8wkJyfryJEjPu/RdpZS+zYdz1xyuVyKi4vrUUiybQV8MIKxz2Dr75pD9fMMx7GWqDuchGPNEnWboFfXgamvr1dFRYVSUlKUkZGhqKgo7d+/3/v8yZMnVVlZqczMTElSZmamjh8/Lrfb7W1TUlKiuLg4paametscOHDAZz8lJSXe9wAAAPArwKxdu1YHDx7UV199pf/5n//RggULZFmWbr31VsXHx2vGjBkqKirSgQMHVFZWpuXLlysrK8sbPpxOp1JTU7VkyRJ9+umn2rdvnzZu3KjZs2crOjpakjRr1ixVVFRo3bp1+uyzz7Rjxw7t2bNHc+bM6evaAQCAofxaQvr666/10EMP6fTp0xo+fLiuueYavfzyyxo+fLgkafny5bIsSwsXLlRTU5OcTqdWrlzpfX1ERIQ2b96sVatWaebMmYqJiVFeXp4WLlzobTNmzBht2bJFhYWF2r59u0aOHKk1a9ZwCjUAAPBy2LYpq10943IF9iDe5OT4gO4z2Pqy5sjI8wfxvlR+RFXnGjRicKzuSJ2ompp6tbSE3kG84TbWEnWHU93hWLNE3aFQd1tfusK9kAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGKdXAeaf//mflZ6erl/96lfebY2NjcrPz1d2draysrL0wAMPyOVy+byusrJS8+bN06RJk5STk6O1a9eqpaXFp82HH36ovLw8ZWRkaNq0adq9e3dvugoAAAaQHgeYI0eOqLi4WOnp6T7bCwoK9N5772njxo168cUXVVVVpQULFnifb21t1fz589Xc3Kzi4mIVFRXp1Vdf1aZNm7xtKioqNH/+fGVnZ+v111/XXXfdpRUrVmjfvn097S4AABhAehRg6uvr9fDDD2vNmjUaOnSod3tdXZ127dqlZcuWKScnRxkZGSooKNDhw4dVWloqSfrggw9UXl6uxx9/XBMmTNDUqVO1aNEi7dixQ01NTZKk4uJijR49WsuWLdO4ceN0xx136Oabb9bzzz/f64IBAID5ehRgVq9eralTpyo3N9dne1lZmZqbm322jxs3TqNGjfIGmNLSUqWlpSk5Odnbxul06syZMyovL/e2ycnJ8Xlvp9PpfQ8AABDeIv19wVtvvaVPPvlEr7zyyneec7lcioqKUkJCgs/2pKQkVVdXe9u0Dy+SvI+7anPmzBmdO3dOgwcP7nZ/HY5uN+21tn0Fcp/BFqiaQ+0zDcexlqg7nOoOx5ol6g6FurvbB78CzJ///Gf96le/0r/8y79o0KBBPelXwCUlxYfFPoOtP2tOTBzSb+/dW+E41hJ1h5NwrFmibhP4FWA+/vhjud1u3X777d5tra2tOnTokHbs2KGtW7equblZtbW1PrMwbrdbKSkpks7PpBw5csTnfdvOUmrfpuOZSy6XS3FxcX7Nvpzfd51s26+X9JjDcX7wA7nPYOvLmiMirE7DSk1NvVpbPb178z4WjmMtUXc41R2ONUvUHQp1t/WlK34FmB/84Ad64403fLb9wz/8g/7iL/5C99xzjy699FJFRUVp//79uvnmmyVJJ0+eVGVlpTIzMyVJmZmZ2rx5s9xut5KSkiRJJSUliouLU2pqqrfNf/3Xf/nsp6SkxPse/rBtBXwwgrHPYOvvmkP18wzHsZaoO5yEY80SdZvArwATFxentLQ0n22xsbEaNmyYd/uMGTNUVFSkoUOHKi4uTmvWrFFWVpY3fDidTqWmpmrJkiV6+OGHVV1drY0bN2r27NmKjo6WJM2aNUs7duzQunXrNGPGDB04cEB79uzRli1b+qBkAABgOr8P4u3K8uXLZVmWFi5cqKamJjmdTq1cudL7fEREhDZv3qxVq1Zp5syZiomJUV5enhYuXOhtM2bMGG3ZskWFhYXavn27Ro4cqTVr1mjKlCl93V0AAGAgh22bMlnUMy5XYI+BSU6OD+g+g60va46MPH8MzEvlR1R1rkEjBsfqjtSJqqmpV0tL6B0DE25jLVF3ONUdjjVL1B0Kdbf1pSvcCwkAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcSKD3QGgKxERvjnb47Hl8YTRbWIBAN9BgEHIio2Mkse2lZAQ47Pd4/GopqaBEAMAYYwAg5A1OCJSlsOhtytO6FTjWUnS8EExmj7mClmWgwADAGGMAIOQd6rxrKrONQS7GwCAEMJBvAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMaJDHYHEPosyyHLcngfezy2PB47iD0CAIQ7AgwuyrIcSkyMlWV9O1nn8XhUU9NAiAEABA0BBhd1fvbF0tsVJ3Sq8ayGD4rR9DFXyLIcBBgAQNAQYNAtpxrPqupcQ5++Z8elqYgIDskCAHQPAQZB0dnSFAAA3UWAQVB0XJqSpMvjhsk58vtB7hkAwAQEGARV+6Wp4YNigtwbAIApmL8HAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIzjV4DZuXOnbrvtNk2ePFmTJ0/WzJkz9f7773ufb2xsVH5+vrKzs5WVlaUHHnhALpfL5z0qKys1b948TZo0STk5OVq7dq1aWlp82nz44YfKy8tTRkaGpk2bpt27d/eiRAAAMND4FWBGjhypX/7yl9q9e7d27dqlH/zgB7r//vt14sQJSVJBQYHee+89bdy4US+++KKqqqq0YMEC7+tbW1s1f/58NTc3q7i4WEVFRXr11Ve1adMmb5uKigrNnz9f2dnZev3113XXXXdpxYoV2rdvXx+VDAAATOdXgLnxxhs1depUXX755Ro7dqwefPBBxcbGqrS0VHV1ddq1a5eWLVumnJwcZWRkqKCgQIcPH1Zpaakk6YMPPlB5ebkef/xxTZgwQVOnTtWiRYu0Y8cONTU1SZKKi4s1evRoLVu2TOPGjdMdd9yhm2++Wc8//3xf1w4AAAzV43shtba26t///d/V0NCgrKwslZWVqbm5Wbm5ud4248aN06hRo1RaWqrMzEyVlpYqLS1NycnJ3jZOp1OrVq1SeXm5rrzySpWWlionJ8dnX06nUwUFBT3qp8PRs/p6s69A7jOYHI7g1hzMzzncxroNdQe3H4EUjjVL1B0KdXe3D34HmGPHjmnWrFlqbGxUbGysnnnmGaWmpuro0aOKiopSQkKCT/ukpCRVV1dLklwul094keR93FWbM2fO6Ny5cxo8eLBf/U1KiverfV8Ixj4DLTFxiM/jQNfccf/BEg5j3RnqDh/hWLNE3SbwO8CMHTtWr732murq6vS73/1OS5cu1UsvvdQffesTbnedbDsw+3I4zg9+IPfZ3yIirE7DQk1NvVpbPT2u+ULv211t+w+WgTjW3UHd4VN3ONYsUXco1N3Wl674HWCio6N12WWXSZIyMjL0xz/+Udu3b9df/dVfqbm5WbW1tT6zMG63WykpKZLOz6QcOXLE5/3azlJq36bjmUsul0txcXF+z75Ikm0r4IMRjH0GQ/sag/U5B1u4jHVH1B0+wrFmibpN0OvrwHg8HjU1NSkjI0NRUVHav3+/97mTJ0+qsrJSmZmZkqTMzEwdP35cbrfb26akpERxcXFKTU31tjlw4IDPPkpKSrzvAQAA4FeA2bBhgw4dOqSvvvpKx44d04YNG3Tw4EHddtttio+P14wZM1RUVKQDBw6orKxMy5cvV1ZWljd8OJ1OpaamasmSJfr000+1b98+bdy4UbNnz1Z0dLQkadasWaqoqNC6dev02WefaceOHdqzZ4/mzJnT17UDAABD+bWE5Ha7tXTpUlVVVSk+Pl7p6enaunWrrr/+eknS8uXLZVmWFi5cqKamJjmdTq1cudL7+oiICG3evFmrVq3SzJkzFRMTo7y8PC1cuNDbZsyYMdqyZYsKCwu1fft2jRw5UmvWrNGUKVP6qGQAAGA6vwJMV6cyDxo0SCtXrvQJLR1973vf07PPPnvR98nOztZrr73mT9cAAEAY4V5IAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcSKD3QGgJyIifLO3x2PL47GD1BsAQKARYGCU2MgoeWxbCQkxPts9Ho9qahoIMQAQJggwMMrgiEhZDoferjihU41nJUnDB8Vo+pgrZFkOAgwAhAkCDIx0qvGsqs41BLsbAIAg4SBeAABgHAIMAAAwDktI6JGOZwFZlkOtrRx/AgAIDAIM/HKhs4CGDYvlLCAAQMAQYOAXzgICAIQCAgx6hLOAAADBxEG8AADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMaJDHYHMHBERHybhz0eWx6PHcTeAAAGMgIMei02Mkoe21ZCQox3m8fjUU1NAyEGANAvCDDotcERkbIcDr1dcUKnGs9q+KAYTR9zhSzLQYABAPQLAgx8WJZDluXwPm6/LNSVU41nVXWuoT+6hSDp+H0AgFBBgIGXZTmUmBgry+LYblz4++CxbVmWQ62tzK4BCB4CDLzO/7VteZeCJOnyuGFyjvx+kHuGYOjs+9C2POhwOCQRYAAEj19/am/ZskUzZsxQVlaWcnJydN999+nkyZM+bRobG5Wfn6/s7GxlZWXpgQcekMvl8mlTWVmpefPmadKkScrJydHatWvV0tLi0+bDDz9UXl6eMjIyNG3aNO3evbuHJcJfbUtBVecaVNvcGOzuIMjafx/aggwABJtfAebgwYOaPXu2Xn75ZW3btk0tLS2aO3euGhq+Pe6hoKBA7733njZu3KgXX3xRVVVVWrBggff51tZWzZ8/X83NzSouLlZRUZFeffVVbdq0ydumoqJC8+fPV3Z2tl5//XXdddddWrFihfbt29cHJSNQIiIsRUZ++8OxFACAvuLXEtLWrVt9HhcVFSknJ0cff/yxrrvuOtXV1WnXrl1av369cnJyJJ0PNNOnT1dpaakyMzP1wQcfqLy8XNu2bVNycrImTJigRYsWaf369VqwYIGio6NVXFys0aNHa9myZZKkcePG6aOPPtLzzz+vKVOm9FHp6C+dnVYtcWo1AKDv9OoYmLq6OknS0KFDJUllZWVqbm5Wbm6ut824ceM0atQob4ApLS1VWlqakpOTvW2cTqdWrVql8vJyXXnllSotLfUGoPZtCgoK/O6jI4B/9LftK5D7DEUdT6uWvj12IioqQq2tHr/ObvJHoD77cB9rhyO8ag/H8Q7HmiXqDoW6u9uHHgcYj8ejgoICTZ48WWlpaZIkl8ulqKgoJSQk+LRNSkpSdXW1t0378CLJ+7irNmfOnNG5c+c0ePDgbvczKSnev8L6QDD2GYran1Z9oVmZvpSYOKTf3vtCwnWshw0L/GcdCsJxvMOxZom6TdDjAJOfn68TJ05o586dfdmfPud218kO0IqFw3F+8AO5z74UEWH1WwjoOCvTH2c31dTUq7XV06fveSGmj3V3XOz7cPp0vVpaAvNZh4JwGO+OwrFmibpDoe62vnSlRwFm9erV2rt3r1566SWNHDnSuz05OVnNzc2qra31mYVxu91KSUnxtjly5IjP+7WdpdS+Tcczl1wul+Li4vyafZEk21bAByMY+zRF26zM8EH9MxPDWAcGdYePcKxZom4T+HUggm3bWr16td555x298MILGjNmjM/zGRkZioqK0v79+73bTp48qcrKSmVmZkqSMjMzdfz4cbndbm+bkpISxcXFKTU11dvmwIEDPu9dUlLifQ8AABDe/Aow+fn5+u1vf6sNGzZoyJAhqq6uVnV1tc6dOydJio+P14wZM1RUVKQDBw6orKxMy5cvV1ZWljd8OJ1OpaamasmSJfr000+1b98+bdy4UbNnz1Z0dLQkadasWaqoqNC6dev02WefaceOHdqzZ4/mzJnTp8UDAAAz+bWE9K//+q+SpJ///Oc+2wsLC3X77bdLkpYvXy7LsrRw4UI1NTXJ6XRq5cqV3rYRERHavHmzVq1apZkzZyomJkZ5eXlauHCht82YMWO0ZcsWFRYWavv27Ro5cqTWrFnDKdQAAECSnwHm2LFjXbYZNGiQVq5c6RNaOvre976nZ5999qLvk52drddee82f7gEAgDDBXfsAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABinVzdzBEJJ+xtEejw2d70GgAGMAAPjdXajSI/Ho5qaBkIMAAxQBJgwZlkOWda39y1vP4Nhko43ihw+KEbTx1why3IQYABggCLAhCnLcigxMVaWZWZo6UzbjSIBAAMfASZMnZ99sbyzFpJ0edwwOUd+P8g9AwCgawSYMNd+1mL4oJguWgMAEBoGzvoBAAAIGwQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAONEBrsDQH+JiPDN5x6PLY/HDlJvAAB9iQCDASc2Mkoe21ZCQozPdo/Ho5qaBkIMAAwABBgMOIMjImU5HHq74oRONZ6VJA0fFKPpY66QZTkIMAAwABBgMGCdajyrqnMNwe4GAKAfcBAvAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjcBYSAKNZlkOW5fDZxkULgYGPAAPAWJblUGJirCyr41WXuWghMNARYAAY6/zsi8VFC4EwRIABQhjLI93DRQuB8EOAAUIUyyMAcGEEGCBEsTwCABdGgAFCHMsjAPBdXAcGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMbxO8AcOnRI9957r5xOp9LT0/Xuu+/6PG/btp588kk5nU5NnDhRc+bM0Z/+9CefNqdPn9bixYs1efJkXXvttVq+fLnq6+t92nz66af6u7/7O1199dWaOnWqnn32Wf+rAzqIiLAUGXn+x7Icwe4OAKCH/A4wDQ0NSk9P18qVKzt9/tlnn9WLL76oVatW6eWXX1ZMTIzmzp2rxsZGb5tf/vKXKi8v17Zt27R582b993//tx599FHv82fOnNHcuXM1atQo7d69W0uWLNHTTz+t3/zmNz0oEZBiI6PksW0lJMQoMXHI//+JJcQAgKH8vhv11KlTNXXq1E6fs21b27dv1y9+8QvddNNNkqR169YpNzdX7777rm655RZ99tln2rdvn1555RVdffXVkqQVK1Zo3rx5WrJkiS655BL99re/VXNzswoKChQdHa0rrrhCR48e1bZt2zRz5sxelBveLMvh/Qc7IiK8Vg8HR0TKcjj0dsUJnWo8q+GDYjR9zBWyLIc8HjvY3QMA+KlP/xX76quvVF1drdzcXO+2+Ph4TZo0SYcPH5YkHT58WAkJCd7wIkm5ubmyLEtHjhyRJJWWluraa69VdHS0t43T6dTnn3+ub775pi+7HDYsy6HExFjv7ENCQkywuxQUpxrPqupcg041ng12VwAAveD3DMzFVFdXS5KSkpJ8ticlJcnlckmSXC6Xhg8f7tuJyEgNHTrU+3qXy6XRo0f7tElOTvY+N3To0G73yRHAFYK2fQVyn911fvbF8s5AXB43TM6R3w92t0JCT8YrFMY62PsOxe95R33Vx1AY70ALx5ol6g6Furvbhz4NMKEoKSk+LPbZXW0zEMMHhecMTEeJiUN69fpgjXVv+91bw4YFd//d0R+fUSj/t91fwrFmibpN0KcBJiUlRZLkdrs1YsQI73a3263x48dLOj+TcurUKZ/XtbS06JtvvvG+Pjk52Ttj06btcdtMTHe53XWyA3SIg8NxfvADuc/uioiwgv6PXiiqqalXa6vH79cFYqwvNmY97Xdf7f/06Xq1tPTv/rsjUJ9RKP+33V/CsWaJukOh7ra+dKVPA8zo0aOVkpKi/fv3a8KECZLOn1H0hz/8QX/7t38rScrKylJtba3KysqUkZEhSTpw4IA8Ho8mTpwoScrMzNTGjRvV3NysqKgoSVJJSYnGjh3r1/KRJNm2Aj4Ywdgneq43YxXMsQ7md8yU73hf99GUuvtSONYsUbcJ/D6It76+XkePHtXRo0clnT9w9+jRo6qsrJTD4dCdd96pX//61/r973+vY8eOacmSJRoxYoT3rKRx48ZpypQpeuSRR3TkyBF99NFHeuyxx3TLLbfokksukSTddtttioqK0j/+4z/qxIkTevvtt7V9+3bdfffdfVg6AAAwld8zMGVlZbrzzju9jwsLCyVJeXl5Kioq0j333KOzZ8/q0UcfVW1tra655ho999xzGjRokPc169ev12OPPaa77rpLlmXpxz/+sVasWOF9Pj4+Xlu3btXq1at1++23KzExUffddx+nUAMAAEk9CDDZ2dk6duzYBZ93OBxatGiRFi1adME2w4YN04YNGy66n/Hjx2vnzp3+dg8AAISB8LqaGQAAGBAIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGGfA30oAuJiOd+X2eGzuTg0ABiDAICzFRkbJY9vfuSu3x+NRTU0DIQYAQhwBBmFpcESkLIfDe3duSRo+KEbTx1why3IQYAAgxBFgENba7s4NADALB/ECAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiH06gNYFkOWZbD+5irxQIAwh0BJsRZlkOJibGyrG8nyzwej2prz8m27XbbCDUAgPBBgAlx52dfLO8VY0fFxutHl16uYcNifdpxCXwAQDghwBii7YqxwwfFcAl8AEDYI8AYqrNL4Le/szJLSj3H5wgAoY8AMwB0dmdllpT8Z9LnSMgCEO4IMANAxzsrs6TUMyZ8jiaFLADoTwSYAYQ7K/eNUP4cTQhZABAIBJgBrP0yQ/vfYb5QDlkAEAgEmAGos2UGAAAGEgJMiOl41d2ezJx0XGaQpMvjhsk58vt91k8AAIKJABNCOrvqbm+0X2YYPojZGADAwEGACSEdr7orMXMCAEBnCDAhiJkTAAAujlNTAACAcZiBAbrQ8UBqj8f2uRM4ACDwCDDABVzodHSPx6PTp7kGCwAEEwEGuIDOTkdvu/Ktw+Ho4tUAgP5EgAG6cLE7f0dEWNxMEQCCgIN4AT90XFZKTBzy/6/dw4wMAAQSMzCAH7iZIgCEBgIM0AMdl5U6O1OJQAMA/YcAA/TCxc5UqqlpIMQAQD8hwAC9cLEzlXqyrNT+Zp7+3MiTGSAA4YYAA/SBzs5U8ldPbubJDBCAcEWACbKe/sWNgafjzTy7cyPPvp4BAgBTEGCCqCd/cWPga5vN8edGnn0xAwQAJiHABFFP/uKGOTguBQD6DwEmBPTkL26ELo5LAYD+R4AB+tjFjkuJiopQa6tHEjMyANAbBBign7Q/LqWzWRmPx6Pa2nOy7fMhhoO4AaD7CDBAAHSclRkVG68fXXq5hg2LDXbXAMBIBBgggNof79RxmYmDuAGg+wgwQBC1X2bq64O42y9JcbwNgIGGAAMMMN053uZCCDoATEGAAQaY7h5v47FtWQ6H77Z2QediBxV3dcBxxyDU/orT3X0NAFwMASaAOv5PnLNO0J+6c7xN+23dObC409mdLoKQw+FQQsLg71xxuuPruE4OAH8QYAKE2wYg2Do73qbjtvZBp7ODijvO7vgThC4WoDq7Tk538EcAEL4IMAHS8bYBEmedIDR158rQHdt0JwhdLEBd8OrFnc3udLINQPgJ6QCzY8cObd26VdXV1Ro/frweeeQRTZw4Mdjd6pX+POsECCX+3CKjs6sXdza703EbfwQA4Stk51/ffvttFRYW6v7779err76q8ePHa+7cuXK73cHuGoB+0hZ6qs41qLa5scttbY8BhJ+QDTDbtm3Tz372M82YMUOpqanKz8/X4MGDtWvXrmB3rdssy6HISEuRkRZr9QAA9KGQXEJqamrSxx9/rPnz53u3WZal3NxcHT582K/3siypi0tf+M3hcMjRYQ3ets/vS5IiIy1JnZ95MWLwEEU5zm9LjB7ss63j42C3Cfb+6SN97NHr/v+SVfs/Gmxb6njYTMdtF2vTtr2zP0R6+97BaNOd17X9Hhlpef8fGmp97I824TjW7bWNd/fex+7y2lI90d1D3Bx2f+y9l/7v//5PP/zhD1VcXKysrCzv9nXr1unQoUP6t3/7tyD2DgAABBvrGgAAwDghGWASExMVERHxnQN23W63kpOTg9QrAAAQKkIywERHR+uqq67S/v37vds8Ho/279/vs6QEAADCU0gexCtJd999t5YuXaqMjAxNnDhRL7zwgs6ePavbb7892F0DAABBFrIBZvr06Tp16pQ2bdqk6upqTZgwQc899xxLSAAAIDTPQgIAALiYkDwGBgAA4GIIMAAAwDgEGAAAYBwCDAAAMA4Bpo/s2LFDN954o66++mr9zd/8jY4cORLsLvWpp556Sunp6T4/P/nJT7zPNzY2Kj8/X9nZ2crKytIDDzwgl8sVxB7779ChQ7r33nvldDqVnp6ud9991+d527b15JNPyul0auLEiZozZ47+9Kc/+bQ5ffq0Fi9erMmTJ+vaa6/V8uXLVV9fH8Aq/NdV3cuWLfvO2M+dO9enjYl1b9myRTNmzFBWVpZycnJ033336eTJkz5tuvO9rqys1Lx58zRp0iTl5ORo7dq1amlpCWQp3dadmn/+859/Z7wfffRRnzYm1SxJO3fu1G233abJkydr8uTJmjlzpt5//33v8wNtnNt0VbfxY22j19566y37qquusl955RX7xIkT9ooVK+xrr73Wdrlcwe5an9m0aZN9yy232FVVVd4ft9vtff7RRx+1p06dapeUlNh//OMf7Z/97Gf2zJkzg9hj/+3du9d+4okn7P/4j/+w09LS7Hfeecfn+S1bttjXXHON/c4779hHjx617733XvvGG2+0z507520zd+5c+6c//aldWlpqHzp0yJ42bZr90EMPBboUv3RV99KlS+25c+f6jP3p06d92phY99///d/bu3btso8fP24fPXrUvueee+wf/ehHdn19vbdNV9/rlpYW+9Zbb7XnzJljf/LJJ/bevXvt7Oxse8OGDcEoqUvdqfmOO+6wV6xY4TPedXV13udNq9m2bfv3v/+9vXfvXvvzzz+3T548aT/xxBP2VVddZR8/fty27YE3zm26qtv0sSbA9IG//uu/tvPz872PW1tbbafTaW/ZsiWIvepbmzZtsn/60592+lxtba191VVX2Xv27PFuKy8vt9PS0uzDhw8HqId9q+M/5B6Px77++uvt5557zruttrbWzsjIsN98803btr+t+ciRI94277//vp2enm5//fXXget8L1wowPziF7+44GsGQt22bdtut9tOS0uzDx48aNt2977Xe/futcePH29XV1d72+zcudOePHmy3djYGND+90THmm37/D9qa9asueBrTK+5zXXXXWe//PLLYTHO7bXVbdvmjzVLSL3U1NSkjz/+WLm5ud5tlmUpNzdXhw8fDmLP+t4XX3whp9Opv/zLv9TixYtVWVkpSSorK1Nzc7PPZzBu3DiNGjVKpaWlQept3/rqq69UXV3tU2N8fLwmTZrkHefDhw8rISFBV199tbdNbm6uLMsyfknx4MGDysnJ0c0336yVK1eqpqbG+9xAqbuurk6SNHToUEnd+16XlpYqLS3N5wKbTqdTZ86cUXl5eeA630Mda27zxhtvKDs7W7feeqs2bNigs2fPep8zvebW1la99dZbamhoUFZWVliMs/TdutuYPNYheyVeU9TU1Ki1tVVJSUk+25OSkr6ztmyyiRMnqrCwUGPHjlV1dbWeeeYZzZ49W2+88YZcLpeioqKUkJDg85qkpCRVV1cHqcd9q62Ozsa5ba3c5XJp+PDhPs9HRkZq6NChRn8OU6ZM0bRp0zR69GhVVFToiSee0D333KPf/OY3ioiIGBB1ezweFRQUaPLkyUpLS5Okbn2vXS7Xd64O3vY41GvvrGZJuvXWWzVq1CiNGDFCx44d0/r16/X555/r6aeflmRuzceOHdOsWbPU2Nio2NhYPfPMM0pNTdXRo0cH9DhfqG7J/LEmwKBbpk6d6v19/PjxmjRpkm644Qbt2bNHgwcPDmLP0N9uueUW7+9tB/rddNNN3lmZgSA/P18nTpzQzp07g92VgLlQzTNnzvT+np6erpSUFM2ZM0dffvmlvv/97we6m31m7Nixeu2111RXV6ff/e53Wrp0qV566aVgd6vfXaju1NRU48eaJaReSkxMVEREhNxut892t9s9oO/blJCQoMsvv1xffvmlkpOT1dzcrNraWp82brdbKSkpQeph32qr42LjnJycrFOnTvk839LSom+++WbAfA6SNGbMGCUmJuqLL76QZH7dq1ev1t69e/XCCy9o5MiR3u3d+V4nJyd/52yVtsehXPuFau7MpEmTJMlnvE2sOTo6WpdddpkyMjK0ePFijR8/Xtu3bx/Q4yxduO7OmDbWBJheio6O1lVXXaX9+/d7t3k8Hu3fv99nnXGgqa+vV0VFhVJSUpSRkaGoqCifz+DkyZOqrKxUZmZm8DrZh0aPHq2UlBSfGs+cOaM//OEP3nHOyspSbW2tysrKvG0OHDggj8ejiRMnBrzP/eXrr7/W6dOnvf8DM7Vu27a1evVqvfPOO3rhhRc0ZswYn+e7873OzMzU8ePHfYJtSUmJ4uLivNP0oaSrmjtz9OhRSd/+g2VazRfi8XjU1NQ0IMf5Ytrq7oxpY80SUh+4++67tXTpUmVkZGjixIl64YUXdPbsWd1+++3B7lqfWbt2rW644QaNGjVKVVVVeuqpp2RZlm699VbFx8drxowZKioq0tChQxUXF6c1a9YoKyvLqABTX1+vL7/80vv4q6++0tGjRzV06FCNGjVKd955p37961/rsssu0+jRo/Xkk09qxIgRuummmySdP/BvypQpeuSRR5Sfn6/m5mY99thjuuWWW3TJJZcEq6wuXazuoUOH6umnn9bNN9+s5ORkVVRU6PHHH9dll12mKVOmSDK37vz8fL355pv6p3/6Jw0ZMsS7ph8fH6/Bgwd363vtdDqVmpqqJUuW6OGHH1Z1dbU2btyo2bNnKzo6OojVda6rmr/88ku98cYbmjp1qoYNG6Zjx46psLBQ1113ncaPHy/JvJolacOGDfrhD3+oSy+9VPX19XrzzTd18OBBbd26dUCOc5uL1T0Qxpq7UfeRl156SVu3blV1dbUmTJigFStWeKfjBoIHH3xQhw4d0unTpzV8+HBdc801evDBB73rpI2NjSoqKtJbb72lpqYmOZ1OrVy5MiSmGbvrww8/1J133vmd7Xl5eSoqKpJt29q0aZNefvll1dbW6pprrtHKlSs1duxYb9vTp0/rscce03/+53/Ksiz9+Mc/1ooVKzRkyJBAluKXi9W9atUq3X///frkk09UV1enESNG6Prrr9eiRYt8lkhNrDs9Pb3T7YWFhd4/Prrzvf7f//1frVq1SgcPHlRMTIzy8vK0ePFiRUaG3t+HXdX85z//WQ8//LBOnDihhoYGXXrppbrpppt03333KS4uztvepJolafny5Tpw4ICqqqoUHx+v9PR03XPPPbr++uslDbxxbnOxugfCWBNgAACAcTgGBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADj/D8y4eWK9yOESAAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "df[\"post_infill_duration\"].hist(bins=np.linspace(-5, 360, 100))\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.035167Z", "start_time": "2024-05-16T13:59:40.801098Z" }, "execution": { "iopub.execute_input": "2025-06-03T18:50:33.276843Z", "iopub.status.busy": "2025-06-03T18:50:33.276689Z", "iopub.status.idle": "2025-06-03T18:50:34.560313Z", "shell.execute_reply": "2025-06-03T18:50:34.559953Z", "shell.execute_reply.started": "2025-06-03T18:50:33.276830Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "after duration 0.9956505875409133\n", "after infill duration 0.9979245586736063\n", "neg_filter_reaction_play_count 1.0\n", "neg_filter_upvote_count 0.9895\n", "neg_filter_norm_play_frac 1.0\n", "neg_filter_continues 1.0\n", "----------------\n", "pos_filter_continues 1.0\n", "pos_filter_reaction_play_count 1.0\n", "pos_filter_relative_play_count 0.9741\n", "pos_filter_cer_diff_preference 1.0\n", "pos_filter_bad_flags 0.9999\n", "after filter on play counts 0.9742\n", "after filter on higher quality 0.318\n", "----------------\n", "negative 458116 positive 130106\n", "----------------\n", "total pair requests 465925 --> selected pair requests 128741 frac 0.276\n" ] } ], "source": [ "normal_pos_play_count = 3\n", "# this is lower, cause a concat is probably already ensuring that it is good\n", "concat_pos_play_count = 1\n", "# this is a filter on the concated clip\n", "concat_total_play_count = 3\n", "\n", "all_fitlers = (df[\"duration\"] >= 10) & (df[\"duration\"] <= 240)\n", "print(\"after duration\", all_fitlers.sum() / df.shape[0])\n", "infill_duration_filter = (\n", " ~df[\"task\"].isin(\n", " [\n", " \"infill\",\n", " \"infill_intro\",\n", " \"infill_outro\",\n", " ]\n", " )\n", ") | (df[\"post_infill_duration\"] <= 239)\n", "print(\"after infill duration\", infill_duration_filter.sum() / df.shape[0])\n", "# negative fitlers\n", "total_negative = df[~df[\"preference\"]].shape[0]\n", "neg_filter_reaction_play_count = (~df[\"preference\"]) & (df[\"reaction_play_count\"] >= 1)\n", "print(\n", " \"neg_filter_reaction_play_count\",\n", " round(neg_filter_reaction_play_count.sum() / total_negative, 4),\n", ")\n", "neg_filter_upvote_count = (~df[\"preference\"]) & (df[\"upvote_count\"] == 0)\n", "print(\n", " \"neg_filter_upvote_count\",\n", " round(neg_filter_upvote_count.sum() / total_negative, 4),\n", ")\n", "neg_filter_norm_play_frac = (~df[\"preference\"]) & (df[\"norm_play_frac\"] <= 3.1)\n", "print(\n", " \"neg_filter_norm_play_frac\",\n", " round(neg_filter_norm_play_frac.sum() / total_negative, 4),\n", ")\n", "neg_filter_continues = (~df[\"preference\"]) & (\n", " df[\"has_continue_and_start_continue_at\"].isna()\n", ")\n", "print(\n", " \"neg_filter_continues\",\n", " round(neg_filter_continues.sum() / total_negative, 4),\n", ")\n", "\n", "neg_filter_selection_mask = (\n", " all_fitlers\n", " & infill_duration_filter\n", " & neg_filter_reaction_play_count\n", " & neg_filter_upvote_count\n", " & neg_filter_norm_play_frac\n", " & neg_filter_continues\n", ")\n", "\n", "print(\"----------------\")\n", "total_positive = df[df[\"preference\"]].shape[0]\n", "assert total_positive == total_negative\n", "pos_filter_continues = (df[\"preference\"]) & (df[\"good_continue_at\"])\n", "print(\"pos_filter_continues\", round(pos_filter_continues.sum() / total_positive, 4))\n", "pos_filter_reaction_play_count = (df[\"preference\"]) & (df[\"reaction_play_count\"] >= 1)\n", "print(\n", " \"pos_filter_reaction_play_count\",\n", " round(pos_filter_reaction_play_count.sum() / total_positive, 4),\n", ")\n", "pos_filter_relative_play_count = (df[\"preference\"]) & (df[\"play_rel_diff\"] >= 0)\n", "print(\n", " \"pos_filter_relative_play_count\",\n", " round(pos_filter_relative_play_count.sum() / total_positive, 4),\n", ")\n", "pos_filter_cer_diff_preference = (\n", " df[\n", " \"preference\"\n", " ] # & (df[\"pos_diff_preference\"] == 2) # & (df[\"cer_diff_preference\"] < 0.5) & (df[\"cer\"] < 0.99)\n", ")\n", "print(\n", " \"pos_filter_cer_diff_preference\",\n", " round(pos_filter_cer_diff_preference.sum() / total_positive, 4),\n", ")\n", "pos_filter_bad_flags = (\n", " (df[\"preference\"]) & (df[\"flag_count\"] == 0) & (df[\"dislike_count\"] == 0)\n", ")\n", "print(\n", " \"pos_filter_bad_flags\",\n", " round(pos_filter_bad_flags.sum() / total_positive, 4),\n", ")\n", "pos_filter_play_counts = (df[\"preference\"]) & (\n", " (\n", " (df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= concat_pos_play_count)\n", " & (df[\"concat_play_counts\"] >= concat_total_play_count)\n", " )\n", " | (\n", " (~df[\"part_of_concat\"]) & (df[\"reaction_play_count\"] >= normal_pos_play_count)\n", " # & (df[\"norm_play_frac\"] >= 2.1) # this is a bit of a luxury cut...\n", " )\n", " | (df[\"task\"].isin([\"infill\", \"infill_intro\", \"infill_outro\"]))\n", ")\n", "print(\n", " \"after filter on play counts\",\n", " round(pos_filter_play_counts.sum() / total_positive, 4),\n", ")\n", "high_quality_tasks_filter = (\n", " (df[\"task\"].isin([\"cover\", \"extend\", \"artist_consistency\", \"\"]))\n", " & (\n", " (df[\"upvote_count\"] >= 1) # (df[\"upvote_count\"] >= 1)\n", " | (df[\"reaction_play_count\"] >= 5)\n", " | (df[\"concat_play_counts\"] >= 5)\n", " )\n", " & (\n", " (df[\"part_of_concat\"])\n", " | (\n", " (~df[\"part_of_concat\"])\n", " & (df[\"norm_play_frac\"] >= 5.1) # this is a bit of a luxury cut...\n", " & (\n", " df[\"norm_play_frac\"] >= df[\"reaction_play_count\"] / 3\n", " ) # play duration is not low on average\n", " )\n", " )\n", ")\n", "medium_quality_tasks_filter = (\n", " df[\"task\"].isin(\n", " [\n", " \"infill\",\n", " \"infill_intro\",\n", " \"infill_outro\",\n", " ]\n", " )\n", ") & ( # let more infill through only in this case...\n", " (\n", " df[\"upvote_count\"] >= 0\n", " ) # (df[\"upvote_count\"] >= 1) (df[\"pos_diff_preference\"] == 2)\n", " | (df[\"reaction_play_count\"] >= 1)\n", " | (df[\"concat_play_counts\"] >= 1)\n", ")\n", "pos_filter_higher_quality = (df[\"preference\"]) & (\n", " high_quality_tasks_filter | medium_quality_tasks_filter\n", ")\n", "print(\n", " \"after filter on higher quality\",\n", " round(pos_filter_higher_quality.sum() / total_positive, 4),\n", ")\n", "\n", "user_gen_filter = (\n", " df[\"user_n_clips\"] >= 100\n", ") # user needs to have genereated at least 100 over the time period\n", "\n", "print(\"----------------\")\n", "pos_filter_selectin_mask = (\n", " (df[\"preference\"]) # get basics aligned\n", " & all_fitlers\n", " & infill_duration_filter\n", " & pos_filter_continues\n", " & pos_filter_reaction_play_count\n", " & pos_filter_relative_play_count\n", " & pos_filter_cer_diff_preference\n", " & pos_filter_bad_flags\n", " & pos_filter_play_counts\n", " & pos_filter_higher_quality\n", " & user_gen_filter\n", ")\n", "print(\n", " \"negative\",\n", " sum(neg_filter_selection_mask),\n", " \"positive\",\n", " sum(pos_filter_selectin_mask),\n", ")\n", "\n", "neg_filter_requests = df[neg_filter_selection_mask][\"request_id\"].unique()\n", "pos_filter_requests = df[pos_filter_selectin_mask][\"request_id\"].unique()\n", "# looking for very strong signal here:\n", "# listen to the positive/negative more than once\n", "# disliked one of the clips\n", "unique_requests = set(pos_filter_requests).intersection(neg_filter_requests)\n", "print(\"----------------\")\n", "print(\n", " \"total pair requests\",\n", " df[\"request_id\"].nunique(),\n", " \" --> selected pair requests\",\n", " len(unique_requests),\n", " f\"frac {len(unique_requests) / df['request_id'].nunique():.3f}\",\n", ")" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.250737Z", "start_time": "2024-05-16T13:59:41.036434Z" }, "execution": { "iopub.execute_input": "2025-06-03T18:50:34.560925Z", "iopub.status.busy": "2025-06-03T18:50:34.560763Z", "iopub.status.idle": "2025-06-03T18:50:35.016290Z", "shell.execute_reply": "2025-06-03T18:50:35.015931Z", "shell.execute_reply.started": "2025-06-03T18:50:34.560912Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "bluejay_mix_t2 requests 128741 clips 257482 total khrs 11.045; N gpus for 1000 iters 16.093; 4 gpus for x iters 4023.156; n unique users 43058 n pro users 40242\n" ] } ], "source": [ "df_slice = df[df[\"request_id\"].isin(set(unique_requests))].copy()\n", "print(\n", " f\"{os.path.basename(OUT_DATA_DIR)} requests\",\n", " df_slice[\"request_id\"].nunique(),\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"4 gpus for x iters {df_slice.shape[0] / 8 / 2 / 4:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", " f\"n pro users {df_slice[df_slice['is_pro_user']]['user_id'].nunique()}\",\n", ")\n", "# 76171 152342 total khrs 2.880 n gpus for 1250 iters 3.809\n", "# v10 has 78866\n", "# v14 has 110402\n", "# 30b_t4_v12 requests 47755 clips 95510 total khrs 4.555; N gpus for 1000 iters 5.969; 4 gpus for x iters 1492.344; n unique users 13790 n pro users 13526\n", "# 30b_t4_v20 requests 18306 clips 36612 total khrs 1.754; N gpus for 1000 iters 2.288; 4 gpus for x iters 572.062; n unique users 6943 n pro users 6714\n", "# 30b_t5_v5 requests 21318 clips 42636 total khrs 1.938; N gpus for 1000 iters 2.665; 4 gpus for x iters 666.188; n unique users 11661 n pro users 9347\n", "# 30b_t5_v6 requests 22833 clips 45666 total khrs 2.072; N gpus for 1000 iters 2.854; 4 gpus for x iters 713.531; n unique users 12576 n pro users 9910\n", "# 30b_t5_v7 requests 33197 clips 66394 total khrs 2.846; N gpus for 1000 iters 4.150; 4 gpus for x iters 1037.406; n unique users 16665 n pro users 13050\n", "# 30b_t5_v8 requests 20907 clips 41814 total khrs 1.629; N gpus for 1000 iters 2.613; 4 gpus for x iters 653.344; n unique users 11102 n pro users 9052\n", "# 30b_t5_v9 requests 24645 clips 49290 total khrs 1.918; N gpus for 1000 iters 3.081; 4 gpus for x iters 770.156; n unique users 12661 n pro users 10229\n", "# 30b_t5_v11 requests 21975 clips 43950 total khrs 1.508; N gpus for 1000 iters 2.747; 4 gpus for x iters 686.719; n unique users 11173 n pro users 9270\n", "# 30b_t5_v12 requests 21436 clips 42872 total khrs 1.582; N gpus for 1000 iters 2.679; 4 gpus for x iters 669.875; n unique users 11553 n pro users 9374\n", "# 30b_t5_v13 requests 25912 clips 51824 total khrs 1.922; N gpus for 1000 iters 3.239; 4 gpus for x iters 809.750; n unique users 13429 n pro users 10737\n", "# 30b_t5_v14 requests 27687 clips 55374 total khrs 2.036; N gpus for 1000 iters 3.461; 4 gpus for x iters 865.219; n unique users 14196 n pro users 11227\n", "# 30b_t5_v15 requests 29563 clips 59126 total khrs 2.225; N gpus for 1000 iters 3.695; 4 gpus for x iters 923.844; n unique users 14684 n pro users 11642\n", "# 30b_t6_v1 requests 24678 clips 49356 total khrs 1.934; N gpus for 1000 iters 3.085; 4 gpus for x iters 771.188; n unique users 11350 n pro users 10065\n", "# 30b_t6_v2 requests 49341 clips 98682 total khrs 4.332; N gpus for 1000 iters 6.168; 4 gpus for x iters 1541.906; n unique users 19992 n pro users 16972\n", "# 30b_t6_v3 requests 54967 clips 109934 total khrs 4.826; N gpus for 1000 iters 6.871; 4 gpus for x iters 1717.719; n unique users 21653 n pro users 18266\n", "# 30b_t6_v5 requests 29459 clips 58918 total khrs 2.657; N gpus for 1000 iters 3.682; 4 gpus for x iters 920.594; n unique users 14091 n pro users 11884\n", "# 30b_t6_v7 requests 40056 clips 80112 total khrs 3.354; N gpus for 1000 iters 5.007; 4 gpus for x iters 1251.750; n unique users 17440 n pro users 14476\n", "# 30b_t6_v8 requests 31681 clips 63362 total khrs 2.681; N gpus for 1000 iters 3.960; 4 gpus for x iters 990.031; n unique users 15180 n pro users 15078\n", "# 30b_t6_v9 requests 55785 clips 111570 total khrs 4.727; N gpus for 1000 iters 6.973; 4 gpus for x iters 1743.281; n unique users 23114 n pro users 22871\n", "# 30b_t6_v10 requests 52209 clips 104418 total khrs 4.515; N gpus for 1000 iters 6.526; 4 gpus for x iters 1631.531; n unique users 22216 n pro users 21893\n", "# 30b_t6_v11 requests 31338 clips 62676 total khrs 2.493; N gpus for 1000 iters 3.917; 4 gpus for x iters 979.312; n unique users 15190 n pro users 14973\n", "# 30b_t6_v13 requests 50856 clips 101712 total khrs 3.987; N gpus for 1000 iters 6.357; 4 gpus for x iters 1589.250; n unique users 23547 n pro users 23097\n", "# 30b_t6_v15 requests 65213 clips 130426 total khrs 5.217; N gpus for 1000 iters 8.152; 4 gpus for x iters 2037.906; n unique users 28763 n pro users 27964\n", "# 30b_t6_v16 requests 37907 clips 75814 total khrs 2.956; N gpus for 1000 iters 4.738; 4 gpus for x iters 1184.594; n unique users 18448 n pro users 17212\n", "# 30b_t6_v17 requests 51543 clips 103086 total khrs 4.154; N gpus for 1000 iters 6.443; 4 gpus for x iters 1610.719; n unique users 24223 n pro users 21829\n", "# 30b_t6_v18 requests 94965 clips 189930 total khrs 7.483; N gpus for 1000 iters 11.871; 4 gpus for x iters 2967.656; n unique users 23852 n pro users 21767\n", "# 30b_t6_v19 requests 34289 clips 68578 total khrs 2.846; N gpus for 1000 iters 4.286; 4 gpus for x iters 1071.531; n unique users 11802 n pro users 10830\n", "# 30b_t6_v20 requests 53258 clips 106516 total khrs 4.270; N gpus for 1000 iters 6.657; 4 gpus for x iters 1664.312; n unique users 22096 n pro users 21171\n", "# 30b_t6_v22 requests 29971 clips 59942 total khrs 1.992; N gpus for 1000 iters 3.746; 4 gpus for x iters 936.594; n unique users 13611 n pro users 13036\n", "# 30b_t6_v24 requests 45333 clips 90666 total khrs 3.343; N gpus for 1000 iters 5.667; 4 gpus for x iters 1416.656; n unique users 18853 n pro users 18049\n", "# 30b_t6_v25 requests 28154 clips 56308 total khrs 2.401; N gpus for 1000 iters 3.519; 4 gpus for x iters 879.812; n unique users 12946 n pro users 12361\n", "# 30b_t6_v26 requests 30428 clips 60856 total khrs 2.599; N gpus for 1000 iters 3.804; 4 gpus for x iters 950.875; n unique users 13822 n pro users 13163\n", "# 30b_t6_v27 requests 43731 clips 87462 total khrs 3.862; N gpus for 1000 iters 5.466; 4 gpus for x iters 1366.594; n unique users 17898 n pro users 16865\n", "# 30b_t6_v28 requests 56869 clips 113738 total khrs 5.064; N gpus for 1000 iters 7.109; 4 gpus for x iters 1777.156; n unique users 22004 n pro users 20244\n", "# 30b_t6_v29 requests 32729 clips 65458 total khrs 2.604; N gpus for 1000 iters 4.091; 4 gpus for x iters 1022.781; n unique users 13739 n pro users 13503\n", "# 30b_t6_v30 requests 32553 clips 65106 total khrs 2.592; N gpus for 1000 iters 4.069; 4 gpus for x iters 1017.281; n unique users 13712 n pro users 13478\n", "# 30b_t6_v31 requests 41523 clips 83046 total khrs 3.164; N gpus for 1000 iters 5.190; 4 gpus for x iters 1297.594; n unique users 16266 n pro users 15967\n", "# 30b_t6_v32 requests 55942 clips 111884 total khrs 4.361; N gpus for 1000 iters 6.993; 4 gpus for x iters 1748.188; n unique users 20808 n pro users 20271\n", "# 30b_t6_v33 requests 182096 clips 364192 total khrs 13.745; N gpus for 1000 iters 22.762; 4 gpus for x iters 5690.500; n unique users 40437 n pro users 34524\n", "# auk_mix_t1_v2 requests 102002 clips 204004 total khrs 9.191; N gpus for 1000 iters 12.750; 4 gpus for x iters 3187.562; n unique users 36408 n pro users 34038\n", "# auk_mix_t1_v13 requests 53011 clips 106022 total khrs 4.496; N gpus for 1000 iters 6.626; 4 gpus for x iters 1656.594; n unique users 23713 n pro users 23117\n", "# auk_mix_t1_v14 requests 63272 clips 126544 total khrs 5.382; N gpus for 1000 iters 7.909; 4 gpus for x iters 1977.250; n unique users 26926 n pro users 26111" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.277006Z", "start_time": "2024-05-16T13:59:41.252105Z" }, "execution": { "iopub.execute_input": "2025-06-03T18:50:35.016936Z", "iopub.status.busy": "2025-06-03T18:50:35.016692Z", "iopub.status.idle": "2025-06-03T18:50:35.067301Z", "shell.execute_reply": "2025-06-03T18:50:35.066966Z", "shell.execute_reply.started": "2025-06-03T18:50:35.016922Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive in playlist (29458, 164)\n", "task\n", "cover 108844\n", "artist_consistency 107186\n", "infill 41446\n", " 6\n", "Name: count, dtype: int64\n" ] } ], "source": [ "test_mask = (df_slice[\"preference\"]) & (\n", " (df_slice[\"is_in_playlist\"]) | (df_slice[\"concat_in_playlist\"])\n", ")\n", "print(\"positive in playlist\", df_slice[test_mask].shape)\n", "print(df_slice[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:35.067867Z", "iopub.status.busy": "2025-06-03T18:50:35.067676Z", "iopub.status.idle": "2025-06-03T18:50:35.201249Z", "shell.execute_reply": "2025-06-03T18:50:35.200924Z", "shell.execute_reply.started": "2025-06-03T18:50:35.067854Z" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAqfUlEQVR4nO3df3AUdZ7/8dd0fpCE/CAkIHKgcsQE1ghJ1IqJw/LdPVnuRK8KuFuow1UsStgVhLJQ4DgUgmwSULaQ1Vs45JCfx3IL6KpwW3qnHhYguEU2h4tI5HaNl/XyAzAhgfya/v7BpWFCfszAZGY+yfNRldLp+Uz3p9/pJC+6P/1pl23btgAAAAxihboDAAAA/iLAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjRIa6Az2tpqZOwXxYgsslpaQkBH27pqFOvqFOvqFO3aNGvqFOvunJOrWtuzu9PsDYtkJyEIZqu6ahTr6hTr6hTt2jRr6hTr4JZZ24hAQAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOL3+adTomyzLJctyeS3zeGx5PDxeFgB6AwIMeh3Lcik5OU6W5X2C0ePx6Pz5BkIMAPQCBBj0OlfOvlg6UH5G5xovSZIG9ovVQ8PvlGW5CDAA0AsQYNBrnWu8pMrLDaHuBgCgBzCIFwAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxbirA/NM//ZMyMjL005/+1FnW2NiogoIC5ebmKjs7W08//bSqq6u9PldRUaHZs2dr7NixysvL0+rVq9XS0uLV5pNPPtHkyZOVmZmpCRMmaN++fTfTVQAA0IvccIApLS3V7t27lZGR4bW8sLBQH3zwgdatW6ft27ersrJS8+bNc95vbW3VnDlz1NzcrN27d6u4uFj79+/X+vXrnTbl5eWaM2eOcnNz9dZbb+nxxx/XsmXLdOjQoRvtLgAA6EVuKMDU19frueee06pVq5SUlOQsr6ur0969e7VkyRLl5eUpMzNThYWFOnHihEpKSiRJH3/8scrKyvTSSy9p9OjRGj9+vBYsWKCdO3eqqalJkrR7924NGzZMS5Ys0ciRI/Xoo49q4sSJeuONN256hwEAgPluKMCsXLlS48ePV35+vtfykydPqrm52Wv5yJEjNXToUCfAlJSUKD09XampqU4bt9utixcvqqyszGmTl5fntW632+2swx8uV/C/QrVd0756qk7heEyEY5162xd1okbUqffUyReRvjW76t1339Xvf/97/epXv7ruverqakVFRSkxMdFreUpKiqqqqpw214YXSc7r7tpcvHhRly9fVkxMjM/9TUlJ8LltIIVqu6YJdp2Sk/sHdXuBwvHkG+rUPWrkG+rkm1DWya8A86c//Uk//elP9c///M/q169fT/UpoGpq6mTbwduey3XlGxrs7ZqmJ+sUEWF1GlTOn69Xa6snsBvsQRxPvqFO3aNGvqFOvunJOrWtuzt+BZjPPvtMNTU1mjJlirOstbVVx48f186dO7V582Y1NzertrbW6yxMTU2NBg0aJOnKmZTS0lKv9bbdpXRtm/Z3LlVXVys+Pt6vsy+SZNsKyUEYqu2aJhR1MvH7wvHkG+rUPWrkG+rkm1DWya8Ac//99+vtt9/2Wvb3f//3+vM//3M9+eSTuvXWWxUVFaUjR45o4sSJkqSzZ8+qoqJCWVlZkqSsrCxt2LBBNTU1SklJkSQdPnxY8fHxSktLc9r853/+p9d2Dh8+7KwDAAD0bX4FmPj4eKWnp3sti4uL04ABA5zlU6dOVXFxsZKSkhQfH69Vq1YpOzvbCR9ut1tpaWlatGiRnnvuOVVVVWndunWaMWOGoqOjJUnTp0/Xzp07tWbNGk2dOlVHjx7VwYMHtXHjxgDsMgAAMJ3fg3i7s3TpUlmWpfnz56upqUlut1vLly933o+IiNCGDRu0YsUKTZs2TbGxsZo8ebLmz5/vtBk+fLg2btyooqIibdu2TUOGDNGqVas0bty4QHcXAAAYyGXbvfsqX3V18AfxpqYmBH27punJOkVGXhnEu6OsVJWXGyRJg2Pi9GjaGJ0/X6+WFrMG8XI8dY86dY8a+YY6+aYn69S27u7wLCQAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwjl8BZteuXXrkkUeUk5OjnJwcTZs2TR999JHzfmNjowoKCpSbm6vs7Gw9/fTTqq6u9lpHRUWFZs+erbFjxyovL0+rV69WS0uLV5tPPvlEkydPVmZmpiZMmKB9+/bdxC4CAIDexq8AM2TIED377LPat2+f9u7dq/vvv19z587VmTNnJEmFhYX64IMPtG7dOm3fvl2VlZWaN2+e8/nW1lbNmTNHzc3N2r17t4qLi7V//36tX7/eaVNeXq45c+YoNzdXb731lh5//HEtW7ZMhw4dCtAuAwAA0/kVYL7//e9r/PjxuuOOOzRixAg988wziouLU0lJierq6rR3714tWbJEeXl5yszMVGFhoU6cOKGSkhJJ0scff6yysjK99NJLGj16tMaPH68FCxZo586dampqkiTt3r1bw4YN05IlSzRy5Eg9+uijmjhxot54441A7zsAADBU5I1+sLW1Vf/2b/+mhoYGZWdn6+TJk2publZ+fr7TZuTIkRo6dKhKSkqUlZWlkpISpaenKzU11Wnjdru1YsUKlZWV6Tvf+Y5KSkqUl5fntS23263CwsIb6qfLdWP7d6Pathfs7ZomlHUy6XvD8eQb6tQ9auQb6uSbnqyTr+v0O8CcPn1a06dPV2Njo+Li4vTaa68pLS1Np06dUlRUlBITE73ap6SkqKqqSpJUXV3tFV4kOa+7a3Px4kVdvnxZMTExfvU3JSXBr/aBEqrtmibYdUpO7h/U7QUKx5NvqFP3qJFvqJNvQlknvwPMiBEj9Oabb6qurk6/+c1vtHjxYu3YsaMn+hYQNTV1su3gbc/luvINDfZ2TdOTdYqIsDoNKufP16u11RPYDfYgjiffUKfuUSPfUCff9GSd2tbdHb8DTHR0tG6//XZJUmZmpv7rv/5L27Zt01/91V+publZtbW1XmdhampqNGjQIElXzqSUlpZ6ra/tLqVr27S/c6m6ulrx8fF+n32RJNtWSA7CUG3XNKGok4nfF44n31Cn7lEj31An34SyTjc9D4zH41FTU5MyMzMVFRWlI0eOOO+dPXtWFRUVysrKkiRlZWXpiy++UE1NjdPm8OHDio+PV1pamtPm6NGjXts4fPiwsw4AAAC/AszatWt1/Phxff311zp9+rTWrl2rY8eO6ZFHHlFCQoKmTp2q4uJiHT16VCdPntTSpUuVnZ3thA+32620tDQtWrRIn3/+uQ4dOqR169ZpxowZio6OliRNnz5d5eXlWrNmjb788kvt3LlTBw8e1MyZMwO97wAAwFB+XUKqqanR4sWLVVlZqYSEBGVkZGjz5s164IEHJElLly6VZVmaP3++mpqa5Ha7tXz5cufzERER2rBhg1asWKFp06YpNjZWkydP1vz58502w4cP18aNG1VUVKRt27ZpyJAhWrVqlcaNGxegXQYAAKZz2XbvvspXXR38QbypqQlB365perJOkZFXBvHuKCtV5eUGSdLgmDg9mjZG58/Xq6XFrEG8HE/do07do0a+oU6+6ck6ta27OzwLCQAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYx++nUQMmi4i4mtk9HlseD1NtAoCJCDDoE+Iio+SxbSUmxjrLPB6Pzp9vIMQAgIEIMOgTYiIiZblcOlB+RucaL2lgv1g9NPxOWZaLAAMABiLAoE8513jJecAjAMBcDOIFAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOt1EjICzLJctyOa+Z5RYA0JMIMLhpluVScnKcLOvaafqZ5RYA0HMIMLhpV86+WMxyCwAIGgIMAoZZbgEAwcIgXgAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABjHrwCzceNGTZ06VdnZ2crLy9NTTz2ls2fPerVpbGxUQUGBcnNzlZ2draefflrV1dVebSoqKjR79myNHTtWeXl5Wr16tVpaWrzafPLJJ5o8ebIyMzM1YcIE7du37wZ3EehcRISlyMirX5blCnWXAAA+8CvAHDt2TDNmzNCePXu0ZcsWtbS0aNasWWpoaHDaFBYW6oMPPtC6deu0fft2VVZWat68ec77ra2tmjNnjpqbm7V7924VFxdr//79Wr9+vdOmvLxcc+bMUW5urt566y09/vjjWrZsmQ4dOhSAXQakuMgoeWxbiYmxSk7uf81XHCEGAAwQ6U/jzZs3e70uLi5WXl6ePvvsM913332qq6vT3r179fLLLysvL0/SlUDz0EMPqaSkRFlZWfr4449VVlamLVu2KDU1VaNHj9aCBQv08ssva968eYqOjtbu3bs1bNgwLVmyRJI0cuRI/fa3v9Ubb7yhcePGBWjX0ZfFRETKcrl0oPyMzjVekiQN7Berh4bfKctyyeOxQ9xDAEBX/Aow7dXV1UmSkpKSJEknT55Uc3Oz8vPznTYjR47U0KFDnQBTUlKi9PR0paamOm3cbrdWrFihsrIyfec731FJSYkTgK5tU1hY6HcfXUH+x3Tb9oK93XDVWR3CpU7nGi+p8nLDdctD3a824VKncEedukeNfEOdfNOTdfJ1nTccYDwejwoLC5WTk6P09HRJUnV1taKiopSYmOjVNiUlRVVVVU6ba8OLJOd1d20uXryoy5cvKyYmxud+pqQk+LdjARKq7YaT5OT+3bYJxzr50u9gC8c6hSPq1D1q5Bvq5JtQ1umGA0xBQYHOnDmjXbt2BbI/AVdTUyc7iFcDXK4r39BgbzeUIiKsDv/onz9fr9ZWT4ef6ck6ddYfX3XV72Dri8fTjaBO3aNGvqFOvunJOrWtuzs3FGBWrlypDz/8UDt27NCQIUOc5ampqWpublZtba3XWZiamhoNGjTIaVNaWuq1vra7lK5t0/7OperqasXHx/t19kWSbFshOQhDtd1w010NwrVO4dancK1TuKFO3aNGvqFOvgllnfy6C8m2ba1cuVLvvfeetm7dquHDh3u9n5mZqaioKB05csRZdvbsWVVUVCgrK0uSlJWVpS+++EI1NTVOm8OHDys+Pl5paWlOm6NHj3qt+/Dhw846AABA3+ZXgCkoKNCvf/1rrV27Vv3791dVVZWqqqp0+fJlSVJCQoKmTp2q4uJiHT16VCdPntTSpUuVnZ3thA+32620tDQtWrRIn3/+uQ4dOqR169ZpxowZio6OliRNnz5d5eXlWrNmjb788kvt3LlTBw8e1MyZMwO68wAAwEx+XUL6l3/5F0nSj370I6/lRUVFmjJliiRp6dKlsixL8+fPV1NTk9xut5YvX+60jYiI0IYNG7RixQpNmzZNsbGxmjx5subPn++0GT58uDZu3KiioiJt27ZNQ4YM0apVq7iFGgAASPIzwJw+fbrbNv369dPy5cu9Qkt7f/Znf6ZNmzZ1uZ7c3Fy9+eab/nQPAAD0ETwLCQAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcW7qYY4AEGyW5ZJlXX3am8djy2bKVKDPIcAAMIZluZScHCfLunry2OPx6MKF658oDqB3I8AAMMaVsy+WDpSf0bnGSxrYL1YPDb9TLper+w8D6FUIMACMc67xkiovc9YF6MsYxAsAAIxDgAEAAMbhEhKCpv3dIwAA3CgCDIKiw7tHbFuW5VJrK7fAAgD8Q4BBUHR99wgBBgDgHwIMgoq7RwAAgcAgXgAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOExkh5CKiPDO0B6PLY+HmXkBAF0jwCAk4iKj5LFtJSbGei33eDw6f76BEAMA6BIBBiERExEpy+Vyno0kyXk+kmW5CDAAgC4RYBBSPBsJAHAjGMQLAACMQ4ABAADGIcAAAADjEGAAAIBxGMSLHnPtHC/t53sBAOBmEGDgN8tyybJczuv24aSzOV5Mce3+MLEeAIQnAgz8YlkuJSfHybI6P6PS0Rwvd8QPkHvIbcHq5g3pKHgxsR4AhCcCDPxy5eyL5VM4uXaOl4H9fD8bE6ozIO2DFxPrAUD4IsDghtxoOOlKuJwBYXI9AAh/BBiEjc7OgERFRai11eO0Y1wKAIAAg7DTdgaEBz4CADpDgEHY4oGPAIDOEGAQ9hiTAgBoj9nFAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYh7uQYKT2D5BkcjsA6FsIMDAKk9sBACQCDAzD5HYAAIkAA0MxuR0A9G0M4gUAAMYhwAAAAONwCQm9RtudSe3vUAIA9D4EGBivszuTAoVbtgEg/BBgYLz2dybdET9A7iG33fR6uWUbAMIXAQa9RtudSQP7BeZMDLdsA0D4IsAA3eCWbfQEy3LJslzOay5NAv4hwADoVPs/shJ/aAPBslxKTo6TZV0dX8WlScA/BBgAHeroj6zEH9pAuBIMLefyJJcmAf8RYAB0qP0fWYkxQIHG5UngxhFgAHSJP7IAwhEzfgEAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGMfvAHP8+HH9+Mc/ltvtVkZGht5//32v923b1iuvvCK3260xY8Zo5syZ+sMf/uDV5sKFC1q4cKFycnJ07733aunSpaqvr/dq8/nnn+vv/u7vdPfdd2v8+PHatGmT/3sHAAB6Jb8DTENDgzIyMrR8+fIO39+0aZO2b9+uFStWaM+ePYqNjdWsWbPU2NjotHn22WdVVlamLVu2aMOGDfr000/1wgsvOO9fvHhRs2bN0tChQ7Vv3z4tWrRIr776qn75y1/ewC4CAIDexu+J7MaPH6/x48d3+J5t29q2bZt+8pOf6MEHH5QkrVmzRvn5+Xr//fc1adIkffnllzp06JB+9atf6e6775YkLVu2TLNnz9aiRYt0yy236Ne//rWam5tVWFio6Oho3XnnnTp16pS2bNmiadOm3cTuAgCA3iCgM/F+/fXXqqqqUn5+vrMsISFBY8eO1YkTJzRp0iSdOHFCiYmJTniRpPz8fFmWpdLSUk2YMEElJSW69957FR0d7bRxu93atGmTvv32WyUlJfncJ5er+zaB1La9YG8XwReM73E4H0/h1KdwrpO/emofelONehJ18k1P1snXdQY0wFRVVUmSUlJSvJanpKSourpaklRdXa2BAwd6dyIyUklJSc7nq6urNWzYMK82qampznv+BJiUlAT/diJAQrVdBEdycv+gbi/cjqdg7393Bgy40p9wq5O/glFX02sULNTJN6GsU69/FlJNTZ3sID5zzuW68g0N9naDJSLCCrs/XqFw/ny9Wls9Pb6dUB5PXX2vg7X/7XXWpwsX6jVgQH9jfu4624+erGtv/90UKNTJNz1Zp7Z1dyegAWbQoEGSpJqaGg0ePNhZXlNTo1GjRkm6cibl3LlzXp9raWnRt99+63w+NTXVOWPTpu1125kYX9m2QnIQhmq7CI6IiKvj3z0eu8efzByOx1M49aetL+FYJ3/1dP97Q42CgTr5JpR1Cug8MMOGDdOgQYN05MgRZ9nFixf1u9/9TtnZ2ZKk7Oxs1dbW6uTJk06bo0ePyuPxaMyYMZKkrKwsffrpp2pubnbaHD58WCNGjPDr8hEQaHGRUfLYthITY5Wc3P//vuJkWVwwB4Bg8jvA1NfX69SpUzp16pSkKwN3T506pYqKCrlcLj322GP6xS9+oX//93/X6dOntWjRIg0ePNi5K2nkyJEaN26cnn/+eZWWluq3v/2tXnzxRU2aNEm33HKLJOmRRx5RVFSU/uEf/kFnzpzRgQMHtG3bNj3xxBMB3HX4yrJcioy0FBlpeZ156ItiIiJluVw6UH5GO8pKdaD8jCzLIsAAQJD5fQnp5MmTeuyxx5zXRUVFkqTJkyeruLhYTz75pC5duqQXXnhBtbW1uueee/T666+rX79+zmdefvllvfjii3r88cdlWZZ+8IMfaNmyZc77CQkJ2rx5s1auXKkpU6YoOTlZTz31FLdQh4Bluf7vDEPfDi7tnWu8pMrLDaHuBgD0WX4HmNzcXJ0+fbrT910ulxYsWKAFCxZ02mbAgAFau3Ztl9sZNWqUdu3a5W/3EGCW5ZJlWTpQfkbnGi/pjvgBcg+5LdTdAgD0cfyzGj5pO+NQ29zYfWMAAHoYAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxvH7YY4ArhcR4f1vAY/Hlsdjh6g3AND7EWCAmxAXGSWPbSsxMdZrucfj0fnzDYQYAOghBBjgJsRERMpyuXSg/IzONV6SJA3sF6uHht8py3IRYACghxBggAA413hJlZcbQt0NAOgzGMQLAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcbqMGegiz8wJAzyHAAAHG7LwA0PMIMECAMTsvAPQ8AgzQQ5idFwB6DoN4AQCAcQgwAADAOAQYAABgHAIMAAAwDoN4eynLcsmyXF7LmIcEANBbEGB6IctyKTk5TpbVfiI15iEBAPQOBJhe6MrZF4t5SAAAvRYBxkC+Xh5iHpLwc+3jBbikBwA3jgBjmK4uD9XWXpZt29c9gweh19HjBUy+pEcQAxBqBBjDdHR5aGhcgv7frXdowIC4EPcOnWn/eAFTL+n1tiAGwFwEGENde3loYL9Yrz+Od8QPkHvIbSHuITpi+mW93hLEAJiPANOLtP1xHNgvttM2nPpHIJgexACYjwDTR3DqHwDQmxBg+ghO/QMAehMCTB/DqX8AQG9AgOnj2t9yzbgYAIAJCDB9VEdjYiTv+WSk6wMOAADhgADTR7UfEyMxnwwAwBwEmD6uq/lkJDGnTA/jEh4A3BgCDK7TPtQg8Lq6hMet7QDQPQKMAa59eCNjUnqHji7hcWs7APiOABPmOnt4I3qHjm5rvzaktg2mBgB4I8CEufYPb2RMSu/V2WzJEo+AAID2CDCG8OU5RzBb+8tKbXeFSVJycn+nXTiOk2EwMoBgI8AAYebasBru42QYjAwgVAgwQJgL58c/MBgZQKgQYAADhduYmHAOWQB6JwIMYJDOBvreyOWaa2/Pv7qu0IchAPAFAQYwSPtLNjd6uaaz2/MZuwLAFAQYwEDtL9n4exdQ+9vzpatjV6KiItTa6mHSRABhjQADGMzXp4p3FmiuDUKdrQsAwhEBBjCYr08V9+XSUPt1MWkigHBGgAkz7QdWchofvujqqeL+jpNh0kQAJiDAhBGee4RA6mqcDMEYgOkIMGGko4GVnMbHzWJsC4DeiAAThtpfDgBuRkfjZIIRjLubbK+jeWjaY14aAJ0hwAB9RLCCcWeT7V17V5TL5VJiYozX5VKPbctytZ9Yz7d5adrC0s1cGiMsAWYhwAAIqM6eqn3tXVFt2t/x1NW8NNL1AaV9WGp7anf7MNRhOGrfpoOw1P4sESEHCB8EGAA9oqunarcFlvZ3PPk7L01Xl8e6Ckftl3UUljo8S9TuTFJn2gcd7i4EAo8AAyAo/L2E5c/YnY7W3VU4ar+sq7DU3Zmkzi59tQWdjoIQgJsX1gFm586d2rx5s6qqqjRq1Cg9//zzGjNmTKi7BSCIgjF2p6uw5MuZpO4mEpTE3YVAgIXtPwkOHDigoqIizZ07V/v379eoUaM0a9Ys1dTUhLprAHqptrBSeblBtc2NPre5dtnl1hYn6OwoK9XH33zl87oB+C5sA8yWLVv0wx/+UFOnTlVaWpoKCgoUExOjvXv3hrprANCttsBCWAF6RlheQmpqatJnn32mOXPmOMssy1J+fr5OnDjh17osS+pmvJ3fXC6XXO2uedu25HJd+ZKkyEhLHs/V19e26ehz0tWBfYNj+ivKdeX/k6NjvJa1f92TbUK9ffpIH8Np+z3ax/+7NNZ+cG/73xnd/Q7pqk3bEJzISEu2Hdh190SbUG2//e/jcOyjP216at1t/+2Zv7E+trO7G04fAv/7v/+r7373u9q9e7eys7Od5WvWrNHx48f1r//6ryHsHQAACLWwvYQEAADQmbAMMMnJyYqIiLhuwG5NTY1SU1ND1CsAABAuwjLAREdH66677tKRI0ecZR6PR0eOHPG6pAQAAPqmsBzEK0lPPPGEFi9erMzMTI0ZM0Zbt27VpUuXNGXKlFB3DQAAhFjYBpiHHnpI586d0/r161VVVaXRo0fr9ddf5xISAAAIz7uQAAAAuhKWY2AAAAC6QoABAADGIcAAAADjEGAAAIBxCDABtHPnTn3/+9/X3Xffrb/9279VaWlpqLsUUj//+c+VkZHh9fWXf/mXzvuNjY0qKChQbm6usrOz9fTTT6u6ujqEPQ6O48eP68c//rHcbrcyMjL0/vvve71v27ZeeeUVud1ujRkzRjNnztQf/vAHrzYXLlzQwoULlZOTo3vvvVdLly5VfX19EPei53VXpyVLllx3fM2aNcurTW+v08aNGzV16lRlZ2crLy9PTz31lM6ePevVxpefs4qKCs2ePVtjx45VXl6eVq9erZaWlmDuSo/ypU4/+tGPrjueXnjhBa82vb1Ou3bt0iOPPKKcnBzl5ORo2rRp+uijj5z3w+1YIsAEyIEDB1RUVKS5c+dq//79GjVqlGbNmnXdbMJ9zZ133qmPP/7Y+dq1a5fzXmFhoT744AOtW7dO27dvV2VlpebNmxfC3gZHQ0ODMjIytHz58g7f37Rpk7Zv364VK1Zoz549io2N1axZs9TYePWpxs8++6zKysq0ZcsWbdiwQZ9++ul1v2xN112dJGncuHFex9fPfvYzr/d7e52OHTumGTNmaM+ePdqyZYtaWlo0a9YsNTQ0OG26+zlrbW3VnDlz1NzcrN27d6u4uFj79+/X+vXrQ7FLPcKXOknSD3/4Q6/jadGiRc57faFOQ4YM0bPPPqt9+/Zp7969uv/++zV37lydOXNGUhgeSzYC4m/+5m/sgoIC53Vra6vtdrvtjRs3hrBXobV+/Xr7r//6rzt8r7a21r7rrrvsgwcPOsvKysrs9PR0+8SJE0HqYeilp6fb7733nvPa4/HYDzzwgP366687y2pra+3MzEz7nXfesW37ap1KS0udNh999JGdkZFhf/PNN8HrfBC1r5Nt2/bixYvtn/zkJ51+pi/Wqaamxk5PT7ePHTtm27ZvP2cffvihPWrUKLuqqspps2vXLjsnJ8dubGwMav+DpX2dbNu2H330UXvVqlWdfqYv1sm2bfu+++6z9+zZE5bHEmdgAqCpqUmfffaZ8vPznWWWZSk/P18nTpwIYc9C749//KPcbrf+4i/+QgsXLlRFRYUk6eTJk2pubvaq2ciRIzV06FCVlJSEqLeh9/XXX6uqqsqrLgkJCRo7dqxzLJ04cUKJiYm6++67nTb5+fmyLKvPXbY8duyY8vLyNHHiRC1fvlznz5933uuLdaqrq5MkJSUlSfLt56ykpETp6elek4S63W5dvHhRZWVlwet8ELWvU5u3335bubm5evjhh7V27VpdunTJea+v1am1tVXvvvuuGhoalJ2dHZbHUtjOxGuS8+fPq7W1VSkpKV7LU1JSrrvO2peMGTNGRUVFGjFihKqqqvTaa69pxowZevvtt1VdXa2oqCglJiZ6fSYlJUVVVVUh6nHote17R8dS27Xm6upqDRw40Ov9yMhIJSUl9anajRs3ThMmTNCwYcNUXl6un/3sZ3ryySf1y1/+UhEREX2uTh6PR4WFhcrJyVF6erok+fRzVl1dfd0M522v+0qdJOnhhx/W0KFDNXjwYJ0+fVovv/yy/vu//1uvvvqqpL5Tp9OnT2v69OlqbGxUXFycXnvtNaWlpenUqVNhdywRYNBjxo8f7/z/qFGjNHbsWH3ve9/TwYMHFRMTE8KeoTeYNGmS8/9tgy4ffPBB56xMX1NQUKAzZ854jTPD9Tqr07Rp05z/z8jI0KBBgzRz5kx99dVXuu2224LdzZAZMWKE3nzzTdXV1ek3v/mNFi9erB07doS6Wx3iElIAJCcnKyIi4roBuzU1NTy76RqJiYm644479NVXXyk1NVXNzc2qra31alNTU6NBgwaFqIeh17bvXR1LqampOnfunNf7LS0t+vbbb/t07YYPH67k5GT98Y9/lNS36rRy5Up9+OGH2rp1q4YMGeIs9+XnLDU19bo7Sdpe95U6dWTs2LGS5HU89YU6RUdH6/bbb1dmZqYWLlyoUaNGadu2bWF5LBFgAiA6Olp33XWXjhw54izzeDw6cuSIsrOzQ9iz8FJfX6/y8nINGjRImZmZioqK8qrZ2bNnVVFRoaysrNB1MsSGDRumQYMGedXl4sWL+t3vfuccS9nZ2aqtrdXJkyedNkePHpXH49GYMWOC3udw8c033+jChQvOL8q+UCfbtrVy5Uq999572rp1q4YPH+71vi8/Z1lZWfriiy+8QvPhw4cVHx+vtLS0oOxHT+uuTh05deqUpKt/ePtCnTri8XjU1NQUlscSl5AC5IknntDixYuVmZmpMWPGaOvWrbp06ZKmTJkS6q6FzOrVq/W9731PQ4cOVWVlpX7+85/Lsiw9/PDDSkhI0NSpU1VcXKykpCTFx8dr1apVys7O7vUBpr6+Xl999ZXz+uuvv9apU6eUlJSkoUOH6rHHHtMvfvEL3X777Ro2bJheeeUVDR48WA8++KCkKwPnxo0bp+eff14FBQVqbm7Wiy++qEmTJumWW24J1W4FXFd1SkpK0quvvqqJEycqNTVV5eXleumll3T77bdr3LhxkvpGnQoKCvTOO+/oH//xH9W/f39nnEFCQoJiYmJ8+jlzu91KS0vTokWL9Nxzz6mqqkrr1q3TjBkzFB0dHcK9C5zu6vTVV1/p7bff1vjx4zVgwACdPn1aRUVFuu+++zRq1ChJfaNOa9eu1Xe/+13deuutqq+v1zvvvKNjx45p8+bNYXks8TTqANqxY4c2b96sqqoqjR49WsuWLXNOQ/ZFzzzzjI4fP64LFy5o4MCBuueee/TMM88415MbGxtVXFysd999V01NTXK73Vq+fHmvOh3bkU8++USPPfbYdcsnT56s4uJi2bat9evXa8+ePaqtrdU999yj5cuXa8SIEU7bCxcu6MUXX9R//Md/yLIs/eAHP9CyZcvUv3//YO5Kj+qqTitWrNDcuXP1+9//XnV1dRo8eLAeeOABLViwwOuybW+vU0ZGRofLi4qKnH88+fJz9j//8z9asWKFjh07ptjYWE2ePFkLFy5UZGTv+Ddud3X605/+pOeee05nzpxRQ0ODbr31Vj344IN66qmnFB8f77Tv7XVaunSpjh49qsrKSiUkJCgjI0NPPvmkHnjgAUnhdywRYAAAgHEYAwMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcf4/8N4OmPbxC+MAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "df_slice[\"post_infill_duration\"].hist(bins=np.linspace(-5, 300, 100))\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.323409Z", "start_time": "2024-05-16T13:59:41.278278Z" }, "execution": { "iopub.execute_input": "2025-06-03T18:50:35.201815Z", "iopub.status.busy": "2025-06-03T18:50:35.201622Z", "iopub.status.idle": "2025-06-03T18:50:35.215331Z", "shell.execute_reply": "2025-06-03T18:50:35.215061Z", "shell.execute_reply.started": "2025-06-03T18:50:35.201801Z" } }, "outputs": [], "source": [ "# interesting_clips_must_be_positive_mask = (\n", "# (df_slice[\"upvoted\"] == True)\n", "# | (df_slice[\"has_action\"] == True)\n", "# | (df_slice[\"part_of_concat\"] == True)\n", "# )\n", "# interesting_clips_must_be_not_negative_mask = (df_slice[\"downvoted\"] == False) # & (df_slice[\"dislike_count\"] < 1)\n", "# interesting_clips_mask = interesting_clips_must_be_positive_mask & interesting_clips_must_be_not_negative_mask\n", "# assert interesting_clips_mask.eq(df_slice[\"preference\"]).all()" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.392244Z", "start_time": "2024-05-16T13:59:41.324472Z" }, "execution": { "iopub.execute_input": "2025-06-03T18:50:35.215816Z", "iopub.status.busy": "2025-06-03T18:50:35.215644Z", "iopub.status.idle": "2025-06-03T18:50:35.225703Z", "shell.execute_reply": "2025-06-03T18:50:35.225448Z", "shell.execute_reply.started": "2025-06-03T18:50:35.215805Z" } }, "outputs": [], "source": [ "# save positive ids\n", "# positive_preference_ids = df_slice[df_slice[\"preference\"] == False][\"s3_id\"].to_json(orient='values')\n", "# with open('/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id_negative.json', 'w') as file:\n", "# file.write(positive_preference_ids)" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:35.226170Z", "iopub.status.busy": "2025-06-03T18:50:35.226004Z", "iopub.status.idle": "2025-06-03T18:50:35.235969Z", "shell.execute_reply": "2025-06-03T18:50:35.235713Z", "shell.execute_reply.started": "2025-06-03T18:50:35.226159Z" } }, "outputs": [], "source": [ "# df_slice_2 = pd.read_pickle(\"/home/tony/Data/Preference/30b_v6/interesting_clips_v4_t_6_20241118_full_slice.pkl\")\n", "# df_total = pd.concat([df_slice, df_slice_2])\n", "# print(df_total.shape)\n", "# df_slice = df_total" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:35.236465Z", "iopub.status.busy": "2025-06-03T18:50:35.236269Z", "iopub.status.idle": "2025-06-03T18:50:35.246348Z", "shell.execute_reply": "2025-06-03T18:50:35.246094Z", "shell.execute_reply.started": "2025-06-03T18:50:35.236454Z" } }, "outputs": [], "source": [ "# df_slice_prev = pd.read_pickle(\n", "# \"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_v13_20240902_slice.pkl\"\n", "# )\n", "# df_slice_prev = df_slice_prev[\n", "# (\n", "# (df_slice_prev[\"task\"] != \"infill\")\n", "# & (df_slice_prev[\"task\"] != \"cover\")\n", "# & (df_slice_prev[\"task\"] != \"artist_consistency\")\n", "# )\n", "# ].copy()\n", "# df_slice = pd.concat([df_slice, df_slice_prev])\n", "# print(df_slice.shape)\n", "# print(df_slice[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:35.246835Z", "iopub.status.busy": "2025-06-03T18:50:35.246651Z", "iopub.status.idle": "2025-06-03T18:50:35.256718Z", "shell.execute_reply": "2025-06-03T18:50:35.256463Z", "shell.execute_reply.started": "2025-06-03T18:50:35.246823Z" } }, "outputs": [], "source": [ "# df_slice[(df_slice[\"preference\"]) & ((df_slice[\"task\"].str.strip() == \"\") | (df_slice[\"task\"].str.strip() == \"cover\"))].to_pickle(\n", "# \"/home/tony/Data/Preference/30b_v5/interesting_clips_v4_t_5_20241018_full_with_cer_pos_gen.pkl\"\n", "# )\n", "# df_slice[(df_slice[\"preference\"]) & ((df_slice[\"task\"].str.strip() == \"\") | (df_slice[\"task\"].str.strip() == \"cover\"))].shape" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:35.257162Z", "iopub.status.busy": "2025-06-03T18:50:35.257016Z", "iopub.status.idle": "2025-06-03T18:50:35.267032Z", "shell.execute_reply": "2025-06-03T18:50:35.266774Z", "shell.execute_reply.started": "2025-06-03T18:50:35.257151Z" } }, "outputs": [], "source": [ "# df_slice.to_pickle(\"/home/tony/Data/Preference/30b_v6/interesting_clips_v4_h_t_6_20250108_full_slice.pkl\")" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:35.267490Z", "iopub.status.busy": "2025-06-03T18:50:35.267325Z", "iopub.status.idle": "2025-06-03T18:50:35.277243Z", "shell.execute_reply": "2025-06-03T18:50:35.276985Z", "shell.execute_reply.started": "2025-06-03T18:50:35.267479Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Preference/30b_v6/similarities_30b_t6.json\", \"r\") as fp:\n", "# all_similarities = json.load(fp)\n", "\n", "# df_slice[\"similarity\"] = df_slice[\"id\"].map(all_similarities)\n", "# df_cover = df_slice[df_slice[\"task\"] == \"cover\"].copy()\n", "# df_artist = df_slice[df_slice[\"task\"] == \"artist_consistency\"].copy()" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:35.277739Z", "iopub.status.busy": "2025-06-03T18:50:35.277546Z", "iopub.status.idle": "2025-06-03T18:50:35.287627Z", "shell.execute_reply": "2025-06-03T18:50:35.287374Z", "shell.execute_reply.started": "2025-06-03T18:50:35.277728Z" } }, "outputs": [], "source": [ "# df_cover[df_cover[\"preference\"]][\"similarity\"].hist(bins=200, label=\"positive\", alpha=0.5)\n", "# df_cover[~df_cover[\"preference\"]][\"similarity\"].hist(bins=200, label=\"negative\", alpha=0.5)\n", "# plt.legend()\n", "# plt.show()\n", "# df_cover[\"similarity_diff\"] = df_cover[\"similarity\"].diff()\n", "# df_cover[df_cover[\"preference\"]][\"similarity_diff\"].hist(bins=200)\n", "# plt.show()\n", "# df_cover[\"continued_parent\"] = None\n", "# df_cover[\"continue_at\"] = -1\n", "# too_similar = (df_cover[\"similarity\"] >= 0.99)\n", "# too_different = ( df_cover[\"similarity\"] < 0.25)\n", "# too_big_difference = (df_cover[\"similarity_diff\"] < -0.2)\n", "# df_cover_drops_id = df_cover[\n", "# (df_cover[\"preference\"])\n", "# & (too_similar | too_different | too_big_difference)\n", "# ][\"s3_id\"].unique()\n", "# print(\"total rows\", df_cover.shape[0], \"total drops\", len(df_cover_drops_id))" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:35.288082Z", "iopub.status.busy": "2025-06-03T18:50:35.287920Z", "iopub.status.idle": "2025-06-03T18:50:35.298005Z", "shell.execute_reply": "2025-06-03T18:50:35.297748Z", "shell.execute_reply.started": "2025-06-03T18:50:35.288071Z" } }, "outputs": [], "source": [ "# df_artist[df_artist[\"preference\"]][\"similarity\"].hist(bins=200, label=\"positive\", alpha=0.5)\n", "# df_artist[~df_artist[\"preference\"]][\"similarity\"].hist(bins=200, label=\"negative\", alpha=0.5)\n", "# plt.legend()\n", "# plt.show()\n", "# df_artist[\"similarity_diff\"] = df_artist[\"similarity\"].diff()\n", "# df_artist[df_artist[\"preference\"]][\"similarity_diff\"].hist(bins=200)\n", "# plt.show()\n", "# too_similar = (df_artist[\"similarity\"] >= 0.95)\n", "# too_different = ( df_artist[\"similarity\"] < 0.4)\n", "# too_big_difference = (df_artist[\"similarity_diff\"] < -0.2)\n", "# df_artist_drops_id = df_artist[\n", "# (df_artist[\"preference\"])\n", "# & (too_similar | too_different | too_big_difference)\n", "# ][\"s3_id\"].unique()\n", "# print(\"total rows\", df_artist.shape[0], \"total drops\", len(df_artist_drops_id))" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:35.298447Z", "iopub.status.busy": "2025-06-03T18:50:35.298301Z", "iopub.status.idle": "2025-06-03T18:50:35.308335Z", "shell.execute_reply": "2025-06-03T18:50:35.308084Z", "shell.execute_reply.started": "2025-06-03T18:50:35.298436Z" } }, "outputs": [], "source": [ "# print(\"before filter on cover similarity\", df_slice.shape)\n", "# df_slice = df_slice[~df_slice[\"s3_id\"].isin(df_cover_drops_id)].copy()\n", "# print(\"after filter on cover similarity\", df_slice.shape)\n", "# df_slice = df_slice[~df_slice[\"s3_id\"].isin(df_artist_drops_id)].copy()\n", "# print(\"after filter on artist similarity\", df_slice.shape)\n", "# df_slice = df_slice[\n", "# df_slice[\"request_id\"].isin(\n", "# df_slice[\"request_id\"].value_counts().index[df_slice[\"request_id\"].value_counts() == 2]\n", "# )\n", "# ]\n", "# print(\"after filter on request id pairs\", df_slice.shape)\n", "# print(df_slice.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "# assert df_slice.shape[0] == df_slice[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:35.308788Z", "iopub.status.busy": "2025-06-03T18:50:35.308627Z", "iopub.status.idle": "2025-06-03T18:50:35.319960Z", "shell.execute_reply": "2025-06-03T18:50:35.319700Z", "shell.execute_reply.started": "2025-06-03T18:50:35.308777Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "is_public\n", "False 247127\n", "True 10355\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(df_slice[\"is_public\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "execution": { "iopub.execute_input": "2025-06-03T18:50:35.320436Z", "iopub.status.busy": "2025-06-03T18:50:35.320253Z", "iopub.status.idle": "2025-06-03T18:50:35.352175Z", "shell.execute_reply": "2025-06-03T18:50:35.351860Z", "shell.execute_reply.started": "2025-06-03T18:50:35.320425Z" } }, "outputs": [], "source": [ "df_slice[\"npz_path\"] = df_slice[\"s3_id\"].map(lambda x: f\"{NPZ_DIR}/{x}.npz\")" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T14:00:20.866354Z", "start_time": "2024-05-16T14:00:12.443344Z" }, "execution": { "iopub.execute_input": "2025-06-03T18:50:35.352703Z", "iopub.status.busy": "2025-06-03T18:50:35.352523Z", "iopub.status.idle": "2025-06-03T18:50:35.997156Z", "shell.execute_reply": "2025-06-03T18:50:35.995850Z", "shell.execute_reply.started": "2025-06-03T18:50:35.352691Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(216030, 165)\n", "task\n", "cover 108844\n", "artist_consistency 107186\n", "Name: count, dtype: int64\n", "(216030, 165)\n", "task\n", "cover 108844\n", "artist_consistency 107186\n", "Name: count, dtype: int64\n" ] }, { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[37], line 9\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mvalue_counts())\n\u001b[0;32m----> 9\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "# df_slice.to_pickle(\n", "# \"/home/tony/Data/Preference/30b_v6/interesting_clips_v4_h_t_6_20250501_full_long_bluejay_r3.pkl\"\n", "# )\n", "print(df_slice.shape)\n", "print(df_slice[\"task\"].value_counts())\n", "df_slice = df_slice[df_slice[\"task\"].isin([\"cover\", \"artist_consistency\"])].copy()\n", "print(df_slice.shape)\n", "print(df_slice[\"task\"].value_counts())\n", "BREAK" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T10:37:34.609866Z", "iopub.status.busy": "2025-07-01T10:37:34.609514Z", "iopub.status.idle": "2025-07-01T10:37:48.166055Z", "shell.execute_reply": "2025-07-01T10:37:48.165489Z", "shell.execute_reply.started": "2025-07-01T10:37:34.609848Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(416048, 170)\n", "task\n", " 397920\n", "upload_extend 10130\n", "extend 7998\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df_13b_slice = pd.read_pickle(\n", " \"/home/tony/Data/Preference/13b_v32/interesting_clips_v4_h_s_32_20250501_full_long_bluejay_r3.pkl\"\n", ")\n", "print(df_13b_slice.shape)\n", "print(df_13b_slice[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T10:37:48.167037Z", "iopub.status.busy": "2025-07-01T10:37:48.166882Z", "iopub.status.idle": "2025-07-01T10:37:55.671166Z", "shell.execute_reply": "2025-07-01T10:37:55.670593Z", "shell.execute_reply.started": "2025-07-01T10:37:48.167021Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(216030, 165)\n", "task\n", "cover 108844\n", "artist_consistency 107186\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df_30b_slice = pd.read_pickle(\n", " \"/home/tony/Data/Preference/30b_v6/interesting_clips_v4_h_t_6_20250501_full_long_bluejay_r3.pkl\"\n", ")\n", "print(df_30b_slice.shape)\n", "print(df_30b_slice[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T10:37:55.671877Z", "iopub.status.busy": "2025-07-01T10:37:55.671723Z", "iopub.status.idle": "2025-07-01T10:38:01.557402Z", "shell.execute_reply": "2025-07-01T10:38:01.556850Z", "shell.execute_reply.started": "2025-07-01T10:37:55.671861Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(632078, 176)\n", "task\n", " 397920\n", "cover 108844\n", "artist_consistency 107186\n", "upload_extend 10130\n", "extend 7998\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df_total = pd.concat([df_13b_slice, df_30b_slice])\n", "print(df_total.shape)\n", "print(df_total[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T10:38:01.558457Z", "iopub.status.busy": "2025-07-01T10:38:01.558299Z", "iopub.status.idle": "2025-07-01T10:38:03.739193Z", "shell.execute_reply": "2025-07-01T10:38:03.738626Z", "shell.execute_reply.started": "2025-07-01T10:38:01.558442Z" } }, "outputs": [], "source": [ "df_slice = df_total.copy()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932296Z", "start_time": "2024-05-16T13:59:41.932287Z" }, "execution": { "iopub.execute_input": "2025-07-01T10:38:20.135747Z", "iopub.status.busy": "2025-07-01T10:38:20.135375Z", "iopub.status.idle": "2025-07-01T10:38:20.166608Z", "shell.execute_reply": "2025-07-01T10:38:20.166068Z", "shell.execute_reply.started": "2025-07-01T10:38:20.135728Z" } }, "outputs": [], "source": [ "# don't have continue at\n", "df_slice[\"request_id\"] = df_slice[\"request_id\"].astype(str)\n", "# df_slice[\"npz_path\"] = df_slice[\"npz_path\"].apply(lambda x: str(x).replace(\"_npz\", \"_npz/\"))\n", "# df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique(), df_slice[\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T10:38:20.167556Z", "iopub.status.busy": "2025-07-01T10:38:20.167405Z", "iopub.status.idle": "2025-07-01T10:38:22.299994Z", "shell.execute_reply": "2025-07-01T10:38:22.299428Z", "shell.execute_reply.started": "2025-07-01T10:38:20.167541Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(632078, 176)\n", "(632078, 176)\n", "(632078, 176)\n" ] } ], "source": [ "print(df_slice.shape)\n", "df_slice = df_slice[df_slice[\"request_id\"].apply(lambda x: len(x) > 3)]\n", "print(df_slice.shape)\n", "# df_slice = df_slice[df_slice[\"is_pro_user\"]].copy()\n", "print(df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932966Z", "start_time": "2024-05-16T13:59:41.932957Z" }, "execution": { "iopub.execute_input": "2025-07-01T10:38:22.300713Z", "iopub.status.busy": "2025-07-01T10:38:22.300564Z", "iopub.status.idle": "2025-07-01T10:38:22.380196Z", "shell.execute_reply": "2025-07-01T10:38:22.379652Z", "shell.execute_reply.started": "2025-07-01T10:38:22.300698Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "316039\n" ] } ], "source": [ "final_filtered_requests = df_slice[\"request_id\"].astype(str).unique()\n", "# final_filtered_requests = df_slice[df_slice[\"is_pro_user\"]][\"request_id\"].astype(str).unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.933558Z", "start_time": "2024-05-16T13:59:41.933550Z" }, "execution": { "iopub.execute_input": "2025-07-01T10:38:22.381355Z", "iopub.status.busy": "2025-07-01T10:38:22.381198Z", "iopub.status.idle": "2025-07-01T10:38:22.395471Z", "shell.execute_reply": "2025-07-01T10:38:22.395032Z", "shell.execute_reply.started": "2025-07-01T10:38:22.381339Z" } }, "outputs": [], "source": [ "# df_slice.to_csv(\"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_20240902_slice.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T10:38:22.396088Z", "iopub.status.busy": "2025-07-01T10:38:22.395951Z", "iopub.status.idle": "2025-07-01T10:38:22.408254Z", "shell.execute_reply": "2025-07-01T10:38:22.407837Z", "shell.execute_reply.started": "2025-07-01T10:38:22.396073Z" } }, "outputs": [], "source": [ "# df_slice[\"continue_at\"] = -1" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934277Z", "start_time": "2024-05-16T13:59:41.934268Z" }, "execution": { "iopub.execute_input": "2025-07-01T10:38:22.408856Z", "iopub.status.busy": "2025-07-01T10:38:22.408719Z", "iopub.status.idle": "2025-07-01T10:38:25.880269Z", "shell.execute_reply": "2025-07-01T10:38:25.879694Z", "shell.execute_reply.started": "2025-07-01T10:38:22.408843Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "312878 3161\n", "(625756, 176) (6322, 176)\n" ] } ], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df # .reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df # .reset_index()\n", "train_df = train_df.reset_index(drop=True)\n", "val_df = val_df.reset_index(drop=True)\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.935620Z", "start_time": "2024-05-16T13:59:41.935613Z" }, "execution": { "iopub.execute_input": "2025-07-01T10:38:59.831874Z", "iopub.status.busy": "2025-07-01T10:38:59.831326Z", "iopub.status.idle": "2025-07-01T10:39:23.376029Z", "shell.execute_reply": "2025-07-01T10:39:23.375450Z", "shell.execute_reply.started": "2025-07-01T10:38:59.831857Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 625756/625756 [00:23<00:00, 26597.29it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "32,260 hours of 625756 clips, 39.10975 nodes, 611.08984375 iters\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "total_duration = 0\n", "for i, row in tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " total_duration += row[\"duration\"]\n", " except Exception as E:\n", " print(i, row)\n", " print(E)\n", " raise ValueError()\n", "\n", "print(\n", " f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1000} nodes, {train_df.shape[0] / 8 / 8 / 16} iters\"\n", ")" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936268Z", "start_time": "2024-05-16T13:59:41.936260Z" }, "execution": { "iopub.execute_input": "2025-07-01T10:39:23.377017Z", "iopub.status.busy": "2025-07-01T10:39:23.376856Z", "iopub.status.idle": "2025-07-01T10:41:07.522804Z", "shell.execute_reply": "2025-07-01T10:41:07.522246Z", "shell.execute_reply.started": "2025-07-01T10:39:23.377001Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "t_data_memmap is set to: 6016\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 35%|█████████████████████████████████████▉ | 2199/6322 [00:37<01:00, 68.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "weird, /app/suno/data/dpo/30b_npz/c1150a74-6d63-43b1-b3e5-41d74a4b3c63.npz, with only v3.0\n", "2187, 'artist_arr is not a file in the archive', artist_consistency, /app/suno/data/dpo/30b_npz/c1150a74-6d63-43b1-b3e5-41d74a4b3c63.npz.\n", "WTF --> 2187, 'artist_arr is not a file in the archive', artist_consistency, /app/suno/data/dpo/30b_npz/c1150a74-6d63-43b1-b3e5-41d74a4b3c63.npz.\n", "WTF --> 2188, skip, preference: False, 5378d8d5-d336-4934-8fc0-899f1b6bf8e0, task: .\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6322/6322 [01:44<00:00, 60.75it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 6320 clips, 2 different prompts\n", "184 hours of False\n", "182 hours of True\n", "gen: 213.5 hours\n", "artist_consistency: 67.6 hours\n", "extend: 10.6 hours\n", "cover: 73.8 hours\n", "🚨 Error artist_consistency: 1\n", "Done\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "make_dataset(\n", " val_df, OUT_DATA_DIR, is_val=True, npz_dir=NPZ_DIR, t_data_memmap=N_TOKENS_AUDIO\n", ")" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T10:41:07.523494Z", "iopub.status.busy": "2025-07-01T10:41:07.523339Z", "iopub.status.idle": "2025-07-01T10:41:07.545927Z", "shell.execute_reply": "2025-07-01T10:41:07.545479Z", "shell.execute_reply.started": "2025-07-01T10:41:07.523478Z" } }, "outputs": [], "source": [ "# test_npz = np.load(\"/app/suno/data/dpo/30b_npz/26d19085-18da-4701-af43-122684543891.npz\")\n", "# for k in test_npz.keys():\n", "# print(k)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936964Z", "start_time": "2024-05-16T13:59:41.936957Z" }, "execution": { "iopub.execute_input": "2025-07-01T10:41:07.547008Z", "iopub.status.busy": "2025-07-01T10:41:07.546859Z", "iopub.status.idle": "2025-07-01T13:28:21.636494Z", "shell.execute_reply": "2025-07-01T13:28:21.635834Z", "shell.execute_reply.started": "2025-07-01T10:41:07.546993Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "t_data_memmap is set to: 6016\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 6%|██████ | 36586/625756 [09:57<2:31:30, 64.81it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "36574, 'history_arr is not a file in the archive', upload_extend, /app/suno/data/dpo/13b_s32_npz/c8ad50dc-58c3-470f-a094-676e03f25de9.npz.\n", "WTF --> 36575, skip, preference: True, 10754475-08f0-4951-b71a-c2fe0991c409, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 9%|████████▉ | 53601/625756 [14:37<2:23:09, 66.61it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "53588, 'history_arr is not a file in the archive', upload_extend, /app/suno/data/dpo/13b_s32_npz/316ec332-9b2b-4be4-a3e8-66b26fa3beff.npz.\n", "WTF --> 53589, skip, preference: True, f02d5e0c-5083-4375-a989-6d9a7420e0f9, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 34%|███████████████████████████████████ | 212921/625756 [57:52<1:42:04, 67.41it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "weird, /app/suno/data/dpo/30b_npz/025128d0-1ef2-497b-a7aa-b77645e04131.npz, with only v3.0\n", "212909, 'cover_arr is not a file in the archive', cover, /app/suno/data/dpo/30b_npz/025128d0-1ef2-497b-a7aa-b77645e04131.npz.\n", "WTF --> 212909, 'cover_arr is not a file in the archive', cover, /app/suno/data/dpo/30b_npz/025128d0-1ef2-497b-a7aa-b77645e04131.npz.\n", "WTF --> 212910, skip, preference: False, 54ca21bd-79e1-4753-8224-4d78828d80ba, task: .\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 40%|████████████████████████████████████████▋ | 251737/625756 [1:08:16<1:30:53, 68.58it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "251728, 'history_arr is not a file in the archive', upload_extend, /app/suno/data/dpo/13b_s32_npz/6148bda0-531b-4f9d-b12c-7a276545e872.npz.\n", "WTF --> 251729, skip, preference: True, 1cfdccd1-d576-484c-bf18-bfeb7c1a2370, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 54%|██████████████████████████████████████████████████████▎ | 336642/625756 [1:31:05<1:11:07, 67.75it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "336630, 'history_arr is not a file in the archive', upload_extend, /app/suno/data/dpo/13b_s32_npz/cc918963-4a41-4bf1-b846-c9a140d1951b.npz.\n", "WTF --> 336631, skip, preference: True, 409fc6b6-73a1-4884-b80f-6d34ee88ccd1, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 61%|█████████████████████████████████████████████████████████████▉ | 384038/625756 [1:43:58<1:01:14, 65.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "384024, 'history_arr is not a file in the archive', upload_extend, /app/suno/data/dpo/13b_s32_npz/cf0c29c4-267e-4aa3-a3f8-009876019300.npz.\n", "WTF --> 384025, skip, preference: True, 3cb56754-6b0a-4492-9bce-299dff540fe1, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 70%|███████████████████████████████████████████████████████████████████████▉ | 437344/625756 [1:58:01<46:32, 67.46it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "437330, 'history_arr is not a file in the archive', upload_extend, /app/suno/data/dpo/13b_s32_npz/a538ccbe-a313-4acf-99c7-317806e9d062.npz.\n", "WTF --> 437331, skip, preference: True, 83054a12-5b17-42f2-b702-7d025c5e3fe9, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 70%|████████████████████████████████████████████████████████████████████████▎ | 439568/625756 [1:58:36<49:26, 62.77it/s]IOPub message rate exceeded.\n", "The Jupyter server will temporarily stop sending output\n", "to the client in order to avoid crashing it.\n", "To change this limit, set the config variable\n", "`--ServerApp.iopub_msg_rate_limit`.\n", "\n", "Current values:\n", "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n", "ServerApp.rate_limit_window=3.0 (secs)\n", "\n", " 94%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 589004/625756 [2:37:37<08:46, 69.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "588992, 'history_arr is not a file in the archive', upload_extend, /app/suno/data/dpo/13b_s32_npz/1d6fb316-3320-409f-b8a6-189d385b69e6.npz.\n", "WTF --> 588993, skip, preference: True, b33d9b6e-9ab7-4392-8807-fd66ef850551, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 625756/625756 [2:47:13<00:00, 62.37it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 625736 clips, 68 different prompts\n", "18,158 hours of False\n", "17,909 hours of True\n", "artist_consistency: 7040.2 hours\n", "extend: 1013.3 hours\n", "gen: 21103.9 hours\n", "cover: 6910.1 hours\n", "🚨 Error upload_extend: 9\n", "🚨 Error cover: 1\n", "Done\n" ] } ], "source": [ "make_dataset(\n", " train_df, OUT_DATA_DIR, is_val=False, npz_dir=NPZ_DIR, t_data_memmap=N_TOKENS_AUDIO\n", ")" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.937879Z", "start_time": "2024-05-16T13:59:41.937870Z" }, "execution": { "iopub.execute_input": "2025-07-01T13:28:21.637312Z", "iopub.status.busy": "2025-07-01T13:28:21.637142Z", "iopub.status.idle": "2025-07-01T13:28:22.618209Z", "shell.execute_reply": "2025-07-01T13:28:22.617648Z", "shell.execute_reply.started": "2025-07-01T13:28:21.637295Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, N_TOKENS_AUDIO, 13)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000\n", "assert mm[:100, :, 1:].min() >= 0\n", "assert mm[:100, :, 1:].max() <= 2048" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T13:28:22.618972Z", "iopub.status.busy": "2025-07-01T13:28:22.618815Z", "iopub.status.idle": "2025-07-01T13:28:22.638117Z", "shell.execute_reply": "2025-07-01T13:28:22.637639Z", "shell.execute_reply.started": "2025-07-01T13:28:22.618956Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Counter({None: 3971, 'cover': 1144, 'artist_consistency': 1019, 'extend': 186})\n" ] } ], "source": [ "task_counts = Counter()\n", "for test_meta in test_metas:\n", " task_counts[test_meta.get(\"task\")] += 1\n", "print(task_counts)" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.938629Z", "start_time": "2024-05-16T13:59:41.938621Z" }, "execution": { "iopub.execute_input": "2025-07-01T13:28:22.638763Z", "iopub.status.busy": "2025-07-01T13:28:22.638622Z", "iopub.status.idle": "2025-07-01T13:28:22.651631Z", "shell.execute_reply": "2025-07-01T13:28:22.651200Z", "shell.execute_reply.started": "2025-07-01T13:28:22.638748Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as codec_encode,\n", "# decode_stream_to_full_audio as codec_decode,\n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# decode as decode\n", "# )\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "# assert len(test_metas) == len(mm)\n", "# idx_list = list(range(len(test_metas)))\n", "# # random.shuffle(idx_list)\n", "# # idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939205Z", "start_time": "2024-05-16T13:59:41.939198Z" }, "execution": { "iopub.execute_input": "2025-07-01T13:28:22.652234Z", "iopub.status.busy": "2025-07-01T13:28:22.652099Z", "iopub.status.idle": "2025-07-01T13:28:22.663795Z", "shell.execute_reply": "2025-07-01T13:28:22.663365Z", "shell.execute_reply.started": "2025-07-01T13:28:22.652220Z" } }, "outputs": [], "source": [ "# import random\n", "# idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "# assert \"original_duration_s\" in test_metas[idx]\n", "# # positive index should be shifted by 1\n", "# pos_idx = idx + 1\n", "# print(\n", "# \"tags:\",\n", "# test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", "# test_metas[idx].get(\"tags\"),\n", "# )\n", "# arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[: pad_idx_arr[0], :]\n", "# pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pos_pad_idx_arr) > 0:\n", "# pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "# a = decode(arr)\n", "# print(\"\\n negative example \\n\", test_metas[idx])\n", "# a.play(compress=False)\n", "# pos_a = decode(pos_arr)\n", "# print(\"\\n positive example \\n\", test_metas[pos_idx])\n", "# pos_a.play(compress=False)\n", "# print(\n", "# \"text:\",\n", "# test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", "# test_metas[idx].get(\"text\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939977Z", "start_time": "2024-05-16T13:59:41.939969Z" }, "execution": { "iopub.execute_input": "2025-07-01T13:28:22.664410Z", "iopub.status.busy": "2025-07-01T13:28:22.664273Z", "iopub.status.idle": "2025-07-01T13:28:22.675586Z", "shell.execute_reply": "2025-07-01T13:28:22.675163Z", "shell.execute_reply.started": "2025-07-01T13:28:22.664395Z" } }, "outputs": [], "source": [ "# val_df[val_df[\"tags\"] == 'a vibrant blend of experimental jazz fusion, drum-and-bass and swagger fuzzed-out guitars']" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.940610Z", "start_time": "2024-05-16T13:59:41.940603Z" }, "execution": { "iopub.execute_input": "2025-07-01T13:28:22.677333Z", "iopub.status.busy": "2025-07-01T13:28:22.677182Z", "iopub.status.idle": "2025-07-01T13:28:22.688353Z", "shell.execute_reply": "2025-07-01T13:28:22.687939Z", "shell.execute_reply.started": "2025-07-01T13:28:22.677319Z" } }, "outputs": [], "source": [ "# from collections import Counter\n", "# c = Counter()\n", "# for _, row in df_slice.iterrows():\n", "# # print(row[\"metadata\"])\n", "# for k in ast.literal_eval(row[\"metadata\"]).keys():\n", "# c[k] += 1" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941167Z", "start_time": "2024-05-16T13:59:41.941159Z" }, "execution": { "iopub.execute_input": "2025-07-01T13:28:22.691407Z", "iopub.status.busy": "2025-07-01T13:28:22.691286Z", "iopub.status.idle": "2025-07-01T13:28:22.702279Z", "shell.execute_reply": "2025-07-01T13:28:22.701863Z", "shell.execute_reply.started": "2025-07-01T13:28:22.691395Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941801Z", "start_time": "2024-05-16T13:59:41.941793Z" }, "execution": { "iopub.execute_input": "2025-07-01T13:28:22.702879Z", "iopub.status.busy": "2025-07-01T13:28:22.702739Z", "iopub.status.idle": "2025-07-01T13:28:22.717618Z", "shell.execute_reply": "2025-07-01T13:28:22.717167Z", "shell.execute_reply.started": "2025-07-01T13:28:22.702865Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3159 1\n" ] } ], "source": [ "def validation_on_metas(input_metas):\n", " total_bad = 0\n", " total_good = 0\n", " for idx in range(len(input_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if input_metas[idx].get(\"tags\") != input_metas[pos_idx].get(\"tags\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", " print(total_good, total_bad)\n", " return\n", "\n", "\n", "validation_on_metas(test_metas)" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.942520Z", "start_time": "2024-05-16T13:59:41.942511Z" }, "execution": { "iopub.execute_input": "2025-07-01T13:28:22.718427Z", "iopub.status.busy": "2025-07-01T13:28:22.718288Z", "iopub.status.idle": "2025-07-01T13:28:22.777412Z", "shell.execute_reply": "2025-07-01T13:28:22.776974Z", "shell.execute_reply.started": "2025-07-01T13:28:22.718414Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.943072Z", "start_time": "2024-05-16T13:59:41.943065Z" }, "execution": { "iopub.execute_input": "2025-07-01T13:28:22.778019Z", "iopub.status.busy": "2025-07-01T13:28:22.777883Z", "iopub.status.idle": "2025-07-01T13:28:22.811546Z", "shell.execute_reply": "2025-07-01T13:28:22.811107Z", "shell.execute_reply.started": "2025-07-01T13:28:22.778005Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)\n", "# make sure they are offset by 1 and exactly 1\n", "for i, j in zip(n_neg_tr, n_pos_tr):\n", " assert i == j - 1" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.944246Z", "start_time": "2024-05-16T13:59:41.944237Z" }, "execution": { "iopub.execute_input": "2025-07-01T13:28:22.812141Z", "iopub.status.busy": "2025-07-01T13:28:22.812001Z", "iopub.status.idle": "2025-07-01T13:28:22.823749Z", "shell.execute_reply": "2025-07-01T13:28:22.823304Z", "shell.execute_reply.started": "2025-07-01T13:28:22.812127Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total samples 625736 (625756, 176)\n" ] } ], "source": [ "total_iters = len(n_neg_tr) + len(n_pos_tr)\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945249Z", "start_time": "2024-05-16T13:59:41.945241Z" }, "execution": { "iopub.execute_input": "2025-07-01T13:28:22.824350Z", "iopub.status.busy": "2025-07-01T13:28:22.824216Z", "iopub.status.idle": "2025-07-01T13:28:22.836058Z", "shell.execute_reply": "2025-07-01T13:28:22.835605Z", "shell.execute_reply.started": "2025-07-01T13:28:22.824336Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 epoch per batch 4, total 9777.125\n" ] } ], "source": [ "print(\"1 epoch per batch 4, total\", total_iters / 8 / 2 / 4)" ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "execution": { "iopub.execute_input": "2025-07-05T13:46:24.373733Z", "iopub.status.busy": "2025-07-05T13:46:24.373430Z", "iopub.status.idle": "2025-07-05T14:46:24.494909Z", "shell.execute_reply": "2025-07-05T14:46:24.494340Z", "shell.execute_reply.started": "2025-07-05T13:46:24.373715Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Here\n" ] } ], "source": [ "import time\n", "print(\"Here\")\n", "time.sleep(3600 * 1)" ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945972Z", "start_time": "2024-05-16T13:59:41.945964Z" }, "execution": { "iopub.execute_input": "2025-07-05T14:46:24.495882Z", "iopub.status.busy": "2025-07-05T14:46:24.495727Z", "iopub.status.idle": "2025-07-05T14:46:25.821421Z", "shell.execute_reply": "2025-07-05T14:46:25.820781Z", "shell.execute_reply.started": "2025-07-05T14:46:24.495867Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Submitted batch job 7318\n" ] } ], "source": [ "!cd /home/tony/Work/tony/slurm/bluejay && sbatch sbatch_ipo_bluejay" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T13:28:22.851661Z", "iopub.status.busy": "2025-07-01T13:28:22.851525Z", "iopub.status.idle": "2025-07-01T13:28:22.873063Z", "shell.execute_reply": "2025-07-01T13:28:22.872616Z", "shell.execute_reply.started": "2025-07-01T13:28:22.851647Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cache kept!\n" ] } ], "source": [ "import shutil\n", "\n", "# Basic file copy\n", "shutil.copy(\n", " \"/home/tony/Work/tony/Preference/make_dataset_bluejay_r3_13b_30b_preauk.ipynb\",\n", " os.path.join(OUT_DATA_DIR, \"make_dataset.ipynb\"),\n", ")\n", "print(\"Cache kept!\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# some gymathtics loading prev data" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.946562Z", "start_time": "2024-05-16T13:59:41.946555Z" }, "execution": { "iopub.execute_input": "2025-07-01T13:28:22.873683Z", "iopub.status.busy": "2025-07-01T13:28:22.873543Z", "iopub.status.idle": "2025-07-01T13:28:22.884768Z", "shell.execute_reply": "2025-07-01T13:28:22.884338Z", "shell.execute_reply.started": "2025-07-01T13:28:22.873669Z" } }, "outputs": [], "source": [ "# prev_v3_data = \"/app/suno/data/dpo/7v_v20_full/\"\n", "\n", "# test_val_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_val.jsonl\"))\n", "# test_tr_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_tr.jsonl\"))\n", "\n", "# all_ids = set()\n", "# for meta in test_val_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# for meta in test_tr_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# print(len(all_ids), len(test_val_metas) + len(test_tr_metas))\n", "\n", "# all_ids = list(all_ids)\n", "# with open(\"/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id.json\", \"w\") as fp:\n", "# json.dump(all_ids, fp)" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T13:28:22.885516Z", "iopub.status.busy": "2025-07-01T13:28:22.885380Z", "iopub.status.idle": "2025-07-01T13:28:22.896790Z", "shell.execute_reply": "2025-07-01T13:28:22.896365Z", "shell.execute_reply.started": "2025-07-01T13:28:22.885503Z" } }, "outputs": [], "source": [ "# x_data = train_df[train_df[\"preference\"]][\"similarity\"]\n", "# y_data = train_df[~train_df[\"preference\"]][\"similarity\"]\n", "# from matplotlib.colors import LogNorm\n", "\n", "# fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 10))\n", "\n", "# # 2D Histogram\n", "# h = ax1.hist2d(\n", "# x_data,\n", "# y_data,\n", "# bins=(50, 50),\n", "# cmap=\"coolwarm\",\n", "# range=[[0, 1], [0, 1]],\n", "# norm=LogNorm(),\n", "# )\n", "\n", "# ax1.set_xlabel(\"Semantic Distance (Preferred)\")\n", "# ax1.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "# ax1.set_title(\n", "# \"2D Histogram of Semantic Distances: Preferred vs Non-Preferred (Log Scale)\"\n", "# )\n", "\n", "# cbar1 = plt.colorbar(h[3], ax=ax1)\n", "# cbar1.set_label(\"Number of Request IDs (Log Scale)\")\n", "\n", "# # Scatter plot\n", "# ax2.scatter(x_data, y_data, alpha=0.1, s=1)\n", "# ax2.set_xlabel(\"Semantic Distance (Preferred)\")\n", "# ax2.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "# ax2.set_title(\"Scatter Plot of Semantic Distances: Preferred vs Non-Preferred\")\n", "# ax2.set_xlim(0, 1)\n", "# ax2.set_ylim(0, 1)\n", "\n", "# plt.tight_layout()\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T13:28:22.897364Z", "iopub.status.busy": "2025-07-01T13:28:22.897233Z", "iopub.status.idle": "2025-07-01T13:28:22.908309Z", "shell.execute_reply": "2025-07-01T13:28:22.907889Z", "shell.execute_reply.started": "2025-07-01T13:28:22.897350Z" } }, "outputs": [], "source": [ "# train_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_tr.jsonl\"))" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "execution": { "iopub.execute_input": "2025-07-01T13:28:22.908909Z", "iopub.status.busy": "2025-07-01T13:28:22.908775Z", "iopub.status.idle": "2025-07-01T13:28:22.922258Z", "shell.execute_reply": "2025-07-01T13:28:22.921848Z", "shell.execute_reply.started": "2025-07-01T13:28:22.908896Z" } }, "outputs": [ { "data": { "text/plain": [ "dict_keys(['perference_0', 'perference_1'])" ] }, "execution_count": 37, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train_info.keys()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }