{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T01:55:41.833193Z", "iopub.status.busy": "2025-11-15T01:55:41.832864Z", "iopub.status.idle": "2025-11-15T01:55:41.847722Z", "shell.execute_reply": "2025-11-15T01:55:41.847313Z", "shell.execute_reply.started": "2025-11-15T01:55:41.833174Z" } }, "outputs": [], "source": [ "# setup autoload\n", "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.040680Z", "start_time": "2024-05-16T13:58:19.777010Z" }, "execution": { "iopub.execute_input": "2025-11-15T01:55:41.849315Z", "iopub.status.busy": "2025-11-15T01:55:41.849197Z", "iopub.status.idle": "2025-11-15T01:55:45.521826Z", "shell.execute_reply": "2025-11-15T01:55:45.521269Z", "shell.execute_reply.started": "2025-11-15T01:55:41.849302Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The autoreload extension is already loaded. To reload it, use:\n", " %reload_ext autoreload\n" ] } ], "source": [ "import ast\n", "import os\n", "import shutil\n", "import sys\n", "from collections import defaultdict\n", "\n", "import json\n", "import numpy as np\n", "import pandas as pd\n", "from preference_data_preparation_4rvq import *\n", "from preference_helper import *\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n", "from tqdm import tqdm\n", "\n", "pd.set_option(\"display.max_rows\", 500)\n", "pd.set_option(\"display.max_columns\", 500)\n", "pd.set_option(\"display.width\", 1000)\n", "\n", "# setup autoload\n", "%load_ext autoreload\n", "%autoreload 2\n", "\n", "\n", "def custom_parse(x):\n", " try:\n", " return json.loads(x)\n", " except:\n", " return {}" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.082172Z", "start_time": "2024-05-16T13:58:21.041926Z" }, "execution": { "iopub.execute_input": "2025-11-15T02:01:19.266111Z", "iopub.status.busy": "2025-11-15T02:01:19.265700Z", "iopub.status.idle": "2025-11-15T02:01:20.062090Z", "shell.execute_reply": "2025-11-15T02:01:20.061538Z", "shell.execute_reply.started": "2025-11-15T02:01:19.266091Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "N_TOKENS_AUDIO 12000\n" ] } ], "source": [ "OUT_DATA_DIR = \"/app2/suno/data/dpo/crow_t1_v55_4rvq\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "shutil.copyfile(\n", " \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", " os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", ")\n", "NPZ_DIR = \"/app2/suno/data/dpo/crow_t1_npz\"\n", "N_TOKENS_AUDIO = 25 * 8 * 60\n", "print(\"N_TOKENS_AUDIO\", N_TOKENS_AUDIO)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T01:55:45.589949Z", "iopub.status.busy": "2025-11-15T01:55:45.589827Z", "iopub.status.idle": "2025-11-15T01:58:17.020744Z", "shell.execute_reply": "2025-11-15T01:58:17.019968Z", "shell.execute_reply.started": "2025-11-15T01:55:45.589936Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "before dropna (2733682, 215)\n", "after dropna (2733682, 208)\n" ] } ], "source": [ "df = pd.read_pickle(\n", " \"/home/tony/Data/Preference/crow_t1/interesting_clips_crow_t1_20251030_full_slice.pkl\"\n", ")\n", "print(\"before dropna\", df.shape)\n", "df = df.dropna(axis=1, how=\"all\")\n", "print(\"after dropna\", df.shape)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T01:58:17.023245Z", "iopub.status.busy": "2025-11-15T01:58:17.023099Z", "iopub.status.idle": "2025-11-15T01:58:17.041230Z", "shell.execute_reply": "2025-11-15T01:58:17.040678Z", "shell.execute_reply.started": "2025-11-15T01:58:17.023229Z" } }, "outputs": [], "source": [ "df_total = df" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T02:01:05.147392Z", "iopub.status.busy": "2025-11-15T02:01:05.146966Z", "iopub.status.idle": "2025-11-15T02:01:12.493195Z", "shell.execute_reply": "2025-11-15T02:01:12.492549Z", "shell.execute_reply.started": "2025-11-15T02:01:05.147368Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(2733682, 208) (566242, 208)\n", "after date cut (566242, 208)\n" ] } ], "source": [ "# # v3 # 09-23 -- 09-26 217218\n", "# # v4 # 09-26 -- 09-28 166358\n", "# # v5 # 09-28 -- 10-01 156902\n", "# # \n", "# # v6 # 09-23 -- 09-26 232690\n", "# # v7 # 09-26 -- 09-28 184956\n", "# # v8 # 09-28 -- 10-01 231470\n", "# # \n", "# # v9 (v6, v7, v8)\n", "# # v10 # 09-23 -- 09-26 217344\n", "# # v11 # 09-26 -- 09-28 167036\n", "# # v12 # 09-28 -- 10-01 157570\n", "# # \n", "# # data 10-06\n", "# # v13 # 09-23 -- 09-26 251008\n", "# # v14 # 09-26 -- 09-28 205040\n", "# # v15 # 09-28 -- 10-01 274238\n", "# # v16 # 10-01 -- 10-04 246298\n", "# # \n", "# # data 10-09\n", "# # v13 # 09-23 -- 09-26 260700\n", "# # v14 # 09-26 -- 09-28 213860\n", "# # v15 # 09-28 -- 10-01 289968\n", "# # v16 # 10-01 -- 10-04 277330\n", "# # \n", "# # data 10-11\n", "# # v17 # 09-23 -- 09-26 254828\n", "# # v18 # 09-26 -- 09-29 304158\n", "# # v19 # 09-29 -- 10-02 281620\n", "# # v20 # 10-02 -- 10-05 274700\n", "# # \n", "# # data 10-16\n", "# # v21 # 09-23 -- 09-26 266992\n", "# # v22 # 09-26 -- 09-29 320004\n", "# # v23 # 09-29 -- 10-02 299798\n", "# # v24 # 10-02 -- 10-05 299228\n", "# # v25 # 10-05 -- 10-08 283690\n", "# # v26 # 10-08 -- 10-11 277962\n", "# # v27 # 10-11 -- 10-15 320808\n", "# # \n", "# # data 10-25\n", "# # v31 # 09-23 -- 09-26 200286\n", "# # v32 # 09-26 -- 09-29 242158\n", "# # v33 # 09-29 -- 10-02 230224\n", "# # v34 # 10-02 -- 10-05 232714\n", "# # v35 # 10-05 -- 10-11 450570\n", "# # v36 # 10-11 -- 10-17 438742\n", "# # v37 # 10-17 -- 10-26 446970\n", "# # \n", "# # data 10-25\n", "# # v41 # 09-23 -- 09-26 135520\n", "# # v42 # 09-26 -- 09-29 162702\n", "# # v43 # 09-29 -- 10-02 152880\n", "# # v44 # 10-02 -- 10-05 153712\n", "# # v45 # 10-05 -- 10-11 292718\n", "# # v46 # 10-11 -- 10-17 277006\n", "# # v47 # 10-17 -- 10-26 253528\n", "# # \n", "# # data 10-30\n", "# # v51 # 09-23 -- 09-26 209870\n", "# # v52 # 09-26 -- 09-29 254476\n", "# # v53 # 09-29 -- 10-02 242088\n", "# # v54 # 10-02 -- 10-05 245902\n", "# # v55 # 10-05 -- 10-12 566242\n", "# # v56 # 10-12 -- 10-20 625112 \n", "# # v57 # 10-20 -- 10-31 589974\n", "# # \n", "# # data 11-04 (1st full)\n", "# # v58 # 09-23 -- 10-26 462146 \n", "# # v59 # 10-12 -- 10-31 589974\n", "# sth is wrong\n", "# # v61 # 09-23 -- 09-29 454492\n", "# # v62 # 09-29 -- 10-05 474278\n", "# # v63 # 10-05 -- 10-12 546182\n", "# # v64 # 10-12 -- 10-20 595196 \n", "# # v65 # 10-20 -- 10-28 489446\n", "# # v66 # 10-28 -- 11-05 234182 \n", "df_total[\"created_at\"] = pd.to_datetime(df_total[\"created_at\"], utc=True)\n", "start_cutoff_date = pd.to_datetime(\"2025-10-05\", utc=True)\n", "end_cutoff_date = pd.to_datetime(\"2025-10-12\", utc=True)\n", "date_mask = (df_total[\"created_at\"] < end_cutoff_date) & (df_total[\"created_at\"] >= start_cutoff_date)\n", "print(df_total.shape, df_total[date_mask].shape)\n", "df_slice = df_total[date_mask].copy()\n", "print(\"after date cut\", df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T14:00:20.866354Z", "start_time": "2024-05-16T14:00:12.443344Z" }, "execution": { "iopub.execute_input": "2025-11-15T02:01:12.901618Z", "iopub.status.busy": "2025-11-15T02:01:12.901424Z", "iopub.status.idle": "2025-11-15T02:01:13.084767Z", "shell.execute_reply": "2025-11-15T02:01:13.084163Z", "shell.execute_reply.started": "2025-11-15T02:01:12.901601Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(566242, 208)\n", "task\n", " 338528\n", "cover 131758\n", "artist_consistency 38932\n", "artist_cover 20928\n", "extend 13260\n", "infill 6078\n", "playlist_condition 5326\n", "overpainting 2706\n", "artist_extend 2630\n", "stem_condition 2354\n", "upload_extend 2040\n", "underpainting 1702\n", "Name: count, dtype: int64\n" ] }, { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[10], line 6\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mvalue_counts())\n\u001b[0;32m----> 6\u001b[0m \u001b[43mBREAK\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m# (269282, 199)\u001b[39;00m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "# df_total.to_pickle(\n", "# \"/home/tony/Data/Preference/crow_t1/interesting_clips_crow_t1_20251030_full_slice.pkl\"\n", "# )\n", "print(df_slice.shape)\n", "print(df_slice[\"task\"].value_counts())\n", "BREAK\n", "# (269282, 199)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T02:01:24.454583Z", "iopub.status.busy": "2025-11-15T02:01:24.454167Z", "iopub.status.idle": "2025-11-15T02:01:26.251999Z", "shell.execute_reply": "2025-11-15T02:01:26.251385Z", "shell.execute_reply.started": "2025-11-15T02:01:24.454563Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(566242, 208)\n", "(566242, 208)\n", "(566242, 208)\n" ] } ], "source": [ "print(df_slice.shape)\n", "df_slice = df_slice[df_slice[\"request_id\"].apply(lambda x: len(x) > 3)]\n", "print(df_slice.shape)\n", "# df_slice = df_slice[df_slice[\"is_pro_user\"]].copy()\n", "print(df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932296Z", "start_time": "2024-05-16T13:59:41.932287Z" }, "execution": { "iopub.execute_input": "2025-11-15T02:01:26.253140Z", "iopub.status.busy": "2025-11-15T02:01:26.252965Z", "iopub.status.idle": "2025-11-15T02:01:26.296369Z", "shell.execute_reply": "2025-11-15T02:01:26.295804Z", "shell.execute_reply.started": "2025-11-15T02:01:26.253123Z" } }, "outputs": [], "source": [ "# don't have continue at\n", "df_slice[\"request_id\"] = df_slice[\"request_id\"].astype(str)\n", "# df_slice[\"npz_path\"] = df_slice[\"npz_path\"].apply(lambda x: str(x).replace(\"_npz\", \"_npz/\"))\n", "# df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique(), df_slice[\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932966Z", "start_time": "2024-05-16T13:59:41.932957Z" }, "execution": { "iopub.execute_input": "2025-11-15T02:01:26.297111Z", "iopub.status.busy": "2025-11-15T02:01:26.296963Z", "iopub.status.idle": "2025-11-15T02:01:26.402759Z", "shell.execute_reply": "2025-11-15T02:01:26.402171Z", "shell.execute_reply.started": "2025-11-15T02:01:26.297096Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "283121\n" ] } ], "source": [ "final_filtered_requests = df_slice[\"request_id\"].astype(str).unique()\n", "# final_filtered_requests = df_slice[df_slice[\"is_pro_user\"]][\"request_id\"].astype(str).unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934277Z", "start_time": "2024-05-16T13:59:41.934268Z" }, "execution": { "iopub.execute_input": "2025-11-15T02:01:26.403525Z", "iopub.status.busy": "2025-11-15T02:01:26.403371Z", "iopub.status.idle": "2025-11-15T02:01:37.166728Z", "shell.execute_reply": "2025-11-15T02:01:37.166072Z", "shell.execute_reply.started": "2025-11-15T02:01:26.403509Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "280289 2832\n", "(560578, 208) (5664, 208)\n" ] } ], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df # .reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df # .reset_index()\n", "train_df = train_df.reset_index(drop=True)\n", "val_df = val_df.reset_index(drop=True)\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.935620Z", "start_time": "2024-05-16T13:59:41.935613Z" }, "execution": { "iopub.execute_input": "2025-11-15T02:01:37.168210Z", "iopub.status.busy": "2025-11-15T02:01:37.168035Z", "iopub.status.idle": "2025-11-15T02:02:09.491532Z", "shell.execute_reply": "2025-11-15T02:02:09.490958Z", "shell.execute_reply.started": "2025-11-15T02:01:37.168192Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 560578/560578 [00:32<00:00, 17354.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "28,259 hours of 560578 clips, 35.036125 nodes, 729.9192708333334 iters\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "total_duration = 0\n", "for i, row in tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " total_duration += row[\"duration\"]\n", " except Exception as E:\n", " print(i, row)\n", " print(E)\n", " raise ValueError()\n", "\n", "print(\n", " f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1000} nodes, {train_df.shape[0] / 8 / 6 / 16} iters\"\n", ")" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936268Z", "start_time": "2024-05-16T13:59:41.936260Z" }, "execution": { "iopub.execute_input": "2025-11-15T02:02:09.492306Z", "iopub.status.busy": "2025-11-15T02:02:09.492152Z", "iopub.status.idle": "2025-11-15T02:04:28.473108Z", "shell.execute_reply": "2025-11-15T02:04:28.472561Z", "shell.execute_reply.started": "2025-11-15T02:02:09.492290Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "t_data_memmap is set to: 12000\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 5655/5664 [02:18<00:00, 43.35it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "5646, Failed to load conditioning codes for stem from 3a09d334-0dd3-4601-91b4-148d1e117c07: Semantic codes not found: /app2/suno/data/semantic_code/crow/3a09d334-0dd3-4601-91b4-148d1e117c07.npz, stem_condition, /app2/suno/data/semantic_code/crow/c82c102c-b6bb-4acd-b61a-0ead2cd8983b.npz.\n", "WTF --> 5647, skip, preference: True, 7816deb0-b0d8-46ac-8f10-55a7090d546c, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5664/5664 [02:18<00:00, 40.77it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 5662 clips, 0 different prompts, 0 different tags, 0 different negative tags\n", "191 hours of False\n", "192 hours of True\n", "cover: 124.7 hours\n", "gen: 174.9 hours\n", "extend: 7.9 hours\n", "infill: 1.2 hours\n", "artist_consistency: 41.5 hours\n", "artist_cover: 22.9 hours\n", "playlist_condition: 4.1 hours\n", "stem_condition: 2.1 hours\n", "overpainting: 1.9 hours\n", "artist_extend: 1.4 hours\n", "underpainting: 1.3 hours\n", "🚨 Error stem_condition: 1\n", "\n", "--- Gender Distribution ---\n", " female: 468 (8.3%)\n", " male: 834 (14.7%)\n", " unspecified: 4,361 (77.0%)\n", "\n", "--- Negative Tags Usage ---\n", " has_neg_tags: 330 (5.8%)\n", " no_neg_tags: 5,333 (94.2%)\n", "\n", "--- Control Slider Usage ---\n", " has_control_slider: 1,786 (31.5% of clips)\n", " no_control_slider: 3,877 (68.5% of clips)\n", "Done\n" ] } ], "source": [ "make_dataset(\n", " val_df, OUT_DATA_DIR, is_val=True, t_data_memmap=N_TOKENS_AUDIO\n", ")" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T02:04:28.473817Z", "iopub.status.busy": "2025-11-15T02:04:28.473667Z", "iopub.status.idle": "2025-11-15T02:04:28.491300Z", "shell.execute_reply": "2025-11-15T02:04:28.490848Z", "shell.execute_reply.started": "2025-11-15T02:04:28.473802Z" } }, "outputs": [], "source": [ "# test_npz = np.load(\"/app/suno/data/dpo/30b_npz/26d19085-18da-4701-af43-122684543891.npz\")\n", "# for k in test_npz.keys():\n", "# print(k)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936964Z", "start_time": "2024-05-16T13:59:41.936957Z" }, "execution": { "iopub.execute_input": "2025-11-15T02:04:28.491955Z", "iopub.status.busy": "2025-11-15T02:04:28.491813Z", "iopub.status.idle": "2025-11-15T06:00:24.312052Z", "shell.execute_reply": "2025-11-15T06:00:24.311439Z", "shell.execute_reply.started": "2025-11-15T02:04:28.491941Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "t_data_memmap is set to: 12000\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 4%|███▉ | 20902/560578 [08:36<3:21:09, 44.71it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "20894, Semantic codes not found: /app2/suno/data/semantic_code/crow/m_181658cb-8374-4e61-b297-34ccd1e53d8c.npz, upload_extend, /app2/suno/data/semantic_code/crow/4f0999c2-d4a5-439b-aca3-fb8fd5bb9145.npz.\n", "WTF --> 20895, skip, preference: True, 6f238ea7-3057-4729-8a3a-48fa617ad0a7, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 27%|██████████████████████████▊ | 148885/560578 [1:01:42<2:34:43, 44.34it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "148878, Failed to load conditioning codes for stem from 02973ef0-5848-41e6-a097-adce5437d997: Semantic codes not found: /app2/suno/data/semantic_code/crow/02973ef0-5848-41e6-a097-adce5437d997.npz, stem_condition, /app2/suno/data/semantic_code/crow/c7f52f0c-e840-4443-acd7-a9fc69bd73b8.npz.\n", "WTF --> 148879, skip, preference: True, e88ed516-9a88-4d39-91dc-aa6bb6c92aaa, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 29%|█████████████████████████████ | 161156/560578 [1:06:43<2:28:30, 44.83it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "161146, Failed to load conditioning codes for stem from 0112f71f-e443-4cc7-994a-ee73c0a90479: Semantic codes not found: /app2/suno/data/semantic_code/crow/0112f71f-e443-4cc7-994a-ee73c0a90479.npz, stem_condition, /app2/suno/data/semantic_code/crow/6ee92797-f48a-45e3-9818-b829c30d5ae1.npz.\n", "WTF --> 161147, skip, preference: True, 5112dc9f-77c9-4e88-a1ff-f23c8722858f, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 29%|█████████████████████████████▌ | 163753/560578 [1:07:47<2:18:56, 47.60it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "163744, Failed to load conditioning codes for stem from 05095755-66f4-4ce1-84b4-b9ca4e3acfa0: Semantic codes not found: /app2/suno/data/semantic_code/crow/05095755-66f4-4ce1-84b4-b9ca4e3acfa0.npz, stem_condition, /app2/suno/data/semantic_code/crow/42422090-a10e-49fd-a086-b5b9353e35b8.npz.\n", "WTF --> 163745, skip, preference: True, b392aa3e-a3f1-4888-bf52-9d5bf35d8352, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 30%|██████████████████████████████▏ | 167356/560578 [1:09:14<2:34:55, 42.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "167348, Failed to load conditioning codes for stem from 0081dc64-8e6b-4b2b-bd3c-451c474e5bf7: Semantic codes not found: /app2/suno/data/semantic_code/crow/0081dc64-8e6b-4b2b-bd3c-451c474e5bf7.npz, stem_condition, /app2/suno/data/semantic_code/crow/64611445-ce39-4ac1-81ae-cedc3ba7bd55.npz.\n", "WTF --> 167349, skip, preference: True, 21fd1a3c-9af7-44b0-86dd-3a1391d02ea7, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 31%|███████████████████████████████▏ | 173206/560578 [1:11:34<2:34:00, 41.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "173198, Failed to load conditioning codes for stem from 028c9002-eefb-4286-9d06-7315679003ae: Semantic codes not found: /app2/suno/data/semantic_code/crow/028c9002-eefb-4286-9d06-7315679003ae.npz, stem_condition, /app2/suno/data/semantic_code/crow/55bfafe8-ffb4-4b11-85a4-282e0426c688.npz.\n", "WTF --> 173199, skip, preference: True, ac721b98-472a-4216-abc5-dbbd3eb5154a, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 31%|███████████████████████████████▋ | 175554/560578 [1:12:31<2:36:35, 40.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "175548, Semantic codes not found: /app2/suno/data/semantic_code/crow/m_5de6c7fe-5032-46c0-82c7-a4dfa359164d.npz, upload_extend, /app2/suno/data/semantic_code/crow/ba740eb7-24b6-42f1-aba6-49cfa2725d27.npz.\n", "WTF --> 175549, skip, preference: True, 16118feb-200a-4649-92ee-62b55573cfde, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 33%|█████████████████████████████████▏ | 183971/560578 [1:15:52<2:27:35, 42.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Overflow at 183966: 13095 > 12000\n", "Overflow at 183967: 13095 > 12000\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 36%|███████████████████████████████████▊ | 199070/560578 [1:21:56<2:10:36, 46.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "199064, Semantic codes not found: /app2/suno/data/semantic_code/crow/m_8ba90bfe-2338-45e8-9770-4a5e3cb3e06e.npz, upload_extend, /app2/suno/data/semantic_code/crow/0d809c22-57a3-4b10-b0a1-b50f89091a4a.npz.\n", "WTF --> 199065, skip, preference: True, 55f0e351-0c11-4167-8d37-1a16172ce86f, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 36%|████████████████████████████████████ | 200249/560578 [1:22:25<2:15:34, 44.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "200240, Failed to load conditioning codes for stem from 0b47f36e-f2e8-4dd1-908b-9075257f34dd: Semantic codes not found: /app2/suno/data/semantic_code/crow/0b47f36e-f2e8-4dd1-908b-9075257f34dd.npz, stem_condition, /app2/suno/data/semantic_code/crow/bd4b8fe6-a0e9-40d8-a4e4-bd236dc003be.npz.\n", "WTF --> 200241, skip, preference: True, 3bee3347-b2d0-451d-97b5-07a09c896d88, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 36%|████████████████████████████████████▎ | 201695/560578 [1:23:00<1:57:36, 50.86it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "201692, Failed to load conditioning codes for stem from 0be02759-18dc-4fb5-810d-880c7878ae89: Semantic codes not found: /app2/suno/data/semantic_code/crow/0be02759-18dc-4fb5-810d-880c7878ae89.npz, stem_condition, /app2/suno/data/semantic_code/crow/59d49ab7-664e-4909-839c-ae2cedbc88ea.npz.\n", "WTF --> 201693, skip, preference: True, da303d82-1049-40eb-a866-308441333864, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 39%|███████████████████████████████████████▋ | 220576/560578 [1:32:31<2:18:29, 40.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "220568, Failed to load conditioning codes for stem from 0d79978b-0d17-482d-96c1-947850db41a4: Semantic codes not found: /app2/suno/data/semantic_code/crow/0d79978b-0d17-482d-96c1-947850db41a4.npz, stem_condition, /app2/suno/data/semantic_code/crow/ce06086b-ee44-4b6e-8b08-893430f2a6b6.npz.\n", "WTF --> 220569, skip, preference: True, 5f95544c-4be9-4d0d-95d1-b421631e6e22, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 40%|████████████████████████████████████████▏ | 223281/560578 [1:33:41<1:59:13, 47.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "223272, Failed to load conditioning codes for stem from 0f8524b0-1c3e-4b82-97b0-cb54c5cebae3: Semantic codes not found: /app2/suno/data/semantic_code/crow/0f8524b0-1c3e-4b82-97b0-cb54c5cebae3.npz, stem_condition, /app2/suno/data/semantic_code/crow/3073134b-5b28-420d-8399-cd4171559793.npz.\n", "WTF --> 223273, skip, preference: True, 520679ac-db3c-4faf-99fa-db12590f12c5, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 42%|██████████████████████████████████████████▏ | 233892/560578 [1:38:25<1:57:08, 46.48it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "233886, Failed to load conditioning codes for stem from 0161b525-595b-49bc-b65a-47ddbe468bcd: Semantic codes not found: /app2/suno/data/semantic_code/crow/0161b525-595b-49bc-b65a-47ddbe468bcd.npz, stem_condition, /app2/suno/data/semantic_code/crow/08b98d1c-2ed3-4156-9c50-f26d234ebedf.npz.\n", "WTF --> 233887, skip, preference: True, 385a9155-43a4-4a07-a818-29adfa9fa15f, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 42%|██████████████████████████████████████████▌ | 235942/560578 [1:39:14<2:00:01, 45.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "235934, Failed to load conditioning codes for stem from 0dd59e3b-ceb7-4fc3-a315-9ccd34772cac: Semantic codes not found: /app2/suno/data/semantic_code/crow/0dd59e3b-ceb7-4fc3-a315-9ccd34772cac.npz, stem_condition, /app2/suno/data/semantic_code/crow/0cca87bb-8794-4307-829c-6af5f517d0db.npz.\n", "WTF --> 235935, skip, preference: True, b4f94021-c940-4685-9f18-02a990bf0506, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 43%|███████████████████████████████████████████▎ | 240565/560578 [1:41:04<1:45:44, 50.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "240560, Failed to load conditioning codes for stem from 05dc1c38-6715-464c-bba2-8ea5ee4ad906: Semantic codes not found: /app2/suno/data/semantic_code/crow/05dc1c38-6715-464c-bba2-8ea5ee4ad906.npz, stem_condition, /app2/suno/data/semantic_code/crow/724bcf7d-6dd0-4dc2-8c4d-acf8fd0fe2ce.npz.\n", "WTF --> 240561, skip, preference: True, c2e532e4-5b6c-4135-bb77-eaede30e04bd, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 44%|███████████████████████████████████████████▉ | 244202/560578 [1:42:31<1:51:24, 47.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "244192, Failed to load conditioning codes for stem from 0f6f8c0e-0151-4027-b42a-c9415ce6663e: Semantic codes not found: /app2/suno/data/semantic_code/crow/0f6f8c0e-0151-4027-b42a-c9415ce6663e.npz, stem_condition, /app2/suno/data/semantic_code/crow/3c43755e-e588-41b1-83c1-c210c2ce4524.npz.\n", "WTF --> 244193, skip, preference: True, bf9ae6ff-7385-4023-8dea-403a4a2c672c, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 44%|████████████████████████████████████████████▍ | 246711/560578 [1:43:31<1:48:06, 48.39it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "246700, Failed to load conditioning codes for stem from 1368138b-12b3-4def-98a8-2e99bcae0534: Semantic codes not found: /app2/suno/data/semantic_code/crow/1368138b-12b3-4def-98a8-2e99bcae0534.npz, stem_condition, /app2/suno/data/semantic_code/crow/d64d6687-9873-4b67-bd9c-2de1ee1af62f.npz.\n", "WTF --> 246701, skip, preference: True, 3a2638bf-8280-4745-8b5d-25f4c9515c71, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 46%|██████████████████████████████████████████████▊ | 259980/560578 [1:48:51<1:49:28, 45.76it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "259970, Failed to load conditioning codes for stem from 12a75aca-b707-4e21-8244-b65fedb99d44: Semantic codes not found: /app2/suno/data/semantic_code/crow/12a75aca-b707-4e21-8244-b65fedb99d44.npz, stem_condition, /app2/suno/data/semantic_code/crow/95940130-c86b-4da2-a6db-fbf3251a51a2.npz.\n", "WTF --> 259971, skip, preference: True, 36369bf7-7ad1-4fe5-8fe7-6363e0f9993e, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 48%|████████████████████████████████████████████████▏ | 267687/560578 [1:51:59<1:48:05, 45.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "267682, Failed to load conditioning codes for stem from 02aab4ef-7aed-4c0e-9bbe-c538c4775132: Semantic codes not found: /app2/suno/data/semantic_code/crow/02aab4ef-7aed-4c0e-9bbe-c538c4775132.npz, stem_condition, /app2/suno/data/semantic_code/crow/f371e361-076c-465f-8528-e08926de7153.npz.\n", "WTF --> 267683, skip, preference: True, 6aab97e1-6034-4627-b32f-af05ecd2cfd2, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 48%|████████████████████████████████████████████████▉ | 271809/560578 [1:53:38<1:49:29, 43.96it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "271800, Failed to load conditioning codes for stem from 074afcfc-05a9-414f-85a4-a14f168c6503: Semantic codes not found: /app2/suno/data/semantic_code/crow/074afcfc-05a9-414f-85a4-a14f168c6503.npz, stem_condition, /app2/suno/data/semantic_code/crow/a7895819-b7f5-463a-8f1b-24117c5f9338.npz.\n", "WTF --> 271801, skip, preference: True, d9d929a6-47bf-4213-b582-0c1cad97631a, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 49%|█████████████████████████████████████████████████▍ | 274047/560578 [1:54:33<1:44:35, 45.66it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "274042, Failed to load conditioning codes for stem from 05917a83-0fa2-430a-8a60-b552e774c20c: Semantic codes not found: /app2/suno/data/semantic_code/crow/05917a83-0fa2-430a-8a60-b552e774c20c.npz, stem_condition, /app2/suno/data/semantic_code/crow/eb9f5fe8-6586-4dc7-8894-d795b0bdcb84.npz.\n", "WTF --> 274043, skip, preference: True, a2000b03-6c60-4e2b-bc84-69436e00b27b, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 52%|████████████████████████████████████████████████████▍ | 291040/560578 [2:01:44<1:43:19, 43.47it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "291034, Failed to load conditioning codes for stem from 05eea717-1df8-4c9c-9d67-6664083d48b8: Semantic codes not found: /app2/suno/data/semantic_code/crow/05eea717-1df8-4c9c-9d67-6664083d48b8.npz, stem_condition, /app2/suno/data/semantic_code/crow/b1e139be-3ab0-4c5d-96ef-1faabbcdceeb.npz.\n", "WTF --> 291035, skip, preference: True, 1d8f309e-0085-46fd-b6b1-dea0a418ce30, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 53%|█████████████████████████████████████████████████████▍ | 296831/560578 [2:04:38<1:35:16, 46.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "296826, Failed to load conditioning codes for stem from 0b3ce30e-30e1-439d-b132-6485d0a06063: Semantic codes not found: /app2/suno/data/semantic_code/crow/0b3ce30e-30e1-439d-b132-6485d0a06063.npz, stem_condition, /app2/suno/data/semantic_code/crow/97f64d87-9653-4f2b-a7b0-acc98077a6a3.npz.\n", "WTF --> 296827, skip, preference: True, 35b85368-cf12-4684-84e1-c4a74007c1a1, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 54%|██████████████████████████████████████████████████████▍ | 302228/560578 [2:06:49<1:35:53, 44.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "302222, Failed to load conditioning codes for stem from 0bc5855d-b71c-49cf-b7c2-5a5aa4e6f915: Semantic codes not found: /app2/suno/data/semantic_code/crow/0bc5855d-b71c-49cf-b7c2-5a5aa4e6f915.npz, stem_condition, /app2/suno/data/semantic_code/crow/8f396b02-437c-43a4-adaf-dfcf98b7ab27.npz.\n", "WTF --> 302223, skip, preference: True, ec4bbe33-dee4-48a0-aed5-2b13fa229e8d, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 55%|███████████████████████████████████████████████████████▍ | 307684/560578 [2:09:01<1:26:48, 48.56it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "307674, Failed to load conditioning codes for stem from 12497d06-9bc3-4533-a2a4-bbdcf89322d8: Semantic codes not found: /app2/suno/data/semantic_code/crow/12497d06-9bc3-4533-a2a4-bbdcf89322d8.npz, stem_condition, /app2/suno/data/semantic_code/crow/e727f75a-016b-45a5-898b-8f8c24cec157.npz.\n", "WTF --> 307675, skip, preference: True, 116b105e-dba7-4c72-929a-098a0eed1b6b, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 55%|███████████████████████████████████████████████████████▌ | 308323/560578 [2:09:17<1:32:36, 45.40it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "308314, Failed to load conditioning codes for stem from 0080925a-bf20-44c5-91de-4b5e21bdffba: Semantic codes not found: /app2/suno/data/semantic_code/crow/0080925a-bf20-44c5-91de-4b5e21bdffba.npz, stem_condition, /app2/suno/data/semantic_code/crow/fdb5a4b4-ccfd-4b54-9fea-07e70b3ea0a6.npz.\n", "WTF --> 308315, skip, preference: True, 31be6954-7c12-4a65-9c01-02f5d2c26076, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 56%|████████████████████████████████████████████████████████▏ | 311964/560578 [2:10:44<1:27:05, 47.58it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "311958, Failed to load conditioning codes for stem from 0a112c35-0967-4f22-a6c9-3e5959ed6d0b: Semantic codes not found: /app2/suno/data/semantic_code/crow/0a112c35-0967-4f22-a6c9-3e5959ed6d0b.npz, stem_condition, /app2/suno/data/semantic_code/crow/66e8f58f-aaaf-4f0f-9d66-2d94daba3c62.npz.\n", "WTF --> 311959, skip, preference: True, c5a07adc-0c1c-417c-a65e-814c5841778b, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 57%|█████████████████████████████████████████████████████████▎ | 318143/560578 [2:13:16<1:36:49, 41.73it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "318138, Failed to load conditioning codes for stem from 20d1ecbf-f1bc-4535-82b5-6bd3295c5c88: Semantic codes not found: /app2/suno/data/semantic_code/crow/20d1ecbf-f1bc-4535-82b5-6bd3295c5c88.npz, stem_condition, /app2/suno/data/semantic_code/crow/078bd478-faed-406f-bff3-55aa3f0d2cf1.npz.\n", "WTF --> 318139, skip, preference: True, 9416361c-a766-46ff-a48c-bbb69214fa09, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 57%|█████████████████████████████████████████████████████████▍ | 318676/560578 [2:13:30<1:33:28, 43.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "318668, Failed to load conditioning codes for stem from 1431c9e3-fcd9-41a9-a1c1-16230afbc980: Semantic codes not found: /app2/suno/data/semantic_code/crow/1431c9e3-fcd9-41a9-a1c1-16230afbc980.npz, stem_condition, /app2/suno/data/semantic_code/crow/83673f45-bb72-4c10-854f-8bcfbd31b2cf.npz.\n", "WTF --> 318669, skip, preference: True, 8ea3f5ea-8fdc-45c7-921a-1021c219bfc1, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 58%|██████████████████████████████████████████████████████████▉ | 326870/560578 [2:16:52<1:29:10, 43.68it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "326864, Failed to load conditioning codes for stem from 1e80c203-1000-42f1-a953-a4264dd2a4fc: Semantic codes not found: /app2/suno/data/semantic_code/crow/1e80c203-1000-42f1-a953-a4264dd2a4fc.npz, stem_condition, /app2/suno/data/semantic_code/crow/998cdbc5-c468-439a-ac82-d9665dd6ad29.npz.\n", "WTF --> 326865, skip, preference: True, 695cc1ca-608a-4673-8bd5-f62d6bfa4de0, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 59%|███████████████████████████████████████████████████████████▏ | 328223/560578 [2:17:24<1:23:35, 46.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "328216, Failed to load conditioning codes for stem from 0cb7626c-14d8-40ea-baa7-63ba47460016: Semantic codes not found: /app2/suno/data/semantic_code/crow/0cb7626c-14d8-40ea-baa7-63ba47460016.npz, stem_condition, /app2/suno/data/semantic_code/crow/5fea82c0-243c-4548-9dab-349b03528517.npz.\n", "WTF --> 328217, skip, preference: True, 33d6153e-5177-4e18-a684-93b92dd7c322, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 59%|███████████████████████████████████████████████████████████▏ | 328482/560578 [2:17:30<1:29:15, 43.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "328474, Failed to load conditioning codes for stem from 008e2a92-2d02-41b2-9014-eef14ceb7141: Semantic codes not found: /app2/suno/data/semantic_code/crow/008e2a92-2d02-41b2-9014-eef14ceb7141.npz, stem_condition, /app2/suno/data/semantic_code/crow/64897c6a-1603-4b4e-a2c2-71b73381901f.npz.\n", "WTF --> 328475, skip, preference: True, 0cbac77f-a435-4234-a8e4-ba8cadc3de3b, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 59%|████████████████████████████████████████████████████████████ | 333499/560578 [2:19:32<1:31:19, 41.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Overflow at 333492: 12698 > 12000\n", "Overflow at 333493: 12698 > 12000\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 60%|████████████████████████████████████████████████████████████▉ | 337928/560578 [2:21:25<1:27:16, 42.52it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "337918, Failed to load conditioning codes for stem from 0d7068be-e739-4dde-a22b-6222bc843124: Semantic codes not found: /app2/suno/data/semantic_code/crow/0d7068be-e739-4dde-a22b-6222bc843124.npz, stem_condition, /app2/suno/data/semantic_code/crow/f05b5021-2f2f-4d91-b9f7-26efdeb740f7.npz.\n", "WTF --> 337919, skip, preference: True, 6f9931f2-c114-4910-aded-69d84a7ec1b7, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 61%|█████████████████████████████████████████████████████████████▎ | 340165/560578 [2:22:20<1:21:27, 45.10it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "340158, Failed to load conditioning codes for stem from 0fc9f052-d6a3-47ab-aeaf-76826592d03d: Semantic codes not found: /app2/suno/data/semantic_code/crow/0fc9f052-d6a3-47ab-aeaf-76826592d03d.npz, stem_condition, /app2/suno/data/semantic_code/crow/ece58527-4f2d-44bb-ab4f-bd04d4b22855.npz.\n", "WTF --> 340159, skip, preference: True, 3898712d-08ee-4a0b-9e85-db5db56c739a, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 62%|██████████████████████████████████████████████████████████████▉ | 348984/560578 [2:25:55<1:10:41, 49.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "348980, Failed to load conditioning codes for stem from 0d2c15d0-7ea5-4b3f-ab44-990352bafb34: Semantic codes not found: /app2/suno/data/semantic_code/crow/0d2c15d0-7ea5-4b3f-ab44-990352bafb34.npz, stem_condition, /app2/suno/data/semantic_code/crow/b68a18e5-0c81-48c7-a9a2-200779550126.npz.\n", "WTF --> 348981, skip, preference: True, 201650ef-eb4e-4b27-87b1-892f69af9eba, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 63%|███████████████████████████████████████████████████████████████▌ | 352691/560578 [2:27:24<1:16:28, 45.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "352684, Failed to load conditioning codes for stem from 07d484b0-d417-49de-ba82-7174c938e6d9: Semantic codes not found: /app2/suno/data/semantic_code/crow/07d484b0-d417-49de-ba82-7174c938e6d9.npz, stem_condition, /app2/suno/data/semantic_code/crow/81ef5ab0-d419-4bf7-a7ab-ac6b28e1c410.npz.\n", "WTF --> 352685, skip, preference: True, 375aa63b-714b-4c0b-8851-d6067e855e1d, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 63%|███████████████████████████████████████████████████████████████▋ | 353179/560578 [2:27:36<1:16:22, 45.26it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "353170, Failed to load conditioning codes for stem from 0d822950-b6f9-46c8-87af-3a4f129b9155: Semantic codes not found: /app2/suno/data/semantic_code/crow/0d822950-b6f9-46c8-87af-3a4f129b9155.npz, stem_condition, /app2/suno/data/semantic_code/crow/f1a05c9b-aae0-4229-9f21-6a9171a3fb30.npz.\n", "WTF --> 353171, skip, preference: True, 7e46f435-c8fb-4ed6-bc7a-7d404455b330, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 63%|███████████████████████████████████████████████████████████████▊ | 354443/560578 [2:28:07<1:15:42, 45.38it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "354436, Failed to load conditioning codes for stem from 202b7489-ddf6-49f4-a094-d8b6bff7a978: Semantic codes not found: /app2/suno/data/semantic_code/crow/202b7489-ddf6-49f4-a094-d8b6bff7a978.npz, stem_condition, /app2/suno/data/semantic_code/crow/e4810443-dd6a-4581-b862-45a3824cc4ab.npz.\n", "WTF --> 354437, skip, preference: True, 45444626-e47c-4e06-ba7a-81957df2db36, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 63%|████████████████████████████████████████████████████████████████ | 355766/560578 [2:28:38<1:11:53, 47.48it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "355760, Failed to load conditioning codes for stem from 040d6441-15b8-409a-a606-286d906a2d6d: Semantic codes not found: /app2/suno/data/semantic_code/crow/040d6441-15b8-409a-a606-286d906a2d6d.npz, stem_condition, /app2/suno/data/semantic_code/crow/999b11af-96f2-4da8-a854-36a24902c6db.npz.\n", "WTF --> 355761, skip, preference: True, a4283747-94a3-4766-8d73-8b2f0d7ba22d, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 63%|████████████████████████████████████████████████████████████████▏ | 355955/560578 [2:28:43<1:12:44, 46.88it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "355946, Failed to load conditioning codes for stem from 10e7e6f8-9bb7-4f64-8af0-93b6e733a70a: Semantic codes not found: /app2/suno/data/semantic_code/crow/10e7e6f8-9bb7-4f64-8af0-93b6e733a70a.npz, stem_condition, /app2/suno/data/semantic_code/crow/f6a8a700-cb76-4f85-965f-24c6ec9109e3.npz.\n", "WTF --> 355947, skip, preference: True, d9fc0a37-376c-4df0-b0d5-57c93c78f077, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 64%|████████████████████████████████████████████████████████████████▏ | 356375/560578 [2:28:53<1:23:44, 40.64it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "356370, Failed to load conditioning codes for stem from 05af8045-5c62-4601-a0a1-7b0550983646: Semantic codes not found: /app2/suno/data/semantic_code/crow/05af8045-5c62-4601-a0a1-7b0550983646.npz, stem_condition, /app2/suno/data/semantic_code/crow/4257bf8a-cf76-4375-938b-125ab02a0839.npz.\n", "WTF --> 356371, skip, preference: True, 71b4e262-d367-4598-8b48-f98f4d2161cd, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 64%|████████████████████████████████████████████████████████████████▉ | 360172/560578 [2:30:25<1:17:10, 43.28it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "360162, Failed to load conditioning codes for stem from 0fcedf6d-a392-4892-a1d2-ead322b81169: Semantic codes not found: /app2/suno/data/semantic_code/crow/0fcedf6d-a392-4892-a1d2-ead322b81169.npz, stem_condition, /app2/suno/data/semantic_code/crow/0abc5dc1-c7d4-4c31-850d-d0bc9a970d1e.npz.\n", "WTF --> 360163, skip, preference: True, d8ae92d9-201c-4288-a714-0595abf496a9, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 65%|█████████████████████████████████████████████████████████████████▎ | 362549/560578 [2:31:24<1:16:53, 42.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "362540, Failed to load conditioning codes for stem from 2215ea27-4699-48a7-bf62-4081ef3ca7fd: Semantic codes not found: /app2/suno/data/semantic_code/crow/2215ea27-4699-48a7-bf62-4081ef3ca7fd.npz, stem_condition, /app2/suno/data/semantic_code/crow/48007aeb-1aa6-413c-a2fa-aef590dc57e3.npz.\n", "WTF --> 362541, skip, preference: True, 1d92f202-15c6-4da5-b001-199554d6cc94, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 65%|█████████████████████████████████████████████████████████████████▋ | 364385/560578 [2:32:13<1:09:29, 47.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "364376, Failed to load conditioning codes for stem from 1e5ac4ac-7675-4c8f-8fcb-7bbea769d487: Semantic codes not found: /app2/suno/data/semantic_code/crow/1e5ac4ac-7675-4c8f-8fcb-7bbea769d487.npz, stem_condition, /app2/suno/data/semantic_code/crow/bb0f6559-f605-4d1f-8135-ea018de5736c.npz.\n", "WTF --> 364377, skip, preference: True, 2db37d08-6282-4eb6-a872-02a09a2b008d, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 65%|█████████████████████████████████████████████████████████████████▊ | 365181/560578 [2:32:40<1:16:48, 42.40it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "365172, Failed to load conditioning codes for stem from 21d07295-98aa-4bcc-baed-c29f74bb0491: Semantic codes not found: /app2/suno/data/semantic_code/crow/21d07295-98aa-4bcc-baed-c29f74bb0491.npz, stem_condition, /app2/suno/data/semantic_code/crow/f354537c-5f32-4816-a7c2-46ca4cc9f900.npz.\n", "WTF --> 365173, skip, preference: True, 52a39932-1ae6-455f-aa62-8dd01fd5ea8b, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 66%|██████████████████████████████████████████████████████████████████▏ | 367348/560578 [2:33:33<1:12:15, 44.57it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "367340, Failed to load conditioning codes for stem from 15ddab34-dab0-40fb-a08d-632744ce4f1c: Semantic codes not found: /app2/suno/data/semantic_code/crow/15ddab34-dab0-40fb-a08d-632744ce4f1c.npz, stem_condition, /app2/suno/data/semantic_code/crow/e3ba6356-d972-43df-89b2-87abd1f94368.npz.\n", "WTF --> 367341, skip, preference: True, a56313e8-21e0-43c0-8e96-f51691520e37, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 67%|███████████████████████████████████████████████████████████████████▋ | 375450/560578 [2:37:12<1:11:51, 42.94it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "375440, Failed to load conditioning codes for stem from 2a5785b4-f937-4684-b645-6253e2c106ee: Semantic codes not found: /app2/suno/data/semantic_code/crow/2a5785b4-f937-4684-b645-6253e2c106ee.npz, stem_condition, /app2/suno/data/semantic_code/crow/b4d9d78f-6ab2-4247-baec-62e7cea50539.npz.\n", "WTF --> 375441, skip, preference: True, 843346fb-70fc-44ad-ae25-eee6d1b19a9a, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 68%|████████████████████████████████████████████████████████████████████▎ | 379264/560578 [2:40:07<1:04:40, 46.72it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "379256, Failed to load conditioning codes for stem from 28eae981-b922-4d4a-a1e1-85b6d8ba4a77: Semantic codes not found: /app2/suno/data/semantic_code/crow/28eae981-b922-4d4a-a1e1-85b6d8ba4a77.npz, stem_condition, /app2/suno/data/semantic_code/crow/c96b4a62-f8ff-4a60-86ea-1e1b74507fd3.npz.\n", "WTF --> 379257, skip, preference: True, 8fb43124-29d0-4197-ba54-fb5333842f3c, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 69%|█████████████████████████████████████████████████████████████████████▉ | 388284/560578 [2:44:08<1:00:57, 47.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "388274, Failed to load conditioning codes for stem from 29cb9b45-4174-4a20-a804-a43c4f57aabb: Semantic codes not found: /app2/suno/data/semantic_code/crow/29cb9b45-4174-4a20-a804-a43c4f57aabb.npz, stem_condition, /app2/suno/data/semantic_code/crow/8f9397c6-2dcc-4238-90a3-3e3dad5ac9b1.npz.\n", "WTF --> 388275, skip, preference: True, 9f32e3b4-46a8-4e5d-9fb7-2a91eb84d02d, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 70%|████████████████████████████████████████████████████████████████████████▏ | 392676/560578 [2:46:52<59:45, 46.83it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "392670, Failed to load conditioning codes for stem from 1f25447e-db27-4241-a05b-d08170b0ed06: Semantic codes not found: /app2/suno/data/semantic_code/crow/1f25447e-db27-4241-a05b-d08170b0ed06.npz, stem_condition, /app2/suno/data/semantic_code/crow/f279d648-0b0a-4ede-8aff-819b80c38f98.npz.\n", "WTF --> 392671, skip, preference: True, 181e5996-8d6d-4e95-a788-87a1d4fc024c, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 71%|███████████████████████████████████████████████████████████████████████▎ | 395897/560578 [2:48:25<1:04:29, 42.55it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "395888, Failed to load conditioning codes for stem from 19dad3a9-a492-4769-804f-4be9013b3b79: Semantic codes not found: /app2/suno/data/semantic_code/crow/19dad3a9-a492-4769-804f-4be9013b3b79.npz, stem_condition, /app2/suno/data/semantic_code/crow/e0df3973-3114-46e9-bb16-535c3ac33d82.npz.\n", "WTF --> 395889, skip, preference: True, df22a680-4b3a-4987-9b1d-cf088de98170, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 71%|███████████████████████████████████████████████████████████████████████▊ | 398426/560578 [2:49:38<1:02:58, 42.91it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "398420, Failed to load conditioning codes for stem from 063818b1-e155-4567-8699-03d8909bfbe9: Semantic codes not found: /app2/suno/data/semantic_code/crow/063818b1-e155-4567-8699-03d8909bfbe9.npz, stem_condition, /app2/suno/data/semantic_code/crow/9bdd50ca-3d27-481c-904a-0b19450fafa9.npz.\n", "WTF --> 398421, skip, preference: True, 6c0e1406-c0a8-4698-af27-eb783af4cf3b, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 71%|█████████████████████████████████████████████████████████████████████████▍ | 399693/560578 [2:50:14<57:48, 46.38it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "399686, Semantic codes not found: /app2/suno/data/semantic_code/crow/m_fe3e0158-65b8-4ae0-97e0-bedfac5a06da.npz, upload_extend, /app2/suno/data/semantic_code/crow/a3e92870-0af8-445f-aade-995a17f853e3.npz.\n", "WTF --> 399687, skip, preference: True, 954a179f-72ee-442a-bc7f-05a53c405906, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 72%|█████████████████████████████████████████████████████████████████████████▊ | 401705/560578 [2:51:15<58:56, 44.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "401700, Failed to load conditioning codes for stem from 1045d6f1-458c-4f65-b227-2341c44bce7a: Semantic codes not found: /app2/suno/data/semantic_code/crow/1045d6f1-458c-4f65-b227-2341c44bce7a.npz, stem_condition, /app2/suno/data/semantic_code/crow/4968f566-5c9a-470c-bec1-226b9239d719.npz.\n", "WTF --> 401701, skip, preference: True, 307c5e8c-673e-41cf-b3eb-7901f280e8c5, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 72%|█████████████████████████████████████████████████████████████████████████▉ | 402459/560578 [2:51:38<58:35, 44.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "402454, Failed to load conditioning codes for stem from 2be8589f-ac92-400c-9bb5-8389cd012c40: Semantic codes not found: /app2/suno/data/semantic_code/crow/2be8589f-ac92-400c-9bb5-8389cd012c40.npz, stem_condition, /app2/suno/data/semantic_code/crow/671e9c33-6853-417c-9a31-3837247cfa1b.npz.\n", "WTF --> 402455, skip, preference: True, 5b1c54e8-7013-4120-a914-965b8b734c78, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 73%|███████████████████████████████████████████████████████████████████████████▌ | 411116/560578 [2:55:27<55:59, 44.49it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "411110, Failed to load conditioning codes for stem from 27997aa8-c931-4fcd-bb14-d648e71d3a5b: Semantic codes not found: /app2/suno/data/semantic_code/crow/27997aa8-c931-4fcd-bb14-d648e71d3a5b.npz, stem_condition, /app2/suno/data/semantic_code/crow/8a1ba050-b73f-48cc-93dc-13c956cfda32.npz.\n", "WTF --> 411111, skip, preference: True, cdd14d37-f0c7-4732-9a58-23db3f79588a, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 73%|███████████████████████████████████████████████████████████████████████████▌ | 411147/560578 [2:55:28<56:04, 44.42it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "411140, Semantic codes not found: /app2/suno/data/semantic_code/crow/m_8d22f9f8-7abd-4ad4-8742-7c990af5d4f4.npz, upload_extend, /app2/suno/data/semantic_code/crow/40bcc205-05f9-4d93-a7a0-9926fe9c24bc.npz.\n", "WTF --> 411141, skip, preference: True, 595d0fc2-efb8-4d6a-a343-dbcddd2dd5a5, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 74%|███████████████████████████████████████████████████████████████████████████▋ | 412204/560578 [2:55:57<58:25, 42.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "412198, Failed to load conditioning codes for stem from 1637f946-5f3e-4214-bffb-8f5970e508ae: Semantic codes not found: /app2/suno/data/semantic_code/crow/1637f946-5f3e-4214-bffb-8f5970e508ae.npz, stem_condition, /app2/suno/data/semantic_code/crow/eb260e60-e7a4-4065-81a3-9ff6736ddc63.npz.\n", "WTF --> 412199, skip, preference: True, e318860b-520c-4b0b-a0f6-1b1118410b92, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 74%|████████████████████████████████████████████████████████████████████████████ | 413696/560578 [2:56:36<56:30, 43.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "413688, Failed to load conditioning codes for stem from 00419e26-9e5a-4470-b12f-8e6b8762ad8b: Semantic codes not found: /app2/suno/data/semantic_code/crow/00419e26-9e5a-4470-b12f-8e6b8762ad8b.npz, stem_condition, /app2/suno/data/semantic_code/crow/1007d761-a339-4383-9213-a1e990d139bb.npz.\n", "WTF --> 413689, skip, preference: True, 16fcb6e6-04c1-430b-a7cd-099ad10c2e6c, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 74%|████████████████████████████████████████████████████████████████████████████▏ | 414826/560578 [2:57:04<52:12, 46.52it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "414818, Failed to load conditioning codes for stem from 320e334e-d404-4d99-b56a-601ec5b38e35: Semantic codes not found: /app2/suno/data/semantic_code/crow/320e334e-d404-4d99-b56a-601ec5b38e35.npz, stem_condition, /app2/suno/data/semantic_code/crow/9f01ab91-54ec-4995-9b90-86ece5e59aa5.npz.\n", "WTF --> 414819, skip, preference: True, 96b1db22-6cae-4612-91de-8ab585a4bc43, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 74%|████████████████████████████████████████████████████████████████████████████▍ | 415852/560578 [2:57:31<54:28, 44.28it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "415844, Failed to load conditioning codes for stem from 0671b0f3-151f-4e76-9130-0acff3986e95: Semantic codes not found: /app2/suno/data/semantic_code/crow/0671b0f3-151f-4e76-9130-0acff3986e95.npz, stem_condition, /app2/suno/data/semantic_code/crow/9425ea2f-f8ed-40fb-97f2-8eff0e1695ae.npz.\n", "WTF --> 415845, skip, preference: True, c523c1dd-95cc-4e7e-afb9-71c93b5878ef, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 75%|████████████████████████████████████████████████████████████████████████████▊ | 418234/560578 [2:58:30<47:21, 50.09it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "418228, Failed to load conditioning codes for stem from 16b6b757-7627-4b3a-8ae8-31f9057b7923: Semantic codes not found: /app2/suno/data/semantic_code/crow/16b6b757-7627-4b3a-8ae8-31f9057b7923.npz, stem_condition, /app2/suno/data/semantic_code/crow/7e0426c4-fe09-433e-99df-eaac627f99be.npz.\n", "WTF --> 418229, skip, preference: True, 8bb0ebb7-0b77-431f-90e3-625d1f1dec21, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 75%|████████████████████████████████████████████████████████████████████████████▉ | 419015/560578 [2:58:48<52:28, 44.96it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "419008, Failed to load conditioning codes for stem from 34227436-74bd-4ef8-b561-9c6f18dcd02f: Semantic codes not found: /app2/suno/data/semantic_code/crow/34227436-74bd-4ef8-b561-9c6f18dcd02f.npz, stem_condition, /app2/suno/data/semantic_code/crow/a0ec1e0b-3e8f-48da-8ff8-4c5f8669fffa.npz.\n", "WTF --> 419009, skip, preference: True, 0d1faedf-34c5-4250-8538-a91a6876879f, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 75%|█████████████████████████████████████████████████████████████████████████████▎ | 420958/560578 [2:59:34<50:26, 46.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "420948, Failed to load conditioning codes for stem from 2fff7bae-4e9e-41af-9d06-4a7110e81140: Semantic codes not found: /app2/suno/data/semantic_code/crow/2fff7bae-4e9e-41af-9d06-4a7110e81140.npz, stem_condition, /app2/suno/data/semantic_code/crow/81143565-c7b2-47db-b082-796d7840f185.npz.\n", "WTF --> 420949, skip, preference: True, a56fbb70-27fd-49ed-87b5-099bc9f507ee, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 75%|█████████████████████████████████████████████████████████████████████████████▍ | 421727/560578 [2:59:52<52:12, 44.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "421722, Semantic codes not found: /app2/suno/data/semantic_code/crow/m_9771a33c-e8d9-4301-87b7-b75ea34c8b93.npz, upload_extend, /app2/suno/data/semantic_code/crow/951df033-ca7f-425c-a32c-23cea8347d69.npz.\n", "WTF --> 421723, skip, preference: True, afa6d61b-47e6-460e-bcda-da55c2cb8243, task: upload_extend.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 75%|█████████████████████████████████████████████████████████████████████████████▌ | 421997/560578 [2:59:58<46:17, 49.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "421990, Failed to load conditioning codes for stem from 117ee109-2345-4c35-887f-c0d184aeae6f: Semantic codes not found: /app2/suno/data/semantic_code/crow/117ee109-2345-4c35-887f-c0d184aeae6f.npz, stem_condition, /app2/suno/data/semantic_code/crow/0f874474-0d64-4ab2-85e2-7e77b49c71a0.npz.\n", "WTF --> 421991, skip, preference: True, 7027400d-a9e0-47c7-94af-a60a4a1ecda5, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 76%|█████████████████████████████████████████████████████████████████████████████▉ | 424374/560578 [3:00:54<52:01, 43.63it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "424366, Failed to load conditioning codes for stem from 234b95f3-d73b-4757-88c5-0530d0eaf749: Semantic codes not found: /app2/suno/data/semantic_code/crow/234b95f3-d73b-4757-88c5-0530d0eaf749.npz, stem_condition, /app2/suno/data/semantic_code/crow/01dc6c1d-c8b7-4b6a-a1df-380d67d77050.npz.\n", "WTF --> 424367, skip, preference: True, ea4d5d56-e3f6-484e-b640-740f34408cc2, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 76%|██████████████████████████████████████████████████████████████████████████████▎ | 426049/560578 [3:01:33<51:25, 43.61it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "426040, Failed to load conditioning codes for stem from 2e6ab276-211f-4603-bb1e-39dcfd9a97ee: Semantic codes not found: /app2/suno/data/semantic_code/crow/2e6ab276-211f-4603-bb1e-39dcfd9a97ee.npz, stem_condition, /app2/suno/data/semantic_code/crow/388433d1-73d9-4d6d-88e2-3a1a9bb9732d.npz.\n", "WTF --> 426041, skip, preference: True, b69f4d9e-4041-4f3e-8332-1e43dd793288, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 76%|██████████████████████████████████████████████████████████████████████████████▌ | 427306/560578 [3:02:03<48:36, 45.70it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "427296, Failed to load conditioning codes for stem from 2f79296a-77b0-483c-ba76-2c9d50e8e157: Semantic codes not found: /app2/suno/data/semantic_code/crow/2f79296a-77b0-483c-ba76-2c9d50e8e157.npz, stem_condition, /app2/suno/data/semantic_code/crow/106c00a2-d482-4414-99dd-b522547b4be2.npz.\n", "WTF --> 427297, skip, preference: True, 566d9337-f967-4153-94e4-cad4fb2647b1, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 76%|██████████████████████████████████████████████████████████████████████████████▌ | 427696/560578 [3:02:12<49:29, 44.75it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "427686, Failed to load conditioning codes for stem from 2cdb0bff-791d-4b23-9baf-460be57d5eb7: Semantic codes not found: /app2/suno/data/semantic_code/crow/2cdb0bff-791d-4b23-9baf-460be57d5eb7.npz, stem_condition, /app2/suno/data/semantic_code/crow/90252cc8-bb82-45a1-8fb7-572c55da2df7.npz.\n", "WTF --> 427687, skip, preference: True, cd180de9-ca08-4e2e-a29a-be196ea42763, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 77%|███████████████████████████████████████████████████████████████████████████████ | 430573/560578 [3:03:21<48:27, 44.71it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "430566, Failed to load conditioning codes for stem from 05b254d8-46e6-464e-b805-7927c63bf2f0: Semantic codes not found: /app2/suno/data/semantic_code/crow/05b254d8-46e6-464e-b805-7927c63bf2f0.npz, stem_condition, /app2/suno/data/semantic_code/crow/9b678698-830c-4e53-b1ef-0abda2367e75.npz.\n", "WTF --> 430567, skip, preference: True, 0b5ec39d-32e8-49e1-9251-6974cd2ae41c, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 77%|███████████████████████████████████████████████████████████████████████████████▏ | 430952/560578 [3:03:30<48:50, 44.23it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "430946, Failed to load conditioning codes for stem from 143602cb-8852-4aa3-8d54-b0f399cd4e33: Semantic codes not found: /app2/suno/data/semantic_code/crow/143602cb-8852-4aa3-8d54-b0f399cd4e33.npz, stem_condition, /app2/suno/data/semantic_code/crow/e87744cb-c514-4ff7-b043-1b96b93535a3.npz.\n", "WTF --> 430947, skip, preference: True, d6a597bb-1beb-4637-ae55-bae20fb8ebfa, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 78%|███████████████████████████████████████████████████████████████████████████████▊ | 434474/560578 [3:05:08<48:04, 43.72it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "434464, Failed to load conditioning codes for stem from 3020e129-ada4-463d-870f-f646dbf83f9a: Semantic codes not found: /app2/suno/data/semantic_code/crow/3020e129-ada4-463d-870f-f646dbf83f9a.npz, stem_condition, /app2/suno/data/semantic_code/crow/d2d9a0fd-ee00-43f3-aeb2-5dbea3ac51ab.npz.\n", "WTF --> 434465, skip, preference: True, 0c9651c2-dea9-4dc5-b442-8485e3ededf2, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 78%|███████████████████████████████████████████████████████████████████████████████▉ | 435034/560578 [3:05:21<45:29, 45.99it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "435030, Failed to load conditioning codes for stem from 2eccdb19-2e86-403e-85fc-e729bcf78318: Semantic codes not found: /app2/suno/data/semantic_code/crow/2eccdb19-2e86-403e-85fc-e729bcf78318.npz, stem_condition, /app2/suno/data/semantic_code/crow/f47d097b-ea38-42d3-b99f-8ec046a8c211.npz.\n", "WTF --> 435031, skip, preference: True, 359654b8-5465-4273-ad9d-2ffcde02a71d, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 78%|████████████████████████████████████████████████████████████████████████████████▎ | 437302/560578 [3:06:17<45:57, 44.71it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "437294, Failed to load conditioning codes for stem from 296bba59-7e58-4f7e-9f91-f340b01b7e6d: Semantic codes not found: /app2/suno/data/semantic_code/crow/296bba59-7e58-4f7e-9f91-f340b01b7e6d.npz, stem_condition, /app2/suno/data/semantic_code/crow/b0aa5dc1-b227-4e24-9b1d-7b429550c715.npz.\n", "WTF --> 437295, skip, preference: True, a4339f68-9c63-4724-877d-4b56d9802188, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 79%|███████████████████████████████████████████████████████████████████████████████▎ | 440123/560578 [3:07:31<1:46:12, 18.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "440116, Failed to load conditioning codes for stem from 135eb392-47ff-4df9-86c9-e4f93a59ec03: Semantic codes not found: /app2/suno/data/semantic_code/crow/135eb392-47ff-4df9-86c9-e4f93a59ec03.npz, stem_condition, /app2/suno/data/semantic_code/crow/46cbb9b4-3064-4a67-9076-3039598d0d9e.npz.\n", "WTF --> 440117, skip, preference: True, 6de4e966-a78e-4e9d-ad12-068f2a7228b5, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 79%|█████████████████████████████████████████████████████████████████████████████████▏ | 441579/560578 [3:08:10<45:57, 43.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "441570, Failed to load conditioning codes for stem from 191b3dfb-c2ae-46ce-8e85-a00fddaf20d2: Semantic codes not found: /app2/suno/data/semantic_code/crow/191b3dfb-c2ae-46ce-8e85-a00fddaf20d2.npz, stem_condition, /app2/suno/data/semantic_code/crow/a1b7dfe2-b266-4fdc-af97-ed9b6ad220bb.npz.\n", "WTF --> 441571, skip, preference: True, 526e64e5-4196-4379-a9d4-7d8b9954763a, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 79%|█████████████████████████████████████████████████████████████████████████████████▏ | 441693/560578 [3:08:13<43:08, 45.94it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "441686, Failed to load conditioning codes for stem from 0abae080-41a6-48b2-9004-a2caa99f8a72: Semantic codes not found: /app2/suno/data/semantic_code/crow/0abae080-41a6-48b2-9004-a2caa99f8a72.npz, stem_condition, /app2/suno/data/semantic_code/crow/c7bbad61-9f64-4b32-9975-7da9055e7b12.npz.\n", "WTF --> 441687, skip, preference: True, d5362ff6-5944-4287-bccc-591a5814905d, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 79%|█████████████████████████████████████████████████████████████████████████████████▎ | 442635/560578 [3:08:38<45:13, 43.46it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "442628, Failed to load conditioning codes for stem from 1aac0822-3318-44d0-8382-a7cad07539ec: Semantic codes not found: /app2/suno/data/semantic_code/crow/1aac0822-3318-44d0-8382-a7cad07539ec.npz, stem_condition, /app2/suno/data/semantic_code/crow/e4c48592-605c-4c6f-89d9-fb914493d55f.npz.\n", "WTF --> 442629, skip, preference: True, 218b0abd-f113-4a91-a6a5-e5d6dd255925, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 79%|█████████████████████████████████████████████████████████████████████████████████▍ | 443019/560578 [3:08:49<39:03, 50.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "443012, Failed to load conditioning codes for stem from 0fefeafd-6161-4423-b0d3-ea33d0e49f04: Semantic codes not found: /app2/suno/data/semantic_code/crow/0fefeafd-6161-4423-b0d3-ea33d0e49f04.npz, stem_condition, /app2/suno/data/semantic_code/crow/9f048f47-cb09-4e14-a3fd-c8570a01829f.npz.\n", "WTF --> 443013, skip, preference: True, 05c6860b-0208-4dd9-81b5-626e91993602, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 79%|█████████████████████████████████████████████████████████████████████████████████▊ | 445549/560578 [3:09:50<38:13, 50.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "445542, Failed to load conditioning codes for stem from 15c16814-2d52-4c93-a2e3-ac6031ec35f6: Semantic codes not found: /app2/suno/data/semantic_code/crow/15c16814-2d52-4c93-a2e3-ac6031ec35f6.npz, stem_condition, /app2/suno/data/semantic_code/crow/eb6e8e5d-d475-46b4-8ee9-a263141ecb33.npz.\n", "WTF --> 445543, skip, preference: True, e737e365-c8b1-4f9f-8bc8-8c21a7329c07, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 80%|██████████████████████████████████████████████████████████████████████████████████▎ | 448222/560578 [3:10:54<48:21, 38.72it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Overflow at 448216: 12599 > 12000\n", "Overflow at 448217: 12599 > 12000\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 81%|███████████████████████████████████████████████████████████████████████████████████▊ | 455906/560578 [3:13:56<37:56, 45.97it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "455898, Failed to load conditioning codes for stem from 0d64be0c-68d3-44a7-85df-94d05c757875: Semantic codes not found: /app2/suno/data/semantic_code/crow/0d64be0c-68d3-44a7-85df-94d05c757875.npz, stem_condition, /app2/suno/data/semantic_code/crow/6a00b7c1-971a-4663-9172-70b14a53249b.npz.\n", "WTF --> 455899, skip, preference: True, 28edccb3-2c46-4ee0-9360-514d95f098a4, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 82%|███████████████████████████████████████████████████████████████████████████████████▉ | 457129/560578 [3:14:59<42:23, 40.68it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "457130, Failed to load conditioning codes for stem from 3a4f2fc9-5f26-45d2-be39-949c60d277de: Semantic codes not found: /app2/suno/data/semantic_code/crow/3a4f2fc9-5f26-45d2-be39-949c60d277de.npz, stem_condition, /app2/suno/data/semantic_code/crow/42715f65-318a-402b-b0f2-7c4fc0e1b62e.npz.\n", "WTF --> 457131, skip, preference: True, 0946cf57-6903-4ace-8206-2c3b7efbecfd, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 82%|████████████████████████████████████████████████████████████████████████████████████▍ | 459447/560578 [3:16:03<34:43, 48.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "459438, Failed to load conditioning codes for stem from 2846ece7-31e2-4848-ad20-1d8de7830600: Semantic codes not found: /app2/suno/data/semantic_code/crow/2846ece7-31e2-4848-ad20-1d8de7830600.npz, stem_condition, /app2/suno/data/semantic_code/crow/880a72a9-cf43-4165-900e-9da0694d180d.npz.\n", "WTF --> 459439, skip, preference: True, ce69b7a2-4ed5-447d-8c22-9161ee20bb29, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 82%|████████████████████████████████████████████████████████████████████████████████████▉ | 462135/560578 [3:17:07<34:37, 47.39it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "462128, Failed to load conditioning codes for stem from 0e654d16-c841-4c47-ba17-29ea6c4f92ca: Semantic codes not found: /app2/suno/data/semantic_code/crow/0e654d16-c841-4c47-ba17-29ea6c4f92ca.npz, stem_condition, /app2/suno/data/semantic_code/crow/5dbe6bd8-f4df-4f2b-a8f5-18e1a354b659.npz.\n", "WTF --> 462129, skip, preference: True, 659158b7-0856-440c-bc0c-85edf8112a31, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 83%|█████████████████████████████████████████████████████████████████████████████████████ | 463282/560578 [3:17:33<35:39, 45.49it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "463276, Failed to load conditioning codes for stem from 245e51cf-21ea-49ec-a035-3a1de9472a7b: Semantic codes not found: /app2/suno/data/semantic_code/crow/245e51cf-21ea-49ec-a035-3a1de9472a7b.npz, stem_condition, /app2/suno/data/semantic_code/crow/dbb2756a-365b-44ce-bc95-50b2e9369c35.npz.\n", "WTF --> 463277, skip, preference: True, 5eb1df67-260e-4efd-b8bb-78a30642166a, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 83%|█████████████████████████████████████████████████████████████████████████████████████▎ | 464223/560578 [3:17:55<36:30, 43.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "464216, Failed to load conditioning codes for stem from 043b0f7d-de8c-44da-9594-0efde4d88fbe: Semantic codes not found: /app2/suno/data/semantic_code/crow/043b0f7d-de8c-44da-9594-0efde4d88fbe.npz, stem_condition, /app2/suno/data/semantic_code/crow/da04b8b9-49cc-41da-8bb7-68f1cb87c7ae.npz.\n", "WTF --> 464217, skip, preference: True, 4213869c-12c8-42d9-85bc-b6371c158949, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 83%|█████████████████████████████████████████████████████████████████████████████████████▎ | 464609/560578 [3:18:04<31:54, 50.12it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "464600, Failed to load conditioning codes for stem from 0f0c3963-fea4-482d-9e18-f581e594c1d4: Semantic codes not found: /app2/suno/data/semantic_code/crow/0f0c3963-fea4-482d-9e18-f581e594c1d4.npz, stem_condition, /app2/suno/data/semantic_code/crow/dc4f2b51-9916-40e8-814c-f0a70e339e47.npz.\n", "WTF --> 464601, skip, preference: True, 5927ec6b-983f-4b68-bd6f-518aab59885d, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 83%|█████████████████████████████████████████████████████████████████████████████████████▎ | 464648/560578 [3:18:05<36:25, 43.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "464640, Failed to load conditioning codes for stem from 26cacbe0-cc21-4ad2-9f7b-18e9fbc79dc6: Semantic codes not found: /app2/suno/data/semantic_code/crow/26cacbe0-cc21-4ad2-9f7b-18e9fbc79dc6.npz, stem_condition, /app2/suno/data/semantic_code/crow/c9d3581d-4265-4e55-8576-9880ca5df2f1.npz.\n", "WTF --> 464641, skip, preference: True, b4eb3b6f-67d2-40de-92d4-694bcf5c5917, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 84%|██████████████████████████████████████████████████████████████████████████████████████ | 468401/560578 [3:19:32<29:57, 51.28it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "468396, Failed to load conditioning codes for stem from 38804dd3-74e7-4736-a8fe-43ba9a426fd3: Semantic codes not found: /app2/suno/data/semantic_code/crow/38804dd3-74e7-4736-a8fe-43ba9a426fd3.npz, stem_condition, /app2/suno/data/semantic_code/crow/3ceac71c-8441-4838-adaa-b81ec8f4e169.npz.\n", "WTF --> 468397, skip, preference: True, b9356772-7e22-4798-bb56-4795a65025d9, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 84%|██████████████████████████████████████████████████████████████████████████████████████▎ | 469901/560578 [3:20:07<31:54, 47.37it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "469892, Failed to load conditioning codes for stem from 3974b375-faa8-4472-80ea-0f286f265725: Semantic codes not found: /app2/suno/data/semantic_code/crow/3974b375-faa8-4472-80ea-0f286f265725.npz, stem_condition, /app2/suno/data/semantic_code/crow/e2be7ef2-00fa-45c8-8559-cc172ba185b4.npz.\n", "WTF --> 469893, skip, preference: True, 60f42f22-af29-458e-9bef-e1a39a040666, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 84%|██████████████████████████████████████████████████████████████████████████████████████▌ | 470996/560578 [3:20:32<31:41, 47.12it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "470988, Failed to load conditioning codes for stem from 21dfd8f4-55b8-4227-b864-8216087c1e75: Semantic codes not found: /app2/suno/data/semantic_code/crow/21dfd8f4-55b8-4227-b864-8216087c1e75.npz, stem_condition, /app2/suno/data/semantic_code/crow/1ed70acb-bd8f-476e-a6e2-b48c8e73d587.npz.\n", "WTF --> 470989, skip, preference: True, a92e7bda-c9ea-40d0-a675-415cef317e5c, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 84%|██████████████████████████████████████████████████████████████████████████████████████▋ | 471663/560578 [3:20:48<33:14, 44.58it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "471652, Failed to load conditioning codes for stem from 22b2ed9e-3a5b-452e-8f7d-c98e3bf85764: Semantic codes not found: /app2/suno/data/semantic_code/crow/22b2ed9e-3a5b-452e-8f7d-c98e3bf85764.npz, stem_condition, /app2/suno/data/semantic_code/crow/b551dfdb-43fa-436b-942a-c44052361244.npz.\n", "WTF --> 471653, skip, preference: True, ddbc359a-236e-495e-a3a8-6ae2804287ad, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 85%|███████████████████████████████████████████████████████████████████████████████████████ | 474098/560578 [3:21:46<32:42, 44.06it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "474092, Failed to load conditioning codes for stem from 3314785d-5cca-44d0-9264-af28a16ae0c0: Semantic codes not found: /app2/suno/data/semantic_code/crow/3314785d-5cca-44d0-9264-af28a16ae0c0.npz, stem_condition, /app2/suno/data/semantic_code/crow/9d5306e6-537e-425d-ab0d-c73aecd7c37c.npz.\n", "WTF --> 474093, skip, preference: True, af84bb45-6af8-4297-b19b-67b3e599cea9, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 85%|███████████████████████████████████████████████████████████████████████████████████████▍ | 475681/560578 [3:22:22<32:30, 43.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "475672, Failed to load conditioning codes for stem from 28361557-d791-451c-bd3c-6d3859d1abcc: Semantic codes not found: /app2/suno/data/semantic_code/crow/28361557-d791-451c-bd3c-6d3859d1abcc.npz, stem_condition, /app2/suno/data/semantic_code/crow/e01ae887-a268-4a15-918e-53dd71a3ddac.npz.\n", "WTF --> 475673, skip, preference: True, b718064d-8e53-452e-b556-fb398837a99c, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 85%|███████████████████████████████████████████████████████████████████████████████████████▋ | 477258/560578 [3:22:59<30:37, 45.34it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "477250, Failed to load conditioning codes for stem from 203351e4-f5be-4aea-9ed4-456d251bb6ae: Semantic codes not found: /app2/suno/data/semantic_code/crow/203351e4-f5be-4aea-9ed4-456d251bb6ae.npz, stem_condition, /app2/suno/data/semantic_code/crow/19738c1a-747e-4d75-ab99-2793e20e85ef.npz.\n", "WTF --> 477251, skip, preference: True, 0444bc3e-c07e-4972-be46-2f9eb75cbbc5, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 85%|███████████████████████████████████████████████████████████████████████████████████████▉ | 478467/560578 [3:23:27<27:43, 49.35it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "478456, Failed to load conditioning codes for stem from 2c514620-81a0-463e-9f77-ccada4542ae3: Semantic codes not found: /app2/suno/data/semantic_code/crow/2c514620-81a0-463e-9f77-ccada4542ae3.npz, stem_condition, /app2/suno/data/semantic_code/crow/eb5ff6de-82dd-4908-b2a7-0ca833cabdc1.npz.\n", "WTF --> 478457, skip, preference: True, 4f71df96-9800-4897-b5a6-ccb6e01657eb, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 86%|████████████████████████████████████████████████████████████████████████████████████████▋ | 482819/560578 [3:25:08<27:16, 47.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "482808, Failed to load conditioning codes for stem from 3f2d0d42-795d-4d87-be58-44cc12559c17: Semantic codes not found: /app2/suno/data/semantic_code/crow/3f2d0d42-795d-4d87-be58-44cc12559c17.npz, stem_condition, /app2/suno/data/semantic_code/crow/200afd20-8972-4617-a1c6-816ff3979cbd.npz.\n", "WTF --> 482809, skip, preference: True, cccb241d-63be-409a-9ae2-c8a0aa9e8d8d, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 86%|████████████████████████████████████████████████████████████████████████████████████████▊ | 483238/560578 [3:25:18<27:09, 47.45it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "483232, Failed to load conditioning codes for stem from 187d728b-d516-43ca-a67a-c99beaf417e9: Semantic codes not found: /app2/suno/data/semantic_code/crow/187d728b-d516-43ca-a67a-c99beaf417e9.npz, stem_condition, /app2/suno/data/semantic_code/crow/cf6c3eec-7c7a-4978-a2d4-34dce368f647.npz.\n", "WTF --> 483233, skip, preference: True, cbce6f16-4b47-4428-9c68-ee6dc10602cd, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 86%|████████████████████████████████████████████████████████████████████████████████████████▊ | 483593/560578 [3:25:26<28:02, 45.76it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "483586, Failed to load conditioning codes for stem from 1015a70b-3d7d-43b6-80ae-94ca7fb314b0: Semantic codes not found: /app2/suno/data/semantic_code/crow/1015a70b-3d7d-43b6-80ae-94ca7fb314b0.npz, stem_condition, /app2/suno/data/semantic_code/crow/76a74e9d-ca39-42ce-a200-f0441f35dbc2.npz.\n", "WTF --> 483587, skip, preference: True, 678b2932-7587-4bcb-ad50-1e149b217c97, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 87%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 486804/560578 [3:26:42<27:12, 45.20it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "486794, Failed to load conditioning codes for stem from 2d333aab-672a-4e55-b9d7-9d378111ddde: Semantic codes not found: /app2/suno/data/semantic_code/crow/2d333aab-672a-4e55-b9d7-9d378111ddde.npz, stem_condition, /app2/suno/data/semantic_code/crow/fce620e7-5513-42ec-8d51-db91225d6992.npz.\n", "WTF --> 486795, skip, preference: True, 094b07b8-525b-4b05-9abf-fee4374c9ae9, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 87%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 489047/560578 [3:27:33<24:41, 48.29it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "489042, Failed to load conditioning codes for stem from 227db657-787f-4cbb-a871-f269f8b8850b: Semantic codes not found: /app2/suno/data/semantic_code/crow/227db657-787f-4cbb-a871-f269f8b8850b.npz, stem_condition, /app2/suno/data/semantic_code/crow/453d2d65-7da4-456d-b20e-d69673f7d605.npz.\n", "WTF --> 489043, skip, preference: True, d92dadc2-f37c-4ba7-9e95-af94000d3682, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 87%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 489682/560578 [3:27:47<25:49, 45.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "489676, Failed to load conditioning codes for stem from 33d8251a-532c-4a86-ab5b-f4350f4f0ed4: Semantic codes not found: /app2/suno/data/semantic_code/crow/33d8251a-532c-4a86-ab5b-f4350f4f0ed4.npz, stem_condition, /app2/suno/data/semantic_code/crow/22cc681b-cbc4-44f4-be45-c4645015cbd2.npz.\n", "WTF --> 489677, skip, preference: True, fb7eb145-7843-4548-b28b-8a8ea601327b, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 88%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 490925/560578 [3:28:16<25:32, 45.45it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "490916, Failed to load conditioning codes for stem from 055a6503-e837-4de8-bcc9-ed2064f53fd6: Semantic codes not found: /app2/suno/data/semantic_code/crow/055a6503-e837-4de8-bcc9-ed2064f53fd6.npz, stem_condition, /app2/suno/data/semantic_code/crow/3f956456-481b-4714-ae15-c58c7e241a89.npz.\n", "WTF --> 490917, skip, preference: True, 77f5eb98-ed28-42fe-9b4e-3995a964bf14, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 88%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 491671/560578 [3:28:33<23:06, 49.69it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "491664, Failed to load conditioning codes for stem from 2504991b-b49c-4924-814d-2bb45abdbe3e: Semantic codes not found: /app2/suno/data/semantic_code/crow/2504991b-b49c-4924-814d-2bb45abdbe3e.npz, stem_condition, /app2/suno/data/semantic_code/crow/771e77a8-4c13-448a-96cf-f7f8731c2f0f.npz.\n", "WTF --> 491665, skip, preference: True, a8f9c4f3-364a-4cb8-b083-204b14572128, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 88%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 491782/560578 [3:28:35<22:41, 50.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "491776, Failed to load conditioning codes for stem from 04ecb6d1-a0ba-44e8-85bb-645d24d713e8: Semantic codes not found: /app2/suno/data/semantic_code/crow/04ecb6d1-a0ba-44e8-85bb-645d24d713e8.npz, stem_condition, /app2/suno/data/semantic_code/crow/3d173c8a-7f00-4ae2-909b-43c77c33834f.npz.\n", "WTF --> 491777, skip, preference: True, ce082a1c-6912-4ae9-9ec5-7dbfb6eb672a, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 88%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 493359/560578 [3:29:12<25:05, 44.65it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "493354, Failed to load conditioning codes for stem from 0f55b030-a4ba-46b5-99a2-de3d274bb1d4: Semantic codes not found: /app2/suno/data/semantic_code/crow/0f55b030-a4ba-46b5-99a2-de3d274bb1d4.npz, stem_condition, /app2/suno/data/semantic_code/crow/eb40abd0-a113-48f4-b6d2-4f4eee36c0e3.npz.\n", "WTF --> 493355, skip, preference: True, 2c64759a-5205-4659-8f79-5dead2d1238c, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 88%|███████████████████████████████████████████████████████████████████████████████████████████ | 495303/560578 [3:29:57<21:20, 50.99it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "495294, Failed to load conditioning codes for stem from 1657988d-adb8-48e8-a905-5dfe8d6d8ca7: Semantic codes not found: /app2/suno/data/semantic_code/crow/1657988d-adb8-48e8-a905-5dfe8d6d8ca7.npz, stem_condition, /app2/suno/data/semantic_code/crow/3a5ec928-2d09-4b9c-862e-6aba74a1f233.npz.\n", "WTF --> 495295, skip, preference: True, 2205c4c9-c784-45c0-970a-37185d946292, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 89%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 498863/560578 [3:31:18<20:28, 50.23it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "498856, Failed to load conditioning codes for stem from 1010d0cf-86b1-417b-98a5-3eda480a5a32: Semantic codes not found: /app2/suno/data/semantic_code/crow/1010d0cf-86b1-417b-98a5-3eda480a5a32.npz, stem_condition, /app2/suno/data/semantic_code/crow/e42fac87-633c-4071-9a38-d55780ad56b6.npz.\n", "WTF --> 498857, skip, preference: True, 47f8e8ea-caeb-498e-a2db-e00cec50f46e, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 89%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 499184/560578 [3:31:25<23:21, 43.81it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "499178, Failed to load conditioning codes for stem from 24692acf-57b1-4872-89d6-dea5c6351be4: Semantic codes not found: /app2/suno/data/semantic_code/crow/24692acf-57b1-4872-89d6-dea5c6351be4.npz, stem_condition, /app2/suno/data/semantic_code/crow/2ddd74b7-da69-4fca-b77b-c22f2f872fe4.npz.\n", "WTF --> 499179, skip, preference: True, 9f28992c-b5ce-40de-8c1e-e088d77a8cf9, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 89%|████████████████████████████████████████████████████████████████████████████████████████████ | 500735/560578 [3:32:01<19:44, 50.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "500728, Failed to load conditioning codes for stem from 04bef2f5-89aa-49d0-8beb-7a1dfefd760f: Semantic codes not found: /app2/suno/data/semantic_code/crow/04bef2f5-89aa-49d0-8beb-7a1dfefd760f.npz, stem_condition, /app2/suno/data/semantic_code/crow/1af10503-1d3c-4186-9001-27f80f073288.npz.\n", "WTF --> 500729, skip, preference: True, 4b053845-69f0-4ef6-b429-2184a97466c8, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 90%|█████████████████████████████████████████████████████████████████████████████████████████████ | 506689/560578 [3:34:18<19:12, 46.75it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "506680, Failed to load conditioning codes for stem from 227e7fe1-df78-4eef-ac41-194ac58d8c8a: Semantic codes not found: /app2/suno/data/semantic_code/crow/227e7fe1-df78-4eef-ac41-194ac58d8c8a.npz, stem_condition, /app2/suno/data/semantic_code/crow/6d8c05a3-45c6-423d-ad50-039b34cd65f1.npz.\n", "WTF --> 506681, skip, preference: True, 495ee383-83e0-4256-9c04-bfcb0fb927b9, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 90%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 507068/560578 [3:34:27<18:49, 47.40it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "507058, Failed to load conditioning codes for stem from 2abc353d-f81a-46cd-80e9-a85101e52474: Semantic codes not found: /app2/suno/data/semantic_code/crow/2abc353d-f81a-46cd-80e9-a85101e52474.npz, stem_condition, /app2/suno/data/semantic_code/crow/f01d1776-231a-4409-b752-49403a3d57f7.npz.\n", "WTF --> 507059, skip, preference: True, dad54254-5fc0-4b69-acba-704fe7bff0b4, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 91%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 509274/560578 [3:35:16<19:01, 44.93it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "509268, Failed to load conditioning codes for stem from 0be0b7b3-0c76-46c3-99cc-b950973b4a50: Semantic codes not found: /app2/suno/data/semantic_code/crow/0be0b7b3-0c76-46c3-99cc-b950973b4a50.npz, stem_condition, /app2/suno/data/semantic_code/crow/12224694-587f-4542-a14e-d1479610ff4a.npz.\n", "WTF --> 509269, skip, preference: True, 1349afb7-995d-4d0d-a9a1-396d17646b7c, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 91%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 510092/560578 [3:35:35<17:23, 48.39it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "510084, Failed to load conditioning codes for stem from 25d2c78e-a5fd-4e06-98e6-38f33d5e0347: Semantic codes not found: /app2/suno/data/semantic_code/crow/25d2c78e-a5fd-4e06-98e6-38f33d5e0347.npz, stem_condition, /app2/suno/data/semantic_code/crow/47f7df78-67a6-48e0-8a99-781ffae971e0.npz.\n", "WTF --> 510085, skip, preference: True, 63284b39-e37d-4a25-8feb-b2148198bda1, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 91%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 510164/560578 [3:35:37<17:33, 47.87it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "510154, Failed to load conditioning codes for stem from 01c295f0-8b85-4872-93b3-b9069b4b2a4b: Semantic codes not found: /app2/suno/data/semantic_code/crow/01c295f0-8b85-4872-93b3-b9069b4b2a4b.npz, stem_condition, /app2/suno/data/semantic_code/crow/401b42eb-3b9e-4413-8c31-a490a331b10e.npz.\n", "WTF --> 510155, skip, preference: True, 5239bd4b-5d92-4436-b612-25ec7d050989, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 91%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 510592/560578 [3:35:46<19:11, 43.42it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "510586, Failed to load conditioning codes for stem from 3971c225-78fc-4de4-8efb-a28790537bfd: Semantic codes not found: /app2/suno/data/semantic_code/crow/3971c225-78fc-4de4-8efb-a28790537bfd.npz, stem_condition, /app2/suno/data/semantic_code/crow/3627ff1e-28dd-41a3-bbfc-b24d0dde666e.npz.\n", "WTF --> 510587, skip, preference: True, b4821d25-0b92-4d63-a6ab-c2e52ccdb17f, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 91%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 511483/560578 [3:36:07<17:02, 48.02it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "511474, Failed to load conditioning codes for stem from 1bd9ef83-43a0-43c1-a34a-ff6979810fd8: Semantic codes not found: /app2/suno/data/semantic_code/crow/1bd9ef83-43a0-43c1-a34a-ff6979810fd8.npz, stem_condition, /app2/suno/data/semantic_code/crow/b807cf65-bac8-4d1f-ae51-b662348491ee.npz.\n", "WTF --> 511475, skip, preference: True, 7f572a4e-fef4-4eec-bf80-c1edfea3db68, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 91%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 512813/560578 [3:36:38<16:53, 47.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "512806, Failed to load conditioning codes for stem from 011872df-9208-476b-afb3-648b89573179: Semantic codes not found: /app2/suno/data/semantic_code/crow/011872df-9208-476b-afb3-648b89573179.npz, stem_condition, /app2/suno/data/semantic_code/crow/25f2ace9-26db-41ae-94eb-69cdc660a3ab.npz.\n", "WTF --> 512807, skip, preference: True, 28f49b35-cbc3-465e-a97a-e659408fa896, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 92%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 513106/560578 [3:36:44<16:21, 48.34it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "513100, Failed to load conditioning codes for stem from 08f384df-4858-4980-a3a7-c0cb923ff5f9: Semantic codes not found: /app2/suno/data/semantic_code/crow/08f384df-4858-4980-a3a7-c0cb923ff5f9.npz, stem_condition, /app2/suno/data/semantic_code/crow/317505a6-b161-450f-9c3a-0470a175fec1.npz.\n", "WTF --> 513101, skip, preference: True, 82669a31-e84d-4c30-b40c-6d1c78cb3486, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 92%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 514303/560578 [3:37:11<16:21, 47.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "514292, Failed to load conditioning codes for stem from 42b6d547-dd25-4ef8-a3b4-335811af09a6: Semantic codes not found: /app2/suno/data/semantic_code/crow/42b6d547-dd25-4ef8-a3b4-335811af09a6.npz, stem_condition, /app2/suno/data/semantic_code/crow/f718eebe-d44d-4551-81af-dc9305cfc0fa.npz.\n", "WTF --> 514293, skip, preference: True, dd2a8875-f641-417f-935f-882b430e732a, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 92%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 515127/560578 [3:37:30<16:18, 46.45it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "515116, Failed to load conditioning codes for stem from 0571a354-e039-4e8a-92c5-00b0b0474e40: Semantic codes not found: /app2/suno/data/semantic_code/crow/0571a354-e039-4e8a-92c5-00b0b0474e40.npz, stem_condition, /app2/suno/data/semantic_code/crow/a8bf5c4b-6c5b-42f0-b498-af312971ae39.npz.\n", "WTF --> 515117, skip, preference: True, da7e3f8b-c30d-4644-998f-eb867f553ce7, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 92%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 515443/560578 [3:37:38<14:51, 50.61it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "515434, Failed to load conditioning codes for stem from 10f8a6fb-d622-4564-933e-7ede3ea06a6f: Semantic codes not found: /app2/suno/data/semantic_code/crow/10f8a6fb-d622-4564-933e-7ede3ea06a6f.npz, stem_condition, /app2/suno/data/semantic_code/crow/dd3b918a-6206-4309-9831-c08bf29d4062.npz.\n", "WTF --> 515435, skip, preference: True, 08f839ef-69ef-4a98-9b15-51599ac9ff47, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 92%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 516342/560578 [3:37:58<15:21, 48.01it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "516332, Failed to load conditioning codes for stem from 1a473d6b-173b-4ac5-b5eb-f80b17fef2ac: Semantic codes not found: /app2/suno/data/semantic_code/crow/1a473d6b-173b-4ac5-b5eb-f80b17fef2ac.npz, stem_condition, /app2/suno/data/semantic_code/crow/f5dfdb25-9f8e-4d0d-ae4e-5df15d549afc.npz.\n", "WTF --> 516333, skip, preference: True, 07b9796b-135c-4300-bec2-635ae0b59269, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 92%|███████████████████████████████████████████████████████████████████████████████████████████████ | 517652/560578 [3:38:29<15:07, 47.29it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "517644, Failed to load conditioning codes for stem from 3bf9f017-8905-4590-9aae-5e4e2e5e7392: Semantic codes not found: /app2/suno/data/semantic_code/crow/3bf9f017-8905-4590-9aae-5e4e2e5e7392.npz, stem_condition, /app2/suno/data/semantic_code/crow/8810c785-f288-4160-9350-5a9ba853beac.npz.\n", "WTF --> 517645, skip, preference: True, 003a23d6-0980-4721-bcce-f986d4426e5d, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 92%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 517756/560578 [3:38:31<14:52, 47.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "517746, Failed to load conditioning codes for stem from 40d9e5e4-daf3-4ab7-b594-332db0d2d8a2: Semantic codes not found: /app2/suno/data/semantic_code/crow/40d9e5e4-daf3-4ab7-b594-332db0d2d8a2.npz, stem_condition, /app2/suno/data/semantic_code/crow/aca0d146-1594-4305-bb74-1f73e685e26e.npz.\n", "WTF --> 517747, skip, preference: True, 826c5f8c-ecfb-41fe-9a0c-0f7b955f0756, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 93%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 519518/560578 [3:39:11<14:52, 46.02it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "519512, Failed to load conditioning codes for stem from 068f3c27-e668-4cde-ac39-33424479a433: Semantic codes not found: /app2/suno/data/semantic_code/crow/068f3c27-e668-4cde-ac39-33424479a433.npz, stem_condition, /app2/suno/data/semantic_code/crow/a7843fb4-e2ca-4c91-8fb8-ec93c6bb8293.npz.\n", "WTF --> 519513, skip, preference: True, 3664ef9f-8935-47de-9c45-4036836bceec, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 93%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 520384/560578 [3:39:32<14:37, 45.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "520374, Failed to load conditioning codes for stem from 090cce82-1659-4ae0-bf6b-25c76887f8f6: Semantic codes not found: /app2/suno/data/semantic_code/crow/090cce82-1659-4ae0-bf6b-25c76887f8f6.npz, stem_condition, /app2/suno/data/semantic_code/crow/6e330a42-9bb7-4dd7-80f1-8a6d49edb9e5.npz.\n", "WTF --> 520375, skip, preference: True, 0d081b17-c4eb-44e0-82bc-782d00b6c60a, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 93%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 521462/560578 [3:39:56<13:20, 48.87it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "521454, Failed to load conditioning codes for stem from 27e05c1b-2d0d-4317-9177-7514b2c8c275: Semantic codes not found: /app2/suno/data/semantic_code/crow/27e05c1b-2d0d-4317-9177-7514b2c8c275.npz, stem_condition, /app2/suno/data/semantic_code/crow/94eb15ee-b289-4606-9967-06c4d3320cab.npz.\n", "WTF --> 521455, skip, preference: True, de2ac9a4-8e98-47b6-8ea4-495c325f7213, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 94%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 527470/560578 [3:42:20<11:22, 48.48it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "527460, Failed to load conditioning codes for stem from 453f90e5-4779-4a60-8116-cd5228cf8c16: Semantic codes not found: /app2/suno/data/semantic_code/crow/453f90e5-4779-4a60-8116-cd5228cf8c16.npz, stem_condition, /app2/suno/data/semantic_code/crow/6a383527-3d3b-4734-850e-892ff00282c6.npz.\n", "WTF --> 527461, skip, preference: True, 1bd1f33f-8ea3-4cab-8f9b-d61c05c03197, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 94%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 528122/560578 [3:42:35<12:14, 44.20it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "528112, Failed to load conditioning codes for stem from 336d78b1-3c72-44de-98b7-aa7381bfb9f9: Semantic codes not found: /app2/suno/data/semantic_code/crow/336d78b1-3c72-44de-98b7-aa7381bfb9f9.npz, stem_condition, /app2/suno/data/semantic_code/crow/040d7f82-eaba-48a9-8d15-090ee3e0a04e.npz.\n", "WTF --> 528113, skip, preference: True, 8dd42c8c-2ef6-48ab-bb31-f049516c050b, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 94%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 529021/560578 [3:42:56<11:35, 45.40it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "529016, Failed to load conditioning codes for stem from 44e7da48-cea1-4662-9a79-ba2a5fc1a898: Semantic codes not found: /app2/suno/data/semantic_code/crow/44e7da48-cea1-4662-9a79-ba2a5fc1a898.npz, stem_condition, /app2/suno/data/semantic_code/crow/1059fbbc-0ff8-4501-b947-37376bf8f314.npz.\n", "WTF --> 529017, skip, preference: True, c12f7d24-885f-4571-8143-7207357650af, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 94%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 529425/560578 [3:43:05<10:57, 47.38it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "529418, Failed to load conditioning codes for stem from 3eff8734-6f2a-4900-87c1-8253667e214c: Semantic codes not found: /app2/suno/data/semantic_code/crow/3eff8734-6f2a-4900-87c1-8253667e214c.npz, stem_condition, /app2/suno/data/semantic_code/crow/f2ff2836-e813-47e6-9d7b-f604d2df42ed.npz.\n", "WTF --> 529419, skip, preference: True, fa3b9a3b-01f3-4af9-acba-b5b7f77ddd37, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 95%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 529925/560578 [3:43:17<11:08, 45.82it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "529916, Failed to load conditioning codes for stem from 37913309-69a3-41fe-aa67-7ae707af3630: Semantic codes not found: /app2/suno/data/semantic_code/crow/37913309-69a3-41fe-aa67-7ae707af3630.npz, stem_condition, /app2/suno/data/semantic_code/crow/59c45176-1307-4f1f-a88d-cac3f56012e6.npz.\n", "WTF --> 529917, skip, preference: True, 79f5361a-dd35-49ad-8df6-ac477b6b1cbd, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 95%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 531773/560578 [3:44:00<10:03, 47.73it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "531766, Failed to load conditioning codes for stem from 28f05dfc-acf4-440b-b94b-cae9e5f84642: Semantic codes not found: /app2/suno/data/semantic_code/crow/28f05dfc-acf4-440b-b94b-cae9e5f84642.npz, stem_condition, /app2/suno/data/semantic_code/crow/99672087-e97f-4e4a-b288-429207f74e84.npz.\n", "WTF --> 531767, skip, preference: True, 0e5c33b9-fc4d-48ba-a30e-f0c624aee46b, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 95%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 532899/560578 [3:44:28<09:40, 47.71it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "532890, Failed to load conditioning codes for stem from 473997e3-41d8-4c6b-bbc3-e0b8d6fbfba2: Semantic codes not found: /app2/suno/data/semantic_code/crow/473997e3-41d8-4c6b-bbc3-e0b8d6fbfba2.npz, stem_condition, /app2/suno/data/semantic_code/crow/637f7b19-3ff6-48db-b159-4d6593f358a2.npz.\n", "WTF --> 532891, skip, preference: True, a58f60c0-30b7-4e89-98d3-d5f79cfcb4c2, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 95%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 533015/560578 [3:44:31<09:17, 49.46it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "533008, Failed to load conditioning codes for stem from 1d40855b-01c0-4dd5-b36b-de61d803a84d: Semantic codes not found: /app2/suno/data/semantic_code/crow/1d40855b-01c0-4dd5-b36b-de61d803a84d.npz, stem_condition, /app2/suno/data/semantic_code/crow/9b0e64e0-a042-4f75-a7f6-d35e9100ada6.npz.\n", "WTF --> 533009, skip, preference: True, 38d8ba8a-fb92-43a3-b015-877a56d215e3, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 96%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 535869/560578 [3:45:36<08:42, 47.29it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "535860, Failed to load conditioning codes for stem from 0de0590e-8f67-4da1-9f93-7038c0ca6c49: Semantic codes not found: /app2/suno/data/semantic_code/crow/0de0590e-8f67-4da1-9f93-7038c0ca6c49.npz, stem_condition, /app2/suno/data/semantic_code/crow/44492ef1-8788-4010-9956-049dbb38e6bc.npz.\n", "WTF --> 535861, skip, preference: True, bee60888-7c33-4d80-8a38-34ace7fac95c, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 96%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 540398/560578 [3:47:21<07:33, 44.49it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "540392, Failed to load conditioning codes for stem from 406f387b-0476-4c3b-bac6-3464378ce099: Semantic codes not found: /app2/suno/data/semantic_code/crow/406f387b-0476-4c3b-bac6-3464378ce099.npz, stem_condition, /app2/suno/data/semantic_code/crow/ebfb75e4-a0e7-485c-92d6-c76e43ef4ca3.npz.\n", "WTF --> 540393, skip, preference: True, 6b49752f-88c7-4aaf-8e9a-12903cfedd1e, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 97%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 541275/560578 [3:47:49<06:53, 46.63it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "541266, Failed to load conditioning codes for stem from 3f9e8d2e-3cd1-4ec9-be5a-631785f72c23: Semantic codes not found: /app2/suno/data/semantic_code/crow/3f9e8d2e-3cd1-4ec9-be5a-631785f72c23.npz, stem_condition, /app2/suno/data/semantic_code/crow/39a4513f-2a0d-4cad-a125-71a920aea784.npz.\n", "WTF --> 541267, skip, preference: True, 5dbee7c8-de24-46c3-8e83-3136b1323537, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 97%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 541377/560578 [3:47:52<07:20, 43.60it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "541372, Failed to load conditioning codes for stem from 1127806e-4a5b-4cd5-9b75-c7ad1099db0c: Semantic codes not found: /app2/suno/data/semantic_code/crow/1127806e-4a5b-4cd5-9b75-c7ad1099db0c.npz, stem_condition, /app2/suno/data/semantic_code/crow/96f56fd9-f460-47ca-a854-b4903a8e92a7.npz.\n", "WTF --> 541373, skip, preference: True, 8d1252b2-d6b4-4512-a238-bb39d99fccde, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 97%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 541849/560578 [3:48:03<06:52, 45.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "541842, Failed to load conditioning codes for stem from 3f3f62a2-80c4-4b7e-b5d2-e2f9048cbbc1: Semantic codes not found: /app2/suno/data/semantic_code/crow/3f3f62a2-80c4-4b7e-b5d2-e2f9048cbbc1.npz, stem_condition, /app2/suno/data/semantic_code/crow/bc1ee336-f816-4e30-9916-af2128a16bbc.npz.\n", "WTF --> 541843, skip, preference: True, 176a388e-3a0e-4835-b04b-826802c8a5c2, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 97%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 542360/560578 [3:48:15<08:10, 37.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "542352, Failed to load conditioning codes for stem from 37ab2317-6dcb-4494-998b-006078580dfb: Semantic codes not found: /app2/suno/data/semantic_code/crow/37ab2317-6dcb-4494-998b-006078580dfb.npz, stem_condition, /app2/suno/data/semantic_code/crow/bba3be8f-e4ed-4c44-b48c-c50baa2baa23.npz.\n", "WTF --> 542353, skip, preference: True, 4e6c14f7-ab92-41c3-a59a-a043b8b50123, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 97%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 545185/560578 [3:49:58<08:57, 28.65it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "545174, Failed to load conditioning codes for stem from 0952a51a-9efb-4a61-8efa-bc2ad842900c: Semantic codes not found: /app2/suno/data/semantic_code/crow/0952a51a-9efb-4a61-8efa-bc2ad842900c.npz, stem_condition, /app2/suno/data/semantic_code/crow/d17623d1-5b21-4873-aef8-de83bed5cd47.npz.\n", "WTF --> 545175, skip, preference: True, 47ca1be1-0739-4637-84a0-83e7a0ef5676, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 98%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 547021/560578 [3:50:41<04:27, 50.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "547012, Failed to load conditioning codes for stem from 0013172a-aa54-4b66-8d64-f23a6fc26d84: Semantic codes not found: /app2/suno/data/semantic_code/crow/0013172a-aa54-4b66-8d64-f23a6fc26d84.npz, stem_condition, /app2/suno/data/semantic_code/crow/5ea7a0f1-fb94-418d-80ce-68f58c8ed396.npz.\n", "WTF --> 547013, skip, preference: True, fbbe38de-0757-4a8c-9469-da4034453ed6, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 551656/560578 [3:52:29<03:15, 45.71it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "551646, Failed to load conditioning codes for stem from 304252a6-9923-4f3a-8fdc-d869956d1a2d: Semantic codes not found: /app2/suno/data/semantic_code/crow/304252a6-9923-4f3a-8fdc-d869956d1a2d.npz, stem_condition, /app2/suno/data/semantic_code/crow/c48a7e9f-f1f5-4258-9b8f-1c139197c61d.npz.\n", "WTF --> 551647, skip, preference: True, 83559158-71d2-4179-a4b7-03f74c8d3e80, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 553310/560578 [3:53:06<02:22, 51.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "553306, Failed to load conditioning codes for stem from 3c2fd264-313f-4a17-967e-a2909918701b: Semantic codes not found: /app2/suno/data/semantic_code/crow/3c2fd264-313f-4a17-967e-a2909918701b.npz, stem_condition, /app2/suno/data/semantic_code/crow/93d4c5a0-c461-4e82-a5d7-fcc22269b47b.npz.\n", "WTF --> 553307, skip, preference: True, 5d74171a-2c54-4d65-8293-f02b1696beb5, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 555472/560578 [3:53:56<01:43, 49.45it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "555466, Failed to load conditioning codes for stem from 43601ec9-c56f-4eb4-b5a7-549e51b91564: Semantic codes not found: /app2/suno/data/semantic_code/crow/43601ec9-c56f-4eb4-b5a7-549e51b91564.npz, stem_condition, /app2/suno/data/semantic_code/crow/a4ffb3ca-48f9-4f78-a596-d569a0c03d79.npz.\n", "WTF --> 555467, skip, preference: True, 10f584af-2346-42ba-84e3-24f32ef107ac, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎| 556940/560578 [3:54:30<01:17, 47.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "556930, Failed to load conditioning codes for stem from 2faa35a4-64c8-4794-ba44-f635df836be3: Semantic codes not found: /app2/suno/data/semantic_code/crow/2faa35a4-64c8-4794-ba44-f635df836be3.npz, stem_condition, /app2/suno/data/semantic_code/crow/6f28f665-c382-4771-999c-d988007ab7d1.npz.\n", "WTF --> 556931, skip, preference: True, 371ae36e-23e3-4fb6-9988-4a02f0b86b13, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍| 557249/560578 [3:54:37<01:17, 43.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "557240, Failed to load conditioning codes for stem from 010b0a98-d20e-4021-b2a2-56a8aa132dfb: Semantic codes not found: /app2/suno/data/semantic_code/crow/010b0a98-d20e-4021-b2a2-56a8aa132dfb.npz, stem_condition, /app2/suno/data/semantic_code/crow/b44247e0-23e8-4021-8e52-d0f73739629f.npz.\n", "WTF --> 557241, skip, preference: True, f220fe20-bee9-4ead-8468-d227d3d4ac6b, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍| 557420/560578 [3:54:41<01:06, 47.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "557412, Failed to load conditioning codes for stem from 21fff50f-5af1-4d5d-aadb-9fb2299a35ed: Semantic codes not found: /app2/suno/data/semantic_code/crow/21fff50f-5af1-4d5d-aadb-9fb2299a35ed.npz, stem_condition, /app2/suno/data/semantic_code/crow/913b139a-a31d-4049-bd06-d4a8a7c0c746.npz.\n", "WTF --> 557413, skip, preference: True, ca1fad53-7ae8-4039-ae9c-002f08097fea, task: stem_condition.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 560578/560578 [3:55:54<00:00, 39.60it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 560276 clips, 0 different prompts, 3 different tags, 0 different negative tags\n", "18,985 hours of False\n", "19,067 hours of True\n", "gen: 17128.3 hours\n", "cover: 12493.9 hours\n", "infill: 221.9 hours\n", "artist_consistency: 3900.3 hours\n", "artist_cover: 2399.9 hours\n", "extend: 764.3 hours\n", "overpainting: 232.4 hours\n", "underpainting: 136.5 hours\n", "artist_extend: 190.2 hours\n", "playlist_condition: 458.0 hours\n", "stem_condition: 126.2 hours\n", "🚨 Error upload_extend: 6\n", "🚨 Error stem_condition: 145\n", "\n", "--- Gender Distribution ---\n", " female: 51,004 (9.1%)\n", " male: 81,396 (14.5%)\n", " unspecified: 428,027 (76.4%)\n", "\n", "--- Negative Tags Usage ---\n", " has_neg_tags: 29,967 (5.3%)\n", " no_neg_tags: 530,460 (94.7%)\n", "\n", "--- Control Slider Usage ---\n", " has_control_slider: 173,758 (31.0% of clips)\n", " no_control_slider: 386,669 (69.0% of clips)\n", "Done\n" ] } ], "source": [ "make_dataset(\n", " train_df, OUT_DATA_DIR, is_val=False, t_data_memmap=N_TOKENS_AUDIO\n", ")" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.937879Z", "start_time": "2024-05-16T13:59:41.937870Z" }, "execution": { "iopub.execute_input": "2025-11-15T06:00:24.312985Z", "iopub.status.busy": "2025-11-15T06:00:24.312826Z", "iopub.status.idle": "2025-11-15T06:00:24.512124Z", "shell.execute_reply": "2025-11-15T06:00:24.511559Z", "shell.execute_reply.started": "2025-11-15T06:00:24.312969Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, N_TOKENS_AUDIO, 4)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T06:00:24.512869Z", "iopub.status.busy": "2025-11-15T06:00:24.512720Z", "iopub.status.idle": "2025-11-15T06:00:24.532883Z", "shell.execute_reply": "2025-11-15T06:00:24.532400Z", "shell.execute_reply.started": "2025-11-15T06:00:24.512855Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Counter({None: 3442, 'cover': 1282, 'artist_consistency': 414, 'artist_cover': 202, 'extend': 152, 'playlist_condition': 46, 'infill': 44, 'stem_condition': 24, 'overpainting': 20, 'artist_extend': 20, 'underpainting': 16})\n" ] } ], "source": [ "task_counts = Counter()\n", "for test_meta in test_metas:\n", " task_counts[test_meta.get(\"task\")] += 1\n", "print(task_counts)" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.938629Z", "start_time": "2024-05-16T13:59:41.938621Z" }, "execution": { "iopub.execute_input": "2025-11-15T06:00:24.533521Z", "iopub.status.busy": "2025-11-15T06:00:24.533380Z", "iopub.status.idle": "2025-11-15T06:00:24.547899Z", "shell.execute_reply": "2025-11-15T06:00:24.547445Z", "shell.execute_reply.started": "2025-11-15T06:00:24.533506Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as codec_encode,\n", "# decode_stream_to_full_audio as codec_decode,\n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# decode as decode\n", "# )\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "# assert len(test_metas) == len(mm)\n", "# idx_list = list(range(len(test_metas)))\n", "# # random.shuffle(idx_list)\n", "# # idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939205Z", "start_time": "2024-05-16T13:59:41.939198Z" }, "execution": { "iopub.execute_input": "2025-11-15T06:00:24.548511Z", "iopub.status.busy": "2025-11-15T06:00:24.548372Z", "iopub.status.idle": "2025-11-15T06:00:24.562571Z", "shell.execute_reply": "2025-11-15T06:00:24.562142Z", "shell.execute_reply.started": "2025-11-15T06:00:24.548498Z" } }, "outputs": [], "source": [ "# import random\n", "# idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "# assert \"original_duration_s\" in test_metas[idx]\n", "# # positive index should be shifted by 1\n", "# pos_idx = idx + 1\n", "# print(\n", "# \"tags:\",\n", "# test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", "# test_metas[idx].get(\"tags\"),\n", "# )\n", "# arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[: pad_idx_arr[0], :]\n", "# pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pos_pad_idx_arr) > 0:\n", "# pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "# a = decode(arr)\n", "# print(\"\\n negative example \\n\", test_metas[idx])\n", "# a.play(compress=False)\n", "# pos_a = decode(pos_arr)\n", "# print(\"\\n positive example \\n\", test_metas[pos_idx])\n", "# pos_a.play(compress=False)\n", "# print(\n", "# \"text:\",\n", "# test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", "# test_metas[idx].get(\"text\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939977Z", "start_time": "2024-05-16T13:59:41.939969Z" }, "execution": { "iopub.execute_input": "2025-11-15T06:00:24.563145Z", "iopub.status.busy": "2025-11-15T06:00:24.563015Z", "iopub.status.idle": "2025-11-15T06:00:24.575419Z", "shell.execute_reply": "2025-11-15T06:00:24.574998Z", "shell.execute_reply.started": "2025-11-15T06:00:24.563132Z" } }, "outputs": [], "source": [ "# val_df[val_df[\"tags\"] == 'a vibrant blend of experimental jazz fusion, drum-and-bass and swagger fuzzed-out guitars']" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.940610Z", "start_time": "2024-05-16T13:59:41.940603Z" }, "execution": { "iopub.execute_input": "2025-11-15T06:00:24.577823Z", "iopub.status.busy": "2025-11-15T06:00:24.577485Z", "iopub.status.idle": "2025-11-15T06:00:24.589493Z", "shell.execute_reply": "2025-11-15T06:00:24.589072Z", "shell.execute_reply.started": "2025-11-15T06:00:24.577809Z" } }, "outputs": [], "source": [ "# from collections import Counter\n", "# c = Counter()\n", "# for _, row in df_slice.iterrows():\n", "# # print(row[\"metadata\"])\n", "# for k in ast.literal_eval(row[\"metadata\"]).keys():\n", "# c[k] += 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941167Z", "start_time": "2024-05-16T13:59:41.941159Z" }, "execution": { "iopub.execute_input": "2025-11-15T06:00:24.592230Z", "iopub.status.busy": "2025-11-15T06:00:24.592111Z", "iopub.status.idle": "2025-11-15T06:00:24.603881Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941801Z", "start_time": "2024-05-16T13:59:41.941793Z" }, "execution": { "iopub.execute_input": "2025-11-15T06:00:24.605600Z", "iopub.status.busy": "2025-11-15T06:00:24.605473Z", "iopub.status.idle": "2025-11-15T06:00:24.624449Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2831 0\n" ] } ], "source": [ "def validation_on_metas(input_metas):\n", " total_bad = 0\n", " total_good = 0\n", " for idx in range(len(input_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if input_metas[idx].get(\"tags\") != input_metas[pos_idx].get(\"tags\"):\n", " print(\n", " input_metas[idx].get(\"id\"),\n", " input_metas[idx].get(\"tags\"),\n", " input_metas[pos_idx].get(\"id\"),\n", " input_metas[pos_idx].get(\"tags\"),\n", " )\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " elif input_metas[idx].get(\"text\") != input_metas[pos_idx].get(\"text\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", " print(total_good, total_bad)\n", " return\n", "\n", "\n", "validation_on_metas(test_metas)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.942520Z", "start_time": "2024-05-16T13:59:41.942511Z" }, "execution": { "iopub.execute_input": "2025-11-15T06:00:24.626330Z", "iopub.status.busy": "2025-11-15T06:00:24.626202Z", "iopub.status.idle": "2025-11-15T06:00:24.676717Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.943072Z", "start_time": "2024-05-16T13:59:41.943065Z" }, "execution": { "iopub.execute_input": "2025-11-15T06:00:24.678416Z", "iopub.status.busy": "2025-11-15T06:00:24.678288Z", "iopub.status.idle": "2025-11-15T06:00:24.710815Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)\n", "# make sure they are offset by 1 and exactly 1\n", "for i, j in zip(n_neg_tr, n_pos_tr):\n", " assert i == j - 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.944246Z", "start_time": "2024-05-16T13:59:41.944237Z" }, "execution": { "iopub.execute_input": "2025-11-15T06:00:24.711589Z", "iopub.status.busy": "2025-11-15T06:00:24.711324Z", "iopub.status.idle": "2025-11-15T06:00:24.724824Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total samples 560276 (560578, 208)\n" ] } ], "source": [ "total_iters = len(n_neg_tr) + len(n_pos_tr)\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T06:00:24.725397Z", "iopub.status.busy": "2025-11-15T06:00:24.725266Z", "iopub.status.idle": "2025-11-15T06:00:44.090997Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "280138 0\n" ] } ], "source": [ "metas_tr = read_jsonl(os.path.join(OUT_DATA_DIR, \"meta_tr.jsonl\"))\n", "validation_on_metas(metas_tr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T06:00:44.091719Z", "iopub.status.busy": "2025-11-15T06:00:44.091567Z", "iopub.status.idle": "2025-11-15T06:00:44.107239Z" } }, "outputs": [], "source": [ "# new_metas_tr = []\n", "# for index, l in enumerate(metas_tr):\n", "# if index % 2 == 1:\n", "# last_l = new_metas_tr[-1]\n", "# if l[\"tags\"] != last_l[\"tags\"]:\n", "# print(l[\"tags\"], last_l[\"tags\"])\n", "# l[\"tags\"] = last_l[\"tags\"]\n", "# new_metas_tr.append(l)\n", "# validation_on_metas(new_metas_tr)\n", "# write_jsonl(new_metas_tr, os.path.join(OUT_DATA_DIR, \"meta_tr.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945249Z", "start_time": "2024-05-16T13:59:41.945241Z" }, "execution": { "iopub.execute_input": "2025-11-15T06:00:44.107954Z", "iopub.status.busy": "2025-11-15T06:00:44.107822Z", "iopub.status.idle": "2025-11-15T06:00:44.121675Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 epoch per batch 4, total 2188.578125\n", "1 epoch per batch 6, total 729.5260416666666\n", "1 epoch per batch 8, total 547.14453125\n" ] } ], "source": [ "print(\"1 epoch per batch 4, total\", total_iters / 8 / 8 / 4)\n", "print(\"1 epoch per batch 6, total\", total_iters / 16 / 8 / 6)\n", "print(\"1 epoch per batch 8, total\", total_iters / 16 / 8 / 8)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T06:00:44.122264Z", "iopub.status.busy": "2025-11-15T06:00:44.122133Z", "iopub.status.idle": "2025-11-15T06:00:44.133472Z" } }, "outputs": [], "source": [ "# import time\n", "# time.sleep(60 * 60 * 1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T06:00:44.134054Z", "iopub.status.busy": "2025-11-15T06:00:44.133918Z", "iopub.status.idle": "2025-11-15T06:00:44.145294Z" } }, "outputs": [], "source": [ "# df[\"control_tags\"].value_counts()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945972Z", "start_time": "2024-05-16T13:59:41.945964Z" }, "execution": { "iopub.execute_input": "2025-11-15T06:00:44.145953Z", "iopub.status.busy": "2025-11-15T06:00:44.145823Z", "iopub.status.idle": "2025-11-15T06:00:44.157195Z" } }, "outputs": [], "source": [ "# !cd /home/tony/Work/tony/slurm/crow && sbatch sbatch_ipo_crow" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T06:00:44.157880Z", "iopub.status.busy": "2025-11-15T06:00:44.157640Z", "iopub.status.idle": "2025-11-15T06:00:44.181911Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cache kept!\n" ] } ], "source": [ "import shutil\n", "\n", "# Basic file copy\n", "shutil.copy(\n", " \"/home/tony/Work/tony/Preference/make_dataset_crow_t1_4rvq_dup2.ipynb\",\n", " os.path.join(OUT_DATA_DIR, \"make_dataset.ipynb\"),\n", ")\n", "print(\"Cache kept!\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-15T06:00:44.182449Z", "iopub.status.busy": "2025-11-15T06:00:44.182320Z", "iopub.status.idle": "2025-11-15T06:00:44.203466Z" } }, "outputs": [ { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[38], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "BREAK" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# some gymathtics loading prev data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.946562Z", "start_time": "2024-05-16T13:59:41.946555Z" }, "execution": { "iopub.status.busy": "2025-11-15T06:00:44.203948Z", "iopub.status.idle": "2025-11-15T06:00:44.204143Z" } }, "outputs": [], "source": [ "# prev_v3_data = \"/app/suno/data/dpo/7v_v20_full/\"\n", "\n", "# test_val_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_val.jsonl\"))\n", "# test_tr_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_tr.jsonl\"))\n", "\n", "# all_ids = set()\n", "# for meta in test_val_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# for meta in test_tr_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# print(len(all_ids), len(test_val_metas) + len(test_tr_metas))\n", "\n", "# all_ids = list(all_ids)\n", "# with open(\"/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id.json\", \"w\") as fp:\n", "# json.dump(all_ids, fp)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.204530Z", "iopub.status.idle": "2025-11-15T06:00:44.204761Z" } }, "outputs": [], "source": [ "# x_data = train_df[train_df[\"preference\"]][\"similarity\"]\n", "# y_data = train_df[~train_df[\"preference\"]][\"similarity\"]\n", "# from matplotlib.colors import LogNorm\n", "\n", "# fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 10))\n", "\n", "# # 2D Histogram\n", "# h = ax1.hist2d(\n", "# x_data,\n", "# y_data,\n", "# bins=(50, 50),\n", "# cmap=\"coolwarm\",\n", "# range=[[0, 1], [0, 1]],\n", "# norm=LogNorm(),\n", "# )\n", "\n", "# ax1.set_xlabel(\"Semantic Distance (Preferred)\")\n", "# ax1.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "# ax1.set_title(\n", "# \"2D Histogram of Semantic Distances: Preferred vs Non-Preferred (Log Scale)\"\n", "# )\n", "\n", "# cbar1 = plt.colorbar(h[3], ax=ax1)\n", "# cbar1.set_label(\"Number of Request IDs (Log Scale)\")\n", "\n", "# # Scatter plot\n", "# ax2.scatter(x_data, y_data, alpha=0.1, s=1)\n", "# ax2.set_xlabel(\"Semantic Distance (Preferred)\")\n", "# ax2.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "# ax2.set_title(\"Scatter Plot of Semantic Distances: Preferred vs Non-Preferred\")\n", "# ax2.set_xlim(0, 1)\n", "# ax2.set_ylim(0, 1)\n", "\n", "# plt.tight_layout()\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.205088Z", "iopub.status.idle": "2025-11-15T06:00:44.205236Z" } }, "outputs": [], "source": [ "# train_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_tr.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.205653Z", "iopub.status.idle": "2025-11-15T06:00:44.205796Z" } }, "outputs": [], "source": [ "# train_info.keys()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.206039Z", "iopub.status.idle": "2025-11-15T06:00:44.206179Z" } }, "outputs": [], "source": [ "# import torch\n", "\n", "# a = torch.tensor([6.2500e-04, 3.9062e-05, 2.3462e-03])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.206537Z", "iopub.status.idle": "2025-11-15T06:00:44.206681Z" } }, "outputs": [], "source": [ "# a.mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.207129Z", "iopub.status.idle": "2025-11-15T06:00:44.207270Z" } }, "outputs": [], "source": [ "# import time\n", "# time.sleep(3600 * 3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.207503Z", "iopub.status.idle": "2025-11-15T06:00:44.207655Z" } }, "outputs": [], "source": [ "# !cd /home/tony/Work/tony/slurm/diffusion && sbatch run_diffusion_infill.sh" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.207931Z", "iopub.status.idle": "2025-11-15T06:00:44.208071Z" } }, "outputs": [], "source": [ "def get_control_slider(metadata):\n", " if \"param_experiment\" in metadata:\n", " exp = metadata.get(\"param_experiment\", \"\")\n", " if exp:\n", " if exp == \"mask_control_slider\":\n", " if not metadata.get(\"control_sliders\", None):\n", " return False\n", " else:\n", " return True\n", " return None\n", "\n", "df[\"mask_control\"] = df.apply(\n", " lambda row: get_control_slider(row[\"metadata\"]), axis=1\n", ")\n", "masked_request_id = df[~df[\"mask_control\"].isna()][\"request_id\"].unique()\n", "subset_df = df[df[\"request_id\"].isin(masked_request_id)].copy()\n", "print(\"check subset shape\", df.shape, subset_df.shape)\n", "# control masked\n", "user_pref_pct = (\n", " subset_df[subset_df[\"mask_control\"] == True].groupby(\"user_id\")[\"preference\"]\n", " .apply(lambda x: x.mean())\n", ")\n", "print(user_pref_pct.head())\n", "# Plot a histogram of the user preference percentages\n", "plt.figure(figsize=(6, 4))\n", "plt.hist(user_pref_pct, bins=np.linspace(0, 1, 100), edgecolor=\"black\")\n", "plt.xlabel(\"Preference % for mask_control\")\n", "plt.ylabel(\"Number of Users\")\n", "plt.title(\"Histogram of User Preference for 'mask_control'\")\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.208552Z", "iopub.status.idle": "2025-11-15T06:00:44.208695Z" } }, "outputs": [], "source": [ "user_counts = subset_df.groupby(\"user_id\")[\"preference\"].count()\n", "eligible_users = user_counts[user_counts >= 4].index\n", "len(eligible_users)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.208935Z", "iopub.status.idle": "2025-11-15T06:00:44.209071Z" } }, "outputs": [], "source": [ "# control masked\n", "eligible_user_pref_pct = (\n", " subset_df[subset_df[\"user_id\"].isin(eligible_users) & (subset_df[\"mask_control\"] == True)].groupby(\"user_id\")[\"preference\"]\n", " .apply(lambda x: x.mean())\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.209463Z", "iopub.status.idle": "2025-11-15T06:00:44.209609Z" } }, "outputs": [], "source": [ "# Plot a histogram of the user preference percentages\n", "plt.figure(figsize=(6, 4))\n", "plt.hist(eligible_user_pref_pct, bins=np.linspace(0, 1, 20), edgecolor=\"black\")\n", "plt.xlabel(\"Preference % for mask_control\")\n", "plt.ylabel(\"Number of Users\")\n", "plt.title(\"Histogram of User Preference for 'mask_control'\")\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.209885Z", "iopub.status.idle": "2025-11-15T06:00:44.210022Z" } }, "outputs": [], "source": [ "eligible_user_pref_pct[172918] # kakermix" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## sliders" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.210363Z", "iopub.status.idle": "2025-11-15T06:00:44.210503Z" } }, "outputs": [], "source": [ "from typing import Any\n", "import pandas as pd\n", "\n", "def map_control_sliders_to_df(df: pd.DataFrame) -> pd.DataFrame:\n", " \"\"\"Extracts 'style_weight' and 'weirdness_constraint' from the 'control_sliders' dict in the 'metadata' column\n", " and adds them as new columns to the DataFrame.\n", "\n", " Args:\n", " df (pd.DataFrame): DataFrame with a 'metadata' column containing a 'control_sliders' dict.\n", "\n", " Returns:\n", " pd.DataFrame: DataFrame with added 'style_weight' and 'weirdness_constraint' columns.\n", "\n", " Raises:\n", " KeyError: If 'control_sliders', 'style_weight', or 'weirdness_constraint' are missing in any row.\n", " TypeError: If the extracted values are not floats.\n", "\n", " Example:\n", " >>> import pandas as pd\n", " >>> data = [{'metadata': {'control_sliders': {'style_weight': 0.89, 'weirdness_constraint': 0.8}}}]\n", " >>> df = pd.DataFrame(data)\n", " >>> df = map_control_sliders_to_df(df)\n", " >>> df[['style_weight', 'weirdness_constraint']].iloc[0].tolist()\n", " [0.89, 0.8]\n", " \"\"\"\n", " # Vectorized extraction for performance\n", " sliders = df[\"metadata\"].map(lambda m: m.get(\"control_sliders\", {}))\n", " style_weight = sliders.map(lambda s: s.get(\"style_weight\", None))\n", " weirdness_constraint = sliders.map(lambda s: s.get(\"weirdness_constraint\", None))\n", " audio_weight = sliders.map(lambda s: s.get(\"audio_weight\", None))\n", "\n", " df[\"style_weight\"] = style_weight\n", " df[\"weirdness_constraint\"] = weirdness_constraint\n", " df[\"audio_weight\"] = audio_weight\n", " return df\n", "\n", "def print_percentiles(\n", " data: np.ndarray,\n", " percentiles: list[float] = [5, 10, 20, 25, 50, 75, 80, 90, 95]\n", ") -> None:\n", " \"\"\"Prints specified percentiles of the data.\n", "\n", " Args:\n", " data (np.ndarray): Array of values to compute percentiles for.\n", " percentiles (list[float], optional): List of percentiles to print. Defaults to [5, 10, 20, 25, 50, 75, 80, 90, 95].\n", "\n", " Example:\n", " >>> print_percentiles(np.array([1, 2, 3, 4, 5]))\n", " \"\"\"\n", " data = data[~data.isna()]\n", " results = np.percentile(data, percentiles)\n", " print(\"Percentiles:\")\n", " for p, v in zip(percentiles, results):\n", " print(f\" {p:>3}%: {v:.4f}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.210784Z", "iopub.status.idle": "2025-11-15T06:00:44.210920Z" } }, "outputs": [], "source": [ "df_total = map_control_sliders_to_df(df_total)\n", "df_total[[\"style_weight\",\"weirdness_constraint\",\"audio_weight\"]].describe()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.211212Z", "iopub.status.idle": "2025-11-15T06:00:44.211351Z" } }, "outputs": [], "source": [ "plt.figure(figsize=(8, 5))\n", "df_total[\"style_weight\"].hist(bins=np.linspace(0, 1, 100), color=\"skyblue\", edgecolor=\"black\")\n", "print_percentiles(df_total[\"style_weight\"])\n", "plt.title(\"Distribution of Style Weight\")\n", "plt.xlabel(\"Style Weight\")\n", "plt.ylabel(\"Counts\")\n", "plt.grid(True, linestyle=\"--\", alpha=0.6)\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.211841Z", "iopub.status.idle": "2025-11-15T06:00:44.211995Z" } }, "outputs": [], "source": [ "plt.figure(figsize=(8, 5))\n", "df_total[\"weirdness_constraint\"].hist(bins=np.linspace(0, 1, 100), color=\"skyblue\", edgecolor=\"black\")\n", "print_percentiles(df_total[\"weirdness_constraint\"])\n", "plt.title(\"Distribution of Weirdness\")\n", "plt.xlabel(\"weirdness_constraint\")\n", "plt.ylabel(\"Counts\")\n", "plt.grid(True, linestyle=\"--\", alpha=0.6)\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.212231Z", "iopub.status.idle": "2025-11-15T06:00:44.212372Z" } }, "outputs": [], "source": [ "plt.figure(figsize=(8, 5))\n", "df_total[\"audio_weight\"].hist(bins=np.linspace(0, 1, 100), color=\"skyblue\", edgecolor=\"black\")\n", "print_percentiles(df_total[\"audio_weight\"])\n", "plt.title(\"Distribution of audio_weight\")\n", "plt.xlabel(\"audio_weight\")\n", "plt.ylabel(\"Counts\")\n", "plt.grid(True, linestyle=\"--\", alpha=0.6)\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Fetch other parameters" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.212782Z", "iopub.status.idle": "2025-11-15T06:00:44.212925Z" } }, "outputs": [], "source": [ "# df_total[df_total[\"task\"] == \"stem_condition\"][[\"s3_id\", \"preference\", \"request_id\", \"control_tags\"]].head(n=100)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.213150Z", "iopub.status.idle": "2025-11-15T06:00:44.213285Z" } }, "outputs": [], "source": [ "len(metas_tr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.213784Z", "iopub.status.idle": "2025-11-15T06:00:44.213938Z" } }, "outputs": [], "source": [ "c = Counter()\n", "for data_meta in metas_tr:\n", " c[data_meta.get(\"control_tags\", \"\") if isinstance(data_meta.get(\"control_tags\", \"\"), str) else \"\"] += 1\n", "print(c)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.214550Z", "iopub.status.idle": "2025-11-15T06:00:44.214714Z" } }, "outputs": [], "source": [ "# with open(\"/app2/suno/data/dpo/crow_t1_v3/dodo_t23_2025-08-28_01-30-09_cached_loss.json\", \"r\") as fp:\n", "# old_loss = json.load(fp)\n", "# with open(\"/app2/suno/data/dpo/crow_t1_v3/dodo_t23_2025-08-28_01-30-09_redo3_cached_loss.json\", \"r\") as fp:\n", "# new_loss = json.load(fp)\n", "\n", "# sem_losses = []\n", "# for curr_id, o_v in old_loss[\"train\"].items():\n", "# n_v = new_loss[\"train\"][curr_id]\n", "# for k in o_v:\n", "# assert o_v[k] == n_v[k]\n", "# if k == \"semantic_0\":\n", "# if o_v[\"orig_idx\"] % 2 == 1:\n", "# sem_losses.append(o_v[k])\n", "# # if o_v[k] > 4 and o_v[\"orig_idx\"] % 2 == 1:\n", "# # print(o_v, o_v[\"orig_idx\"] % 3360, o_v[\"orig_idx\"] % 3360 // 4)\n", "# sem_losses.sort()\n", "# plt.hist(sem_losses, bins=np.linspace(0, 10, 100))\n", "# plt.yscale(\"log\")\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.215020Z", "iopub.status.idle": "2025-11-15T06:00:44.215160Z" } }, "outputs": [], "source": [ "# df_total[df_total[\"is_public\"]].to_pickle(\"/home/tony/Data/Preference/crow_t1/interesting_clips_crow_t1_20251030_public_only.pkl\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2025-11-15T06:00:44.215565Z", "iopub.status.idle": "2025-11-15T06:00:44.215709Z" } }, "outputs": [], "source": [ "df[df[\"is_public\"] & (df[\"tags\"].str.len() < 1)]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }