{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.040680Z", "start_time": "2024-05-16T13:58:19.777010Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:49:02.439236Z", "iopub.status.busy": "2024-07-19T04:49:02.439077Z", "iopub.status.idle": "2024-07-19T04:49:03.701223Z", "shell.execute_reply": "2024-07-19T04:49:03.700650Z", "shell.execute_reply.started": "2024-07-19T04:49:02.439210Z" } }, "outputs": [], "source": [ "import ast\n", "import os\n", "import shutil\n", "import sys\n", "from collections import defaultdict\n", "\n", "import numpy as np\n", "import pandas as pd\n", "from preference_data_preparation_4min_13b import *\n", "from preference_helper import *\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n", "from tqdm import tqdm\n", "\n", "pd.set_option(\"display.max_rows\", 500)\n", "pd.set_option(\"display.max_columns\", 500)\n", "pd.set_option(\"display.width\", 1000)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.082172Z", "start_time": "2024-05-16T13:58:21.041926Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:49:03.702140Z", "iopub.status.busy": "2024-07-19T04:49:03.701929Z", "iopub.status.idle": "2024-07-19T04:49:03.723961Z", "shell.execute_reply": "2024-07-19T04:49:03.723468Z", "shell.execute_reply.started": "2024-07-19T04:49:03.702122Z" } }, "outputs": [], "source": [ "OUT_DATA_DIR = \"/app/suno/data/dpo/13b_2h_ft1_v3/\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "shutil.copyfile(\n", " \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", " os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", ")\n", "NPZ_DIR = \"/app/suno/data/dpo/13b_npz\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:53.962528Z", "start_time": "2024-05-16T13:58:21.105919Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:49:03.724758Z", "iopub.status.busy": "2024-07-19T04:49:03.724606Z", "iopub.status.idle": "2024-07-19T04:49:08.170462Z", "shell.execute_reply": "2024-07-19T04:49:08.169866Z", "shell.execute_reply.started": "2024-07-19T04:49:03.724741Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Preference data shape (154513, 52)\n" ] } ], "source": [ "df = pd.read_csv(\n", " \"/home/tony/Data/Preference/13b_v0/interesting_clips_ft_1_20240718.csv\"\n", ") # , engine='python')\n", "print(\"Preference data shape\", df.shape)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.199480Z", "start_time": "2024-05-16T13:58:53.963687Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:49:08.171325Z", "iopub.status.busy": "2024-07-19T04:49:08.171163Z", "iopub.status.idle": "2024-07-19T04:51:14.396602Z", "shell.execute_reply": "2024-07-19T04:51:14.395803Z", "shell.execute_reply.started": "2024-07-19T04:49:08.171308Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5367038\n", "5367038\n", "pre-downloaded df (154513, 52)\n", "downloaded df (154513, 52)\n" ] } ], "source": [ "converted_paths = os.listdir(NPZ_DIR)\n", "print(len(converted_paths))\n", "\n", "converted_paths = set([f.replace(\".npz\", \"\") for f in converted_paths])\n", "print(len(converted_paths))\n", "\n", "print(\"pre-downloaded df\", df.shape)\n", "df[df[\"s3_id\"].isin(converted_paths)].shape\n", "df = df[df[\"s3_id\"].isin(converted_paths)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.467253Z", "start_time": "2024-05-16T13:58:56.207647Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:14.397693Z", "iopub.status.busy": "2024-07-19T04:51:14.397520Z", "iopub.status.idle": "2024-07-19T04:51:14.658716Z", "shell.execute_reply": "2024-07-19T04:51:14.658078Z", "shell.execute_reply.started": "2024-07-19T04:51:14.397673Z" } }, "outputs": [ { "data": { "text/plain": [ "is_13b\n", "True 154513\n", "Name: count, dtype: int64" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"is_13b\"] = df[\"model_name\"].str.contains(\"v3p5\")\n", "df[\"is_13b\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.592883Z", "start_time": "2024-05-16T13:58:56.470781Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:14.661110Z", "iopub.status.busy": "2024-07-19T04:51:14.660598Z", "iopub.status.idle": "2024-07-19T04:51:14.721888Z", "shell.execute_reply": "2024-07-19T04:51:14.721209Z", "shell.execute_reply.started": "2024-07-19T04:51:14.661089Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "preference model_name \n", "False chirp-v3p5-engine-ft-1 77772\n", "True chirp-v3p5-engine-ft-1 76741\n", "Name: count, dtype: int64\n", "(154513, 53)\n", "(154513, 53)\n" ] } ], "source": [ "## for 13b this is easy for now\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "print(df.shape)\n", "df = df[df[\"model_name\"].isin([\"chirp-v3p5-engine-ft-1\"])]\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.909539Z", "start_time": "2024-05-16T13:58:56.595736Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:14.722912Z", "iopub.status.busy": "2024-07-19T04:51:14.722746Z", "iopub.status.idle": "2024-07-19T04:51:14.851191Z", "shell.execute_reply": "2024-07-19T04:51:14.850441Z", "shell.execute_reply.started": "2024-07-19T04:51:14.722892Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(154513, 53)\n", "(145314, 53)\n", "preference model_name \n", "False chirp-v3p5-engine-ft-1 72657\n", "True chirp-v3p5-engine-ft-1 72657\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(df.shape)\n", "df = df[\n", " df[\"request_id\"].isin(\n", " df[\"request_id\"].value_counts().index[df[\"request_id\"].value_counts() == 2]\n", " )\n", "]\n", "print(df.shape)\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "assert df.shape[0] == df[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:36.043975Z", "start_time": "2024-05-16T13:58:56.910958Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:14.852263Z", "iopub.status.busy": "2024-07-19T04:51:14.852086Z", "iopub.status.idle": "2024-07-19T04:51:48.228742Z", "shell.execute_reply": "2024-07-19T04:51:48.227982Z", "shell.execute_reply.started": "2024-07-19T04:51:14.852244Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "unique_requests 72657\n" ] } ], "source": [ "# Let's use the old selection for now -- for quality assurance\n", "# expand the metadata columns -- this takes forever...~ 6 mins\n", "test_slice = df[\"metadata\"].apply(lambda x: ast.literal_eval(x))\n", "test_slice_series = test_slice.apply(pd.Series)\n", "df = pd.concat([df, test_slice_series], axis=1, join=\"inner\")\n", "print(\"unique_requests\", df[\"request_id\"].nunique())" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:36.393047Z", "start_time": "2024-05-16T13:59:36.048831Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:48.229820Z", "iopub.status.busy": "2024-07-19T04:51:48.229647Z", "iopub.status.idle": "2024-07-19T04:51:48.246101Z", "shell.execute_reply": "2024-07-19T04:51:48.245513Z", "shell.execute_reply.started": "2024-07-19T04:51:48.229801Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "unique_requests 72657\n" ] } ], "source": [ "# GPT requests are also fine for now\n", "print(\"unique_requests\", df[\"request_id\"].nunique())" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T04:51:48.246967Z", "iopub.status.busy": "2024-07-19T04:51:48.246818Z", "iopub.status.idle": "2024-07-19T04:51:48.654246Z", "shell.execute_reply": "2024-07-19T04:51:48.653488Z", "shell.execute_reply.started": "2024-07-19T04:51:48.246950Z" } }, "outputs": [], "source": [ "df = df.loc[:,~df.columns.duplicated()].copy()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:40.799375Z", "start_time": "2024-05-16T13:59:36.394236Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:48.655581Z", "iopub.status.busy": "2024-07-19T04:51:48.655411Z", "iopub.status.idle": "2024-07-19T04:51:49.161760Z", "shell.execute_reply": "2024-07-19T04:51:49.160982Z", "shell.execute_reply.started": "2024-07-19T04:51:48.655562Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5540\n", "good_continue_at\n", "True 145156\n", "False 158\n", "Name: count, dtype: int64\n", "\n", " Check some basics... \n", " preference\n", "False 72657\n", "True 72657\n", "Name: count, dtype: int64 is_13b\n", "True 145314\n", "Name: count, dtype: int64 model_name\n", "chirp-v3p5-engine-ft-1 145314\n", "Name: count, dtype: int64 preference model_name \n", "False chirp-v3p5-engine-ft-1 72657\n", "True chirp-v3p5-engine-ft-1 72657\n", "Name: count, dtype: int64\n" ] } ], "source": [ "# get the original duration of the clips, if they are concacted\n", "df[\"original_duration_s\"] = df[\"total_start_s\"] + df[\"duration\"]\n", "# classify the continue at behavoirs by the duration choice\n", "audio_prompt_id_to_continue_at = {}\n", "for _, row in df[~df[\"audio_prompt_id\"].isna()].iterrows():\n", " audio_prompt_id = row[\"audio_prompt_id\"]\n", " if audio_prompt_id not in audio_prompt_id_to_continue_at:\n", " audio_prompt_id_to_continue_at[audio_prompt_id] = row[\"continue_at\"]\n", " else:\n", " # pick the max\n", " audio_prompt_id = max(\n", " audio_prompt_id_to_continue_at[audio_prompt_id], row[\"continue_at\"]\n", " )\n", "print(len(audio_prompt_id_to_continue_at))\n", "df[\"has_continue_and_start_continue_at\"] = df[\"id\"].apply(\n", " lambda x: audio_prompt_id_to_continue_at.get(x)\n", ")\n", "# we want continue at to be at most of the clip...\n", "df[\"good_continue_at\"] = (\n", " (df[\"has_continue_and_start_continue_at\"] / df[\"duration\"]) > 0.9\n", ") | df[\"has_continue_and_start_continue_at\"].isna()\n", "print(df[\"good_continue_at\"].value_counts())\n", "\n", "\n", "print(\n", " \"\\n Check some basics... \\n\",\n", " df[\"preference\"].value_counts(),\n", " df[\"is_13b\"].value_counts(),\n", " df[\"model_name\"].value_counts(),\n", " df.groupby([\"preference\"])[\"model_name\"].value_counts(),\n", ")\n", "\n", "df = df.sort_values(by=[\"request_id\", \"preference\"])\n", "df[\"duration_rel_diff\"] = df[\"duration\"].diff()\n", "df[\"play_rel_diff\"] = df[\"reaction_play_count\"].diff()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.035167Z", "start_time": "2024-05-16T13:59:40.801098Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:49.162850Z", "iopub.status.busy": "2024-07-19T04:51:49.162666Z", "iopub.status.idle": "2024-07-19T04:51:49.287511Z", "shell.execute_reply": "2024-07-19T04:51:49.286744Z", "shell.execute_reply.started": "2024-07-19T04:51:49.162830Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "negative 70942 positive 21699\n", "total pair requests 72657 selected pair requests 21309 frac 0.293\n" ] } ], "source": [ "normal_pos_play_count = 5\n", "# this is lower, cause a concat is probably already ensuring that it is good\n", "concat_pos_play_count = 1\n", "# this is a filter on the concated clip\n", "concat_total_play_count = 5\n", "\n", "neg_filter_selection_mask = (\n", " (df[\"preference\"] == False) # get basics aligned\n", " & (df[\"reaction_play_count\"] >= 1) # has to be played once\n", " # & (df[\"play_count\"] <= 3) # if it is actually bad, shouldn't be listened often\n", " & (df[\"duration\"] >= 10) # can't be too short, otherwise it is obvious\n", " & (df[\"duration\"] <= 240) # can't be badly long\n", " & (df[\"has_continue_and_start_continue_at\"].isna()) # won't have any continues\n", " # & (df[\"dislike_count\"] >= 1) # this is kinda strict\n", " # & (\n", " # (df_slice[\"is_in_playlist\"] == False)\n", " # & (df_slice[\"concat_in_playlist\"] == False)\n", " # ) # can't be part of a playlist -- otherwise there are some like signal in it?\n", ")\n", "pos_filter_selectin_mask = (\n", " (df[\"preference\"] == True) # get basics aligned\n", " & (\n", " df[\"good_continue_at\"] == True\n", " ) # if continue, needs to continue off a certain percentage\n", " & (df[\"reaction_play_count\"] >= 1)\n", " # & (df[\"play_rel_diff\"] >= 0) # this is more like quality assurance\n", " & (df[\"duration\"] >= 10) # can't be too short, otherwise it is obvious\n", " & (df[\"duration\"] <= 240) # can't be badly long\n", " & (df[\"dislike_count\"] == 0) # can't have dislikes\n", " & (df[\"flag_count\"] == 0) # can't have issues\n", " & (\n", " (\n", " (df[\"part_of_concat\"] == True)\n", " & (df[\"play_count\"] >= concat_pos_play_count)\n", " & (df[\"concat_play_counts\"] >= concat_total_play_count)\n", " )\n", " | (\n", " (df[\"part_of_concat\"] == False)\n", " & (df[\"play_count\"] >= normal_pos_play_count)\n", " )\n", " )\n", " & (df[\"user_n_clips\"] >= 10) # user needs to have genereated at least 20\n", " # & (df[\"duration_rel_diff\"] < 10) # positive isn't just longer\n", " # & (df[\"upvote_count\"] >= 1)\n", ")\n", "print(\n", " \"negative\",\n", " sum(neg_filter_selection_mask),\n", " \"positive\",\n", " sum(pos_filter_selectin_mask),\n", ")\n", "\n", "neg_filter_requests = df[neg_filter_selection_mask][\"request_id\"].unique()\n", "pos_filter_requests = df[pos_filter_selectin_mask][\"request_id\"].unique()\n", "# looking for very strong signal here:\n", "# listen to the positive/negative more than once\n", "# disliked one of the clips\n", "unique_requests = set(pos_filter_requests).intersection(neg_filter_requests)\n", "print(\n", " \"total pair requests\",\n", " df[\"request_id\"].nunique(),\n", " \"selected pair requests\",\n", " len(unique_requests),\n", " f\"frac {len(unique_requests) / df['request_id'].nunique():.3f}\",\n", ")" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.250737Z", "start_time": "2024-05-16T13:59:41.036434Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:49.288562Z", "iopub.status.busy": "2024-07-19T04:51:49.288386Z", "iopub.status.idle": "2024-07-19T04:51:49.344903Z", "shell.execute_reply": "2024-07-19T04:51:49.344166Z", "shell.execute_reply.started": "2024-07-19T04:51:49.288542Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "requests 21309 clips 42618 total khrs 2.522; N gpus for 1000 iters 2.664; n unique users 18060\n" ] } ], "source": [ "df_slice = df[df[\"request_id\"].isin(set(unique_requests))].copy()\n", "print(\n", " \"requests\",\n", " df_slice[\"request_id\"].nunique(),\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", ")\n", "# 76171 152342 total khrs 2.880 n gpus for 1250 iters 3.809" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.277006Z", "start_time": "2024-05-16T13:59:41.252105Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:49.345925Z", "iopub.status.busy": "2024-07-19T04:51:49.345763Z", "iopub.status.idle": "2024-07-19T04:51:49.352221Z", "shell.execute_reply": "2024-07-19T04:51:49.351630Z", "shell.execute_reply.started": "2024-07-19T04:51:49.345906Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive in playlist (2429, 80)\n" ] } ], "source": [ "test_mask = (df_slice[\"preference\"] == True) & (\n", " (df_slice[\"is_in_playlist\"] == True) | (df_slice[\"concat_in_playlist\"] == True)\n", ")\n", "print(\"positive in playlist\", df_slice[test_mask].shape)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.323409Z", "start_time": "2024-05-16T13:59:41.278278Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:49.353115Z", "iopub.status.busy": "2024-07-19T04:51:49.352951Z", "iopub.status.idle": "2024-07-19T04:51:49.390076Z", "shell.execute_reply": "2024-07-19T04:51:49.389593Z", "shell.execute_reply.started": "2024-07-19T04:51:49.353098Z" } }, "outputs": [], "source": [ "# interesting_clips_must_be_positive_mask = (\n", "# (df_slice[\"upvoted\"] == True)\n", "# | (df_slice[\"has_action\"] == True)\n", "# | (df_slice[\"part_of_concat\"] == True)\n", "# )\n", "# interesting_clips_must_be_not_negative_mask = (df_slice[\"downvoted\"] == False) # & (df_slice[\"dislike_count\"] < 1)\n", "# interesting_clips_mask = interesting_clips_must_be_positive_mask & interesting_clips_must_be_not_negative_mask\n", "# assert interesting_clips_mask.eq(df_slice[\"preference\"]).all()" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.392244Z", "start_time": "2024-05-16T13:59:41.324472Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:49.390843Z", "iopub.status.busy": "2024-07-19T04:51:49.390707Z", "iopub.status.idle": "2024-07-19T04:51:49.434961Z", "shell.execute_reply": "2024-07-19T04:51:49.434480Z", "shell.execute_reply.started": "2024-07-19T04:51:49.390828Z" } }, "outputs": [], "source": [ "# save positive ids\n", "# positive_preference_ids = df_slice[df_slice[\"preference\"] == False][\"s3_id\"].to_json(orient='values')\n", "# with open('/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id_negative.json', 'w') as file:\n", "# file.write(positive_preference_ids)" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T14:00:20.866354Z", "start_time": "2024-05-16T14:00:12.443344Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:49.435747Z", "iopub.status.busy": "2024-07-19T04:51:49.435607Z", "iopub.status.idle": "2024-07-19T04:51:49.475062Z", "shell.execute_reply": "2024-07-19T04:51:49.474588Z", "shell.execute_reply.started": "2024-07-19T04:51:49.435731Z" } }, "outputs": [], "source": [ "# df_slice.to_csv(\"/home/tony/Data/Preference/13b_v0/interesting_clips_20240623_slice.csv\")\n", "# BREAK" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932296Z", "start_time": "2024-05-16T13:59:41.932287Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:49.476051Z", "iopub.status.busy": "2024-07-19T04:51:49.475916Z", "iopub.status.idle": "2024-07-19T04:51:49.546863Z", "shell.execute_reply": "2024-07-19T04:51:49.546309Z", "shell.execute_reply.started": "2024-07-19T04:51:49.476037Z" } }, "outputs": [ { "data": { "text/plain": [ "20003" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# don't have continue at\n", "df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932966Z", "start_time": "2024-05-16T13:59:41.932957Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:49.547750Z", "iopub.status.busy": "2024-07-19T04:51:49.547604Z", "iopub.status.idle": "2024-07-19T04:51:49.574864Z", "shell.execute_reply": "2024-07-19T04:51:49.574304Z", "shell.execute_reply.started": "2024-07-19T04:51:49.547734Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "21309\n" ] } ], "source": [ "final_filtered_requests = df_slice[\"request_id\"].unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.933558Z", "start_time": "2024-05-16T13:59:41.933550Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:49.575689Z", "iopub.status.busy": "2024-07-19T04:51:49.575539Z", "iopub.status.idle": "2024-07-19T04:51:49.613619Z", "shell.execute_reply": "2024-07-19T04:51:49.613136Z", "shell.execute_reply.started": "2024-07-19T04:51:49.575673Z" } }, "outputs": [], "source": [ "# df_slice.to_csv(\"/home/tony/Data/Preference/7b_v2/7b_before_recode_20240412\", index=False)" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934277Z", "start_time": "2024-05-16T13:59:41.934268Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:49.616641Z", "iopub.status.busy": "2024-07-19T04:51:49.616118Z", "iopub.status.idle": "2024-07-19T04:51:49.742324Z", "shell.execute_reply": "2024-07-19T04:51:49.741579Z", "shell.execute_reply.started": "2024-07-19T04:51:49.616623Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "21095 214\n", "(42190, 81) (428, 81)\n" ] } ], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df.reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df.reset_index()\n", "\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934954Z", "start_time": "2024-05-16T13:59:41.934946Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:49.743561Z", "iopub.status.busy": "2024-07-19T04:51:49.743395Z", "iopub.status.idle": "2024-07-19T04:51:49.745816Z", "shell.execute_reply": "2024-07-19T04:51:49.745333Z", "shell.execute_reply.started": "2024-07-19T04:51:49.743542Z" } }, "outputs": [], "source": [ "# val_df[[\"request_id\", \"metadata\", \"updated_at\", \"user_id\", \"preference\"]].head()" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.935620Z", "start_time": "2024-05-16T13:59:41.935613Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:49.746731Z", "iopub.status.busy": "2024-07-19T04:51:49.746587Z", "iopub.status.idle": "2024-07-19T04:51:51.042393Z", "shell.execute_reply": "2024-07-19T04:51:51.041629Z", "shell.execute_reply.started": "2024-07-19T04:51:49.746715Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 42190/42190 [00:01<00:00, 33763.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2,497 hours of 42190 clips, 1.3184375 nodes\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "total_duration = 0\n", "for i, row in tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " except:\n", " print(i, row)\n", " total_duration += row[\"duration\"]\n", "print(\n", " f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 4 / 1000} nodes\"\n", ")" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936268Z", "start_time": "2024-05-16T13:59:41.936260Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:51.043422Z", "iopub.status.busy": "2024-07-19T04:51:51.043254Z", "iopub.status.idle": "2024-07-19T04:51:59.059221Z", "shell.execute_reply": "2024-07-19T04:51:59.058478Z", "shell.execute_reply.started": "2024-07-19T04:51:51.043402Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 428/428 [00:07<00:00, 53.56it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 428 clips\n", "13 hours of False\n", "13 hours of True\n", "Done\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "make_dataset(val_df, OUT_DATA_DIR, is_val=True, npz_dir=NPZ_DIR)" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936964Z", "start_time": "2024-05-16T13:59:41.936957Z" }, "execution": { "iopub.execute_input": "2024-07-19T04:51:59.060299Z", "iopub.status.busy": "2024-07-19T04:51:59.060121Z", "iopub.status.idle": "2024-07-19T05:04:24.669990Z", "shell.execute_reply": "2024-07-19T05:04:24.669372Z", "shell.execute_reply.started": "2024-07-19T04:51:59.060280Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 42190/42190 [12:25<00:00, 56.59it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 42190 clips\n", "1,250 hours of False\n", "1,248 hours of True\n", "Done\n" ] } ], "source": [ "make_dataset(train_df, OUT_DATA_DIR, is_val=False, npz_dir=NPZ_DIR)" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.937879Z", "start_time": "2024-05-16T13:59:41.937870Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:24.670929Z", "iopub.status.busy": "2024-07-19T05:04:24.670759Z", "iopub.status.idle": "2024-07-19T05:04:24.690310Z", "shell.execute_reply": "2024-07-19T05:04:24.689766Z", "shell.execute_reply.started": "2024-07-19T05:04:24.670909Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, 6016, 13)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000\n", "assert mm[:100, :, 1:].min() >= 0\n", "assert mm[:100, :, 1:].max() <= 2048" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.938629Z", "start_time": "2024-05-16T13:59:41.938621Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:24.691178Z", "iopub.status.busy": "2024-07-19T05:04:24.691020Z", "iopub.status.idle": "2024-07-19T05:04:24.721059Z", "shell.execute_reply": "2024-07-19T05:04:24.720615Z", "shell.execute_reply.started": "2024-07-19T05:04:24.691159Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as codec_encode,\n", "# decode_stream_to_full_audio as codec_decode,\n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# decode as decode\n", "# )\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "# assert len(test_metas) == len(mm)\n", "# idx_list = list(range(len(test_metas)))\n", "# # random.shuffle(idx_list)\n", "# # idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939205Z", "start_time": "2024-05-16T13:59:41.939198Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:24.721998Z", "iopub.status.busy": "2024-07-19T05:04:24.721854Z", "iopub.status.idle": "2024-07-19T05:04:24.758831Z", "shell.execute_reply": "2024-07-19T05:04:24.758397Z", "shell.execute_reply.started": "2024-07-19T05:04:24.721982Z" } }, "outputs": [], "source": [ "# import random\n", "# idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "# assert \"original_duration_s\" in test_metas[idx]\n", "# # positive index should be shifted by 1\n", "# pos_idx = idx + 1\n", "# print(\n", "# \"tags:\",\n", "# test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", "# test_metas[idx].get(\"tags\"),\n", "# )\n", "# arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[: pad_idx_arr[0], :]\n", "# pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pos_pad_idx_arr) > 0:\n", "# pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "# a = decode(arr)\n", "# print(\"\\n negative example \\n\", test_metas[idx])\n", "# a.play(compress=False)\n", "# pos_a = decode(pos_arr)\n", "# print(\"\\n positive example \\n\", test_metas[pos_idx])\n", "# pos_a.play(compress=False)\n", "# print(\n", "# \"text:\",\n", "# test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", "# test_metas[idx].get(\"text\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939977Z", "start_time": "2024-05-16T13:59:41.939969Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:24.759563Z", "iopub.status.busy": "2024-07-19T05:04:24.759417Z", "iopub.status.idle": "2024-07-19T05:04:24.807102Z", "shell.execute_reply": "2024-07-19T05:04:24.806687Z", "shell.execute_reply.started": "2024-07-19T05:04:24.759547Z" } }, "outputs": [], "source": [ "# val_df[val_df[\"tags\"] == 'a vibrant blend of experimental jazz fusion, drum-and-bass and swagger fuzzed-out guitars']" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.940610Z", "start_time": "2024-05-16T13:59:41.940603Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:24.807809Z", "iopub.status.busy": "2024-07-19T05:04:24.807674Z", "iopub.status.idle": "2024-07-19T05:04:24.844310Z", "shell.execute_reply": "2024-07-19T05:04:24.843889Z", "shell.execute_reply.started": "2024-07-19T05:04:24.807793Z" } }, "outputs": [], "source": [ "# from collections import Counter\n", "# c = Counter()\n", "# for _, row in df_slice.iterrows():\n", "# # print(row[\"metadata\"])\n", "# for k in ast.literal_eval(row[\"metadata\"]).keys():\n", "# c[k] += 1" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941167Z", "start_time": "2024-05-16T13:59:41.941159Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:24.845149Z", "iopub.status.busy": "2024-07-19T05:04:24.845011Z", "iopub.status.idle": "2024-07-19T05:04:24.882641Z", "shell.execute_reply": "2024-07-19T05:04:24.882245Z", "shell.execute_reply.started": "2024-07-19T05:04:24.845134Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941801Z", "start_time": "2024-05-16T13:59:41.941793Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:24.883299Z", "iopub.status.busy": "2024-07-19T05:04:24.883162Z", "iopub.status.idle": "2024-07-19T05:04:24.925291Z", "shell.execute_reply": "2024-07-19T05:04:24.924800Z", "shell.execute_reply.started": "2024-07-19T05:04:24.883283Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "214 0\n" ] } ], "source": [ "def validation_on_metas(input_metas):\n", "\n", " total_bad = 0\n", " total_good = 0\n", " for idx in range(len(input_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if input_metas[idx].get(\"tags\") != input_metas[pos_idx].get(\"tags\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", " print(total_good, total_bad)\n", " return\n", "\n", "\n", "validation_on_metas(test_metas)" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.942520Z", "start_time": "2024-05-16T13:59:41.942511Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:24.926032Z", "iopub.status.busy": "2024-07-19T05:04:24.925893Z", "iopub.status.idle": "2024-07-19T05:04:24.969432Z", "shell.execute_reply": "2024-07-19T05:04:24.968926Z", "shell.execute_reply.started": "2024-07-19T05:04:24.926016Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.943072Z", "start_time": "2024-05-16T13:59:41.943065Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:24.970339Z", "iopub.status.busy": "2024-07-19T05:04:24.970198Z", "iopub.status.idle": "2024-07-19T05:04:25.015065Z", "shell.execute_reply": "2024-07-19T05:04:25.014647Z", "shell.execute_reply.started": "2024-07-19T05:04:24.970323Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.944246Z", "start_time": "2024-05-16T13:59:41.944237Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:25.015769Z", "iopub.status.busy": "2024-07-19T05:04:25.015629Z", "iopub.status.idle": "2024-07-19T05:04:25.059341Z", "shell.execute_reply": "2024-07-19T05:04:25.058917Z", "shell.execute_reply.started": "2024-07-19T05:04:25.015752Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total samples 42190 (42190, 81)\n" ] } ], "source": [ "total_iters = len(n_neg_tr) + len(n_pos_tr)\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945249Z", "start_time": "2024-05-16T13:59:41.945241Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:25.060060Z", "iopub.status.busy": "2024-07-19T05:04:25.059919Z", "iopub.status.idle": "2024-07-19T05:04:25.110545Z", "shell.execute_reply": "2024-07-19T05:04:25.110085Z", "shell.execute_reply.started": "2024-07-19T05:04:25.060044Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 epoch per batch 4, total 1318.4375\n" ] } ], "source": [ "print(\"1 epoch per batch 4, total\", total_iters / 8 / 2 / 2)" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945972Z", "start_time": "2024-05-16T13:59:41.945964Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:25.111239Z", "iopub.status.busy": "2024-07-19T05:04:25.111102Z", "iopub.status.idle": "2024-07-19T05:04:25.395463Z", "shell.execute_reply": "2024-07-19T05:04:25.394834Z", "shell.execute_reply.started": "2024-07-19T05:04:25.111223Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Submitted batch job 1112\n" ] } ], "source": [ "!cd /home/tony/Work/tony/slurm/13b_2h_ft && sbatch sbatch_ipo_13b_2h" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# some gymathtics loading prev data" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.946562Z", "start_time": "2024-05-16T13:59:41.946555Z" }, "execution": { "iopub.execute_input": "2024-07-19T05:04:25.396579Z", "iopub.status.busy": "2024-07-19T05:04:25.396401Z", "iopub.status.idle": "2024-07-19T05:04:25.399289Z", "shell.execute_reply": "2024-07-19T05:04:25.398886Z", "shell.execute_reply.started": "2024-07-19T05:04:25.396560Z" } }, "outputs": [], "source": [ "# prev_v3_data = \"/app/suno/data/dpo/7v_v20_full/\"\n", "\n", "# test_val_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_val.jsonl\"))\n", "# test_tr_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_tr.jsonl\"))\n", "\n", "# all_ids = set()\n", "# for meta in test_val_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# for meta in test_tr_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# print(len(all_ids), len(test_val_metas) + len(test_tr_metas))\n", "\n", "# all_ids = list(all_ids)\n", "# with open(\"/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id.json\", \"w\") as fp:\n", "# json.dump(all_ids, fp)" ] }, { "cell_type": "code", "execution_count": 39, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T05:04:25.400031Z", "iopub.status.busy": "2024-07-19T05:04:25.399883Z", "iopub.status.idle": "2024-07-19T05:04:25.630943Z", "shell.execute_reply": "2024-07-19T05:04:25.630474Z", "shell.execute_reply.started": "2024-07-19T05:04:25.400015Z" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
indexidcreated_atupdated_attime_usedmetadatauser_idstatusdiscord_message_idprompt_idrequest_idis_generateds3_idupvote_countbatch_indexmodel_nameprompt_textdaily_theme_idis_deletedimage_s3_idis_publicdislike_countflag_countplay_countskip_counttitleslugis_pro_useris_in_playlistcontinued_parentdurationsourceuser_n_clipsupvoteddownvotedhas_continuedpart_of_concathas_actionflaggeddeletedpos_preferenceneg_preferencediff_preferencepreferencereaction_play_countreaction_pro_play_counttotal_start_stotal_clip_sconcat_play_countsconcat_in_playlistconcat_likesconcat_dislikesis_13btagstypeinfillpromptstreamhistoryoptionspriorityhas_vocalexperimentgpt_promptcontinue_atconcat_historyrefund_creditsaudio_prompt_idmake_instrumentalcontinued_from_promptgpt_description_promptis_audio_upload_tos_acceptedparam_experimenterror_typeerror_messageoriginal_duration_shas_continue_and_start_continue_atgood_continue_atduration_rel_diffplay_rel_diff
53113032616151f85e6-0c43-465f-8103-e091e30f821d2024-07-15 06:26:53.814163+00:002024-07-15 06:26:53.814176+00:00210.729800{'tags': 'soulful hip hop melancholic', 'type'...10394491completeNaNNaN000bbfeb-c293-48ef-87d7-4c81a53858f3True151f85e6-0c43-465f-8103-e091e30f821d00chirp-v3p5-engine-ft-1NaNNaNFalseimage_151f85e6-0c43-465f-8103-e091e30f821dFalse0010Crazy WorldNaNTrueFalseNaN240.00web224FalseFalseFalseFalseFalseFalseFalseFalseFalse0False1.01.0NaNNaNNaNNaNNaNNaNTruesoulful hip hop melancholicgenNaN[Verse]\\nWorld on fire, insanity climbs, chaos...TrueNaNNaN0NaN[chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1]NoneNaNNaNFalseNaNFalseNaNA sad long verse rap song about how everything...NaNtemp_semantic_70NaNNaNNaNNaNTrue23.76-3.0
541130326146bbb7acb-2f9c-42f1-97d5-593b4fb29c702024-07-15 06:26:53.814261+00:002024-07-15 06:26:53.814268+00:00222.105723{'tags': 'soulful hip hop melancholic', 'type'...10394491completeNaNNaN000bbfeb-c293-48ef-87d7-4c81a53858f3True6bbb7acb-2f9c-42f1-97d5-593b4fb29c7011chirp-v3p5-engine-ft-1NaNNaNFalseimage_6bbb7acb-2f9c-42f1-97d5-593b4fb29c70True00120Crazy WorldNaNTrueFalseNaN240.00web224TrueFalseFalseFalseFalseFalseFalseTrueFalse1True12.012.0NaNNaNNaNNaNNaNNaNTruesoulful hip hop melancholicgenNaN[Verse]\\nWorld on fire, insanity climbs, chaos...TrueNaNNaN0NaN[chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1]NoneNaNNaNFalseNaNFalseNaNA sad long verse rap song about how everything...NaNNaNNaNNaNNaNNaNTrue0.0011.0
67107108906b2b73b14-289f-448b-855f-a91ad7f8a1732024-07-11 14:59:19.668245+00:002024-07-11 14:59:19.668251+00:00105.287227{'tags': 'Soft rock, Nu-gaze, neo-soul, wonky,...10505134completeNaNNaN00129b4f-e2aa-41ba-9d62-6864ed72c2c6Trueb2b73b14-289f-448b-855f-a91ad7f8a17301chirp-v3p5-engine-ft-1NaNNaNFalseimage_b2b73b14-289f-448b-855f-a91ad7f8a173False0090Have you heard 04NaNTrueFalsedf0118fb-9d91-45d5-8a3d-90e9c427dda2105.84web282FalseFalseFalseFalseFalseFalseFalseFalseFalse0False9.09.0NaNNaNNaNNaNNaNNaNTrueSoft rock, Nu-gaze, neo-soul, wonky, Layered d...genFalseTrue[{'id': 'df0118fb-9d91-45d5-8a3d-90e9c427dda2'...NaN10NaN[chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1]None119.96NaNFalsedf0118fb-9d91-45d5-8a3d-90e9c427dda2FalseNoneNaNtemp_semantic_70NaNNaNNaNNaNTrue-14.167.0
681012975503d517781-aeef-47d7-a921-d8e359b53b1f2024-07-11 14:59:19.668141+00:002024-07-11 14:59:19.668159+00:00134.651532{'tags': 'Soft rock, Nu-gaze, neo-soul, wonky,...10505134completeNaNNaN00129b4f-e2aa-41ba-9d62-6864ed72c2c6True3d517781-aeef-47d7-a921-d8e359b53b1f00chirp-v3p5-engine-ft-1NaNNaNFalseimage_3d517781-aeef-47d7-a921-d8e359b53b1fFalse0060Have you heard 01NaNTrueFalsedf0118fb-9d91-45d5-8a3d-90e9c427dda2104.36web282FalseFalseFalseFalseTrueFalseFalseTrueFalse1True6.06.0NaNNaNNaNNaNNaNNaNTrueSoft rock, Nu-gaze, neo-soul, wonky, Layered d...genFalseTrue[{'id': 'df0118fb-9d91-45d5-8a3d-90e9c427dda2'...NaN10NaN[chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1]None119.96NaNFalsedf0118fb-9d91-45d5-8a3d-90e9c427dda2FalseNoneNaNNaNNaNNaNNaNNaNTrue-1.48-3.0
81248630664d9edf40-48c4-419c-a02c-0b36159ff91d2024-07-12 20:20:33.260807+00:002024-07-12 20:20:33.260824+00:00198.776230{'tags': 'vibrant pop', 'type': 'gen', 'prompt...10034444completeNaNNaN0019726b-5a49-4f87-b970-b0113ed792f1True4d9edf40-48c4-419c-a02c-0b36159ff91d00chirp-v3p5-engine-ft-1NaNNaNTrueimage_4d9edf40-48c4-419c-a02c-0b36159ff91dFalse0040Fred l'Homme MagnifiqueNaNTrueFalseNaN240.00web148FalseFalseFalseFalseFalseFalseTrueFalseTrue-1False4.04.0NaNNaNNaNNaNNaNNaNTruevibrant popgenNaN[Verse]\\nFred l'homme magnifique\\nChaque jour ...TrueNaNNaN10NaN[chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1]NoneNaNNaNFalseNaNFalseNaNGénère une chanson en anglais sur un homme nom...NaNtemp_semantic_70NaNNaNNaNNaNTrue0.002.0
...................................................................................................................................................................................................................................................
154421104754500389b8569-397b-4d90-8680-04c5356b5d142024-07-13 21:10:10.951336+00:002024-07-13 21:10:10.951356+00:00229.741042{'tags': '80s, psychedelic, electro, dark', 't...12010228completeNaNNaNffd425e2-55a8-441a-8739-71be19b7a1b1True389b8569-397b-4d90-8680-04c5356b5d1400chirp-v3p5-engine-ft-1NaNNaNFalseimage_389b8569-397b-4d90-8680-04c5356b5d14False0080Celestial universe (E1)NaNTrueFalseNaN240.00web293FalseFalseFalseFalseFalseFalseFalseFalseFalse0True8.08.0NaNNaNNaNNaNNaNNaNTrue80s, psychedelic, electro, darkgenNaNTrueNaNNaN10NaN[chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1]NoneNaNNaNFalseNaNFalseNaNNoneNaNNaNNaNNaNNaNNaNTrue0.001.0
154469117546780cc13794b-0a7a-48c6-a196-175f217e52812024-07-15 21:56:20.615037+00:002024-07-15 21:56:20.615048+00:00230.237290{'tags': 'guzheng chinese traditional fast', '...26283790completeNaNNaNffe9331d-9800-493a-965b-2e31849a3a50Truecc13794b-0a7a-48c6-a196-175f217e528100chirp-v3p5-engine-ft-1NaNNaNTrueimage_cc13794b-0a7a-48c6-a196-175f217e5281False0070Kung Fu SpiritNaNFalseFalseNaN240.00web23FalseFalseFalseFalseFalseFalseTrueFalseTrue-1False7.0NaNNaNNaNNaNNaNNaNNaNTrueguzheng chinese traditional fastgenNaN[Instrumental]TrueNaNNaN0NaN[chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1]NoneNaNNaNFalseNaNTrueNaNchinese traditional song, with guzheng. In e m...NaNNaNNaNNaNNaNNaNTrue0.005.0
1544701175467812efab511-e2f1-4537-acf6-29ad77d8421a2024-07-15 21:56:20.615120+00:002024-07-15 21:56:20.615125+00:00215.984247{'tags': 'guzheng chinese traditional fast', '...26283790completeNaNNaNffe9331d-9800-493a-965b-2e31849a3a50True2efab511-e2f1-4537-acf6-29ad77d8421a01chirp-v3p5-engine-ft-1NaNNaNFalseimage_2efab511-e2f1-4537-acf6-29ad77d8421aFalse0050Kung Fu SpiritNaNFalseFalseNaN240.00web23FalseFalseFalseFalseFalseFalseFalseFalseFalse0True5.0NaNNaNNaNNaNNaNNaNNaNTrueguzheng chinese traditional fastgenNaN[Instrumental]TrueNaNNaN0NaN[chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1]NoneNaNNaNFalseNaNTrueNaNchinese traditional song, with guzheng. In e m...NaNtemp_semantic_95NaNNaNNaNNaNTrue0.00-2.0
15451110883909376c133d5-7a29-4d36-89e8-0f6e655a06a62024-07-17 08:16:24.805738+00:002024-07-17 08:16:24.805751+00:00266.409319{'tags': 'post rock industrial dark indie folk...288960completeNaNNaNfffebb95-7985-44ce-9c01-69eb73b07056True76c133d5-7a29-4d36-89e8-0f6e655a06a600chirp-v3p5-engine-ft-1NaNNaNFalseimage_76c133d5-7a29-4d36-89e8-0f6e655a06a6False0020The Shade of TomorrowNaNFalseFalseNaN240.00web111FalseFalseFalseFalseFalseFalseFalseFalseFalse0False2.0NaNNaNNaNNaNNaNNaNNaNTruepost rock industrial dark indie folk doom jazzgenNaN[Instrumental]TrueNaNNaN0NaN[chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1]NoneNaNNaNFalseNaNTrueNaNdepressing gloomy complex industrial Post rock...NaNNaNNaNNaNNaNNaNTrue0.00-2.0
154512108890193430052ff-5f5a-449f-8884-f53d652982db2024-07-17 08:16:24.805833+00:002024-07-17 08:16:24.805840+00:00256.468231{'tags': 'post rock industrial dark indie folk...288960completeNaNNaNfffebb95-7985-44ce-9c01-69eb73b07056True430052ff-5f5a-449f-8884-f53d652982db11chirp-v3p5-engine-ft-1NaNNaNFalseimage_430052ff-5f5a-449f-8884-f53d652982dbTrue0080The Shade of TomorrowNaNFalseFalseNaN240.00web111TrueFalseFalseFalseFalseFalseFalseTrueFalse1True8.0NaNNaNNaNNaNNaNNaNNaNTruepost rock industrial dark indie folk doom jazzgenNaN[Instrumental]TrueNaNNaN0NaN[chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1]NoneNaNNaNFalseNaNTrueNaNdepressing gloomy complex industrial Post rock...NaNNaNNaNNaNNaNNaNTrue0.006.0
\n", "

11912 rows × 80 columns

\n", "
" ], "text/plain": [ " index id created_at updated_at time_used metadata user_id status discord_message_id prompt_id request_id is_generated s3_id upvote_count batch_index model_name prompt_text daily_theme_id is_deleted image_s3_id is_public dislike_count flag_count play_count skip_count title slug is_pro_user is_in_playlist continued_parent duration source user_n_clips upvoted downvoted has_continued part_of_concat has_action flagged deleted pos_preference neg_preference diff_preference preference reaction_play_count reaction_pro_play_count total_start_s total_clip_s concat_play_counts concat_in_playlist concat_likes concat_dislikes is_13b \\\n", "53 113032616 151f85e6-0c43-465f-8103-e091e30f821d 2024-07-15 06:26:53.814163+00:00 2024-07-15 06:26:53.814176+00:00 210.729800 {'tags': 'soulful hip hop melancholic', 'type'... 10394491 complete NaN NaN 000bbfeb-c293-48ef-87d7-4c81a53858f3 True 151f85e6-0c43-465f-8103-e091e30f821d 0 0 chirp-v3p5-engine-ft-1 NaN NaN False image_151f85e6-0c43-465f-8103-e091e30f821d False 0 0 1 0 Crazy World NaN True False NaN 240.00 web 224 False False False False False False False False False 0 False 1.0 1.0 NaN NaN NaN NaN NaN NaN True \n", "54 113032614 6bbb7acb-2f9c-42f1-97d5-593b4fb29c70 2024-07-15 06:26:53.814261+00:00 2024-07-15 06:26:53.814268+00:00 222.105723 {'tags': 'soulful hip hop melancholic', 'type'... 10394491 complete NaN NaN 000bbfeb-c293-48ef-87d7-4c81a53858f3 True 6bbb7acb-2f9c-42f1-97d5-593b4fb29c70 1 1 chirp-v3p5-engine-ft-1 NaN NaN False image_6bbb7acb-2f9c-42f1-97d5-593b4fb29c70 True 0 0 12 0 Crazy World NaN True False NaN 240.00 web 224 True False False False False False False True False 1 True 12.0 12.0 NaN NaN NaN NaN NaN NaN True \n", "67 107108906 b2b73b14-289f-448b-855f-a91ad7f8a173 2024-07-11 14:59:19.668245+00:00 2024-07-11 14:59:19.668251+00:00 105.287227 {'tags': 'Soft rock, Nu-gaze, neo-soul, wonky,... 10505134 complete NaN NaN 00129b4f-e2aa-41ba-9d62-6864ed72c2c6 True b2b73b14-289f-448b-855f-a91ad7f8a173 0 1 chirp-v3p5-engine-ft-1 NaN NaN False image_b2b73b14-289f-448b-855f-a91ad7f8a173 False 0 0 9 0 Have you heard 04 NaN True False df0118fb-9d91-45d5-8a3d-90e9c427dda2 105.84 web 282 False False False False False False False False False 0 False 9.0 9.0 NaN NaN NaN NaN NaN NaN True \n", "68 101297550 3d517781-aeef-47d7-a921-d8e359b53b1f 2024-07-11 14:59:19.668141+00:00 2024-07-11 14:59:19.668159+00:00 134.651532 {'tags': 'Soft rock, Nu-gaze, neo-soul, wonky,... 10505134 complete NaN NaN 00129b4f-e2aa-41ba-9d62-6864ed72c2c6 True 3d517781-aeef-47d7-a921-d8e359b53b1f 0 0 chirp-v3p5-engine-ft-1 NaN NaN False image_3d517781-aeef-47d7-a921-d8e359b53b1f False 0 0 6 0 Have you heard 01 NaN True False df0118fb-9d91-45d5-8a3d-90e9c427dda2 104.36 web 282 False False False False True False False True False 1 True 6.0 6.0 NaN NaN NaN NaN NaN NaN True \n", "81 24863066 4d9edf40-48c4-419c-a02c-0b36159ff91d 2024-07-12 20:20:33.260807+00:00 2024-07-12 20:20:33.260824+00:00 198.776230 {'tags': 'vibrant pop', 'type': 'gen', 'prompt... 10034444 complete NaN NaN 0019726b-5a49-4f87-b970-b0113ed792f1 True 4d9edf40-48c4-419c-a02c-0b36159ff91d 0 0 chirp-v3p5-engine-ft-1 NaN NaN True image_4d9edf40-48c4-419c-a02c-0b36159ff91d False 0 0 4 0 Fred l'Homme Magnifique NaN True False NaN 240.00 web 148 False False False False False False True False True -1 False 4.0 4.0 NaN NaN NaN NaN NaN NaN True \n", "... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... \n", "154421 104754500 389b8569-397b-4d90-8680-04c5356b5d14 2024-07-13 21:10:10.951336+00:00 2024-07-13 21:10:10.951356+00:00 229.741042 {'tags': '80s, psychedelic, electro, dark', 't... 12010228 complete NaN NaN ffd425e2-55a8-441a-8739-71be19b7a1b1 True 389b8569-397b-4d90-8680-04c5356b5d14 0 0 chirp-v3p5-engine-ft-1 NaN NaN False image_389b8569-397b-4d90-8680-04c5356b5d14 False 0 0 8 0 Celestial universe (E1) NaN True False NaN 240.00 web 293 False False False False False False False False False 0 True 8.0 8.0 NaN NaN NaN NaN NaN NaN True \n", "154469 117546780 cc13794b-0a7a-48c6-a196-175f217e5281 2024-07-15 21:56:20.615037+00:00 2024-07-15 21:56:20.615048+00:00 230.237290 {'tags': 'guzheng chinese traditional fast', '... 26283790 complete NaN NaN ffe9331d-9800-493a-965b-2e31849a3a50 True cc13794b-0a7a-48c6-a196-175f217e5281 0 0 chirp-v3p5-engine-ft-1 NaN NaN True image_cc13794b-0a7a-48c6-a196-175f217e5281 False 0 0 7 0 Kung Fu Spirit NaN False False NaN 240.00 web 23 False False False False False False True False True -1 False 7.0 NaN NaN NaN NaN NaN NaN NaN True \n", "154470 117546781 2efab511-e2f1-4537-acf6-29ad77d8421a 2024-07-15 21:56:20.615120+00:00 2024-07-15 21:56:20.615125+00:00 215.984247 {'tags': 'guzheng chinese traditional fast', '... 26283790 complete NaN NaN ffe9331d-9800-493a-965b-2e31849a3a50 True 2efab511-e2f1-4537-acf6-29ad77d8421a 0 1 chirp-v3p5-engine-ft-1 NaN NaN False image_2efab511-e2f1-4537-acf6-29ad77d8421a False 0 0 5 0 Kung Fu Spirit NaN False False NaN 240.00 web 23 False False False False False False False False False 0 True 5.0 NaN NaN NaN NaN NaN NaN NaN True \n", "154511 108839093 76c133d5-7a29-4d36-89e8-0f6e655a06a6 2024-07-17 08:16:24.805738+00:00 2024-07-17 08:16:24.805751+00:00 266.409319 {'tags': 'post rock industrial dark indie folk... 288960 complete NaN NaN fffebb95-7985-44ce-9c01-69eb73b07056 True 76c133d5-7a29-4d36-89e8-0f6e655a06a6 0 0 chirp-v3p5-engine-ft-1 NaN NaN False image_76c133d5-7a29-4d36-89e8-0f6e655a06a6 False 0 0 2 0 The Shade of Tomorrow NaN False False NaN 240.00 web 111 False False False False False False False False False 0 False 2.0 NaN NaN NaN NaN NaN NaN NaN True \n", "154512 108890193 430052ff-5f5a-449f-8884-f53d652982db 2024-07-17 08:16:24.805833+00:00 2024-07-17 08:16:24.805840+00:00 256.468231 {'tags': 'post rock industrial dark indie folk... 288960 complete NaN NaN fffebb95-7985-44ce-9c01-69eb73b07056 True 430052ff-5f5a-449f-8884-f53d652982db 1 1 chirp-v3p5-engine-ft-1 NaN NaN False image_430052ff-5f5a-449f-8884-f53d652982db True 0 0 8 0 The Shade of Tomorrow NaN False False NaN 240.00 web 111 True False False False False False False True False 1 True 8.0 NaN NaN NaN NaN NaN NaN NaN True \n", "\n", " tags type infill prompt stream history options priority has_vocal experiment gpt_prompt continue_at concat_history refund_credits audio_prompt_id make_instrumental continued_from_prompt gpt_description_prompt is_audio_upload_tos_accepted param_experiment error_type error_message original_duration_s has_continue_and_start_continue_at good_continue_at duration_rel_diff play_rel_diff \n", "53 soulful hip hop melancholic gen NaN [Verse]\\nWorld on fire, insanity climbs, chaos... True NaN NaN 0 NaN [chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1] None NaN NaN False NaN False NaN A sad long verse rap song about how everything... NaN temp_semantic_70 NaN NaN NaN NaN True 23.76 -3.0 \n", "54 soulful hip hop melancholic gen NaN [Verse]\\nWorld on fire, insanity climbs, chaos... True NaN NaN 0 NaN [chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1] None NaN NaN False NaN False NaN A sad long verse rap song about how everything... NaN NaN NaN NaN NaN NaN True 0.00 11.0 \n", "67 Soft rock, Nu-gaze, neo-soul, wonky, Layered d... gen False True [{'id': 'df0118fb-9d91-45d5-8a3d-90e9c427dda2'... NaN 10 NaN [chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1] None 119.96 NaN False df0118fb-9d91-45d5-8a3d-90e9c427dda2 False None NaN temp_semantic_70 NaN NaN NaN NaN True -14.16 7.0 \n", "68 Soft rock, Nu-gaze, neo-soul, wonky, Layered d... gen False True [{'id': 'df0118fb-9d91-45d5-8a3d-90e9c427dda2'... NaN 10 NaN [chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1] None 119.96 NaN False df0118fb-9d91-45d5-8a3d-90e9c427dda2 False None NaN NaN NaN NaN NaN NaN True -1.48 -3.0 \n", "81 vibrant pop gen NaN [Verse]\\nFred l'homme magnifique\\nChaque jour ... True NaN NaN 10 NaN [chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1] None NaN NaN False NaN False NaN Génère une chanson en anglais sur un homme nom... NaN temp_semantic_70 NaN NaN NaN NaN True 0.00 2.0 \n", "... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... \n", "154421 80s, psychedelic, electro, dark gen NaN True NaN NaN 10 NaN [chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1] None NaN NaN False NaN False NaN None NaN NaN NaN NaN NaN NaN True 0.00 1.0 \n", "154469 guzheng chinese traditional fast gen NaN [Instrumental] True NaN NaN 0 NaN [chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1] None NaN NaN False NaN True NaN chinese traditional song, with guzheng. In e m... NaN NaN NaN NaN NaN NaN True 0.00 5.0 \n", "154470 guzheng chinese traditional fast gen NaN [Instrumental] True NaN NaN 0 NaN [chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1] None NaN NaN False NaN True NaN chinese traditional song, with guzheng. In e m... NaN temp_semantic_95 NaN NaN NaN NaN True 0.00 -2.0 \n", "154511 post rock industrial dark indie folk doom jazz gen NaN [Instrumental] True NaN NaN 0 NaN [chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1] None NaN NaN False NaN True NaN depressing gloomy complex industrial Post rock... NaN NaN NaN NaN NaN NaN True 0.00 -2.0 \n", "154512 post rock industrial dark indie folk doom jazz gen NaN [Instrumental] True NaN NaN 0 NaN [chirp-v3p5-engine-ft-1, chirp-v3p5-engine-ft-1] None NaN NaN False NaN True NaN depressing gloomy complex industrial Post rock... NaN NaN NaN NaN NaN NaN True 0.00 6.0 \n", "\n", "[11912 rows x 80 columns]" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_slice[df_slice[\"prompt_text\"].str.strip() != df_slice[\"prompt_text\"].str.strip()]" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T05:04:25.631814Z", "iopub.status.busy": "2024-07-19T05:04:25.631665Z", "iopub.status.idle": "2024-07-19T05:04:25.635137Z", "shell.execute_reply": "2024-07-19T05:04:25.634741Z", "shell.execute_reply.started": "2024-07-19T05:04:25.631797Z" } }, "outputs": [ { "data": { "text/plain": [ "Index(['index', 'id', 'created_at', 'updated_at', 'time_used', 'metadata', 'user_id', 'status', 'discord_message_id', 'prompt_id', 'request_id', 'is_generated', 's3_id', 'upvote_count', 'batch_index', 'model_name', 'prompt_text', 'daily_theme_id', 'is_deleted', 'image_s3_id', 'is_public', 'dislike_count', 'flag_count', 'play_count', 'skip_count', 'title', 'slug', 'is_pro_user', 'is_in_playlist', 'continued_parent', 'duration', 'source', 'user_n_clips', 'upvoted', 'downvoted', 'has_continued', 'part_of_concat', 'has_action', 'flagged', 'deleted', 'pos_preference', 'neg_preference', 'diff_preference', 'preference', 'reaction_play_count', 'reaction_pro_play_count', 'total_start_s', 'total_clip_s', 'concat_play_counts', 'concat_in_playlist', 'concat_likes', 'concat_dislikes', 'is_13b', 'tags', 'type', 'infill', 'prompt', 'stream', 'history', 'options', 'priority', 'has_vocal', 'experiment', 'gpt_prompt', 'continue_at', 'concat_history', 'refund_credits', 'audio_prompt_id',\n", " 'make_instrumental', 'continued_from_prompt', 'gpt_description_prompt', 'is_audio_upload_tos_accepted', 'param_experiment', 'error_type', 'error_message', 'original_duration_s', 'has_continue_and_start_continue_at', 'good_continue_at', 'duration_rel_diff', 'play_rel_diff'],\n", " dtype='object')" ] }, "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_slice.columns" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T05:04:25.635859Z", "iopub.status.busy": "2024-07-19T05:04:25.635721Z", "iopub.status.idle": "2024-07-19T05:04:25.800742Z", "shell.execute_reply": "2024-07-19T05:04:25.800242Z", "shell.execute_reply.started": "2024-07-19T05:04:25.635843Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(21309, 80) (21309, 80)\n" ] } ], "source": [ "negative_df_slice = df_slice[::2].copy()\n", "positive_df_slice = df_slice[1::2].copy()\n", "print(negative_df_slice.shape, positive_df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "execution": { "iopub.execute_input": "2024-07-19T05:04:25.801536Z", "iopub.status.busy": "2024-07-19T05:04:25.801395Z", "iopub.status.idle": "2024-07-19T05:04:25.841821Z", "shell.execute_reply": "2024-07-19T05:04:25.841382Z", "shell.execute_reply.started": "2024-07-19T05:04:25.801521Z" } }, "outputs": [ { "data": { "text/plain": [ "303" ] }, "execution_count": 42, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sum(negative_df_slice.reset_index()[\"prompt\"] != positive_df_slice.reset_index()[\"prompt\"])" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }