{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#filepath = \"/home/tony/Data/Preference/up_v2_d3/interesting_clips_ahi_d3_20250504.pkl\"\n", "filepath = \"/home/tony/Data/Preference/up_v2_d3/interesting_clips_ahi_d3_20250519_slice.pkl\"\n", "filepath = \"/home/tony/Data/Preference/up_v2_d3/interesting_clips_ahi_d3_20250528.pkl\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from suno_utils.audio import Audio\n", "\n", "metas = pd.read_pickle(filepath)\n", "\n", "metas.head()\n", "print(len(metas))\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "source_metas_map = {}\n", "\n", "# iterate over the rows in the metas dataframe\n", "for idx, row in metas.iterrows():\n", " source_metas_map[row[\"id\"]] = {\n", " \"id\": row[\"id\"],\n", " \"upsample_clip_id\": row[\"metadata\"][\"upsample_clip_id\"],\n", " \"text\": row[\"metadata\"][\"prompt\"],\n", " \"tags\": row[\"metadata\"][\"tags\"],\n", " }\n", "print(len(source_metas_map))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"5\"\n", "\n", "from suno_utils.tasks.mert_25 import (\n", " preload_models as preload_semantic_models,\n", " encode as encode_semantic,\n", ")\n", "\n", "semantic_model_filepath = \"s3://suno-data/georg/models/semantic/mert_25.pt\"\n", "semantic_clusters_filepath = \"s3://suno-data/georg/models/semantic/mert_25_2x4k.npy\"\n", "\n", "_ = preload_semantic_models(semantic_model_filepath, semantic_clusters_filepath)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import torch\n", "import numpy as np\n", "\n", "vocab_size = 4001\n", "input_dim = 768\n", "\n", "def create_embedding(centroid_path, vocab_size, input_dim):\n", " embedding = torch.nn.Embedding(vocab_size, input_dim)\n", "\n", " centroids = np.load(centroid_path)[0]\n", " # create a new tensor with an extra row for the pad token\n", " centroids = np.concatenate([centroids, np.random.randn(1, centroids.shape[1])], axis=0)\n", " print(embedding.weight.shape, centroids.shape)\n", "\n", " assert embedding.weight.shape == centroids.shape\n", " embedding.weight.data.copy_(torch.tensor(centroids, dtype=torch.float32))\n", " return embedding\n", "\n", "semantic_centroid_path=\"/app/suno/models/semantic/mert_25_2x4k.npy\"\n", "embedding = create_embedding(semantic_centroid_path, vocab_size, input_dim) \n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "CHUNKSIZE = 16\n", "OUTPUT_STR = \"interesting_clips_ahi_d3_20250528\"\n", "\n", "import funcy\n", "import numpy as np\n", "from tqdm import tqdm\n", "from suno_utils.utils.s3 import list_s3_dir, read_from_s3\n", "\n", "filepath = (\n", " \"/home/tony/Data/Preference/up_v2_d3/interesting_clips_ahi_d3_20250528.pkl\"\n", ")\n", "metas = pd.read_pickle(filepath)\n", "\n", "work_items = []\n", "odd_ids = np.arange(0, len(metas), 2)\n", "pbar = tqdm(odd_ids)\n", "for odd_id in pbar:\n", "\n", " neg_request_id = metas.iloc[odd_id][\"request_id\"]\n", " pos_request_id = metas.iloc[odd_id + 1][\"request_id\"]\n", "\n", " assert neg_request_id == pos_request_id\n", "\n", " neg_id = metas.iloc[odd_id][\"id\"]\n", " pos_id = metas.iloc[odd_id + 1][\"id\"]\n", "\n", " work_items.append((neg_request_id, neg_id, pos_id))\n", "\n", "print(f\"Total work items: {len(work_items)}\")\n", "\n", "work_items = list(funcy.chunks(CHUNKSIZE, work_items))\n", "print(f\"Chunksize: {CHUNKSIZE}, total chunks: {len(work_items)}\")\n", "\n", "# add a work_item_idx to each work item\n", "work_items = [(i, work_item) for i, work_item in enumerate(work_items)]\n", "print(\"Total chunks: \", len(work_items))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# first check for existing ids in the output path\n", "existing_ids = list_s3_dir(f\"s3://suno-data/christian/outputs/{OUTPUT_STR}/\",\n", ")\n", "\n", "existing_ids = [int(f[0].split(\"/\")[-1].replace(\".json\", \"\")) for f in existing_ids]\n", "existing_ids = list(set(existing_ids))\n", "print(f\"Total existing ids: {len(existing_ids)}\")\n", "\n", "## now remove these from the work items\n", "work_items = [f for f in work_items if f[0] not in existing_ids]\n", "\n", "print(f\"Total work items remaining: {len(work_items)}\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# aws s3 sync s3://suno-data/christian/outputs/interesting_clips_ahi_d3_20250528/ /home/christian/audio/interesting_clips_ahi_d3_20250528\n", "import os\n", "import json\n", "import pandas as pd\n", "dirpath = \"/home/christian/audio/interesting_clips_ahi_d3_20250528\"\n", "# find all files in the directory\n", "files = [f for f in os.listdir(dirpath) if f.endswith('.json')]\n", "print(len(files))\n", "\n", "# load all the json files\n", "data = {}\n", "for f in files:\n", " with open(os.path.join(dirpath, f), \"r\") as file:\n", " data.update(json.load(file))\n", "\n", "data_list = [{\"request_id\": k, \"pos_id\": v[\"pos_id\"], \"neg_id\": v[\"neg_id\"], \"mean_similarity\": v[\"mean_similarity\"], \"neg_ear_score\": v[\"neg_ear_score\"], \"pos_ear_score\": v[\"pos_ear_score\"]} for k, v in data.items()]\n", "\n", "# convert this to a dataframe\n", "data_df = pd.DataFrame(data_list)\n", "print(data_df.describe())" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# add the upsample_clip_id to the dataframe, we can get this from the \n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ear score difference \n", "data_df[\"ear_score_diff\"] = data_df[\"pos_ear_score\"] - data_df[\"neg_ear_score\"]\n", "print(data_df.describe())\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "data_df.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# lets do a little filtering here\n", "# lets make sure the mean similarity is less than 0.75\n", "data_df = data_df[data_df[\"mean_similarity\"] < 0.725]\n", "print(len(data_df))\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "data_df['pref_aligns_with_ear'] = data_df['pos_ear_score'] > data_df['neg_ear_score']\n", "alignment_rate = data_df['pref_aligns_with_ear'].mean()\n", "print(f\"Preference aligned with higher EAR score in {alignment_rate:.2%} of cases\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# now lets create an info file and save as json\n", "info_dict = {\"dataset\" : []}\n", "for idx, row in tqdm(data_df.iterrows()):\n", " valid_clip_ids = [row[\"pos_id\"], row[\"neg_id\"]]\n", " info_dict[\"dataset\"].append(valid_clip_ids)\n", "\n", "print(len(info_dict[\"dataset\"]))\n", "# save the info dict to a json file\n", "with open(f\"/home/christian/code/christian/metadata/dpo_splits/interesting_clips_ahi_d3_20250528_t1.json\", \"w\") as file:\n", " json.dump(info_dict, file)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "fig, ax = plt.subplots(figsize=(5, 3))\n", "plt.hist(data_df[\"mean_similarity\"], bins=250, zorder=10)\n", "plt.xlabel(\"Mean cosine similarity\")\n", "plt.ylabel(\"Frequency\")\n", "plt.grid(c=\"lightgray\", zorder=0)\n", "plt.tight_layout()\n", "plt.show()\n", "\n", "# lets create a new dataframe that compares the even and odd rows \n", "# we will compare the mean score for the even and odd rows\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# cut out items where the abolsut eear score difference is greater than 0.5\n", "data_df = data_df[data_df[\"ear_score_diff\"].abs() > 0.5]\n", "print(len(data_df))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "fig, ax = plt.subplots(figsize=(5, 3))\n", "plt.hist(data_df[\"ear_score_diff\"], bins=250, zorder=10)\n", "plt.xlabel(\"Ear score difference\")\n", "plt.ylabel(\"Frequency\")\n", "plt.grid(c=\"lightgray\", zorder=0)\n", "plt.tight_layout()\n", "plt.show()\n", "\n", "# lets create a new dataframe that compares the even and odd rows \n", "# we will compare the mean score for the even and odd rows\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import torch\n", "from tqdm import tqdm\n", "from sklearn.metrics.pairwise import cosine_similarity\n", "from suno_utils.utils.s3 import read_from_s3\n", "\n", "compare_type = \"codes\"\n", "\n", "\n", "\n", "results = []\n", "\n", "def load_semantic_codes(id):\n", " s3_filepath = f\"s3://suno-data-uploads/studio/uploads/{id}.npz\"\n", " try:\n", " data = read_from_s3(s3_filepath, read_f=np.load)\n", " except Exception as e:\n", " print(f\"Error loading {s3_filepath}: {e}\")\n", " return None\n", "\n", " if \"v3.0_raw\" in data:\n", " codes = data[\"v3.0_raw\"]\n", " elif \"v3.5_raw\" in data:\n", " codes = data[\"v3.5_raw\"]\n", " elif \"v4.0_raw\" in data:\n", " codes = data[\"v4.0_raw\"]\n", " elif \"v4.5_raw\" in data:\n", " codes = data[\"v4.5_raw\"]\n", " elif \"v5.0_raw\" in data:\n", " codes = data[\"v5.0_raw\"]\n", " else:\n", " raise ValueError(\"No codes found\")\n", "\n", " return codes\n", "\n", "# get a list of all odd row ids \n", "odd_ids = np.arange(0, len(metas), 2)\n", "pbar = tqdm(odd_ids[0:100])\n", "for odd_id in pbar:\n", "\n", " neg_id = metas.iloc[odd_id]['id']\n", " pos_id = metas.iloc[odd_id + 1]['id']\n", "\n", " if compare_type == \"audio\":\n", " neg_s3_path = f\"s3://suno-data-uploads/studio/uploads/{neg_id}.mp3\"\n", " pos_s3_path = f\"s3://suno-data-uploads/studio/uploads/{pos_id}.mp3\"\n", "\n", " neg_audio = Audio.from_s3(neg_s3_path, n_channels=1)\n", " pos_audio = Audio.from_s3(pos_s3_path, n_channels=1)\n", "\n", " semantic = encode_semantic([neg_audio, pos_audio], do_clustering=False)\n", " neg_semantic = semantic[0]\n", " pos_semantic = semantic[1]\n", "\n", " # Compute cosine similarity for each token (each row in the matrix)\n", " # This gives a matrix of shape (2698, 2698), but we only want the diagonal\n", " pairwise_sim_matrix = cosine_similarity(neg_semantic, pos_semantic)\n", "\n", " # Get the similarity of corresponding tokens (i.e., diagonal)\n", " tokenwise_similarities = np.diag(pairwise_sim_matrix)\n", "\n", " # Take the mean of the token-wise similarities\n", " mean_similarity = tokenwise_similarities.mean()\n", " else:\n", " # grab the semantic codes from s3\n", " neg_semantic_filepath = f\"s3://suno-data-uploads/studio/uploads/{neg_id}.npz\"\n", " pos_semantic_filepath = f\"s3://suno-data-uploads/studio/uploads/{pos_id}.npz\"\n", "\n", " neg_semantic_codes = load_semantic_codes(neg_id)\n", " pos_semantic_codes = load_semantic_codes(pos_id)\n", "\n", " if neg_semantic_codes is None or pos_semantic_codes is None:\n", " continue\n", "\n", " neg_semantic_codes = torch.tensor(neg_semantic_codes[:,0], dtype=torch.long)\n", " pos_semantic_codes = torch.tensor(pos_semantic_codes[:,0], dtype=torch.long)\n", " \n", " # convert the tokens back to embeddings with the clusters \n", " with torch.no_grad():\n", " neg_semantic_codes = embedding(neg_semantic_codes)\n", " pos_semantic_codes = embedding(pos_semantic_codes)\n", "\n", " mean_similarity = cosine_similarity(neg_semantic_codes, pos_semantic_codes).mean()\n", "\n", "\n", " results.append({\n", " \"mean_similarity\": mean_similarity,\n", " \"neg_id\": neg_id,\n", " \"pos_id\": pos_id,\n", " \"neg_audio\": neg_audio,\n", " \"pos_audio\": pos_audio,\n", " })\n", "\n", " pbar.set_description(f\"Mean similarity: {mean_similarity:.4f}\")\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#histogram of the mean similarity\n", "import matplotlib.pyplot as plt\n", "plt.hist([r[\"mean_similarity\"] for r in results], bins=25)\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# find the the result with the highest mean similarity\n", "best_result = max(results, key=lambda x: x[\"mean_similarity\"])\n", "print(best_result)\n", "\n", "# play the audio\n", "best_result[\"neg_audio\"].play()\n", "best_result[\"pos_audio\"].play()\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import json\n", "with open(\"/home/tony/Data/Preference/up_v2_d3/full_pair_quality.json\", \"r\") as file:\n", " full_pair_quality = json.load(file)\n", "print(\"Total pair quality scores:\", len(full_pair_quality))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "list(full_pair_quality.keys())[0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.text import write_jsonl\n", "from tqdm import tqdm\n", "# create a jsonl file with the metas\n", "# iterate over the rows in the metas dataframe\n", "base_s3_path = \"s3://suno-data-uploads/studio/uploads/\"\n", "new_metas = []\n", "for idx, row in tqdm(metas.iterrows()):\n", " new_metas.append({\n", " \"id\": row[\"id\"],\n", " \"s3_filepath\": f\"{base_s3_path}{row['id']}.mp3\",\n", " })\n", "\n", "# save to my local metadata directory\n", "local_metadata_dir = \"/home/christian/code/christian/metadata/dpo/interesting_clips_ahi_d3_20250504.jsonl\"\n", "write_jsonl(new_metas, local_metadata_dir)\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# analyze ear score and merge with original dataframe\n", "import os\n", "import json\n", "import pandas as pd\n", "from tqdm import tqdm\n", "\n", "\n", "filepath = \"/home/tony/Data/Preference/up_v2_d3/interesting_clips_ahi_d3_20250504.pkl\"\n", "\n", "metas = pd.read_pickle(filepath)\n", "\n", "metas.head()\n", "print(len(metas))\n", "\n", "dataset_name = \"interesting_clips_ahi_d3_20250504\"\n", "\n", "base_dir = f\"/home/christian/ear_scores/{dataset_name}\"\n", "os.makedirs(base_dir, exist_ok=True)\n", "# find all json files in the base_dir\n", "json_files = [f for f in os.listdir(base_dir) if f.endswith('.json')]\n", "print(len(json_files))\n", "json_filepaths = [os.path.join(base_dir, f) for f in json_files]\n", "\n", "# read the json files and store into one dict\n", "data = {}\n", "for f in tqdm(json_filepaths):\n", " data.update(json.load(open(f)))\n", "\n", "# create a dataframe with just the track id and mean score\n", "rows = []\n", "for track_id, track_data in data.items():\n", " row = {\n", " 'id': track_id,\n", " 'mean_score': track_data['mean_score']\n", " }\n", " rows.append(row)\n", "\n", "ear_scores_df = pd.DataFrame(rows)\n", "\n", "# Merge with the original dataframe\n", "# First reset index if id is the index in metas\n", "if metas.index.name == 'id':\n", " metas_reset = metas.reset_index()\n", "else:\n", " metas_reset = metas.copy()\n", "\n", "# Merge the dataframes\n", "merged_df = pd.merge(metas_reset, ear_scores_df, on='id', how='left')\n", "\n", "# Set the index back to id if it was originally\n", "if metas.index.name == 'id':\n", " merged_df.set_index('id', inplace=True)\n", "\n", "# Update the original metas dataframe\n", "metas = merged_df.copy()\n", "\n", "print(metas.head())\n", "print(\"Ear score statistics:\")\n", "print(metas['mean_score'].describe())" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "filepath = \"/home/tony/Data/Preference/up_v2_d3/interesting_clips_ahi_d3_20250519_slice.pkl\"\n", "metas = pd.read_pickle(filepath)\n", "\n", "metas.head()\n", "print(len(metas))\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for col in metas.columns:\n", " print(col)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(metas.iloc[2][\"pair_quality\"])\n", "print(metas.iloc[3][\"pair_quality\"])\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import json\n", "with open(\"/home/tony/Data/Preference/up_v2_d3/full_pair_quality.json\", \"r\") as file:\n", " full_pair_quality = json.load(file)\n", "print(\"Total pair quality scores:\", len(full_pair_quality))\n", "\n", "unpacked_pair_quality = {}\n", "for request_id, pairs_of_qualities in full_pair_quality.items():\n", " for clip_id, pair_quality in pairs_of_qualities.items():\n", " unpacked_pair_quality[clip_id] = pair_quality\n", "print(\"Total unpacked pair quality scores:\", len(unpacked_pair_quality))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "stats = []\n", "# convert the full_pair_quality to a dataframe\n", "for key, value in unpacked_pair_quality.items():\n", " if value is not None:\n", " value[\"id\"] = key\n", " stats.append(value)\n", "\n", "stats_df = pd.DataFrame(stats)\n", "\n", "stats_df.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "# create a mean_ear_v2_quality_score column in unpacked_pair_quality_df\n", "# Fix: Apply mean to each row's ear_v2_quality_scores list instead of the entire column\n", "stats_df[\"mean_ear_v2_quality_score\"] = stats_df[\"ear_v2_quality_scores\"].apply(lambda x: sum(x)/len(x) if isinstance(x, list) else x)\n", "stats_df.head()\n", "\n", "# merge the stats_df into metas dataframe\n", "metas = pd.merge(metas, stats_df, on=\"id\", how=\"left\")\n", "metas.head()\n", "\n", "# remove any rows where mean_ear_v2_quality_score is None\n", "#metas = metas[metas['mean_ear_v2_quality_score'].notna()]\n", "#metas = metas[metas[\"stereo_width\"].notna()]\n", "#rint(len(metas))\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "metas.iloc[0][\"stereo_width\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Show all columns\n", "pd.set_option('display.max_columns', None)\n", "\n", "metas.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "shimmer_scores = stats_df[\"shimmer_score\"].tolist()\n", "print(shimmer_scores[:10])\n", "plt.hist(shimmer_scores, bins=100)\n", "plt.yscale(\"log\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# lets create a new dataframe that compares the even and odd rows \n", "# we will compare the mean score for the even and odd rows\n", "# Create a new dataframe for comparison\n", "comparison_df = pd.DataFrame()\n", "\n", "# Assuming the data is organized in pairs (even/odd rows)\n", "# First, ensure we have an even number of rows to work with\n", "n_rows = len(metas)\n", "if n_rows % 2 != 0:\n", " n_rows -= 1 # Exclude the last row if total count is odd\n", "\n", "# Extract even and odd rows\n", "# even is negative and odd is positive \n", "# so we want negative difference for ear score\n", "# so we want positive difference for shimmer schore\n", "even_rows = metas.iloc[0:n_rows:2].reset_index()\n", "odd_rows = metas.iloc[1:n_rows:2].reset_index()\n", "\n", "# Create the comparison dataframe\n", "#comparison_df['upsample_clip_id'] = even_rows['upsample_clip_id']\n", "comparison_df['even_id'] = even_rows['id']\n", "comparison_df['odd_id'] = odd_rows['id']\n", "\n", "for feature in [\"mean_ear_v2_quality_score\", \"stereo_width\", \"shimmer_score\", \"play_count\", \"sum_total_play_duration_0\", \"spectral_centroid\"]:\n", " print(feature)\n", " comparison_df[f\"even_{feature}\"] = even_rows[feature].astype(float)\n", " comparison_df[f\"odd_{feature}\"] = odd_rows[feature].astype(float)\n", "\n", "\n", " # Calculate the score difference (even - odd)\n", " # even is the negative preference and odd is the positive preference\n", " # so a negative score means ear score of the positive is higher than the negative\n", " comparison_df[f\"diff_{feature}\"] = comparison_df[f\"even_{feature}\"] - comparison_df[f\"odd_{feature}\"]\n", " comparison_df[\"user_n_clips\"] = even_rows[\"user_n_clips\"]\n", "\n", "# Determine preference (1 if even is preferred, 0 if odd is preferred)\n", "#comparison_df['preference'] = (comparison_df['score_diff'] > 0).astype(int)\n", "\n", "# Calculate statistics\n", "#print(\"Comparison statistics:\")\n", "#print(f\"Total pairs: {len(comparison_df)}\")\n", "#print(f\"Pairs where even score > odd score: {comparison_df['preference'].sum()} ({comparison_df['preference'].mean()*100:.2f}%)\")\n", "#print(f\"Average score difference: {comparison_df['score_diff'].mean():.4f}\")\n", "#print(f\"Median score difference: {comparison_df['score_diff'].median():.4f}\")\n", "\n", "# Display the first few rows of the comparison dataframe\n", "print(\"\\nSample of comparison data:\")\n", "comparison_df.head()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "comparison_df['play_count_aligns'] = comparison_df['odd_play_count'] > comparison_df['even_play_count']\n", "play_count_alignment_rate = comparison_df['play_count_aligns'].mean()\n", "print(f\"Preference aligns with higher play count in {play_count_alignment_rate:.2%} of comparisons\")\n", "\n", "\n", "comparison_df['play_duration_aligns'] = comparison_df['odd_sum_total_play_duration_0'] > comparison_df['even_sum_total_play_duration_0']\n", "play_duration_alignment_rate = comparison_df['play_duration_aligns'].mean()\n", "print(f\"Preference aligns with higher play duration in {play_duration_alignment_rate:.2%} of comparisons\")\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# sort rows by the play count difference\n", "comparison_df = comparison_df.sort_values(by='diff_play_count', ascending=False)\n", "comparison_df.head(10)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "plt.hist(comparison_df['user_n_clips'], bins=250)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "import numpy as np\n", "from scipy.stats import gaussian_kde\n", "\n", "feature_name = \"stereo_width\"\n", "\n", "filtered_comparison_df = comparison_df[comparison_df['user_n_clips'] > 15000]\n", "print(len(filtered_comparison_df))\n", "\n", "# Extract values\n", "preferred = filtered_comparison_df[f'odd_{feature_name}'].dropna()\n", "non_preferred = filtered_comparison_df[f'even_{feature_name}'].dropna()\n", "\n", "mean_preferred = np.mean(preferred)\n", "mean_non_preferred = np.mean(non_preferred)\n", "delta_mean = mean_preferred - mean_non_preferred\n", "print(f\"Mean preferred: {mean_preferred:.2f}\")\n", "print(f\"Mean non-preferred: {mean_non_preferred:.2f}\")\n", "print(f\"Delta mean: {delta_mean:.2f}\")\n", "\n", "# plot a histogram of the preferred and non_preferred\n", "plt.hist(non_preferred, bins=100, alpha=0.5, label='Non-preferred')\n", "plt.hist(preferred, bins=100, alpha=0.5, label='Preferred')\n", "\n", "plt.legend()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "filtered_comparison_df.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# lets create a new dataframe that compares the even and odd rows \n", "# we will compare the mean score for the even and odd rows\n", "# Create a new dataframe for comparison\n", "comparison_df = pd.DataFrame()\n", "\n", "# Assuming the data is organized in pairs (even/odd rows)\n", "# First, ensure we have an even number of rows to work with\n", "n_rows = len(metas)\n", "if n_rows % 2 != 0:\n", " n_rows -= 1 # Exclude the last row if total count is odd\n", "\n", "# Extract even and odd rows\n", "# even is negative and odd is positive \n", "# so we want negative difference for ear score\n", "# so we want positive difference for shimmer schore\n", "even_rows = metas.iloc[0:n_rows:2].reset_index()\n", "odd_rows = metas.iloc[1:n_rows:2].reset_index()\n", "\n", "# Create the comparison dataframe\n", "#comparison_df['upsample_clip_id'] = even_rows['upsample_clip_id']\n", "comparison_df['even_id'] = even_rows['id']\n", "comparison_df['odd_id'] = odd_rows['id']\n", "\n", "for feature in [\"mean_ear_v2_quality_score\", \"stereo_width\", \"shimmer_score\", \"play_count\", \"sum_total_play_duration_0\", \"spectral_centroid\"]:\n", " print(feature)\n", " comparison_df[f\"even_{feature}\"] = even_rows[feature].astype(float)\n", " comparison_df[f\"odd_{feature}\"] = odd_rows[feature].astype(float)\n", "\n", "\n", " # Calculate the score difference (even - odd)\n", " # even is the negative preference and odd is the positive preference\n", " # so a negative score means ear score of the positive is higher than the negative\n", " comparison_df[f\"diff_{feature}\"] = comparison_df[f\"even_{feature}\"] - comparison_df[f\"odd_{feature}\"]\n", " comparison_df[\"user_n_clips\"] = even_rows[\"user_n_clips\"]\n", " comparison_df[\"user_id\"] = even_rows[\"user_id\"]\n", "\n", "# Determine preference (1 if even is preferred, 0 if odd is preferred)\n", "#comparison_df['preference'] = (comparison_df['score_diff'] > 0).astype(int)\n", "\n", "# Calculate statistics\n", "#print(\"Comparison statistics:\")\n", "#print(f\"Total pairs: {len(comparison_df)}\")\n", "#print(f\"Pairs where even score > odd score: {comparison_df['preference'].sum()} ({comparison_df['preference'].mean()*100:.2f}%)\")\n", "#print(f\"Average score difference: {comparison_df['score_diff'].mean():.4f}\")\n", "#print(f\"Median score difference: {comparison_df['score_diff'].median():.4f}\")\n", "\n", "# Display the first few rows of the comparison dataframe\n", "print(\"\\nSample of comparison data:\")\n", "comparison_df.head()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "comparison_df['user_n_clips'].describe()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# let's plot the delta mean for spectral centroid as a function of user_n_clips\n", "\n", "#feature_name = \"spectral_centroid\"\n", "#feature_name = \"mean_ear_v2_quality_score\"\n", "#feature_name = \"stereo_width\"\n", "feature_name = \"shimmer_score\"\n", "\n", "delta_means = []\n", "for n in np.linspace(1, 15000, 250):\n", " filtered_comparison_df = comparison_df[comparison_df['user_n_clips'] > n]\n", " preferred = filtered_comparison_df[f'odd_{feature_name}'].dropna()\n", " non_preferred = filtered_comparison_df[f'even_{feature_name}'].dropna()\n", " mean_preferred = np.mean(preferred)\n", " mean_non_preferred = np.mean(non_preferred)\n", " delta_mean = mean_preferred - mean_non_preferred\n", " delta_means.append(delta_mean)\n", "\n", "plt.plot(np.linspace(1, 15000, 250), delta_means)\n", "plt.xlabel(\"User n clips > n\")\n", "plt.ylabel(\"Delta mean\")\n", "plt.title(f\"Delta mean for {feature_name}\")\n", "plt.show()\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "comparison_df['pref_aligns_with_ear'] = comparison_df['odd_mean_ear_v2_quality_score'] > comparison_df['even_mean_ear_v2_quality_score']\n", "\n", "\n", "\n", "# Group by user and compute alignment rate\n", "alignment_by_user = comparison_df.groupby('user_id')['pref_aligns_with_ear'].agg(\n", " align_rate='mean',\n", " n_comparisons='count'\n", ").sort_values(by='align_rate', ascending=False)\n", "\n", "plt.hist(alignment_by_user['n_comparisons'], bins=100)\n", "plt.show()\n", "\n", "# filter out user with less than 10 comparisons\n", "alignment_by_user = alignment_by_user[alignment_by_user['n_comparisons'] >= 10]\n", "print(len(alignment_by_user))\n", "\n", "# make a histogram of the align rate\n", "plt.hist(alignment_by_user['align_rate'], bins=100)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "top_users = alignment_by_user[alignment_by_user['n_comparisons'] >= 10].sort_values('align_rate', ascending=False).head(20)\n", "\n", "def wilson_interval(p, n, z=1.96):\n", " if n == 0:\n", " return (0, 0)\n", " denominator = 1 + z**2 / n\n", " centre = p + z**2 / (2 * n)\n", " margin = z * np.sqrt((p * (1 - p) / n) + (z**2 / (4 * n**2)))\n", " lower = (centre - margin) / denominator\n", " upper = (centre + margin) / denominator\n", " return lower, upper\n", "\n", "\n", "# Create columns for lower and upper bounds\n", "def compute_wilson_bounds(row):\n", " k = int(row['align_rate'] * row['n_comparisons']) # successes\n", " n = int(row['n_comparisons']) # total comparisons\n", " lower, upper = proportion_confint(count=k, nobs=n, method='wilson')\n", " return pd.Series({'wilson_lower': lower, 'wilson_upper': upper})\n", "\n", "# Apply to your DataFrame\n", "# Assuming df_user_alignments has align_rate and n_comparisons\n", "top_users[['wilson_lower', 'wilson_upper']] = top_users.apply(\n", " lambda row: pd.Series(wilson_interval(row['align_rate'], row['n_comparisons'])),\n", " axis=1\n", ")\n", "\n", "print(top_users)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# get comparisions only for user_id 48161320\n", "top_user_comparisons = comparison_df[comparison_df['user_id'] == 84727947]\n", "top_user_comparisons.head()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# cut the dataframe to only include rows with score difference greater than 0.5 abs\n", "comparison_df = comparison_df[comparison_df['diff_mean_ear_v2_quality_score'].abs() > 1.0]\n", "comparison_df = comparison_df[comparison_df['odd_shimmer_score'] < 0.5]\n", "comparison_df = comparison_df[comparison_df['odd_stereo_width'] < 0.5]\n", "\n", "print(len(comparison_df))\n", "# download all the stuff from s3\n", "print(comparison_df.describe())\n", "# make a hsitrogram of the score difference\n", "plt.figure(figsize=(5, 3))\n", "plt.hist(comparison_df['diff_mean_ear_v2_quality_score'], bins=50, edgecolor='black')\n", "plt.title('Histogram of Score Difference')\n", "plt.xlabel('Score Difference')\n", "plt.ylabel('Frequency')\n", "# add some text that counts the number of rows below -1.0 and above 1.0\n", "# Create a text box in the top left\n", "props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)\n", "textstr = f\"Number of rows below -1.0: {len(comparison_df[comparison_df['diff_mean_ear_v2_quality_score'] < -1.0])}\\n\"\n", "textstr += f\"Number of rows above 1.0: {len(comparison_df[comparison_df['diff_mean_ear_v2_quality_score'] > 1.0])}\"\n", "plt.text(0.05, 0.95, textstr, transform=plt.gca().transAxes, fontsize=10, verticalalignment='top', bbox=props)\n", "plt.tight_layout()\n", "plt.show()\n", "\n", "plt.figure(figsize=(5, 3))\n", "plt.hist(comparison_df[\"diff_shimmer_score\"], bins=50, edgecolor='black')\n", "plt.title('Histogram of Score Difference')\n", "plt.xlabel('Score Difference')\n", "plt.ylabel('Frequency')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# download all the stuff from s3\n", "s3_filepaths = [row['even_id'] for idx, row in comparison_df.iterrows()]\n", "s3_filepaths.extend([row['odd_id'] for idx, row in comparison_df.iterrows()])\n", "\n", "print(len(s3_filepaths))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "s3_filepaths[0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "output_dir = \"/app/suno/data/diff_syn_dpo/interesting_clips_ahi_d3_20250504/npz\"\n", "\n", "def download_from_s3(s3_filepaths):\n", " for s3_filepath in tqdm(s3_filepaths):\n", " for name in [\"vae\", \"\"]\n", " local_filepath = os.path.join(output_dir, f\"{s3_filepath}_{name}.npz\")\n", " print(local_filepath)\n", " os.makedirs(output_dir, exist_ok=True)\n", " os.system(f\"aws s3 cp {s3_filepath} {local_filepath}\")\n", "\n", "download_from_s3(s3_filepaths[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# sort the comparison dataframe by score difference\n", "#comparison_df = comparison_df.sort_values(by='score_diff', ascending=False)\n", "# print the first 10 rows of the sorted dataframe\n", "#comparison_df.head(10)\n", "data_df.head(10)\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.s3 import read_from_s3\n", "from suno_utils.audio import Audio\n", "\n", "idx = 100\n", "row = comparison_df.iloc[idx]\n", "audio_neg = Audio.from_s3(row['even_id'], n_channels=2)\n", "audio_pos = Audio.from_s3(row['odd_id'], n_channels=2)\n", "\n", "print(row['even_score'])\n", "audio_neg.play()\n", "print(row['odd_score'])\n", "audio_pos.play()\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "OUT_DATA_DIR = \"/app/suno/data/diff_dpo_cs/interesting_clips_ahi_d3_20250528\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "\n", "# save out the metas first\n", "# split the comparision_df into train and val\n", "# lets put 300 itmes in the val set\n", "# and the rest in the train set\n", "train_df = data_df.iloc[:-150]\n", "val_df = data_df.iloc[-150:]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# check for file ids in npz_dir\n", "npz_dir = \"/app2/suno/data/dpo/diff2_v2_d3\"\n", "\n", "count = 0\n", "even_ids = comparison_df[\"even_id\"]\n", "pbar = tqdm(even_ids)\n", "for even_id in pbar:\n", " filepath = os.path.join(npz_dir, even_id + \".npz\")\n", " if os.path.isfile(filepath):\n", " count += 1\n", " pbar.set_de\n", "print(f\"Found {count}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# to create a memmap for sft we will select the higheset scoring upsample_id for each base_s3_id\n", "# we also need to grab the correct vae latents and semantic codes and text prompt\n", "from tqdm import tqdm\n", "from suno_utils.utils.text import write_jsonl\n", "import gc\n", "import os\n", "import sys\n", "import shutil\n", "import numpy as np\n", "\n", "# aws s3 sync s3://suno-data/christian/outputs/corrupt/genius_t6_sampled_10k/ /app/suno/data/diff_syn_dpo/genius_t6_sampled_10k_corrupted/npz\n", "\n", "SEMANTIC_RATE_HZ = 25\n", "CHUNK_SIZE_S = 30\n", "CHUNK_SIZE = int(CHUNK_SIZE_S * SEMANTIC_RATE_HZ)\n", "OUT_DATA_DIR = \"/app/suno/data/diff_dpo_cs/interesting_clips_ahi_d3_20250528/t7\"\n", "\n", "#BASE_S3_DIR = \"s3://suno-data/christian/outputs/v2-infill-data-v1\"\n", "#BASE_LOCAL_DIR = \"/app/suno/data/diff_dpo/interesting_clips_ahi_d3_20250504/npz\"\n", "BASE_LOCAL_DIR = \"/app2/suno/data/dpo/diff2_v2_d3\"\n", "\n", "if not os.path.exists(OUT_DATA_DIR):\n", " os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "else:\n", " #shutil.rmtree(OUT_DATA_DIR)\n", " os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "\n", "for dset_type in [\"tr\"]:\n", "\n", " if dset_type == \"tr\":\n", " subset_df = train_df\n", " else:\n", " subset_df = val_df\n", "\n", " new_metas = []\n", "\n", " out_mm_semantic_filepath = os.path.join(OUT_DATA_DIR, f\"data_semantic_{dset_type}.bin\")\n", " out_mm_vae_filepath = os.path.join(OUT_DATA_DIR, f\"data_vae_{dset_type}.bin\")\n", " out_metas_filepath = os.path.join(OUT_DATA_DIR, f\"metas_{dset_type}.jsonl\")\n", "\n", " n_offs_v = 0\n", " n_offs_s = 0\n", " to_write_len_v = 0\n", " to_write_len_s = 0\n", " total_hours = 0 # Counter for total hours of audio\n", "\n", " out_mm_semantic = np.memmap(\n", " out_mm_semantic_filepath, dtype=np.uint16, mode=\"w+\", shape=(1,)\n", " )\n", "\n", " out_mm_vae = np.memmap(\n", " out_mm_vae_filepath, dtype=np.float16, mode=\"w+\", shape=(1,)\n", " )\n", "\n", " # clear the metas file\n", " with open(out_metas_filepath, \"w\") as f:\n", " f.write(\"\")\n", "\n", " # Create a tqdm progress bar with hours counter\n", " pbar = tqdm(subset_df.iterrows(), total=len(subset_df))\n", " pbar.set_description(\"Hours: 0.00\")\n", "\n", " for idx, row in pbar:\n", "\n", " # look up the upsample_clip_id in the sample_metas_map\n", " upsample_clip_id = source_metas_map[row[\"pos_id\"]][\"upsample_clip_id\"]\n", "\n", " # load semantic codes from disk\n", " semantic_codes_filepath = f\"{BASE_LOCAL_DIR}/{upsample_clip_id}.npz\"\n", " pos_vae_latents_filepath = f\"{BASE_LOCAL_DIR}/{row['pos_id']}_vae.npz\"\n", " neg_vae_latents_filepath = f\"{BASE_LOCAL_DIR}/{row['neg_id']}_vae.npz\"\n", "\n", " # ensure all files exist\n", " if not os.path.exists(semantic_codes_filepath):\n", " print(f\"File {semantic_codes_filepath} does not exist\")\n", " continue\n", " if not os.path.exists(pos_vae_latents_filepath):\n", " print(f\"File {pos_vae_latents_filepath} does not exist\")\n", " continue\n", " if not os.path.exists(neg_vae_latents_filepath):\n", " print(f\"File {neg_vae_latents_filepath} does not exist\")\n", " continue\n", "\n", " semantic_data = np.load(semantic_codes_filepath)\n", "\n", " if \"v3.0_raw\" in semantic_data:\n", " semantic_data = semantic_data[\"v3.0_raw\"]\n", " elif \"v3.5_raw\" in semantic_data:\n", " semantic_data = semantic_data[\"v3.5_raw\"]\n", " elif \"v4.0_raw\" in semantic_data:\n", " semantic_data = semantic_data[\"v4.0_raw\"]\n", " elif \"v5.0_raw\" in semantic_data:\n", " semantic_data = semantic_data[\"v5.0_raw\"]\n", " else:\n", " continue\n", "\n", " semantic_data = semantic_data[:,0]\n", " #neg_semantic_data = np.load(neg_semantic_codes_filepath)[\"codes\"][:, 0]\n", " pos_vae_data = np.load(pos_vae_latents_filepath)[\"vae_latents\"].astype(np.float16)\n", " neg_vae_data = np.load(neg_vae_latents_filepath)[\"vae_latents\"].astype(np.float16)\n", "\n", " num_sem_chunks = semantic_data.shape[0] // CHUNK_SIZE\n", " num_pos_vae_chunks = pos_vae_data.shape[0] // CHUNK_SIZE\n", " num_neg_vae_chunks = neg_vae_data.shape[0] // CHUNK_SIZE\n", " \n", " num_chunks = min([num_sem_chunks, num_pos_vae_chunks, num_neg_vae_chunks])\n", "\n", " if num_chunks == 0:\n", " #print(f\"No chunks for {row['pos_id']}\")\n", " continue\n", "\n", " # use no more than 1 chunks\n", " num_chunks = 1\n", "\n", " to_write_len_s = semantic_data[:750].size * num_chunks * 2\n", " to_write_len_v = pos_vae_data[:750, :].size * num_chunks * 2\n", " \n", " if to_write_len_s == 0:\n", " continue\n", "\n", " if to_write_len_v == 0:\n", " continue\n", " \n", " out_mm_semantic = np.memmap(\n", " out_mm_semantic_filepath,\n", " dtype=np.uint16,\n", " mode=\"r+\",\n", " shape=(n_offs_s + to_write_len_s,),\n", " )\n", "\n", " out_mm_vae = np.memmap(\n", " out_mm_vae_filepath,\n", " dtype=np.float16,\n", " mode=\"r+\",\n", " shape=(n_offs_v + to_write_len_v,),\n", " )\n", " # Add to total hours counter\n", " audio_duration_hours = (num_chunks * CHUNK_SIZE_S) / 3600\n", " total_hours += audio_duration_hours\n", " \n", " # Update progress bar description with current total hours\n", " pbar.set_description(f\"Hours: {total_hours:.2f}\")\n", "\n", " # we will write the positive and negative as interleaved chunks\n", " for i in np.arange(0, num_chunks):\n", " i = int(i)\n", " for vae_data in [neg_vae_data, pos_vae_data]:\n", " # get the tags and text from the map\n", " source_meta = source_metas_map[row[\"pos_id\"]]\n", " text = source_meta[\"text\"]\n", " tags = source_meta[\"tags\"]\n", " if text is None:\n", " text = \"\"\n", " if tags is None:\n", " tags = []\n", " # create a new meta\n", " new_meta = {\n", " \"id\": row[\"pos_id\"],\n", " \"start_s\": i*CHUNK_SIZE_S,\n", " \"end_s\": (i+1)*CHUNK_SIZE_S,\n", " \"original_duration_s\": semantic_data.shape[0] / SEMANTIC_RATE_HZ,\n", " \"n_vae_tokens\": CHUNK_SIZE,\n", " \"n_semantic_tokens\": CHUNK_SIZE,\n", " \"text\" : text,\n", " \"tags\" : tags,\n", " }\n", " new_metas.append(new_meta)\n", "\n", " semantic_chunk = semantic_data[i*CHUNK_SIZE:(i+1)*CHUNK_SIZE]\n", "\n", " out_mm_semantic[n_offs_s : n_offs_s + semantic_chunk.size] = semantic_chunk.reshape(\n", " -1,\n", " )\n", " n_offs_s += semantic_chunk.size\n", "\n", " vae_chunk = vae_data[i*CHUNK_SIZE:(i+1)*CHUNK_SIZE, :]\n", " out_mm_vae[n_offs_v : n_offs_v + vae_chunk.size] = vae_chunk.reshape(\n", " -1,\n", " )\n", " n_offs_v += vae_chunk.size\n", "\n", " print(f\"Total hours of audio added: {total_hours:.4f} for {dset_type} set\")\n", " \n", " write_jsonl(\n", " new_metas,\n", " os.path.join(out_metas_filepath),\n", " do_append=True\n", " )\n", "\n", " out_mm_semantic.flush()\n", " out_mm_vae.flush()\n", " del out_mm_semantic, out_mm_vae, f\n", " gc.collect()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 2 }