{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#filepath = \"/home/tony/Data/Preference/auk/interesting_clips_exp_20250427_diff_ab.pkl\"\n",
    "filepath = \"/home/tony/Data/Preference/up_v2_d5/interesting_clips_20250721_flow.pkl\"\n",
    "df = pd.read_pickle(filepath)\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in df[\"model_name\"]:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "comparisons = []\n",
    "# Loop through the dataframe in pairs (even index = negative preference, odd index = positive preference)\n",
    "for i in range(0, len(df), 2):\n",
    "    if i+1 < len(df):  # Make sure we have a pair\n",
    "        model_neg = df.iloc[i][\"model_name\"]\n",
    "        model_pos = df.iloc[i+1][\"model_name\"]\n",
    "\n",
    "        if model_neg == model_pos:\n",
    "            comparisons.append({\n",
    "                \"negative_id\": df.iloc[i][\"id\"],\n",
    "                \"positive_id\": df.iloc[i+1][\"id\"],\n",
    "                \"negative_model\": model_neg,\n",
    "                \"positive_model\": model_pos,\n",
    "            })\n",
    "        \n",
    "print(len(comparisons))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "comparisons[1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get audio \n",
    "from suno_utils.audio import Audio\n",
    "idx = 100\n",
    "print(comparisons[idx])\n",
    "negative_id = comparisons[idx][\"negative_id\"]\n",
    "positive_id = comparisons[idx][\"positive_id\"]\n",
    "\n",
    "negative_audio = Audio.from_s3(\"s3://suno-data-uploads/studio/uploads/\" + str(negative_id) + \".mp3\", n_channels=2)\n",
    "positive_audio = Audio.from_s3(\"s3://suno-data-uploads/studio/uploads/\" + str(positive_id) + \".mp3\", n_channels=2)\n",
    "\n",
    "print(\"positive_audio\")\n",
    "negative_audio.play()\n",
    "\n",
    "print(\"negative_audio\")\n",
    "positive_audio.play()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_vae_latents(s3_id):\n",
    "    filepath = f\"s3://suno-data-uploads/studio/uploads/{s3_id}_vae.npz\"\n",
    "    data = read_from_s3(filepath, read_f=np.load)\n",
    "    vae_latents = data[\"vae_latents\"]\n",
    "    # get the first 30s (750 tokens)\n",
    "    vae_latents_30s = vae_latents[:750, :]\n",
    "    # now compute std and mean of the latents\n",
    "    mean = vae_latents_30s.mean()\n",
    "    std = vae_latents_30s.std()\n",
    "    return mean, std\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the vae for positive model \n",
    "from suno_utils.utils.s3 import read_from_s3\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "def process_comparison(comparison):\n",
    "    if comparison[\"positive_model\"] == \"chirp-v4-up-u-7\":\n",
    "        pos_s3_id = comparison[\"positive_id\"]\n",
    "        neg_s3_id = comparison[\"negative_id\"]\n",
    "        \n",
    "        pos_mean, pos_std = get_vae_latents(pos_s3_id)\n",
    "        neg_mean, neg_std = get_vae_latents(neg_s3_id)\n",
    "        \n",
    "        return pos_mean, neg_mean, pos_std, neg_std\n",
    "    return None\n",
    "\n",
    "# Filter comparisons for chirp-v4-up-u-7 model\n",
    "filtered_comparisons = [comp for comp in comparisons if comp[\"positive_model\"] == \"chirp-v4-up-u-7\"]\n",
    "\n",
    "# Process comparisons in parallel\n",
    "results = Parallel(n_jobs=-1)(delayed(process_comparison)(comparison) for comparison in tqdm(filtered_comparisons))\n",
    "\n",
    "# Extract results\n",
    "pos_means = []\n",
    "neg_means = []\n",
    "pos_stds = []\n",
    "neg_stds = []\n",
    "\n",
    "for result in results:\n",
    "    if result is not None:\n",
    "        pos_mean, neg_mean, pos_std, neg_std = result\n",
    "        pos_means.append(pos_mean)\n",
    "        neg_means.append(neg_mean)\n",
    "        pos_stds.append(pos_std)\n",
    "        neg_stds.append(neg_std)\n",
    "    \n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_deltas = np.array(pos_means) - np.array(neg_means)\n",
    "print(\"mean_deltas\", np.mean(mean_deltas))\n",
    "\n",
    "std_deltas = np.array(pos_stds) - np.array(neg_stds)\n",
    "print(\"std_deltas\", np.mean(std_deltas))\n",
    "\n",
    "# make a boxplot of the pos and neg means as seperated boxplots\n",
    "plt.boxplot(std_deltas, vert=False)\n",
    "# add a line at 0\n",
    "plt.axvline(0, color='black', linewidth=0.5)\n",
    "plt.xlabel(\"Model\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "# make a boxplot of the pos and neg means as seperated boxplots\n",
    "plt.boxplot([pos_means, neg_means], labels=[\"Positive\", \"Negative\"], vert=False)\n",
    "# add a line at 0\n",
    "plt.axvline(0, color='black', linewidth=0.5)\n",
    "plt.xlabel(\"Model\")\n",
    "plt.ylabel(\"Mean\")\n",
    "plt.show()\n",
    "\n",
    "print(\"pos_means\", np.mean(pos_means))\n",
    "print(\"neg_means\", np.mean(neg_means))\n",
    "\n",
    "\n",
    "# new plot for the positive and negattive stds\n",
    "plt.boxplot([pos_stds, neg_stds], labels=[\"Positive\", \"Negative\"], vert=False)\n",
    "plt.axvline(0, color='black', linewidth=0.5)\n",
    "plt.xlabel(\"Model\")\n",
    "plt.ylabel(\"Std\")\n",
    "plt.show()\n",
    "\n",
    "print(\"pos_stds\", np.mean(pos_stds))\n",
    "print(\"neg_stds\", np.mean(neg_stds))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import os\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "s3_ids = df[\"id\"].tolist()\n",
    "print(len(s3_ids))\n",
    "\n",
    "OUT_DIR = \"/home/christian/data/ab/interesting_clips_exp_20250427_diff_ab\"\n",
    "OUR_DIR = \"\"\n",
    "os.makedirs(OUT_DIR, exist_ok=True)\n",
    "\n",
    "s3_filepaths = [f\"s3://suno-data-uploads/studio/uploads/{s3_id}.mp3\" for s3_id in s3_ids]\n",
    "\n",
    "def download_audio(s3_filepath):\n",
    "    output_filepath = os.path.join(OUT_DIR, os.path.basename(s3_filepath))\n",
    "    if os.path.exists(output_filepath):\n",
    "        return\n",
    "    os.system(f\"aws s3 cp {s3_filepath} {output_filepath} > /dev/null 2>&1\")\n",
    "\n",
    "Parallel(n_jobs=-1)(delayed(download_audio)(s3_filepath) for s3_filepath in s3_filepaths)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_subset = df.iloc[100:110]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "OUT_DIR = \"/home/christian/data/ab/interesting_clips_exp_20250427_diff_ab\"\n",
    "\n",
    "import pyloudnorm as pyln\n",
    "import soundfile as sf\n",
    "import pandas as pd\n",
    "\n",
    "meter = pyln.Meter(48000)\n",
    "\n",
    "def get_loudness_from_row(row):\n",
    "    # load filepath\n",
    "    s3_id = row.id\n",
    "    audio, sr = sf.read(os.path.join(OUT_DIR, f\"{s3_id}.mp3\"))\n",
    "    loudness_db = meter.integrated_loudness(audio)\n",
    "    return {\"id\": s3_id, \"loudness_db\": loudness_db}\n",
    "    \n",
    "# Get loudness values and file IDs\n",
    "loudness_data = Parallel(n_jobs=-1)(delayed(get_loudness_from_row)(row) for row in df.itertuples())\n",
    "\n",
    "# Convert the list of dictionaries to a DataFrame\n",
    "loudness_df = pd.DataFrame(loudness_data)\n",
    "\n",
    "# merge the loudness data with the original dataframe\n",
    "df_with_loudness = df.merge(loudness_df, on='id', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_with_loudness.iloc[101]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter for rows where the model is either chirp-v4-up-u-7 or chirp-v4-up-u-d-2-i-1\n",
    "model_a = \"chirp-v4-up-u-7\"\n",
    "model_b = \"chirp-v4-up-u-d-2-i-1\"\n",
    "\n",
    "# Create a list to store the comparison pairs\n",
    "comparisons = []\n",
    "\n",
    "# Loop through the dataframe in pairs (even index = negative preference, odd index = positive preference)\n",
    "for i in range(0, len(df), 2):\n",
    "    if i+1 < len(df):  # Make sure we have a pair\n",
    "        model_neg = df.iloc[i][\"model_name\"]\n",
    "        model_pos = df.iloc[i+1][\"model_name\"]\n",
    "        \n",
    "        # Check if this pair contains our target models\n",
    "        if (model_neg == model_a and model_pos == model_b) or (model_neg == model_b and model_pos == model_a):\n",
    "            neg_id = df.iloc[i][\"id\"]\n",
    "            pos_id = df.iloc[i+1][\"id\"]\n",
    "            \n",
    "            # Record which model was preferred\n",
    "            preferred_model = model_pos\n",
    "            \n",
    "            # Get loudness values if available\n",
    "            neg_loudness = df_with_loudness[df_with_loudness[\"id\"] == neg_id][\"loudness_db\"].values[0] if neg_id in df_with_loudness[\"id\"].values else None\n",
    "            pos_loudness = df_with_loudness[df_with_loudness[\"id\"] == pos_id][\"loudness_db\"].values[0] if pos_id in df_with_loudness[\"id\"].values else None\n",
    "            \n",
    "            comparisons.append({\n",
    "                \"negative_id\": neg_id,\n",
    "                \"positive_id\": pos_id,\n",
    "                \"negative_model\": model_neg,\n",
    "                \"positive_model\": model_pos,\n",
    "                \"preferred_model\": preferred_model,\n",
    "                \"negative_loudness_db\": neg_loudness,\n",
    "                \"positive_loudness_db\": pos_loudness\n",
    "            })\n",
    "\n",
    "# Convert to DataFrame for easier analysis\n",
    "comparison_df = pd.DataFrame(comparisons)\n",
    "\n",
    "# Display the results\n",
    "if len(comparison_df) > 0:\n",
    "    print(f\"Found {len(comparison_df)} comparisons between {model_a} and {model_b}\")\n",
    "    print(comparison_df.head())\n",
    "    \n",
    "    # Count preferences\n",
    "    model_a_wins = sum(comparison_df[\"preferred_model\"] == model_a)\n",
    "    model_b_wins = sum(comparison_df[\"preferred_model\"] == model_b)\n",
    "    print(f\"\\nPreference counts:\")\n",
    "    print(f\"{model_a}: {model_a_wins}\")\n",
    "    print(f\"{model_b}: {model_b_wins}\")\n",
    "    \n",
    "    # Calculate average loudness for each model\n",
    "    model_a_loudness = comparison_df[comparison_df[\"positive_model\"] == model_a][\"positive_loudness_db\"].mean()\n",
    "    model_b_loudness = comparison_df[comparison_df[\"positive_model\"] == model_b][\"positive_loudness_db\"].mean()\n",
    "    print(f\"\\nAverage loudness:\")\n",
    "    print(f\"{model_a}: {model_a_loudness:.2f} dB\")\n",
    "    print(f\"{model_b}: {model_b_loudness:.2f} dB\")\n",
    "else:\n",
    "    print(f\"No comparisons found between {model_a} and {model_b}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, row in comparison_df.iterrows():\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# now print all pairs of rows where chirp-v4-up-u-7 is the positive model\n",
    "chirp_v4_up_u_7_positive = comparison_df[comparison_df[\"positive_model\"] == model_a]\n",
    "print(f\"\\nFound {len(chirp_v4_up_u_7_positive)} comparisons where {model_a} is the positive model:\")\n",
    "#chirp_v4_up_u_7_positive.head()\n",
    "\n",
    "# add a column to the comparison_df that is the difference in loudness between the positive and negative model\n",
    "comparison_df[\"loudness_difference\"] = comparison_df[\"positive_loudness_db\"] - comparison_df[\"negative_loudness_db\"]\n",
    "\n",
    "# print the top 10 rows of the comparison_df\n",
    "# average negative_loudness_db and positive_loudness_db\n",
    "print(comparison_df[\"negative_loudness_db\"].mean())\n",
    "print(comparison_df[\"positive_loudness_db\"].mean())\n",
    "print(comparison_df[\"loudness_difference\"].mean())\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets go through all the clips, download the audio, and measure the loudness \n",
    "comparison_df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Filter data for each model as positive\n",
    "model_a_positive = comparison_df[comparison_df[\"positive_model\"] == model_a][\"loudness_difference\"]\n",
    "model_b_positive = comparison_df[comparison_df[\"positive_model\"] == model_b][\"loudness_difference\"]\n",
    "\n",
    "# Create boxplot for both models\n",
    "plt.figure(figsize=(5, 3))\n",
    "boxplot_data = [model_a_positive, model_b_positive]\n",
    "labels = [f\"{model_a} is positive\", f\"{model_b} is positive\"]\n",
    "plt.boxplot(boxplot_data, vert=False, labels=labels)\n",
    "plt.title(\"Loudness Difference by Positive Model\")\n",
    "plt.xlabel(\"Loudness Difference (dB)\")\n",
    "plt.grid(True)\n",
    "plt.show()\n",
    "\n",
    "# positive means the positive model is louder\n",
    "# negative means the negative model is louder\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_diff",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}