{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "8f1efec6", "metadata": {}, "outputs": [], "source": [ "# load the metas\n", "import os\n", "import pandas as pd\n", "\n", "from suno_utils.audio import Audio\n", "import numpy as np\n", "import os\n", "base_dir = \"/app2/suno/data/christian/outputs/v3-distill-data-ctx-t2/\"\n", "\n", "# get all files in each directory\n", "\n", "#model_name = \"16n_2b_flow_distill_bs1_N5_c5e5_g1e6_alt_dmd_cfg_2_residual_sft_220k\"\n", "model_name = \"v3_flow_distill_s3177_lm_t1_0_7_cut_history_4x_1E6_beta100_n8_bt2_acc4_2k_last\"\n", "\n", "def process_dir(dirpath):\n", " a_metadata_filepath = os.path.join(base_dir, dirpath, f\"{dirpath}_{model_name}_0__metadata.npz\")\n", " a_upsampled_vae_filepath = os.path.join(base_dir, dirpath, f\"{dirpath}_{model_name}_0_upsampled_vae.npz\")\n", " b_metadata_filepath = os.path.join(base_dir, dirpath, f\"{dirpath}_{model_name}_1__metadata.npz\")\n", " b_upsampled_vae_filepath = os.path.join(base_dir, dirpath, f\"{dirpath}_{model_name}_1_upsampled_vae.npz\")\n", "\n", " a_metadata = np.load(a_metadata_filepath, allow_pickle=True)\n", " b_metadata = np.load(b_metadata_filepath, allow_pickle=True)\n", "\n", " # put metadata into dict\n", " a_metadata_dict = {}\n", " b_metadata_dict = {}\n", " for key in a_metadata.keys():\n", " a_metadata_dict[key] = a_metadata[key].tolist()\n", " for key in b_metadata.keys():\n", " b_metadata_dict[key] = b_metadata[key].tolist()\n", "\n", " a_upsampled_vae = np.load(a_upsampled_vae_filepath)\n", " b_upsampled_vae = np.load(b_upsampled_vae_filepath)\n", "\n", " return a_metadata_dict, a_upsampled_vae, b_metadata_dict, b_upsampled_vae" ] }, { "cell_type": "code", "execution_count": 2, "id": "f7c759be", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "29970\n" ] } ], "source": [ "# get all directories in base_dir\n", "dirs = os.listdir(base_dir)\n", "print(len(dirs))" ] }, { "cell_type": "code", "execution_count": 3, "id": "8f1efec6", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 29970/29970 [00:41<00:00, 718.32it/s] \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Processed 29822 valid, with history 17822\n" ] } ], "source": [ "# create metas # in this case using ear score\n", "from tqdm import tqdm\n", "from joblib import Parallel, delayed\n", "\n", "def process_single_dir(dirpath):\n", " try: \n", " a_metadata, a_upsampled_vae, b_metadata, b_upsampled_vae = process_dir(dirpath)\n", " except Exception as e:\n", " #print(f\"Error processing {dirpath}: {e}\")\n", " return None, 0\n", "\n", " tags = a_metadata[\"tags\"]\n", " text = a_metadata[\"text\"]\n", " semantic_codes_filepath = os.path.join(base_dir, dirpath, f\"{dirpath}_semantic.npz\")\n", "\n", " pos_metadata = a_metadata\n", " neg_metadata = b_metadata\n", " pos_vae_latents_filepath = os.path.join(base_dir, dirpath, f\"{dirpath}_{model_name}_0_upsampled_vae.npz\")\n", " neg_vae_latents_filepath = os.path.join(base_dir, dirpath, f\"{dirpath}_{model_name}_1_upsampled_vae.npz\")\n", "\n", " history_vae_latents_filepath = os.path.join(base_dir, dirpath, f\"{dirpath}_history_vae.npz\")\n", " if os.path.exists(history_vae_latents_filepath):\n", " # Only load if you need to, but don't keep in meta to save memory\n", " # history_vae_latents = np.load(history_vae_latents_filepath)[\"vae_latents\"]\n", " with_history = 1\n", " else:\n", " history_vae_latents_filepath = None\n", " with_history = 0\n", "\n", " meta = {\n", " \"id\": dirpath,\n", " \"tags\": str(tags),\n", " \"text\": str(text),\n", " \"pos_metadata\": pos_metadata,\n", " \"neg_metadata\": neg_metadata,\n", " \"pos_vae_latents_filepath\": pos_vae_latents_filepath,\n", " \"neg_vae_latents_filepath\": neg_vae_latents_filepath,\n", " \"history_vae_latents_filepath\": history_vae_latents_filepath,\n", " \"semantic_codes_filepath\": semantic_codes_filepath,\n", " }\n", " return meta, with_history\n", "\n", "# Use joblib to parallelize\n", "results = Parallel(n_jobs=16)(\n", " delayed(process_single_dir)(dirpath) for dirpath in tqdm(dirs)\n", ")\n", "\n", "# Filter out failed (None) results and count with_history\n", "metas = []\n", "with_history = 0\n", "for meta, wh in results:\n", " if meta is not None:\n", " metas.append(meta)\n", " with_history += wh\n", "\n", "print(f\"Processed {len(metas)} valid, with history {with_history}\")" ] }, { "cell_type": "code", "execution_count": 4, "id": "17ae2f6f", "metadata": {}, "outputs": [], "source": [ "# now we want to measure 5% and 95% quantiles of the absolute feature scores and also the deltas\n", "\n", "import pandas as pd\n", "\n", "# feature list\n", "feature_list = [\"ear_score\", \"shimmer_score\", \"stereo_width\", \"hoot_cer\"]\n", "\n", "comparison_rows = []\n", "for meta in metas:\n", " row = {\"example_id\": meta[\"id\"]}\n", " for feature in feature_list:\n", " row[f\"pos_{feature}\"] = meta[\"pos_metadata\"].get(feature, None)\n", " row[f\"neg_{feature}\"] = meta[\"neg_metadata\"].get(feature, None)\n", " pos_val = row[f\"pos_{feature}\"]\n", " neg_val = row[f\"neg_{feature}\"]\n", " if pos_val is not None and neg_val is not None:\n", " row[f\"{feature}_delta\"] = pos_val - neg_val\n", " row[f\"{feature}_abs_delta\"] = abs(pos_val - neg_val)\n", " row[f\"{feature}_avg\"] = (pos_val + neg_val) / 2\n", " else:\n", " row[f\"{feature}_delta\"] = 0\n", " row[f\"{feature}_abs_delta\"] = 0\n", " comparison_rows.append(row)\n", "\n", "df_comparison = pd.DataFrame(comparison_rows)\n", "# add a column for the sum of the absolute deltas\n", "df_comparison[\"sum_abs_delta\"] = df_comparison.apply(lambda row: sum(row[f\"{feature}_abs_delta\"] for feature in feature_list), axis=1)" ] }, { "cell_type": "code", "execution_count": 5, "id": "a44462f7", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
example_idpos_ear_scoreneg_ear_scoreear_score_deltaear_score_abs_deltaear_score_avgpos_shimmer_scoreneg_shimmer_scoreshimmer_score_deltashimmer_score_abs_delta...neg_stereo_widthstereo_width_deltastereo_width_abs_deltastereo_width_avgpos_hoot_cerneg_hoot_cerhoot_cer_deltahoot_cer_abs_deltahoot_cer_avgsum_abs_delta
13781411d5d60-14f8-42b2-91c5-9d357188f60516.50297119.854829-3.3518583.35185818.17890028.67706753.124183-24.44711624.447116...0.154852-0.0363810.0363810.1366621.0001.0000.0000.0001.000027.835356
226088fa509de-e4e5-405a-9abb-093bb5db07fa21.06751516.9648054.1027104.10271019.0161608.49320831.807897-23.31468923.314689...0.2256530.0040870.0040870.2276970.8870.8820.0050.0050.884527.426486
2425400ca6d47-67ef-4f94-b2d1-0e841d80022923.23976619.9772073.2625593.26255921.60848627.9110138.32667519.58433819.584338...0.187582-0.0093300.0093300.182917NaNNaN0.0000.000NaN22.856228
2171353eda757-52a4-4d28-908c-f49602269a1b21.05678415.9863945.0703905.07039018.52158917.1196433.69704313.42259913.422599...0.234047-0.0760880.0760880.1960030.9070.9070.0000.0000.907018.569078
157387b999f9b-a88e-47a8-be6f-b00d0211741416.06343917.268896-1.2054571.20545716.66616819.2845782.26485517.01972317.019723...0.0490760.0286170.0286170.063385NaNNaN0.0000.000NaN18.253797
..................................................................
135732d0d9cf-6c16-4675-a376-3804dbd869ed19.87079019.872692-0.0019020.00190219.8717410.3663740.3663740.0000000.000000...0.089719-0.0168270.0168270.0813060.8850.8840.0010.0010.88450.019729
6129dd8296c8-4ef0-4c6e-ad55-604e6075efb922.02821822.037521-0.0093030.00930322.0328700.0000000.0000000.0000000.000000...0.0448570.0088830.0088830.0492990.8840.8840.0000.0000.88400.018186
27632a8165fb4-11ea-417f-ad81-861ac3b49af617.49396817.4873390.0066300.00663017.4906540.0000000.0000000.0000000.000000...0.0259330.0038600.0038600.0278620.9120.9080.0040.0040.91000.014489
28170c0d9ea8b-97c6-4729-8343-eccdb2278aa117.81884517.8153190.0035260.00352617.8170820.0000000.0000000.0000000.000000...0.0237300.0062740.0062740.0268680.8450.8450.0000.0000.84500.009801
902365b565e2-65ce-401a-899a-aa9a45343cce18.97319318.974776-0.0015840.00158418.9739850.0000000.0000000.0000000.000000...0.057658-0.0034850.0034850.0559150.9150.916-0.0010.0010.91550.006069
\n", "

29822 rows × 22 columns

\n", "
" ], "text/plain": [ " example_id pos_ear_score neg_ear_score \\\n", "13781 411d5d60-14f8-42b2-91c5-9d357188f605 16.502971 19.854829 \n", "22608 8fa509de-e4e5-405a-9abb-093bb5db07fa 21.067515 16.964805 \n", "24254 00ca6d47-67ef-4f94-b2d1-0e841d800229 23.239766 19.977207 \n", "21713 53eda757-52a4-4d28-908c-f49602269a1b 21.056784 15.986394 \n", "15738 7b999f9b-a88e-47a8-be6f-b00d02117414 16.063439 17.268896 \n", "... ... ... ... \n", "1357 32d0d9cf-6c16-4675-a376-3804dbd869ed 19.870790 19.872692 \n", "6129 dd8296c8-4ef0-4c6e-ad55-604e6075efb9 22.028218 22.037521 \n", "27632 a8165fb4-11ea-417f-ad81-861ac3b49af6 17.493968 17.487339 \n", "28170 c0d9ea8b-97c6-4729-8343-eccdb2278aa1 17.818845 17.815319 \n", "9023 65b565e2-65ce-401a-899a-aa9a45343cce 18.973193 18.974776 \n", "\n", " ear_score_delta ear_score_abs_delta ear_score_avg pos_shimmer_score \\\n", "13781 -3.351858 3.351858 18.178900 28.677067 \n", "22608 4.102710 4.102710 19.016160 8.493208 \n", "24254 3.262559 3.262559 21.608486 27.911013 \n", "21713 5.070390 5.070390 18.521589 17.119643 \n", "15738 -1.205457 1.205457 16.666168 19.284578 \n", "... ... ... ... ... \n", "1357 -0.001902 0.001902 19.871741 0.366374 \n", "6129 -0.009303 0.009303 22.032870 0.000000 \n", "27632 0.006630 0.006630 17.490654 0.000000 \n", "28170 0.003526 0.003526 17.817082 0.000000 \n", "9023 -0.001584 0.001584 18.973985 0.000000 \n", "\n", " neg_shimmer_score shimmer_score_delta shimmer_score_abs_delta ... \\\n", "13781 53.124183 -24.447116 24.447116 ... \n", "22608 31.807897 -23.314689 23.314689 ... \n", "24254 8.326675 19.584338 19.584338 ... \n", "21713 3.697043 13.422599 13.422599 ... \n", "15738 2.264855 17.019723 17.019723 ... \n", "... ... ... ... ... \n", "1357 0.366374 0.000000 0.000000 ... \n", "6129 0.000000 0.000000 0.000000 ... \n", "27632 0.000000 0.000000 0.000000 ... \n", "28170 0.000000 0.000000 0.000000 ... \n", "9023 0.000000 0.000000 0.000000 ... \n", "\n", " neg_stereo_width stereo_width_delta stereo_width_abs_delta \\\n", "13781 0.154852 -0.036381 0.036381 \n", "22608 0.225653 0.004087 0.004087 \n", "24254 0.187582 -0.009330 0.009330 \n", "21713 0.234047 -0.076088 0.076088 \n", "15738 0.049076 0.028617 0.028617 \n", "... ... ... ... \n", "1357 0.089719 -0.016827 0.016827 \n", "6129 0.044857 0.008883 0.008883 \n", "27632 0.025933 0.003860 0.003860 \n", "28170 0.023730 0.006274 0.006274 \n", "9023 0.057658 -0.003485 0.003485 \n", "\n", " stereo_width_avg pos_hoot_cer neg_hoot_cer hoot_cer_delta \\\n", "13781 0.136662 1.000 1.000 0.000 \n", "22608 0.227697 0.887 0.882 0.005 \n", "24254 0.182917 NaN NaN 0.000 \n", "21713 0.196003 0.907 0.907 0.000 \n", "15738 0.063385 NaN NaN 0.000 \n", "... ... ... ... ... \n", "1357 0.081306 0.885 0.884 0.001 \n", "6129 0.049299 0.884 0.884 0.000 \n", "27632 0.027862 0.912 0.908 0.004 \n", "28170 0.026868 0.845 0.845 0.000 \n", "9023 0.055915 0.915 0.916 -0.001 \n", "\n", " hoot_cer_abs_delta hoot_cer_avg sum_abs_delta \n", "13781 0.000 1.0000 27.835356 \n", "22608 0.005 0.8845 27.426486 \n", "24254 0.000 NaN 22.856228 \n", "21713 0.000 0.9070 18.569078 \n", "15738 0.000 NaN 18.253797 \n", "... ... ... ... \n", "1357 0.001 0.8845 0.019729 \n", "6129 0.000 0.8840 0.018186 \n", "27632 0.004 0.9100 0.014489 \n", "28170 0.000 0.8450 0.009801 \n", "9023 0.001 0.9155 0.006069 \n", "\n", "[29822 rows x 22 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_comparison.sort_values(\"sum_abs_delta\", ascending=False)" ] }, { "cell_type": "code", "execution_count": 6, "id": "9df1732c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "24772\n", "23826\n", "22379\n", "20544\n" ] } ], "source": [ "# make the cuts here\n", "\n", "# cut on the difference, so sum_abs_delta > 1.0\n", "df_filtered = df_comparison[df_comparison[\"sum_abs_delta\"] > 1.0]\n", "print(len(df_filtered))\n", "\n", "# cut on the average ear score > 15\n", "df_filtered = df_filtered[df_filtered[\"ear_score_avg\"] > 15]\n", "print(len(df_filtered))\n", "\n", "# cut on the average shimmer_score < 0.5\n", "df_filtered = df_filtered[df_filtered[\"shimmer_score_avg\"] < 6]\n", "print(len(df_filtered))\n", "\n", "# cut on the average stereo width > 0.05\n", "df_filtered = df_filtered[df_filtered[\"stereo_width_avg\"] > 0.05]\n", "print(len(df_filtered))\n", "\n" ] }, { "cell_type": "code", "execution_count": 7, "id": "ca8defd5", "metadata": {}, "outputs": [], "source": [ "# save out dataframe\n", "out_dir = \"/home/christian/code/christian/metadata/labelmaker/t2\"\n", "df_filtered.to_csv(os.path.join(out_dir, \"filtered_pairs.csv\"), index=False)" ] }, { "cell_type": "code", "execution_count": 8, "id": "a0f90f5c", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
pos_ear_scoreneg_ear_scoreear_score_deltaear_score_abs_deltaear_score_avgpos_shimmer_scoreneg_shimmer_scoreshimmer_score_deltashimmer_score_abs_deltashimmer_score_avg...neg_stereo_widthstereo_width_deltastereo_width_abs_deltastereo_width_avgpos_hoot_cerneg_hoot_cerhoot_cer_deltahoot_cer_abs_deltahoot_cer_avgsum_abs_delta
count20544.00000020544.00000020544.00000020544.00000020544.00000020544.00000020544.00000020544.00000020544.00000020544.000000...20544.00000020544.00000020544.00000020544.00000020474.00000020474.00000020544.00000020544.00000020474.00000020544.000000
mean18.93178518.954507-0.0227222.09503518.9431461.4646821.478343-0.0136610.9778271.471513...0.1337730.0002850.0221450.1339160.8583220.858507-0.0001840.0068630.8584143.101870
std2.3108292.3013972.5014041.3668041.9375041.5970181.6150571.4743561.1035041.426887...0.0541980.0294580.0194270.0521970.0997280.0997440.0151390.0134950.0994471.647720
min11.42193712.293645-9.9460260.00078215.0002430.0000000.000000-10.7913700.0000000.000000...0.000149-0.2201130.0000020.0500000.0000000.000000-0.2830000.0000000.0000001.000139
25%17.25120417.285129-1.8545341.07684617.5237700.2664540.266454-0.6328270.1998400.349720...0.091078-0.0168280.0080390.0925530.8210000.821000-0.0030000.0010000.8210001.840091
50%18.85207518.873774-0.0151141.82180818.8579150.8992810.8992810.0000000.6328270.999201...0.1279830.0002390.0172530.1282090.8690000.8690000.0000000.0030000.8685002.728913
75%20.51348420.5137391.7941772.86451920.2453332.1649352.1649350.5995211.3322682.164935...0.1694160.0176540.0308530.1686750.9120000.9120000.0030000.0080000.9118753.978946
max28.64537928.0551369.4935079.94602626.90798410.45830311.15774410.45830310.7913705.995206...0.5118050.2402620.2402620.5094811.0000001.0000000.4050000.4050001.00000014.700588
\n", "

8 rows × 21 columns

\n", "
" ], "text/plain": [ " pos_ear_score neg_ear_score ear_score_delta ear_score_abs_delta \\\n", "count 20544.000000 20544.000000 20544.000000 20544.000000 \n", "mean 18.931785 18.954507 -0.022722 2.095035 \n", "std 2.310829 2.301397 2.501404 1.366804 \n", "min 11.421937 12.293645 -9.946026 0.000782 \n", "25% 17.251204 17.285129 -1.854534 1.076846 \n", "50% 18.852075 18.873774 -0.015114 1.821808 \n", "75% 20.513484 20.513739 1.794177 2.864519 \n", "max 28.645379 28.055136 9.493507 9.946026 \n", "\n", " ear_score_avg pos_shimmer_score neg_shimmer_score \\\n", "count 20544.000000 20544.000000 20544.000000 \n", "mean 18.943146 1.464682 1.478343 \n", "std 1.937504 1.597018 1.615057 \n", "min 15.000243 0.000000 0.000000 \n", "25% 17.523770 0.266454 0.266454 \n", "50% 18.857915 0.899281 0.899281 \n", "75% 20.245333 2.164935 2.164935 \n", "max 26.907984 10.458303 11.157744 \n", "\n", " shimmer_score_delta shimmer_score_abs_delta shimmer_score_avg ... \\\n", "count 20544.000000 20544.000000 20544.000000 ... \n", "mean -0.013661 0.977827 1.471513 ... \n", "std 1.474356 1.103504 1.426887 ... \n", "min -10.791370 0.000000 0.000000 ... \n", "25% -0.632827 0.199840 0.349720 ... \n", "50% 0.000000 0.632827 0.999201 ... \n", "75% 0.599521 1.332268 2.164935 ... \n", "max 10.458303 10.791370 5.995206 ... \n", "\n", " neg_stereo_width stereo_width_delta stereo_width_abs_delta \\\n", "count 20544.000000 20544.000000 20544.000000 \n", "mean 0.133773 0.000285 0.022145 \n", "std 0.054198 0.029458 0.019427 \n", "min 0.000149 -0.220113 0.000002 \n", "25% 0.091078 -0.016828 0.008039 \n", "50% 0.127983 0.000239 0.017253 \n", "75% 0.169416 0.017654 0.030853 \n", "max 0.511805 0.240262 0.240262 \n", "\n", " stereo_width_avg pos_hoot_cer neg_hoot_cer hoot_cer_delta \\\n", "count 20544.000000 20474.000000 20474.000000 20544.000000 \n", "mean 0.133916 0.858322 0.858507 -0.000184 \n", "std 0.052197 0.099728 0.099744 0.015139 \n", "min 0.050000 0.000000 0.000000 -0.283000 \n", "25% 0.092553 0.821000 0.821000 -0.003000 \n", "50% 0.128209 0.869000 0.869000 0.000000 \n", "75% 0.168675 0.912000 0.912000 0.003000 \n", "max 0.509481 1.000000 1.000000 0.405000 \n", "\n", " hoot_cer_abs_delta hoot_cer_avg sum_abs_delta \n", "count 20544.000000 20474.000000 20544.000000 \n", "mean 0.006863 0.858414 3.101870 \n", "std 0.013495 0.099447 1.647720 \n", "min 0.000000 0.000000 1.000139 \n", "25% 0.001000 0.821000 1.840091 \n", "50% 0.003000 0.868500 2.728913 \n", "75% 0.008000 0.911875 3.978946 \n", "max 0.405000 1.000000 14.700588 \n", "\n", "[8 rows x 21 columns]" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_filtered.describe()" ] }, { "cell_type": "code", "execution_count": null, "id": "c46c8027", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio\n", "\n", "# get the audio files for an id\n", "item_id = \"47159907-5a9b-438f-94df-04d9cea7c331\"\n", "\n", "# get the audio files for the item\n", "audio_files = [f for f in os.listdir(os.path.join(base_dir, item_id)) if f.endswith(\".mp3\")]\n", "print(audio_files)\n", "\n", "file0 = Audio.from_file(os.path.join(base_dir, item_id, audio_files[0]), n_channels=2).play()\n", "file1 = Audio.from_file(os.path.join(base_dir, item_id, audio_files[1]), n_channels=2).play()" ] }, { "cell_type": "code", "execution_count": null, "id": "e75a84b1", "metadata": {}, "outputs": [], "source": [ "#make a histogram of sum_abs_delta\n", "\n", "import matplotlib.pyplot as plt\n", "\n", "df_deltas = df_comparison.dropna()\n", "\n", "feature = \"stereo_width_avg\"\n", "\n", "# 95% and 5% quantiles of sum_abs_delta\n", "print(np.quantile(df_deltas[f\"{feature}\"], 0.10))\n", "print(np.quantile(df_deltas[f\"{feature}\"], 0.95))\n", "\n", "# make a histogram of sum_abs_delta\n", "# plot the 5% and 95% quantiles\n", "plt.hist(df_deltas[f\"{feature}\"], bins=100)\n", "plt.axvline(np.quantile(df_deltas[f\"{feature}\"], 0.05), color=\"red\")\n", "plt.axvline(np.quantile(df_deltas[f\"{feature}\"], 0.95), color=\"red\")\n", "plt.show()\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "24de6de2", "metadata": {}, "outputs": [], "source": [ "# write a function to load to audio files from local and then compute the similarity score\n", "\n", "import torch\n", "import torch.nn.functional as F\n", "import torchaudio\n", "from torchaudio import transforms as T\n", "\n", "def _load_audio_mono(path: str, target_sr: int) -> tuple[torch.Tensor, int]:\n", " \"\"\"\n", " Load an audio file, convert to mono (1 x T), and resample to target_sr.\n", " Returns (waveform, sample_rate).\n", " \"\"\"\n", " waveform, sr = torchaudio.load(path) # waveform: (channels, T)\n", " # to mono\n", " if waveform.size(0) > 1:\n", " # unfold to only one channel\n", " waveform = torch.cat([waveform[0], waveform[1]], dim=0)\n", " # resample if needed\n", " if sr != target_sr:\n", " waveform = torchaudio.functional.resample(waveform, sr, target_sr)\n", " sr = target_sr\n", " return waveform, sr\n", "\n", "def _mel_vector(\n", " waveform: torch.Tensor,\n", " sr: int,\n", " n_mels: int = 128,\n", " n_fft: int = 1024,\n", " hop_length: int = 512,\n", ") -> torch.Tensor:\n", " \"\"\"\n", " Compute a time-averaged mel vector (size n_mels) from a mono waveform (1 x T).\n", " \"\"\"\n", " mel = T.MelSpectrogram(\n", " sample_rate=sr,\n", " n_fft=n_fft,\n", " hop_length=hop_length,\n", " n_mels=n_mels,\n", " )(waveform)\n", " return mel.squeeze(0)\n", "\n", "def _mfcc_vector(\n", " waveform: torch.Tensor,\n", " sr: int,\n", " n_mfcc: int = 40,\n", " n_fft: int = 1024,\n", " hop_length: int = 512,\n", " n_mels: int = 64,\n", " log_mels: bool = True,\n", " peak_norm: bool = True,\n", ") -> torch.Tensor:\n", " \"\"\"\n", " Compute a time-averaged MFCC vector (size n_mfcc) from a mono waveform (1 x T).\n", " \"\"\"\n", "\n", " if peak_norm:\n", " waveform = waveform / waveform.abs().max().clamp(min=1e-6)\n", "\n", " mfcc = T.MFCC(\n", " sample_rate=sr,\n", " n_mfcc=n_mfcc,\n", " melkwargs={\n", " \"n_fft\": n_fft,\n", " \"hop_length\": hop_length,\n", " \"n_mels\": n_mels,\n", " \"center\": True,\n", " \"power\": 2.0,\n", " \"norm\": \"slaney\",\n", " \"mel_scale\": \"htk\",\n", " },\n", " log_mels=log_mels,\n", " )(waveform) # shape: (1, n_mfcc, frames)\n", "\n", " # Mean-pool over time and drop channel dim -> (n_mfcc,)\n", " return mfcc.squeeze(0)#.mean(dim=-1)\n", "\n", "def cosine_similarity_mfcc(\n", " file_a: str,\n", " file_b: str,\n", " sample_rate: int = 48000,\n", " n_mfcc: int = 40,\n", " n_fft: int = 1024,\n", " hop_length: int = 512,\n", " n_mels: int = 64,\n", " device: str | torch.device = \"cpu\",\n", ") -> float:\n", " \"\"\"\n", " Load two MP3s and compute cosine similarity between their mean-pooled MFCC vectors.\n", "\n", " Returns:\n", " float in [-1, 1], where 1.0 means identical direction in MFCC space.\n", " \"\"\"\n", " device = torch.device(device)\n", "\n", " # Load & prep audio\n", " wav_a, sr_a = _load_audio_mono(file_a, sample_rate)\n", " wav_b, sr_b = _load_audio_mono(file_b, sample_rate)\n", " assert sr_a == sr_b == sample_rate\n", "\n", " wav_a = wav_a.to(device)\n", " wav_b = wav_b.to(device)\n", "\n", " # Compute MFCC vectors\n", " vec_a = _mel_vector(wav_a, sample_rate, n_mels=n_mels, n_fft=n_fft, hop_length=hop_length).to(device)\n", " vec_b = _mel_vector(wav_b, sample_rate, n_mels=n_mels, n_fft=n_fft, hop_length=hop_length).to(device)\n", "\n", " # Cosine similarity\n", " sim = F.cosine_similarity(vec_a.unsqueeze(0), vec_b.unsqueeze(0)).mean()\n", " return float(sim)\n", "\n", "# --- Example usage ---\n", "# sim = cosine_similarity_mfcc(\"audio1.mp3\", \"audio2.mp3\")\n", "# print(f\"Cosine similarity (MFCC): {sim:.4f}\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "574a738f", "metadata": {}, "outputs": [], "source": [ "from joblib import Parallel, delayed\n", "import json\n", "\n", "def compute_similarity_for_dir(main_dir):\n", " dir_path = os.path.join(base_dir, main_dir)\n", " files = [f for f in os.listdir(dir_path) if f.endswith(\".mp3\")]\n", " if len(files) != 2:\n", " return None # skip if not exactly 2 files\n", " file0 = os.path.join(dir_path, files[0])\n", " file1 = os.path.join(dir_path, files[1])\n", " similarity_score = cosine_similarity_mfcc(file0, file1)\n", " return {\"id\": main_dir, \"similarity_score\": similarity_score}\n", "\n", "results = Parallel(n_jobs=-1)(\n", " delayed(compute_similarity_for_dir)(main_dir) for main_dir in main_dirs\n", ")\n", "\n", "# Filter out None results (dirs that didn't have exactly 2 mp3s)\n", "results = [r for r in results if r is not None]\n", "\n", "# Convert to dict mapping id -> similarity_score\n", "similarity_dict = {r[\"id\"]: r[\"similarity_score\"] for r in results}\n", "\n", "# Save to JSON\n", "with open(\"/app2/suno/data/christian/outputs/v3-distill-data-ctx-t1/similarity_scores.json\", \"w\") as f:\n", " json.dump(similarity_dict, f, indent=2)\n", "\n", "# Optionally print results\n", "for k, v in similarity_dict.items():\n", " print(f\"{k}: {v:.4f}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "00e29d3a", "metadata": {}, "outputs": [], "source": [ "# make histogram of the similarity scores\n", "import json\n", "import matplotlib.pyplot as plt\n", "\n", "# make histogram of the similarity scores\n", "plt.hist(list(similarity_dict.values()), bins=100)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "id": "efa9ae95", "metadata": {}, "outputs": [], "source": [ "# count number of ids with similarity score < 0.85\n", "filtered_sim_scores = [v for v in similarity_dict.values() if v < 0.85]\n", "print(len(filtered_sim_scores))" ] }, { "cell_type": "code", "execution_count": null, "id": "76946e9f", "metadata": {}, "outputs": [], "source": [ "# sort the similarity scores by similarity score\n", "sorted_similarity_scores = sorted(similarity_dict.items(), key=lambda x: x[1], reverse=True)\n", "\n", "# print the top 100 similarity scores\n", "for k, v in sorted_similarity_scores[:10]:\n", " print(k, v)" ] }, { "cell_type": "code", "execution_count": null, "id": "cd666d06", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_diff", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.9" } }, "nbformat": 4, "nbformat_minor": 5 }