{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "#filepath = \"/home/christian/code/christian/metadata/genius_hq_audio_production_features_v2.csv\"\n",
    "#filepath = \"/home/christian/code/christian/metadata/youtube_music_audio_production_features_v2.csv\"\n",
    "#filepath = \"/home/christian/code/christian/metadata/imslp_audio_production_features_v2.csv\"\n",
    "filepath = \"/home/christian/code/christian/metadata/discogs_audio_production_features_v2.csv\"\n",
    "\n",
    "df = pd.read_csv(filepath, index_col=0)\n",
    "\n",
    "cols = [\n",
    "    \"spectral_centroid\",\n",
    "    \"bass\", \n",
    "    \"mid\", \n",
    "    \"high\", \n",
    "    \"stereo_width\", \n",
    "    \"spectral_flatness\", \n",
    "    \"crest_factor\",\n",
    "    \"silence_percentage\",\n",
    "    \"loudness\",\n",
    "]\n",
    "\n",
    "for col in cols:\n",
    "    print(col)\n",
    "    # create new column with standard normalized values\n",
    "    df[f\"{col}_normalized\"] = (df[col] - df[col].mean()) / df[col].std()\n",
    "\n",
    "\n",
    "df.describe()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the original metas from genius\n",
    "from suno_utils.utils.text import read_jsonl\n",
    "\n",
    "#metas = read_jsonl(\"/home/christian/code/christian/metadata/genius_hq_metas.jsonl\")\n",
    "metas = read_jsonl(\"/home/christian/code/christian/metadata/discogs_metas.jsonl\")\n",
    "\n",
    "print(len(metas))\n",
    "# find extreme values\n",
    "# create a metas map\n",
    "metas_map = {meta[\"id\"]: meta for meta in metas}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(df[\"silence_percentage\"], bins=5)\n",
    "plt.yscale(\"log\")\n",
    "plt.title(f\"genius_hq (len={len(df)})\")\n",
    "plt.xlabel(\"Silence Percentage\")\n",
    "plt.ylabel(\"Count\")\n",
    "plt.show()\n",
    "\n",
    "# count number of samples with silence percentage > 0.9\n",
    "count = len(df[df[\"silence_percentage\"] >= 80.0])\n",
    "percentage = count / len(df)\n",
    "\n",
    "silent_s = 0\n",
    "silent_df = df[df[\"silence_percentage\"] >= 80.0]\n",
    "for meta_id in silent_df.index:\n",
    "    meta = metas_map[meta_id]\n",
    "    duration_s = meta[\"duration_s\"]\n",
    "    silent_s += duration_s\n",
    "\n",
    "print(f\"Total silent time: {silent_s / 3600:.2f} hours\")\n",
    "print(f\"Number of samples with silence percentage > 80%: {count} ({percentage:.3%})\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(metas[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add a new column with the duration_s from the metas\n",
    "df[\"duration_s\"] = [metas_map[meta_id][\"duration_s\"] for meta_id in df.index]\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# can we make an xy plot of the silence percentage vs the youtube_views\n",
    "# first we have to create a new dataframe with the silence percentage added\n",
    "new_df = df.copy()\n",
    "new_df[\"youtube_views\"] = [metas_map[meta_id][\"youtube_views\"] for meta_id in new_df.index]\n",
    "new_df.head()\n",
    "\n",
    "# remove any rows with NaN values\n",
    "new_df = new_df.dropna()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# histogram of view_count\n",
    "view_count = new_df[\"youtube_views\"]\n",
    "# clip 95 percentile\n",
    "max_view_count = np.percentile(view_count, 95)\n",
    "min_view_count = np.percentile(view_count, 5)\n",
    "print(max_view_count / 1e6) # 4M\n",
    "print(min_view_count) # 2000\n",
    "view_count_clip = np.clip(view_count, min_view_count, max_view_count)\n",
    "plt.hist(view_count_clip, bins=100)\n",
    "plt.show()\n",
    "view_count_log = np.log10(view_count_clip)\n",
    "view_count_log_norm = view_count_log / view_count_log.max()\n",
    "#view_count_log_norm = np.clip(view_count_log_norm, 0.1, 1)\n",
    "plt.hist(view_count_log_norm, bins=100)\n",
    "plt.ylabel(\"Count\")\n",
    "plt.xlabel(\"View Count Log Normalized, sampling weight\")\n",
    "plt.show()\n",
    "\n",
    "# add these back to the new_df\n",
    "new_df[\"view_count_log\"] = view_count_log\n",
    "new_df[\"view_count_log_norm\"] = view_count_log_norm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sort by view_count_log_norm\n",
    "new_df = new_df.sort_values(by=\"view_count_log_norm\", ascending=True)\n",
    "# only print id and view_count_log_norm\n",
    "print(new_df[[\"view_count_log_norm\"]].head(15))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print the any rows with silence percentage > 80% and then sort by youtube views\n",
    "# get list of top 10 youtube views\n",
    "import os\n",
    "top_10_youtube_views = new_df[new_df[\"silence_percentage\"] > 80.0].sort_values(by=\"youtube_views\", ascending=False)[\"youtube_views\"].head(5)\n",
    "print(top_10_youtube_views)\n",
    "\n",
    "for meta_id in top_10_youtube_views.index:\n",
    "    print(metas_map[meta_id])\n",
    "    # get the audio\n",
    "    s3_filepath = metas_map[meta_id][\"audio_filepath\"]\n",
    "    print(s3_filepath)\n",
    "    # download the audio\n",
    "    os.system(f\"aws s3 cp {s3_filepath} /home/christian/code/christian/tmp/{meta_id}.mp3\")\n",
    "    # print the filepath locally\n",
    "    print(f\"/home/christian/code/christian/tmp/{meta_id}.mp3\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# sort by silence percentage\n",
    "df = df.sort_values(by=\"silence_percentage\", ascending=False)\n",
    "\n",
    "# get the first row\n",
    "for i in range(10):\n",
    "    row = df.iloc[i]\n",
    "    meta_id = row.name\n",
    "    # get the audio\n",
    "    s3_filepath = metas_map[meta_id][\"s3_filepath\"]\n",
    "    # download the audio\n",
    "    extension = \"mp3\" # s3_filepath.split(\".\")[-1]\n",
    "    os.system(f\"aws s3 cp {s3_filepath} /home/christian/code/christian/tmp/{meta_id}.{extension}\")\n",
    "\n",
    "    # print the filepath locally\n",
    "    print(f\"/home/christian/code/christian/tmp/{meta_id}.{extension}\", row[\"silence_percentage\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For a single feature:\n",
    "def get_feature_bounds(df, feature, multiplier=1.5):\n",
    "    Q1 = df[feature].quantile(0.25)\n",
    "    Q3 = df[feature].quantile(0.75)\n",
    "    IQR = Q3 - Q1\n",
    "    lower = Q1 - multiplier * IQR\n",
    "    upper = Q3 + multiplier * IQR\n",
    "    return lower, upper\n",
    "\n",
    "def get_feature_bounds_std(df, feature, multiplier=2.0):\n",
    "    std = df[feature].std()\n",
    "    mean = df[feature].mean()\n",
    "    lower = mean - multiplier * std\n",
    "    upper = mean + multiplier * std\n",
    "    return lower, upper\n",
    "\n",
    "for i, col in enumerate(cols):\n",
    "    lower, upper = get_feature_bounds(df, col, multiplier=2.0)\n",
    "    print(f\"\\n{col}:\")\n",
    "    print(f\"Lower bound: {lower:.2f}\")\n",
    "    print(f\"Upper bound: {upper:.2f}\")\n",
    "    print(f\"Values outside bounds: {df[~df[col].between(lower, upper)][col].count()} ({df[~df[col].between(lower, upper)][col].count() / len(df):.2%})\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filt_df = df[df[\"duration_s\"] > 60.0] # filter out short samples\n",
    "\n",
    "# make a multiplot histogram of all the normalized columns\n",
    "\n",
    "fig, axs = plt.subplots(nrows=len(cols), ncols=1, figsize=(10, 20), sharex=False, sharey=False)\n",
    "for i, col in enumerate(cols):\n",
    "    axs[i].hist(filt_df[f\"{col}\"], bins=500, zorder=10)\n",
    "    lower, upper = get_feature_bounds(df, f\"{col}\", multiplier=1.5)\n",
    "    #axs[i].axvline(x=lower, color='r', linestyle='--', label=f'Lower Bound: {lower:.2f}')\n",
    "    #axs[i].axvline(x=upper, color='r', linestyle='--', label=f'Upper Bound: {upper:.2f}')\n",
    "    axs[i].set_title(col)\n",
    "\n",
    "    #lower, upper = get_feature_bounds_std(filt_df, f\"{col}\", multiplier=2.0)\n",
    "    #print(f\"Values outside bounds: {df[~df[f'{col}'].between(lower, upper)][f'{col}'].count()} ({df[~df[f'{col}'].between(lower, upper)][f'{col}'].count() / len(df):.2%})\")\n",
    "    #axs[i].axvline(x=lower, color='g', linestyle='--', label=f'Lower Bound: {lower:.2f}')\n",
    "    #axs[i].axvline(x=upper, color='g', linestyle='--', label=f'Upper Bound: {upper:.2f}')\n",
    "    axs[i].set_yscale(\"log\")\n",
    "    axs[i].grid(color=\"lightgray\", zorder=0)\n",
    "plt.tight_layout()\n",
    "#plt.show()\n",
    "plt.savefig(\"discogs_feature_histograms.png\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find extreme values\n",
    "feature_cols = [f\"{col}_normalized\" for col in cols]\n",
    "\n",
    "# first filter out samples with silence percentage > 80%\n",
    "filtered_df = df[df[\"silence_percentage\"] < 80.0]\n",
    "# For standardized data, sum all the absolute values of the features\n",
    "filtered_df[\"sum_abs\"] = filtered_df[feature_cols].abs().sum(axis=1) / len(feature_cols)\n",
    "\n",
    "# now sort the dataframe by the sum of the absolute values\n",
    "filtered_df = filtered_df.sort_values(by=\"sum_abs\", ascending=False)\n",
    "print(filtered_df[\"sum_abs\"].describe())\n",
    "\n",
    "# histogram of the sum of the absolute values\n",
    "plt.hist(filtered_df[\"sum_abs\"], bins=np.linspace(0, 5, 200))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "\n",
    "# get the first row\n",
    "row = filtered_df.iloc[26]\n",
    "print(row)\n",
    "meta_id = row.name\n",
    "meta = metas_map[meta_id]\n",
    "for key, value in meta.items():\n",
    "    print(f\"{key}: {value}\")\n",
    "\n",
    "# get the audio\n",
    "s3_filepath = metas_map[meta_id][\"s3_filepath\"]\n",
    "print(s3_filepath)\n",
    "\n",
    "# download the audio\n",
    "extension = \"mp3\" # s3_filepath.split(\".\")[-1]  #\"mp3\"\n",
    "os.system(f\"aws s3 cp {s3_filepath} /home/christian/code/christian/tmp/{meta_id}.{extension}\")\n",
    "\n",
    "# print the filepath locally\n",
    "print(f\"/home/christian/code/christian/tmp/{meta_id}.{extension}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "feature = \"loudness\"\n",
    "# high spectral flatnees is noise-like\n",
    "# sort the dataframe by the feature\n",
    "df = df.sort_values(by=feature, ascending=False)\n",
    "\n",
    "# get the first row\n",
    "row = df.iloc[20]\n",
    "print(row)\n",
    "meta_id = row.name\n",
    "meta = metas_map[meta_id]\n",
    "for key, value in meta.items():\n",
    "    print(f\"{key}: {value}\")\n",
    "\n",
    "# get the audio\n",
    "s3_filepath = metas_map[meta_id][\"s3_filepath\"]\n",
    "print(s3_filepath)\n",
    "\n",
    "# download the audio\n",
    "os.system(f\"aws s3 cp {s3_filepath} /home/christian/code/christian/tmp/{meta_id}.mp3\")\n",
    "\n",
    "# print the filepath locally\n",
    "print(f\"/home/christian/code/christian/tmp/{meta_id}.mp3\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}