{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from suno_utils.audio import Audio\n",
    "from suno_utils.utils.text import read_jsonl, read_json, write_jsonl, write_json\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from joblib import Parallel, delayed\n",
    "from pathlib import Path\n",
    "import time\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def summarize_meta(ds_path):\n",
    "    metas = read_jsonl(ds_path)\n",
    "    print(f\"{len(metas):,} tracks with {sum([m['duration_s'] for m in metas])/60/60:,.1f}h total\")\n",
    "    print(metas[0].keys())\n",
    "    return metas\n",
    "\n",
    "def sample_meta(metas):\n",
    "    random_index = random.randint(0, len(metas ) - 1)\n",
    "    sample = metas[random_index]\n",
    "    for k,v in sample.items():\n",
    "        print(f\"{k}: {v}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2",
   "metadata": {},
   "outputs": [],
   "source": [
    "metas = summarize_meta(\"/app2/suno/data/sara/sfx_get_beats/combined_v3_w_extreme_metas_v0_aligned.jsonl\")\n",
    "sample_meta(metas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(metas)\n",
    "short = df[df['duration_s'] < 1.5]\n",
    "print(len(short) * 100. / len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4",
   "metadata": {},
   "outputs": [],
   "source": [
    "id_to_metas = {}\n",
    "for meta in metas:\n",
    "    id = meta['id']\n",
    "    meta['is_reliable'] = True\n",
    "    id_to_metas[id] = meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5",
   "metadata": {},
   "outputs": [],
   "source": [
    "consolidated_results = read_jsonl(\"/app2/suno/data/sara/sfx_audio_analysis_consolidated.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6",
   "metadata": {},
   "outputs": [],
   "source": [
    "id_to_stats = {}\n",
    "for stats in consolidated_results:\n",
    "    id = stats['filename'].split(\"_analysis\")[0]\n",
    "    id_to_stats[id] = stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7",
   "metadata": {},
   "outputs": [],
   "source": [
    "for id, meta in id_to_metas.items(): # v0\n",
    "    stats = id_to_stats.get(id, None)\n",
    "    if stats is not None:\n",
    "        if stats['centroid'] and stats['centroid'] > 12_500:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['spread'] and stats['spread'] > 7_000:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['flatness'] and stats['flatness'] > 0.6:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['clipped_samples'] and stats['clipped_samples'] > 1_000:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['duration_s'] and stats['duration_s'] < 0.04:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['lufs_db'] and (stats['lufs_db'] > 0.0 or stats['lufs_db'] < -60.0):\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['stereo_width'] and stats['stereo_width'] > 0.95:\n",
    "            meta['is_reliable'] = False\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8",
   "metadata": {},
   "outputs": [],
   "source": [
    "for id, meta in id_to_metas.items(): # v1, more filtering\n",
    "    stats = id_to_stats.get(id, None)\n",
    "    if stats is not None:\n",
    "        if stats['centroid'] and stats['centroid'] > 12_500:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['spread'] and stats['spread'] > 6_000:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['flatness'] and stats['flatness'] > 0.5:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['clipped_samples'] and stats['clipped_samples'] > 500:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['duration_s'] and stats['duration_s'] < 0.04:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['lufs_db'] and (stats['lufs_db'] > 0.0 or stats['lufs_db'] < -50.0):\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['stereo_width'] and stats['stereo_width'] > 0.90:\n",
    "            meta['is_reliable'] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9",
   "metadata": {},
   "outputs": [],
   "source": [
    "for id, meta in id_to_metas.items(): # v3, middle ground\n",
    "    stats = id_to_stats.get(id, None)\n",
    "    if meta['dataset'] == \"pond_sfx\":\n",
    "        continue\n",
    "    if stats is not None:\n",
    "        if stats['centroid'] and stats['centroid'] > 12_500:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['spread'] and stats['spread'] > 7_000:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['flatness'] and stats['flatness'] > 0.6:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['clipped_samples'] and stats['clipped_samples'] > 500:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['duration_s'] and stats['duration_s'] < 0.04:\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['lufs_db'] and (stats['lufs_db'] > 0.0 or stats['lufs_db'] < -50.0):\n",
    "            meta['is_reliable'] = False\n",
    "        if stats['stereo_width'] and stats['stereo_width'] > 0.95:\n",
    "            meta['is_reliable'] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_fails = 0\n",
    "for id, meta in id_to_metas.items():\n",
    "    if not meta['is_reliable']:\n",
    "        total_fails += 1\n",
    "print(total_fails)\n",
    "print (100. * total_fails / len(metas))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11",
   "metadata": {},
   "outputs": [],
   "source": [
    "write_jsonl(metas, \"combined_v3_w_extreme_metas_v0_aligned_filtered.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12",
   "metadata": {},
   "outputs": [],
   "source": [
    "info_all_v1 = {}\n",
    "for m in metas:\n",
    "    dataset = m['dataset']\n",
    "    is_reliable = m['is_reliable']\n",
    "    id = m['id']\n",
    "    if dataset not in info_all_v1:\n",
    "        info_all_v1[dataset] = []\n",
    "    if is_reliable:\n",
    "        info_all_v1[dataset].append(id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13",
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json(info_all_v1, \"info_all_v2.json\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14",
   "metadata": {},
   "source": [
    "### Explore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_id = \"freesound_520432\"\n",
    "test_analysis = f\"/home/sara/sfx_analysis/{test_id}_analysis.json\"\n",
    "data = read_json(test_analysis)\n",
    "data_meta = id_to_metas[test_id]\n",
    "print(data_meta['tags'])\n",
    "Audio.from_s3(data_meta['s3_filepath']).play()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16",
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, val in data.items():\n",
    "    if isinstance(val, list):\n",
    "        numpy_data = np.asarray(val)\n",
    "        print(key, numpy_data.shape)\n",
    "        print(numpy_data)\n",
    "    else:\n",
    "        print(key, val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17",
   "metadata": {},
   "outputs": [],
   "source": [
    "average_spectrum_db = np.asarray(data['average_spectrum_db'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_spectral_features(spectrum_db, sample_rate=48000):\n",
    "    \"\"\"\n",
    "    Calculate spectral features from magnitude spectrum in dB\n",
    "    \n",
    "    Returns:\n",
    "    dict with spectral features in Hz\n",
    "    \"\"\"\n",
    "    \n",
    "    # Handle different input shapes\n",
    "    if len(spectrum_db.shape) == 2:\n",
    "        spectrum = spectrum_db[0]\n",
    "    else:\n",
    "        spectrum = spectrum_db\n",
    "    \n",
    "    # Convert from dB to linear magnitude\n",
    "    magnitude = 10 ** (spectrum / 20.0)\n",
    "    \n",
    "    # Create frequency bins\n",
    "    n_bins = len(spectrum)\n",
    "    nyquist = sample_rate / 2\n",
    "    frequencies = np.linspace(0, nyquist, n_bins)\n",
    "    \n",
    "    # Normalize magnitude\n",
    "    total_magnitude = np.sum(magnitude)\n",
    "    if total_magnitude == 0:\n",
    "        return None\n",
    "    \n",
    "    normalized_mag = magnitude / total_magnitude\n",
    "    \n",
    "    # Calculate features\n",
    "    centroid = np.sum(frequencies * normalized_mag)\n",
    "    spread = np.sqrt(np.sum(((frequencies - centroid) ** 2) * normalized_mag))\n",
    "    \n",
    "    # Rolloff (85% energy cutoff)\n",
    "    cumulative_mag = np.cumsum(normalized_mag)\n",
    "    rolloff_idx = np.where(cumulative_mag >= 0.85)[0]\n",
    "    rolloff = frequencies[rolloff_idx[0]] if len(rolloff_idx) > 0 else nyquist\n",
    "    \n",
    "    # Flatness (tonal vs noise)\n",
    "    geometric_mean = np.exp(np.mean(np.log(magnitude[magnitude > 0])))\n",
    "    arithmetic_mean = np.mean(magnitude)\n",
    "    flatness = geometric_mean / arithmetic_mean if arithmetic_mean > 0 else 0\n",
    "    \n",
    "    return {\n",
    "        'centroid': centroid,\n",
    "        'spread': spread,\n",
    "        'rolloff': rolloff,\n",
    "        'flatness': flatness,\n",
    "        'peak_freq': frequencies[np.argmax(magnitude)],\n",
    "    }\n",
    "\n",
    "def plot_spectrum_analysis(spectrum_db, sample_rate=48000):\n",
    "    \"\"\"Plot spectrum with key features marked\"\"\"\n",
    "    \n",
    "    spectrum = spectrum_db[0] if len(spectrum_db.shape) == 2 else spectrum_db\n",
    "    features = calculate_spectral_features(spectrum_db, sample_rate)\n",
    "    \n",
    "    if features is None:\n",
    "        print(\"Cannot calculate features - spectrum has no energy\")\n",
    "        return\n",
    "    \n",
    "    # Create frequency axis\n",
    "    n_bins = len(spectrum)\n",
    "    frequencies = np.linspace(0, sample_rate/2, n_bins)\n",
    "    \n",
    "    # Plot\n",
    "    plt.figure(figsize=(6, 4))\n",
    "    plt.plot(frequencies, spectrum, linewidth=2, color='#2E86AB', label='Spectrum')\n",
    "    plt.fill_between(frequencies, spectrum, alpha=0.3, color='#2E86AB')\n",
    "    \n",
    "    # Mark features\n",
    "    plt.axvline(features['centroid'], color='red', linestyle='--', linewidth=2, \n",
    "                label=f\"Centroid: {features['centroid']:.0f} Hz\")\n",
    "    plt.axvline(features['rolloff'], color='orange', linestyle='--', linewidth=2, \n",
    "                label=f\"Rolloff: {features['rolloff']:.0f} Hz\")\n",
    "    plt.axvline(features['peak_freq'], color='green', linestyle=':', linewidth=2, \n",
    "                label=f\"Peak: {features['peak_freq']:.0f} Hz\")\n",
    "    \n",
    "    plt.xlabel('Frequency (Hz)', fontsize=12, fontweight='bold')\n",
    "    plt.ylabel('Amplitude (dB)', fontsize=12, fontweight='bold')\n",
    "    plt.title('Audio Spectrum Analysis', fontsize=14, fontweight='bold')\n",
    "    plt.grid(True, alpha=0.3)\n",
    "    plt.legend()\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    \n",
    "    return features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19",
   "metadata": {},
   "outputs": [],
   "source": [
    "features = plot_spectrum_analysis(average_spectrum_db, sample_rate=48000)\n",
    "print(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_single_json(json_file_path, sample_rate=48000):\n",
    "    \"\"\"Process a single JSON file and return features\"\"\"\n",
    "    \n",
    "    try:\n",
    "        data = read_json(json_file_path)\n",
    "        \n",
    "        # Extract average_spectrum_db\n",
    "        if 'average_spectrum_db' not in data:\n",
    "            return None\n",
    "        \n",
    "        features = {}\n",
    "        if data.get('average_spectrum_db', None) is not None:\n",
    "            spectrum_db = np.asarray(data['average_spectrum_db'])\n",
    "            features = calculate_spectral_features(spectrum_db, sample_rate)\n",
    "\n",
    "        mean_abs_stereo_diff = 0.0\n",
    "        if data.get('average_stereo_spectrum_side', None) is not None:\n",
    "            stereo_diff = np.asarray(data['average_stereo_spectrum_side'])\n",
    "            mean_abs_stereo_diff = float(np.mean(np.abs(stereo_diff)))\n",
    "        \n",
    "        # Return filename and features\n",
    "        return {\n",
    "            'filename': Path(json_file_path).name,\n",
    "            'centroid': features.get('centroid', None),\n",
    "            'spread': features.get('spread', None),\n",
    "            'rolloff': features.get('rolloff', None),\n",
    "            'flatness': features.get('flatness', None),\n",
    "            'mean_abs_stereo_diff': mean_abs_stereo_diff,\n",
    "            'duration_s': data.get('duration_seconds', None),\n",
    "            'lufs_db': data.get('lufs_db',None), \n",
    "            'lufs_db_factor': data.get('lufs_db_factor',None), \n",
    "            'rms_loudness_db': data.get('rms_loudness_db',None), \n",
    "            'peak_loudness_db': data.get('peak_loudness_db',None), \n",
    "            'stereo_width': data.get('stereo_width',None), \n",
    "            'clipped_samples': data.get('clipped_samples',None), \n",
    "        }\n",
    "        \n",
    "    except Exception as e:\n",
    "        # Return error info for debugging if needed\n",
    "        print(e)\n",
    "        return None\n",
    "\n",
    "def process_json_folder_parallel(folder_path, sample_rate=48000, n_jobs=10, batch_size=1000, progress_update=10000, max_files=None):\n",
    "    \"\"\"\n",
    "    Process all JSON files in parallel with progress tracking\n",
    "    \n",
    "    Parameters:\n",
    "    folder_path: Path to folder containing JSON files\n",
    "    sample_rate: Audio sample rate in Hz\n",
    "    n_jobs: Number of parallel jobs (-1 for all cores)\n",
    "    batch_size: Number of files to process in each batch\n",
    "    progress_update: Print progress every N files\n",
    "    \n",
    "    Returns:\n",
    "    dict with lists of all features\n",
    "    \"\"\"\n",
    "    \n",
    "    folder = Path(folder_path)\n",
    "    if not folder.exists():\n",
    "        print(f\"Folder {folder_path} does not exist!\")\n",
    "        return None\n",
    "    \n",
    "    # Get all JSON files\n",
    "    json_files = list(folder.glob('*.json'))\n",
    "    if max_files is not None:\n",
    "        json_files = json_files[:max_files]\n",
    "    total_files = len(json_files)\n",
    "    \n",
    "    print(f\"Found {total_files:,} JSON files\")\n",
    "    print(f\"Using {n_jobs if n_jobs > 0 else os.cpu_count()} parallel jobs\")\n",
    "    print(f\"Processing in batches of {batch_size:,} files\")\n",
    "    \n",
    "    # Initialize results\n",
    "    all_results = []\n",
    "    processed_count = 0\n",
    "    start_time = time.time()\n",
    "    \n",
    "    # Process files in batches to manage memory\n",
    "    for i in range(0, total_files, batch_size):\n",
    "        batch_files = json_files[i:i + batch_size]\n",
    "        batch_start = time.time()\n",
    "        \n",
    "        # Process batch in parallel\n",
    "        batch_results = Parallel(n_jobs=n_jobs, backend='threading')(\n",
    "            delayed(process_single_json)(json_file, sample_rate) \n",
    "            for json_file in batch_files\n",
    "        )\n",
    "        \n",
    "        # Filter out None results and add to main results\n",
    "        valid_results = [r for r in batch_results if r is not None]\n",
    "        all_results.extend(valid_results)\n",
    "        \n",
    "        processed_count += len(batch_files)\n",
    "        batch_time = time.time() - batch_start\n",
    "        \n",
    "        # Progress update\n",
    "        if processed_count % progress_update == 0 or processed_count == total_files:\n",
    "            elapsed_time = time.time() - start_time\n",
    "            rate = processed_count / elapsed_time\n",
    "            eta = (total_files - processed_count) / rate if rate > 0 else 0\n",
    "            \n",
    "            print(f\"Processed {processed_count:,}/{total_files:,} files \"\n",
    "                  f\"({processed_count/total_files*100:.1f}%) | \"\n",
    "                  f\"Valid: {len(all_results):,} | \"\n",
    "                  f\"Rate: {rate:.0f} files/sec | \"\n",
    "                  f\"ETA: {eta/60:.1f} min | \"\n",
    "                  f\"Batch time: {batch_time:.1f}s\")\n",
    "    \n",
    "    # Convert to the expected format\n",
    "    if not all_results:\n",
    "        print(\"No valid results found!\")\n",
    "        return None\n",
    "    \n",
    "    total_time = time.time() - start_time\n",
    "    print(f\"\\nCompleted! Processed {len(all_results):,} valid files in {total_time/60:.1f} minutes\")\n",
    "    print(f\"Average rate: {len(all_results)/total_time:.0f} files/sec\")\n",
    "    \n",
    "    return all_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = process_json_folder_parallel(\"/home/sara/sfx_analysis\", max_files=40_000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(results)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_histograms(df, exclude_columns=None):\n",
    "    \"\"\"\n",
    "    Create histograms for each numeric feature with mean, median, and std dev.\n",
    "    \n",
    "    Parameters:\n",
    "    df: pandas DataFrame with numeric columns\n",
    "    exclude_columns: list of column names to exclude, or None to include all\n",
    "    \"\"\"\n",
    "    \n",
    "    # Determine which columns to plot\n",
    "    if exclude_columns is None:\n",
    "        exclude_columns = []\n",
    "    \n",
    "    columns_to_plot = [col for col in df.columns if col not in exclude_columns]\n",
    "    \n",
    "    if not columns_to_plot:\n",
    "        print(\"No columns to plot after exclusions\")\n",
    "        return\n",
    "    \n",
    "    n_cols = len(columns_to_plot)\n",
    "    n_rows = (n_cols + 2) // 3  # 3 columns per row\n",
    "    \n",
    "    # Create subplot grid based only on columns we're actually plotting\n",
    "    fig, axes = plt.subplots(n_rows, min(3, n_cols), figsize=(5 * min(3, n_cols), 5 * n_rows))\n",
    "    \n",
    "    # Handle different subplot configurations\n",
    "    if n_rows == 1 and n_cols == 1:\n",
    "        axes = [axes]\n",
    "    elif n_rows == 1:\n",
    "        axes = axes if n_cols > 1 else [axes]\n",
    "    else:\n",
    "        axes = axes.flatten()\n",
    "    \n",
    "    # Only process the columns we're actually plotting\n",
    "    for i, column in enumerate(columns_to_plot):\n",
    "        ax = axes[i]\n",
    "        \n",
    "        # Filter out NaN and infinity values\n",
    "        data_clean = df[column].replace([np.inf, -np.inf], np.nan).dropna()\n",
    "        \n",
    "        if len(data_clean) == 0:\n",
    "            ax.text(0.5, 0.5, 'No valid data\\n(all NaN/inf)', ha='center', va='center', transform=ax.transAxes)\n",
    "            ax.set_title(f'{column} (No valid data)')\n",
    "            continue\n",
    "        \n",
    "        # Calculate statistics\n",
    "        mean_val = data_clean.mean()\n",
    "        median_val = data_clean.median()\n",
    "\n",
    "        # Create histogram\n",
    "        ax.hist(data_clean, bins=30, alpha=0.7, edgecolor='black', color='skyblue')\n",
    "        \n",
    "        # Add vertical lines for statistics\n",
    "        ax.axvline(mean_val, color='red', linestyle='--', linewidth=2, label=f'Mean: {mean_val:.2f}')\n",
    "        ax.axvline(median_val, color='green', linestyle='--', linewidth=2, label=f'Median: {median_val:.2f}')\n",
    "        \n",
    "        ax.set_title(f'Distribution of {column}')\n",
    "        ax.set_xlabel(column)\n",
    "        ax.set_ylabel('Frequency')\n",
    "        ax.legend()\n",
    "    \n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    \n",
    "    print(f\"Plotted {len(columns_to_plot)} columns. Excluded: {exclude_columns}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_histograms(df, exclude_columns=['filename'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filename'] = df['filename'].apply(lambda x: x.split(\"_analysis\")[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = df[(df['centroid'] > 12_500)] # could consider also a duration cutoff here to not include hihats and such but I think this may be sufficient\n",
    "print(len(test))\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = test.sample(n=1).iloc[0]\n",
    "print(sample)\n",
    "id = sample['filename']\n",
    "s3_fp = id_to_metas[id]['s3_filepath']\n",
    "print(id_to_metas[id]['tags'])\n",
    "audio = Audio.from_s3(s3_fp)\n",
    "audio.play()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = df[df['spread'] > 7000] # above 7000 seems just noise\n",
    "print(len(test))\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = test.sample(n=1).iloc[0]\n",
    "print(sample)\n",
    "id = sample['filename']\n",
    "s3_fp = id_to_metas[id]['s3_filepath']\n",
    "print(id_to_metas[id]['tags'])\n",
    "audio = Audio.from_s3(s3_fp)\n",
    "audio.play()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = df[df['rolloff'] > 17_500] # eh this is hit or miss, mabye don't filter on this\n",
    "print(len(test))\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = test.sample(n=1).iloc[0]\n",
    "print(sample)\n",
    "id = sample['filename']\n",
    "s3_fp = id_to_metas[id]['s3_filepath']\n",
    "print(id_to_metas[id]['tags'])\n",
    "audio = Audio.from_s3(s3_fp)\n",
    "audio.play()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = df[df['flatness'] > 0.6] # above 0.6 basically noise\n",
    "print(len(test))\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = test.sample(n=1).iloc[0]\n",
    "print(sample)\n",
    "id = sample['filename']\n",
    "s3_fp = id_to_metas[id]['s3_filepath']\n",
    "print(id_to_metas[id]['tags'])\n",
    "audio = Audio.from_s3(s3_fp)\n",
    "audio.play()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['clips_per_s'] = df['clipped_samples'] / df['duration_s']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36",
   "metadata": {},
   "outputs": [],
   "source": [
    "#test = df[df['clips_per_s'] > (48_000 * 0.1)]\n",
    "\n",
    "test = df[df['clipped_samples'] > 1000]\n",
    "print(len(test))\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = test.sample(n=1).iloc[0]\n",
    "print(sample)\n",
    "id = sample['filename']\n",
    "s3_fp = id_to_metas[id]['s3_filepath']\n",
    "print(id_to_metas[id]['tags'])\n",
    "audio = Audio.from_s3(s3_fp)\n",
    "audio.play()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = df[df['mean_abs_stereo_diff'] > 175] # this is just detecting things that are panned all the way. Probably should just reformat these as mono instead of deleting\n",
    "print(len(test))\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = test.sample(n=1).iloc[0]\n",
    "print(sample)\n",
    "id = sample['filename']\n",
    "s3_fp = id_to_metas[id]['s3_filepath']\n",
    "print(id_to_metas[id]['tags'])\n",
    "audio = Audio.from_s3(s3_fp)\n",
    "audio.play()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = df[df['duration_s'] < 0.04] # filtering out things less than 1 token\n",
    "print(len(test))\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = test.sample(n=1).iloc[0]\n",
    "print(sample)\n",
    "id = sample['filename']\n",
    "s3_fp = id_to_metas[id]['s3_filepath']\n",
    "print(id_to_metas[id]['tags'])\n",
    "audio = Audio.from_s3(s3_fp)\n",
    "audio.play()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = df[(df['lufs_db'] < -60) | (df['lufs_db'] > 0)] # this is very quiet or very loud\n",
    "print(len(test))\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = test.sample(n=1).iloc[0]\n",
    "print(sample)\n",
    "id = sample['filename']\n",
    "s3_fp = id_to_metas[id]['s3_filepath']\n",
    "print(id_to_metas[id]['tags'])\n",
    "audio = Audio.from_s3(s3_fp)\n",
    "audio.play()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = df[df['stereo_width'] > 0.95] # very panned to one side; again probably don't filter based on this but could adjust the audio? or filter for 0.9 + \n",
    "print(len(test))\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = test.sample(n=1).iloc[0]\n",
    "print(sample)\n",
    "id = sample['filename']\n",
    "s3_fp = id_to_metas[id]['s3_filepath']\n",
    "print(id_to_metas[id]['tags'])\n",
    "audio = Audio.from_s3(s3_fp)\n",
    "audio.play()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_clean",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}