{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import torch\n", "import torchaudio\n", "from suno_utils.tasks.mert_25 import (\n", " preload_models as preload_semantic_models,\n", " load_model as load_semantic_model,\n", " encode as encode_semantic,\n", " EMBEDDING_RATE as SEMANTIC_HZ,\n", ")\n", "import numpy as np\n", "\n", "semantic_model_filepath=\"/home/georg/notebooks/gpu_nb/tmp/mert_25.pt\"\n", "semantic_clusters_filepath=\"/home/georg/notebooks/gpu_nb/tmp/mert_25_2x4k.npy\"\n", "\n", "_ = preload_semantic_models(semantic_model_filepath, semantic_clusters_filepath)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "x, sr = torchaudio.load(\"/home/christian/audio/reference-audio-wav/02 Dreams.wav\")\n", "print(x.shape)\n", "\n", "# crop to 30s\n", "x = x[:, :30 * sr]\n", "\n", "# convert to mono and resample to 24 khz\n", "x = x.mean(dim=0).unsqueeze(0)\n", "x = torchaudio.functional.resample(x, sr, 24000)\n", "\n", "print(x.shape)\n", "import IPython.display as ipd\n", "ipd.display(ipd.Audio(x.numpy(), rate=24000))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import IPython.display as ipd\n", "import matplotlib.pyplot as plt\n", "from matplotlib.colors import ListedColormap\n", "\n", "# create reference codes for the original audio\n", "reference_codes = encode_semantic(x)[:, 0]\n", "print(reference_codes.shape)\n", "\n", "# Store results for visualization\n", "noise_levels = [1e-6, 1e-5, 5e-5, 1e-4, 2e-4, 1e-3]\n", "diff_percentages = []\n", "diff_positions = []\n", "\n", "plt.figure(figsize=(8, 7))\n", "\n", "for i, noise_level in enumerate(noise_levels):\n", " # create white noise with specified level\n", " noise = torch.randn_like(x) * noise_level\n", " noisy_x = x + noise\n", " # encode semantic\n", " semantic_codes = encode_semantic(noisy_x)[:, 0]\n", " \n", " # compute the number of codes that are different\n", " differences = reference_codes != semantic_codes\n", " different_codes = np.sum(differences)\n", " percent_different = different_codes / reference_codes.shape[0]\n", " diff_percentages.append(percent_different)\n", " diff_positions.append(np.where(differences)[0])\n", " \n", " print(f\"{noise_level}: Number of different codes: {different_codes} ({percent_different:.2%})\")\n", " \n", " # Plot which tokens changed\n", " plt.subplot(len(noise_levels), 1, i+1)\n", " plt.imshow(differences.reshape(1, -1), cmap=ListedColormap(['lightgray', 'red']), aspect='auto')\n", " plt.title(f\"Noise level: {noise_level}, Different tokens: {different_codes} ({percent_different:.2%})\")\n", " plt.ylabel(\"Tokens\")\n", " if i == len(noise_levels) - 1:\n", " plt.xlabel(\"Token position\")\n", " plt.yticks([])\n", " \n", " # let's listen to the audio\n", " #ipd.display(ipd.Audio(noisy_x.numpy(), rate=24000))\n", "\n", "plt.tight_layout()\n", "plt.savefig(\"plots/semantic_sensitivity_noise.png\")\n", "\n", "# Plot summary of percentage differences\n", "#plt.figure(figsize=(5, 3))\n", "#plt.plot(range(len(noise_levels)), diff_percentages, 'o-')\n", "#plt.xticks(range(len(noise_levels)), [f\"{level}\" for level in noise_levels])\n", "#plt.xlabel(\"Noise Level\")\n", "#plt.ylabel(\"Percentage of Different Tokens\")\n", "#plt.title(\"Token Difference vs Noise Level\")\n", "#plt.grid(True)\n", "#plt.savefig(\"plots/semantic_sensitivity_noise.png\")\n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import IPython.display as ipd\n", "import matplotlib.pyplot as plt\n", "from matplotlib.colors import ListedColormap\n", "import torchaudio.functional as F\n", "\n", "# create reference codes for the original audio\n", "reference_codes = encode_semantic(x)[:, 0]\n", "print(reference_codes.shape)\n", "\n", "# Store results for visualization\n", "cutoff_frequencies = [2, 3, 4, 5, 10, 20]\n", "diff_percentages = []\n", "diff_positions = []\n", "\n", "plt.figure(figsize=(8, 7))\n", "\n", "sample_rate = 24000 # Assuming 24kHz sample rate, adjust if different\n", "\n", "for i, cutoff_freq in enumerate(cutoff_frequencies):\n", " # Apply highpass filter with increasing cutoff frequency\n", " filtered_x = F.highpass_biquad(x, sample_rate, cutoff_freq)\n", " \n", " # encode semantic\n", " semantic_codes = encode_semantic(filtered_x)[:, 0]\n", " \n", " # compute the number of codes that are different\n", " differences = reference_codes != semantic_codes\n", " different_codes = np.sum(differences)\n", " percent_different = different_codes / reference_codes.shape[0]\n", " diff_percentages.append(percent_different)\n", " diff_positions.append(np.where(differences)[0])\n", " \n", " print(f\"{cutoff_freq}Hz: Number of different codes: {different_codes} ({percent_different:.2%})\")\n", " \n", " # Plot which tokens changed\n", " plt.subplot(len(cutoff_frequencies), 1, i+1)\n", " plt.imshow(differences.reshape(1, -1), cmap=ListedColormap(['lightgray', 'red']), aspect='auto')\n", " plt.title(f\"Highpass cutoff: {cutoff_freq}Hz, Different tokens: {different_codes} ({percent_different:.2%})\")\n", " plt.ylabel(\"Tokens\")\n", " if i == len(cutoff_frequencies) - 1:\n", " plt.xlabel(\"Token position\")\n", " plt.yticks([])\n", " \n", " # let's listen to the audio\n", " ipd.display(ipd.Audio(filtered_x.numpy(), rate=24000))\n", "\n", "plt.tight_layout()\n", "plt.savefig(\"plots/semantic_sensitivity_highpass.png\")\n", "\n", "# Plot summary of percentage differences\n", "#plt.figure(figsize=(5, 3))\n", "#plt.plot(range(len(cutoff_frequencies)), diff_percentages, 'o-')\n", "#plt.xticks(range(len(cutoff_frequencies)), [f\"{freq}Hz\" for freq in cutoff_frequencies])\n", "#plt.xlabel(\"Highpass Cutoff Frequency\")#\n", "#plt.ylabel(\"Percentage of Different Tokens\")\n", "#plt.title(\"Token Difference vs Highpass Cutoff\")\n", "#plt.grid(True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_diff", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.9" } }, "nbformat": 4, "nbformat_minor": 2 }