{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Consolidate midi datasets\n", "\n", "This notebook consolidates the midi datasets into a single dataset. It also serves as documentation for the midi datasets.\n", "\n", "It produces a metadata file pointing to the midi and audio files.\n", "\n", "## Datasets\n", "\n", "- Infinite synthetic MIDI\n", "- Mew\n", "\t- `/app2/suno/data/mew/scoretube`\n", "\t- `/app2/suno/data/mew/mmd_chunks`\n", "\t- Match midi instruments to stems.\n", "- A dataset of loosely paired real music with arrangements\n", "\t- [Slack](https://suno-main.slack.com/archives/C03A7347CSK/p1719682717375969)\n", "\t- 20k songs matched with high confidence\n", "\t- [The AI workspace that works for you. | Notion](https://www.notion.so/suno-ai/MooTube-187b01573ccf8065915ecb04af262d6c#187b01573ccf80de941bc4412c900eca)\n", "- Hooktheory melodys, 50k hooks only\n", "\t- [The AI workspace that works for you. | Notion](https://www.notion.so/suno-ai/hooktheory-91267f75401f4087b506f086abefc65f)\n", "- Trombone champ melodys. 4k full songs\n", "\t- [The AI workspace that works for you. | Notion](https://www.notion.so/suno-ai/Trombone-Champ-206b01573ccf8009aa1fe22f3440687c)\n", "- Piano Maestro\n", "\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# mew\n", "\n", "Victors's personal stash" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "import glob\n", "import random\n", "from suno_utils.audio import Audio\n", "from suno_utils.audio.midi import Midi\n", "from tqdm import tqdm\n", "\n", "out_stem_dir = \"/app2/suno/data/victor/midi_stems\"\n", "\n", "\n", "class MidiPair:\n", " def __init__(self, midi_file: str, audio_file: str):\n", " self.midi_file = midi_file\n", " self.audio_file = audio_file\n", " self.midi = None\n", " self.audio = None\n", "\n", " def load_midi(self):\n", " if self.midi is not None:\n", " return self.midi\n", " self.midi = Midi.from_path(self.midi_file)\n", " return self.midi\n", "\n", " def load_audio(self):\n", " if self.audio is not None:\n", " return self.audio\n", " self.audio = Audio.from_file(self.audio_file)\n", " return self.audio\n", "\n", " def load_all(self):\n", " self.load_midi()\n", " self.load_audio()\n", "\n", " def __str__(self):\n", " return f\"MidiPair(midi_file={self.midi_file}, audio_file={self.audio_file})\"\n", "\n", " def __repr__(self):\n", " return self.__str__()\n", "\n", " def play(self):\n", " self.load_all()\n", " stereo_audio = self.midi.make_stereo_comparison(self.audio)\n", " stereo_audio.play()\n", "\n", "\n", "scoretube_dir = \"/app2/suno/data/victor/mew/scoretube\"\n", "\n", "\n", "def load_pairs(dir: str, audio_ext: str = \"mp3\", midi_ext: str = \"mid\"):\n", " midi_files = glob.glob(os.path.join(dir, \"**\", f\"*.{midi_ext}\"), recursive=True)\n", " audio_files = glob.glob(os.path.join(dir, \"**\", f\"*.{audio_ext}\"), recursive=True)\n", "\n", " # Create a mapping of base filenames to audio files\n", " audio_map = {}\n", " for audio_file in audio_files:\n", " base_name = os.path.splitext(os.path.basename(audio_file))[0]\n", " audio_map[base_name] = audio_file\n", "\n", " pairs = []\n", " for midi_file in midi_files:\n", " base_name = os.path.splitext(os.path.basename(midi_file))[0]\n", " if base_name in audio_map:\n", " pairs.append(MidiPair(midi_file, audio_map[base_name]))\n", "\n", " return pairs" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## trombone champ" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "trombone_champ_dir = \"/app2/suno/data/victor/trombone_champ\"\n", "trombone_champ_pairs = load_pairs(trombone_champ_dir, audio_ext=\"opus\")\n", "print(f\"Loaded {len(trombone_champ_pairs)} trombone champ pairs\")\n", "\n", "# play random pair\n", "random_pair = random.choice(trombone_champ_pairs)\n", "print(random_pair)\n", "# random_pair.play()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Check stems" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# !du -sh /app2/suno/data/victor/midi_stems/" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# count number of stems in each dataset\n", "pairs_to_process = {\n", " # \"scoretube\": scoretube_pairs,\n", " \"trombone_champ\": trombone_champ_pairs,\n", " # \"hooktheory\": hooktheory_pairs,\n", " # \"mootube\": mootube_pairs,\n", " # \"mmd\": mmd_pairs,\n", "} # ignore maestro\n", "for dataset in pairs_to_process.keys():\n", " # count number of stems in each dataset in the dir\n", " dir = f\"{out_stem_dir}/{dataset}\"\n", " # count number of files in the dir\n", " num_files = len(os.listdir(dir))\n", " print(f\"{dataset}: {num_files} stemmed\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# pair up midis with stems\n", "def process_pair(args):\n", " pair, dataset = args\n", " # find stems\n", " id = pair.audio_file.split(\"/\")[-1].split(\".\")[0]\n", " stem_dir = f\"{out_stem_dir}/{dataset}/{id}\"\n", " if not os.path.exists(stem_dir):\n", " # print(f\"{id} not found\")\n", " return None\n", " # count number of files in the dir\n", " files = os.listdir(stem_dir)\n", " files = [stem_dir + \"/\" + file for file in files]\n", " assert len(files)\n", " return pair, files\n", "\n", "\n", "def pair_up_midis(dataset, pairs):\n", " from concurrent.futures import ThreadPoolExecutor\n", "\n", " with ThreadPoolExecutor(max_workers=20) as executor:\n", " args = [(pair, dataset) for pair in pairs]\n", " results = executor.map(process_pair, args)\n", " for result in results:\n", " if result is not None:\n", " yield result\n", "\n", "\n", "stem_pairs = []\n", "for dataset, pairs in pairs_to_process.items():\n", " print(f\"Processing {dataset}...\")\n", " stem_pairs.extend(list(tqdm(pair_up_midis(dataset, pairs), total=len(pairs))))\n", "\n", "print(f\"Found {len(stem_pairs)} stem pairs\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Count stems and stem types\n", "from collections import defaultdict\n", "import matplotlib.pyplot as plt\n", "\n", "stem_counts = defaultdict(int)\n", "stem_type_counts = defaultdict(int)\n", "\n", "for pair, files in stem_pairs:\n", " stem_counts[\"total_pairs\"] += 1\n", " for file in files:\n", " # Extract stem type from filename (assuming format like \"vocals.wav\", \"drums.wav\", etc.)\n", " stem_type = file.split(\"/\")[-1].split(\".\")[0]\n", " stem_type_counts[stem_type] += 1\n", " stem_counts[\"total_stems\"] += 1\n", "\n", "print(f\"Total pairs with stems: {stem_counts['total_pairs']}\")\n", "print(f\"Total stems: {stem_counts['total_stems']}\")\n", "print(\"\\nStem type distribution:\")\n", "for stem_type, count in sorted(stem_type_counts.items()):\n", " print(f\" {stem_type}: {count}\")\n", "\n", "# Plot stem type distribution\n", "plt.figure(figsize=(12, 6))\n", "sorted_items = sorted(stem_type_counts.items(), key=lambda x: x[1], reverse=True)\n", "stem_types = [item[0] for item in sorted_items]\n", "counts = [item[1] for item in sorted_items]\n", "\n", "plt.bar(stem_types, counts)\n", "plt.title(\"Stem Type Distribution\")\n", "plt.xlabel(\"Stem Type\")\n", "plt.ylabel(\"Count\")\n", "plt.xticks(rotation=45, ha=\"right\")\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# filter out pairs with no vocals\n", "\n", "stem_pairs_filtered = []\n", "for pair, files in stem_pairs:\n", " files = [file for file in files if \"Vocals.opus\" in file]\n", " if len(files) == 0:\n", " continue\n", " stem_pairs_filtered.append((pair, files[0]))\n", "\n", "print(f\"Filtered {len(stem_pairs_filtered)} pairs\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import librosa\n", "import numpy as np\n", "import scipy.spatial.distance\n", "import copy\n", "# from dtw import dtw\n", "\n", "\n", "# Audio/CQT parameters\n", "FS = 22050\n", "NOTE_START = 36\n", "N_NOTES = 84\n", "HOP_LENGTH = 1024\n", "# DTW parameters\n", "GULLY = 0.96\n", "\n", "\n", "def plot_cqt(cqt: np.ndarray):\n", " \"\"\"Plot the CQT\"\"\"\n", " import matplotlib.pyplot as plt\n", "\n", " plt.figure(figsize=(12, 6))\n", " plt.imshow(cqt.T, aspect=\"auto\", origin=\"lower\", interpolation=\"nearest\")\n", " plt.colorbar(label=\"Magnitude (dB)\")\n", " plt.xlabel(\"Time (frames)\")\n", " plt.ylabel(\"MIDI Note\")\n", " plt.title(\"Constant-Q Transform\")\n", " plt.tight_layout()\n", " plt.show()\n", "\n", "\n", "def compute_cqt(audio: Audio):\n", " \"\"\"Compute the CQT and frame times for some audio data\"\"\"\n", " audio_data = audio.resample(FS).array_float\n", " # Compute CQT\n", " cqt = librosa.cqt(\n", " audio_data,\n", " sr=FS,\n", " fmin=librosa.midi_to_hz(NOTE_START),\n", " n_bins=N_NOTES,\n", " hop_length=HOP_LENGTH,\n", " tuning=0.0,\n", " )\n", " # Compute the time of each frame\n", " times = librosa.frames_to_time(\n", " np.arange(cqt.shape[1]), sr=FS, hop_length=HOP_LENGTH\n", " )\n", " # Compute log-amplitude\n", " cqt = librosa.amplitude_to_db(np.abs(cqt), ref=np.abs(cqt).max())\n", " # Plot the CQT\n", " # Normalize and return\n", " return librosa.util.normalize(cqt, axis=0).T, times" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## fix octave shifts" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "stem_pairs_filtered" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test = \"01040_0\"\n", "midi_pair, stems = [pair for pair in stem_pairs_filtered if test in pair[0].midi_file][\n", " 0\n", "]\n", "\n", "midi_pair, stems = random.choice(stem_pairs_filtered)\n", "\n", "midi = midi_pair.load_midi()\n", "midi_audio = midi.to_audio().resample(22050)\n", "stem = Audio.from_file(stems).resample(22050)\n", "\n", "midi.make_stereo_comparison(stem).play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import torchaudio\n", "import pesto\n", "import tempfile\n", "\n", "with tempfile.NamedTemporaryFile(suffix=\".wav\") as f:\n", " stem.write_wav(f.name)\n", "\n", " # Load audio (ensure mono; stereo channels are treated as separate batch dimensions)\n", " x, sr = torchaudio.load(f.name)\n", " x = x.mean(dim=0) # PESTO takes mono audio as input\n", "\n", " # Predict pitch. x can be (num_samples) or (batch, num_samples)\n", " timesteps, pitch, confidence, activations = pesto.predict(x, sr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pitch_pesto = pitch.clone()\n", "confidence_pesto = confidence.clone()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import torchcrepe\n", "import tempfile\n", "import torch\n", "import torchaudio\n", "\n", "# Process in 60s chunks to avoid OOM\n", "chunk_duration = 60\n", "total_duration = stem.duration_s\n", "all_results = []\n", "all_results_periodicity = []\n", "\n", "for start_time in range(0, int(total_duration), chunk_duration):\n", " end_time = min(start_time + chunk_duration, total_duration)\n", "\n", " with tempfile.NamedTemporaryFile(suffix=\".wav\") as f:\n", " stem.get_slice(start_time, end_time).write_wav(f.name)\n", " chunk_results, periodicity = torchcrepe.predict_from_file(\n", " f.name,\n", " device=\"cuda:3\",\n", " hop_length=240,\n", " decoder=torchcrepe.decode.weighted_argmax,\n", " pad=False,\n", " return_periodicity=True,\n", " )\n", " all_results.append(chunk_results)\n", " all_results_periodicity.append(periodicity)\n", "\n", "# Concatenate results\n", "pitch_data = torch.cat([r[0] for r in all_results], dim=0).unsqueeze(0)\n", "periodicity = torch.cat([r[0] for r in all_results_periodicity], dim=0).unsqueeze(0)\n", "print(pitch_data)\n", "print(periodicity)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "win_length = 3\n", "\n", "# Median filter noisy confidence value\n", "# periodicity = torchcrepe.filter.median(periodicity, win_length)\n", "\n", "# Remove inharmonic regions\n", "# pitch_data = torchcrepe.threshold.At(.05)(pitch_data, periodicity)\n", "# pitch_data = pitch_data.nan_to_num(nan=20)\n", "\n", "# Optionally smooth pitch to remove quantization artifacts\n", "pitch_data = torchcrepe.filter.mean(pitch_data, win_length)\n", "results = pitch_data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "import librosa\n", "\n", "# Convert frequencies to MIDI note numbers (pitches)\n", "# frequencies = pitch_pesto #torchcrepe.filter.median(results, 30)[0]\n", "frequencies = pitch_pesto # pitch_data[0]\n", "\n", "\n", "# zero out sections that are silent\n", "hz = 100\n", "for i in range(int(stem.duration_s)):\n", " if stem.get_slice(i, i + 1).loudness < -35:\n", " frequencies[i * 100 : (i + 1) * 100] = 20\n", " # elif confidence_pesto[i * 100 : (i + 1) * 100].mean() < 0.:\n", " # frequencies[i * 100 : (i + 1) * 100] = 20\n", "\n", "pitches = librosa.hz_to_midi(frequencies)\n", "\n", "# Create time axis\n", "time_axis = [t / 100 for t in range(len(frequencies))]\n", "\n", "plt.figure(figsize=(12, 6))\n", "plt.plot(time_axis, pitches, label=\"Vocal F0\")\n", "\n", "# filter out notes that are silent\n", "for i in reversed(range(len(midi.pmidi.instruments[0].notes))):\n", " audio_slice = stem.get_slice(\n", " midi.pmidi.instruments[0].notes[i].start,\n", " midi.pmidi.instruments[0].notes[i].end + 1,\n", " )\n", " if np.mean(audio_slice.array_float**2) < 1e-4:\n", " midi.pmidi.instruments[0].notes.pop(i)\n", "\n", "# Overlay MIDI notes\n", "for note in midi.pmidi.instruments[0].notes:\n", " start_time = note.start\n", " end_time = note.end\n", " pitch = note.pitch\n", " plt.hlines(\n", " pitch,\n", " start_time,\n", " end_time,\n", " colors=\"red\",\n", " linewidth=2,\n", " alpha=0.7,\n", " label=\"MIDI Notes\" if note == midi.pmidi.instruments[0].notes[0] else \"\",\n", " )\n", "\n", "plt.xlabel(\"Time (s)\")\n", "plt.ylabel(\"MIDI Note Number\")\n", "plt.legend()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# viterbi octave correction\n", "Ks = range(-3, +4) # allow shifts of ±2 octaves\n", "N = len(midi.pmidi.instruments[0].notes)\n", "λ = 500.0 # tune this to discourage frequent jumps\n", "\n", "# dp[i][k] = min cost up to note i ending with shift k\n", "dp = np.full((N, len(Ks)), np.inf)\n", "prev = np.zeros_like(dp, dtype=int)\n", "note_estimates = []\n", "for note in midi.pmidi.instruments[0].notes:\n", " pitches = frequencies[int(note.start * 100) : int(note.end * 100)]\n", " # filter out pitches that are silent\n", " pitches = pitches[pitches > 20]\n", " median_pitch = np.median(pitches)\n", " note_estimates.append(librosa.hz_to_midi(median_pitch))\n", "\n", "notes = midi.pmidi.instruments[0].notes\n", "print(note_estimates)\n", "# init\n", "for ik, k in enumerate(Ks):\n", " if np.isnan(note_estimates[0]):\n", " dp[0, ik] = 0\n", " else:\n", " dp[0, ik] = (note_estimates[0] - (notes[0].pitch + 12 * k)) ** 2\n", "\n", "# fill\n", "for i in range(1, N):\n", " for ik, k in enumerate(Ks):\n", " if np.isnan(note_estimates[i]):\n", " obs = 0\n", " else:\n", " obs = (note_estimates[i] - (notes[i].pitch + 12 * k)) ** 2\n", " # find best predecessor\n", " costs = dp[i - 1, :] + [λ * abs(k - kp) for kp in Ks]\n", " dp[i, ik] = obs + np.min(costs)\n", " prev[i, ik] = np.argmin(costs)\n", "\n", "# backtrack\n", "best_path = np.zeros(N, dtype=int)\n", "best_path[-1] = np.argmin(dp[-1, :])\n", "for i in range(N - 2, -1, -1):\n", " best_path[i] = prev[i + 1, best_path[i + 1]]\n", "\n", "print(best_path)\n", "\n", "new_midi = copy.deepcopy(midi)\n", "# apply shifts\n", "for i, note in enumerate(new_midi.pmidi.instruments[0].notes):\n", " note.pitch += 12 * Ks[best_path[i]]\n", "new_midi.make_stereo_comparison(stem).play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "import librosa\n", "\n", "# Convert frequencies to MIDI note numbers (pitches)\n", "# frequencies = torchcrepe.filter.median(results, 30)[0]\n", "frequencies = pitch_pesto # pitch_data[0]\n", "\n", "# zero out sections that are silent\n", "hz = 100\n", "for i in range(int(stem.duration_s)):\n", " if stem.get_slice(i, i + 1).loudness < -35:\n", " frequencies[i * 100 : (i + 1) * 100] = 20\n", "\n", "\n", "pitches = librosa.hz_to_midi(frequencies)\n", "\n", "# Create time axis\n", "time_axis = [t / 100 for t in range(len(frequencies))]\n", "\n", "plt.figure(figsize=(12, 6))\n", "plt.plot(time_axis, pitches, label=\"Vocal F0\")\n", "\n", "# filter out notes that are silent\n", "for i in reversed(range(len(new_midi.pmidi.instruments[0].notes))):\n", " audio_slice = stem.get_slice(\n", " midi.pmidi.instruments[0].notes[i].start,\n", " midi.pmidi.instruments[0].notes[i].end + 1,\n", " )\n", " if np.mean(audio_slice.array_float**2) < 1e-6:\n", " midi.pmidi.instruments[0].notes.pop(i)\n", "\n", "# Overlay MIDI notes\n", "for note in new_midi.pmidi.instruments[0].notes:\n", " start_time = note.start\n", " end_time = note.end\n", " pitch = note.pitch\n", " plt.hlines(\n", " pitch,\n", " start_time,\n", " end_time,\n", " colors=\"red\",\n", " linewidth=2,\n", " alpha=0.7,\n", " label=\"MIDI Notes\" if note == new_midi.pmidi.instruments[0].notes[0] else \"\",\n", " )\n", "\n", "plt.xlabel(\"Time (s)\")\n", "plt.ylabel(\"MIDI Note Number\")\n", "plt.legend()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "midi_cqt, _ = compute_cqt(midi_audio)\n", "stem_cqt, _ = compute_cqt(stem)\n", "\n", "print(\"MIDI CQT\")\n", "plot_cqt(midi_cqt)\n", "print(\"STEM CQT\")\n", "plot_cqt(stem_cqt)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import librosa\n", "\n", "stem_f0, voiced_flag_stem, voiced_probs_stem = librosa.pyin(\n", " stem.mono().array_float,\n", " sr=stem.sample_rate,\n", " fmin=librosa.note_to_hz(\"C2\"),\n", " fmax=librosa.note_to_hz(\"C7\"),\n", ")\n", "\n", "print(f\"Stem f0 shape: {stem_f0.shape}\")\n", "\n", "# Plot the fundamental frequency (f0) for both MIDI and stem\n", "plt.figure(figsize=(12, 8))\n", "\n", "# Create time axes\n", "stem_times = librosa.frames_to_time(range(len(stem_f0)), sr=stem.sample_rate)\n", "\n", "plt.subplot(2, 1, 2)\n", "plt.plot(stem_times, stem_f0, \"r-\", alpha=0.7, label=\"Stem f0\")\n", "plt.xlabel(\"Time (s)\")\n", "plt.ylabel(\"Frequency (Hz)\")\n", "plt.title(\"Stem Fundamental Frequency\")\n", "plt.grid(True, alpha=0.3)\n", "plt.legend()\n", "\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# print(midi_audio.array_float.shape[0] / midi_f0.shape[0])\n", "# print(stem.array_float.shape[0] / stem_f0.shape[0])\n", "\n", "# for each note, find the average pitch of the notes in the stem\n", "for instrument in midi.pmidi.instruments:\n", " for note in instrument.notes:\n", " buf_s = 0.2\n", " start_frame = int((note.start - buf_s) * FS / 512)\n", " end_frame = int((note.end + buf_s) * FS / 512)\n", " note_pitch = np.nanmean(stem_f0[start_frame:end_frame])\n", " # convert to midi pitch\n", " midi_pitch = librosa.hz_to_midi(note_pitch)\n", " if np.isnan(midi_pitch):\n", " continue\n", " diff = midi_pitch - note.pitch\n", " # round diff to 12\n", " octave_diff = round(diff / 12) * 12\n", " print(\n", " f\"Time: {note.start}, Stem pitch: {midi_pitch}, MIDI pitch: {note.pitch}, diff: {diff}, octave diff: {octave_diff}\"\n", " )" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# examine some random pairs\n", "\n", "random_pairs = random.sample(stem_pairs_filtered, 3)\n", "for pair, files in random_pairs:\n", " print(pair)\n", " midi = pair.load_midi()\n", " stem = Audio.from_file(files)\n", " audio_cqt, audio_times = compute_cqt(stem)\n", "\n", " # when the audio is silent, remove the notes from the midi\n", " # find silent frames\n", " silent_frames = np.max(audio_cqt, axis=1) < -0.8\n", " print(f\"Silent frames: {silent_frames.shape}, {silent_frames.mean()}\")\n", " if silent_frames.mean() > 0.7:\n", " print(\"Skipping pair because audio is mostly silent\")\n", " continue\n", " # find when midi is silent, zero out the audio\n", " audio_arr = stem.array_float\n", " audio_mask = np.zeros_like(audio_arr)\n", " for instrument in midi.pmidi.instruments:\n", " for note in instrument.notes:\n", " buf_s = 1\n", " start_frame = int((note.start - buf_s) * stem.sample_rate)\n", " end_frame = int((note.end + buf_s) * stem.sample_rate)\n", " audio_mask[start_frame:end_frame] = 1\n", " audio_arr *= audio_mask\n", "\n", " print(f\"Audio arr: {audio_arr.shape}, {audio_arr.dtype}\")\n", " stem = Audio.from_array_float(audio_arr, sample_rate=stem.sample_rate)\n", "\n", " def note_is_silent(note):\n", " start, end = note.start, note.end\n", " start_frame = int(start * FS / HOP_LENGTH)\n", " end_frame = int(end * FS / HOP_LENGTH)\n", " frames = silent_frames[start_frame:end_frame]\n", " if len(frames) == 0:\n", " return True\n", " return frames.mean() > 0.8\n", "\n", " # remove notes that overlap with >80% silence\n", " for instrument in midi.pmidi.instruments:\n", " instrument.notes = [\n", " note for note in instrument.notes if not note_is_silent(note)\n", " ]\n", "\n", " # now fix octave errors\n", " stem_f0, voiced_flag_stem, voiced_probs_stem = librosa.pyin(\n", " stem.mono().array_float,\n", " sr=stem.sample_rate,\n", " fmin=librosa.note_to_hz(\"C2\"),\n", " fmax=librosa.note_to_hz(\"C7\"),\n", " )\n", "\n", " for instrument in midi.pmidi.instruments:\n", " for note in instrument.notes:\n", " buf_s = 0.2\n", " start_frame = int((note.start - buf_s) * FS / HOP_LENGTH)\n", " end_frame = int((note.end + buf_s) * FS / HOP_LENGTH)\n", " note_pitch = np.nanmean(stem_f0[start_frame:end_frame])\n", " midi_pitch = librosa.hz_to_midi(note_pitch)\n", " diff = midi_pitch - note.pitch\n", " if not np.isnan(diff):\n", " # round diff to 12\n", " octave_diff = round(diff / 12) * 12\n", " print(\n", " f\"Time: {note.start}, Stem pitch: {midi_pitch}, MIDI pitch: {note.pitch}, diff: {diff}, octave diff: {octave_diff}\"\n", " )\n", " note.pitch += octave_diff\n", "\n", " midi.make_stereo_comparison(stem).play()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_clean", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" } }, "nbformat": 4, "nbformat_minor": 2 }