{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "import torch\n", "import json\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "from tqdm import tqdm\n", "from suno_utils.utils.text import read_jsonl, write_jsonl\n", "\n", "METAS_DIR = \"/app/suno/tmp\"" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# fundctions for lyrics de-duplication\n", "\n", "import re\n", "from collections import defaultdict\n", "from tqdm import tqdm\n", "\n", "def clean_text(text):\n", " \"\"\"Clean text by removing punctuation and extra spaces.\"\"\"\n", " text = re.sub(r'[^\\w\\s]', '', text.lower())\n", " return ' '.join(text.split())\n", "\n", "def get_text_hash(text):\n", " \"\"\"Create a simple hash from text by taking first and last words plus length.\"\"\"\n", " words = clean_text(text).split()\n", " if len(words) < 2:\n", " return words[0] if words else ''\n", " return f\"{words[0]}_{words[-1]}_{len(words)}\"\n", "\n", "def find_similar_lyrics(lyrics_list):\n", " \"\"\"Find similar lyrics using a simple hashing approach.\"\"\"\n", " print(f\"Processing {len(lyrics_list)} lyrics...\")\n", " \n", " # Group lyrics by hash\n", " hash_groups = defaultdict(list)\n", " \n", " # Process each lyric with progress bar\n", " for idx, lyric in tqdm(enumerate(lyrics_list), total=len(lyrics_list)):\n", " text_hash = get_text_hash(lyric)\n", " hash_groups[text_hash].append((idx, lyric))\n", " \n", " # Filter to only groups with multiple entries\n", " similar_groups = [group for group in hash_groups.values() if len(group) > 1]\n", " \n", " print(f\"\\nFound {len(similar_groups)} groups of similar lyrics\")\n", " return similar_groups" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# load all the base metas that we want to filter from pre-training \n", "meta_info_map = {\n", " \"discogs_subset\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"clean_discogs_subset_v0_metas.jsonl\"))},\n", " \"genius\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"clean_genius_v0_metas.jsonl\"))},\n", " #\"pond5\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"clean_pond5_v0_metas.jsonl\"))},\n", " \"imslp\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"clean_imslp_v0_metas.jsonl\"))},\n", " \"deezer\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"clean_deezer_v0_metas.jsonl\"))},\n", "}\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "first_deezer_id = list(meta_info_map[\"deezer\"].keys())[5]\n", "print(meta_info_map[\"deezer\"][first_deezer_id].keys())\n", "\n", "# lets count how many have alignments \n", "count = 0\n", "for meta in meta_info_map[\"deezer\"].values():\n", " if \"alignments\" in meta:\n", " count += 1\n", "\n", "print(count)\n", "\n", "\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "raw_meta_info_map = {\n", " \"discogs_subset\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"raw_discogs_subset_metas.jsonl\"))},\n", " \"genius\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"raw_genius_metas.jsonl\"))},\n", " #\"pond5\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"raw_pond5_metas.jsonl\"))},\n", " \"imslp\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"raw_imslp_metas.jsonl\"))},\n", " \"deezer\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"raw_deezer_metas.jsonl\"))},\n", "}\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# now load the raw metas for each subset and merge in the view count\n", "for subset in [\"genius\", \"discogs_subset\", \"deezer\"]:\n", " raw_metas = raw_meta_info_map[subset]\n", " for meta_id, meta in tqdm(raw_metas.items()):\n", " meta_info_map[subset][meta_id][\"views\"] = meta[\"views\"]\n", "\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "# load some additional metadata to filter on \n", "discogs_grammy_metas = {m[\"id\"]: m for m in read_jsonl(\"/home/christian/code/christian/metadata/popularity/discogs_subset_grammy_metas.jsonl\")}\n", "discogs_chart_metas = {m[\"id\"]: m for m in read_jsonl(\"/home/christian/code/christian/metadata/popularity/discogs_subset_chart_metas.jsonl\")}\n", "\n", "discogs_audio_features = pd.read_csv(\"/home/christian/code/christian/metadata/v4/discogs_subset_audio_production_features_v2.csv\")\n", "imslp_audio_features = pd.read_csv(\"/home/christian/code/christian/metadata/v4/imslp_audio_production_features_v2.csv\")\n", "genius_audio_features = pd.read_csv(\"/home/christian/code/christian/metadata/v4/genius_audio_production_features_v2.csv\")" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "# load the artist sampled tracks \n", "with open(\"/home/christian/code/christian/metadata/sampled_tracks.json\", \"r\") as fp:\n", " sampled_tracks = json.load(fp)\n", "\n", "\n", "for key in sampled_tracks.keys():\n", " sampled_tracks[key] = set(sampled_tracks[key])\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# load the reliable cover ids\n", "with open(\"/home/christian/code/christian/metadata/v4/reliable_cover_ids.txt\", \"r\") as f:\n", " reliable_cover_ids = f.readlines()\n", "\n", "print(len(reliable_cover_ids))\n", "\n" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "# old genius id jsonl\n", "old_genius_metas = read_jsonl(\"/home/christian/code/christian/metadata/genius_hq_metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "genius_id_map = {meta[\"original_id\"]: meta[\"id\"] for meta in old_genius_metas}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# load some alignemnts info (h5 alignments)\n", "genius_alignments_filepath = (\n", " \"/home/tony/Work/tony/hoot/tmp/genius_hq_alignments_h5_t480_v1.jsonl\"\n", ")\n", "discogs_alignments_filepath = (\n", " \"/home/tony/Work/tony/hoot/tmp/discogs_hq_alignments_h5_t480_v1.jsonl\"\n", ")\n", "deezer_alignments_filepath = (\n", " \"/home/tony/Work/tony/hoot/tmp/deezer_hq_alignments_h5_t480_v1.jsonl\"\n", ")\n", "\n", "genius_alignments = read_jsonl(genius_alignments_filepath, progress=False)\n", "print(len(genius_alignments))\n", "discogs_alignments = read_jsonl(discogs_alignments_filepath, progress=False)\n", "print(len(discogs_alignments))\n", "deezer_alignments = read_jsonl(deezer_alignments_filepath, progress=False)\n", "print(len(deezer_alignments))\n", "\n", "# make alignments into a dictionary \n", "genius_alignments_map = {k: {\"texts\": v, \"cer\": cer} for k, v, cer in genius_alignments}\n", "discogs_alignments_map = {k: {\"texts\": v, \"cer\": cer} for k, v, cer in discogs_alignments}\n", "deezer_alignments_map = {k: {\"texts\": v, \"cer\": cer} for k, v, cer in deezer_alignments}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "# import counter\n", "from collections import Counter\n", "\n", "counter = Counter()\n", "cer_by_lang = {}\n", "\n", "# then merge into the raw metas map\n", "for subset in [\"discogs_subset\", \"genius\", \"deezer\"]:\n", " raw_metas = meta_info_map[subset]\n", " counter[subset] = 0\n", " for meta_id, meta in tqdm(raw_metas.items()):\n", " if subset == \"discogs_subset\":\n", " alignment_info = discogs_alignments_map.get(meta_id, None)\n", " if alignment_info is not None:\n", " meta[\"alignments\"] = alignment_info[\"texts\"]\n", " meta[\"cer\"] = alignment_info[\"cer\"]\n", " counter[subset] += 1\n", " text_lang = meta.get(\"lang\", None)\n", " if text_lang is not None:\n", " if text_lang not in cer_by_lang:\n", " cer_by_lang[text_lang] = []\n", " cer_by_lang[text_lang].append(meta[\"cer\"])\n", " elif subset == \"deezer\":\n", " alignment_info = deezer_alignments_map.get(meta_id, None)\n", " if alignment_info is not None:\n", " meta[\"alignments\"] = alignment_info[\"texts\"]\n", " meta[\"cer\"] = alignment_info[\"cer\"]\n", " counter[subset] += 1\n", " text_lang = meta.get(\"lang\", None)\n", " if text_lang is not None:\n", " if text_lang not in cer_by_lang:\n", " cer_by_lang[text_lang] = []\n", " cer_by_lang[text_lang].append(meta[\"cer\"])\n", " elif subset == \"genius\":\n", " # map the meta_id to the new meta_id\n", " tmp_id = genius_id_map.get(meta_id, None)\n", " alignment_info = genius_alignments_map.get(tmp_id, None)\n", " if alignment_info is not None:\n", " meta[\"alignments\"] = alignment_info[\"texts\"]\n", " meta[\"cer\"] = alignment_info[\"cer\"]\n", " counter[subset] += 1\n", " text_lang = meta.get(\"lang\", None)\n", " if text_lang is not None:\n", " if text_lang not in cer_by_lang:\n", " cer_by_lang[text_lang] = []\n", " cer_by_lang[text_lang].append(meta[\"cer\"])\n", " else:\n", " raise ValueError(f\"Unknown subset: {subset}\")\n", "\n", "for subset in counter:\n", " print(f\"{subset}: {counter[subset]}\")\n", "\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "cer_cutoff_by_lang = {}\n", "\n", "for lang, cers in cer_by_lang.items():\n", " print(f\"{lang}: {np.mean(cers)} {np.percentile(cers, 90)}\")\n", " cer_cutoff_by_lang[lang] = np.percentile(cers, 90)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "# put audio features into a dictionary \n", "audio_features_map = {\n", " \"discogs_subset\": {m[\"id\"]: m for m in discogs_audio_features.to_dict(orient=\"records\")},\n", " \"imslp\": {m[\"id\"]: m for m in imslp_audio_features.to_dict(orient=\"records\")},\n", " \"genius\": {m[\"id\"]: m for m in genius_audio_features.to_dict(orient=\"records\")},\n", "}\n", "\n", "feature_bounds = {\n", " \"loudness\": [-32, -4],\n", " \"spectral_centroid\": [1750, 5000],\n", " \"spectral_flatness\": [0.02, 0.3],\n", " \"crest_factor\": [1.0, 3],\n", " #\"bass\" : [0.1, 0.5],\n", " #\"mid\" : [0.4, 1.0],\n", " # \"high\" : [0.15, 1.25],\n", " \"stereo_width\" : [0.1, 0.4]\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# summary of how we cut the data\n", "# remove songs without lyrics if they are in discogs or genius\n", "# remove songs with less than 10 characters\n", "# remove songs with CER higher than 90% for the associated language\n", "# remove songs with no tags\n", "# remove songs with urls in the lyrics\n", "# remove songs with duration less than 60 seconds\n", "# remove songs with out of bounds audio features\n", "\n", "# side note: after we get a final list of ids here we can try to improve\n", "# the annotations we have via gpt only on those " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "results = {}\n", "\n", "for idx, (subset_name, subset_metas) in enumerate(meta_info_map.items()):\n", " if subset_name not in results:\n", " results[subset_name] = []\n", "\n", " print(f\"subset: {subset_name} {len(subset_metas)}\")\n", " pbar = tqdm(subset_metas.values())\n", " for meta in pbar:\n", " # this is global filter \n", " if meta.get(\"duration_s\", 0) < 60:\n", " continue\n", "\n", " # check audio features\n", " if subset_name in audio_features_map:\n", " audio_features = audio_features_map[subset_name].get(meta[\"id\"], None)\n", " if audio_features is not None:\n", " pass_audio_filter = True\n", " for feature in feature_bounds:\n", " if audio_features[feature] < feature_bounds[feature][0] or audio_features[feature] > feature_bounds[feature][1]:\n", " pass_audio_filter = False\n", " break\n", " if not pass_audio_filter:\n", " continue\n", "\n", " # we will have separate filters for each dataset \n", " if subset_name in [\"discogs_subset\", \"genius\"]:\n", " meta_id = meta[\"id\"]\n", " # check if the song is in the sampled tracks\n", " # but only for english lang\n", " text_lang = meta.get(\"lang\", None)\n", " if text_lang is not None and text_lang != \"en\": \n", " if meta_id not in sampled_tracks[subset_name]:\n", " continue\n", " if meta.get(\"text\", \"\").__len__() < 10:\n", " continue\n", " if meta.get(\"text\", \"\").__len__() > 10_000:\n", " continue\n", " if meta.get(\"tags\", []).__len__() < 3:\n", " continue\n", " if \"http\" in meta.get(\"text\", \"\"):\n", " continue\n", " tags = meta.get(\"tags\", [])\n", " for tag in tags:\n", " if \"http\" in tag:\n", " continue\n", " # get alignment info\n", " tmp_meta = meta_info_map[subset_name].get(meta[\"id\"], None)\n", " \n", " if tmp_meta is None:\n", " continue\n", " else:\n", " text_lang = tmp_meta.get(\"lang\", None)\n", " if text_lang is None:\n", " continue\n", " if \"cer\" in tmp_meta:\n", " if tmp_meta[\"cer\"] > cer_cutoff_by_lang[text_lang]:\n", " continue\n", " \n", " elif subset_name == \"deezer\":\n", " # deezer doesn't have any tags\n", " alignments = meta.get(\"alignments\", [])\n", " # check the cer\n", " cer = meta.get(\"cer\", None)\n", " # get the lang\n", " text_lang = meta.get(\"lang\", None)\n", " # get the views\n", " views = meta.get(\"views\", None)\n", " \n", " if len(alignments) == 0:\n", " continue\n", " if views < 50_000:\n", " continue\n", " if text_lang is None:\n", " continue\n", " if cer > cer_cutoff_by_lang[text_lang]:\n", " continue\n", "\n", " \n", " elif subset_name in [\"pond5\", \"imslp\"]:\n", " tags = meta.get(\"tags\", [])\n", " if len(tags) < 3:\n", " continue\n", " bad_tags = {\"http\", \"uplifting\", \"inspirational\", \"positive\", \"energetic\",\n", " \"happy\", \"motivational\", \"corporate\", \"business\", \"advertising\", \n", " \"fun\", \"background\", \"hopeful\", \"optimistic\", \"bright\", \"reflective\"}\n", " if any(bad_tag.lower() in tag.lower() for tag in tags for bad_tag in bad_tags):\n", " continue\n", "\n", " results[subset_name].append(meta[\"id\"])\n", " #pbar.set_postfix({\"subset\": subset_name, \"n_results\": len(results[subset_name])})\n", "\n", " percent_kept = len(results[subset_name]) / len(subset_metas) * 100\n", " hours_kept = sum(meta_info_map[subset_name][id][\"duration_s\"] for id in results[subset_name]) / 3600\n", " print(f\"subset {subset_name}: {len(results[subset_name])} ({percent_kept:.2f}%) {hours_kept:.2f} hrs\")\n", " print(\"--------------------------------\")\n", "\n" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "# save this set of ids out to disk\n", "output_filepath = \"/home/christian/code/christian/metadata/v45_splits/ids_keep_sets_v9.json\"\n", "with open(output_filepath, \"w\") as f:\n", " json.dump(results, f)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# audit the final split based on tags, lyrics, and language\n", "\n", "tag_count = {}\n", "\n", "for subset_name in results:\n", " print(f\"subset: {subset_name}\")\n", " for meta_id in tqdm(results[subset_name]):\n", " meta = meta_info_map[subset_name][meta_id]\n", " text_lang = meta.get(\"lang\", None)\n", " text = meta.get(\"text\", None)\n", " tags = meta.get(\"tags\", None)\n", " if tags is None:\n", " continue\n", " for tag in tags:\n", " if tag not in tag_count:\n", " tag_count[tag] = 0\n", " tag_count[tag] += 1\n", "\n", "\n", "#sorted_tag_count = sorted(tag_count.items(), key=lambda x: x[1], reverse=True)\n", "\n", "\n", "#for idx, (tag, count) in enumerate(sorted_tag_count[:100]):\n", "# print(f\"{idx}: {tag}: {count}\")\n", "\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# first lets audit the tags and lyrics metadata that we have \n", "\n", "results = {}\n", "\n", "for idx, (subset_name, subset_metas) in enumerate(meta_info_map.items()):\n", "\n", " #if subset_name != \"imslp\":\n", " # continue\n", "\n", " results[subset_name] = {\n", " \"tags_count\": [],\n", " \"lyrics_count\": [],\n", " \"n_lyrics_chars\": [],\n", " \"ids_keep\": [],\n", " \"lyrics_ratio\": [],\n", " \"duration_s\": [],\n", " \"ids_failed_audio_filter\": [],\n", " \"ids_failed_url_filter\": [],\n", " \"instrumentals\": [],\n", " \"views\" : [],\n", " \"duration_s_keep\": []\n", " }\n", " print(f\"subset: {subset_name} {len(subset_metas)}\")\n", "\n", " #if subset_name != \"deezer\":\n", " # continue\n", "\n", " for meta in subset_metas.values():\n", " has_tags = False\n", " atleast_60s = False\n", " lyrics_ratio = 0\n", " duration_s = meta.get(\"duration_s\", 0)\n", " results[subset_name][\"duration_s\"].append(duration_s)\n", " if duration_s < 60:\n", " continue\n", "\n", " if subset_name == \"pond5\" or subset_name == \"imslp\":\n", " views = 1_000_000 # this will pass the filter \n", " elif \"views\" in meta:\n", " views = meta[\"views\"]\n", " else:\n", " views = 0\n", "\n", " results[subset_name][\"views\"].append(views)\n", "\n", " if \"tags\" in meta:\n", " # count the number of unique tags \n", " unique_tags = set(meta[\"tags\"])\n", " results[subset_name][\"tags_count\"].append(len(unique_tags))\n", " has_tags = True\n", " if \"text\" in meta:\n", " # count the number of unique lyrics \n", " results[subset_name][\"lyrics_count\"].append(True)\n", " has_lyrics = True\n", " # check if the lyrics are \"Instrumental\"\n", " if \"Instrumental\" in meta[\"text\"] and len(meta[\"text\"]) < 50:\n", " # for this case we can ignore the lyrics ratio \n", " lyrics_ratio = 10 # this will pass the filter \n", " n_lyric_chars = 101\n", " # instrumental count\n", " results[subset_name][\"instrumentals\"].append(True)\n", " elif subset_name == \"pond5\" or subset_name == \"imslp\" or subset_name == \"deezer\":\n", " # for these datasets we can ignore the lyrics ratio \n", " lyrics_ratio = 10 # this will pass the filter \n", " n_lyric_chars = 101\n", " # instrumental count\n", " results[subset_name][\"instrumentals\"].append(True)\n", " else:\n", " n_lyric_chars = len(meta[\"text\"])\n", " lyrics_ratio = n_lyric_chars / meta[\"duration_s\"]\n", "\n", " # check for URL in lyrics \n", " pass_url_filter = True\n", " if \"http\" in meta[\"text\"]:\n", " pass_url_filter = False\n", " results[subset_name][\"ids_failed_url_filter\"].append(meta[\"id\"])\n", "\n", " results[subset_name][\"n_lyrics_chars\"].append(n_lyric_chars)\n", " results[subset_name][\"lyrics_ratio\"].append(lyrics_ratio)\n", "\n", " # get features and check if they pass audio filter\n", " pass_audio_filter = True\n", " if subset_name in audio_features_map:\n", " audio_features = audio_features_map[subset_name].get(meta[\"id\"], None)\n", " if audio_features is not None:\n", " # Check all features against their bounds\n", " for feature, (min_bound, max_bound) in feature_bounds.items():\n", " # Skip if feature doesn't exist in audio_features\n", " if feature not in audio_features:\n", " continue\n", " \n", " # Check if the feature is out of bounds\n", " if audio_features[feature] < min_bound or audio_features[feature] > max_bound:\n", " #print(f\"feature {feature} out of bounds: {audio_features[feature]}, {min_bound}, {max_bound}\")\n", " pass_audio_filter = False\n", " results[subset_name][\"ids_failed_audio_filter\"].append(meta[\"id\"])\n", " break\n", "\n", " if subset_name == \"imslp\":\n", " has_tags = True\n", " has_lyrics = True\n", " n_lyric_chars = 1000\n", " lyrics_ratio = 10\n", "\n", " if subset_name == \"deezer\":\n", " if \"text_lines\" in meta:\n", " has_lyrics = True\n", " has_tags = True\n", " n_lyric_chars = 1000\n", " lyrics_ratio = 10\n", " pass_audio_filter = True\n", " #views = 1_000_000\n", "\n", "\n", " #print(f\"has_tags: {has_tags} has_lyrics: {has_lyrics} n_lyric_chars: {n_lyric_chars} lyrics_ratio: {lyrics_ratio} views: {views} pass_audio_filter: {pass_audio_filter} pass_url_filter: {pass_url_filter}\")\n", " #print(meta['text_lines'])\n", "\n", " if (has_tags and \n", " has_lyrics and \n", " n_lyric_chars > 100 and\n", " n_lyric_chars < 5000 and \n", " lyrics_ratio > 2 and \n", " lyrics_ratio < 14 and\n", " views > 250_000 and\n", " pass_audio_filter and\n", " pass_url_filter):\n", " results[subset_name][\"ids_keep\"].append(meta[\"id\"])\n", " results[subset_name][\"duration_s_keep\"].append(duration_s)\n", "\n", " n_songs_with_tags = len(results[subset_name][\"tags_count\"])\n", " n_songs_with_lyrics = len(results[subset_name][\"lyrics_count\"])\n", " n_songs_with_no_lyrics = len([l for l in results[subset_name][\"lyrics_count\"] if l == False])\n", " n_songs_with_tags = len(results[subset_name][\"tags_count\"])\n", " n_songs_with_no_tags = n_songs_with_tags - len(results[subset_name][\"ids_keep\"])\n", " n_songs_with_tags_and_lyrics = len(results[subset_name][\"ids_keep\"])\n", " n_songs_failed_audio_filter = len(results[subset_name][\"ids_failed_audio_filter\"])\n", " n_songs_instrumentals = len(results[subset_name][\"instrumentals\"])\n", " n_songs_failed_views_filter = len([v for v in results[subset_name][\"views\"] if v < 400_000])\n", " total_duration_hrs = sum(results[subset_name][\"duration_s_keep\"]) / 3600\n", " total_songs = len(subset_metas)\n", " total_ids_keep = len(results[subset_name][\"ids_keep\"])\n", " keep_percent = (total_ids_keep / total_songs) * 100\n", " print(f\"subset {subset_name}: before {total_songs} after {total_ids_keep} ({keep_percent:.2f}%)\")\n", " print(f\"subset {subset_name}: n_songs_failed_views_filter {n_songs_failed_views_filter}\")\n", " print(f\"subset {subset_name}: n_songs_with_no_lyrics {n_songs_with_no_lyrics}\")\n", " print(f\"subset {subset_name}: n_songs_with_no_tags {n_songs_with_no_tags}\")\n", " print(f\"subset {subset_name}: n_songs_failed_audio_filter {n_songs_failed_audio_filter}\")\n", " print(f\"subset {subset_name}: n_songs_instrumentals {n_songs_instrumentals}\")\n", " print(f\"subset {subset_name}: ids_failed_url_filter {len(results[subset_name]['ids_failed_url_filter'])}\")\n", " print(f\"subset {subset_name}: n_songs_failed_views_filter {n_songs_failed_views_filter}\")\n", " print(f\"subset {subset_name}: total duration {total_duration_hrs:.2f} hrs\")\n", " print()\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "ids_keep = results[subset_name][\"ids_keep\"]\n", "for idx in ids_keep:\n", " print(idx)\n", " break\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# as a final step, de-duplicate the lyrics \n", "for subset_name in results:\n", " print(f\"subset: {subset_name}\")\n", " lyrics_list = [meta_info_map[subset_name][id][\"text\"] for id in results[subset_name][\"ids_keep\"]]\n", " ids_keep = results[subset_name][\"ids_keep\"].copy()\n", " similar_groups = find_similar_lyrics(lyrics_list)\n", " print(len(similar_groups))\n", "\n", " # iterate over all groups and remove duplicates\n", " # keep the first one and remove the rest in each group\n", "\n", " tot_n_removed = 0\n", " for group in tqdm(similar_groups):\n", " if len(group) > 1:\n", " for idx, lyric in group[1:]:\n", " # get the id from the ids_keep list \n", " meta_id = ids_keep[idx]\n", " results[subset_name][\"ids_keep\"].remove(meta_id)\n", " tot_n_removed += 1\n", "\n", " print(f\"total n_removed: {tot_n_removed}\")\n", "\n", " # double check for duplicates \n", " lyrics_list = [meta_info_map[subset_name][id][\"text\"] for id in results[subset_name][\"ids_keep\"]]\n", " similar_groups = find_similar_lyrics(lyrics_list)\n", " print(len(similar_groups))\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(results.keys())" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# convert ids_keep to a set \n", "ids_keep_sets = {}\n", "for subset_name in results:\n", " ids_keep_set = set(results[subset_name])\n", " ids_keep_sets[subset_name] = ids_keep_set\n", " print(f\"subset {subset_name}: ids_keep_set {len(ids_keep_set)}\")\n" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [], "source": [ "# save the ids_keep_sets to disk \n", "with open(\"/home/christian/code/christian/metadata/v45_splits/ids_keep_sets_v8.json\", \"w\") as f:\n", " json.dump(results, f)\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# save the ids_keep_sets to disk \n", "with open(\"/home/christian/code/christian/metadata/v45_splits/ids_keep_sets_v8.json\", \"r\") as f:\n", " results = json.load(f)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# convert ids_keep to a set \n", "ids_keep_sets = {}\n", "for subset_name in results:\n", " ids_keep_set = set(results[subset_name][\"ids_keep\"])\n", " ids_keep_sets[subset_name] = ids_keep_set\n", " print(f\"subset {subset_name}: ids_keep_set {len(ids_keep_set)}\")\n" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [], "source": [ "import polars as pl" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# preload the source metas\n", "source_metas = {}\n", "for subset in [\"val\", \"tr\"]:\n", " # Read the data and convert to dictionaries immediately\n", " df = pl.read_ndjson(f\"/app/suno/data/chirp_v5_ft/v2/metas_{subset}.jsonl\")\n", " source_metas[subset] = df.to_dicts()\n", " print(f\"subset {subset}: {len(source_metas[subset])}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for meta in source_metas[\"val\"]:\n", " if \"dataset\" in meta:\n", " dataset = meta[\"dataset\"]\n", " if \"deezer\" in dataset:\n", " print(meta)\n", " break\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "dataset_names = [meta[\"dataset\"] for meta in source_metas[\"tr\"]]\n", "dataset_names = list(set(dataset_names))\n", "print(dataset_names)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "source_metas_keep_ids.keys()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "ids_keep_sets[\"discogs_subset\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# load the list of covers\n", "reliable_covers_filepath = \"/home/christian/code/christian/metadata/v4/reliable_cover_ids.txt\"\n", "with open(reliable_covers_filepath, \"r\") as f:\n", " reliable_cover_ids = f.readlines()\n", "\n", "reliable_cover_ids = [id.strip() for id in reliable_cover_ids]\n", "\n", "print(len(reliable_cover_ids))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "reliable_cover_ids_set = set(reliable_cover_ids)\n", "subset = \"tr\"\n", "\n", "with open(f\"/app/suno/data/chirp_v5_ft/v2/info_{subset}.json\", \"r\") as f:\n", " original_info = json.load(f)\n", "\n", "# first print number of source covers\n", "n_source_covers = len(original_info[\"covers\"][\"idx_map\"])\n", "print(f\"n_source_covers: {n_source_covers}\")\n", "\n", "# now compute total number of covers by summing over all the idx_map values\n", "n_covers = sum([len(idx_list) for idx_list in original_info[\"covers\"][\"idx_map\"].values()])\n", "print(f\"n_covers: {n_covers}\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "results.keys()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for key, val in original_info.items():\n", " print(key)\n", " idx_list = val.get(\"idx_list\", None)\n", " if idx_list is not None:\n", " print(len(idx_list))\n", " else:\n", " idx_map = val.get(\"idx_map\", None)\n", " if idx_map is not None:\n", " print(len(idx_map))\n", " print()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# save out ids for finetuning\n", "# load the original training/val metas\n", "\n", "for subset in [\"tr\"]:\n", " source_metas_keep_ids = {\n", " #\"covers\" : {\"idx_map\": {}, \"task\": \"covers\"},\n", " \"discogs_subset_lyrics_foreign\" : {\"idx_list\": [], \"task\": \"default\"},\n", " \"discogs_subset_lyrics\" : {\"idx_list\": [], \"task\": \"default\"},\n", " \"discogs_subset\" : {\"idx_list\": [], \"task\": \"default\"},\n", " \"genius\" : {\"idx_list\": [], \"task\": \"default\"},\n", " \"genius_lyrics\" : {\"idx_list\": [], \"task\": \"default\"},\n", " \"genius_lyrics_foreign\" : {\"idx_list\": [], \"task\": \"default\"},\n", " \"imslp\" : {\"idx_list\": [], \"task\": \"default\"},\n", " #\"pond5\" : {\"idx_list\": [], \"task\": \"default\"},\n", " \"deezer_lyrics_aligned\" : {\"idx_list\": [], \"task\": \"default\"},\n", " \"deezer\" : {\"idx_list\": [], \"task\": \"default\"},\n", " \"discogs_lyrics\" : {\"idx_list\": [], \"task\": \"default\"},\n", " \"discogs_lyrics_foreign\" : {\"idx_list\": [], \"task\": \"default\"},\n", " }\n", " source_metas_subset = source_metas[subset]\n", " \n", " with open(f\"/app/suno/data/chirp_v5_ft/v2/info_{subset}.json\", \"r\") as f:\n", " original_info = json.load(f)\n", " \n", " for meta_idx, meta in enumerate(tqdm(source_metas_subset)):\n", " dataset = meta[\"dataset\"]\n", " task = meta[\"task\"]\n", "\n", " if \"discogs_subset\" in dataset:\n", " short_dataset = \"discogs_subset\"\n", " #elif dataset == \"discogs\":\n", " # short_dataset = \"discogs_subset\"\n", " elif \"imslp\" in dataset:\n", " short_dataset = \"imslp\"\n", " elif \"genius\" in dataset:\n", " short_dataset = \"genius\"\n", " elif \"deezer\" in dataset:\n", " short_dataset = \"deezer\"\n", " else:\n", " short_dataset = dataset\n", "\n", " if \"imslp\" in dataset:\n", " # just copy the idx_list from the original info\n", " source_metas_keep_ids[dataset][\"idx_list\"] = original_info[dataset][\"idx_list\"]\n", " continue\n", "\n", " # we need to check if this id is in the results[dataset][\"ids_keep\"]\n", " if short_dataset not in results:\n", " if dataset == \"covers\":\n", " source_metas_keep_ids[dataset] = original_info[dataset]\n", " #print(f\"dataset {short_dataset} not in results\")\n", " continue\n", " # keep everything from datasets that we didn't filter on \n", " else: # we filtered on this dataset \n", " #print(f\"dataset {short_dataset} in results ({len(ids_keep_sets[short_dataset])}\")\n", " if meta[\"id\"] in ids_keep_sets[short_dataset]:\n", " source_metas_keep_ids[dataset][\"idx_list\"].append(meta_idx)\n", "\n", " #for dataset in source_metas_keep_ids:\n", " # print(f\"dataset {dataset}: {len(source_metas_keep_ids[dataset]['idx_list'])}\")\n", "\n", " final_source_metas_keep_ids = {}\n", " for dataset in source_metas_keep_ids:\n", "\n", " # special case for covers \n", " if dataset == \"covers\":\n", " final_source_metas_keep_ids[dataset] = original_info[dataset]\n", " print(f\"dataset {dataset}: {len(final_source_metas_keep_ids[dataset]['idx_map'])}\")\n", "\n", " # print the total number of sub-covers\n", " n_sub_covers = sum([len(idx_list) for idx_list in final_source_metas_keep_ids[dataset]['idx_map'].values()])\n", " print(f\"n_sub_covers: {n_sub_covers}\")\n", "\n", " # now we will filter the idx_map here based on the idx_list\n", " reliable_cover_ids_set = set(reliable_cover_ids)\n", " \n", " for key, idx_list in final_source_metas_keep_ids[dataset][\"idx_map\"].items():\n", " for meta_id in idx_list:\n", " if meta_id not in reliable_cover_ids_set:\n", " final_source_metas_keep_ids[dataset][\"idx_map\"][key].remove(meta_id)\n", "\n", " # check for any idx_map that is empty and remove it\n", " final_source_metas_keep_ids[dataset][\"idx_map\"] = {k: v for k, v in final_source_metas_keep_ids[dataset][\"idx_map\"].items() if len(v) > 0}\n", " print(f\"dataset {dataset}: {len(final_source_metas_keep_ids[dataset]['idx_map'])}\")\n", " # print the total number of sub-covers\n", " n_sub_covers = sum([len(idx_list) for idx_list in final_source_metas_keep_ids[dataset]['idx_map'].values()])\n", " print(f\"n_sub_covers: {n_sub_covers}\")\n", "\n", " else:\n", " idx_list = source_metas_keep_ids[dataset].get(\"idx_list\", None)\n", " if idx_list is None:\n", " # check for idx_map\n", " idx_map = source_metas_keep_ids[dataset].get(\"idx_map\", None)\n", " print(f\"dataset {dataset}: {len(idx_map)}\")\n", " final_source_metas_keep_ids[dataset] = {\n", " \"idx_map\": idx_map,\n", " \"task\": source_metas_keep_ids[dataset].get(\"task\", \"default\")\n", " }\n", " elif idx_list.__len__() == 0:\n", " # remove this dataset from the source_metas_keep_ids\n", " pass\n", " else:\n", " print(f\"dataset {dataset}: {len(idx_list)}\")\n", " final_source_metas_keep_ids[dataset] = {\n", " \"idx_list\": idx_list,\n", " \"task\": source_metas_keep_ids[dataset].get(\"task\", \"default\")\n", " }\n", " \n", "\n", " # write this to disk as a json file\n", " with open(f\"/home/christian/code/christian/metadata/v45_splits/info_{subset}_ft_v11.json\", \"w\") as f:\n", " json.dump(final_source_metas_keep_ids, f)\n" ] }, { "cell_type": "code", "execution_count": 73, "metadata": {}, "outputs": [], "source": [ "# let's save out a json file with the set of metas that we are keeping\n", "with open(f\"/home/christian/code/christian/metadata/v45_splits/info_{subset}_ft_v11_ids.json\", \"w\") as f:\n", " json.dump(results, f)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "results[\"discogs_subset\"][\"ids_keep\"][0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "raw" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "raw_meta_info_map[\"discogs_subset\"][results[\"discogs_subset\"][\"ids_keep\"][6443]]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "meta_info_map[\"discogs_subset\"][results[\"discogs_subset\"][\"ids_keep\"][6443]]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# after we are done, count the number of ids in each dataset\n", "for dataset in final_source_metas_keep_ids:\n", " print(f\"dataset {dataset}: {len(final_source_metas_keep_ids[dataset]['idx_list'])}\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# remove any keys from the dict if the length of idx_list is 0\n", "new_data_v4 = {}\n", "for key, val in final_source_metas_keep_ids.items():\n", " if key == \"pond5\":\n", " continue\n", " if \"idx_list\" in val:\n", " if len(val[\"idx_list\"]) > 0:\n", " new_data_v4[key] = val\n", " else:\n", " new_data_v4[key] = val\n", "\n", "for key, val in new_data_v4.items():\n", " print(key)\n", "\n", "# write this to disk as a json file\n", "with open(f\"/home/christian/code/christian/metadata/v45_splits/info_{subset}_ft_v11_f1.json\", \"w\") as f:\n", " json.dump(final_source_metas_keep_ids, f)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for dataset in source_metas_keep_ids:\n", " idx_list = source_metas_keep_ids[dataset].get(\"idx_list\", [])\n", " if idx_list.__len__() == 0:\n", " # check for idx_map\n", " idx_map = source_metas_keep_ids[dataset].get(\"idx_map\", {})\n", " print(f\"dataset {dataset}: {len(idx_map)}\")\n", " else:\n", " print(f\"dataset {dataset}: {len(idx_list)}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for dataset, val in source_metas_keep_ids.items():\n", " print(dataset, val.keys())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Data analysis " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "fig, axs = plt.subplots(1, 1, figsize=(6, 6), sharex=True, sharey=False)\n", "n_bins = 100\n", "\n", "subset_name = \"discogs_subset\"\n", "\n", "# histogram of tags length distribution \n", "min_len = min(results[subset_name][\"n_lyrics_chars\"])\n", "max_len = max(results[subset_name][\"n_lyrics_chars\"])\n", "median_len = np.median(results[subset_name][\"n_lyrics_chars\"])\n", "print(f\"subset {subset_name}: min {min_len} max {max_len} median {median_len}\")\n", "\n", "# count number of songs with less than 100 characters \n", "n_songs_less_than_100 = len([l for l in results[subset_name][\"n_lyrics_chars\"] if l < 100])\n", "print(f\"subset {subset_name}: n_songs_less_than_100 {n_songs_less_than_100}\")\n", "\n", "# count number of songs with no tags\n", "n_songs_with_no_tags = len([l for l in results[subset_name][\"tags_count\"] if l == 0])\n", "print(f\"subset {subset_name}: n_songs_with_no_tags {n_songs_with_no_tags}\")\n", "\n", "# count number of songs less than 60s\n", "n_songs_less_than_60s = len([l for l in results[subset_name][\"duration_s\"] if l < 60])\n", "print(f\"subset {subset_name}: n_songs_less_than_60s {n_songs_less_than_60s}\")\n", "\n", "# compute irq\n", "#irq = np.percentile(results[subset_name][\"n_lyrics_chars\"], [25, 75])\n", "Q1 = np.percentile(results[subset_name][\"n_lyrics_chars\"], 25)\n", "Q3 = np.percentile(results[subset_name][\"n_lyrics_chars\"], 75)\n", "IQR = Q3 - Q1\n", "\n", "k = 1.5\n", "# Calculate bounds\n", "lower_bound = Q1 - k * IQR\n", "upper_bound = Q3 + k * IQR\n", "print(f\"subset {subset_name}: lower_bound {lower_bound} upper_bound {upper_bound}\")\n", "\n", "# add line to axs\n", "axs.axvline(x=lower_bound, color=\"red\", linestyle=\"--\", linewidth=2, zorder=11)\n", "axs.axvline(x=upper_bound, color=\"red\", linestyle=\"--\", linewidth=2, zorder=11)\n", "\n", "axs.hist(results[subset_name][\"n_lyrics_chars\"], bins=100, density=False, zorder=10)\n", "axs.set_title(f\"{subset_name}\")\n", "axs.set_xlabel(\"Number of Lyrics Characters\")\n", "axs.set_ylabel(\"Frequency\")\n", "axs.set_yscale(\"log\")\n", "axs.grid(c=\"lightgray\", zorder=0)\n", "\n", "plt.tight_layout()\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": 165, "metadata": {}, "outputs": [], "source": [ "# sort discogs subset by number of lyrics characters \n", "subset_name = \"genius\"\n", "sorted_discogs_subset = sorted(meta_info_map[subset_name].values(), key=lambda x: x.get(\"text\", \"\").__len__(), reverse=True)\n", "# remove ones with no lyrics \n", "sorted_discogs_subset = [m for m in sorted_discogs_subset if m.get(\"text\", \"\").__len__() > 0]\n", "# remove ones with no duration \n", "sorted_discogs_subset = [m for m in sorted_discogs_subset if m.get(\"duration_s\", 0) > 0]\n", "\n", "# sort discogs subset by lyrics ratio \n", "#orted_discogs_subset = sorted(sorted_discogs_subset, key=lambda x: len(x.get(\"text\", \"\")) / x.get(\"duration_s\", 0), reverse=True)\n", "\n", "# filter out ones with duration less than 60 seconds \n", "#sorted_discogs_subset = [m for m in sorted_discogs_subset if m.get(\"duration_s\", 0) > 60]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "idx = len(sorted_discogs_subset) - 35_000\n", "print(sorted_discogs_subset[idx])\n", "\n", "print(f\"lyrics chars: {len(sorted_discogs_subset[idx]['text'])}\")\n", "print(f\"duration: {sorted_discogs_subset[idx]['duration_s'] / 60:.2f} min\")\n", "print(f\"ratio: {len(sorted_discogs_subset[idx]['text']) / sorted_discogs_subset[idx]['duration_s']:.2f}\")\n", "print()\n", "print(f\"lyrics: {sorted_discogs_subset[idx]['text']}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# get all ids that failed the url filter \n", "subset_name = \"genius\"\n", "tmp_subset = sorted(meta_info_map[subset_name].values(), key=lambda x: x.get(\"text\", \"\"), reverse=True)\n", "tmp_subset = [m for m in tmp_subset if \"http\" in m.get(\"text\", \"\")]\n", "print(f\"tmp_subset: {len(tmp_subset)}\")\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "tmp_subset[400]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# get a list of all metas that are instrumentals \n", "instrumentals = [m for m in sorted_discogs_subset if \"Instrumental\" in m.get(\"text\", \"\") and m.get(\"text\", \"\").__len__() < 100]\n", "print(f\"instrumentals: {len(instrumentals)}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "instrumentals[10]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "subset_name = \"genius\"\n", "min_ratio = min(results[subset_name][\"lyrics_ratio\"])\n", "max_ratio = max(results[subset_name][\"lyrics_ratio\"])\n", "mean_ratio = np.mean(results[subset_name][\"lyrics_ratio\"])\n", "median_ratio = np.median(results[subset_name][\"lyrics_ratio\"])\n", "Q1 = np.percentile(results[subset_name][\"lyrics_ratio\"], 25)\n", "Q3 = np.percentile(results[subset_name][\"lyrics_ratio\"], 75)\n", "IQR = Q3 - Q1\n", "\n", "k = 1.5\n", "# Calculate bounds\n", "lower_bound = Q1 - k * IQR\n", "upper_bound = Q3 + k * IQR\n", "\n", "print(f\"min ratio: {min_ratio} max ratio: {max_ratio}\")\n", "print(f\"mean ratio: {mean_ratio} median ratio: {median_ratio}\")\n", "print(f\"irq: {IQR} upper_bound: {upper_bound} lower_bound: {lower_bound}\")\n", "\n", "# remove ones with ratio less than 8 and greater than 18" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "dataset_features = genius_audio_features\n", "feature_cols = dataset_features.columns[1:]\n", "\n", "conservative_feature_bounds = {\n", " \"loudness\": [-32, -4],\n", " \"spectral_centroid\": [1250, 6000],\n", " \"spectral_flatness\": [0.01, 0.25],\n", " \"crest_factor\": [0.5, 4],\n", " \"bass\" : [0.01, 0.6],\n", " \"mid\" : [0.2, 1.0],\n", " \"high\" : [0.05, 1.5],\n", " \"stereo_width\" : [0.05, 0.5]\n", "}\n", "\n", "feature_bounds = {\n", " \"loudness\": [-32, -4],\n", " \"spectral_centroid\": [1750, 5000],\n", " \"spectral_flatness\": [0.02, 0.22],\n", " \"crest_factor\": [1.0, 3],\n", " \"bass\" : [0.1, 0.5],\n", " \"mid\" : [0.4, 1.0],\n", " \"high\" : [0.15, 1.25],\n", " \"stereo_width\" : [0.05, 0.3]\n", "}\n", "\n", "for feature in feature_cols:\n", " print(f\"feature: {feature}\")\n", " plt.hist(dataset_features[feature], bins=500, density=False, zorder=10)\n", " #plt.title(f\"{subset_name} {feature}\")\n", "\n", " # add bounds to axs\n", " plt.axvline(x=feature_bounds[feature][0], color=\"red\", linestyle=\"--\", linewidth=2, zorder=11)\n", " plt.axvline(x=feature_bounds[feature][1], color=\"red\", linestyle=\"--\", linewidth=2, zorder=11)\n", "\n", " # count how many are out of bounds \n", " n_out_of_bounds = len(dataset_features[dataset_features[feature] < feature_bounds[feature][0]]) + len(dataset_features[dataset_features[feature] > feature_bounds[feature][1]])\n", " print(f\"subset {subset_name}: n_out_of_bounds {n_out_of_bounds}\")\n", "\n", " plt.xlabel(feature)\n", " plt.ylabel(\"Frequency\")\n", " #plt.yscale(\"log\")\n", " plt.grid(c=\"lightgray\", zorder=0)\n", " plt.show()\n", " \n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 2 }