{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "debe114a", "metadata": {}, "outputs": [], "source": [ "import os\n", "import json\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "from tqdm import tqdm\n", "from suno_utils.utils.text import read_jsonl, write_jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "03186a7f", "metadata": {}, "outputs": [], "source": [ "# load the raw metadata files that we will use for pretraining\n", "# lets start from the clean ones since i dont think we need other metadata\n", "metas_dir = \"/app/suno/tmp\"\n", "\n", "discogs_subset_filename = \"clean_discogs_subset_v0_metas.jsonl\"\n", "genius_filename = \"clean_genius_v0_metas.jsonl\"\n", "imslp_filename = \"clean_imslp_v0_metas.jsonl\"\n", "#deezer_filename = \"clean_deezer_v0_metas.jsonl\"\n", "\n", "discogs_subset_filepath = os.path.join(metas_dir, discogs_subset_filename)\n", "genius_filepath = os.path.join(metas_dir, genius_filename)\n", "imslp_filepath = os.path.join(metas_dir, imslp_filename)\n", "#deezer_filepath = os.path.join(metas_dir, deezer_filename)\n", "\n", "# load the metas\n", "discogs_subset_metas = read_jsonl(discogs_subset_filepath)\n", "print(f\"discogs_subset_metas {len(discogs_subset_metas):,}\")\n", "genius_metas = read_jsonl(genius_filepath)\n", "print(f\"genius_metas {len(genius_metas):,}\")\n", "imslp_metas = read_jsonl(imslp_filepath)\n", "print(f\"imslp_metas {len(imslp_metas):,}\")\n", "#deezer_metas = read_jsonl_buffered(deezer_filepath)\n", "#print(\"deezer_metas\", len(deezer_metas))\n", "\n", "local_filepath_dir = \"/app2/suno/data/raw_audio_opus_v0\"" ] }, { "cell_type": "code", "execution_count": null, "id": "a6c5a05e", "metadata": {}, "outputs": [], "source": [ "version = \"5\"\n", "\n", "# load some alignemnts info (h5 alignments)\n", "genius_alignments_filepath = (\n", " f\"/home/tony/Work/tony/hoot/tmp/genius_hq_alignments_h5_t480_v{version}.jsonl\"\n", ")\n", "discogs_alignments_filepath = (\n", " f\"/home/tony/Work/tony/hoot/tmp/discogs_hq_alignments_h5_t480_v{version}.jsonl\"\n", ")\n", "#deezer_alignments_filepath = (\n", "# f\"/home/tony/Work/tony/hoot/tmp/deezer_hq_alignments_h5_t480_v{version}.jsonl\"\n", "#)\n", "\n", "# Materialize the generator into a list before subscript\n", "genius_alignments = read_jsonl(genius_alignments_filepath)\n", "print(len(genius_alignments))\n", "discogs_alignments = read_jsonl(discogs_alignments_filepath)\n", "print(len(discogs_alignments))\n", "#deezer_alignments = list(read_jsonl_buffered(deezer_alignments_filepath))\n", "#deezer_alignments_map = {meta[\"id\"]: meta for meta in deezer_alignments}\n", "#print(len(deezer_alignments_map))\n", "\n", "def build_alignment_map(data):\n", " result = {}\n", " for k, v, cer in data:\n", " meta = v[0]\n", " lines, starts, ends = (\n", " meta[\"line_text\"],\n", " meta[\"line_start_s\"],\n", " meta[\"line_end_s\"],\n", " )\n", "\n", " line_entries = []\n", " for text, start, end in zip(lines, starts, ends):\n", " if start is None or end is None:\n", " continue\n", " line_entries.append(\n", " (\n", " start,\n", " end,\n", " text,\n", " )\n", " )\n", "\n", " result[k] = {\n", " \"lines\": line_entries,\n", " \"cer\": cer,\n", " \"text\": meta.get(\"text\"),\n", " \"start_s\": meta.get(\"start_s\"),\n", " \"end_s\": meta.get(\"end_s\"),\n", " \"vocal_start_s\": meta.get(\"vocal_start_s\"),\n", " \"vocal_end_s\": meta.get(\"vocal_end_s\"),\n", " }\n", " return result\n", "\n", "\n", "# usage\n", "genius_alignments_map = build_alignment_map(genius_alignments)\n", "discogs_alignments_map = build_alignment_map(discogs_alignments)\n", "#deezer_alignments_map = build_alignment_map(deezer_alignments)" ] }, { "cell_type": "code", "execution_count": null, "id": "705338ca", "metadata": {}, "outputs": [], "source": [ "genius_alignments_map = {}\n", "discogs_alignments_map = {}" ] }, { "cell_type": "code", "execution_count": null, "id": "ffa5083b", "metadata": {}, "outputs": [], "source": [ "# audio production metadata\n", "filepath = \"/home/christian/code/christian/metadata/organized/audio_production/combined_audio_production_features_v2.csv\"\n", "df = pd.read_csv(filepath)\n", "# convert this to a dictionary indexed by id\n", "audio_production_dict = df.set_index(\"id\").to_dict(orient=\"index\")" ] }, { "cell_type": "code", "execution_count": null, "id": "3763c442", "metadata": {}, "outputs": [], "source": [ "# load stuff here that will become tags\n", "discogs_subset_chart_metas_filepath = \"/home/christian/code/christian/metadata/organized/charts/discogs_subset_chart_metas.jsonl\"\n", "discogs_subset_grammy_metas_filepath = \"/home/christian/code/christian/metadata/organized/charts/discogs_subset_grammy_metas.jsonl\"\n", "discogs_subset_grammy_metas = [meta for meta in read_jsonl(discogs_subset_grammy_metas_filepath)]\n", "discogs_subset_chart_metas = [meta for meta in read_jsonl(discogs_subset_chart_metas_filepath)]\n", "\n", "# then convert these to dictionaries indexed by id\n", "discogs_subset_chart_metas_dict = {meta[\"id\"]: meta for meta in discogs_subset_chart_metas}\n", "discogs_subset_grammy_metas_dict = {meta[\"id\"]: meta for meta in discogs_subset_grammy_metas}\n", "\n", "# load the genius llm tags\n", "discogs_gpt_tags_filepath = \"/home/christian/code/christian/metadata/organized/llm/gpt-4_1-tagging/gpt-4_1-tagging_results_discogs_subset.json\"\n", "genius_gpt_tags_filepath = \"/home/christian/code/christian/metadata/organized/llm/gpt-4_1-tagging/gpt-4_1-tagging_results_genius.json\"\n", "imslp_gpt_tags_filepath = \"/home/christian/code/christian/metadata/organized/llm/gpt-4_1-tagging/gpt-4_1-tagging_results_imslp.json\"\n", "\n", "with open(discogs_gpt_tags_filepath, \"r\") as f:\n", " discogs_gpt_tags = json.load(f)\n", "\n", "with open(genius_gpt_tags_filepath, \"r\") as f:\n", " genius_gpt_tags = json.load(f)\n", "\n", "with open(imslp_gpt_tags_filepath, \"r\") as f:\n", " imslp_gpt_tags = json.load(f)\n", "\n", "# clean up the gpt tags by remove the \"gpt_tags\" key for each element but keep it as dict with id as key\n", "discogs_gpt_tags = {id: tags[\"gpt_tag\"] for id, tags in discogs_gpt_tags.items()}\n", "genius_gpt_tags = {id: tags[\"gpt_tag\"] for id, tags in genius_gpt_tags.items()}\n", "imslp_gpt_tags = {id: tags[\"gpt_tag\"] for id, tags in imslp_gpt_tags.items()}\n", "\n", "print(discogs_gpt_tags[list(discogs_gpt_tags.keys())[0]])\n", "print(genius_gpt_tags[list(genius_gpt_tags.keys())[0]])\n", "print(imslp_gpt_tags[list(imslp_gpt_tags.keys())[0]])\n", "\n", "# load the titles \n", "discogs_subset_titles_filepath = \"/home/christian/code/christian/metadata/organized/titles/discogs_subset_title_terms_map.json\"\n", "genius_titles_filepath = \"/home/christian/code/christian/metadata/organized/titles/genius_title_terms_map.json\"\n", "\n", "with open(discogs_subset_titles_filepath, \"r\") as f:\n", " discogs_subset_titles = json.load(f)\n", "\n", "with open(genius_titles_filepath, \"r\") as f:\n", " genius_titles = json.load(f)\n", "\n", "\n", "# Merge all relevant tags into a single list per meta_id\n", "# Collect all unique ids from all metadata sources\n", "all_ids = set().union(\n", " discogs_subset_titles.keys(),\n", " genius_titles.keys(),\n", " discogs_gpt_tags.keys(),\n", " genius_gpt_tags.keys(),\n", " imslp_gpt_tags.keys(),\n", " discogs_subset_chart_metas_dict.keys(),\n", " discogs_subset_grammy_metas_dict.keys()\n", ")\n", "print(f\"Total unique ids: {len(all_ids)}\")\n", "\n", "tag_metadata = {}\n", "\n", "for meta_id in tqdm(all_ids):\n", " # Gather possible tag lists\n", " tags = []\n", " # Only append if the value is not None/empty\n", " if discogs_subset_titles.get(meta_id):\n", " tags += discogs_subset_titles[meta_id]\n", " if genius_titles.get(meta_id):\n", " tags += genius_titles[meta_id]\n", " if discogs_gpt_tags.get(meta_id):\n", " tags.append(discogs_gpt_tags[meta_id])\n", " if genius_gpt_tags.get(meta_id):\n", " tags.append(genius_gpt_tags[meta_id])\n", " if imslp_gpt_tags.get(meta_id):\n", " tags.append(imslp_gpt_tags[meta_id])\n", " # Optionally add chart and grammy metas if they have any 'tags' keys, or skip if only want string tags\n", " # Example: if 'tags' in discogs_subset_chart_metas_dict.get(meta_id, {}): ...\n", "\n", " tag_metadata[meta_id] = tags" ] }, { "cell_type": "code", "execution_count": null, "id": "2d88df59", "metadata": {}, "outputs": [], "source": [ "output_metas = []" ] }, { "cell_type": "code", "execution_count": null, "id": "e9ea710b", "metadata": {}, "outputs": [], "source": [ "for metas in [genius_metas, discogs_subset_metas, imslp_metas]:\n", " for idx, meta in enumerate(tqdm(metas)):\n", " new_meta = meta.copy()\n", "\n", " meta_id = meta[\"id\"]\n", " # as we iterate here, we need to merge in the other metadata based on the id\n", " # metadata we need\n", " # 1. alignments\n", " # 2. audio production\n", " # 3. tags from : titles, gpt (llm), \n", "\n", " if meta_id in genius_alignments_map:\n", " new_meta[\"text_aligned\"] = genius_alignments_map[meta_id][\"lines\"]\n", " new_meta[\"cer\"] = genius_alignments_map[meta_id][\"cer\"]\n", " new_meta[\"vocal_start_s\"] = genius_alignments_map[meta_id][\"vocal_start_s\"]\n", " new_meta[\"vocal_end_s\"] = genius_alignments_map[meta_id][\"vocal_end_s\"]\n", " if meta_id in discogs_alignments_map:\n", " new_meta[\"text_aligned\"] = discogs_alignments_map[meta_id][\"lines\"]\n", " new_meta[\"cer\"] = discogs_alignments_map[meta_id][\"cer\"]\n", " new_meta[\"vocal_start_s\"] = discogs_alignments_map[meta_id][\"vocal_start_s\"]\n", " new_meta[\"vocal_end_s\"] = discogs_alignments_map[meta_id][\"vocal_end_s\"]\n", "\n", " # merge in any tags\n", " new_meta.get(\"tags\", []).extend(tag_metadata.get(meta_id, []))\n", "\n", " # add in audio production\n", " new_meta[\"audio_stats\"] = audio_production_dict.get(meta_id, {})\n", "\n", " # we also need to construct the local_filepath\n", " local_filepath = os.path.join(local_filepath_dir, f\"{meta_id}.opus\")\n", " new_meta[\"local_filepath\"] = local_filepath\n", "\n", " output_metas.append(new_meta)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "994c8a13", "metadata": {}, "outputs": [], "source": [ "# write the current unfiltered metas to \n", "out_filepath = \"/home/christian/code/christian/metadata/diffusion_pretrain/unfiltered_metas_v0.jsonl\"\n", "write_jsonl(output_metas, out_filepath)" ] }, { "cell_type": "code", "execution_count": null, "id": "fd761e03", "metadata": {}, "outputs": [], "source": [ "# now apply the filtering here\n", "# filter on\n", "# 1. duration (must be between 30 and 480 seconds)\n", "# 2. audio stats\n", "# - loudness (must be between -20 and -6)\n", "# - silence_percentage (must be less than 4%)\n", "# 3. if we have alignments, then we ensure CER is less than 0.8\n", "\n", "filtered_metas = []\n", "\n", "for meta in output_metas:\n", " if meta[\"duration_s\"] < 30 or meta[\"duration_s\"] > 480:\n", " continue\n", " if meta[\"audio_stats\"][\"loudness\"] < -20 or meta[\"audio_stats\"][\"loudness\"] > -6:\n", " continue\n", " if meta[\"audio_stats\"][\"silence_percentage\"] > 4:\n", " continue\n", " if meta.get(\"text_aligned\"):\n", " if meta[\"cer\"] > 0.8:\n", " continue\n", " \n", " filtered_metas.append(meta)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "bed3a644", "metadata": {}, "outputs": [], "source": [ "# write the current unfiltered metas to \n", "out_filepath = \"/home/christian/code/christian/metadata/diffusion_pretrain/filtered_metas_v0.jsonl\"\n", "write_jsonl(filtered_metas, out_filepath)" ] }, { "cell_type": "markdown", "id": "4dc934f3", "metadata": {}, "source": [] }, { "cell_type": "code", "execution_count": null, "id": "29da95d4", "metadata": {}, "outputs": [], "source": [ "# Do this for multiple features (loudness and silence_percentage)\n", "features = [\"loudness\", \"silence_percentage\"]\n", "\n", "for feature in features:\n", " plt.figure()\n", " plt.hist(df[feature], bins=250)\n", " plt.title(f\"Histogram of {feature}\")\n", " plt.xlabel(feature)\n", " plt.ylabel(\"Count\")\n", " plt.show()\n", "\n", " print(f\"{feature.capitalize()} column statistics:\")\n", " print(f\" Mean : {df[feature].mean():.3f}\")\n", " print(f\" Min : {df[feature].min():.3f}\")\n", " print(f\" Max : {df[feature].max():.3f}\")\n", " print(f\" Median : {df[feature].median():.3f}\")\n", " print(f\" 95th % : {df[feature].quantile(0.95):.3f}\")\n", " print(f\" 5th % : {df[feature].quantile(0.05):.3f}\")\n", " print(\"\")\n", "\n", "# cut everything below -20\n", "# and everything above -6" ] }, { "cell_type": "code", "execution_count": null, "id": "e6b17d0f", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 5 }