{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "d31d8850", "metadata": { "ExecuteTime": { "end_time": "2024-02-14T02:28:19.043823Z", "start_time": "2024-02-14T02:28:19.039598Z" } }, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "bd365f5e", "metadata": { "ExecuteTime": { "end_time": "2024-02-14T02:28:19.385689Z", "start_time": "2024-02-14T02:28:19.384280Z" } }, "outputs": [], "source": [ "# prep metas for each dataset with one row for each entry and (eg non-music will be filtered out)\n", "# genius_hq, youtube_music, freesound, jamendo, imslp, deezer, ytm_tagged\n", "\n", "# metas will contain text and tags:\n", "# text will be a simple string (eg for foreign etc)\n", "# text_segments will be a list of timestamped segments for aligned\n", "# tags will be (maybe empty) list of strings which could be any description (genre, title etc)\n", "\n", "# save this all to METAS_DIR" ] }, { "cell_type": "code", "execution_count": 3, "id": "7cd4c0dd", "metadata": { "ExecuteTime": { "end_time": "2024-02-14T02:28:24.246742Z", "start_time": "2024-02-14T02:28:19.480060Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_2756292/1317798345.py:12: DeprecationWarning: \n", "Pyarrow will become a required dependency of pandas in the next major release of pandas (pandas 3.0),\n", "(to allow more performant data types, such as the Arrow string type, and better interoperability with other libraries)\n", "but was not found to be installed on your system.\n", "If this would cause problems for you,\n", "please provide us feedback at https://github.com/pandas-dev/pandas/issues/54466\n", " \n", " import pandas as pd\n" ] } ], "source": [ "import tqdm\n", "import math\n", "import torch\n", "import random\n", "import funcy\n", "import copy\n", "import gc\n", "import re\n", "import json\n", "import tempfile\n", "import collections\n", "import pandas as pd\n", "import numpy as np\n", "import fasttext\n", "from joblib import Parallel, delayed\n", "\n", "from suno_utils.audio import Audio\n", "from suno_utils.utils.display import capture_output\n", "from suno_utils.utils.text import write_jsonl, read_jsonl, write_json, read_json, normalize_whitespace\n", "from suno_utils.utils.s3 import read_from_s3, check_s3_file_exists, open_from_s3\n", "from suno_utils.utils.tokenizers import tokenize\n", "from suno_utils.harvest.youtube.constants.text_lang import BASE_TO_FASTTEXT_REMAP\n", "\n", "LANG_ID_MODEL_FP = \"s3://suno-data/georg/trained_models/chirp_v1/lid.176.bin\"\n", "with capture_output():\n", " text_lang_model = read_from_s3(LANG_ID_MODEL_FP, read_f=fasttext.load_model)\n", "\n", "def _get_text_lang(text):\n", " \"\"\"get probability of input language for text\"\"\"\n", " text = text.replace(\"’\", \"'\").lower()\n", " text = re.sub(r\"\\[.+?\\]\", \" \", text)\n", " text = normalize_whitespace(text)\n", " out = text_lang_model.predict(text, k=1)\n", " lang_str = out[0][0]\n", " p_lang = out[1][0]\n", " lang = lang_str.split(\"__\")[-1]\n", " lang = BASE_TO_FASTTEXT_REMAP.get(lang, lang)\n", "# if p_lang >= 0.8:\n", "# return lang\n", " return p_lang, lang\n", "\n", "def _clean_tag(s):\n", " if s is None or not isinstance(s, str):\n", " return \"\"\n", " # remove symbold with special meaning\n", " s = re.sub(r\"[\\{\\}\\_\\[\\]]\", \" \", s)\n", " # squash whitespace\n", " return normalize_whitespace(s)\n", "\n", "\n", "TMP_DIR = os.path.join(os.getcwd(), \"tmp\")\n", "METAS_DIR = os.path.join(os.getcwd(), \"metadata\")\n", "os.makedirs(METAS_DIR, exist_ok=True)\n", "os.makedirs(TMP_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 4, "id": "2f190219", "metadata": { "ExecuteTime": { "end_time": "2024-02-14T02:28:24.252676Z", "start_time": "2024-02-14T02:28:24.249057Z" } }, "outputs": [ { "data": { "text/plain": [ "'/home/tony/Work/tony/FineTuning_chirp_v3/metadata'" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "METAS_DIR" ] }, { "cell_type": "code", "execution_count": 5, "id": "a4e8a0c4", "metadata": { "ExecuteTime": { "end_time": "2024-02-14T02:28:24.315047Z", "start_time": "2024-02-14T02:28:24.253660Z" } }, "outputs": [], "source": [ "# TODO: add 's3_filepath' and 'duration_s' to each entry for convenience\n", "# TODO: exclude other translations such as \"Tradução Em Português\" or \"Übersetzung\"\n", "# TODO: check hoot for youtube music and deezer lyrics\n", "# TODO: added lower filters for \"zh\" and \"ja\", verify that still enough included after" ] }, { "cell_type": "markdown", "id": "d9d50793", "metadata": {}, "source": [ "## genius_hq" ] }, { "cell_type": "code", "execution_count": 6, "id": "bef6fd2a", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:27:24.156388Z", "start_time": "2024-01-19T20:27:24.046653Z" } }, "outputs": [], "source": [ "from suno_utils.utils.lyrics import remove_speakers" ] }, { "cell_type": "code", "execution_count": 7, "id": "3bf63af8", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:28:41.184654Z", "start_time": "2024-01-19T20:27:24.157967Z" } }, "outputs": [], "source": [ "# 1m 13secs\n", "base_metas = read_from_s3(\"s3://suno-data/datasets/bundles/v1/genius_hq/metas_plus.jsonl\", read_f=read_jsonl)" ] }, { "cell_type": "code", "execution_count": 8, "id": "38aef11e", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:28:41.189010Z", "start_time": "2024-01-19T20:28:41.187031Z" } }, "outputs": [], "source": [ "prod_tag_version = \"v3\"" ] }, { "cell_type": "code", "execution_count": 9, "id": "283f089b", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:29:20.741392Z", "start_time": "2024-01-19T20:28:41.190955Z" } }, "outputs": [], "source": [ "aligned_lyrics = read_jsonl(os.path.join(TMP_DIR, f\"genius_hq_alignments_{prod_tag_version}.jsonl\"))\n", "aligned_lyrics_map = {k: v for k, v in aligned_lyrics}\n", "del aligned_lyrics\n", "gc.collect();" ] }, { "cell_type": "code", "execution_count": 10, "id": "b5d10840", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:29:20.746253Z", "start_time": "2024-01-19T20:29:20.743997Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "SUP bro\n" ] } ], "source": [ "print(\"SUP bro\")" ] }, { "cell_type": "code", "execution_count": 11, "id": "9324363e", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:29:21.780784Z", "start_time": "2024-01-19T20:29:20.747602Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████| 2090009/2090009 [00:00<00:00, 2126424.29it/s]\n" ] } ], "source": [ "for m in tqdm.tqdm(base_metas):\n", " if m[\"genius_slug\"] == \"Taylor-swift-youre-on-your-own-kid-lyrics\":\n", " print(m)" ] }, { "cell_type": "code", "execution_count": 12, "id": "d879fe57", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:29:21.784227Z", "start_time": "2024-01-19T20:29:21.782551Z" } }, "outputs": [], "source": [ "# TODO: discarding 35% here.... adjust filters? not obvious which ones" ] }, { "cell_type": "code", "execution_count": 13, "id": "d3e6866e", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:30:38.566417Z", "start_time": "2024-01-19T20:29:21.786290Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████| 2090009/2090009 [01:16<00:00, 27238.56it/s]\n" ] } ], "source": [ "metas = []\n", "seen_ids = set()\n", "blocked_tags = set(\n", " [\n", " t.lower()\n", " for t in [\n", " \"Non-Music\",\n", " \"Spoken Word\",\n", " \"Harsh Noise\",\n", " \"Noise\",\n", " \"Noise Pop\",\n", " \"Noise Rock\",\n", " ]\n", " ]\n", ")\n", "for m in tqdm.tqdm(base_metas):\n", " if (\n", " m[\"genius_views\"] < 50\n", " or m[\"youtube_views\"] < 100 # default is 50\n", " or m[\"duration_s\"] < 1 * 60 # default is 100\n", " or m[\"duration_s\"] > 8 * 60\n", " or len(m[\"lyrics\"]) > 6144\n", " or len(m[\"lyrics\"]) < 50\n", " ):\n", " continue\n", " # check if it has description\n", "# if \"[\" not in m[\"lyrics\"] or \"(\" not in m[\"lyrics\"]:\n", "# continue\n", " if m[\"id\"] in seen_ids or m[\"id\"] not in aligned_lyrics_map:\n", " continue\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"original_id\": m[\"genius_slug\"],\n", " \"views\": m[\"youtube_views\"],\n", " \"lang\": m[\"lang\"].split(\"-\")[0].lower(),\n", " }\n", " text_segments = []\n", " for mm in aligned_lyrics_map[m[\"id\"]]:\n", " text_segments.append(\n", " {\n", " \"text\": remove_speakers(mm[\"text\"]),\n", " \"private_text\": mm[\"text\"],\n", " \"start_s\": mm[\"start_s\"],\n", " \"end_s\": mm[\"end_s\"],\n", " \"vocal_start_s\": mm[\"vocal_start_s\"],\n", " \"vocal_end_s\": mm[\"vocal_end_s\"],\n", " }\n", " )\n", " new_m[\"text_segments\"] = text_segments\n", " new_m[\"duration_s\"] = m[\"duration_s\"]\n", " new_m[\"private_text\"] = m[\"lyrics\"]\n", " new_m[\"text\"] = remove_speakers(m[\"lyrics\"])\n", " # add tags\n", " tags = [\n", " normalize_whitespace(t_clean.replace(\"Genius\", \" \"))\n", " for t in m.get(\"tags_text\", [])\n", " if len(t_clean := _clean_tag(t)) > 0\n", " ]\n", " if len(tags) == 0:\n", " # we don't want to fine tune on things with no tags\n", " continue\n", " if len(tags) > 0:\n", " new_m[\"tags\"] = list(set(tags))\n", "# if any(t.lower() in blocked_tags for t in new_m[\"tags\"]):\n", "# continue\n", " private_tag = _clean_tag(m[\"youtube_title\"])\n", " private_tags = tags\n", " if len(private_tag) > 0:\n", " private_tags += [private_tag]\n", " if len(private_tags) > 0:\n", " new_m[\"private_tags\"] = list(set(private_tags))\n", " metas.append(new_m)\n", " seen_ids.add(m[\"id\"])\n", "# v13\n", "# 1,237,956 total clips\n", "# 1,237,956 aligned clips\n", "# 793,697 en clips\n", "# 444,259 foreign clips\n", "# 42,882 hours of aligned en clips\n", "# 23,194 hours of aligned non-en clips\n", "# 139 hours of aligned zh clips\n", "# ======\n", "# v1 \n", "# 1,255,485 total clips\n", "# 1,255,485 aligned clips\n", "# 809,155 en clips\n", "# 446,330 foreign clips\n", "# 38,000 hours of aligned en clips\n", "# 20,561 hours of aligned non-en clips\n", "# 122 hours of aligned zh clips\n", "# after the extension\n", "# 1,261,850 total clips\n", "# 1,261,850 aligned clips\n", "# 813,838 en clips\n", "# 448,012 foreign clips\n", "# 43,824 hours of aligned en clips\n", "# 23,725 hours of aligned non-en clips\n", "# 143 hours of aligned zh clips\n", "# v2\n", "# 1,262,813 total clips\n", "# 1,262,813 aligned clips\n", "# 814,507 en clips\n", "# 448,306 foreign clips\n", "# 44,019 hours of aligned en clips\n", "# 23,781 hours of aligned non-en clips\n", "# 143 hours of aligned zh clips\n", "# 1,262,813 clips with start 1,118,811 clips with end\n", "# v3\n", "# 1,262,927 total clips\n", "# 1,262,927 aligned clips\n", "# 814,530 en clips\n", "# 448,397 foreign clips\n", "# 44,020 hours of aligned en clips\n", "# 23,787 hours of aligned non-en clips\n", "# 143 hours of aligned zh clips\n", "# 1,262,927 clips with start 1,118,962 clips with end" ] }, { "cell_type": "code", "execution_count": 14, "id": "52f752d4", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:30:40.771457Z", "start_time": "2024-01-19T20:30:38.567924Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1,262,927 total clips\n", "1,262,927 aligned clips\n", "814,530 en clips\n", "448,397 foreign clips\n", "44,020 hours of aligned en clips\n", "23,787 hours of aligned non-en clips\n", "143 hours of aligned zh clips\n", "1,262,927 clips with start 1,118,962 clips with end\n" ] } ], "source": [ "print(f\"{len(metas):,} total clips\")\n", "print(f\"{len([m for m in metas if 'text_segments' in m]):,} aligned clips\")\n", "print(f\"{len([m for m in metas if 'text' in m if m['lang'] == 'en']):,} en clips\")\n", "print(f\"{len([m for m in metas if 'text' in m if m['lang'] != 'en']):,} foreign clips\")\n", "en_dur_s = 0\n", "non_en_dur_s = 0\n", "zh_dur_s = 0\n", "n_vocal_start = 0\n", "n_vocal_end = 0\n", "for m in metas:\n", " for mm in m.get(\"text_segments\", []):\n", " time_diff = mm[\"end_s\"] - mm[\"start_s\"]\n", " if time_diff > 120.01:\n", " print(\"WTF\", time_diff)\n", " if m[\"lang\"] == \"en\":\n", " en_dur_s += time_diff\n", " else:\n", " non_en_dur_s += time_diff\n", " if m[\"lang\"] == \"zh\":\n", " zh_dur_s += time_diff\n", " if mm[\"vocal_start_s\"] is not None:\n", " n_vocal_start += 1\n", " if mm[\"vocal_end_s\"] is not None:\n", " n_vocal_end += 1\n", "print(f\"{en_dur_s/60/60:,.0f} hours of aligned en clips\")\n", "print(f\"{non_en_dur_s/60/60:,.0f} hours of aligned non-en clips\")\n", "print(f\"{zh_dur_s/60/60:,.0f} hours of aligned zh clips\")\n", "print(f\"{n_vocal_start:,} clips with start\", f\"{n_vocal_end:,} clips with end\")" ] }, { "cell_type": "code", "execution_count": 15, "id": "02e294f4", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:33:29.300720Z", "start_time": "2024-01-19T20:30:40.779519Z" } }, "outputs": [], "source": [ "write_jsonl(metas, os.path.join(METAS_DIR, f\"genius_hq_{prod_tag_version}.jsonl\"))" ] }, { "cell_type": "code", "execution_count": 16, "id": "0d667725", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:33:29.304992Z", "start_time": "2024-01-19T20:33:29.303134Z" } }, "outputs": [], "source": [ "# with open(\"/app/suno/data/preprocessing/meta_cutoff_freq.json\", \"r\") as fp:\n", "# meta_cutoff_freq = json.load(fp)" ] }, { "cell_type": "code", "execution_count": 17, "id": "a3b95a12", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:33:29.353678Z", "start_time": "2024-01-19T20:33:29.307081Z" } }, "outputs": [], "source": [ "# selected_cutoffs = []\n", "# for m in metas:\n", "# cutoff_freq = meta_cutoff_freq.get(m[\"id\"])\n", "# selected_cutoffs.append(cutoff_freq)" ] }, { "cell_type": "code", "execution_count": 18, "id": "7f9cdab2", "metadata": { "ExecuteTime": { "end_time": "2024-01-19T20:33:29.398362Z", "start_time": "2024-01-19T20:33:29.355018Z" } }, "outputs": [], "source": [ "# import matplotlib.pyplot as plt\n", "# plt.clf()\n", "# plt.hist(selected_cutoffs, bins=list(range(0, 22000, 100)))\n", "# plt.text(0.1, 0.8, \"\\n\".join(f\"{100 * x} %: {np.quantile(selected_cutoffs, x):.1f}\" for x in [0.1, 0.15, 0.2, 0.3]))\n", "# plt.xlabel(\"99% roll_off_median, hz\")\n", "# plt.ylabel(\"number of genius linked audios\")\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": null, "id": "4d5a5e90", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "0a54c6a3", "metadata": {}, "source": [ "## youtube_music" ] }, { "cell_type": "code", "execution_count": 81, "id": "cf38bda5", "metadata": {}, "outputs": [], "source": [ "base_metas = read_from_s3(\"s3://suno-data/datasets/bundles/v1/youtube_music/metas.jsonl\", read_f=read_jsonl)" ] }, { "cell_type": "code", "execution_count": 42, "id": "382f0c73", "metadata": {}, "outputs": [], "source": [ "# TODO: use hoot/whisper on lyrics to filter" ] }, { "cell_type": "code", "execution_count": 82, "id": "384dac11", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2,101,712 clips\n", "2,013,545 music clips\n" ] } ], "source": [ "print(f\"{len(base_metas):,} clips\")\n", "allowed_ids = set()\n", "for m in base_metas:\n", " if m[\"view_count\"] < 1000:\n", " continue\n", " if re.search(r\"\\blive\\b\", m[\"title\"], flags=re.IGNORECASE):\n", " continue\n", " allowed_ids.add(m[\"id\"])\n", "print(f\"{len(allowed_ids):,} music clips\")\n", "# 2,101,712 clips\n", "# 2,013,545 music clips" ] }, { "cell_type": "code", "execution_count": 94, "id": "d183738b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2,013,545 entries\n", "674,466 entries with lyrics\n" ] } ], "source": [ "metas = []\n", "seen_ids = set()\n", "for m in base_metas:\n", " if m[\"id\"] in seen_ids or m[\"id\"] not in allowed_ids:\n", " continue\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"views\": m[\"view_count\"],\n", " }\n", " # get tags\n", " tags = [\n", " m.get(\"genre_guess\"),\n", " m.get(\"mood_guess\"),\n", " ]\n", " tags = [t_clean for t in tags if len(t_clean:=_clean_tag(t)) > 0]\n", " if len(tags) > 0:\n", " new_m[\"tags\"] = list(set(tags))\n", " # get private tags\n", " private_tags = [\n", " m.get(\"title\"),\n", " m.get(\"artists\"),\n", " m.get(\"album\"),\n", " ]\n", " private_tags = [t_clean for t in private_tags if len(t_clean:=_clean_tag(t)) > 0]\n", " private_tags += tags\n", " if len(private_tags) > 0:\n", " new_m[\"private_tags\"] = list(set(private_tags))\n", " # get lyrics\n", " if (\n", " \"lang_guess\" in m and \n", " \"lyrics\" in m and \n", "# m[\"lang_guess\"].lower() != \"en\" and\n", "# _get_text_lang(m[\"lyrics\"])[1] != \"en\" and\n", "# len(m[\"lang_guess\"].lower().split(\"-\")[0]) >= 2 and\n", " len(m[\"lyrics\"]) <= 5120 # empirically verified\n", " ):\n", " use_lyrics = False\n", " if m[\"lang_guess\"][:2] in (\"zh\", \"ja\"):\n", " if len(m[\"lyrics\"]) < 256:\n", " use_lyrics = True\n", " else:\n", " if len(m[\"lyrics\"]) < 512:\n", " use_lyrics = True\n", " if use_lyrics:\n", " lang_guess_1 = m[\"lang_guess\"].split(\"-\")[0].lower()\n", " lang_guess_2 = _get_text_lang(m[\"lyrics\"])[1].split(\"-\")[0].lower()\n", " if (lang_guess_1 == \"en\") == (lang_guess_2 == \"en\"):\n", " # make sure they agree on english or not\n", " lyrics = m[\"lyrics\"]\n", " new_m[\"text\"] = m[\"lyrics\"]\n", " new_m[\"lang\"] = lang_guess_1\n", " metas.append(new_m)\n", " seen_ids.add(m[\"id\"])\n", "print(f\"{len(metas):,} entries\")\n", "print(f\"{len([m for m in metas if 'text' in m]):,} entries with lyrics\")\n", "# 2,013,545 entries\n", "# 674,466 entries with lyrics" ] }, { "cell_type": "code", "execution_count": 95, "id": "d35468bd", "metadata": {}, "outputs": [], "source": [ "write_jsonl(metas, os.path.join(METAS_DIR, \"youtube_music.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "be5b3d68", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f5860a2c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7e537400", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "2e349879", "metadata": {}, "source": [ "## freesound" ] }, { "cell_type": "code", "execution_count": 24, "id": "1e4b5427", "metadata": {}, "outputs": [], "source": [ "base_metas = read_from_s3(\"s3://suno-data/datasets/bundles/v1/freesound/metas.jsonl\", read_f=read_jsonl) " ] }, { "cell_type": "code", "execution_count": 26, "id": "de99a839", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "510,491 clips\n", "2,993 hours of audio\n", "34,225 music clips\n", "411 hours of music\n" ] } ], "source": [ "print(f\"{len(base_metas):,} clips\")\n", "print(f\"{np.sum([m['duration_s'] for m in base_metas])/60/60:,.0f} hours of audio\")\n", "needle_tags = [\n", " \"music\", \"synth\", \"bpm\", \"drums\", \"drum-loop\", \"guitar\", \"piano\", \"soundtrack\", \"song\", \n", " \"violin\", \"melody\", \"bassline\",\n", "]\n", "\n", "needle_re = re.compile(r\"(\\b\" + r\"\\b)|(\\b\".join([s.lower() for s in needle_tags]) + r\"\\b)\")\n", "l = []\n", "music_ids = set()\n", "for m in base_metas:\n", " if m[\"duration_s\"] < 10:\n", " continue\n", " tag_str = \" \".join(m[\"tags\"]).lower()\n", " if needle_re.search(tag_str):\n", " l.append(m[\"duration_s\"])\n", " music_ids.add(m[\"id\"])\n", "print(f\"{len(music_ids):,} music clips\")\n", "print(f\"{np.sum(l)/60/60:,.0f} hours of music\")\n", "# 510,491 clips\n", "# 2,993 hours of audio\n", "# 34,225 music clips\n", "# 411 hours of music" ] }, { "cell_type": "code", "execution_count": 27, "id": "ff92dfed", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "34,225 entries\n" ] } ], "source": [ "metas = []\n", "seen_ids = set()\n", "for m in base_metas:\n", " if m[\"id\"] in seen_ids or m[\"id\"] not in music_ids:\n", " continue\n", " new_m = {\"id\": m[\"id\"]}\n", " tags = [\n", " _clean_tag(m.get(\"cap1\")),\n", " _clean_tag(m.get(\"cap2\")),\n", " ]\n", " tags.extend([_clean_tag(s) for s in m.get(\"tags\", [])])\n", " tags = [t for t in tags if len(t) > 0]\n", " if len(tags) > 0:\n", " new_m[\"tags\"] = list(set(tags))\n", " metas.append(new_m)\n", " seen_ids.add(m[\"id\"])\n", "print(f\"{len(metas):,} entries\")\n", "# 34,225 entries" ] }, { "cell_type": "code", "execution_count": 28, "id": "6b03ad9a", "metadata": {}, "outputs": [], "source": [ "write_jsonl(metas, os.path.join(METAS_DIR, \"freesound.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "d61eb037", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1749760a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "358398cd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "fba8655f", "metadata": {}, "source": [ "## jamendo" ] }, { "cell_type": "code", "execution_count": 29, "id": "46ba131e", "metadata": {}, "outputs": [], "source": [ "base_metas = read_from_s3(\"s3://suno-data/datasets/bundles/v1/jamendo/metas.jsonl\", read_f=read_jsonl)" ] }, { "cell_type": "code", "execution_count": 30, "id": "7c0cf8bd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "55,609 entries\n" ] } ], "source": [ "metas = []\n", "seen_ids = set()\n", "for m in base_metas:\n", " if m[\"id\"] in seen_ids:\n", " continue\n", " new_m = {\"id\": m[\"id\"]}\n", " tags = []\n", " tags.extend([_clean_tag(s) for s in m.get(\"genre\", \"\").split(\";\")])\n", " tags.extend([_clean_tag(s) for s in m.get(\"instrument\", \"\").split(\";\")])\n", " tags.extend([_clean_tag(s) for s in m.get(\"mood/theme\", \"\").split(\";\")])\n", " tags = [t for t in tags if len(t) > 0]\n", " if len(tags) > 0:\n", " new_m[\"tags\"] = list(set(tags))\n", " metas.append(new_m)\n", " seen_ids.add(m[\"id\"])\n", "print(f\"{len(metas):,} entries\")\n", "# 55,609 entries" ] }, { "cell_type": "code", "execution_count": 31, "id": "e8d95346", "metadata": {}, "outputs": [], "source": [ "write_jsonl(metas, os.path.join(METAS_DIR, \"jamendo.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "d6297c0a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "52809176", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1c5cf161", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "335506ec", "metadata": {}, "source": [ "## imslp" ] }, { "cell_type": "code", "execution_count": 19, "id": "1e7b4a80", "metadata": { "ExecuteTime": { "end_time": "2024-01-20T15:22:29.170557Z", "start_time": "2024-01-20T15:22:26.433462Z" } }, "outputs": [], "source": [ "base_metas_imslp = read_from_s3(\"s3://suno-data/datasets/bundles/v1/imslp/metas.jsonl\", read_f=read_jsonl)" ] }, { "cell_type": "code", "execution_count": 20, "id": "de23d794", "metadata": { "ExecuteTime": { "end_time": "2024-01-20T15:22:29.867328Z", "start_time": "2024-01-20T15:22:29.864661Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "WTF\n" ] } ], "source": [ "print(\"WTF\")" ] }, { "cell_type": "code", "execution_count": 24, "id": "f14670d9", "metadata": { "ExecuteTime": { "end_time": "2024-01-20T15:24:10.294853Z", "start_time": "2024-01-20T15:24:08.117463Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "278,620 clips\n", "208,971 music clips\n", "22.251300277777776 khr\n" ] } ], "source": [ "print(f\"{len(base_metas_imslp):,} clips\")\n", "blocked_tags = [\"vocal\", \"chorus\"]\n", "blocked_re = re.compile(r\"(\" + r\")|(\".join([s.lower() for s in blocked_tags]) + r\")\")\n", "allowed_ids = set()\n", "total_duration = 0\n", "durations = []\n", "for m in base_metas_imslp:\n", " tag_str = (\n", " m.get(\"title\", \"\")\n", " + \" \"\n", " + m.get(\"track_title\", \"\")\n", " + \" \"\n", " + m.get(\"composer\", \"\")\n", " + \" \"\n", " + m.get(\"language\", \"\")\n", " + \" \"\n", " + m.get(\"recording_category\", \"\")\n", " + \" \"\n", " + m.get(\"genre\", \"\")\n", " + \" \"\n", " + m.get(\"instruments\", \"\")\n", " ).lower()\n", " total_duration += m[\"duration_s\"]\n", " durations.append(m[\"duration_s\"])\n", " if blocked_re.search(tag_str):\n", " continue\n", " if len(normalize_whitespace(tag_str)) == 0:\n", " continue\n", " allowed_ids.add(m[\"id\"])\n", "print(f\"{len(allowed_ids):,} music clips\")\n", "print(f\"{total_duration / 3600 / 1000} khr\")\n", "# 278,620 clips\n", "# 274,275 music clips" ] }, { "cell_type": "code", "execution_count": 26, "id": "79e63a58", "metadata": { "ExecuteTime": { "end_time": "2024-01-20T15:24:33.207751Z", "start_time": "2024-01-20T15:24:31.964710Z" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGwCAYAAABLvHTgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAv9ElEQVR4nO3de3RU1f3//1dCyCRcknAxM0QDpF64i0AUI4hW8iFqpFJpLRgBEUExVCKIwNeSeuNSWKjgBaq1wmcJAq4KKiCYT0CoEAJEglwDVRAqTKhCMoASINm/P/xxykDUgJNMsnk+1pq1mLPfc2afvVaY19pnn3NCjDFGAAAAlgkNdgcAAAAqAyEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKYcHuQDCVlZXpwIEDql+/vkJCQoLdHQAAUAHGGB09elRxcXEKDf3x+ZpLOuQcOHBA8fHxwe4GAAC4CPv379cVV1zxo+2XdMipX7++pB8GKSoqKsi9AQAAFeHz+RQfH+/8jv+YSzrknDlFFRUVRcgBAKCG+bmlJiw8BgAAViLkAAAAKxFyAACAlQg5AADASoQcAABgpQsOOatXr1bPnj0VFxenkJAQLVq0yK/dGKPMzEw1adJEkZGRSk5O1u7du/1qDh8+rLS0NEVFRSkmJkaDBg3SsWPH/Go+//xz3XzzzYqIiFB8fLwmT558Xl/effddtWzZUhEREWrXrp2WLl16oYcDAAAsdcEh5/jx42rfvr1effXVctsnT56s6dOna+bMmcrNzVXdunWVkpKiEydOODVpaWnatm2bsrKytHjxYq1evVpDhgxx2n0+n3r06KFmzZopLy9PU6ZM0dNPP63XX3/dqVm7dq369u2rQYMGadOmTerVq5d69eqlrVu3XughAQAAG5lfQJJZuHCh876srMx4PB4zZcoUZ1tRUZFxuVzmnXfeMcYYs337diPJbNiwwan56KOPTEhIiPn666+NMca89tprpkGDBqakpMSpGT16tGnRooXz/t577zWpqal+/encubN5+OGHK9z/4uJiI8kUFxdX+DMAACC4Kvr7HdA1OXv27JHX61VycrKzLTo6Wp07d1ZOTo4kKScnRzExMUpMTHRqkpOTFRoaqtzcXKemW7duCg8Pd2pSUlJUUFCgI0eOODVnf8+ZmjPfU56SkhL5fD6/FwAAsFNAQ47X65Ukud1uv+1ut9tp83q9io2N9WsPCwtTw4YN/WrK28fZ3/FjNWfayzNx4kRFR0c7L55bBQCAvS6pq6vGjh2r4uJi57V///5gdwkAAFSSgIYcj8cjSSosLPTbXlhY6LR5PB4dOnTIr/306dM6fPiwX015+zj7O36s5kx7eVwul/OcKp5XBQCA3QIachISEuTxeJSdne1s8/l8ys3NVVJSkiQpKSlJRUVFysvLc2pWrFihsrIyde7c2alZvXq1Tp065dRkZWWpRYsWatCggVNz9vecqTnzPQAA4NJ2wSHn2LFjys/PV35+vqQfFhvn5+dr3759CgkJUUZGhp5//nl98MEH2rJli/r376+4uDj16tVLktSqVSvdfvvtGjx4sNavX681a9Zo2LBh6tOnj+Li4iRJ9913n8LDwzVo0CBt27ZN8+fP17Rp0zRixAinH8OHD9eyZcs0depU7dy5U08//bQ2btyoYcOG/fJRAQAANd+FXra1cuVKI+m814ABA4wxP1xGPm7cOON2u43L5TLdu3c3BQUFfvv49ttvTd++fU29evVMVFSUGThwoDl69KhfzebNm03Xrl2Ny+Uyl19+uZk0adJ5fVmwYIG55pprTHh4uGnTpo1ZsmTJBR0Ll5ADAFDzVPT3O8QYY4KYsYLK5/MpOjpaxcXFrM/5Ec3HLDlv295JqUHoCQAAP6jo73dYFfYJljg3+Fxs6CFAAQAqEyEHvxhhBQBQHV1S98kBAACXDmZy4Ke8WRkAAGoiZnIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJW44zGqBHdSBgBUNWZyAACAlZjJQaVg5gYAEGzM5AAAACsRcgAAgJU4XXUJOfcU0t5JqUHqCQAAlY+ZHAAAYCVCDgAAsBIhBwAAWImQAwAArMTC40sY97IBANiMmRwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYKSzYHQDO1nzMEr/3eyelBqknAICajpkcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArcQm5pc69FBsAgEsNMzkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKzE1VWo1sq7SoyHdgIAKoKZHAAAYCVCDgAAsBIhBwAAWImQAwAArBTwkFNaWqpx48YpISFBkZGRuvLKK/Xcc8/JGOPUGGOUmZmpJk2aKDIyUsnJydq9e7fffg4fPqy0tDRFRUUpJiZGgwYN0rFjx/xqPv/8c918882KiIhQfHy8Jk+eHOjDAQAANVTAr676y1/+ohkzZmj27Nlq06aNNm7cqIEDByo6OlqPPfaYJGny5MmaPn26Zs+erYSEBI0bN04pKSnavn27IiIiJElpaWk6ePCgsrKydOrUKQ0cOFBDhgzR3LlzJUk+n089evRQcnKyZs6cqS1btujBBx9UTEyMhgwZEujDqtZ4ThUAAOcLMWdPsQTAXXfdJbfbrTfffNPZ1rt3b0VGRurtt9+WMUZxcXEaOXKknnjiCUlScXGx3G63Zs2apT59+mjHjh1q3bq1NmzYoMTEREnSsmXLdOedd+rf//634uLiNGPGDD311FPyer0KDw+XJI0ZM0aLFi3Szp07y+1bSUmJSkpKnPc+n0/x8fEqLi5WVFRUIIehSl1qIYdLyAHg0ubz+RQdHf2zv98BP1110003KTs7W7t27ZIkbd68WZ9++qnuuOMOSdKePXvk9XqVnJzsfCY6OlqdO3dWTk6OJCknJ0cxMTFOwJGk5ORkhYaGKjc316np1q2bE3AkKSUlRQUFBTpy5Ei5fZs4caKio6OdV3x8fGAPHgAAVBsBP101ZswY+Xw+tWzZUrVq1VJpaanGjx+vtLQ0SZLX65Ukud1uv8+53W6nzev1KjY21r+jYWFq2LChX01CQsJ5+zjT1qBBg/P6NnbsWI0YMcJ5f2YmBwAA2CfgIWfBggWaM2eO5s6dqzZt2ig/P18ZGRmKi4vTgAEDAv11F8TlcsnlcgW1DwAAoGoEPOSMGjVKY8aMUZ8+fSRJ7dq101dffaWJEydqwIAB8ng8kqTCwkI1adLE+VxhYaGuu+46SZLH49GhQ4f89nv69GkdPnzY+bzH41FhYaFfzZn3Z2oAAMClK+Brcr777juFhvrvtlatWiorK5MkJSQkyOPxKDs722n3+XzKzc1VUlKSJCkpKUlFRUXKy8tzalasWKGysjJ17tzZqVm9erVOnTrl1GRlZalFixblnqqCPZqPWeL3AgCgPAEPOT179tT48eO1ZMkS7d27VwsXLtQLL7yg3/72t5KkkJAQZWRk6Pnnn9cHH3ygLVu2qH///oqLi1OvXr0kSa1atdLtt9+uwYMHa/369VqzZo2GDRumPn36KC4uTpJ03333KTw8XIMGDdK2bds0f/58TZs2zW/NDQAAuHQF/HTVyy+/rHHjxunRRx/VoUOHFBcXp4cffliZmZlOzZNPPqnjx49ryJAhKioqUteuXbVs2TLnHjmSNGfOHA0bNkzdu3dXaGioevfurenTpzvt0dHR+vjjj5Wenq5OnTqpcePGyszMvOTukQMAAMoX8Pvk1CQVvc6+urvUT9lw3xwAuLQE7T45AAAA1QEhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYKeDPrgKqWnmPteBRDwAAZnIAAICVCDkAAMBKhBwAAGAl1uTUQOWtQQEAAP6YyQEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwUliwOwBUhuZjlvi93zspNUg9AQAECzM5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJB3TiknDuAzslHtoJALYj5FRz5f04AwCAn8fpKgAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArhQW7A0CwNB+zxO/93kmpQeoJAKAyMJMDAACsVCkh5+uvv9b999+vRo0aKTIyUu3atdPGjRuddmOMMjMz1aRJE0VGRio5OVm7d+/228fhw4eVlpamqKgoxcTEaNCgQTp27Jhfzeeff66bb75ZERERio+P1+TJkyvjcAAAQA0U8JBz5MgRdenSRbVr19ZHH32k7du3a+rUqWrQoIFTM3nyZE2fPl0zZ85Ubm6u6tatq5SUFJ04ccKpSUtL07Zt25SVlaXFixdr9erVGjJkiNPu8/nUo0cPNWvWTHl5eZoyZYqefvppvf7664E+JAAAUAOFGGNMIHc4ZswYrVmzRv/85z/LbTfGKC4uTiNHjtQTTzwhSSouLpbb7dasWbPUp08f7dixQ61bt9aGDRuUmJgoSVq2bJnuvPNO/fvf/1ZcXJxmzJihp556Sl6vV+Hh4c53L1q0SDt37qxQX30+n6Kjo1VcXKyoqKgAHH3gnbtuBJWHNTkAUDNU9Pc74DM5H3zwgRITE/X73/9esbGx6tChg9544w2nfc+ePfJ6vUpOTna2RUdHq3PnzsrJyZEk5eTkKCYmxgk4kpScnKzQ0FDl5uY6Nd26dXMCjiSlpKSooKBAR44cKbdvJSUl8vl8fi8AAGCngIecL7/8UjNmzNDVV1+t5cuXa+jQoXrsscc0e/ZsSZLX65Ukud1uv8+53W6nzev1KjY21q89LCxMDRs29Kspbx9nf8e5Jk6cqOjoaOcVHx//C48WAABUVwEPOWVlZerYsaMmTJigDh06aMiQIRo8eLBmzpwZ6K+6YGPHjlVxcbHz2r9/f7C7BAAAKknA75PTpEkTtW7d2m9bq1at9I9//EOS5PF4JEmFhYVq0qSJU1NYWKjrrrvOqTl06JDfPk6fPq3Dhw87n/d4PCosLPSrOfP+TM25XC6XXC7XRR4ZbFfe+ifW6QBAzRXwmZwuXbqooKDAb9uuXbvUrFkzSVJCQoI8Ho+ys7Oddp/Pp9zcXCUlJUmSkpKSVFRUpLy8PKdmxYoVKisrU+fOnZ2a1atX69SpU05NVlaWWrRo4XclFwAAuDQFPOQ8/vjjWrdunSZMmKB//etfmjt3rl5//XWlp6dLkkJCQpSRkaHnn39eH3zwgbZs2aL+/fsrLi5OvXr1kvTDzM/tt9+uwYMHa/369VqzZo2GDRumPn36KC4uTpJ03333KTw8XIMGDdK2bds0f/58TZs2TSNGjAj0IQEAgBoo4Kerrr/+ei1cuFBjx47Vs88+q4SEBL300ktKS0tzap588kkdP35cQ4YMUVFRkbp27aply5YpIiLCqZkzZ46GDRum7t27KzQ0VL1799b06dOd9ujoaH388cdKT09Xp06d1LhxY2VmZvrdSwcAAFy6An6fnJqE++Tg57AmBwCqn6DdJwcAAKA6IOQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwUliwOwB/zccsCXYXAACwAjM5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVuBkg8BPKuznj3kmpQegJAOBCMZMDAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFbisQ7ABTr3UQ885gEAqidmcgAAgJUIOQAAwEqEHAAAYCXW5AC/0LlrdCTW6QBAdcBMDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAAr8RRyoBKc+2RynkoOAFWv0mdyJk2apJCQEGVkZDjbTpw4ofT0dDVq1Ej16tVT7969VVhY6Pe5ffv2KTU1VXXq1FFsbKxGjRql06dP+9V88skn6tixo1wul6666irNmjWrsg8HAADUEJUacjZs2KC//vWvuvbaa/22P/744/rwww/17rvvatWqVTpw4IDuuecep720tFSpqak6efKk1q5dq9mzZ2vWrFnKzMx0avbs2aPU1FT9+te/Vn5+vjIyMvTQQw9p+fLllXlIAACghqi0kHPs2DGlpaXpjTfeUIMGDZztxcXFevPNN/XCCy/otttuU6dOnfTWW29p7dq1WrdunSTp448/1vbt2/X222/ruuuu0x133KHnnntOr776qk6ePClJmjlzphISEjR16lS1atVKw4YN0+9+9zu9+OKLlXVIAACgBqm0kJOenq7U1FQlJyf7bc/Ly9OpU6f8trds2VJNmzZVTk6OJCknJ0ft2rWT2+12alJSUuTz+bRt2zan5tx9p6SkOPsoT0lJiXw+n98LAADYqVIWHs+bN0+fffaZNmzYcF6b1+tVeHi4YmJi/La73W55vV6n5uyAc6b9TNtP1fh8Pn3//feKjIw877snTpyoZ5555qKPCwAA1BwBn8nZv3+/hg8frjlz5igiIiLQu/9Fxo4dq+LiYue1f//+YHcJAABUkoCHnLy8PB06dEgdO3ZUWFiYwsLCtGrVKk2fPl1hYWFyu906efKkioqK/D5XWFgoj8cjSfJ4POddbXXm/c/VREVFlTuLI0kul0tRUVF+LwAAYKeAn67q3r27tmzZ4rdt4MCBatmypUaPHq34+HjVrl1b2dnZ6t27tySpoKBA+/btU1JSkiQpKSlJ48eP16FDhxQbGytJysrKUlRUlFq3bu3ULF261O97srKynH0A1cm5982RuHcOAFS2gIec+vXrq23btn7b6tatq0aNGjnbBw0apBEjRqhhw4aKiorSH//4RyUlJenGG2+UJPXo0UOtW7dWv379NHnyZHm9Xv3pT39Senq6XC6XJOmRRx7RK6+8oieffFIPPvigVqxYoQULFmjJkvN/TAAAwKUnKHc8fvHFFxUaGqrevXurpKREKSkpeu2115z2WrVqafHixRo6dKiSkpJUt25dDRgwQM8++6xTk5CQoCVLlujxxx/XtGnTdMUVV+hvf/ubUlJSgnFIAACgmgkxxphgdyJYfD6foqOjVVxcXG3W55R3WgN24nQVAFyciv5+84BOAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArBeU+OQDOv10Al5QDQGAxkwMAAKxEyAEAAFYi5AAAACuxJgeoJnhSOQAEFiEniHhOFQAAlYfTVQAAwEqEHAAAYCVCDgAAsBJrcoAahMXJAFBxhBygGmNxOgBcPE5XAQAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACuFBbsDAH6Z5mOW/GzN3kmpVdATAKhemMkBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASl5ADl4DyLjPnsnIAtmMmBwAAWImQAwAArETIAQAAVmJNDnCJOnedDmt0ANiGmRwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACtxCTkASTz6AYB9mMkBAABWIuQAAAArcboKwI+qyF2RuXMygOqKmRwAAGAlQg4AALASIQcAAFiJkAMAAKzEwmMAFVbevXQAoLoK+EzOxIkTdf3116t+/fqKjY1Vr169VFBQ4Fdz4sQJpaenq1GjRqpXr5569+6twsJCv5p9+/YpNTVVderUUWxsrEaNGqXTp0/71XzyySfq2LGjXC6XrrrqKs2aNSvQhwMAAGqogIecVatWKT09XevWrVNWVpZOnTqlHj166Pjx407N448/rg8//FDvvvuuVq1apQMHDuiee+5x2ktLS5WamqqTJ09q7dq1mj17tmbNmqXMzEynZs+ePUpNTdWvf/1r5efnKyMjQw899JCWL18e6EMCAAA1UIgxxlTmF/znP/9RbGysVq1apW7duqm4uFiXXXaZ5s6dq9/97neSpJ07d6pVq1bKycnRjTfeqI8++kh33XWXDhw4ILfbLUmaOXOmRo8erf/85z8KDw/X6NGjtWTJEm3dutX5rj59+qioqEjLli0rty8lJSUqKSlx3vt8PsXHx6u4uFhRUVGVOArlY+ofNuI+OQAqm8/nU3R09M/+flf6wuPi4mJJUsOGDSVJeXl5OnXqlJKTk52ali1bqmnTpsrJyZEk5eTkqF27dk7AkaSUlBT5fD5t27bNqTl7H2dqzuyjPBMnTlR0dLTzio+PD8xBAnA0H7PkvBcABEOlhpyysjJlZGSoS5cuatu2rSTJ6/UqPDxcMTExfrVut1ter9epOTvgnGk/0/ZTNT6fT99//325/Rk7dqyKi4ud1/79+3/xMQIAgOqpUq+uSk9P19atW/Xpp59W5tdUmMvlksvlCnY3AABAFai0mZxhw4Zp8eLFWrlypa644gpnu8fj0cmTJ1VUVORXX1hYKI/H49Sce7XVmfc/VxMVFaXIyMhAHw4AAKhhAh5yjDEaNmyYFi5cqBUrVighIcGvvVOnTqpdu7ays7OdbQUFBdq3b5+SkpIkSUlJSdqyZYsOHTrk1GRlZSkqKkqtW7d2as7ex5maM/sAAACXtoBfXfXoo49q7ty5ev/999WiRQtne3R0tDPDMnToUC1dulSzZs1SVFSU/vjHP0qS1q5dK+mHS8ivu+46xcXFafLkyfJ6verXr58eeughTZgwQdIPl5C3bdtW6enpevDBB7VixQo99thjWrJkiVJSUirU14quzq4sLMjEpYyrsABcrKBdXTVjxgwVFxfr1ltvVZMmTZzX/PnznZoXX3xRd911l3r37q1u3brJ4/Hovffec9pr1aqlxYsXq1atWkpKStL999+v/v3769lnn3VqEhIStGTJEmVlZal9+/aaOnWq/va3v1U44AAAALtV+n1yqjNmcoDqg5kdABVVbe6TAwAAEAyEHAAAYCWeQg6gWijv9C2nsAD8EszkAAAAKzGTA6DaOnd2h5kdABeCmRwAAGAlQg4AALASIQcAAFiJNTlViJv/Ab8MV2ABuBDM5AAAACsRcgAAgJU4XQWgRuMycwA/hpkcAABgJWZyAFiFxckAzmAmBwAAWImQAwAArETIAQAAVmJNDgDrcQUWcGliJgcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEpcQl5Jyru1PIDqoaJ/n1xqDtRszOQAAAArMZMDAD+CmwgCNRszOQAAwEqEHAAAYCVCDgAAsBJrcgCggipyVRbrdoDqg5kcAABgJUIOAACwEiEHAABYiTU5ABBArNsBqg9mcgAAgJUIOQAAwEqEHAAAYCXW5ABAFStv3Q7rdIDAYyYHAABYiZkcAKgGuCoLCDxmcgAAgJWYyQGAGoK1PMCFIeQAQA12bvAh9AD/xekqAABgJWZyAMAinNIC/ouQAwDgtBesRMgBAMtdTICpyIwQs0ao7gg5AHCJqcg9eQAbEHIAAAHD7A6qE0IOAKBCqnIGiLCEQCDkAAAqVUXWBHEKDZWBkAMAqFIXG2i4AgwXipADAKiROKWFn0PIAQBYg6e542yEHADAJeViT5cRjmoeQg4AABVwMeGIYBRchBwAACoJ64aCi5ADAEAVuthL6glHF46QAwBAEFX0NNjFLKoO1Pqjmvossxofcl599VVNmTJFXq9X7du318svv6wbbrgh2N0CAKDKBeqmihXZT024gWNosDvwS8yfP18jRozQn//8Z3322Wdq3769UlJSdOjQoWB3DQAABFmNDjkvvPCCBg8erIEDB6p169aaOXOm6tSpo7///e/B7hoAAAiyGnu66uTJk8rLy9PYsWOdbaGhoUpOTlZOTk65nykpKVFJSYnzvri4WJLk8/kC3r+yku8Cvk8AAGqSyvh9PXu/xpifrKuxIeebb75RaWmp3G6333a3262dO3eW+5mJEyfqmWeeOW97fHx8pfQRAIBLWfRLlbv/o0ePKjo6+kfba2zIuRhjx47ViBEjnPdlZWU6fPiwGjVqpJCQkIB9j8/nU3x8vPbv36+oqKiA7ddGjFXFMVYXhvGqOMaq4hiriqvMsTLG6OjRo4qLi/vJuhobcho3bqxatWqpsLDQb3thYaE8Hk+5n3G5XHK5XH7bYmJiKquLioqK4o+gghirimOsLgzjVXGMVcUxVhVXWWP1UzM4Z9TYhcfh4eHq1KmTsrOznW1lZWXKzs5WUlJSEHsGAACqgxo7kyNJI0aM0IABA5SYmKgbbrhBL730ko4fP66BAwcGu2sAACDIanTI+cMf/qD//Oc/yszMlNfr1XXXXadly5adtxi5qrlcLv35z38+79QYzsdYVRxjdWEYr4pjrCqOsaq46jBWIebnrr8CAACogWrsmhwAAICfQsgBAABWIuQAAAArEXIAAICVCDmV4NVXX1Xz5s0VERGhzp07a/369cHuUpWaOHGirr/+etWvX1+xsbHq1auXCgoK/GpOnDih9PR0NWrUSPXq1VPv3r3Pu7Hjvn37lJqaqjp16ig2NlajRo3S6dOnq/JQqtykSZMUEhKijIwMZxtj9V9ff/217r//fjVq1EiRkZFq166dNm7c6LQbY5SZmakmTZooMjJSycnJ2r17t98+Dh8+rLS0NEVFRSkmJkaDBg3SsWPHqvpQKl1paanGjRunhIQERUZG6sorr9Rzzz3n96yfS3W8Vq9erZ49eyouLk4hISFatGiRX3ugxuXzzz/XzTffrIiICMXHx2vy5MmVfWgB91NjderUKY0ePVrt2rVT3bp1FRcXp/79++vAgQN++wjqWBkE1Lx580x4eLj5+9//brZt22YGDx5sYmJiTGFhYbC7VmVSUlLMW2+9ZbZu3Wry8/PNnXfeaZo2bWqOHTvm1DzyyCMmPj7eZGdnm40bN5obb7zR3HTTTU776dOnTdu2bU1ycrLZtGmTWbp0qWncuLEZO3ZsMA6pSqxfv940b97cXHvttWb48OHOdsbqB4cPHzbNmjUzDzzwgMnNzTVffvmlWb58ufnXv/7l1EyaNMlER0ebRYsWmc2bN5vf/OY3JiEhwXz//fdOze23327at29v1q1bZ/75z3+aq666yvTt2zcYh1Spxo8fbxo1amQWL15s9uzZY959911Tr149M23aNKfmUh2vpUuXmqeeesq89957RpJZuHChX3sgxqW4uNi43W6TlpZmtm7dat555x0TGRlp/vrXv1bVYQbET41VUVGRSU5ONvPnzzc7d+40OTk55oYbbjCdOnXy20cwx4qQE2A33HCDSU9Pd96XlpaauLg4M3HixCD2KrgOHTpkJJlVq1YZY374w6hdu7Z59913nZodO3YYSSYnJ8cY88MfVmhoqPF6vU7NjBkzTFRUlCkpKanaA6gCR48eNVdffbXJysoyt9xyixNyGKv/Gj16tOnateuPtpeVlRmPx2OmTJnibCsqKjIul8u88847xhhjtm/fbiSZDRs2ODUfffSRCQkJMV9//XXldT4IUlNTzYMPPui37Z577jFpaWnGGMbrjHN/uAM1Lq+99ppp0KCB39/g6NGjTYsWLSr5iCpPeYHwXOvXrzeSzFdffWWMCf5YcboqgE6ePKm8vDwlJyc720JDQ5WcnKycnJwg9iy4iouLJUkNGzaUJOXl5enUqVN+49SyZUs1bdrUGaecnBy1a9fO78aOKSkp8vl82rZtWxX2vmqkp6crNTXVb0wkxupsH3zwgRITE/X73/9esbGx6tChg9544w2nfc+ePfJ6vX5jFR0drc6dO/uNVUxMjBITE52a5ORkhYaGKjc3t+oOpgrcdNNNys7O1q5duyRJmzdv1qeffqo77rhDEuP1YwI1Ljk5OerWrZvCw8OdmpSUFBUUFOjIkSNVdDRVr7i4WCEhIc5zIYM9VjX6jsfVzTfffKPS0tLz7rjsdru1c+fOIPUquMrKypSRkaEuXbqobdu2kiSv16vw8PDzHo7qdrvl9XqdmvLG8UybTebNm6fPPvtMGzZsOK+NsfqvL7/8UjNmzNCIESP0//7f/9OGDRv02GOPKTw8XAMGDHCOtbyxOHusYmNj/drDwsLUsGFDq8ZKksaMGSOfz6eWLVuqVq1aKi0t1fjx45WWliZJjNePCNS4eL1eJSQknLePM20NGjSolP4H04kTJzR69Gj17dvXeSBnsMeKkINKlZ6erq1bt+rTTz8Ndleqpf3792v48OHKyspSREREsLtTrZWVlSkxMVETJkyQJHXo0EFbt27VzJkzNWDAgCD3rvpZsGCB5syZo7lz56pNmzbKz89XRkaG4uLiGC8E3KlTp3TvvffKGKMZM2YEuzsOTlcFUOPGjVWrVq3zrnwpLCyUx+MJUq+CZ9iwYVq8eLFWrlypK664wtnu8Xh08uRJFRUV+dWfPU4ej6fccTzTZou8vDwdOnRIHTt2VFhYmMLCwrRq1SpNnz5dYWFhcrvdjNX/r0mTJmrdurXftlatWmnfvn2S/nusP/X35/F4dOjQIb/206dP6/Dhw1aNlSSNGjVKY8aMUZ8+fdSuXTv169dPjz/+uCZOnCiJ8foxgRqXS+XvUvpvwPnqq6+UlZXlzOJIwR8rQk4AhYeHq1OnTsrOzna2lZWVKTs7W0lJSUHsWdUyxmjYsGFauHChVqxYcd40ZKdOnVS7dm2/cSooKNC+ffuccUpKStKWLVv8/jjO/PGc+0NXk3Xv3l1btmxRfn6+80pMTFRaWprzb8bqB126dDnvVgS7du1Ss2bNJEkJCQnyeDx+Y+Xz+ZSbm+s3VkVFRcrLy3NqVqxYobKyMnXu3LkKjqLqfPfddwoN9f8vvlatWiorK5PEeP2YQI1LUlKSVq9erVOnTjk1WVlZatGihVWnqs4EnN27d+v//u//1KhRI7/2oI/VL166DD/z5s0zLpfLzJo1y2zfvt0MGTLExMTE+F35YruhQ4ea6Oho88knn5iDBw86r++++86peeSRR0zTpk3NihUrzMaNG01SUpJJSkpy2s9cFt2jRw+Tn59vli1bZi677DLrLosuz9lXVxnDWJ2xfv16ExYWZsaPH292795t5syZY+rUqWPefvttp2bSpEkmJibGvP/+++bzzz83d999d7mX/nbo0MHk5uaaTz/91Fx99dU1/pLo8gwYMMBcfvnlziXk7733nmncuLF58sknnZpLdbyOHj1qNm3aZDZt2mQkmRdeeMFs2rTJuSIoEONSVFRk3G636devn9m6dauZN2+eqVOnTo27hPynxurkyZPmN7/5jbniiitMfn6+3//3Z18pFcyxIuRUgpdfftk0bdrUhIeHmxtuuMGsW7cu2F2qUpLKfb311ltOzffff28effRR06BBA1OnTh3z29/+1hw8eNBvP3v37jV33HGHiYyMNI0bNzYjR440p06dquKjqXrnhhzG6r8+/PBD07ZtW+NyuUzLli3N66+/7tdeVlZmxo0bZ9xut3G5XKZ79+6moKDAr+bbb781ffv2NfXq1TNRUVFm4MCB5ujRo1V5GFXC5/OZ4cOHm6ZNm5qIiAjzq1/9yjz11FN+Pz6X6nitXLmy3P+jBgwYYIwJ3Lhs3rzZdO3a1bhcLnP55ZebSZMmVdUhBsxPjdWePXt+9P/7lStXOvsI5liFGHPW7S8BAAAswZocAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAF+3WW29VRkZG0L7/k08+UUhIyHkPMAUAiZADoIYoL1DddNNNOnjwoKKjo4PTKQDVGiEHQFCd/eThCxUeHi6Px6OQkJAA9giALQg5ACrk+PHj6t+/v+rVq6cmTZpo6tSp59WEhIRo0aJFfttiYmI0a9YsSdLevXsVEhKi+fPn65ZbblFERITmzJmjb7/9Vn379tXll1+uOnXqqF27dnrnnXecfTzwwANatWqVpk2bppCQEIWEhGjv3r3lnq76xz/+oTZt2sjlcql58+bn9bN58+aaMGGCHnzwQdWvX19NmzbV66+/fkFj8dVXX6lnz55q0KCB6tatqzZt2mjp0qVO+9atW3XHHXeoXr16crvd6tevn7755hunvaysTJMnT9ZVV10ll8ulpk2bavz48RfUBwA/j5ADoEJGjRqlVatW6f3339fHH3+sTz75RJ999tlF7WvMmDEaPny4duzYoZSUFJ04cUKdOnXSkiVLtHXrVg0ZMkT9+vXT+vXrJUnTpk1TUlKSBg8erIMHD+rgwYOKj48/b795eXm699571adPH23ZskVPP/20xo0b54SsM6ZOnarExERt2rRJjz76qIYOHaqCgoIK9z89PV0lJSVavXq1tmzZor/85S+qV6+eJKmoqEi33XabOnTooI0bN2rZsmUqLCzUvffe63x+7NixmjRpksaNG6ft27dr7ty5crvdFzGSAH5SQJ5lDsBqR48eNeHh4WbBggXOtm+//dZERkaa4cOHO9skmYULF/p9Njo62rz11lvGGGP27NljJJmXXnrpZ78zNTXVjBw50nl/yy23+H2XMcasXLnSSDJHjhwxxhhz3333mf/5n//xqxk1apRp3bq1875Zs2bm/vvvd96XlZWZ2NhYM2PGjJ/t0xnt2rUzTz/9dLltzz33nOnRo4fftv379xtJpqCgwPh8PuNyucwbb7xR4e8DcHHCghuxANQEX3zxhU6ePKnOnTs72xo2bKgWLVpc1P4SExP93peWlmrChAlasGCBvv76a508eVIlJSWqU6fOBe13x44duvvuu/22denSRS+99JJKS0tVq1YtSdK1117rtIeEhMjj8ejQoUMV/p7HHntMQ4cO1ccff6zk5GT17t3b2efmzZu1cuVKZ2bnbF988YWKiopUUlKi7t27X9CxAbhwnK4CEDAhISEyxvhtK29hcd26df3eT5kyRdOmTdPo0aO1cuVK5efnKyUlRSdPnqyUftauXdvvfUhIiMrKyir8+Yceekhffvml+vXrpy1btigxMVEvv/yyJOnYsWPq2bOn8vPz/V67d+9Wt27dFBkZGdBjAfDjCDkAftaVV16p2rVrKzc319l25MgR7dq1y6/usssu08GDB533u3fv1nffffez+1+zZo3uvvtu3X///Wrfvr1+9atfnbfv8PBwlZaW/uR+WrVqpTVr1py372uuucaZxQmU+Ph4PfLII3rvvfc0cuRIvfHGG5Kkjh07atu2bWrevLmuuuoqv1fdunV19dVXKzIyUtnZ2QHtD4DzEXIA/Kx69epp0KBBGjVqlFasWKGtW7fqgQceUGio/38ht912m1555RVt2rRJGzdu1COPPHLerEl5rr76amVlZWnt2rXasWOHHn74YRUWFvrVNG/eXLm5udq7d6+++eabcmdeRo4cqezsbD333HPatWuXZs+erVdeeUVPPPHELxuAc2RkZGj58uXas2ePPvvsM61cuVKtWrWS9MOi5MOHD6tv377asGGDvvjiCy1fvlwDBw5UaWmpIiIiNHr0aD355JP63//9X33xxRdat26d3nzzzYD2EQAhB0AFTZkyRTfffLN69uyp5ORkde3aVZ06dfKrmTp1quLj43XzzTfrvvvu0xNPPFGhdTV/+tOf1LFjR6WkpOjWW2+Vx+NRr169/GqeeOIJ1apVS61bt9Zll12mffv2nbefjh07asGCBZo3b57atm2rzMxMPfvss3rggQcu6FhvvfXWn/xMaWmp0tPT1apVK91+++265ppr9Nprr0mS4uLitGbNGpWWlqpHjx5q166dMjIyFBMT44TCcePGaeTIkcrMzFSrVq30hz/84YLWBAGomBBz7gl0ALjENWvWTM8888wFhyMA1QszOQBwlm3btik6Olr9+/cPdlcA/ELM5AAAACsxkwMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArPT/AZmcDBSlsFAdAAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "plt.clf()\n", "plt.hist(durations, bins=np.linspace(0, 60 * 10 * 2, 100))\n", "plt.xlabel(\"duration, sec\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 27, "id": "f45ab32c", "metadata": { "ExecuteTime": { "end_time": "2024-01-20T15:24:41.827299Z", "start_time": "2024-01-20T15:24:41.539217Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Counter({'romantic': 125232, 'baroque': 70338, 'classical': 48660, 'early 20th century': 21860, 'modern': 8575, 'renaissance': 3310, 'medieval': 167, 'traditional (folk)': 137, 'jazz': 101, 'chinese': 96, 'western/chinese/japanese/classic/pop/mixed': 50, 'traditional': 39, 'ancient': 19, 'early_20th_century': 19, 'non-western classical': 15, 'bossa nova': 1, '': 1})\n", "ferrari, carlotta\n", "beethoven, ludwig van\n", "purcell, henry\n", "schumann, robert\n", "mozart, wolfgang amadeus\n", "weber, carl maria von\n", "brahms, johannes\n", "bach, johann sebastian\n", "simpson, daniel léo\n", "sousa, john philip\n", "bellini, vincenzo\n", "bizet, georges\n", "liszt, franz\n", "bruckner, anton\n", "handel, george frideric\n", "verdi, giuseppe\n", "monteverdi, claudio\n", "grieg, edvard\n", "various\n", "zhang, shuwen\n", "prokofiev, sergey\n", "rachmaninoff, sergei\n", "chopin, frédéric\n", "bartók, béla\n", "granados, enrique\n", "schütz, heinrich\n", "strauss, richard\n", "schubert, franz\n", "falla, manuel de\n", "gershwin, george\n", "mendelssohn, felix\n", "rondeau, michel\n", "strauss jr., johann\n", "respighi, ottorino\n", "couperin, françois\n", "frescobaldi, girolamo\n", "debussy, claude\n", "fauré, gabriel\n", "rossini, gioacchino\n", "haydn, joseph\n", "vivaldi, antonio\n", "saint-saëns, camille\n", "telemann, georg philipp\n", "dvořák, antonín\n", "mussorgsky, modest\n", "puccini, giacomo\n", "ravel, maurice\n", "paganini, niccolò\n", "elgar, edward\n", "villa-lobos, heitor\n", "berlioz, hector\n", "massenet, jules\n", "rimsky-korsakov, nikolay\n", "offenbach, jacques\n", "franck, césar\n", "tchaikovsky, pyotr\n", "reger, max\n", "corelli, arcangelo\n", "rameau, jean-philippe\n", "delibes, léo\n", "sullivan, arthur\n", "sor, fernando\n", "donizetti, gaetano\n", "schoenberg, arnold\n", "stravinsky, igor\n", "scriabin, aleksandr\n", "satie, erik\n", "bach, carl philipp emanuel\n", "gounod, charles\n", "sibelius, jean\n", "wagner, richard\n", "pergolesi, giovanni battista\n", "scarlatti, domenico\n", "mahler, gustav\n", "lehár, franz\n", "gluck, christoph willibald\n" ] } ], "source": [ "genre_counter = collections.Counter()\n", "instruments_counter = collections.Counter()\n", "composer_counter = collections.Counter()\n", "for m in base_metas_imslp:\n", " genre_counter[m.get(\"genre\", \"\").lower()] += 1\n", " instruments_counter[m.get(\"instruments\", \"\").lower()] += 1\n", " composer_counter[m.get(\"composer\", \"\").lower()] += 1\n", "# if m.get(\"composer\", \"\").lower() == 'mahler, gustav':\n", "# print(m.get(\"title\", \"\"), m.get(\"track_title\", \"\"))\n", "# if m.get(\"title\", \"\") == \"Symphony No. 5 in C-Sharp Minor\":\n", "# print(\"--->\", m)\n", "print(genre_counter)\n", "block_list = set(\n", " [\n", " \"simpson, daniel léo\",\n", " \"sousa, john philip\",\n", " \"various\",\n", " \"zhang, shuwen\",\n", " \"bartók, béla\",\n", " \"strauss, richard\",\n", " \"prokofiev, sergey\",\n", " ]\n", ")\n", "for composer in composer_counter:\n", " if composer_counter[composer] > 500:\n", " print(composer)" ] }, { "cell_type": "code", "execution_count": 38, "id": "b534970f", "metadata": { "ExecuteTime": { "end_time": "2024-01-20T15:32:37.013234Z", "start_time": "2024-01-20T15:32:37.009010Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[(1600, 'claude le jeune'), (1643, 'monteverdi, claudio'), (1695, 'purcell, henry'), (1713, 'corelli, arcangelo'), (1741, 'antonio vivaldi'), (1750, 'bach, johann sebastian'), (1757, 'scarlatti, domenico'), (1759, 'handel, george frideric'), (1767, 'georg philipp telemann'), (1767, 'telemann, georg philipp'), (1787, 'gluck, christoph willibald'), (1788, 'bach, carl philipp emanuel'), (1809, 'haydn, joseph'), (1827, 'beethoven, ludwig van'), (1828, 'franz schubert'), (1835, 'vincenzo bellini'), (1839, 'sor, fernando'), (1840, 'paganini, niccolò'), (1847, 'felix mendelssohn'), (1848, 'donizetti, gaetano'), (1856, 'schumann, robert'), (1868, 'rossini, gioacchino'), (1869, 'berlioz, hector'), (1875, 'georges bizet'), (1880, 'offenbach, jacques'), (1881, 'modest mussorgsky'), (1883, 'wagner, richard'), (1886, 'liszt, franz'), (1890, 'franck, césar'), (1893, 'gounod, charles'), (1893, 'tchaikovsky, pyotr'), (1897, 'brahms, johannes'), (1899, 'johann strauss jr.'), (1900, 'sullivan, arthur'), (1901, 'verdi, giuseppe'), (1904, 'antonín dvořák'), (1911, 'mahler, gustav'), (1912, 'massenet, jules'), (1915, 'scriabin, aleksandr'), (1916, 'granados, enrique'), (1918, 'debussy, claude')]\n", "{'debussy, claude', 'monteverdi, claudio', 'gounod, charles', 'haydn, joseph', 'brahms, johannes', 'beethoven, ludwig van', 'felix mendelssohn', 'corelli, arcangelo', 'johann strauss jr.', 'berlioz, hector', 'rossini, gioacchino', 'vincenzo bellini', 'georg philipp telemann', 'sullivan, arthur', 'granados, enrique', 'schumann, robert', 'sor, fernando', 'tchaikovsky, pyotr', 'offenbach, jacques', 'mahler, gustav', 'scriabin, aleksandr', 'claude le jeune', 'modest mussorgsky', 'bach, carl philipp emanuel', 'wagner, richard', 'purcell, henry', 'paganini, niccolò', 'antonio vivaldi', 'georges bizet', 'franz schubert', 'telemann, georg philipp', 'donizetti, gaetano', 'scarlatti, domenico', 'liszt, franz', 'antonín dvořák', 'massenet, jules', 'handel, george frideric', 'gluck, christoph willibald', 'verdi, giuseppe', 'franck, césar', 'bach, johann sebastian'}\n" ] } ], "source": [ "composers_and_death = \"\"\"\n", "monteverdi, claudio - 1643\n", "purcell, henry - 1695\n", "corelli, arcangelo - 1713\n", "bach, johann sebastian - 1750\n", "telemann, georg philipp - 1767\n", "scarlatti, domenico - 1757\n", "handel, george frideric - 1759\n", "beethoven, ludwig van - 1827\n", "bach, carl philipp emanuel - 1788\n", "gluck, christoph willibald - 1787\n", "haydn, joseph - 1809\n", "sor, fernando - 1839\n", "donizetti, gaetano - 1848\n", "schumann, robert - 1856\n", "rossini, gioacchino - 1868\n", "liszt, franz - 1886\n", "wagner, richard - 1883\n", "brahms, johannes - 1897\n", "verdi, giuseppe - 1901\n", "sullivan, arthur - 1900\n", "debussy, claude - 1918\n", "granados, enrique - 1916\n", "mahler, gustav - 1911\n", "puccini, giacomo - 1924\n", "satie, erik - 1925\n", "franck, césar - 1890\n", "tchaikovsky, pyotr - 1893\n", "gounod, charles - 1893\n", "prokofiev, sergey - 1953\n", "rachmaninoff, sergei - 1943\n", "scriabin, aleksandr - 1915\n", "paganini, niccolò - 1840\n", "elgar, edward - 1934\n", "villa-lobos, heitor - 1959\n", "berlioz, hector - 1869\n", "massenet, jules - 1912\n", "rimsky-korsakov, nikolay - 1908\n", "offenbach, jacques - 1880\n", "franz lehár - 1948\n", "saint-saëns, camille - 1921\n", "sibelius, jean - 1957\n", "franz schubert - 1828\n", "manuel de falla - 1946\n", "george gershwin - 1937\n", "felix mendelssohn - 1847\n", "michel rondeau - Unknown\n", "johann strauss jr. - 1899\n", "john philip sousa - 1932\n", "vincenzo bellini - 1835\n", "georges bizet - 1875\n", "georg philipp telemann - 1767\n", "antonín dvořák - 1904\n", "modest mussorgsky - 1881\n", "antonio vivaldi - 1741\n", "camille saint-saëns - 1921\n", "jean sibelius - 1957\n", "pierre-montan berton - 1780\n", "bryan d. hoyt - Unknown\n", "claude le jeune - 1600\n", "alexander nakarada - Unknown\n", "\"\"\"\n", "composers = []\n", "allowed_composers = set()\n", "for l in composers_and_death.split(\"\\n\"):\n", " outs = l.split(\"-\")\n", " if len(outs) < 2:\n", " continue\n", " composer = outs[0].strip()\n", " date = outs[1].strip()\n", " if not date.isnumeric():\n", " continue\n", " if int(date) <= 1923:\n", " composers.append((int(date), composer))\n", " allowed_composers.add(composer)\n", "print(sorted(composers))\n", "print(allowed_composers)" ] }, { "cell_type": "code", "execution_count": 39, "id": "de881ffe", "metadata": { "ExecuteTime": { "end_time": "2024-01-20T15:32:39.267131Z", "start_time": "2024-01-20T15:32:37.773345Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "95,685 entries, total duration 8,342 hr\n" ] } ], "source": [ "metas = []\n", "seen_ids = set()\n", "total_duration = 0\n", "for m in base_metas_imslp:\n", " if m[\"id\"] in seen_ids or m[\"id\"] not in allowed_ids:\n", " continue\n", " if m.get(\"composer\", \"\").lower() not in allowed_composers:\n", " continue\n", " new_m = {\"id\": m[\"id\"]}\n", "# tags = [\n", "# _clean_tag(m.get(\"composer\", \"\")),\n", "# _clean_tag(m.get(\"genre\", \"\")),\n", "# _clean_tag(m.get(\"title\")),\n", "# _clean_tag(m.get(\"instruments\", \"\")),\n", "# ]\n", " # original tags\n", " tags = [\n", " m.get(\"composer\", \"\"),\n", " m.get(\"genre\", \"\"),\n", " m.get(\"title\", \"\"),\n", " m.get(\"instruments\", \"\"),\n", " ]\n", " spaced_tags = []\n", " for t in tags:\n", " spaced_tags.extend(t.split(\", \"))\n", " tags = spaced_tags\n", " tags = [t for t in tags if len(t) > 0]\n", " if any(\"voice\" in tag for tag in tags) or any(\"chorus\" in tag for tag in tags):\n", " continue\n", " if len(tags) > 0:\n", " new_m[\"tags\"] = list(set(tags))\n", " private_tags = [\n", " _clean_tag(m.get(\"title\")),\n", " _clean_tag(m.get(\"track_title\", \"\")),\n", " _clean_tag(m.get(\"composer\", \"\")),\n", " ]\n", " spaced_tags = []\n", " for t in private_tags:\n", " spaced_tags.extend(t.split(\", \"))\n", " private_tags = spaced_tags\n", " private_tags += tags\n", " private_tags = [t for t in private_tags if len(t) > 0]\n", " if len(private_tags) > 0:\n", " new_m[\"private_tags\"] = list(set(private_tags))\n", " metas.append(new_m)\n", " seen_ids.add(m[\"id\"])\n", " total_duration += m[\"duration_s\"]\n", "print(f\"{len(metas):,} entries, total duration {total_duration/60/60:,.0f} hr\")" ] }, { "cell_type": "code", "execution_count": 42, "id": "238042df", "metadata": { "ExecuteTime": { "end_time": "2024-01-20T15:33:15.317232Z", "start_time": "2024-01-20T15:33:15.088310Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[('baroque', 37960), ('romantic', 36951), ('piano', 31867), ('bach', 31050), ('johann sebastian', 30545), ('classical', 19311), ('ludwig van', 18198), ('beethoven', 18198), ('orchestra', 18153), ('violin', 10353), ('cello', 9757), ('johannes', 8951), ('brahms', 8951), ('robert', 8207), ('schumann', 8207), ('strings', 8030), ('continuo', 6928), ('organ', 5667), ('keyboard', 5049), ('clavier', 5004), ('2 violins', 4898), ('pyotr', 4349), ('tchaikovsky', 4349), ('debussy', 4026), ('claude', 4026), ('viola', 4006), ('haydn', 3982), ('joseph', 3982), ('george frideric', 3771), ('handel', 3771), ('harpsichord', 3718), ('liszt', 3668), ('franz', 3668), ('2 clarinets', 2912), ('2 oboes', 2903), ('2 trumpets', 2663), ('timpani', 2332), ('goldberg variations', 2252), ('the well-tempered clavier', 2154), ('bwv 988', 1919), ('flute', 1787), ('op. 6', 1747), ('telemann', 1625), ('georg philipp', 1625), ('no. 2', 1498), ('early 20th century', 1463), ('book 1', 1456), ('no. 1', 1422), ('op. 120', 1412), ('3 trombones', 1331), ('2 bassoons\\n2 horns', 1327), ('book 2', 1301), ('op. 9', 1180), ('op. 15', 1143), ('carnaval', 1133), ('paganini', 1061), ('niccolò', 1061), ('césar', 1038), ('franck', 1038), ('op. 1', 1033), ('piano 4-hands', 1024), ('op. 35', 1003), ('op. 13', 992), ('no. 3', 989), ('aleksandr', 972), ('scriabin', 972), ('op. 68', 889), ('corelli', 882), ('arcangelo', 882), ('orchestra: 2 flutes', 800), ('2 flutes', 781), ('2 bassoons\\n4 horns', 756), ('\"diabelli variations\"', 748), ('recorder', 743), ('oboe', 731), ('concertino: 2 violins', 714), ('cello\\nripieno: strings', 714), ('domenico', 703), ('scarlatti', 703), ('but most likely keyboard', 697), ('unspecified', 697), ('preludes', 684), ('24 caprices', 667), ('clarinet', 658), ('bwv 846-869', 655), ('21 hungarian dances', 642), ('guitar', 630), ('hector', 620), ('berlioz', 620), ('op. 2', 617), ('op. 24', 610), ('op. 11', 594), ('trumpet', 592), ('op. 14', 582), ('préludes', 578), ('33 variations on a waltz by diabelli in c major', 576), ('triangle', 572), ('soprano', 570), ('fernando', 568), ('sor', 568)]\n" ] } ], "source": [ "tag_counter = collections.Counter()\n", "for m in metas:\n", " for tag in m.get(\"tags\"):\n", " tag_counter[tag.lower()] += 1\n", "print(tag_counter.most_common(100))" ] }, { "cell_type": "code", "execution_count": 43, "id": "5de82694", "metadata": { "ExecuteTime": { "end_time": "2024-01-20T15:33:23.070450Z", "start_time": "2024-01-20T15:33:22.529463Z" } }, "outputs": [], "source": [ "write_jsonl(metas, os.path.join(METAS_DIR, \"imslp_filtered.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "12d5c590", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "29b05eb1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "01862f25", "metadata": {}, "source": [ "### pond5 music" ] }, { "cell_type": "code", "execution_count": 15, "id": "204baedd", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T03:59:44.790194Z", "start_time": "2024-01-18T03:58:35.017703Z" } }, "outputs": [], "source": [ "pd5_base_metas = read_from_s3(\"s3://suno-data/datasets/bundles/v2/pond5_music/metas.jsonl\", read_f=read_jsonl)\n", "pd5_extra_metas = read_from_s3(\"s3://suno-data/datasets/harvest/pond5_music/pond5_metas.jsonl\", read_f=read_jsonl)" ] }, { "cell_type": "code", "execution_count": 16, "id": "b197bc37", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T03:59:44.793480Z", "start_time": "2024-01-18T03:59:44.791972Z" } }, "outputs": [], "source": [ "# TODO: anything we wanna filter out??\n", "# allowed_ids" ] }, { "cell_type": "code", "execution_count": 17, "id": "e48d6ff8", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T03:59:53.949941Z", "start_time": "2024-01-18T03:59:44.794964Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 2072969/2072969 [00:09<00:00, 228483.86it/s]\n" ] } ], "source": [ "extra_info = {}\n", "seen_ids = set()\n", "genre_counter = collections.Counter()\n", "for m in tqdm.tqdm(pd5_extra_metas):\n", " if \"s3_filepath\" not in m:\n", " continue\n", " if m[\"id\"] in seen_ids:\n", " continue\n", " if m[\"duration\"] < 30:\n", " continue\n", " if m[\"duration\"] > 60 * 4:\n", " continue\n", " name = m.get(\"name\", \"\")\n", " description = m.get(\"description\", \"\")\n", " tags = m.get(\"tags\", [])\n", " genre = m.get(\"genre\", \"\")\n", " if not genre:\n", " continue\n", " genre_counter[genre.lower().strip()] += 1\n", " if not description or not tags or not name:\n", " continue\n", " if len(description.strip().split()) < 10:\n", " continue\n", " if len(description.strip().split()) > 100:\n", " continue\n", " if not isinstance(tags, list) or len(tags) < 5 or len(tags) > 50:\n", " continue\n", " # if there are weird tags\n", " if max(len(tag) for tag in tags) > 30:\n", " continue\n", " extra_info[m[\"id\"]] = {\n", " \"name\": name if isinstance(name, str) else \"\",\n", " \"description\": (description if isinstance(description, str) else \"\"),\n", " \"tags\": ([genre] if genre else []) + tags if isinstance(tags, list) else \"\",\n", " }\n", " seen_ids.add(m[\"id\"])" ] }, { "cell_type": "code", "execution_count": 18, "id": "34b7933e", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T03:59:53.953880Z", "start_time": "2024-01-18T03:59:53.951191Z" } }, "outputs": [ { "data": { "text/plain": [ "(2072969,\n", " 836127,\n", " [('medium tempo', 572064),\n", " ('slow tempo', 155173),\n", " ('fast tempo', 135355),\n", " ('available for musical works', 55097),\n", " ('seamless looping', 38962),\n", " ('summer', 6448),\n", " ('2000s', 5042),\n", " ('slow motion', 3937),\n", " ('night', 3392),\n", " ('lyrics', 3146)])" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(pd5_extra_metas), len(extra_info), genre_counter.most_common(10)" ] }, { "cell_type": "code", "execution_count": 19, "id": "a2e0d453", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T04:00:17.419213Z", "start_time": "2024-01-18T03:59:53.954970Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████| 2068743/2068743 [00:21<00:00, 94358.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "281,609 entries, total duration 8,709 hr\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "metas = []\n", "total_duration = 0\n", "n_c =0\n", "for m in tqdm.tqdm(pd5_base_metas):\n", "# if n_c > 5:\n", "# break\n", " if m[\"id\"] not in seen_ids:\n", " continue\n", " if random.random() < 0.66:\n", " continue\n", " n_c += 1\n", " info = extra_info[m[\"id\"]]\n", " new_m = {\"id\": m[\"id\"]}\n", " # print(m, info)\n", " tags = [\n", " info[\"name\"],\n", " info[\"description\"],\n", " ]\n", " tags.extend(info[\"tags\"])\n", " tags = [_clean_tag(t) for t in tags]\n", " tags = [t for t in tags if len(t) > 0]\n", " if len(tags) > 10:\n", " new_m[\"tags\"] = list(set(tags))\n", " metas.append(new_m)\n", " total_duration += m[\"duration_s\"]\n", "print(f\"{len(metas):,} entries, total duration {total_duration/60/60:,.0f} hr\")" ] }, { "cell_type": "code", "execution_count": 20, "id": "a2dd2c7d", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T04:00:21.174813Z", "start_time": "2024-01-18T04:00:17.420467Z" } }, "outputs": [], "source": [ "write_jsonl(metas, os.path.join(METAS_DIR, \"pond5_music_v1.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "d1b2f3fc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bced417a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "bb193df8", "metadata": {}, "source": [ "## deezer" ] }, { "cell_type": "code", "execution_count": 13, "id": "d27c1bd4", "metadata": {}, "outputs": [], "source": [ "base_metas = read_from_s3(\"s3://suno-data/datasets/bundles/v2/deezer/metas_plus.jsonl\", read_f=read_jsonl)" ] }, { "cell_type": "code", "execution_count": 50, "id": "b3209b25", "metadata": {}, "outputs": [], "source": [ "# TODO: use lyrics\n", "# TODO: get youtube title for private tags" ] }, { "cell_type": "code", "execution_count": null, "id": "23c774c7", "metadata": {}, "outputs": [], "source": [ "# TODO: use aligned lyrics" ] }, { "cell_type": "code", "execution_count": 197, "id": "5a8b74b0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "768,515 clips\n", "535,018 music clips\n" ] } ], "source": [ "print(f\"{len(base_metas):,} clips\")\n", "allowed_ids = set()\n", "for m in base_metas:\n", " # TODO: the threshold here is low...\n", " if m[\"views\"] < 50:\n", " continue\n", "# if re.search(r\"\\blive\\b\", m[\"title\"], flags=re.IGNORECASE):\n", "# continue\n", " allowed_ids.add(m[\"id\"])\n", "print(f\"{len(allowed_ids):,} music clips\")\n", "# 768,515 clips\n", "# 535,018 music clips" ] }, { "cell_type": "code", "execution_count": 199, "id": "8c0a5602", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "508,804 entries\n", "508,804 entries with lyrics\n" ] } ], "source": [ "metas = []\n", "seen_ids = set()\n", "for m in base_metas:\n", " if m[\"id\"] in seen_ids or m[\"id\"] not in allowed_ids:\n", " continue\n", " if (\n", " \"lang_guess\" not in m or \n", " len(m[\"lang_guess\"]) < 2 or\n", " \"lyrics\" not in m or \n", " len(m[\"lyrics\"]) > 5120 # empirically verified\n", " ):\n", " continue\n", " if m[\"lang_guess\"][:2] in (\"zh\", \"ja\"):\n", " if len(m[\"lyrics\"]) < 256:\n", " continue\n", " else:\n", " if len(m[\"lyrics\"]) < 512:\n", " continue\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"views\": m[\"views\"],\n", " \"text\": m[\"lyrics\"],\n", " \"lang\": m[\"lang_guess\"].split(\"-\")[0].lower(),\n", " }\n", " # get private tags\n", " private_tags = [\n", " m.get(\"youtube_title\"),\n", " m.get(\"youtube_artist\"),\n", " ]\n", " private_tags = [t_clean for t in private_tags if len(t_clean:=_clean_tag(t)) > 0]\n", " if len(private_tags) > 0:\n", " new_m[\"private_tags\"] = list(set(private_tags))\n", " metas.append(new_m)\n", " seen_ids.add(m[\"id\"])\n", "print(f\"{len(metas):,} entries\")\n", "# 508,804 entries" ] }, { "cell_type": "code", "execution_count": 200, "id": "1adfd6d9", "metadata": {}, "outputs": [], "source": [ "write_jsonl(metas, os.path.join(METAS_DIR, \"deezer.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "2ebe9b7e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8768446b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6a850869", "metadata": {}, "source": [ "## ytm_tagged" ] }, { "cell_type": "code", "execution_count": 206, "id": "18304cc8", "metadata": {}, "outputs": [], "source": [ "base_metas = read_from_s3(\"s3://suno-data/datasets/bundles/v2/ytm_tagged/metas.jsonl\", read_f=read_jsonl)" ] }, { "cell_type": "code", "execution_count": 228, "id": "8076d9ce", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2,772,393 clips\n", "2,255,305 music clips\n" ] } ], "source": [ "print(f\"{len(base_metas):,} clips\")\n", "allowed_ids = set()\n", "for m in base_metas:\n", " if m[\"views\"] < 100:\n", " continue\n", "# if re.search(r\"\\blive\\b\", m[\"title\"], flags=re.IGNORECASE):\n", "# continue\n", " allowed_ids.add(m[\"id\"])\n", "print(f\"{len(allowed_ids):,} music clips\")\n", "# 2,772,393 clips\n", "# 2,255,305 music clips" ] }, { "cell_type": "code", "execution_count": 229, "id": "4474ec44", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2,255,305 entries\n" ] } ], "source": [ "metas = []\n", "seen_ids = set()\n", "for m in base_metas:\n", " if m[\"id\"] in seen_ids or m[\"id\"] not in allowed_ids:\n", " continue\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"views\": m[\"views\"],\n", " }\n", " # get tags\n", " tags = m.get(\"tags\", [])\n", " tags = [t_clean for t in tags if len(t_clean:=_clean_tag(t)) > 0]\n", " if len(tags) > 0:\n", " new_m[\"tags\"] = list(set(tags))\n", " # get private tags\n", " private_tags = m.get(\"private_tags\", [])\n", " private_tags = [t_clean for t in private_tags if len(t_clean:=_clean_tag(t)) > 0]\n", " if len(private_tags) > 0:\n", " new_m[\"private_tags\"] = list(set(private_tags))\n", " metas.append(new_m)\n", " seen_ids.add(m[\"id\"])\n", "print(f\"{len(metas):,} entries\")\n", "# 2,255,305 entries" ] }, { "cell_type": "code", "execution_count": 234, "id": "8f2c4f3f", "metadata": {}, "outputs": [], "source": [ "write_jsonl(metas, os.path.join(METAS_DIR, \"ytm_tagged.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "10faf712", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "cf0b6253", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d569d249", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "523dc5af", "metadata": {}, "source": [ "## musescore" ] }, { "cell_type": "code", "execution_count": 1, "id": "7ea44d0a", "metadata": {}, "outputs": [], "source": [ "# TODO: add language using classifier" ] }, { "cell_type": "code", "execution_count": 10, "id": "5423b231", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3,219 music clips\n", "204.3 hours\n" ] } ], "source": [ "base_metas = read_from_s3(\"s3://suno-data/datasets/bundles/v2/musescore/metas.jsonl\", read_f=read_jsonl)\n", "print(f\"{len(base_metas):,} music clips\")\n", "print(f\"{sum([m['duration_s'] for m in base_metas])/60/60:,.1f} hours\")" ] }, { "cell_type": "code", "execution_count": 11, "id": "cc11999c", "metadata": {}, "outputs": [], "source": [ "import re\n", "import funcy\n", "\n", "def _max_notes_sub(m, max_notes=5):\n", " return \"[\" + \";\".join(m.group(1).split(\";\")[:max_notes]) + \"]\"\n", "\n", "def create_pitched_lyrics(notes_metas, max_notes=None, include_duration=False):\n", " is_word_bound = True\n", " text = \"\"\n", " for m in notes_metas:\n", " if \"text\" in m:\n", " if m[\"text_part\"] == \"begin\":\n", " is_word_bound = True\n", " if is_word_bound:\n", " text += \" \"\n", " text += m[\"text\"]\n", " if m[\"text_part\"] in {\"single\", \"end\"}:\n", " is_word_bound = True\n", " else:\n", " is_word_bound = False\n", " pitch_str = f\"{m['pitch']}\"\n", " if include_duration:\n", " pitch_str += f\"_{m['duration']}\"\n", " text += f\"[{pitch_str}]\"\n", " text = text.replace(\"][\", \";\").strip()\n", " if max_notes is not None:\n", " # add max ~5 notes in a bracket?\n", " _max_notes_sub_p = funcy.partial(_max_notes_sub, max_notes=max_notes)\n", " text = re.sub(r\"\\[(.*?)\\]\", _max_notes_sub_p, text)\n", " return text" ] }, { "cell_type": "code", "execution_count": 16, "id": "0b7c213e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████| 3219/3219 [00:01<00:00, 2464.96it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "3,219 clips\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "metas = []\n", "for m in tqdm.tqdm(base_metas):\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " }\n", " # get notes/text\n", " text = create_pitched_lyrics(m[\"notes\"], max_notes=5, include_duration=True)\n", " new_m[\"text\"] = text\n", " metas.append(new_m)\n", "print(f\"{len(metas):,} clips\")" ] }, { "cell_type": "code", "execution_count": 18, "id": "da89cdcd", "metadata": {}, "outputs": [], "source": [ "write_jsonl(metas, os.path.join(METAS_DIR, \"musescore.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "fefebdd5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "979ee4d9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "7380e289", "metadata": {}, "source": [ "## ytm_mb" ] }, { "cell_type": "code", "execution_count": null, "id": "344672f6", "metadata": {}, "outputs": [], "source": [ "base_metas = read_from_s3(\"s3://suno-data/datasets/bundles/v2/ytm_mb/metas.jsonl\", read_f=read_jsonl)" ] }, { "cell_type": "code", "execution_count": null, "id": "a389bde9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "956ea735", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1d5ae207", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "311d6bdc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "76e67767", "metadata": {}, "source": [ "## Make tokenizer" ] }, { "cell_type": "code", "execution_count": 1, "id": "ce26faf9", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import tqdm\n", "import random\n", "from collections import Counter\n", "\n", "from suno_utils.utils.text import read_jsonl\n", "\n", "from transformers import PreTrainedTokenizerFast\n", "from transformers import BertTokenizerFast\n", "from tokenizers import (\n", " decoders,\n", " models,\n", " normalizers,\n", " pre_tokenizers,\n", " processors,\n", " trainers,\n", " Tokenizer,\n", ")\n", "\n", "METAS_DIR = os.path.join(os.getcwd(), \"metadata\")" ] }, { "cell_type": "code", "execution_count": 2, "id": "5a5035b2", "metadata": {}, "outputs": [], "source": [ "# TODO: do we want more augment?" ] }, { "cell_type": "code", "execution_count": 3, "id": "940e780b", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████| 8/8 [11:33<00:00, 86.75s/it]\n" ] } ], "source": [ "# create tokenizer data\n", "\n", "def _augment_case(s):\n", " if random.random() >= 0.95:\n", " s = s.upper()\n", " elif random.random() >= 0.95:\n", " s = s.lower()\n", " return s\n", "\n", "def _augment_tag(s):\n", " if random.random() >= 0.95:\n", " s = s.upper()\n", " elif random.random() >= 0.95:\n", " s = s.capitalize()\n", " elif random.random() >= 0.9:\n", " s = s.title()\n", " elif random.random() >= 0.9:\n", " s = s.lower()\n", " if random.random() >= 0.5:\n", " s = s.replace(\"-\", \" \").strip()\n", " return s\n", "\n", "def _space_repl(m):\n", " s = m.group()\n", " n_newline = s.count(\"\\n\")\n", " if n_newline >= 2:\n", " return \"\\n\\n\"\n", " elif n_newline == 1:\n", " return \"\\n\"\n", " return \" \"\n", "\n", "\n", "def _simplify_whitespace(text, retain_newlines=True):\n", " \"\"\"simplify while respecting up to 2 newlines\"\"\"\n", " if retain_newlines:\n", " text = re.sub(r\"\\s+\", _space_repl, text).strip()\n", " else:\n", " text = re.sub(r\"\\s+\", \" \", text).strip()\n", " return text\n", "\n", "with open(\"data/tokenizer_train.txt\", \"w\") as f:\n", " for fn in tqdm.tqdm([\n", " \"genius_hq\",\n", " \"youtube_music\",\n", " \"freesound\",\n", " \"jamendo\",\n", " \"imslp\",\n", " \"pond5_music\",\n", " \"deezer\",\n", " \"ytm_tagged\",\n", " ]):\n", " for m in read_jsonl(os.path.join(METAS_DIR, f\"{fn}.jsonl\")):\n", " text_key = (\n", " \"text\" if \"private_text\" not in m or random.random() >= 0.2 else \"private_text\"\n", " )\n", " if text_key in m:\n", " f.write(\n", " _simplify_whitespace(_augment_case(m[text_key])) + \n", " \"\\n\"\n", " )\n", " text_key = (\n", " \"text_segments\" \n", " if \"private_text_segments\" not in m or random.random() >= 0.2 else \"private_text_segments\"\n", " )\n", " if text_key in m:\n", " for mm in m[text_key]:\n", " f.write(\n", " _simplify_whitespace(_augment_case(mm[\"text\"])) + \n", " \"\\n\"\n", " )\n", " text_key = (\n", " \"tags\" if \"private_tags\" not in m or random.random() >= 0.2 else \"private_tags\"\n", " )\n", " if text_key in m:\n", " join_char = random.choice([\", \", \"; \", \" \"])\n", " f.write(\n", " _simplify_whitespace(join_char.join([_augment_tag(t) for t in m[text_key]])) + \n", " \"\\n\"\n", " )" ] }, { "cell_type": "code", "execution_count": 4, "id": "2b844506", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "6.7G\tdata/tokenizer_train.txt\r\n" ] } ], "source": [ "!du -hs data/tokenizer_train.txt\n", "# 6.7G" ] }, { "cell_type": "code", "execution_count": 5, "id": "39701df2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "15462 unique chars\n", "13045 unique chars after filtering\n" ] } ], "source": [ "min_char_n = 3\n", "with open(\"data/tokenizer_train.txt\") as f:\n", " corpus = f.read()\n", "char_counts = Counter(corpus)\n", "print(len(char_counts), \"unique chars\")\n", "trans_table = str.maketrans({k: \" \" for k, v in char_counts.items() if v < min_char_n})\n", "corpus = corpus.translate(trans_table)\n", "char_counts_2 = Counter(corpus)\n", "print(len(char_counts_2), \"unique chars after filtering\")\n", "vocab = [c for c, _ in char_counts_2.most_common()]\n", "corpus_lines = corpus.split(\"\\n\")\n", "corpus_lines = [line_clean for line in corpus_lines if len(line_clean:=line.strip()) > 0]\n", "random.seed(6006)\n", "sample_corpus_lines = random.choices(corpus_lines, k=1_000_000)\n", "with open(\"data/tokenizer_train_clean.txt\", \"w\") as f:\n", " for line in corpus_lines:\n", " f.write(line + \"\\n\")\n", "# del corpus, corpus_lines\n", "# gc.collect()" ] }, { "cell_type": "code", "execution_count": 6, "id": "ff176544", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['I like the way your lips move when you look at me',\n", " 'How many shots will it take to make son drop?',\n", " 'And hearts will be glowing when loved ones are near',\n", " 'वो ज़माना याद है',\n", " '[Chorus]',\n", " 'Я как-то бежал наобум, все вокруг говорило: «How do you do?» (А-а-а)',\n", " 'Seiso',\n", " 'Ma sta merda so che mi',\n", " 'Que saudade dos mano, vivência de cria',\n", " \"I don't know where to start\"]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sample_corpus_lines[:10]" ] }, { "cell_type": "code", "execution_count": null, "id": "54c6f464", "metadata": {}, "outputs": [], "source": [ "# add newline to tokenizer\n", "# don't tokenize across newline or whitespace" ] }, { "cell_type": "code", "execution_count": null, "id": "8889ef74", "metadata": {}, "outputs": [], "source": [ "# TODO: define certain tokens to always be by themselves like brackets?" ] }, { "cell_type": "markdown", "id": "8348f908", "metadata": {}, "source": [ "#### Train and save" ] }, { "cell_type": "code", "execution_count": 8, "id": "ba2e418b", "metadata": {}, "outputs": [], "source": [ "tokenizer = Tokenizer(models.WordPiece(unk_token=\"[UNK]\"))\n", "tokenizer.normalizer = normalizers.BertNormalizer(lowercase=False)\n", "tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()\n", "special_tokens = [\"[UNK]\", \"[PAD]\"]\n", "trainer = trainers.WordPieceTrainer(vocab_size=60_000, special_tokens=special_tokens)" ] }, { "cell_type": "code", "execution_count": 9, "id": "7ffeee5c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "\n", "\n" ] } ], "source": [ "tokenizer.train([\"data/tokenizer_train_clean.txt\"], trainer=trainer)" ] }, { "cell_type": "code", "execution_count": 10, "id": "e6ea2c0d", "metadata": {}, "outputs": [], "source": [ "# tokenizer.save(\"tmp/tokenizer_60k.json\")" ] }, { "cell_type": "code", "execution_count": null, "id": "5a5e52f6", "metadata": {}, "outputs": [], "source": [ "# !aws s3 cp tmp/tokenizer_60k.json s3://suno-data/georg/trained_models/chirp_v2/tokenizer_60k.json" ] }, { "cell_type": "code", "execution_count": null, "id": "962cdd79", "metadata": {}, "outputs": [], "source": [ "wrapped_tokenizer = PreTrainedTokenizerFast(\n", " tokenizer_object=tokenizer,\n", "# tokenizer_file=\"tmp/tokenizer_60k.json\", # You can load from the tokenizer file, alternatively\n", " unk_token=\"[UNK]\",\n", " pad_token=\"[PAD]\",\n", ")" ] }, { "cell_type": "code", "execution_count": 17, "id": "63561243", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[33356, 18, 33356, 19]" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "wrapped_tokenizer.encode(\"Line 1\\nLine 2\")" ] }, { "cell_type": "code", "execution_count": 36, "id": "62202a2a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from tokenizers import AddedToken\n", "wrapped_tokenizer.add_special_tokens({\"additional_special_tokens\": [AddedToken(\"\\n\")]})" ] }, { "cell_type": "code", "execution_count": 37, "id": "6ede2fee", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[33356, 18, 60000, 33356, 19]" ] }, "execution_count": 37, "metadata": {}, "output_type": "execute_result" } ], "source": [ "wrapped_tokenizer.encode(\"Line 1\\nLine 2\")" ] }, { "cell_type": "code", "execution_count": 38, "id": "2bf2c78e", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[33356, 18, 60000, 33356, 19]" ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "wrapped_tokenizer.encode(\"Line 1\\n Line 2\")" ] }, { "cell_type": "code", "execution_count": 18, "id": "fe3e3e9f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "10289032" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "encodings = wrapped_tokenizer(sample_corpus_lines)\n", "sum([len(e) for e in encodings[\"input_ids\"]])" ] }, { "cell_type": "markdown", "id": "75a3b18c", "metadata": {}, "source": [ "#### compare to bert" ] }, { "cell_type": "code", "execution_count": 19, "id": "05f25d40", "metadata": {}, "outputs": [], "source": [ "BERT_TOKENIZER = BertTokenizerFast.from_pretrained(\"bert-base-multilingual-cased\")" ] }, { "cell_type": "code", "execution_count": 20, "id": "1419d7d8", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Token indices sequence length is longer than the specified maximum sequence length for this model (939 > 512). Running this sequence through the model will result in indexing errors\n" ] }, { "data": { "text/plain": [ "13776455" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "encodings = BERT_TOKENIZER(sample_corpus_lines)\n", "sum([len(e) for e in encodings[\"input_ids\"]])" ] }, { "cell_type": "code", "execution_count": 21, "id": "59fce8e4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'input_ids': [[42, 19183, 19079, 19362, 19150, 22702, 20276, 19313, 19078, 19509, 19278, 19089], [20080, 20531, 23290, 19386, 19108, 19464, 19092, 19381, 20001, 20699, 32]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "wrapped_tokenizer(sample_corpus_lines[:2])" ] }, { "cell_type": "code", "execution_count": 35, "id": "549a9141", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'👅 🍭'" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "wrapped_tokenizer.decode(wrapped_tokenizer.encode(\"👅 🍭\"))" ] }, { "cell_type": "code", "execution_count": 72, "id": "7a4455d7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 2.46 s, sys: 0 ns, total: 2.46 s\n", "Wall time: 2.46 s\n" ] } ], "source": [ "%%time\n", "for _ in range(1000):\n", " for _ in range(2):\n", " _ = BERT_TOKENIZER(line)" ] }, { "cell_type": "code", "execution_count": 73, "id": "b7a9cb06", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 4.76 s, sys: 0 ns, total: 4.76 s\n", "Wall time: 1.64 s\n" ] } ], "source": [ "%%time\n", "for _ in range(1000):\n", " _ = BERT_TOKENIZER([line]*2, return_tensors=\"pt\")" ] }, { "cell_type": "code", "execution_count": 82, "id": "1871108c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[ 101, 10531, 102, 0, 0, 0, 0, 0, 0, 0],\n", " [ 101, 10111, 10531, 102, 0, 0, 0, 0, 0, 0]])" ] }, "execution_count": 82, "metadata": {}, "output_type": "execute_result" } ], "source": [ "BERT_TOKENIZER(\n", " [\"this\", \"and this\"], \n", " return_tensors=\"pt\", \n", " truncation=True, \n", " max_length=10,\n", " padding=\"max_length\"\n", ")[\"input_ids\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "3367157c", "metadata": {}, "outputs": [], "source": [ "# # compare tokens for different vocab sizes\n", "# 10619558 - 50k\n", "# 9926038 - 100k\n", "# 9806614 - 120k\n", "# 13753621 - 120k (bert)" ] }, { "cell_type": "markdown", "id": "1c5edf72", "metadata": {}, "source": [ "### Test Loading" ] }, { "cell_type": "code", "execution_count": 1, "id": "f7522373", "metadata": {}, "outputs": [], "source": [ "from transformers import PreTrainedTokenizerFast\n", "from tokenizers import AddedToken\n", "\n", "custom_tokenizer = PreTrainedTokenizerFast(\n", " tokenizer_file=\"tmp/tokenizer_60k.json\",\n", " unk_token=\"[UNK]\",\n", " pad_token=\"[PAD]\",\n", ")\n", "custom_tokenizer.add_special_tokens({\"additional_special_tokens\": [AddedToken(\"\\n\")]});" ] }, { "cell_type": "code", "execution_count": 10, "id": "72a11f6d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[33356, 18, 60000, 33356, 19]" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "custom_tokenizer.encode(\"Line 1\\n Line 2\")" ] }, { "cell_type": "code", "execution_count": null, "id": "8f3b5378", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ff0fc103", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "da9e69d9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "d273a78b", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 2, "id": "b75d5305", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"" ] }, { "cell_type": "code", "execution_count": 25, "id": "a3983b8f", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "from suno_utils.utils.text import read_jsonl, read_json" ] }, { "cell_type": "code", "execution_count": 4, "id": "7663d683", "metadata": {}, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(\"/mnt/data/georg/data/chirp_v1/combo_val.bin\", dtype=np.uint16, mode=\"r\")\n", "metas = read_jsonl(\"/mnt/data/georg/data/chirp_v1/combo_metas_val.jsonl\")\n", "mm = mm.reshape(-1, 3375, 9)\n", "assert(len(mm) == len(metas))\n", "assert(mm[:100,:,0].min() >= 0)\n", "assert(mm[:100,:,0].max() <= 1000+1)\n", "assert(mm[:100,:,1:].min() >= 0)\n", "assert(mm[:100,:,1:].max() <= 4096+2)\n", "assert(mm[:,1::3,1:].min() == 4096)\n", "assert(mm[:,2::3,1:].min() == 4096)" ] }, { "cell_type": "code", "execution_count": 5, "id": "b65ea3de", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "6200\n" ] } ], "source": [ "# randomly listen to some stuff\n", "from suno_utils.tasks.dac import decode as codec_decode, EMBEDDING_RATE as CODEC_EMBEDDING_RATE\n", "mm = np.memmap(\"/mnt/data/georg/data/chirp_v1/combo_val.bin\", dtype=np.uint16, mode=\"r\")\n", "mm = mm.reshape(-1, 3375, 9)\n", "test_metas = read_jsonl(\"/mnt/data/georg/data/chirp_v1/combo_metas_val.jsonl\")\n", "test_info = read_json(\"/mnt/data/georg/data/chirp_v1/combo_info_val.json\")\n", "assert(len(test_metas) == len(mm))\n", "idx_list = list(range(len(test_metas)))\n", "#random.shuffle(idx_list)\n", "#idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "print(len(mm))" ] }, { "cell_type": "code", "execution_count": 23, "id": "72b96934", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "combo_info_tr.json combo_metas_tr.jsonl combo_tr.bin\r\n", "combo_info_val.json combo_metas_val.jsonl combo_val.bin\r\n" ] } ], "source": [ "!ls /mnt/data/georg/data/chirp_v1" ] }, { "cell_type": "code", "execution_count": 6, "id": "ca145de1", "metadata": {}, "outputs": [], "source": [ "from suno_utils.tasks.dac import preload_models as preload_codec_models\n", "_ = preload_codec_models(\"/mnt/data/georg/tmp/d_codec_25x8.pt\")" ] }, { "cell_type": "code", "execution_count": 45, "id": "1a8d5790", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tags: ['Rap']\n", "text: None\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import random\n", "# show text and audio\n", "# idx = random.choice(idx_list)\n", "idx = random.choice(test_info[\"genius_hq_foreign\"][\"idx_list\"])\n", "# print(len(test_metas[idx][\"tokens\"]), \"tokens\")\n", "print(\"tags:\", test_metas[idx].get(\"tags\"))\n", "print(\"text:\", test_metas[idx].get(\"text\"))\n", "arr = mm[idx,1:].copy().astype(np.int16)[:,1:]\n", "pad_idx_arr = np.where(arr == 4096)[0]\n", "if len(pad_idx_arr) > 0:\n", " arr = arr[:pad_idx_arr[0],:]\n", "codec_decode(arr[2::3]).play()" ] }, { "cell_type": "code", "execution_count": null, "id": "0fe02f5f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7ab96f2e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "890ee199", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }