{
"cells": [
{
"cell_type": "code",
"execution_count": 6,
"id": "15416422",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"import re\n",
"import random\n",
"import shutil\n",
"import uuid\n",
"import multiprocessing\n",
"from copy import deepcopy\n",
"\n",
"import numpy as np\n",
"import pandas as pd\n",
"import tqdm\n",
"\n",
"from suno_utils.audio import Audio\n",
"from suno_utils.audio.conversion import get_duration_s\n",
"from suno_utils.utils.text import normalize_whitespace\n",
"\n",
"DATA_DIR = \"/data/suno/data/academia/rap_de\"\n",
"AUDIO_DIR = os.path.join(DATA_DIR, \"audio\")"
]
},
{
"cell_type": "markdown",
"id": "264fd819",
"metadata": {},
"source": [
"## convert audio"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "19765333",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"28674 items\n"
]
}
],
"source": [
"RAW_DATA_DIR = os.path.join(DATA_DIR, \"raw_data\")\n",
"\n",
"ALLOWED_AUDIO_EXTENSION = set([\"webm\", \"m4a\"])\n",
"\n",
"queue_items = []\n",
"for fn in os.listdir(RAW_DATA_DIR):\n",
" file_ext = fn.split(\".\")[-1]\n",
" if file_ext not in ALLOWED_AUDIO_EXTENSION:\n",
" continue\n",
" file_id = fn[:-len(file_ext)-1]\n",
" from_fp = os.path.join(RAW_DATA_DIR, fn)\n",
" to_fp = os.path.join(AUDIO_DIR, file_id + \".wav\")\n",
" queue_items.append((from_fp, to_fp))\n",
"print(len(queue_items), \"items\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "fa228472",
"metadata": {},
"outputs": [],
"source": [
"shutil.rmtree(AUDIO_DIR, ignore_errors=True)\n",
"os.makedirs(AUDIO_DIR, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "5b67783f",
"metadata": {},
"outputs": [],
"source": [
"def _convert_audio(queue_item):\n",
" from_fp, to_fp = queue_item\n",
" try:\n",
" Audio.from_file(from_fp, sample_rate=16_000, byte_width=2).to_wav(to_fp)\n",
" except:\n",
" pass\n",
" \n",
"p = multiprocessing.Pool(10)\n",
"_ = p.map(_convert_audio, queue_items, chunksize=1)\n",
"p.close()\n",
"p.join()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "f1ddf23c",
"metadata": {},
"outputs": [],
"source": [
"# !ls -1 /data/suno/data/academia/rap_de/audio | wc -l"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "89732a2a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1544.4 hours of audio.\n",
"1402.0 hours of song audio.\n"
]
}
],
"source": [
"durations_s = []\n",
"for fn in os.listdir(AUDIO_DIR):\n",
" durations_s.append(get_duration_s(os.path.join(AUDIO_DIR, fn)))\n",
"s = pd.Series(durations_s)\n",
"print(round(s.sum() / 60 / 60, 1), \"hours of audio.\")\n",
"print(round(s[s<=5*60].sum() / 60 / 60, 1), \"hours of song audio.\")"
]
},
{
"cell_type": "markdown",
"id": "5a3a707b",
"metadata": {},
"source": [
"## assemble pairs"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "61955220",
"metadata": {},
"outputs": [],
"source": [
"youtube_audio_id_set = set()\n",
"for fn in os.listdir(AUDIO_DIR):\n",
" if fn.endswith(\".wav\"):\n",
" youtube_audio_id_set.add(fn[:-4])"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "b5b610b7",
"metadata": {},
"outputs": [],
"source": [
"youtube_info_map = {}\n",
"with open(os.path.join(DATA_DIR, \"youtube_metas.jsonl\")) as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" m = json.loads(line)\n",
" if m is None or not m[\"success\"] or m[\"id\"] not in youtube_audio_id_set:\n",
" continue\n",
" youtube_info_map[m[\"id\"]] = {\n",
" \"id\": m[\"id\"],\n",
" \"views\": m[\"meta\"][\"view_count\"],\n",
" \"likes\": m[\"meta\"].get(\"like_count\"),\n",
" \"title\": m[\"meta\"][\"title\"],\n",
" \"artist\": m[\"meta\"].get(\"artist\"),\n",
" \"album\": m[\"meta\"].get(\"album\"),\n",
" \"is_live\": m[\"meta\"][\"is_live\"],\n",
" \"description\": m[\"meta\"][\"description\"],\n",
" \"categories\": m[\"meta\"][\"categories\"],\n",
" \"tags\": m[\"meta\"][\"tags\"],\n",
" \"release_date\": m[\"meta\"].get(\"release_date\"),\n",
" }"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "743d58b8",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"26024 song details fetched.\n"
]
}
],
"source": [
"song_metas = []\n",
"with open(os.path.join(DATA_DIR, \"song_metas.jsonl\")) as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" m = json.loads(line)\n",
" if m is None:\n",
" continue\n",
" youtube_id = m[\"youtube_url\"].split(\"=\")[-1]\n",
" if (\n",
" youtube_id not in youtube_audio_id_set or \n",
" int(m[\"youtube_start\"]) != 0 or\n",
" m[\"lang\"] != \"de\" or\n",
" m[\"lang_lyrics\"] != \"de\" or\n",
" youtube_id not in youtube_info_map\n",
" ):\n",
" continue\n",
" song_metas.append({\n",
" \"uri\": os.path.join(AUDIO_DIR, f\"{youtube_id}.wav\"),\n",
" \"duration_s\": get_duration_s(os.path.join(AUDIO_DIR, f\"{youtube_id}.wav\")),\n",
" \"genius_info\": {\n",
" \"id\": m[\"id\"],\n",
" \"slug\": m[\"slug\"],\n",
" \"views\": m[\"views\"],\n",
" \"tags\": m[\"tags\"],\n",
" },\n",
" \"youtube_info\": youtube_info_map[youtube_id],\n",
" \"lyrics\": m[\"lyrics\"],\n",
" })\n",
"print(len(song_metas), \"song details fetched.\")"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "cce025ba",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(DATA_DIR, \"pair_metas.jsonl\"), \"w\") as f:\n",
" for m in song_metas:\n",
" f.write(json.dumps(m) + \"\\n\")"
]
},
{
"cell_type": "markdown",
"id": "800430dc",
"metadata": {},
"source": [
"## investigate and filter"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "7e1f1775",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"26024 song details.\n",
"1400.3 hours of audio.\n"
]
}
],
"source": [
"song_metas = []\n",
"with open(os.path.join(DATA_DIR, \"pair_metas.jsonl\")) as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" song_metas.append(json.loads(line))\n",
"print(len(song_metas), \"song details.\")\n",
"print(round(np.sum([m[\"duration_s\"] for m in song_metas]) / 60 / 60, 1), \"hours of audio.\")"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "80916a09",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"20099 song details.\n",
"1061.5 hours of audio.\n"
]
}
],
"source": [
"def _parse_lyrics_lines(lyrics):\n",
" lines = []\n",
" for line in lyrics.split(\"\\n\"):\n",
" line = line.strip()\n",
" if \"[\" in line:\n",
" line = re.sub(r\"\\[.*?\\]\", \" \", line)\n",
" line = re.sub(r\"\\(.*?\\)\", \" \", line)\n",
" line = normalize_whitespace(line)\n",
" if len(line) == 0:\n",
" continue\n",
" lines.append(line)\n",
" return lines\n",
"\n",
"filtered_song_metas = []\n",
"for m in song_metas:\n",
" m = deepcopy(m)\n",
" if m[\"duration_s\"] < 30 or m[\"duration_s\"] > 8 * 60:\n",
" continue\n",
" if m[\"genius_info\"][\"views\"] < 20:\n",
" continue\n",
" if len(set(m[\"genius_info\"][\"tags\"]) & set([1658, 2895])) == 0: # Deutschland, Oesterreich\n",
" continue\n",
" if (\n",
" re.search(r\"^\\s*\\[.*\\b2x\\b\", m[\"lyrics\"]) or # repetitions are hard to parse\n",
" re.search(r\"\\[[\\s\\?]+\\]\", m[\"lyrics\"]) # some have [?] - unintelligible\n",
" ):\n",
" continue\n",
" lyrics_lines = _parse_lyrics_lines(m[\"lyrics\"])\n",
" m[\"lyrics_lines\"] = lyrics_lines\n",
" fulltext = \" \".join(lyrics_lines)\n",
" if len(re.findall(r\"[a-zA-Z]\", fulltext)) < 250 or len(fulltext) > 6000:\n",
" continue\n",
" upper_frac = len(re.findall(r\"[A-Z]\", fulltext)) / len(re.findall(r\"[a-zA-Z]\", fulltext))\n",
" if upper_frac <= 0.01 or upper_frac >= 0.8:\n",
" continue\n",
" filtered_song_metas.append(m)\n",
"print(len(filtered_song_metas), \"song details.\")\n",
"print(round(np.sum([m[\"duration_s\"] for m in filtered_song_metas]) / 60 / 60, 1), \"hours of audio.\")"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "b6170d7c",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(DATA_DIR, \"filtered_pair_metas.jsonl\"), \"w\") as f:\n",
" for m in filtered_song_metas:\n",
" f.write(json.dumps(m) + \"\\n\")"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "8c24c596",
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"-- 6889 -- Du-maroc-ap-mit-brilliz-lyrics -- 325\n",
"\n",
"Ey, ey\n",
"Die Cops halten mich an\n",
"Ich sitze bei Nacht in 'nem schwarzen Lamborghini\n",
"Ey, sie fragen, was ich mach'\n",
"Wohin ich geh' und ich sag', „Zur Familie“\n",
"Achtet auf die AP mit Brillis\n",
"Ich acht' auf, dass er nix mitkriegt von meiner Neunmilli\n",
"Nicht registriert wie der Paddy, ich bin auf Rillé\n",
"Deutsch-Rapper sitzen\n",
"Jeder Zweite um dich rum ist V-Mann\n",
"Ekho, sag mir, wem du noch vertrauen kannst\n",
"Seh' ich um mich rum, seh' ich nur Auge\n",
"Sag, wem von dein'n Brüdern kannst du glauben?\n",
"Paar Riesen im Spiel, es wird\n",
"Eklig hier draußen, Akhi inhaftiert\n",
"Warte auf Zahltag bis etwas passiert\n",
"Du weißt, es kann immer alles passier'n\n",
"Die Cops halten mich an\n",
"Ich sitze bei Nacht in 'nem schwarzen Lamborghini\n",
"Ey, sie fragen, was ich mach'\n",
"Wohin ich geh' und ich sag', „Zur Familie“\n",
"Achtet auf die AP mit Brillis\n",
"Meine AP mit Brillis\n",
"Sie scheint den Bullen ins Gesicht bei Mondlicht\n",
"Die AP mit Brillis\n",
"Die AP mit Brillis, die AP mit Brillis\n",
"AP mit Brillis, die AP mit Brillis\n",
"Ah, Diamanten, sie schein'n\n",
"Der Bulle schiebt Hass mit sei'm Beamtengehalt\n",
"Guck, die Straße macht Männer reich\n",
"Und der Staat lässt sie fallen, Akhi, es ist immer gleich\n",
"Ah, dreh die Boxen auf, MO\n",
"Fensterscheiben runter vom Lambo\n",
"Ganz softe Tempo, fick Drecksbull'n\n",
"Dreißiger Zone, SOS, du weißt, wo ich wohne\n",
"Ah, und ich fahr' auf entspannt\n",
"Hab' den Universal-Deal plus die Stadt in der Hand\n",
"Doch die Iba'ash nervt und sie macht uns krank\n",
"Halt Melon'n und wer wartet da?\n",
"Die Cops halten mich an\n",
"Ich sitze bei Nacht in 'nem schwarzen Lamborghini\n",
"Ey, sie fragen, was ich mach'\n",
"Wohin ich geh' und ich sag', „Zur Familie“\n",
"Achtet auf die AP mit Brillis\n",
"Meine AP mit Brillis\n",
"Sie scheint den Bullen ins Gesicht bei Mondlicht\n",
"Die AP mit Brillis\n",
"Die AP mit Brillis, die AP mit Brillis\n",
"----------\n"
]
}
],
"source": [
"l = filtered_song_metas[:]\n",
"# l = [m for m in filtered_song_metas if \"[\" not in m[\"lyrics\"]]\n",
"\n",
"idx_list = list(range(len(l)))\n",
"random.shuffle(idx_list)\n",
"for idx in idx_list[:1]:\n",
" m = l[idx]\n",
" Audio.from_file(m[\"uri\"]).play()\n",
" print(\"--\", idx, \"--\", m[\"genius_info\"][\"slug\"], \"--\", m[\"genius_info\"][\"views\"])\n",
" print()\n",
" print(\"\\n\".join(m[\"lyrics_lines\"]))\n",
" print(\"-\"*10)"
]
},
{
"cell_type": "markdown",
"id": "caad044f",
"metadata": {},
"source": [
"## Make nvidia distribution corpus"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "aa487288",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"20099 song details.\n",
"1061.5 hours of audio.\n"
]
}
],
"source": [
"song_metas = []\n",
"with open(os.path.join(DATA_DIR, \"filtered_pair_metas.jsonl\")) as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" song_metas.append(json.loads(line))\n",
"print(len(song_metas), \"song details.\")\n",
"print(round(np.sum([m[\"duration_s\"] for m in song_metas]) / 60 / 60, 1), \"hours of audio.\")\n",
"# 20099 song details.\n",
"# 1061.5 hours of audio."
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "3af5ee1e",
"metadata": {},
"outputs": [],
"source": [
"OUT_DIR = os.path.join(DATA_DIR, \"de_rap_corpus_distribution\")\n",
"OUT_AUDIO_DIR = os.path.join(OUT_DIR, \"audio\")\n",
"\n",
"shutil.rmtree(OUT_DIR, ignore_errors=True)\n",
"os.makedirs(OUT_DIR, exist_ok=True)\n",
"os.makedirs(OUT_AUDIO_DIR, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "b14a10db",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|█████████████████████████████████████████████████████████████████| 20099/20099 [30:54<00:00, 10.84it/s]\n"
]
}
],
"source": [
"dist_metas = []\n",
"id_map = []\n",
"for m in tqdm.tqdm(song_metas):\n",
" uid = str(uuid.uuid4())\n",
" id_map.append((m[\"genius_info\"][\"id\"], m[\"youtube_info\"][\"id\"], uid))\n",
" from_fp = m[\"uri\"]\n",
" to_fn = f\"{uid}.wav\"\n",
" to_fp = os.path.join(OUT_AUDIO_DIR, to_fn)\n",
" shutil.copy(from_fp, to_fp)\n",
" to_rel_fp = \"/\".join(to_fp.split(\"/\")[-2:])\n",
" dist_metas.append({\n",
" \"id\": uid,\n",
" \"uri\": to_rel_fp,\n",
" \"duration_s\": m[\"duration_s\"],\n",
" \"lyrics\": \"\\n\".join(m[\"lyrics_lines\"]),\n",
" })"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "b6aaf28f",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(DATA_DIR, \"dist_id_map.json\"), \"w\") as f:\n",
" json.dump(id_map, f)\n",
"with open(os.path.join(OUT_DIR, \"song_metadata.jsonl\"), \"w\") as f:\n",
" for m in dist_metas:\n",
" f.write(json.dumps(m) + \"\\n\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "006c4a16",
"metadata": {},
"outputs": [],
"source": [
"# s4cmd dsync -r /data/suno/data/academia/rap_de/de_rap_corpus_distribution s3://suno-data/datasets/suno_de_rap_corpus"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b140fe49",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "f3fa30ab",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "f6253270",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "a1d18fcc",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "e0e50c6c",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "882af5f9",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "b601daa2",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "5f2391ea",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "3280f09f",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}