{
"cells": [
{
"cell_type": "code",
"execution_count": 151,
"id": "15416422",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"import re\n",
"import random\n",
"import shutil\n",
"import uuid\n",
"import multiprocessing\n",
"from copy import deepcopy\n",
"\n",
"import numpy as np\n",
"import pandas as pd\n",
"import tqdm\n",
"\n",
"from suno_utils.audio import Audio\n",
"from suno_utils.audio.conversion import get_duration_s\n",
"from suno_utils.utils.text import normalize_whitespace\n",
"\n",
"DATA_DIR = \"/mnt/data-ssd-1/data/custom/rap_de\"\n",
"AUDIO_DIR = os.path.join(DATA_DIR, \"audio\")"
]
},
{
"cell_type": "markdown",
"id": "264fd819",
"metadata": {},
"source": [
"## convert audio"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "19765333",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"28674 items\n"
]
}
],
"source": [
"RAW_DATA_DIR = os.path.join(DATA_DIR, \"raw_data\")\n",
"\n",
"ALLOWED_AUDIO_EXTENSION = set([\"webm\", \"m4a\"])\n",
"\n",
"queue_items = []\n",
"for fn in os.listdir(RAW_DATA_DIR):\n",
" file_ext = fn.split(\".\")[-1]\n",
" if file_ext not in ALLOWED_AUDIO_EXTENSION:\n",
" continue\n",
" file_id = fn[:-len(file_ext)-1]\n",
" from_fp = os.path.join(RAW_DATA_DIR, fn)\n",
" to_fp = os.path.join(AUDIO_DIR, file_id + \".wav\")\n",
" queue_items.append((from_fp, to_fp))\n",
"print(len(queue_items), \"items\")"
]
},
{
"cell_type": "code",
"execution_count": 49,
"id": "fa228472",
"metadata": {},
"outputs": [],
"source": [
"shutil.rmtree(AUDIO_DIR, ignore_errors=True)\n",
"os.makedirs(AUDIO_DIR, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": 50,
"id": "5b67783f",
"metadata": {},
"outputs": [],
"source": [
"def _convert_autio(queue_item):\n",
" from_fp, to_fp = queue_item\n",
" try:\n",
" Audio.from_file(from_fp, sample_rate=16_000, byte_width=2).to_wav(to_fp)\n",
" except:\n",
" pass\n",
" \n",
"p = multiprocessing.Pool(10)\n",
"_ = p.map(_convert_autio, queue_items, chunksize=1)\n",
"p.close()\n",
"p.join()"
]
},
{
"cell_type": "code",
"execution_count": 51,
"id": "f1ddf23c",
"metadata": {},
"outputs": [],
"source": [
"# !ls -1 /mnt/data-ssd-1/data/custom/rap_de/audio | wc -l"
]
},
{
"cell_type": "code",
"execution_count": 52,
"id": "89732a2a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1544.4 hours of audio.\n",
"1402.0 hours of song audio.\n"
]
}
],
"source": [
"durations_s = []\n",
"for fn in os.listdir(AUDIO_DIR):\n",
" durations_s.append(get_duration_s(os.path.join(AUDIO_DIR, fn)))\n",
"s = pd.Series(durations_s)\n",
"print(round(s.sum() / 60 / 60, 1), \"hours of audio.\")\n",
"print(round(s[s<=5*60].sum() / 60 / 60, 1), \"hours of song audio.\")"
]
},
{
"cell_type": "markdown",
"id": "5a3a707b",
"metadata": {},
"source": [
"## assemble pairs"
]
},
{
"cell_type": "code",
"execution_count": 55,
"id": "61955220",
"metadata": {},
"outputs": [],
"source": [
"youtube_audio_id_set = set()\n",
"for fn in os.listdir(AUDIO_DIR):\n",
" if fn.endswith(\".wav\"):\n",
" youtube_audio_id_set.add(fn[:-4])"
]
},
{
"cell_type": "code",
"execution_count": 104,
"id": "b5b610b7",
"metadata": {},
"outputs": [],
"source": [
"youtube_info_map = {}\n",
"with open(os.path.join(DATA_DIR, \"youtube_metas.jsonl\")) as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" m = json.loads(line)\n",
" if m is None or not m[\"success\"] or m[\"id\"] not in youtube_audio_id_set:\n",
" continue\n",
" youtube_info_map[m[\"id\"]] = {\n",
" \"id\": m[\"id\"],\n",
" \"views\": m[\"meta\"][\"view_count\"],\n",
" \"likes\": m[\"meta\"].get(\"like_count\"),\n",
" \"title\": m[\"meta\"][\"title\"],\n",
" \"artist\": m[\"meta\"].get(\"artist\"),\n",
" \"album\": m[\"meta\"].get(\"album\"),\n",
" \"is_live\": m[\"meta\"][\"is_live\"],\n",
" \"description\": m[\"meta\"][\"description\"],\n",
" \"categories\": m[\"meta\"][\"categories\"],\n",
" \"tags\": m[\"meta\"][\"tags\"],\n",
" \"release_date\": m[\"meta\"].get(\"release_date\"),\n",
" }"
]
},
{
"cell_type": "code",
"execution_count": 109,
"id": "743d58b8",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"26024 song details fetched.\n"
]
}
],
"source": [
"song_metas = []\n",
"with open(os.path.join(DATA_DIR, \"song_metas.jsonl\")) as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" m = json.loads(line)\n",
" if m is None:\n",
" continue\n",
" youtube_id = m[\"youtube_url\"].split(\"=\")[-1]\n",
" if (\n",
" youtube_id not in youtube_audio_id_set or \n",
" int(m[\"youtube_start\"]) != 0 or\n",
" m[\"lang\"] != \"de\" or\n",
" m[\"lang_lyrics\"] != \"de\" or\n",
" youtube_id not in youtube_info_map\n",
" ):\n",
" continue\n",
" song_metas.append({\n",
" \"uri\": os.path.join(AUDIO_DIR, f\"{youtube_id}.wav\"),\n",
" \"duration_s\": get_duration_s(os.path.join(AUDIO_DIR, f\"{youtube_id}.wav\")),\n",
" \"genius_info\": {\n",
" \"id\": m[\"id\"],\n",
" \"slug\": m[\"slug\"],\n",
" \"views\": m[\"views\"],\n",
" \"tags\": m[\"tags\"],\n",
" },\n",
" \"youtube_info\": youtube_info_map[youtube_id],\n",
" \"lyrics\": m[\"lyrics\"],\n",
" })\n",
"print(len(song_metas), \"song details fetched.\")"
]
},
{
"cell_type": "code",
"execution_count": 110,
"id": "cce025ba",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(DATA_DIR, \"pair_metas.jsonl\"), \"w\") as f:\n",
" for m in song_metas:\n",
" f.write(json.dumps(m) + \"\\n\")"
]
},
{
"cell_type": "markdown",
"id": "800430dc",
"metadata": {},
"source": [
"## investigate and filter"
]
},
{
"cell_type": "code",
"execution_count": 190,
"id": "7e1f1775",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"26024 song details.\n",
"1400.3 hours of audio.\n"
]
}
],
"source": [
"song_metas = []\n",
"with open(os.path.join(DATA_DIR, \"pair_metas.jsonl\")) as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" song_metas.append(json.loads(line))\n",
"print(len(song_metas), \"song details.\")\n",
"print(round(np.sum([m[\"duration_s\"] for m in song_metas]) / 60 / 60, 1), \"hours of audio.\")"
]
},
{
"cell_type": "code",
"execution_count": 191,
"id": "80916a09",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"20099 song details.\n",
"1061.5 hours of audio.\n"
]
}
],
"source": [
"def _parse_lyrics_lines(lyrics):\n",
" lines = []\n",
" for line in lyrics.split(\"\\n\"):\n",
" line = line.strip()\n",
" if \"[\" in line:\n",
" line = re.sub(r\"\\[.*?\\]\", \" \", line)\n",
" line = re.sub(r\"\\(.*?\\)\", \" \", line)\n",
" line = normalize_whitespace(line)\n",
" if len(line) == 0:\n",
" continue\n",
" lines.append(line)\n",
" return lines\n",
"\n",
"filtered_song_metas = []\n",
"for m in song_metas:\n",
" m = deepcopy(m)\n",
" if m[\"duration_s\"] < 30 or m[\"duration_s\"] > 8 * 60:\n",
" continue\n",
" if m[\"genius_info\"][\"views\"] < 20:\n",
" continue\n",
" if len(set(m[\"genius_info\"][\"tags\"]) & set([1658, 2895])) == 0: # Deutschland, Oesterreich\n",
" continue\n",
" if (\n",
" re.search(r\"^\\s*\\[.*\\b2x\\b\", m[\"lyrics\"]) or # repetitions are hard to parse\n",
" re.search(r\"\\[[\\s\\?]+\\]\", m[\"lyrics\"]) # some have [?] - unintelligible\n",
" ):\n",
" continue\n",
" lyrics_lines = _parse_lyrics_lines(m[\"lyrics\"])\n",
" m[\"lyrics_lines\"] = lyrics_lines\n",
" fulltext = \" \".join(lyrics_lines)\n",
" if len(re.findall(r\"[a-zA-Z]\", fulltext)) < 250 or len(fulltext) > 6000:\n",
" continue\n",
" upper_frac = len(re.findall(r\"[A-Z]\", fulltext)) / len(re.findall(r\"[a-zA-Z]\", fulltext))\n",
" if upper_frac <= 0.01 or upper_frac >= 0.8:\n",
" continue\n",
" filtered_song_metas.append(m)\n",
"print(len(filtered_song_metas), \"song details.\")\n",
"print(round(np.sum([m[\"duration_s\"] for m in filtered_song_metas]) / 60 / 60, 1), \"hours of audio.\")"
]
},
{
"cell_type": "code",
"execution_count": 192,
"id": "b6170d7c",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(DATA_DIR, \"filtered_pair_metas.jsonl\"), \"w\") as f:\n",
" for m in filtered_song_metas:\n",
" f.write(json.dumps(m) + \"\\n\")"
]
},
{
"cell_type": "code",
"execution_count": 193,
"id": "8c24c596",
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"-- 1109 -- Ak-ausserkontrolle-immer-wenn-es-nacht-wird-lyrics -- 14225\n",
"\n",
"AK ist Mafia, Westberlin – Kapstadt\n",
"Ob Rap oder Chabdas, die Besten mit Abstand\n",
"Jetzt auf einmal macht hier jeder auf Brecher\n",
"Doch jeder weiß: Apple-Store, das war'n die mit Bandana\n",
"Breche ein bei dem Schlippsträger\n",
"Ganz allein ohne Mittäter, das heißt, ich teil' nur mit Tippgeber\n",
"Blitzbrecher, die alles Mitnehmen, Babas\n",
"Gas geben, AMG, Wildleder Nappa\n",
"CLS Benz, Chiptuning, drückt Brudi\n",
"Fünfhundert PS, gib Gummi\n",
"Hektik, Stress, im Fokus, bremms' nicht\n",
"Bestes Flex, hochprozentig\n",
"Du kennst mich, mit Cap und Bandana\n",
"Echter Verbrecher, das ist Rap nur für Männer\n",
"Lines für mein Pensum, Weißes im Zentrum\n",
"Sechs-fünf, jeder scheiß Bulle kennt uns\n",
"Immer wenn es Nacht wird, heul'n im Block die Sirenen\n",
"Die Cops woll'n meinen Kopf wie Hyänen\n",
"Immer wenn es Nacht wird, woll'n wir an die Scheine ran (ah)\n",
"Und ziehen uns die Highjacks an\n",
"Immer wenn es Nacht wird, fahren wir im Benzer vor\n",
"Meine Jungs fahren mit dem Vectra in den Apple-Store\n",
"Immer wenn es Nacht wird (ah)\n",
"Immer wenn es Nacht wird, immer wenn das Taş wirkt\n",
"Mustang-Rap, klopf' den Staub von den Schultern weg\n",
"Ich pump' im Unterhemd und der Brustumfang wächst\n",
"Ausserkontrolle, das ist pure Gewalt\n",
"Das ist Ruhm auf der Straße, auf der Suche nach Gold\n",
"Absolute Verfolgung, Blicke, die weichen\n",
"Vermummte Gesichter, dunkle Gestalten\n",
"Im Audi durch Fürstenwald, mit dreihundert Sachen\n",
"Wir bringen Daule das Fürchten bei\n",
"Jetzt geht die Post ab, Ghetto Cosa Nostra\n",
"Kopf krank, stürme mit Pumpgun die Postbank\n",
"Harbi 7bdas auf Abiat und Darby\n",
"Gunshots, Kapstadt, Abstammung Mardin\n",
"Jetset-Diplomatie\n",
"Streck' Flex, Lidocain – innovativ\n",
"Anklage, schwarz auf weiß (schwarz auf weiß)\n",
"Seit Tag eins – Staat sein Feind!\n",
"Immer wenn es Nacht wird, heul'n im Block die Sirenen\n",
"Die Cops woll'n meinen Kopf wie Hyänen\n",
"Immer wenn es Nacht wird, woll'n wir an die Scheine ran (ah)\n",
"Und ziehen uns die Highjacks an\n",
"Immer wenn es Nacht wird, fahren wir im Benzer vor\n",
"Meine Jungs fahren mit dem Vectra in den Apple-Store\n",
"Immer wenn es Nacht wird (ah)\n",
"Immer wenn es Nacht wird, immer wenn das Taş wirkt\n",
"----------\n"
]
}
],
"source": [
"l = filtered_song_metas[:]\n",
"# l = [m for m in filtered_song_metas if \"[\" not in m[\"lyrics\"]]\n",
"\n",
"idx_list = list(range(len(l)))\n",
"random.shuffle(idx_list)\n",
"for idx in idx_list[:1]:\n",
" m = l[idx]\n",
" Audio.from_file(m[\"uri\"]).play()\n",
" print(\"--\", idx, \"--\", m[\"genius_info\"][\"slug\"], \"--\", m[\"genius_info\"][\"views\"])\n",
" print()\n",
" print(\"\\n\".join(m[\"lyrics_lines\"]))\n",
" print(\"-\"*10)"
]
},
{
"cell_type": "markdown",
"id": "caad044f",
"metadata": {},
"source": [
"## Make nvidia distribution corpus"
]
},
{
"cell_type": "code",
"execution_count": 194,
"id": "aa487288",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"20099 song details.\n",
"1061.5 hours of audio.\n"
]
}
],
"source": [
"song_metas = []\n",
"with open(os.path.join(DATA_DIR, \"filtered_pair_metas.jsonl\")) as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" song_metas.append(json.loads(line))\n",
"print(len(song_metas), \"song details.\")\n",
"print(round(np.sum([m[\"duration_s\"] for m in song_metas]) / 60 / 60, 1), \"hours of audio.\")"
]
},
{
"cell_type": "code",
"execution_count": 195,
"id": "3af5ee1e",
"metadata": {},
"outputs": [],
"source": [
"OUT_DIR = os.path.join(DATA_DIR, \"de_rap_corpus_distribution\")\n",
"OUT_AUDIO_DIR = os.path.join(OUT_DIR, \"audio\")\n",
"\n",
"shutil.rmtree(OUT_DIR, ignore_errors=True)\n",
"os.makedirs(OUT_DIR, exist_ok=True)\n",
"os.makedirs(OUT_AUDIO_DIR, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": 196,
"id": "b14a10db",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20099/20099 [13:34<00:00, 24.66it/s]\n"
]
}
],
"source": [
"dist_metas = []\n",
"id_map = []\n",
"for m in tqdm.tqdm(song_metas):\n",
" uid = str(uuid.uuid4())\n",
" id_map.append((m[\"genius_info\"][\"id\"], m[\"youtube_info\"][\"id\"], uid))\n",
" from_fp = m[\"uri\"]\n",
" to_fn = f\"{uid}.wav\"\n",
" to_fp = os.path.join(OUT_AUDIO_DIR, to_fn)\n",
" shutil.copy(from_fp, to_fp)\n",
" to_rel_fp = \"/\".join(to_fp.split(\"/\")[-2:])\n",
" dist_metas.append({\n",
" \"id\": uid,\n",
" \"uri\": to_rel_fp,\n",
" \"duration_s\": m[\"duration_s\"],\n",
" \"lyrics\": \"\\n\".join(m[\"lyrics_lines\"]),\n",
" })"
]
},
{
"cell_type": "code",
"execution_count": 203,
"id": "b6aaf28f",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(DATA_DIR, \"dist_id_map.json\"), \"w\") as f:\n",
" json.dump(id_map, f)\n",
"with open(os.path.join(OUT_DIR, \"song_metadata.jsonl\"), \"w\") as f:\n",
" for m in dist_metas:\n",
" f.write(json.dumps(m) + \"\\n\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "006c4a16",
"metadata": {},
"outputs": [],
"source": [
"# s4cmd dsync -r /mnt/data-ssd-1/data/custom/rap_de/de_rap_corpus_distribution s3://suno-data/datasets/suno_de_rap_corpus"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b140fe49",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "f3fa30ab",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "f6253270",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "a1d18fcc",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "e0e50c6c",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "882af5f9",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "b601daa2",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "5f2391ea",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "3280f09f",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}