{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "# given new alignments we will merge them with existing metas.jsonl\n", "import os\n", "from suno_utils.utils.text import read_jsonl\n", "from tqdm import tqdm" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2054302\n", "1799066\n" ] } ], "source": [ "base_dir = \"/home/tony/Work/tony/hoot/tmp\"\n", "discogs_filepath = os.path.join(base_dir, \"discogs_hq_alignments_h4_t30_v1.jsonl\")\n", "genius_filepath = os.path.join(base_dir, \"genius_hq_alignments_h4_t30_v1.jsonl\")\n", "#deezer_filepath = os.path.join(base_dir, \"deezer_hq_alignments_h4_t30_v1.jsonl\")\n", "\n", "\n", "discogs_alignments = read_jsonl(discogs_filepath)\n", "print(len(discogs_alignments))\n", "genius_alignments = read_jsonl(genius_filepath)\n", "print(len(genius_alignments))\n", "#deezer_alignments = read_jsonl(deezer_filepath)\n", "#print(len(deezer_alignments))\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# create maps from genius alignments lookup via id\n", "genius_alignments_map = {k: {\"alignment\": v, \"cer_score\": cer_score} for k, v, cer_score in genius_alignments}\n", "discogs_alignments_map = {k: {\"alignment\": v, \"cer_score\": cer_score} for k, v, cer_score in discogs_alignments}\n", "#deezer_alignments_map = {k: {\"alignment\": v, \"cer_score\": cer_score} for k, v, cer_score in deezer_alignments}" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2090009\n", "2916657\n" ] } ], "source": [ "genius_metas = read_jsonl(\"/home/christian/code/christian/metadata/genius_hq_metas.jsonl\")\n", "genius_metas_map = {meta[\"id\"]: meta for meta in genius_metas}\n", "print(len(genius_metas))\n", "# create an id mapping to convert id to original_id\n", "original_id_map = {meta[\"original_id\"]: meta[\"id\"] for meta in genius_metas}\n", "\n", "\n", "discogs_metas = read_jsonl(\"/app/suno/tmp/clean_discogs_subset_v0_metas.jsonl\")\n", "discogs_metas_map = {meta[\"id\"]: meta for meta in discogs_metas}\n", "print(len(discogs_metas))\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "ISO_LANGUAGES = {\n", " 'aa': 'Afar', 'ab': 'Abkhazian', 'af': 'Afrikaans', 'ak': 'Akan',\n", " 'sq': 'Albanian', 'am': 'Amharic', 'ar': 'Arabic', 'an': 'Aragonese',\n", " 'hy': 'Armenian', 'as': 'Assamese', 'av': 'Avaric', 'ae': 'Avestan',\n", " 'ay': 'Aymara', 'az': 'Azerbaijani', 'ba': 'Bashkir', 'bm': 'Bambara',\n", " 'eu': 'Basque', 'be': 'Belarusian', 'bn': 'Bengali', 'bh': 'Bihari',\n", " 'bi': 'Bislama', 'bs': 'Bosnian', 'br': 'Breton', 'bg': 'Bulgarian',\n", " 'my': 'Burmese', 'ca': 'Catalan', 'ch': 'Chamorro', 'ce': 'Chechen',\n", " 'zh': 'Chinese', 'cu': 'Church Slavic', 'cv': 'Chuvash', 'kw': 'Cornish',\n", " 'co': 'Corsican', 'cr': 'Cree', 'cs': 'Czech', 'da': 'Danish',\n", " 'dv': 'Divehi', 'nl': 'Dutch', 'dz': 'Dzongkha', 'en': 'English',\n", " 'eo': 'Esperanto', 'et': 'Estonian', 'ee': 'Ewe', 'fo': 'Faroese',\n", " 'fj': 'Fijian', 'fi': 'Finnish', 'fr': 'French', 'fy': 'Western Frisian',\n", " 'ff': 'Fulah', 'ka': 'Georgian', 'de': 'German', 'gd': 'Gaelic',\n", " 'ga': 'Irish', 'gl': 'Galician', 'gv': 'Manx', 'el': 'Greek',\n", " 'gn': 'Guarani', 'gu': 'Gujarati', 'ht': 'Haitian', 'ha': 'Hausa',\n", " 'he': 'Hebrew', 'hz': 'Herero', 'hi': 'Hindi', 'ho': 'Hiri Motu',\n", " 'hr': 'Croatian', 'hu': 'Hungarian', 'ig': 'Igbo', 'is': 'Icelandic',\n", " 'io': 'Ido', 'ii': 'Sichuan Yi', 'iu': 'Inuktitut', 'ie': 'Interlingue',\n", " 'ia': 'Interlingua', 'id': 'Indonesian', 'ik': 'Inupiaq', 'it': 'Italian',\n", " 'jv': 'Javanese', 'ja': 'Japanese', 'kl': 'Kalaallisut', 'kn': 'Kannada',\n", " 'ks': 'Kashmiri', 'kr': 'Kanuri', 'kk': 'Kazakh', 'km': 'Central Khmer',\n", " 'ki': 'Kikuyu', 'rw': 'Kinyarwanda', 'ky': 'Kirghiz', 'kv': 'Komi',\n", " 'kg': 'Kongo', 'ko': 'Korean', 'kj': 'Kuanyama', 'ku': 'Kurdish',\n", " 'lo': 'Lao', 'la': 'Latin', 'lv': 'Latvian', 'li': 'Limburgan',\n", " 'ln': 'Lingala', 'lt': 'Lithuanian', 'lb': 'Luxembourgish',\n", " 'lu': 'Luba-Katanga', 'lg': 'Ganda', 'mk': 'Macedonian',\n", " 'mh': 'Marshallese', 'ml': 'Malayalam', 'mi': 'Maori', 'mr': 'Marathi',\n", " 'ms': 'Malay', 'mg': 'Malagasy', 'mt': 'Maltese', 'mn': 'Mongolian',\n", " 'na': 'Nauru', 'nv': 'Navajo', 'nr': 'South Ndebele',\n", " 'nd': 'North Ndebele', 'ng': 'Ndonga', 'ne': 'Nepali', 'nn': 'Norwegian',\n", " 'nb': 'Norwegian', 'no': 'Norwegian', 'ny': 'Chichewa', 'oc': 'Occitan',\n", " 'oj': 'Ojibwa', 'or': 'Oriya', 'om': 'Oromo', 'os': 'Ossetian',\n", " 'pa': 'Panjabi', 'fa': 'Persian', 'pi': 'Pali', 'pl': 'Polish',\n", " 'pt': 'Portuguese', 'ps': 'Pushto', 'qu': 'Quechua', 'rm': 'Romansh',\n", " 'ro': 'Romanian', 'rn': 'Rundi', 'ru': 'Russian', 'sg': 'Sango',\n", " 'sa': 'Sanskrit', 'si': 'Sinhala', 'sk': 'Slovak', 'sl': 'Slovenian',\n", " 'se': 'Northern Sami', 'sm': 'Samoan', 'sn': 'Shona', 'sd': 'Sindhi',\n", " 'so': 'Somali', 'st': 'Southern Sotho', 'es': 'Spanish', 'sc': 'Sardinian',\n", " 'sr': 'Serbian', 'ss': 'Swati', 'su': 'Sundanese', 'sw': 'Swahili',\n", " 'sv': 'Swedish', 'ty': 'Tahitian', 'ta': 'Tamil', 'tt': 'Tatar',\n", " 'te': 'Telugu', 'tg': 'Tajik', 'tl': 'Tagalog', 'th': 'Thai',\n", " 'bo': 'Tibetan', 'ti': 'Tigrinya', 'to': 'Tonga', 'tn': 'Tswana',\n", " 'ts': 'Tsonga', 'tk': 'Turkmen', 'tr': 'Turkish', 'tw': 'Twi',\n", " 'ug': 'Uighur', 'uk': 'Ukrainian', 'ur': 'Urdu', 'uz': 'Uzbek',\n", " 've': 'Venda', 'vi': 'Vietnamese', 'vo': 'Volapük', 'cy': 'Welsh',\n", " 'wa': 'Walloon', 'wo': 'Wolof', 'xh': 'Xhosa', 'yi': 'Yiddish',\n", " 'yo': 'Yoruba', 'za': 'Zhuang', 'zu': 'Zulu'\n", "}\n", "\n", "def get_language_name(code):\n", " return ISO_LANGUAGES.get(code.lower(), 'Unknown')" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'alignment': [{'text': '(Ghosty)\\n(Ay Geno these kids say they know you)\\n\\nAyy, big batty girl oh lord (bunda)\\nTell her come flick with the stars\\n',\n", " 'start_s': 0,\n", " 'end_s': 30},\n", " {'text': \"Tell her come flick with the stars\\nShe wants third leg in the dance\\nSlow down miss and you might have a chance\\nBro made an arm discharge\\nCock back, blast, make everyone run fast\\nLack on the block me and bro just laugh\\nWas talking about which one I'd opt\\nI'm out on a glide and I'm hunting now\\nJumping out tryna run man down\\nRun it up that's someone down\\nBrucking beef no bluffing now\\nIt's T Trap, beat that\\nYou ever seen a gunman frown\\nHop out the ride with a wu-du-du-du-du-du-du\\nYou shouldn't of run that mound\\nHow many times did we do it, do it\\nDon't get chewed like truits, truits\\nThat's a new outfit, ruined, we're screwing\\nPull up on blocks we're doing\\n\",\n", " 'start_s': 30,\n", " 'end_s': 60},\n", " {'text': \"Pull up on blocks we're doing\\nNuff gyal love arse so I'm doing them up\\nIn the booth smoke weed, make musicc\\nShould've knew this we ain't stupid\\nIf it ain't that it's a video shootin'\\nThe whole of North London knows (ayy)\\nBroadwater Farm where the gunsdem go\\nHave you ever been out hunting bro with a hunting knife we just come and go (ching ching)\\nCome on bro you ain't on that smoke\\nPull up slow let the thunder go\\nBeen loyal to my bros, and everyone knows\\nGotta get to the bag it's just hunger bro\\n\\nInterview room I'm more than silent\\nNo comment that's my answer\\nI see bro fling a leg in dance so violent, never knew he was a dancer\\nHB loud and bars of cookie or runts put you in a rabba\\n\",\n", " 'start_s': 60,\n", " 'end_s': 90},\n", " {'text': \"HB loud and bars of cookie or runts put you in a rabba\\nAnd bro hoped out way faster circle up and I got there after\\nAnd if he's advanced, he's overpissed\\nThem youngboys doing up bare fuckery\\nNow I'm big dog, I'm the older kid\\nI'm the older man\\nTold my ex get over man\\nHe denied he's owned a gang\\nWhy you talk so much who knows this man\\nFlick of the wrist if I slap this hammer\\nWatch man lean and stagger\\nThe Junebug put nuff blood on his dagger\\nMake M1llionz and I'm screaming Lagga\\nMy west ting just things I'm a rapper\\nShe knows what's good but my East ting's badder\\nNot on nothing the mandem's madder\\nHaha, been in the field like KaKa\\n\",\n", " 'start_s': 90,\n", " 'end_s': 120},\n", " {'text': \"Haha, been in the field like KaKa\\nBro on seats so Gucci\\nGissele's been boujie\\nHold up, excuse me, told you I'm done with these groupies\\nBro said he'll crew out some movies\\nSit back out when he shoots out the two piece\\nSlim and Coolie always moody, always been picky and choosy\",\n", " 'start_s': 120,\n", " 'end_s': 150}],\n", " 'cer_score': 0.394}" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "discogs_alignments_map[\"---2ayLSzvo\"]" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Distribution of CER scores for genius:\n", "Number of alignments: 1799066\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 0/1799066 [00:00