{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "1f82190c", "metadata": {}, "outputs": [], "source": [ "## ytm_tagged\n", "# s3://suno-data/datasets/harvest/ytm_tagged/ytm_tagged_metas.jsonl\n", "# s3://suno-data/datasets/harvest/ytm_tagged/ytm_tagged_metas_filtered.jsonl\n", "\n", "## ytm_synced_lyrics\n", "# s3://suno-data/datasets/harvest/ytm_synced_lyrics/lyrics_dl.jsonl" ] }, { "cell_type": "code", "execution_count": 1, "id": "bfa27f90", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "a1f47dfa", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n" ] } ], "source": [ "import tqdm\n", "import math\n", "import torch\n", "import random\n", "import funcy\n", "import copy\n", "import gc\n", "import re\n", "import json\n", "import tempfile\n", "import collections\n", "import pandas as pd\n", "import numpy as np\n", "import fasttext\n", "from joblib import Parallel, delayed\n", "\n", "from suno_utils.audio import Audio\n", "from suno_utils.utils.text import write_jsonl, read_jsonl, write_json, read_json, normalize_whitespace\n", "from suno_utils.utils.s3 import read_from_s3, check_s3_file_exists, open_from_s3\n", "from suno_utils.harvest.youtube.constants.text_lang import BASE_TO_FASTTEXT_REMAP\n", "\n", "LANG_ID_MODEL_FP = \"s3://suno-data/georg/trained_models/chirp_v1/lid.176.bin\"\n", "text_lang_model = read_from_s3(LANG_ID_MODEL_FP, read_f=fasttext.load_model)\n", "\n", "def _get_text_lang(text):\n", " \"\"\"get probability of input language for text\"\"\"\n", " text = text.replace(\"’\", \"'\").lower()\n", " text = re.sub(r\"\\[.+?\\]\", \" \", text)\n", " text = normalize_whitespace(text)\n", " out = text_lang_model.predict(text, k=1)\n", " lang_str = out[0][0]\n", " p_lang = out[1][0]\n", " lang = lang_str.split(\"__\")[-1]\n", " lang = BASE_TO_FASTTEXT_REMAP.get(lang, lang)\n", "# if p_lang >= 0.8:\n", "# return lang\n", " return p_lang, lang" ] }, { "cell_type": "code", "execution_count": null, "id": "1865ecfa", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "82e94221", "metadata": {}, "source": [ "### ytm_tagged" ] }, { "cell_type": "code", "execution_count": 4, "id": "a2466f6a", "metadata": {}, "outputs": [], "source": [ "# ytm_metas = read_from_s3(\n", "# \"s3://suno-data/datasets/harvest/ytm_tagged/ytm_tagged_metas.jsonl\", \n", "# read_f=read_jsonl,\n", "# )\n", "# ytm_metas_filtered = read_from_s3(\n", "# \"s3://suno-data/datasets/harvest/ytm_tagged/ytm_tagged_metas_filtered.jsonl\", \n", "# read_f=read_jsonl,\n", "# )\n", "ytm_metas = read_jsonl(\"/mnt/data/georg/tmp/ytm_tagged_metas.jsonl\")\n", "# ytm_metas_filtered = read_jsonl(\"/mnt/data/georg/tmp/ytm_tagged_metas_filtered.jsonl\")" ] }, { "cell_type": "code", "execution_count": 39, "id": "13f9f1ef", "metadata": {}, "outputs": [], "source": [ "# # release_country\n", "\n", "# from suno_utils.harvest.youtube.constants.harvest import GL_LOCS\n", "# c_map = {k.lower(): v for k, v in GL_LOCS.items()}\n", "# for idx in s.index:\n", "# if idx.lower() in c_map:\n", "# print(idx, c_map[idx.lower()])" ] }, { "cell_type": "code", "execution_count": null, "id": "71dccb17", "metadata": {}, "outputs": [], "source": [ "import regex\n", "\n", "def _is_clean(t, blocked_str):\n", "# for s in t.split():\n", " if regex.search(r\"(\\b\" + regex.escape(t.lower()) + r\"\\b){e<=1}\", blocked_str.lower()):\n", " return False\n", " return True\n", "\n", "metas = []\n", "for m in tqdm.tqdm(ytm_metas):\n", " if not m.get(\"success\", False):\n", " continue\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"s3_filepath\": os.path.join(\"s3://suno-data/datasets/harvest/ytm_tagged/audio/\", m[\"audio_filename\"]),\n", " \"duration_s\": m[\"yt_dlp\"][\"meta\"][\"duration\"],\n", " \"views\": m[\"yt_dlp\"][\"meta\"][\"view_count\"],\n", " }\n", " private_tags = []\n", " for s in [\n", " \"recording_tags\", \"recording_genres\",\n", " \"artist_tags\", \"artist_genres\",\n", " \"release_tags\", \"release_genres\",\n", " \"release_group_tags\", \"release_group_genres\",\n", " \"artist_gender\", \"artist_credit_name\", \"recording_instruments\",\n", " ]:\n", " if s in m and isinstance(m[s], list):\n", " private_tags.extend(m[s])\n", " elif s in m and isinstance(m[s], str):\n", " private_tags.append(m[s])\n", " private_tags = [ss for s in private_tags if len(ss:=normalize_whitespace(s)) >= 3]\n", " private_tags = list(set(private_tags))\n", " blocked_str = \" \".join([\n", " m[\"artist_credit_name\"] or \"\",\n", " m[\"release_name\"] or \"\",\n", " m[\"recording_name\"] or \"\",\n", " m[\"yt_dlp\"][\"meta\"].get(\"description\") or \"\",\n", " m[\"yt_dlp\"][\"meta\"].get(\"title\") or \"\",\n", " m[\"yt_dlp\"][\"meta\"].get(\"artist\") or \"\",\n", " ])\n", " clean_tags = []\n", " for t in private_tags:\n", " if _is_clean(t, blocked_str):\n", " clean_tags.append(t) \n", " clean_tags = [ss for s in clean_tags if len(ss:=normalize_whitespace(s)) >= 3]\n", " clean_tags = list(set(clean_tags)) \n", " if len(private_tags) > 0:\n", " new_m[\"private_tags\"] = private_tags \n", " if len(clean_tags) > 0:\n", " new_m[\"tags\"] = clean_tags\n", " metas.append(new_m)\n", "print(f\"{len(metas):,} clips\")\n", "print(f\"{sum(m['duration_s'] for m in metas)/60/60:,.0f} hours\")\n", "print(f\"{sum(m['duration_s'] for m in metas if m['views'] >= 100)/60/60:,.0f} hours popular\")\n", "tags = []\n", "private_tags = []\n", "for m in metas:\n", " tags.extend(m.get(\"tags\", []))\n", " private_tags.extend(m.get(\"private_tags\", []))\n", "print(f\"{len(set(tags)):,} unique tags\")\n", "print(f\"{len(set(private_tags)):,} unique private tags\")\n", "# 2,772,393 clips\n", "# 187,983 hours\n", "# 153,034 hours popular\n", "# 63,664 unique tags\n", "# 281,772 unique private tags" ] }, { "cell_type": "code", "execution_count": null, "id": "ed92eec7", "metadata": {}, "outputs": [], "source": [ "# TODO: lots of low count tags. prob filter to >= 5 at least or something" ] }, { "cell_type": "code", "execution_count": 100, "id": "ad0e50d1", "metadata": {}, "outputs": [], "source": [ "# vc = pd.Series(tags).value_counts()\n", "# private_vc = pd.Series(private_tags).value_counts()\n", "# private_vc.loc[[s for s in private_vc.index if \"britney\" in s.lower()]]\n", "# vc.loc[[s for s in private_vc.index if \"britney\" in s.lower()]]" ] }, { "cell_type": "code", "execution_count": 115, "id": "623de8e6", "metadata": {}, "outputs": [], "source": [ "# write_jsonl(metas, \"tmp/ytm_tagged.jsonl\")" ] }, { "cell_type": "code", "execution_count": 56, "id": "2c321e00", "metadata": {}, "outputs": [], "source": [ "# !aws s3 cp tmp/ytm_tagged.jsonl s3://suno-data/datasets/bundles/v2/ytm_tagged/metas.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "7595fda8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0147ac4d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e81de18a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "bfefb573", "metadata": {}, "source": [ "### deezer" ] }, { "cell_type": "code", "execution_count": null, "id": "76761de7", "metadata": {}, "outputs": [], "source": [ "# https://www.deezer.com/search/Biene%20Maja%20(Main%20Title)/track" ] }, { "cell_type": "code", "execution_count": 119, "id": "a83353d5", "metadata": {}, "outputs": [], "source": [ "# deezer_metas = read_from_s3(\n", "# \"s3://suno-data/datasets/harvest/ytm_synced_lyrics/lyrics_dl.jsonl\",\n", "# read_f=read_jsonl,\n", "# )\n", "deezer_metas = read_jsonl(\"/mnt/data/georg/tmp/lyrics_dl.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "d4ce59ff", "metadata": {}, "outputs": [], "source": [ "# TODO: add youtube title" ] }, { "cell_type": "code", "execution_count": 125, "id": "abe235da", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "768,515 clips\n", "42,975 hours\n", "28,878 hours of english\n", "27,005 hours popular\n", "15,551 hours of english popular\n" ] } ], "source": [ "seen_ids = set()\n", "metas = []\n", "for m in tqdm.tqdm(deezer_metas):\n", " if not m[\"ytm\"].get(\"success\", False) or m[\"ytm\"][\"id\"] in seen_ids:\n", " continue\n", " p_lang, lang = _get_text_lang(m[\"data\"][\"track\"][\"lyrics\"][\"text\"])\n", " new_m = {\n", " \"id\": m[\"ytm\"][\"id\"],\n", " \"s3_filepath\": os.path.join(\"s3://suno-data/datasets/harvest/ytm_synced_lyrics/audio/\", m[\"ytm\"][\"audio_filename\"]),\n", " \"duration_s\": m[\"ytm\"][\"meta\"][\"duration\"],\n", " \"views\": m[\"ytm\"][\"meta\"][\"view_count\"],\n", " \"youtube_title\": m[\"meta\"][\"SNG_TITLE\"],\n", " \"youtube_artist\": m[\"meta\"][\"ART_NAME\"],\n", " \"lyrics\": m[\"data\"][\"track\"][\"lyrics\"][\"text\"],\n", " # TODO: translation seem like non-english to english (~15k clips)\n", " \"lyrics_lines\": [\n", " {\n", " \"text\": mm[\"line\"],\n", "# \"text_translated\": mm.get(\"lineTranslated\"),\n", " \"start_s\": round(mm[\"milliseconds\"] / 1_000, 2),\n", " \"end_s\": round((mm[\"milliseconds\"] + mm[\"duration\"]) / 1_000, 2),\n", " } for mm in m[\"data\"][\"track\"][\"lyrics\"][\"synchronizedLines\"]\n", " ],\n", " \"lang_guess\": lang,\n", " \"lang_p\": p_lang,\n", " }\n", " # skip if most mean is shorter than 1 second\n", " if np.mean([mm[\"end_s\"] - mm[\"start_s\"] for mm in new_m[\"lyrics_lines\"]]) < 1:\n", " continue\n", " metas.append(new_m)\n", " seen_ids.add(m[\"ytm\"][\"id\"])\n", "# break\n", "print(f\"{len(metas):,} clips\")\n", "print(f\"{sum(m['duration_s'] for m in metas)/60/60:,.0f} hours\")\n", "print(f\"{sum(m['duration_s'] for m in metas if m['lang_guess'] == 'en')/60/60:,.0f} hours of english\")\n", "print(f\"{sum(m['duration_s'] for m in metas if m['views'] >= 100)/60/60:,.0f} hours popular\")\n", "print(\n", " f\"{sum(m['duration_s'] for m in metas if m['lang_guess'] == 'en' and m['views'] >= 100)/60/60:,.0f}\"\n", " f\" hours of english popular\"\n", ")\n", "# 768,515 clips\n", "# 42,975 hours\n", "# 28,878 hours of english\n", "# 27,005 hours popular\n", "# 15,551 hours of english popular" ] }, { "cell_type": "code", "execution_count": 10, "id": "fd2b12b8", "metadata": {}, "outputs": [], "source": [ "# write_jsonl(metas, \"tmp/deezer.jsonl\")" ] }, { "cell_type": "code", "execution_count": 12, "id": "73bc7469", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "upload: tmp/deezer.jsonl to s3://suno-data/datasets/bundles/v2/deezer/metas.jsonl\n" ] } ], "source": [ "# !aws s3 cp tmp/deezer.jsonl s3://suno-data/datasets/bundles/v2/deezer/metas.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "5883419b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8b281d41", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f4292855", "metadata": {}, "source": [ "## (optional) listen to some" ] }, { "cell_type": "code", "execution_count": null, "id": "2f86c962", "metadata": {}, "outputs": [], "source": [ "# LCYthJ3X8d0, dcCAD2pUxw8 has 0 duration lyrics\n", "# Ktzae8-pQo0, LN-Tb5XK61w alignment wrong" ] }, { "cell_type": "code", "execution_count": 4, "id": "f2c7aef7", "metadata": {}, "outputs": [], "source": [ "metas_ytm_tagged = read_jsonl(\"tmp/ytm_tagged.jsonl\") # ~80% more than 100 views\n", "metas_deezer = read_jsonl(\"tmp/deezer.jsonl\") # ~65% more than 100 views" ] }, { "cell_type": "code", "execution_count": 90, "id": "8c87c316", "metadata": {}, "outputs": [], "source": [ "raw_deezer_metas = read_jsonl(\"/mnt/data/georg/tmp/lyrics_dl.jsonl\")" ] }, { "cell_type": "code", "execution_count": 97, "id": "cf63c6d6", "metadata": {}, "outputs": [], "source": [ "metas_deezer = metas_deezer_unique" ] }, { "cell_type": "code", "execution_count": 105, "id": "94960047", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "3" ] }, "execution_count": 105, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len([e for e in raw_deezer_metas if \"success\" not in e[\"ytm\"]])" ] }, { "cell_type": "code", "execution_count": 115, "id": "489b03dc", "metadata": {}, "outputs": [], "source": [ "subset_metas = [m for m in metas_deezer if m[\"id\"] == \"LCYthJ3X8d0\"]\n" ] }, { "cell_type": "code", "execution_count": 110, "id": "69a0fd5f", "metadata": {}, "outputs": [], "source": [ "m = [e for e in raw_deezer_metas if e[\"ytm\"].get(\"success\", False) and e[\"ytm\"][\"id\"] == \"LCYthJ3X8d0\"][0]" ] }, { "cell_type": "code", "execution_count": 111, "id": "a54d908b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'lrcTimestamp': '[00:30.09]',\n", " 'line': 'In meinem Cup ist Bombay (Bom Bom Bombay yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 30090,\n", " 'duration': 90,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[00:34.07]',\n", " 'line': 'Ich bin zu Suff, aber okay (Oh okay yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 34070,\n", " 'duration': 70,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[00:37.89]',\n", " 'line': 'Cali Weed in meine Longpapes (In meinem Papes yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 37890,\n", " 'duration': 90,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[00:41.79]',\n", " 'line': 'Auf meinem T-Shirt Cornflakes (Frosted Flakes) Cornflakes, yeah',\n", " 'lineTranslated': None,\n", " 'milliseconds': 41790,\n", " 'duration': 90,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[00:45.66]',\n", " 'line': 'In meinem Cup ist Bombay (Bom Bom Bombay yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 45660,\n", " 'duration': 130,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[00:49.47]',\n", " 'line': 'Ich bin zu Suff, aber okay (Oh okay yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 49470,\n", " 'duration': 70,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[00:53.40]',\n", " 'line': 'Cali Weed in meine Longpapes (In meinem Papes yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 53400,\n", " 'duration': 120,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[00:57.27]',\n", " 'line': 'Auf meinem T-Shirt Cornflakes (Frosted Flakes) Cornflakes, yeah, yeah',\n", " 'lineTranslated': None,\n", " 'milliseconds': 57270,\n", " 'duration': 90,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:02.54]',\n", " 'line': 'Zünd mir ein Jibbit an (Weed)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 62540,\n", " 'duration': 120,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:03.91]',\n", " 'line': 'Mit Bros, Livestream, Instagram (Aha)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 63910,\n", " 'duration': 120,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:05.87]',\n", " 'line': 'Ich bin ein Rapper, du Zimmermann (UUUH)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 65870,\n", " 'duration': 90,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:07.87]',\n", " 'line': 'Will eine Bitch ohne Kinderkram',\n", " 'lineTranslated': None,\n", " 'milliseconds': 67870,\n", " 'duration': 110,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:10.34]',\n", " 'line': 'Habe das Weed in mein Bag (In mein)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 70340,\n", " 'duration': 110,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:12.13]',\n", " 'line': 'P-Pop eine Xanny, kein Crack (Kein Crack)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 72130,\n", " 'duration': 90,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:14.06]',\n", " 'line': 'Dolce & Gabbana, kein Bape (Kein Fake)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 74060,\n", " 'duration': 140,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:16.04]',\n", " 'line': 'G-Gebe kein Fick ihr seid Fake',\n", " 'lineTranslated': None,\n", " 'milliseconds': 76040,\n", " 'duration': 90,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:18.04]',\n", " 'line': 'Bitch, warum kopiert ihr meine Flows?',\n", " 'lineTranslated': None,\n", " 'milliseconds': 78040,\n", " 'duration': 110,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:19.95]',\n", " 'line': 'Heute ein Millionär damals zu broke (Zu broke)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 79950,\n", " 'duration': 110,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:21.92]',\n", " 'line': 'Damals noch clean, heute bin ich auf Koke (Ey yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 81920,\n", " 'duration': 110,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:23.86]',\n", " 'line': 'Baby, schick mir keine Nudes, ich will Toes',\n", " 'lineTranslated': None,\n", " 'milliseconds': 83860,\n", " 'duration': 110,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:25.72]',\n", " 'line': 'Ficke nur Groupies, kein Rapper (Kein Rapper)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 85720,\n", " 'duration': 160,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:27.52]',\n", " 'line': \"Verbrenne G's, bin kein Gangster (Kein Gangster)\",\n", " 'lineTranslated': None,\n", " 'milliseconds': 87520,\n", " 'duration': 110,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:29.27]',\n", " 'line': 'J-Ja man ich balle im Beamer, chille mit Mango Squad, und rauche Shisha',\n", " 'lineTranslated': None,\n", " 'milliseconds': 89270,\n", " 'duration': 110,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:32.45]',\n", " 'line': 'Ich bin ein Baller (Ein Baller)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 92450,\n", " 'duration': 140,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:34.57]',\n", " 'line': 'Habe das Paper, das Money, das Mula, mein Hoodie von Off-White (Off-White)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 94570,\n", " 'duration': 80,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:38.23]',\n", " 'line': 'Kaufe 4 Jibbit´s und Sippe das Lean, du bist von einen Joint breit (Joint breit)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 98230,\n", " 'duration': 150,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:42.13]',\n", " 'line': 'Smash oder pass, ganz egal, spiele mit ihr als wäre es Fortnite (Fortnite)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 102130,\n", " 'duration': 120,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:45.91]',\n", " 'line': 'Fortnite yeah, yeah',\n", " 'lineTranslated': None,\n", " 'milliseconds': 105910,\n", " 'duration': 170,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:49.05]',\n", " 'line': 'Codein in meinem Double Cup (Codein)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 109050,\n", " 'duration': 90,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:50.94]',\n", " 'line': 'Fruity Haze schmeckt mir nach Bubblegum',\n", " 'lineTranslated': None,\n", " 'milliseconds': 110940,\n", " 'duration': 80,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:52.82]',\n", " 'line': 'Rapper sie spitten mir 1000 Bars (Tausend)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 112820,\n", " 'duration': 110,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:54.89]',\n", " 'line': 'Du bist so Pussy, geh zum Frauenarzt',\n", " 'lineTranslated': None,\n", " 'milliseconds': 114890,\n", " 'duration': 130,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:56.69]',\n", " 'line': 'I-I-In meinem Backpack ist Amsta (Amsta)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 116690,\n", " 'duration': 130,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[01:58.53]',\n", " 'line': 'Ja man ich fick deine Amcas (Amcas)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 118530,\n", " 'duration': 170,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[02:00.62]',\n", " 'line': 'Esse nur Cheddar mit Pasta(Yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 120620,\n", " 'duration': 150,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[02:03.02]',\n", " 'line': 'In meinem Cup ist Bombay (Bom Bom Bombay yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 123020,\n", " 'duration': 110,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[02:06.97]',\n", " 'line': 'Ich bin zu Suff, aber okay (Oh okay yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 126970,\n", " 'duration': 110,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[02:10.81]',\n", " 'line': 'Cali Weed in meine Longpapes (In meinem Papes yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 130810,\n", " 'duration': 70,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[02:14.65]',\n", " 'line': 'Auf meinem T-Shirt Cornflakes (Frosted Flakes) Cornflakes, yeah',\n", " 'lineTranslated': None,\n", " 'milliseconds': 134650,\n", " 'duration': 140,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[02:18.54]',\n", " 'line': 'In meinem Cup ist Bombay (Bom Bom Bombay yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 138540,\n", " 'duration': 110,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[02:22.42]',\n", " 'line': 'Ich bin zu Suff, aber okay (Oh okay yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 142420,\n", " 'duration': 90,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[02:26.28]',\n", " 'line': 'Cali Weed in meine Longpapes (In meinem Papes yeah)',\n", " 'lineTranslated': None,\n", " 'milliseconds': 146280,\n", " 'duration': 120,\n", " '__typename': 'LyricsSynchronizedLine'},\n", " {'lrcTimestamp': '[02:30.19]',\n", " 'line': 'Auf meinem T-Shirt Cornflakes (Frosted Flakes) Cornflakes, yeah, yeah',\n", " 'lineTranslated': None,\n", " 'milliseconds': 150190,\n", " 'duration': 90,\n", " '__typename': 'LyricsSynchronizedLine'}]" ] }, "execution_count": 111, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m[\"data\"][\"track\"][\"lyrics\"][\"synchronizedLines\"]" ] }, { "cell_type": "code", "execution_count": 100, "id": "25cdd8f8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'id': 'WWVtTDUncLg',\n", " 'success': True,\n", " 'retry': False,\n", " 'meta': {'id': 'WWVtTDUncLg',\n", " 'title': 'Just The Way You Are',\n", " 'thumbnail': 'https://i.ytimg.com/vi_webp/WWVtTDUncLg/maxresdefault.webp',\n", " 'description': 'Provided to YouTube by Universal Music Group\\n\\nJust The Way You Are · Barry White\\n\\nBarry White - The Collection\\n\\n℗ A Universal Music Enterprises Release; ℗ 1981 UMG Recordings, Inc.\\n\\nReleased on: 1988-09-12\\n\\nProducer: Barry White\\nComposer Lyricist: Billy Joel\\n\\nAuto-generated by YouTube.',\n", " 'channel_id': 'UCrtwSXAnOmk5TPqxRrmrXTw',\n", " 'channel_url': 'https://www.youtube.com/channel/UCrtwSXAnOmk5TPqxRrmrXTw',\n", " 'duration': 289,\n", " 'view_count': 720123,\n", " 'average_rating': None,\n", " 'age_limit': 0,\n", " 'webpage_url': 'https://www.youtube.com/watch?v=WWVtTDUncLg',\n", " 'categories': ['Music'],\n", " 'tags': ['Barry White',\n", " 'バリーホワイト',\n", " 'バリー・ホワイト',\n", " 'Barry White - The Collection',\n", " 'Just The Way You Are'],\n", " 'playable_in_embed': True,\n", " 'live_status': 'not_live',\n", " 'release_timestamp': None,\n", " '_format_sort_fields': ['quality',\n", " 'res',\n", " 'fps',\n", " 'hdr:12',\n", " 'source',\n", " 'vcodec:vp9.2',\n", " 'channels',\n", " 'acodec',\n", " 'lang',\n", " 'proto'],\n", " 'automatic_captions': {},\n", " 'subtitles': {},\n", " 'album': 'Barry White - The Collection',\n", " 'artist': 'Barry White',\n", " 'track': 'Just The Way You Are',\n", " 'release_date': None,\n", " 'release_year': 1981,\n", " 'comment_count': 64,\n", " 'chapters': None,\n", " 'heatmap': None,\n", " 'like_count': 11853,\n", " 'channel': 'Barry White',\n", " 'channel_follower_count': 444000,\n", " 'channel_is_verified': True,\n", " 'uploader': 'Barry White',\n", " 'uploader_id': None,\n", " 'uploader_url': None,\n", " 'upload_date': '20180807',\n", " 'creator': 'Barry White',\n", " 'alt_title': 'Just The Way You Are',\n", " 'availability': 'public',\n", " 'original_url': 'https://www.youtube.com/watch?v=WWVtTDUncLg',\n", " 'webpage_url_basename': 'watch',\n", " 'webpage_url_domain': 'youtube.com',\n", " 'extractor': 'youtube',\n", " 'extractor_key': 'Youtube',\n", " 'playlist': None,\n", " 'playlist_index': None,\n", " 'display_id': 'WWVtTDUncLg',\n", " 'fulltitle': 'Just The Way You Are',\n", " 'duration_string': '4:49',\n", " 'is_live': False,\n", " 'was_live': False,\n", " 'requested_subtitles': None,\n", " '_has_drm': None,\n", " 'epoch': 1694957846,\n", " 'requested_downloads': [{'asr': 48000,\n", " 'filesize': 5112611,\n", " 'format_id': '251',\n", " 'format_note': 'medium',\n", " 'source_preference': -1,\n", " 'audio_channels': 2,\n", " 'quality': 3.0,\n", " 'has_drm': False,\n", " 'tbr': 141.466,\n", " 'url': 'https://rr5---sn-ab5sznz6.googlevideo.com/videoplayback?expire=1694979445&ei=FQEHZZ6MM8_48gTA3LOoBw&ip=45.114.242.181&id=o-ANiKGt8k2QHlE1Tc1gM6Skv7DQU6xL06zZCV6TFs3Os0&itag=251&source=youtube&requiressl=yes&mh=mV&mm=31%2C29&mn=sn-ab5sznz6%2Csn-ab5l6ndy&ms=au%2Crdu&mv=u&mvi=5&pl=24&gcr=us&spc=UWF9fzcMXNkQkkZ7dp-PNE3buf40iVU&vprv=1&svpuc=1&mime=audio%2Fwebm&gir=yes&clen=5112611&dur=289.121&lmt=1565941851011655&mt=1694956411&fvip=5&keepalive=yes&fexp=24007246&c=ANDROID&txp=1311222&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cgcr%2Cspc%2Cvprv%2Csvpuc%2Cmime%2Cgir%2Cclen%2Cdur%2Clmt&sig=AOq0QJ8wRQIgfP9qzH0lmPiKrGWfHjkQ2tr0keWs6TUMSE54YpfvVpwCIQCBOofuGBzfGJCc11GTg7JngIJVAQtbv1tjY12R5nb0fg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl&lsig=AG3C_xAwRQIhAPyZVJJIVBSzKFMPl6zWA__nhEczIwjmufFd8t2YeermAiAO398EY7WQcLMbth4pRMe_5WuiZ0QCiCKFBtsCT-ojSg%3D%3D',\n", " 'language_preference': -1,\n", " 'ext': 'webm',\n", " 'vcodec': 'none',\n", " 'acodec': 'opus',\n", " 'container': 'webm_dash',\n", " 'downloader_options': {'http_chunk_size': 10485760},\n", " 'protocol': 'https',\n", " 'resolution': 'audio only',\n", " 'http_headers': {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36',\n", " 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n", " 'Accept-Language': 'en-us,en;q=0.5',\n", " 'Sec-Fetch-Mode': 'navigate'},\n", " 'audio_ext': 'webm',\n", " 'video_ext': 'none',\n", " 'vbr': 0,\n", " 'abr': 141.466,\n", " 'format': '251 - audio only (medium)',\n", " '_filename': '/home/victor/web_harvest/web_harvest/ytm/data/audio/WWVtTDUncLg.webm',\n", " 'filename': '/home/victor/web_harvest/web_harvest/ytm/data/audio/WWVtTDUncLg.webm',\n", " '__postprocessors': [],\n", " '__real_download': True,\n", " '__finaldir': '/home/victor/web_harvest/web_harvest/ytm/data/audio',\n", " 'filepath': '/home/victor/web_harvest/web_harvest/ytm/data/audio/WWVtTDUncLg.webm',\n", " '__write_download_archive': True}],\n", " 'asr': 48000,\n", " 'filesize': 5112611,\n", " 'format_id': '251',\n", " 'format_note': 'medium',\n", " 'source_preference': -1,\n", " 'fps': None,\n", " 'audio_channels': 2,\n", " 'height': None,\n", " 'quality': 3.0,\n", " 'has_drm': False,\n", " 'tbr': 141.466,\n", " 'url': 'https://rr5---sn-ab5sznz6.googlevideo.com/videoplayback?expire=1694979445&ei=FQEHZZ6MM8_48gTA3LOoBw&ip=45.114.242.181&id=o-ANiKGt8k2QHlE1Tc1gM6Skv7DQU6xL06zZCV6TFs3Os0&itag=251&source=youtube&requiressl=yes&mh=mV&mm=31%2C29&mn=sn-ab5sznz6%2Csn-ab5l6ndy&ms=au%2Crdu&mv=u&mvi=5&pl=24&gcr=us&spc=UWF9fzcMXNkQkkZ7dp-PNE3buf40iVU&vprv=1&svpuc=1&mime=audio%2Fwebm&gir=yes&clen=5112611&dur=289.121&lmt=1565941851011655&mt=1694956411&fvip=5&keepalive=yes&fexp=24007246&c=ANDROID&txp=1311222&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cgcr%2Cspc%2Cvprv%2Csvpuc%2Cmime%2Cgir%2Cclen%2Cdur%2Clmt&sig=AOq0QJ8wRQIgfP9qzH0lmPiKrGWfHjkQ2tr0keWs6TUMSE54YpfvVpwCIQCBOofuGBzfGJCc11GTg7JngIJVAQtbv1tjY12R5nb0fg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl&lsig=AG3C_xAwRQIhAPyZVJJIVBSzKFMPl6zWA__nhEczIwjmufFd8t2YeermAiAO398EY7WQcLMbth4pRMe_5WuiZ0QCiCKFBtsCT-ojSg%3D%3D',\n", " 'width': None,\n", " 'language': None,\n", " 'language_preference': -1,\n", " 'preference': None,\n", " 'ext': 'webm',\n", " 'vcodec': 'none',\n", " 'acodec': 'opus',\n", " 'dynamic_range': None,\n", " 'container': 'webm_dash',\n", " 'downloader_options': {'http_chunk_size': 10485760},\n", " 'protocol': 'https',\n", " 'resolution': 'audio only',\n", " 'aspect_ratio': None,\n", " 'http_headers': {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36',\n", " 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n", " 'Accept-Language': 'en-us,en;q=0.5',\n", " 'Sec-Fetch-Mode': 'navigate'},\n", " 'audio_ext': 'webm',\n", " 'video_ext': 'none',\n", " 'vbr': 0,\n", " 'abr': 141.466,\n", " 'format': '251 - audio only (medium)'},\n", " 'audio_filename': 'WWVtTDUncLg.webm',\n", " 'runtime_s': 5.0}" ] }, "execution_count": 100, "metadata": {}, "output_type": "execute_result" } ], "source": [ "raw_deezer_metas[0][\"ytm\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "d724fc33", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 87, "id": "ced328bf", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'text': 'In einem unbekannten Land', 'start_s': 1.27, 'end_s': 5.9},\n", " {'text': 'Vor gar nicht allzu langer Zeit', 'start_s': 5.9, 'end_s': 10.84},\n", " {'text': 'War eine Biene sehr bekannt', 'start_s': 10.84, 'end_s': 15.54},\n", " {'text': 'Von der sprach alles weit und breit',\n", " 'start_s': 15.54,\n", " 'end_s': 20.4},\n", " {'text': 'Und diese Biene, die ich meine nennt sich Maja',\n", " 'start_s': 20.4,\n", " 'end_s': 28.05},\n", " {'text': 'Kleine freche schlaue Biene Maja',\n", " 'start_s': 28.05,\n", " 'end_s': 32.63},\n", " {'text': 'Maja fliegt durch ihre Welt', 'start_s': 32.63, 'end_s': 37.86},\n", " {'text': 'Zeigt uns das was ihr gefällt', 'start_s': 37.86, 'end_s': 41.35},\n", " {'text': 'Wir treffen heute unsere Freundin Biene Maja',\n", " 'start_s': 41.35,\n", " 'end_s': 47.15},\n", " {'text': 'Diese kleine freche schlaue Maja',\n", " 'start_s': 47.15,\n", " 'end_s': 52.29},\n", " {'text': 'Maja alle lieben Maja', 'start_s': 52.29, 'end_s': 57.48},\n", " {'text': 'Maja (Maja) Maja (Maja)', 'start_s': 57.48, 'end_s': 62.68},\n", " {'text': 'Maja erzähle uns von dir', 'start_s': 62.68, 'end_s': 73.31}]" ] }, "execution_count": 87, "metadata": {}, "output_type": "execute_result" } ], "source": [ "subset_metas[0][\"lyrics_lines\"]" ] }, { "cell_type": "code", "execution_count": null, "id": "dd848ee8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 66, "id": "816a72c4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "25153" ] }, "execution_count": 66, "metadata": {}, "output_type": "execute_result" } ], "source": [ "subset_metas = [m for m in metas_deezer if m[\"views\"] >= 100 and m[\"lang_guess\"] == \"de\"]\n", "len(subset_metas)" ] }, { "cell_type": "code", "execution_count": 80, "id": "64df3c50", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'id': 'LN-Tb5XK61w',\n", " 's3_filepath': 's3://suno-data/datasets/harvest/ytm_synced_lyrics/audio/LN-Tb5XK61w.webm',\n", " 'duration_s': 182,\n", " 'views': 8324,\n", " 'lang_guess': 'de',\n", " 'lang_p': 0.997}" ] }, "execution_count": 80, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m = random.choice(subset_metas)\n", "{k:v for k, v in m.items() if k != \"lyrics_lines\"}" ] }, { "cell_type": "code", "execution_count": 81, "id": "d9f2ab54", "metadata": {}, "outputs": [], "source": [ "a = Audio.from_s3(m[\"s3_filepath\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "4706d404", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 82, "id": "091cf8e3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ ",Bad Boy‘ akustik, yeah“\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Die dir nie sagen, wo sie grad sind\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Lass deine Eltern nicht den Chatverlauf sehen\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Immer wieder ein' Bad Boy (immer wieder und wieder)\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Weil die Sache für ihn nur low-key ist\n" ] } ], "source": [ "for _ in range(5):\n", " ml = random.choice(m[\"lyrics_lines\"])\n", " print(\"-\"*10)\n", " a.play(from_s=ml[\"start_s\"], to_s=ml[\"end_s\"])\n", " print(ml[\"text\"])\n", "# break" ] }, { "cell_type": "code", "execution_count": null, "id": "6b4a0b2e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e2fada1a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "317d961e", "metadata": {}, "source": [ "### unaligned genius stuff" ] }, { "cell_type": "code", "execution_count": null, "id": "8e2b9615", "metadata": {}, "outputs": [], "source": [ "# !aws s3 ls s3://suno-data/datasets/harvest/genius_hq/" ] }, { "cell_type": "code", "execution_count": null, "id": "098a8dfb", "metadata": {}, "outputs": [], "source": [ "!aws s3 ls s3://suno-data/datasets/harvest/genius/song_details_with_youtube.jsonl" ] }, { "cell_type": "code", "execution_count": 3, "id": "0b92b436", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " PRE audio/\r\n", " PRE bk_audio/\r\n", " PRE de_project/\r\n", " PRE logs/\r\n", " PRE raw_html/\r\n", "2022-10-23 13:30:37 36674349 artist_links.json\r\n", "2022-10-23 13:30:42 98659265 artist_metas.json\r\n", "2022-10-23 13:30:42 93903684 artist_metas_1.json\r\n", "2022-10-23 13:30:36 5404082 artist_metas_2.json\r\n", "2022-10-23 13:33:02 3865528120 artist_songs.jsonl\r\n", "2022-10-23 13:39:15 12018922013 bk_song_details.jsonl\r\n", "2022-10-23 13:37:39 9008190363 pairs_metadata.jsonl\r\n", "2022-10-23 13:39:34 12800551780 song_details.jsonl\r\n", "2022-10-23 13:36:03 6426020721 song_details_with_youtube.jsonl\r\n", "2022-10-23 13:59:17 132338419444 youtube_metas.jsonl\r\n", "2022-10-23 13:44:38 32169150511 youtube_metas_bk.jsonl\r\n" ] } ], "source": [ "!aws s3 ls s3://suno-data/datasets/harvest/genius/song_details_with_youtube.jsonl" ] }, { "cell_type": "code", "execution_count": 4, "id": "24d9b8e6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " PRE audio/\r\n", "2023-05-20 09:31:38 8451494901 pairs_metadata.jsonl\r\n", "2023-05-20 08:25:07 155117285991 youtube_metas.jsonl\r\n" ] } ], "source": [ "!aws s3 ls s3://suno-data/datasets/harvest/genius_hq/pairs_metadata.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "2a20e4c4", "metadata": {}, "outputs": [], "source": [ "import os\n", "from suno_utils.utils.s3 import read_from_s3\n", "from suno_utils.utils.text import read_jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "d89d9463", "metadata": {}, "outputs": [], "source": [ "metas_1 = read_from_s3(\"s3://suno-data/datasets/harvest/genius/song_details.jsonl\", read_f=read_jsonl)" ] }, { "cell_type": "code", "execution_count": null, "id": "4726a44d", "metadata": {}, "outputs": [], "source": [ "metas_2 = read_from_s3(\"s3://suno-data/datasets/harvest/genius_hq/pairs_metadata.jsonl\", read_f=read_jsonl)\n", "metas_3 = read_from_s3(\"s3://suno-data/datasets/harvest/genius_hq/youtube_metas.jsonl\", read_f=read_jsonl)" ] }, { "cell_type": "code", "execution_count": null, "id": "cdf58ab3", "metadata": {}, "outputs": [], "source": [ "metas_4 = read_jsonl(os.path.join(os.getcwd(), \"metadata\", \"genius_hq.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "25114778", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 20, "id": "9ca8966a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5480217\n", "4831903\n" ] } ], "source": [ "print(len(metas_1))\n", "sub_metas = [m for m in metas_1 if len(m.get(\"meta\", {}).get(\"lyrics\", \"\")) > 500]\n", "print(len(sub_metas))" ] }, { "cell_type": "code", "execution_count": null, "id": "3918c5ac", "metadata": {}, "outputs": [], "source": [ "2090009" ] }, { "cell_type": "code", "execution_count": 24, "id": "aa1861c6", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'success': True,\n", " 'song_slug': '00s-psycho-lyrics',\n", " 'meta': {'id': '5971062',\n", " 'youtube_url': 'http://www.youtube.com/watch?v=4YkNeeUIQPc',\n", " 'youtube_start': '0',\n", " 'lang': 'en',\n", " 'lang_lyrics': 'en',\n", " 'views': 1081,\n", " 'tags': [3937, 3930, 1472, 1525, 2633, 2674, 16],\n", " 'title': 'Psycho',\n", " 'artist': '00s (빵빵즈)',\n", " 'lyrics': '[\"Psycho\" Cover 가사]\\n\\n[Intro: Daehwi, Hyunjin]\\nOoh, ooh, hey, yeah\\nHmm, yeah, psycho\\n\\n[Verse 1: Bomin, Daehwi]\\n널 어쩌면 좋을까\\n이런 맘은 또 첨이라\\nUp and down이 좀 심해\\n조절이 자꾸 잘 안돼\\n하나 확실한 건\\nI don’t play the game\\n[Refrain: Hyunjin, Sanha]\\n우리 진짜 별나대\\n그냥 내가 너무 좋아해\\n넌 그걸 너무 잘 알고\\n날 쥐락펴락해\\n나도 마찬가지인걸 (Oh-oh-oh)\\n\\n[Pre-Chorus: Hyunjin, Daehwi, Sanha]\\n우린 참 별나고 이상한 사이야\\n서로를 부서지게 (부서지게)\\n그리곤 또 껴안아 (그리곤 또 껴안아)\\n\\n[Chorus: All]\\nYou got me feeling like a psycho, psycho\\n우릴 보고 말해 자꾸 자꾸\\n다시 안 볼 듯 싸우다가도\\n붙어 다니니 말야\\n이해가 안 간대\\n웃기지도 않대\\n\\n[Post-Chorus: All]\\n맞아 psycho, psycho (Psycho, psycho)\\n서로 좋아 죽는 바보 바보 (바보, 바보)\\n너 없인 어지럽고 슬퍼져\\n기운도 막 없어요\\n둘이 잘 만났대\\nHey, now we’ll be okay\\n\\n\\n[Verse 2: Bomin, Hyunjin, Sanha]\\nHey, trouble\\n경고 따윈 없이 오는 너\\nI’m original, visual\\n우린 원래 이랬어, yeah\\n두렵지는 않아 (흥미로울 뿐)\\nIt’s hot! Let me just hop\\n어떻게 널 다룰까? Ooh, ooh, ooh\\n\\n[Refrain 2: Daehwi, Bomin]\\n어쩔 줄을 몰라 너를 달래고\\n매섭게 발로 차도\\n가끔 내게 미소 짓는 널\\n어떻게 놓겠어? Ooh, ooh\\n\\n[Pre-Chorus: Hyunjin, Sanha, Daehwi]\\n우린 아름답고 참 슬픈 사이야\\n서로를 빛나게 해 (Tell me now)\\n마치 달과 강처럼 (그리곤 또 껴안아, yeah)\\n\\n[Chorus: All]\\nYou got me feeling like a psycho, psycho\\n우릴 보고 말해 자꾸 자꾸\\n다시 안 볼 듯 싸우다가도\\n붙어 다니니 말야\\n이해가 안 간대\\n웃기지도 않대\\n\\n\\n[Post-Chorus: All]\\n맞아 psycho, psycho (맞아 psycho, psycho)\\n서로 좋아 죽는 바보 바보 (바보, 바보)\\n너 없인 어지럽고 슬퍼져\\n기운도 막 없어요\\n둘이 잘 만났대\\nHey, now we’ll be okay\\n\\n[Bridge: Bomin, Sanha]\\nDon’t look back\\n그렇게 우리답게 가보자\\n난 온몸으로 널 느끼고 있어\\nEverything will be okay, woo!\\n\\n[Chorus: Daehwi, All]\\nYou got me feeling like a psycho, psycho\\n우릴 보고 말해 자꾸 자꾸 (보고 말해 자꾸, ooh)\\n다시 안 볼 듯 싸우다가도\\n붙어 다니니 말야\\n둘이 잘 만났대\\nHey, now we’ll be okay\\n\\n[Post-Chorus: All, Bomin, Sanha, Daehwi]\\nHey, now we’ll be okay (We\\'ll be okay)\\nHey, now we’ll be okay (We\\'ll be okay)\\nHey, now we’ll be okay (We\\'ll be okay)\\nHey, now we’ll be okay\\nIt’s alright (It\\'s alright; ooh)\\nIt’s alright (Ooh)\\n\\n[Post-Chorus: All, Daehwi, Hyunjin]\\nHey, now we’ll be okay (Hey)\\nHey, now we’ll be okay (Okay, yeah)\\nHey, now we’ll be okay (Hey)\\nHey, now we’ll be okay (Okay)\\nIt’s alright (Oh, woah, oh)\\nIt’s alright\\n\\n[Outro: Bomin]\\n우린 좀 이상해 (Psycho)',\n", " 'youtube_url_2': 'http://www.youtube.com/watch?v=4YkNeeUIQPc'}}" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sub_metas[123]" ] }, { "cell_type": "code", "execution_count": 22, "id": "7778a07d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2150685" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(metas_2)" ] }, { "cell_type": "code", "execution_count": 23, "id": "be0d1825", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1377329" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(metas_4)" ] }, { "cell_type": "code", "execution_count": null, "id": "42f41dab", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e45cdc71", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "aaf8711f", "metadata": {}, "source": [ "## ytm_mb" ] }, { "cell_type": "code", "execution_count": 59, "id": "53ba6c4f", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████| 37197315/37197315 [21:08<00:00, 29315.44it/s]\n" ] } ], "source": [ "metas = []\n", "seen_ids = set()\n", "with open(\"/mnt/data/georg/tmp/large_dl_raw.jsonl\") as f:\n", " for line in tqdm.tqdm(f, total=37_197_315):\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " if \"ytm\" not in m or not m[\"ytm\"][\"success\"] or m[\"videoId\"] in seen_ids:\n", " continue\n", " new_m = {\n", " \"id\": m[\"videoId\"],\n", " \"s3_filepath\": os.path.join(\n", " \"s3://suno-data/datasets/harvest/ytm_large/audio/\", \n", " m[\"ytm\"][\"audio_filename\"]\n", " ),\n", " \"duration_s\": m[\"ytm\"][\"meta\"][\"duration\"],\n", " \"views\": m[\"ytm\"][\"meta\"][\"view_count\"],\n", " \"title\": m[\"ytm\"][\"meta\"][\"title\"],\n", " }\n", " artist = m[\"ytm\"][\"meta\"].get(\"artist\")\n", " if artist is not None:\n", " new_m[\"artist\"] = artist\n", " metas.append(new_m)\n", " seen_ids.add(m[\"videoId\"])" ] }, { "cell_type": "code", "execution_count": 60, "id": "6e3d0467", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "34,808,457 files retained - 2,240,115 hours total\n", "2,061,482 conservative hours\n" ] } ], "source": [ "random.seed(6006)\n", "random.shuffle(metas)\n", "print(f\"{len(metas):,} files retained - {sum(m['duration_s'] for m in metas)/60/60:,.0f} hours total\")\n", "print(f\"{sum(m['duration_s'] for m in metas if 1*60 <= m['duration_s'] <= 8*60)/60/60:,.0f} conservative hours\")\n", "# 34,808,457 files retained - 2,240,115 hours total\n", "# 2,061,482 conservative hours" ] }, { "cell_type": "code", "execution_count": 82, "id": "9c4fd8da", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "26,564,262 files retained - 1,670,585 hours\n" ] } ], "source": [ "# retain ~80% above views threshold\n", "metas = [m for m in metas if m[\"views\"] >= 25 and 1*60 <= m['duration_s'] <= 8*60]\n", "print(f\"{len(metas):,} files retained - {sum(m['duration_s'] for m in metas)/60/60:,.0f} hours\")\n", "# 26,564,262 files retained - 1,670,585 hours" ] }, { "cell_type": "code", "execution_count": 83, "id": "7ca49fac", "metadata": {}, "outputs": [], "source": [ "# write_jsonl(metas, \"/mnt/data/georg/tmp/metas_ytm_mb.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "6aa36897", "metadata": {}, "outputs": [], "source": [ "# aws s3 cp /mnt/data/georg/tmp/metas_ytm_mb.jsonl s3://suno-data/datasets/bundles/v2/ytm_mb/metas.jsonl" ] }, { "cell_type": "markdown", "id": "bd8b8705", "metadata": {}, "source": [ "#### do some more eda" ] }, { "cell_type": "code", "execution_count": 65, "id": "6e67e81a", "metadata": {}, "outputs": [], "source": [ "# from joblib import Parallel, delayed\n", "# out = Parallel(n_jobs=20, prefer=\"threads\")(\n", "# delayed(check_s3_file_exists)(m[\"s3_filepath\"]) for m in metas[:1000]\n", "# )\n", "# print(f\"{sum(out)/len(out)*100:.1f}% available\")" ] }, { "cell_type": "code", "execution_count": 81, "id": "63ca193d", "metadata": {}, "outputs": [], "source": [ "# s = pd.Series([m[\"views\"] for m in metas])\n", "# s.clip(upper=1_000).hist(bins=30)" ] }, { "cell_type": "code", "execution_count": null, "id": "45b6e581", "metadata": {}, "outputs": [], "source": [ "# # listen to a few with super low view count\n", "# low_views = [m for m in metas if m[\"views\"] <= 10]\n", "# random.choice(low_views)" ] }, { "cell_type": "code", "execution_count": null, "id": "b5399936", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8d988bc7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6ae7996e", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 5 }