{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "d31d8850", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:42:48.498319Z", "start_time": "2024-01-18T02:42:48.496832Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:22:06.162764Z", "iopub.status.busy": "2025-01-14T00:22:06.162624Z", "iopub.status.idle": "2025-01-14T00:22:06.167027Z", "shell.execute_reply": "2025-01-14T00:22:06.166610Z", "shell.execute_reply.started": "2025-01-14T00:22:06.162748Z" } }, "outputs": [], "source": [ "import os\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "7cd4c0dd", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:42:53.666867Z", "start_time": "2024-01-18T02:42:48.821513Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:22:06.167729Z", "iopub.status.busy": "2025-01-14T00:22:06.167600Z", "iopub.status.idle": "2025-01-14T00:22:12.274113Z", "shell.execute_reply": "2025-01-14T00:22:12.273582Z", "shell.execute_reply.started": "2025-01-14T00:22:06.167715Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/tony/anaconda3/envs/suno_env_dev/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n", " warnings.warn(\n" ] } ], "source": [ "import tqdm\n", "import math\n", "import torch\n", "import random\n", "import funcy\n", "import copy\n", "import gc\n", "import re\n", "import json\n", "import tempfile\n", "import collections\n", "from collections import Counter\n", "import pandas as pd\n", "import numpy as np\n", "import fasttext\n", "from joblib import Parallel, delayed\n", "import matplotlib.pyplot as plt\n", "\n", "from suno_utils.audio import Audio\n", "from suno_utils.utils.text import (\n", " write_jsonl,\n", " read_jsonl,\n", " write_json,\n", " read_json,\n", " normalize_whitespace,\n", ")\n", "from suno_utils.utils.lyrics import remove_speakers\n", "from suno_utils.utils.s3 import read_from_s3, check_s3_file_exists, open_from_s3\n", "from suno_utils.utils.tokenizers import tokenize\n", "from suno_utils.harvest.youtube.constants.text_lang import BASE_TO_FASTTEXT_REMAP\n", "from suno_utils.utils.metrics import get_cer\n", "from suno_utils.tasks.hoot import (\n", " parse_lyrics,\n", " legacy_parse_lyrics,\n", " EMBEDDING_RATE as HOOT_EMBEDDING_RATE,\n", ")\n", "from suno_utils.utils.lyrics import remove_speakers" ] }, { "cell_type": "code", "execution_count": 3, "id": "d6518a0b", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:42:56.344509Z", "start_time": "2024-01-18T02:42:53.668461Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:22:12.274982Z", "iopub.status.busy": "2025-01-14T00:22:12.274701Z", "iopub.status.idle": "2025-01-14T00:22:12.277428Z", "shell.execute_reply": "2025-01-14T00:22:12.277039Z", "shell.execute_reply.started": "2025-01-14T00:22:12.274965Z" } }, "outputs": [], "source": [ "# If run language detection, uncomment this\n", "# LANG_ID_MODEL_FP = \"s3://suno-data/georg/trained_models/chirp_v1/lid.176.bin\"\n", "# text_lang_model = read_from_s3(LANG_ID_MODEL_FP, read_f=fasttext.load_model)\n", "# def _get_text_lang(text):\n", "# \"\"\"get probability of input language for text\"\"\"\n", "# text = text.replace(\"’\", \"'\").lower()\n", "# text = re.sub(r\"\\[.+?\\]\", \" \", text)\n", "# text = normalize_whitespace(text)\n", "# out = text_lang_model.predict(text, k=1)\n", "# lang_str = out[0][0]\n", "# p_lang = out[1][0]\n", "# lang = lang_str.split(\"__\")[-1]\n", "# lang = BASE_TO_FASTTEXT_REMAP.get(lang, lang)\n", "# # if p_lang >= 0.8:\n", "# # return lang\n", "# return p_lang, lang" ] }, { "cell_type": "markdown", "id": "d9d50793", "metadata": {}, "source": [ "## load genius_hq info" ] }, { "cell_type": "code", "execution_count": 4, "id": "2cd505d5", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:09:50.716502Z", "start_time": "2024-01-09T15:08:04.096829Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:22:12.277994Z", "iopub.status.busy": "2025-01-14T00:22:12.277868Z", "iopub.status.idle": "2025-01-14T00:23:25.412046Z", "shell.execute_reply": "2025-01-14T00:23:25.411243Z", "shell.execute_reply.started": "2025-01-14T00:22:12.277980Z" } }, "outputs": [], "source": [ "# takes ~ 2 mins\n", "base_metas = read_jsonl(\"/app/suno/tmp/clean_discogs_v0_metas.jsonl\")\n", "# TODO: original IDs are already unique here, but did we resolve correctly by highest genius views?\n", "# could be a way to filter translations that don't contain the word 'translation'" ] }, { "cell_type": "code", "execution_count": 5, "id": "3946b51f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "dict_keys(['id', 's3_filepath', 'duration_s', 'artists'])\n" ] } ], "source": [ "print(base_metas[100].keys())" ] }, { "cell_type": "code", "execution_count": 6, "id": "06460337", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "52220969" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(base_metas)" ] }, { "cell_type": "code", "execution_count": 7, "id": "251edcf4", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:09:54.750823Z", "start_time": "2024-01-09T15:09:50.718362Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:25.413069Z", "iopub.status.busy": "2025-01-14T00:23:25.412912Z", "iopub.status.idle": "2025-01-14T00:23:31.877797Z", "shell.execute_reply": "2025-01-14T00:23:31.877220Z", "shell.execute_reply.started": "2025-01-14T00:23:25.413054Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total, 52220969\n", "lang cut, 45488127 (87.11%)\n", "duration cut, 2948005 (5.65%)\n", "lyrics cut, 45494688 (87.12%)\n" ] } ], "source": [ "total_base_metas = len(base_metas)\n", "n_genius_view_cut = 5\n", "n_youtube_view_cut = 50\n", "max_duration = 8 * 60\n", "min_duration = 1 * 60\n", "max_len_lyrics = 9128\n", "min_len_lyrics = 12\n", "n_total_base_metas = len(base_metas)\n", "print(f\"total, {n_total_base_metas}\")\n", "lang_cut_count = sum(1 if m.get(\"lang\") is None else 0 for m in base_metas)\n", "duration_cut_count = sum(\n", " 1\n", " if (m.get(\"duration_s\", 0) > max_duration or m.get(\"duration_s\", 0) < min_duration)\n", " else 0\n", " for m in base_metas\n", ")\n", "lyrics_cut_count = sum(\n", " 1\n", " if (\n", " len(m.get(\"text\", \"\")) > max_len_lyrics\n", " or len(m.get(\"text\", \"\")) < min_len_lyrics\n", " )\n", " else 0\n", " for m in base_metas\n", ")\n", "\n", "print(\n", " f\"lang cut, {lang_cut_count} ({round(lang_cut_count / n_total_base_metas * 100, 2)}%)\"\n", ")\n", "print(\n", " f\"duration cut, {duration_cut_count} ({round(duration_cut_count / n_total_base_metas * 100, 2)}%)\"\n", ")\n", "print(\n", " f\"lyrics cut, {lyrics_cut_count} ({round(lyrics_cut_count / n_total_base_metas * 100, 2)}%)\"\n", ")" ] }, { "cell_type": "code", "execution_count": 8, "id": "6ac295a0", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:01.309149Z", "start_time": "2024-01-09T15:09:54.752976Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:31.879727Z", "iopub.status.busy": "2025-01-14T00:23:31.879303Z", "iopub.status.idle": "2025-01-14T00:23:39.048856Z", "shell.execute_reply": "2025-01-14T00:23:39.048248Z", "shell.execute_reply.started": "2025-01-14T00:23:31.879710Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "52220969 clips\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 5%|▌ | 2705208/52220969 [00:01<00:22, 2194215.12it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "{'id': 'W3JRaGV42Rg', 'lyrics': 'Pusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on aika suudelmain\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on tunne voimakkain\\n\\nKun kansa tuppisuuna naamat mököttäin\\nTungoksessa hiipii töistä kotiinpäin\\nSilloin toisiamme niin kuin kiusallamme\\nMuiden nähden suudellaan\\n\\nJa mitä nyrpeämmän tyypin näämmekin\\nSuutelemaan juuri siihen jäämmekin\\nSyrjään väisty emme, naurain suutelemme\\nHuomaamatta ruuhkaakaan\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on aika suudelman\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on tunne voimakkain\\n\\nKun töihin hapannaamat ryntää aamuisin\\nOikein hyvin pyyhkii meillä silloinkin\\nKun on tapanamme että aloitamme\\nSuukoin päivän jokaisen\\n\\nKun tiukat merkonomit pitkään katselee\\nJotkut pikkusielut heistä vilkaisee\\nPaheksuen meitä, ärsytämme heitä\\nVielä kerran suudellen\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on aika suudelman\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on tunne voimakkain\\n\\nPusi pusi aamuin, pusi pusi illoin\\nPusi pusi milloin vain\\nPusi pusi silloin kun on aika suudelmain', 'duration_s': 184, 'lang': 'fi', 'audio_filepath': 's3://suno-data/shared/nfdg/W3JRaGV42Rg/W3JRaGV42Rg/audio.webm'}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 52220969/52220969 [00:25<00:00, 2087609.16it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "6727389 filtered clips\n", "419833 hours\n" ] } ], "source": [ "print(len(base_metas), \"clips\")\n", "metas = []\n", "seen_id = set()\n", "seen_path = set()\n", "for m in tqdm.tqdm(base_metas):\n", " # guessed_lang = _get_text_lang(m[\"lyrics\"])[1]\n", " if (\n", " # m[\"genius_views\"] < n_genius_view_cut\n", " # or m[\"youtube_views\"] < n_youtube_view_cut\n", " # m.get(\"lang\") is None\n", " # or \"translation\" in m[\"genius_slug\"].lower()\n", " # or any(\"translation\" in tag.lower() for tag in m[\"tags_text\"])\n", " # or m[\"lang\"].lower()[:2] != \"en\"\n", " # or m.get(\"duration_s\", 0) < min_duration\n", " # or m.get(\"duration_s\", 0) > max_duration\n", " len(m.get(\"text\", \"\")) < min_len_lyrics\n", " # or len(m.get(\"text\", \"\")) > max_len_lyrics\n", " or m[\"id\"] in seen_id\n", " or m[\"s3_filepath\"] in seen_path\n", " # or _get_text_lang(m[\"lyrics\"])[1] != \"en\"\n", " ):\n", " continue\n", " seen_id.add(m[\"id\"])\n", " seen_path.add(m[\"s3_filepath\"])\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"lyrics\": m[\"text\"].strip(),\n", " \"duration_s\": m[\"duration_s\"],\n", " \"lang\": m[\"lang\"],\n", " # \"guess_lang\": guessed_lang,\n", " \"audio_filepath\": m[\"s3_filepath\"],\n", " # \"original_id\": m[\"original_id\"],\n", " # \"genius_slug\": m[\"genius_slug\"],\n", " # \"genius_views\": m[\"genius_views\"],\n", " # \"youtube_views\": m[\"youtube_views\"],\n", " }\n", " metas.append(new_m)\n", "print(len(metas), \"filtered clips\")\n", "print(round(sum(m[\"duration_s\"] for m in metas) / 60 / 60), \"hours\")\n", "metas_map = {m[\"id\"]: m for m in metas}\n", "# Discogs\n", "# 2154863 filtered clips\n", "# 136385 hours\n", "# full dicogs\n", "# 6727389 filtered clips\n", "# 419833 hours" ] }, { "cell_type": "code", "execution_count": 1, "id": "710eb01c", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:01.313613Z", "start_time": "2024-01-09T15:10:01.311434Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:39.049576Z", "iopub.status.busy": "2025-01-14T00:23:39.049427Z", "iopub.status.idle": "2025-01-14T00:23:39.051784Z", "shell.execute_reply": "2025-01-14T00:23:39.051377Z", "shell.execute_reply.started": "2025-01-14T00:23:39.049560Z" } }, "outputs": [], "source": [ "# # only need to save this json once -- or you need to update\n", "# with open(\"/home/tony/Data/Hoot/discogs_full_metas.json\", \"w\") as fp:\n", "# json.dump(metas, fp)" ] }, { "cell_type": "code", "execution_count": null, "id": "aa66a026", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:02.592483Z", "start_time": "2024-01-09T15:10:01.314852Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:39.052404Z", "iopub.status.busy": "2025-01-14T00:23:39.052272Z", "iopub.status.idle": "2025-01-14T00:23:40.445874Z", "shell.execute_reply": "2025-01-14T00:23:40.445288Z", "shell.execute_reply.started": "2025-01-14T00:23:39.052391Z" } }, "outputs": [], "source": [ "# check language counts\n", "c = Counter([m[\"lang\"] for m in metas])\n", "print(c.most_common(10))\n", "# filter out languages with less than 50 clips\n", "metas_map = {m[\"id\"]: m for m in metas if c[m[\"lang\"]] > 50}" ] }, { "cell_type": "code", "execution_count": null, "id": "2b486253", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:19.399035Z", "start_time": "2024-01-09T15:10:02.594795Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:40.446601Z", "iopub.status.busy": "2025-01-14T00:23:40.446446Z", "iopub.status.idle": "2025-01-14T00:23:43.203204Z", "shell.execute_reply": "2025-01-14T00:23:43.202628Z", "shell.execute_reply.started": "2025-01-14T00:23:40.446584Z" } }, "outputs": [], "source": [ "print(len(metas_map), \"filtered clips\")\n", "c = Counter([m[\"lang\"] for m in metas_map.values()])\n", "for k, v in c.most_common(10):\n", " print(\n", " k,\n", " c[k],\n", " round(sum(m[\"duration_s\"] for m in metas if m[\"lang\"] == k) / 60 / 60),\n", " \"hours\",\n", " )" ] }, { "cell_type": "code", "execution_count": null, "id": "c2ce5d00", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:19.404006Z", "start_time": "2024-01-09T15:10:19.401076Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.203888Z", "iopub.status.busy": "2025-01-14T00:23:43.203738Z", "iopub.status.idle": "2025-01-14T00:23:43.206949Z", "shell.execute_reply": "2025-01-14T00:23:43.206498Z", "shell.execute_reply.started": "2025-01-14T00:23:43.203872Z" } }, "outputs": [], "source": [ "for m in metas:\n", " if m[\"lang\"] == \"zh\":\n", " print(m)\n", " break" ] }, { "cell_type": "markdown", "id": "346cc768", "metadata": {}, "source": [ "# Pack the data\n", "format to save to:\n", "{\"audio_filepath\": \"./an4/wav/an4_clstk/fash/an251-fash-b.wav\", \"duration\": 1.0, \"text\": \"yes\"}\n" ] }, { "cell_type": "code", "execution_count": 11, "id": "71df5d56", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:20.258192Z", "start_time": "2024-01-09T15:10:19.486945Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.207648Z", "iopub.status.busy": "2025-01-14T00:23:43.207505Z", "iopub.status.idle": "2025-01-14T00:23:43.244969Z", "shell.execute_reply": "2025-01-14T00:23:43.244585Z", "shell.execute_reply.started": "2025-01-14T00:23:43.207634Z" } }, "outputs": [], "source": [ "# # this will also take 2 min...\n", "# pre_download_val_metas = read_jsonl(\"/app/suno/data/audio_2ch_48khz_lg/metas_val.jsonl\")\n", "# pre_download_train_metas = read_jsonl(\n", "# \"/app/suno/data/audio_2ch_48khz_lg/metas_tr.jsonl\"\n", "# )\n", "# downloaded_ids_to_info = {}\n", "# for download_meta in pre_download_train_metas:\n", "# if download_meta[\"original_id\"] not in downloaded_ids_to_info:\n", "# downloaded_ids_to_info[download_meta[\"original_id\"]] = download_meta\n", "# for download_meta in pre_download_val_metas:\n", "# if download_meta[\"original_id\"] not in downloaded_ids_to_info:\n", "# downloaded_ids_to_info[download_meta[\"original_id\"]] = download_meta\n", "# downloaded_ids = set(downloaded_ids_to_info.keys())\n", "# print(len(downloaded_ids))" ] }, { "cell_type": "code", "execution_count": 12, "id": "9870b504", "metadata": { "execution": { "iopub.execute_input": "2025-01-14T00:23:43.245543Z", "iopub.status.busy": "2025-01-14T00:23:43.245415Z", "iopub.status.idle": "2025-01-14T00:23:43.278868Z", "shell.execute_reply": "2025-01-14T00:23:43.278489Z", "shell.execute_reply.started": "2025-01-14T00:23:43.245530Z" } }, "outputs": [], "source": [ "# sum(\n", "# download_meta[\"dataset\"] == \"genius_hq\"\n", "# for download_meta in pre_download_train_metas\n", "# )\n", "# subset_meta = [\n", "# m for m in metas if (m[\"id\"] in downloaded_ids)\n", "# ] # and m[\"lang\"] == \"en\")]\n", "# print(len(subset_meta))\n", "# 986464" ] }, { "cell_type": "code", "execution_count": 13, "id": "c8227733", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:20.872851Z", "start_time": "2024-01-09T15:10:20.261963Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.279446Z", "iopub.status.busy": "2025-01-14T00:23:43.279310Z", "iopub.status.idle": "2025-01-14T00:23:43.323282Z", "shell.execute_reply": "2025-01-14T00:23:43.322846Z", "shell.execute_reply.started": "2025-01-14T00:23:43.279433Z" } }, "outputs": [], "source": [ "subset_meta = metas.copy()" ] }, { "cell_type": "code", "execution_count": 14, "id": "2f9e4207", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:21.009129Z", "start_time": "2024-01-09T15:10:20.874879Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.324006Z", "iopub.status.busy": "2025-01-14T00:23:43.323874Z", "iopub.status.idle": "2025-01-14T00:23:43.360124Z", "shell.execute_reply": "2025-01-14T00:23:43.359705Z", "shell.execute_reply.started": "2025-01-14T00:23:43.323993Z" } }, "outputs": [], "source": [ "RE_EMOJI = re.compile(\"[\\U00010000-\\U0010ffff]\", flags=re.UNICODE)\n", "\n", "\n", "def clean_text(text):\n", " \"\"\"General text cleaning. A bit tight but makes the content very clean.\"\"\"\n", " text = \"\\n\" + text\n", " text = RE_EMOJI.sub(r\"\", text) # remove emojis\n", " text = text.replace(\"’\", \"'\").lower()\n", " text = text.replace('\"', \"\").lower()\n", " text = re.sub(r\"\\[.+?\\]\", \" \", text) # tags\n", " text = re.sub(r\"\\n.+?\\:\", \" \", text) # new line ends with :\n", " text = re.sub(r\"\\n.+?\\：\", \" \", text) # new line ends with :\n", " text = re.sub(r\"\\n\\(.+?\\)\", \" \", text) # new line with ()\n", " text = re.sub(r\"[\\d]\", \" \", text) # digits\n", " text = re.sub(r\"▁\", \"\", text) # special stuff\n", " text = remove_speakers(text)\n", " text = re.sub(r\"[^\\w\\'\\s]\", \" \", text) # keep only the words\n", " text = normalize_whitespace(text)\n", " return text" ] }, { "cell_type": "code", "execution_count": null, "id": "0d2555e3", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:21.070547Z", "start_time": "2024-01-09T15:10:21.011397Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.360753Z", "iopub.status.busy": "2025-01-14T00:23:43.360609Z", "iopub.status.idle": "2025-01-14T00:23:43.396751Z", "shell.execute_reply": "2025-01-14T00:23:43.396322Z", "shell.execute_reply.started": "2025-01-14T00:23:43.360738Z" } }, "outputs": [], "source": [ "# Quality checks\n", "n_c = 0\n", "for m in metas:\n", " if m[\"lang\"] == \"en\":\n", " n_c += 1\n", " if n_c == 1:\n", " text = m[\"lyrics\"]\n", " print(text)\n", " print(\"-------\")\n", " print(clean_text(text))\n", " break" ] }, { "cell_type": "code", "execution_count": null, "id": "e8b679e7", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:21.167632Z", "start_time": "2024-01-09T15:10:21.072528Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.397353Z", "iopub.status.busy": "2025-01-14T00:23:43.397215Z", "iopub.status.idle": "2025-01-14T00:23:43.455604Z", "shell.execute_reply": "2025-01-14T00:23:43.455194Z", "shell.execute_reply.started": "2025-01-14T00:23:43.397338Z" } }, "outputs": [], "source": [ "# TODO: swap to larger tokenizer\n", "import sentencepiece\n", "\n", "tokenizer = sentencepiece.SentencePieceProcessor()\n", "tokenizer.load(\"tokenizer_spe_bpe_v20480/tokenizer.model\")" ] }, { "cell_type": "code", "execution_count": null, "id": "feacb5f1", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:22.487284Z", "start_time": "2024-01-09T15:10:21.169512Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.456202Z", "iopub.status.busy": "2025-01-14T00:23:43.456070Z", "iopub.status.idle": "2025-01-14T00:23:44.817495Z", "shell.execute_reply": "2025-01-14T00:23:44.816908Z", "shell.execute_reply.started": "2025-01-14T00:23:43.456189Z" } }, "outputs": [], "source": [ "print(len(subset_meta), len(set(m[\"original_id\"] for m in subset_meta)))\n", "random.shuffle(subset_meta)" ] }, { "cell_type": "code", "execution_count": 18, "id": "8d3fa27d-c422-444d-8fbf-149274636627", "metadata": { "execution": { "iopub.status.busy": "2025-01-15T01:20:39.231764Z", "iopub.status.idle": "2025-01-15T01:20:39.231945Z", "shell.execute_reply": "2025-01-15T01:20:39.231859Z", "shell.execute_reply.started": "2025-01-15T01:20:39.231851Z" } }, "outputs": [], "source": [ "## two nodes 35 hrs to download" ] }, { "cell_type": "code", "execution_count": null, "id": "3a3246ca", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:53.367719Z", "start_time": "2024-01-09T15:10:22.489702Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:44.818238Z", "iopub.status.busy": "2025-01-14T00:23:44.818090Z", "iopub.status.idle": "2025-01-14T01:36:38.514831Z", "shell.execute_reply": "2025-01-14T01:36:38.514272Z", "shell.execute_reply.started": "2025-01-14T00:23:44.818222Z" } }, "outputs": [], "source": [ "total_infos = []\n", "n_text_too_short = 0\n", "n_text_too_long = 0\n", "n_audio_too_long = 0\n", "n_lang = collections.defaultdict(int)\n", "n_repeat_ratio_too_high = 0\n", "n_lang_max_cut = 5_000_000 # 10k songs :D, 2_000_000 is max\n", "bad_meta_ids = []\n", "for meta in tqdm.tqdm(subset_meta):\n", " output_path = os.path.join(\n", " \"/app/suno/data/hoot/audios/genius\", f\"{meta['original_id']}.mp3\"\n", " )\n", " if not os.path.exists(output_path):\n", " continue\n", " # output_path = downloaded_ids_to_info[meta[\"id\"]][\"filepath\"]\n", " if (\n", " meta[\"duration_s\"] > 400\n", " ): # this cut is mainly for batching purposes...X.x, but...we probably shouldn't have this.\n", " n_audio_too_long += 1\n", " continue\n", " n_lang[meta[\"lang\"]] += 1\n", " # if n_lang[meta[\"lang\"]] > n_lang_max_cut:\n", " # continue\n", " cleaned_text = clean_text(meta[\"lyrics\"])\n", " # cleaned_text = meta[\"lyrics\"].lower() # clean_text(meta[\"lyrics\"])\n", " # cleaned_tokens = tokenizer.encode(cleaned_text)\n", " # decoded_tokens = tokenizer.decode(cleaned_tokens)\n", " # if decoded_tokens != cleaned_text:\n", " # print(cleaned_text)\n", " # print(decoded_tokens)\n", " if len(cleaned_text) < 20:\n", " n_text_too_short += 1\n", " continue\n", " # check the high freq token frequency\n", " encoded_array = np.array(tokenizer.encode(cleaned_text))\n", " if len(encoded_array) > meta[\"duration_s\"] * 12:\n", " n_text_too_long += 1\n", " continue\n", " repeat_ratio = max(np.bincount(encoded_array)) / (encoded_array.shape[0] + 1)\n", " if repeat_ratio > 0.3:\n", " n_repeat_ratio_too_high += 1\n", " bad_meta_ids.append(meta[\"id\"])\n", " continue\n", " total_infos.append(\n", " {\n", " \"audio_filepath\": output_path,\n", " \"duration\": meta[\"duration_s\"],\n", " \"text\": cleaned_text,\n", " \"lyrics\": meta[\"lyrics\"],\n", " \"lang\": meta[\"lang\"],\n", " \"id\": meta[\"id\"],\n", " \"genius_views\": meta.get(\"genius_views\", 0),\n", " \"youtube_views\": meta.get(\"youtube_views\", 0),\n", " }\n", " )\n", " if not os.path.exists(output_path):\n", " # print(\"WTF\", output_path)\n", " continue\n", "# audio = Audio.from_s3(meta[\"audio_filepath\"])\n", "# audio = audio.convert(16_000, audio.byte_width, n_channels=1)\n", "# audio.to_wav(output_path)\n", "print(\n", " f\"total {len(total_infos)}, too long: {n_audio_too_long}, text too short: {n_text_too_short}, text too long: {n_text_too_long}, text bad: {n_repeat_ratio_too_high}\"\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "618075a0", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:54.930054Z", "start_time": "2024-01-09T16:37:53.368988Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:38.515547Z", "iopub.status.busy": "2025-01-14T01:36:38.515398Z", "iopub.status.idle": "2025-01-14T01:36:40.077169Z", "shell.execute_reply": "2025-01-14T01:36:40.076608Z", "shell.execute_reply.started": "2025-01-14T01:36:38.515532Z" } }, "outputs": [], "source": [ "print(\n", " \"total duration khr\",\n", " sum(m[\"duration\"] for m in total_infos) / 3600000,\n", " min(m[\"duration\"] for m in total_infos),\n", " max(m[\"duration\"] for m in total_infos),\n", " min(len(m[\"text\"]) for m in total_infos),\n", " max(len(m[\"text\"]) for m in total_infos),\n", ")\n", "# 29.15879888888889 120 100 2879\n", "# cut on durations 400: total duration khr 88.80962305555556 60 480 100 5616\n", "# cut on 240: total duration khr 54.709809444444446 60 240 100 2880" ] }, { "cell_type": "code", "execution_count": 21, "id": "76dcc2f9-777f-476e-b190-fddced599117", "metadata": { "execution": { "iopub.execute_input": "2025-01-15T01:20:50.635781Z", "iopub.status.busy": "2025-01-15T01:20:50.635598Z", "iopub.status.idle": "2025-01-15T01:20:50.638227Z", "shell.execute_reply": "2025-01-15T01:20:50.637805Z", "shell.execute_reply.started": "2025-01-15T01:20:50.635766Z" } }, "outputs": [], "source": [ "# exist_info = []\n", "# for test_input in test_info:\n", "# if os.path.exists(test_input[\"audio_filepath\"]):\n", "# exist_info.append(test_input)\n", "# print(len(info), len(exist_info))" ] }, { "cell_type": "code", "execution_count": null, "id": "d8eb67a9", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:55.757800Z", "start_time": "2024-01-09T16:37:54.931303Z" }, "execution": { "iopub.execute_input": "2025-01-15T00:53:38.889667Z", "iopub.status.busy": "2025-01-15T00:53:38.889229Z", "iopub.status.idle": "2025-01-15T00:53:38.893159Z", "shell.execute_reply": "2025-01-15T00:53:38.892581Z", "shell.execute_reply.started": "2025-01-15T00:53:38.889646Z" } }, "outputs": [], "source": [ "# cutoff = -2000\n", "# random.shuffle(info)\n", "# train_info = info[:cutoff]\n", "# test_info = info[cutoff:]\n", "# print(len(train_info), len(test_info))\n", "# # 545838 1000\n", "train_info = total_infos.copy()\n", "test_info = []\n", "print(len(train_info), len(test_info))" ] }, { "cell_type": "code", "execution_count": 23, "id": "72bc9539", "metadata": { "execution": { "iopub.execute_input": "2025-01-14T01:36:40.832617Z", "iopub.status.busy": "2025-01-14T01:36:40.832218Z", "iopub.status.idle": "2025-01-14T01:36:40.834572Z", "shell.execute_reply": "2025-01-14T01:36:40.834176Z", "shell.execute_reply.started": "2025-01-14T01:36:40.832599Z" } }, "outputs": [], "source": [ "# BREAK" ] }, { "cell_type": "code", "execution_count": 24, "id": "a48b2d77", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:55.761119Z", "start_time": "2024-01-09T16:37:55.759035Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:40.835146Z", "iopub.status.busy": "2025-01-14T01:36:40.835018Z", "iopub.status.idle": "2025-01-14T01:36:40.870228Z", "shell.execute_reply": "2025-01-14T01:36:40.869844Z", "shell.execute_reply.started": "2025-01-14T01:36:40.835132Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/all_train_manifest_fast.json\", \"w\") as fp:\n", "# # for l in train_info[:80]:\n", "# for l in train_info[:80]:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')\n", "# with open(\"/home/tony/Data/Hoot/all_test_manifest_fast.json\", \"w\") as fp:\n", "# for l in test_info[:8]:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')" ] }, { "cell_type": "code", "execution_count": 25, "id": "4a7003ff", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:55.836785Z", "start_time": "2024-01-09T16:37:55.762070Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:40.870803Z", "iopub.status.busy": "2025-01-14T01:36:40.870669Z", "iopub.status.idle": "2025-01-14T01:36:40.906745Z", "shell.execute_reply": "2025-01-14T01:36:40.906369Z", "shell.execute_reply.started": "2025-01-14T01:36:40.870789Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/multi_all_104khr_filtered_train_manifest.json\", \"w\") as fp:\n", "# for l in train_info:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')\n", "# with open(\"/home/tony/Data/Hoot/multi_all_104khr_filtered_test_manifest.json\", \"w\") as fp:\n", "# for l in test_info:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')" ] }, { "cell_type": "code", "execution_count": 26, "id": "506b6b20", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:55.891610Z", "start_time": "2024-01-09T16:37:55.838219Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:40.907307Z", "iopub.status.busy": "2025-01-14T01:36:40.907178Z", "iopub.status.idle": "2025-01-14T01:36:40.944089Z", "shell.execute_reply": "2025-01-14T01:36:40.943708Z", "shell.execute_reply.started": "2025-01-14T01:36:40.907294Z" } }, "outputs": [], "source": [ "# with open(DataStoreObject(manifest).get(), 'r') as in_reader:\n", "# for line in in_reader:\n", "# item = json.loads(line)" ] }, { "cell_type": "markdown", "id": "5907d689", "metadata": {}, "source": [ "# Train the tokenizer" ] }, { "cell_type": "markdown", "id": "3a36ab23", "metadata": {}, "source": [ "\n", "\n", "- 0.998 has vocab size of 780\n", "- 0.9995 has vocab size of 1936\n", "- 0.9996 has vocab of 2176\n", "- 0.9999 has vocab of 3878\n", "- 1.0 has vocab of 12402\n", "------------------------\n", "python ./scripts/process_asr_text_tokenizer.py \\\n", " --manifest=\"/home/tony/Data/Hoot/all_train_manifest.json\" \\\n", " --data_root=\"./tokenizers/all/\" \\\n", " --vocab_size=1024 \\\n", " --tokenizer=\"spe\" \\\n", " --spe_character_coverage=0.998 \\\n", " --spe_type=\"bpe\" \\\n", " --log\n", "\n", "------------------------\n", "python ./scripts/process_asr_text_tokenizer.py \\\n", " --manifest=\"/home/tony/Data/Hoot/all_train_manifest.json\" \\\n", " --data_root=\"./tokenizers/multi/\" \\\n", " --vocab_size=5120 \\\n", " --tokenizer=\"spe\" \\\n", " --spe_character_coverage=0.9999 \\\n", " --spe_type=\"bpe\" \\\n", " --log\n", " \n", " ------------------------\n", "python ./scripts/process_asr_text_tokenizer.py \\\n", " --manifest=\"/home/tony/Data/Hoot/multi_filtered_train_manifest.json\" \\\n", " --data_root=\"./tokenizers/multi_filtered/\" \\\n", " --vocab_size=5120 \\\n", " --tokenizer=\"spe\" \\\n", " --spe_character_coverage=0.9999 \\\n", " --spe_type=\"bpe\" \\\n", " --log\n", " \n", " ------------------------\n", " python ./scripts/process_asr_text_tokenizer.py \\\n", " --manifest=\"/home/tony/Data/Hoot/metas.json\" \\\n", " --data_root=\"./tokenizers/full/\" \\\n", " --vocab_size=10240 \\\n", " --tokenizer=\"spe\" \\\n", " --spe_character_coverage=1.0 \\\n", " --spe_type=\"bpe\" \\\n", " --log\n", " \n", " ------------------------\n", " python ./scripts/process_asr_text_tokenizer.py \\\n", " --manifest=\"/home/tony/Data/Hoot/multi_all_104khr_filtered_train_manifest.json\" \\\n", " --data_root=\"./tokenizers/v3/\" \\\n", " --vocab_size=10241 \\\n", " --tokenizer=\"spe\" \\\n", " --spe_character_coverage=1.0 \\\n", " --spe_type=\"bpe\" \\\n", " --log" ] }, { "cell_type": "markdown", "id": "a0616448", "metadata": {}, "source": [ "# Check tokenzier" ] }, { "cell_type": "code", "execution_count": 27, "id": "253035d4", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:55.947077Z", "start_time": "2024-01-09T16:37:55.892947Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:40.944620Z", "iopub.status.busy": "2025-01-14T01:36:40.944495Z", "iopub.status.idle": "2025-01-14T01:36:40.981045Z", "shell.execute_reply": "2025-01-14T01:36:40.980637Z", "shell.execute_reply.started": "2025-01-14T01:36:40.944606Z" } }, "outputs": [], "source": [ "test_text = \"whoa oh oh whoa oh oh whoa oh oh oh oh oh oh whoa oh oh whoa oh oh whoa oh oh oh oh oh oh 취할 것 같은 불빛이 화려한 밤에 누구보다도 자유로워 지기를 바래 아름다운 이 순간이 끝나기 전에 바람소리에 이 음악이 실리길 원해 whoa oh oh whoa oh oh whoa oh oh oh oh oh oh whoa oh oh whoa oh oh whoa oh oh oh oh oh oh lets get it on tonight and set me free tonight 이 순간 즐겨봐 너와 나 소리 높여 lets groove it party time 두근거리는 느낌이 황홀한 밤에 푸른 바다에 달빛을 물들이길 바래 파도가 이 밤을 데려가기 전에 지금 이대로 시간이 멈추기를 원해 whoa oh oh whoa oh oh whoa oh oh oh oh oh oh whoa oh oh whoa oh oh whoa oh oh oh oh oh oh 이 순간 즐겨봐 너와 나 소리 높여 lets groove it party time whoa oh oh whoa oh oh whoa oh oh oh oh oh oh whoa oh oh whoa oh oh whoa oh oh oh oh oh oh 이 순간 즐겨봐 너와 나 소리 높여 lets groove it party time whoa whoa whoa yeah oh whoa oh whoa oh whoa\"" ] }, { "cell_type": "code", "execution_count": null, "id": "d0599dc0", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.061972Z", "start_time": "2024-01-09T16:37:55.948411Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:40.981716Z", "iopub.status.busy": "2025-01-14T01:36:40.981584Z", "iopub.status.idle": "2025-01-14T01:36:41.081968Z", "shell.execute_reply": "2025-01-14T01:36:41.081553Z", "shell.execute_reply.started": "2025-01-14T01:36:40.981702Z" } }, "outputs": [], "source": [ "import sentencepiece\n", "\n", "tokenizer = sentencepiece.SentencePieceProcessor()\n", "tokenizer.load(\"tokenizer_spe_bpe_v20480/tokenizer.model\")" ] }, { "cell_type": "code", "execution_count": null, "id": "6baf7076", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.122437Z", "start_time": "2024-01-09T16:37:56.063015Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.082560Z", "iopub.status.busy": "2025-01-14T01:36:41.082427Z", "iopub.status.idle": "2025-01-14T01:36:41.108656Z", "shell.execute_reply": "2025-01-14T01:36:41.108197Z", "shell.execute_reply.started": "2025-01-14T01:36:41.082546Z" } }, "outputs": [], "source": [ "print(np.array(tokenizer.encode(test_text)).shape)\n", "print(tokenizer.decode(tokenizer.encode(test_text)))" ] }, { "cell_type": "markdown", "id": "6d966633", "metadata": {}, "source": [ "# DEBUG" ] }, { "cell_type": "code", "execution_count": 30, "id": "353d4d0f", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.239480Z", "start_time": "2024-01-09T16:37:56.184175Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.109247Z", "iopub.status.busy": "2025-01-14T01:36:41.109114Z", "iopub.status.idle": "2025-01-14T01:36:41.202044Z", "shell.execute_reply": "2025-01-14T01:36:41.201540Z", "shell.execute_reply.started": "2025-01-14T01:36:41.109233Z" } }, "outputs": [], "source": [ "# x = []\n", "# with open(\"/home/tony/Data/Hoot/all_train_manifest.json\", \"r\") as fp:\n", "# for l in fp:\n", "# x.append(json.loads(l))\n", "\n", "# bad_indices = []\n", "# for i, meta in tqdm.tqdm(enumerate(x)):\n", "# encoded_array = np.array(tokenizer.encode(meta[\"text\"]))\n", "# repeat_ratio = max(np.bincount(encoded_array)) / (encoded_array.shape[0] + 1)\n", "# if repeat_ratio > 0.3:\n", "# # print(repeat_ratio, meta)\n", "# bad_indices.append(i)" ] }, { "cell_type": "code", "execution_count": 31, "id": "e5f54770", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.293353Z", "start_time": "2024-01-09T16:37:56.240396Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.202685Z", "iopub.status.busy": "2025-01-14T01:36:41.202544Z", "iopub.status.idle": "2025-01-14T01:36:41.239169Z", "shell.execute_reply": "2025-01-14T01:36:41.238783Z", "shell.execute_reply.started": "2025-01-14T01:36:41.202670Z" } }, "outputs": [], "source": [ "# len(bad_indices)\n", "\n", "# bad_ids = [\n", "# 36004, 91329, 772492\n", "# ]" ] }, { "cell_type": "code", "execution_count": 32, "id": "f65a307d", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.365226Z", "start_time": "2024-01-09T16:37:56.294287Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.239745Z", "iopub.status.busy": "2025-01-14T01:36:41.239620Z", "iopub.status.idle": "2025-01-14T01:36:41.273980Z", "shell.execute_reply": "2025-01-14T01:36:41.273608Z", "shell.execute_reply.started": "2025-01-14T01:36:41.239732Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/all_train_manifest_debug.json\", \"w\") as fp:\n", "# # for l in train_info[:80]:\n", "# for l in selected:\n", "# json.dump(x[l], fp, ensure_ascii=True)\n", "# fp.write('\\n')" ] }, { "cell_type": "code", "execution_count": 33, "id": "9c374d06", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.420690Z", "start_time": "2024-01-09T16:37:56.366689Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.274539Z", "iopub.status.busy": "2025-01-14T01:36:41.274414Z", "iopub.status.idle": "2025-01-14T01:36:41.312345Z", "shell.execute_reply": "2025-01-14T01:36:41.311969Z", "shell.execute_reply.started": "2025-01-14T01:36:41.274526Z" } }, "outputs": [], "source": [ "# for bad_id in bad_ids:\n", "# print(x[bad_id])\n", "\n", "# test_audio = Audio.from_file('/app/suno/data/hoot/2I8_OV4DtCs.wav')\n", "# test_audio.play(compress=False)\n", "\n", "# for meta in base_metas:\n", "# if meta[\"original_id\"] == \"JpkwJW92UcY\":\n", "# print(meta)" ] }, { "cell_type": "markdown", "id": "e7f8985c", "metadata": {}, "source": [ "# Load the hooted output" ] }, { "cell_type": "code", "execution_count": 34, "id": "a7be6e73", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.505275Z", "start_time": "2024-01-09T16:37:56.421982Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.312896Z", "iopub.status.busy": "2025-01-14T01:36:41.312777Z", "iopub.status.idle": "2025-01-14T01:36:41.350393Z", "shell.execute_reply": "2025-01-14T01:36:41.350007Z", "shell.execute_reply.started": "2025-01-14T01:36:41.312884Z" } }, "outputs": [], "source": [ "# total_infos = dict()\n", "# for i in tqdm.tqdm(range(0, 1836168, 100)):\n", "# if not os.path.exists(f\"/home/tony/Data/Hoot/alignments/outputs/batch_{i}.json\"):\n", "# # print(\"Missing,\", i)\n", "# continue\n", "# try:\n", "# with open(f\"/home/tony/Data/Hoot/alignments/outputs/batch_{i}.json\", \"r\") as fp:\n", "# infos = json.load(fp)\n", "# for x_info in infos:\n", "# if x_info[\"id\"] in total_infos:\n", "# break\n", "# total_infos[x_info[\"id\"]] = x_info[\"cer_val\"]\n", "# except:\n", "# # print(\"WTF,\", i)\n", "# pass\n", "\n", "# plt.hist(total_infos.values(), bins=np.linspace(0, 1, 50))\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": 35, "id": "552c1c73", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.570374Z", "start_time": "2024-01-09T16:37:56.506567Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.351050Z", "iopub.status.busy": "2025-01-14T01:36:41.350923Z", "iopub.status.idle": "2025-01-14T01:36:41.385902Z", "shell.execute_reply": "2025-01-14T01:36:41.385519Z", "shell.execute_reply.started": "2025-01-14T01:36:41.351037Z" } }, "outputs": [], "source": [ "# high_cer_infos = {k:v for k, v in total_infos.items() if v < 0.4}\n", "# print(len(high_cer_infos))\n", "\n", "# high_cer_train_info = []\n", "# for info in train_info:\n", "# if info[\"id\"] in high_cer_infos and info[\"duration\"] < 240:\n", "# high_cer_train_info.append(info)\n", "# print(len(high_cer_train_info))\n", "\n", "# print(\n", "# \"total duration khr\", sum(m[\"duration\"] for m in high_cer_train_info) / 3600000,\n", "# min(m[\"duration\"] for m in high_cer_train_info),\n", "# max(m[\"duration\"] for m in high_cer_train_info),\n", "# min(len(m[\"text\"]) for m in high_cer_train_info),\n", "# max(len(m[\"text\"]) for m in high_cer_train_info),\n", "# )\n", "\n", "# print(Counter(m[\"lang\"] for m in high_cer_train_info))\n", "\n", "# cutoff = -2000\n", "# random.shuffle(high_cer_train_info)\n", "# high_cer_train_info, high_cer_test_info = high_cer_train_info[:cutoff], high_cer_train_info[cutoff:]\n", "# print(len(high_cer_train_info), len(high_cer_test_info))" ] }, { "cell_type": "code", "execution_count": 36, "id": "417c0eb3", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.651676Z", "start_time": "2024-01-09T16:37:56.571763Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.386487Z", "iopub.status.busy": "2025-01-14T01:36:41.386360Z", "iopub.status.idle": "2025-01-14T01:36:41.427365Z", "shell.execute_reply": "2025-01-14T01:36:41.426982Z", "shell.execute_reply.started": "2025-01-14T01:36:41.386474Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/multi_large_vocab_cer_04_train.json\", \"w\") as fp:\n", "# for l in high_cer_train_info:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')\n", "# with open(\"/home/tony/Data/Hoot/multi_large_vocab_cer_04_test.json\", \"w\") as fp:\n", "# for l in high_cer_test_info:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')" ] }, { "cell_type": "markdown", "id": "1642e3b5", "metadata": {}, "source": [ "# 2nd round of hoot data prep setup" ] }, { "cell_type": "code", "execution_count": 4, "id": "52f2a9a1", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:43:46.229977Z", "start_time": "2024-01-18T02:43:05.476371Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.427915Z", "iopub.status.busy": "2025-01-14T01:36:41.427793Z", "iopub.status.idle": "2025-01-14T01:36:42.450600Z", "shell.execute_reply": "2025-01-14T01:36:42.450006Z", "shell.execute_reply.started": "2025-01-14T01:36:41.427903Z" } }, "outputs": [], "source": [ "with open(\"/home/tony/Data/Hoot/discogs_metas.json\", \"r\") as fp:\n", " metas = json.load(fp)\n", "metas_map = {m[\"id\"]: m for m in metas}" ] }, { "cell_type": "code", "execution_count": null, "id": "274e39d7", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:57:19.260628Z", "start_time": "2024-01-18T02:43:49.821152Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:42.451326Z", "iopub.status.busy": "2025-01-14T01:36:42.451172Z", "iopub.status.idle": "2025-01-14T01:47:52.315467Z", "shell.execute_reply": "2025-01-14T01:47:52.314897Z", "shell.execute_reply.started": "2025-01-14T01:36:42.451310Z" } }, "outputs": [], "source": [ "from collections import defaultdict\n", "\n", "total_infos_2 = dict()\n", "lang_cers = defaultdict(list)\n", "for i in tqdm.tqdm(range(0, 2155000, 100)):\n", " try:\n", " with open(\n", " f\"/app/suno/data/hoot/alignments/discogs_v4/batch_{i}.json\", \"r\"\n", " ) as fp:\n", " infos = json.load(fp)\n", " for x_info in infos:\n", " if x_info[\"id\"] in total_infos_2:\n", " break\n", " if x_info[\"id\"] not in metas_map:\n", " continue\n", " # if x_info[\"id\"] not in metas_map:\n", " # this is somehow already filtered out...?\n", " total_infos_2[x_info[\"id\"]] = x_info[\"cer_val\"]\n", " lang_cers[metas_map[x_info[\"id\"]][\"lang\"]].append(x_info[\"cer_val\"])\n", " except:\n", " print(\"WTF,\", i)" ] }, { "cell_type": "code", "execution_count": null, "id": "033dc016", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:57:19.317468Z", "start_time": "2024-01-18T02:57:19.266816Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:47:52.316177Z", "iopub.status.busy": "2025-01-14T01:47:52.316027Z", "iopub.status.idle": "2025-01-14T01:47:52.319363Z", "shell.execute_reply": "2025-01-14T01:47:52.318967Z", "shell.execute_reply.started": "2025-01-14T01:47:52.316161Z" } }, "outputs": [], "source": [ "len(total_infos_2), len(lang_cers)" ] }, { "cell_type": "code", "execution_count": null, "id": "ca6fcca0", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:57:19.825960Z", "start_time": "2024-01-18T02:57:19.318687Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:47:52.320042Z", "iopub.status.busy": "2025-01-14T01:47:52.319914Z", "iopub.status.idle": "2025-01-14T01:47:52.753995Z", "shell.execute_reply": "2025-01-14T01:47:52.753453Z", "shell.execute_reply.started": "2025-01-14T01:47:52.320028Z" } }, "outputs": [], "source": [ "langs = [(k, len(v)) for k, v in lang_cers.items()]\n", "langs.sort(key=lambda x: (-x[1]))\n", "print(\"lang n_songs median cer\")\n", "cer_percentage_threshold = 0.5\n", "langs_cer_cut = {}\n", "for k, l_v in langs:\n", " v = lang_cers[k]\n", " if l_v > 1:\n", " print(k, l_v, round(np.quantile(v, cer_percentage_threshold), 3))\n", " langs_cer_cut[k] = round(np.quantile(v, cer_percentage_threshold), 3)" ] }, { "cell_type": "code", "execution_count": null, "id": "bf85525f", "metadata": {}, "outputs": [], "source": [ "langs = [(k, len(v)) for k, v in lang_cers.items()]\n", "langs.sort(key=lambda x: (-x[1]))\n", "print(\"lang n_songs median cer\")\n", "cer_percentage_threshold = 0.5\n", "langs_cer_cut = {}\n", "for k, l_v in langs:\n", " v = lang_cers[k]\n", " if l_v > 1:\n", " print(k, l_v, round(np.quantile(v, cer_percentage_threshold), 3))\n", " langs_cer_cut[k] = round(np.quantile(v, cer_percentage_threshold), 3)" ] }, { "cell_type": "code", "execution_count": null, "id": "768780a2", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T03:01:31.958520Z", "start_time": "2024-01-18T03:01:04.057834Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:47:52.754672Z", "iopub.status.busy": "2025-01-14T01:47:52.754529Z", "iopub.status.idle": "2025-01-14T01:48:24.789903Z", "shell.execute_reply": "2025-01-14T01:48:24.789336Z", "shell.execute_reply.started": "2025-01-14T01:47:52.754657Z" } }, "outputs": [], "source": [ "rates = []\n", "cers = []\n", "# youtube_views_log = []\n", "for k in tqdm.tqdm(total_infos_2):\n", " duration_s = metas_map[k][\"duration_s\"]\n", " n_lyrics = len(metas_map[k][\"lyrics\"].split())\n", " # print(n_lyrics / duration_s)\n", " cers.append(total_infos_2[k])\n", " rates.append(n_lyrics / duration_s)\n", " # youtube_views_log.append(np.log10(metas_map[k][\"youtube_views\"]))" ] }, { "cell_type": "code", "execution_count": 10, "id": "ddca3db4", "metadata": { "ExecuteTime": { "end_time": "2024-01-12T22:41:25.073971Z", "start_time": "2024-01-12T22:41:25.072066Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:24.790628Z", "iopub.status.busy": "2025-01-14T01:48:24.790474Z", "iopub.status.idle": "2025-01-14T01:48:24.792987Z", "shell.execute_reply": "2025-01-14T01:48:24.792598Z", "shell.execute_reply.started": "2025-01-14T01:48:24.790612Z" } }, "outputs": [], "source": [ "# plt.clf()\n", "# h = plt.hist2d(rates, youtube_views_log, bins=200)\n", "# plt.colorbar(h[3])\n", "# # plt.ylim(0, 4)\n", "# plt.xlim(0, 5)\n", "# plt.xlabel(\"n_words / sec\")\n", "# plt.ylabel(\"youtube_views, log10\")\n", "# plt.title(\"youtube views vs. num words per sec\")\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": 11, "id": "2e5eb800", "metadata": { "ExecuteTime": { "end_time": "2024-01-12T22:41:25.152543Z", "start_time": "2024-01-12T22:41:25.074881Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:24.793569Z", "iopub.status.busy": "2025-01-14T01:48:24.793443Z", "iopub.status.idle": "2025-01-14T01:48:24.832093Z", "shell.execute_reply": "2025-01-14T01:48:24.831718Z", "shell.execute_reply.started": "2025-01-14T01:48:24.793556Z" } }, "outputs": [], "source": [ "# plt.hist([r for r, v in zip(rates, youtube_views_log) if v < 6], bins=100, density=True, label=\"views < 1m\", alpha=0.5)\n", "# plt.hist([r for r, v in zip(rates, youtube_views_log) if v >= 6], bins=100, density=True, label=\"views >= 1m\", alpha=0.5)\n", "# plt.legend()\n", "# plt.xlabel(\"word rate / sec\")\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": 12, "id": "b19130f2", "metadata": { "ExecuteTime": { "end_time": "2024-01-12T22:41:25.212542Z", "start_time": "2024-01-12T22:41:25.153867Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:24.832726Z", "iopub.status.busy": "2025-01-14T01:48:24.832599Z", "iopub.status.idle": "2025-01-14T01:48:24.868630Z", "shell.execute_reply": "2025-01-14T01:48:24.868258Z", "shell.execute_reply.started": "2025-01-14T01:48:24.832713Z" } }, "outputs": [], "source": [ "# plt.clf()\n", "# h = plt.hist2d(cers, youtube_views_log, bins=100)\n", "# plt.colorbar(h[3])\n", "# # plt.ylim(0, 4)\n", "# # plt.xlim(0, 0.99)\n", "# plt.xlabel(\"cer\")\n", "# plt.ylabel(\"youtube_views\")\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": null, "id": "416b615d", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T03:01:32.652951Z", "start_time": "2024-01-18T03:01:31.959970Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:24.869209Z", "iopub.status.busy": "2025-01-14T01:48:24.869078Z", "iopub.status.idle": "2025-01-14T01:48:25.558515Z", "shell.execute_reply": "2025-01-14T01:48:25.558027Z", "shell.execute_reply.started": "2025-01-14T01:48:24.869196Z" } }, "outputs": [], "source": [ "plt.clf()\n", "h = plt.hist2d(cers, rates, bins=100)\n", "plt.colorbar(h[3])\n", "plt.ylim(0, 4)\n", "# plt.xlim(0, 0.99)\n", "plt.xlabel(\"cer\")\n", "plt.ylabel(\"n_words / sec -- higher faster\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "id": "2172be9d", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T03:01:36.428046Z", "start_time": "2024-01-18T03:01:32.654154Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:25.559235Z", "iopub.status.busy": "2025-01-14T01:48:25.559090Z", "iopub.status.idle": "2025-01-14T01:48:30.078970Z", "shell.execute_reply": "2025-01-14T01:48:30.078473Z", "shell.execute_reply.started": "2025-01-14T01:48:25.559220Z" } }, "outputs": [], "source": [ "plt.hist(total_infos_2.values(), bins=np.linspace(0, 1, 50))\n", "plt.grid(True)\n", "plt.title(\"Discogs cer distribution\")\n", "plt.xlabel(\"cer\")\n", "plt.ylabel(\"n_songs\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 15, "id": "097e26a4", "metadata": { "ExecuteTime": { "end_time": "2024-01-12T22:41:30.694878Z", "start_time": "2024-01-12T22:41:30.692934Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:30.079677Z", "iopub.status.busy": "2025-01-14T01:48:30.079522Z", "iopub.status.idle": "2025-01-14T01:48:30.082164Z", "shell.execute_reply": "2025-01-14T01:48:30.081768Z", "shell.execute_reply.started": "2025-01-14T01:48:30.079662Z" } }, "outputs": [], "source": [ "# sub_total_infos = {k: v for k, v in total_infos.items() if k in total_infos_2}\n", "\n", "# len(sub_total_infos)\n", "\n", "# plt.hist(sub_total_infos.values(), bins=np.linspace(0, 1, 50))\n", "# plt.show()\n", "\n", "# np.median(list(sub_total_infos.values())), np.median(list(total_infos_2.values()))\n", "\n", "# interesting_diffs = []\n", "# for k in total_infos_2:\n", "# if k in sub_total_infos and total_infos_2[k] - sub_total_infos[k] < 0.3 and sub_total_infos[k] > 0.6:\n", "# interesting_diffs.append(k)\n", "# print(len(interesting_diffs))\n", "\n", "# for idx in interesting_diffs:\n", "# if idx in metas_map and metas_map[idx][\"lang\"] == \"zh\":\n", "# print(metas_map[idx])\n", "# break" ] }, { "cell_type": "markdown", "id": "42dcbcc8", "metadata": {}, "source": [ "# prepare the 2nd hoot dataset" ] }, { "cell_type": "code", "execution_count": null, "id": "326e1b84", "metadata": { "ExecuteTime": { "end_time": "2024-01-15T18:49:58.544826Z", "start_time": "2024-01-15T18:49:56.487590Z" }, "execution": { "iopub.execute_input": "2025-01-14T15:34:40.602741Z", "iopub.status.busy": "2025-01-14T15:34:40.602572Z", "iopub.status.idle": "2025-01-14T15:34:43.196382Z", "shell.execute_reply": "2025-01-14T15:34:43.195791Z", "shell.execute_reply.started": "2025-01-14T15:34:40.602725Z" } }, "outputs": [], "source": [ "high_cer_infos_2 = {}\n", "for k, v in total_infos_2.items():\n", " k_lang = metas_map[k][\"lang\"]\n", " # should probably also play it safe\n", " cut_threshold = min(langs_cer_cut.get(k_lang, 0.5), 0.8)\n", " if v <= cut_threshold:\n", " high_cer_infos_2[k] = v\n", "print(len(high_cer_infos_2))" ] }, { "cell_type": "code", "execution_count": null, "id": "636c3311", "metadata": { "ExecuteTime": { "end_time": "2024-01-15T18:51:44.029594Z", "start_time": "2024-01-15T18:51:42.398978Z" }, "execution": { "iopub.execute_input": "2025-01-14T15:37:30.232542Z", "iopub.status.busy": "2025-01-14T15:37:30.232210Z", "iopub.status.idle": "2025-01-14T15:37:31.992066Z", "shell.execute_reply": "2025-01-14T15:37:31.991479Z", "shell.execute_reply.started": "2025-01-14T15:37:30.232525Z" } }, "outputs": [], "source": [ "high_cer_train_info_2 = []\n", "for info in train_info:\n", " if (\n", " info[\"id\"] in high_cer_infos_2 and info[\"duration\"] < 360\n", " ): # --> this is how v3 t9 data is prepared\n", " # if info[\"id\"] in high_cer_infos_2 and 210 < info[\"duration\"] < 300:\n", " high_cer_train_info_2.append(info)\n", "print(len(high_cer_train_info_2), len(high_cer_train_info_2) / 2 / 8 / 8)" ] }, { "cell_type": "code", "execution_count": null, "id": "bf451a42", "metadata": { "ExecuteTime": { "end_time": "2024-01-15T18:52:00.685027Z", "start_time": "2024-01-15T18:51:59.480509Z" }, "execution": { "iopub.execute_input": "2025-01-14T15:37:43.551574Z", "iopub.status.busy": "2025-01-14T15:37:43.551251Z", "iopub.status.idle": "2025-01-14T15:37:46.406410Z", "shell.execute_reply": "2025-01-14T15:37:46.405823Z", "shell.execute_reply.started": "2025-01-14T15:37:43.551558Z" } }, "outputs": [], "source": [ "print(\n", " \"total duration khr\",\n", " sum(m[\"duration\"] for m in high_cer_train_info_2) / 3600000,\n", " min(m[\"duration\"] for m in high_cer_train_info_2),\n", " max(m[\"duration\"] for m in high_cer_train_info_2),\n", " min(len(m[\"text\"]) for m in high_cer_train_info_2),\n", " max(len(m[\"text\"]) for m in high_cer_train_info_2),\n", ")\n", "print(Counter(m[\"lang\"] for m in high_cer_train_info_2))" ] }, { "cell_type": "code", "execution_count": null, "id": "063513d5-ddf4-47c5-aa3c-6382d63f35e2", "metadata": { "execution": { "iopub.execute_input": "2025-01-14T16:00:32.470361Z", "iopub.status.busy": "2025-01-14T16:00:32.469783Z", "iopub.status.idle": "2025-01-14T16:00:32.699485Z", "shell.execute_reply": "2025-01-14T16:00:32.698913Z", "shell.execute_reply.started": "2025-01-14T16:00:32.470342Z" } }, "outputs": [], "source": [ "selected_multi_validation_set = read_jsonl(\n", " \"/home/tony/Data/Hoot/v4_multi_validation_set.json\"\n", ")\n", "print(len(selected_multi_validation_set))\n", "selected_en_validation_set = read_jsonl(\n", " \"/home/tony/Data/Hoot/v4_en_validation_set.json\"\n", ")\n", "print(len(selected_en_validation_set))" ] }, { "cell_type": "code", "execution_count": null, "id": "26c6eac1-736a-4f93-9a19-40e1c0152dc5", "metadata": { "execution": { "iopub.execute_input": "2025-01-14T16:00:33.396095Z", "iopub.status.busy": "2025-01-14T16:00:33.395767Z", "iopub.status.idle": "2025-01-14T16:00:33.400562Z", "shell.execute_reply": "2025-01-14T16:00:33.400011Z", "shell.execute_reply.started": "2025-01-14T16:00:33.396079Z" } }, "outputs": [], "source": [ "selected_validation_set_ids = set(x[\"id\"] for x in selected_multi_validation_set).union(\n", " set(x[\"id\"] for x in selected_en_validation_set)\n", ")\n", "print(len(selected_validation_set_ids))" ] }, { "cell_type": "code", "execution_count": null, "id": "5492f723-0ac1-45bd-8539-bfd093a5335b", "metadata": { "execution": { "iopub.execute_input": "2025-01-14T15:59:56.367179Z", "iopub.status.busy": "2025-01-14T15:59:56.366634Z", "iopub.status.idle": "2025-01-14T15:59:57.063118Z", "shell.execute_reply": "2025-01-14T15:59:57.062469Z", "shell.execute_reply.started": "2025-01-14T15:59:56.367125Z" } }, "outputs": [], "source": [ "print(len(high_cer_train_info_2))\n", "high_cer_train_info_2 = [\n", " x for x in high_cer_train_info_2 if x[\"id\"] not in selected_validation_set_ids\n", "]\n", "print(len(high_cer_train_info_2))" ] }, { "cell_type": "code", "execution_count": null, "id": "90466614", "metadata": { "ExecuteTime": { "end_time": "2024-01-15T18:52:12.098221Z", "start_time": "2024-01-15T18:52:11.939584Z" }, "execution": { "iopub.execute_input": "2025-01-14T15:44:14.123946Z", "iopub.status.busy": "2025-01-14T15:44:14.123554Z", "iopub.status.idle": "2025-01-14T15:44:14.517921Z", "shell.execute_reply": "2025-01-14T15:44:14.517273Z", "shell.execute_reply.started": "2025-01-14T15:44:14.123925Z" } }, "outputs": [], "source": [ "cutoff = -4000\n", "random.shuffle(high_cer_train_info_2)\n", "high_cer_train_info_2, high_cer_test_info_2 = (\n", " high_cer_train_info_2[:cutoff],\n", " high_cer_train_info_2[cutoff:],\n", ")\n", "print(len(high_cer_train_info_2), len(high_cer_test_info_2))" ] }, { "cell_type": "code", "execution_count": 55, "id": "74ef0ed9", "metadata": { "ExecuteTime": { "end_time": "2024-01-15T18:52:34.087188Z", "start_time": "2024-01-15T18:52:23.399643Z" }, "execution": { "iopub.execute_input": "2025-01-14T15:44:18.027449Z", "iopub.status.busy": "2025-01-14T15:44:18.027087Z", "iopub.status.idle": "2025-01-14T15:44:53.231893Z", "shell.execute_reply": "2025-01-14T15:44:53.231193Z", "shell.execute_reply.started": "2025-01-14T15:44:18.027431Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/v4_t1_cer_50_long_train.json\", \"w\") as fp:\n", "# for l in high_cer_train_info_2:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')\n", "# with open(\"/home/tony/Data/Hoot/v4_t1_cer_50_long_test.json\", \"w\") as fp:\n", "# for l in high_cer_test_info_2:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')" ] }, { "cell_type": "code", "execution_count": null, "id": "61409fbd", "metadata": { "ExecuteTime": { "end_time": "2024-01-15T18:52:34.091372Z", "start_time": "2024-01-15T18:52:34.089444Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:35.505118Z", "iopub.status.busy": "2025-01-14T01:48:35.504994Z", "iopub.status.idle": "2025-01-14T01:48:35.539824Z", "shell.execute_reply": "2025-01-14T01:48:35.539404Z", "shell.execute_reply.started": "2025-01-14T01:48:35.505106Z" } }, "outputs": [], "source": [ "print(\"Done\")" ] }, { "cell_type": "markdown", "id": "535130a0", "metadata": {}, "source": [ "# Curate a seperate list of long audios, for just evaluation" ] }, { "cell_type": "code", "execution_count": null, "id": "13fa3abf", "metadata": { "ExecuteTime": { "end_time": "2024-01-11T19:10:11.366040Z", "start_time": "2024-01-11T19:10:10.100230Z" }, "execution": { "iopub.execute_input": "2025-01-14T16:00:15.047699Z", "iopub.status.busy": "2025-01-14T16:00:15.047401Z", "iopub.status.idle": "2025-01-14T16:00:17.115621Z", "shell.execute_reply": "2025-01-14T16:00:17.114871Z", "shell.execute_reply.started": "2025-01-14T16:00:15.047681Z" } }, "outputs": [], "source": [ "high_cer_train_info_2_long = []\n", "for info in train_info:\n", " if info[\"id\"] in high_cer_infos_2 and info[\"duration\"] < 300:\n", " high_cer_train_info_2_long.append(info)\n", "print(len(high_cer_train_info_2_long))" ] }, { "cell_type": "code", "execution_count": 58, "id": "5c7f0742", "metadata": { "ExecuteTime": { "end_time": "2024-01-11T19:10:11.424285Z", "start_time": "2024-01-11T19:10:11.367473Z" }, "execution": { "iopub.execute_input": "2025-01-14T16:00:17.116825Z", "iopub.status.busy": "2025-01-14T16:00:17.116651Z", "iopub.status.idle": "2025-01-14T16:00:17.480047Z", "shell.execute_reply": "2025-01-14T16:00:17.479393Z", "shell.execute_reply.started": "2025-01-14T16:00:17.116808Z" } }, "outputs": [], "source": [ "random.shuffle(high_cer_train_info_2_long)" ] }, { "cell_type": "code", "execution_count": null, "id": "cfbf0a99", "metadata": { "ExecuteTime": { "end_time": "2024-01-11T19:10:12.582481Z", "start_time": "2024-01-11T19:10:12.457526Z" }, "execution": { "iopub.execute_input": "2025-01-14T16:00:17.480831Z", "iopub.status.busy": "2025-01-14T16:00:17.480681Z", "iopub.status.idle": "2025-01-14T16:00:18.515817Z", "shell.execute_reply": "2025-01-14T16:00:18.515175Z", "shell.execute_reply.started": "2025-01-14T16:00:17.480815Z" } }, "outputs": [], "source": [ "selected_validation_set = []\n", "c = Counter()\n", "for m in high_cer_train_info_2_long:\n", " if c[m[\"lang\"]] < min(len(lang_cers.get(m[\"lang\"], [])) // 4, 50):\n", " selected_validation_set.append(m)\n", " c[m[\"lang\"]] += 1\n", "print(len(selected_validation_set))" ] }, { "cell_type": "code", "execution_count": 60, "id": "d3c6afd4-c44a-495d-b311-bb8042a9bfe1", "metadata": { "execution": { "iopub.execute_input": "2025-01-15T01:23:44.489159Z", "iopub.status.busy": "2025-01-15T01:23:44.488728Z", "iopub.status.idle": "2025-01-15T01:23:44.789090Z", "shell.execute_reply": "2025-01-15T01:23:44.788560Z", "shell.execute_reply.started": "2025-01-15T01:23:44.489140Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/v4_multi_validation_set.json\", \"w\") as fp:\n", "# for l in selected_validation_set:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write(\"\\n\")" ] }, { "cell_type": "code", "execution_count": null, "id": "f8d88911", "metadata": { "ExecuteTime": { "end_time": "2024-01-11T19:10:43.816910Z", "start_time": "2024-01-11T19:10:43.660712Z" }, "execution": { "iopub.execute_input": "2025-01-14T16:00:18.763232Z", "iopub.status.busy": "2025-01-14T16:00:18.762767Z", "iopub.status.idle": "2025-01-14T16:00:19.678028Z", "shell.execute_reply": "2025-01-14T16:00:19.677390Z", "shell.execute_reply.started": "2025-01-14T16:00:18.763214Z" } }, "outputs": [], "source": [ "selected_validation_set_ids = set(x[\"id\"] for x in selected_validation_set)\n", "selected_en_validation_set = []\n", "c = Counter()\n", "for m in high_cer_train_info_2_long:\n", " if (\n", " m[\"lang\"] == \"en\"\n", " and m[\"id\"] not in selected_validation_set_ids\n", " and c[m[\"lang\"]] < 3000\n", " ):\n", " selected_en_validation_set.append(m)\n", " c[m[\"lang\"]] += 1\n", "print(len(selected_en_validation_set))" ] }, { "cell_type": "code", "execution_count": 62, "id": "c1e7ac97", "metadata": { "execution": { "iopub.execute_input": "2025-01-15T01:23:52.018137Z", "iopub.status.busy": "2025-01-15T01:23:52.017514Z", "iopub.status.idle": "2025-01-15T01:23:52.217614Z", "shell.execute_reply": "2025-01-15T01:23:52.217124Z", "shell.execute_reply.started": "2025-01-15T01:23:52.018119Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/v4_en_validation_set.json\", \"w\") as fp:\n", "# for l in selected_en_validation_set:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write(\"\\n\")" ] }, { "cell_type": "markdown", "id": "2854610a", "metadata": {}, "source": [ "# Validations and checks" ] }, { "cell_type": "code", "execution_count": 5, "id": "a0e31899-aeca-457d-bafa-c313f22180b7", "metadata": {}, "outputs": [], "source": [ "train_infos = read_jsonl(\"/home/tony/Data/Hoot/v4_t1_cer_50_long_train.json\")" ] }, { "cell_type": "code", "execution_count": null, "id": "094c8548", "metadata": {}, "outputs": [], "source": [ "print(len(train_infos))" ] }, { "cell_type": "code", "execution_count": null, "id": "df1ffb98", "metadata": {}, "outputs": [], "source": [ "with open(\"tmp/genius_hq_silent_80_ids.txt\", \"r\") as fp:\n", " ids = set(fp.read().splitlines())\n", "print(len(ids))\n", "n_silent_in_train = 0\n", "for train_info in train_infos:\n", " if train_info[\"id\"] in ids:\n", " n_silent_in_train += 1\n", " print(train_info)\n", "print(n_silent_in_train)" ] }, { "cell_type": "code", "execution_count": null, "id": "755afe1f", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env_dev", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }