{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "d31d8850", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:42:48.498319Z", "start_time": "2024-01-18T02:42:48.496832Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:22:06.162764Z", "iopub.status.busy": "2025-01-14T00:22:06.162624Z", "iopub.status.idle": "2025-01-14T00:22:06.167027Z", "shell.execute_reply": "2025-01-14T00:22:06.166610Z", "shell.execute_reply.started": "2025-01-14T00:22:06.162748Z" } }, "outputs": [], "source": [ "import os\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "7cd4c0dd", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:42:53.666867Z", "start_time": "2024-01-18T02:42:48.821513Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:22:06.167729Z", "iopub.status.busy": "2025-01-14T00:22:06.167600Z", "iopub.status.idle": "2025-01-14T00:22:12.274113Z", "shell.execute_reply": "2025-01-14T00:22:12.273582Z", "shell.execute_reply.started": "2025-01-14T00:22:06.167715Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/tony/anaconda3/envs/suno_env_dev/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n", " warnings.warn(\n" ] } ], "source": [ "import tqdm\n", "import math\n", "import torch\n", "import random\n", "import funcy\n", "import copy\n", "import gc\n", "import re\n", "import json\n", "import tempfile\n", "import collections\n", "from collections import Counter\n", "import pandas as pd\n", "import numpy as np\n", "import fasttext\n", "from joblib import Parallel, delayed\n", "import matplotlib.pyplot as plt\n", "\n", "from suno_utils.audio import Audio\n", "from suno_utils.utils.text import (\n", " write_jsonl,\n", " read_jsonl,\n", " write_json,\n", " read_json,\n", " normalize_whitespace,\n", ")\n", "from suno_utils.utils.lyrics import remove_speakers\n", "from suno_utils.utils.s3 import read_from_s3, check_s3_file_exists, open_from_s3\n", "from suno_utils.utils.tokenizers import tokenize\n", "from suno_utils.harvest.youtube.constants.text_lang import BASE_TO_FASTTEXT_REMAP\n", "from suno_utils.utils.metrics import get_cer\n", "from suno_utils.tasks.hoot import (\n", " parse_lyrics,\n", " legacy_parse_lyrics,\n", " EMBEDDING_RATE as HOOT_EMBEDDING_RATE,\n", ")\n", "from suno_utils.utils.lyrics import remove_speakers" ] }, { "cell_type": "code", "execution_count": 3, "id": "d6518a0b", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:42:56.344509Z", "start_time": "2024-01-18T02:42:53.668461Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:22:12.274982Z", "iopub.status.busy": "2025-01-14T00:22:12.274701Z", "iopub.status.idle": "2025-01-14T00:22:12.277428Z", "shell.execute_reply": "2025-01-14T00:22:12.277039Z", "shell.execute_reply.started": "2025-01-14T00:22:12.274965Z" } }, "outputs": [], "source": [ "# If run language detection, uncomment this\n", "# LANG_ID_MODEL_FP = \"s3://suno-data/georg/trained_models/chirp_v1/lid.176.bin\"\n", "# text_lang_model = read_from_s3(LANG_ID_MODEL_FP, read_f=fasttext.load_model)\n", "# def _get_text_lang(text):\n", "# \"\"\"get probability of input language for text\"\"\"\n", "# text = text.replace(\"’\", \"'\").lower()\n", "# text = re.sub(r\"\\[.+?\\]\", \" \", text)\n", "# text = normalize_whitespace(text)\n", "# out = text_lang_model.predict(text, k=1)\n", "# lang_str = out[0][0]\n", "# p_lang = out[1][0]\n", "# lang = lang_str.split(\"__\")[-1]\n", "# lang = BASE_TO_FASTTEXT_REMAP.get(lang, lang)\n", "# # if p_lang >= 0.8:\n", "# # return lang\n", "# return p_lang, lang" ] }, { "cell_type": "markdown", "id": "d9d50793", "metadata": {}, "source": [ "## load genius_hq info" ] }, { "cell_type": "code", "execution_count": 4, "id": "2cd505d5", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:09:50.716502Z", "start_time": "2024-01-09T15:08:04.096829Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:22:12.277994Z", "iopub.status.busy": "2025-01-14T00:22:12.277868Z", "iopub.status.idle": "2025-01-14T00:23:25.412046Z", "shell.execute_reply": "2025-01-14T00:23:25.411243Z", "shell.execute_reply.started": "2025-01-14T00:22:12.277980Z" } }, "outputs": [], "source": [ "# takes ~ 2 mins\n", "base_metas = read_from_s3(\n", " \"s3://suno-data/datasets/bundles/v1/genius_hq/metas_plus.jsonl\", read_f=read_jsonl\n", ")\n", "# TODO: original IDs are already unique here, but did we resolve correctly by highest genius views?\n", "# could be a way to filter translations that don't contain the word 'translation'" ] }, { "cell_type": "code", "execution_count": 6, "id": "c3c89a17", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "10000" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# # Randomly sample 10000 entries from base_metas\n", "# import random\n", "# random.seed(42) # For reproducibility\n", "# sampled_base_metas = random.sample(base_metas, 10000) if len(base_metas) > 10000 else base_metas\n", "# len(sampled_base_metas) # Display the original count" ] }, { "cell_type": "code", "execution_count": 5, "id": "251edcf4", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:09:54.750823Z", "start_time": "2024-01-09T15:09:50.718362Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:25.413069Z", "iopub.status.busy": "2025-01-14T00:23:25.412912Z", "iopub.status.idle": "2025-01-14T00:23:31.877797Z", "shell.execute_reply": "2025-01-14T00:23:31.877220Z", "shell.execute_reply.started": "2025-01-14T00:23:25.413054Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total, 2090009\n", "lang cut, 23432 (1.12%)\n", "genius view cut, 86048 (4.12%)\n", "youtube view cut, 96343 (4.61%)\n", "duration cut, 53169 (2.54%)\n", "lyrics cut, 34057 (1.63%)\n" ] } ], "source": [ "total_base_metas = len(base_metas)\n", "n_genius_view_cut = 5\n", "n_youtube_view_cut = 50\n", "max_duration = 8 * 60\n", "min_duration = 1 * 60\n", "max_len_lyrics = 9128\n", "min_len_lyrics = 64\n", "n_total_base_metas = len(base_metas)\n", "print(f\"total, {n_total_base_metas}\")\n", "lang_cut_count = sum(1 if m[\"lang\"] is None else 0 for m in base_metas)\n", "genius_view_cut_count = sum(\n", " 1 if m[\"genius_views\"] < n_genius_view_cut else 0 for m in base_metas\n", ")\n", "youtube_view_cut_count = sum(\n", " 1 if m[\"youtube_views\"] < n_youtube_view_cut else 0 for m in base_metas\n", ")\n", "duration_cut_count = sum(\n", " 1 if (m[\"duration_s\"] > max_duration or m[\"duration_s\"] < min_duration) else 0\n", " for m in base_metas\n", ")\n", "lyrics_cut_count = sum(\n", " 1 if (len(m[\"lyrics\"]) > max_len_lyrics or len(m[\"lyrics\"]) < min_len_lyrics) else 0\n", " for m in base_metas\n", ")\n", "\n", "print(\n", " f\"lang cut, {lang_cut_count} ({round(lang_cut_count / n_total_base_metas * 100, 2)}%)\"\n", ")\n", "print(\n", " f\"genius view cut, {genius_view_cut_count} ({round(genius_view_cut_count / n_total_base_metas * 100, 2)}%)\"\n", ")\n", "print(\n", " f\"youtube view cut, {youtube_view_cut_count} ({round(youtube_view_cut_count / n_total_base_metas * 100, 2)}%)\"\n", ")\n", "print(\n", " f\"duration cut, {duration_cut_count} ({round(duration_cut_count / n_total_base_metas * 100, 2)}%)\"\n", ")\n", "print(\n", " f\"lyrics cut, {lyrics_cut_count} ({round(lyrics_cut_count / n_total_base_metas * 100, 2)}%)\"\n", ")" ] }, { "cell_type": "code", "execution_count": 6, "id": "6ac295a0", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:01.309149Z", "start_time": "2024-01-09T15:09:54.752976Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:31.879727Z", "iopub.status.busy": "2025-01-14T00:23:31.879303Z", "iopub.status.idle": "2025-01-14T00:23:39.048856Z", "shell.execute_reply": "2025-01-14T00:23:39.048248Z", "shell.execute_reply.started": "2025-01-14T00:23:31.879710Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2090009 clips\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 2090009/2090009 [00:06<00:00, 301655.66it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "1835250 filtered clips\n", "106967 hours\n" ] } ], "source": [ "print(len(base_metas), \"clips\")\n", "metas = []\n", "seen_id = set()\n", "seen_path = set()\n", "for m in tqdm.tqdm(base_metas):\n", " # guessed_lang = _get_text_lang(m[\"lyrics\"])[1]\n", " if (\n", " m[\"genius_views\"] < n_genius_view_cut\n", " or m[\"youtube_views\"] < n_youtube_view_cut\n", " or m[\"lang\"] is None\n", " or \"translation\" in m[\"genius_slug\"].lower()\n", " or any(\"translation\" in tag.lower() for tag in m[\"tags_text\"])\n", " # or m[\"lang\"].lower()[:2] != \"en\"\n", " or m[\"duration_s\"] < min_duration\n", " or m[\"duration_s\"] > max_duration\n", " or len(m[\"lyrics\"]) < min_len_lyrics\n", " or len(m[\"lyrics\"]) > max_len_lyrics\n", " or m[\"id\"] in seen_id\n", " or m[\"audio_filepath\"] in seen_path\n", " # or _get_text_lang(m[\"lyrics\"])[1] != \"en\"\n", " ):\n", " continue\n", " seen_id.add(m[\"id\"])\n", " seen_path.add(m[\"audio_filepath\"])\n", " new_m = {\n", " \"id\": m[\"id\"],\n", " \"lyrics\": m[\"lyrics\"].strip(),\n", " \"duration_s\": m[\"duration_s\"],\n", " \"lang\": m[\"lang\"],\n", " # \"guess_lang\": guessed_lang,\n", " \"audio_filepath\": m[\"audio_filepath\"],\n", " \"original_id\": m[\"original_id\"],\n", " \"genius_slug\": m[\"genius_slug\"],\n", " \"genius_views\": m[\"genius_views\"],\n", " \"youtube_views\": m[\"youtube_views\"],\n", " }\n", " metas.append(new_m)\n", "print(len(metas), \"filtered clips\")\n", "print(round(sum(m[\"duration_s\"] for m in metas) / 60 / 60), \"hours\")\n", "metas_map = {m[\"id\"]: m for m in metas}\n", "# 2090009 clips\n", "# 1835250 filtered clips\n", "# 106967 hours" ] }, { "cell_type": "code", "execution_count": 7, "id": "710eb01c", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:01.313613Z", "start_time": "2024-01-09T15:10:01.311434Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:39.049576Z", "iopub.status.busy": "2025-01-14T00:23:39.049427Z", "iopub.status.idle": "2025-01-14T00:23:39.051784Z", "shell.execute_reply": "2025-01-14T00:23:39.051377Z", "shell.execute_reply.started": "2025-01-14T00:23:39.049560Z" } }, "outputs": [], "source": [ "# # only need to save this json once -- or you need to update\n", "# with open(\"/home/tony/Data/Hoot/genius_metas.json\", \"w\") as fp:\n", "# json.dump(metas, fp)" ] }, { "cell_type": "code", "execution_count": 8, "id": "aa66a026", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:02.592483Z", "start_time": "2024-01-09T15:10:01.314852Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:39.052404Z", "iopub.status.busy": "2025-01-14T00:23:39.052272Z", "iopub.status.idle": "2025-01-14T00:23:40.445874Z", "shell.execute_reply": "2025-01-14T00:23:40.445288Z", "shell.execute_reply.started": "2025-01-14T00:23:39.052391Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[('en', 1136063), ('es', 109120), ('pt', 78373), ('fr', 75540), ('de', 63397), ('ru', 57311), ('pl', 55307), ('it', 46156), ('tr', 25347), ('ko', 21898)]\n" ] } ], "source": [ "# check language counts\n", "c = Counter([m[\"lang\"] for m in metas])\n", "print(c.most_common(10))\n", "# filter out languages with less than 50 clips\n", "metas_map = {m[\"id\"]: m for m in metas if c[m[\"lang\"]] > 50}" ] }, { "cell_type": "code", "execution_count": 9, "id": "2b486253", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:19.399035Z", "start_time": "2024-01-09T15:10:02.594795Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:40.446601Z", "iopub.status.busy": "2025-01-14T00:23:40.446446Z", "iopub.status.idle": "2025-01-14T00:23:43.203204Z", "shell.execute_reply": "2025-01-14T00:23:43.202628Z", "shell.execute_reply.started": "2025-01-14T00:23:40.446584Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1834141 filtered clips\n", "en 1136063 66835 hours\n", "es 109120 6377 hours\n", "pt 78373 4559 hours\n", "fr 75540 4315 hours\n", "de 63397 3469 hours\n", "ru 57311 2912 hours\n", "pl 55307 3095 hours\n", "it 46156 2561 hours\n", "tr 25347 1490 hours\n", "ko 21898 1325 hours\n" ] } ], "source": [ "print(len(metas_map), \"filtered clips\")\n", "c = Counter([m[\"lang\"] for m in metas_map.values()])\n", "for k, v in c.most_common(10):\n", " print(\n", " k,\n", " c[k],\n", " round(sum(m[\"duration_s\"] for m in metas if m[\"lang\"] == k) / 60 / 60),\n", " \"hours\",\n", " )" ] }, { "cell_type": "code", "execution_count": 10, "id": "c2ce5d00", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:19.404006Z", "start_time": "2024-01-09T15:10:19.401076Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.203888Z", "iopub.status.busy": "2025-01-14T00:23:43.203738Z", "iopub.status.idle": "2025-01-14T00:23:43.206949Z", "shell.execute_reply": "2025-01-14T00:23:43.206498Z", "shell.execute_reply.started": "2025-01-14T00:23:43.203872Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'id': '023347b1-ce44-48fb-a3b9-4d760e3ab73a', 'lyrics': \"合：\\n噗通噗通的心跳\\nI love I love us\\n噗通噗通的心跳\\n孟美岐：\\n开始第一次心跳\\nPick me up 喊我名字\\n你越喜爱 我越可爱\\n吴宣仪：\\n直到第二次的心跳\\n你爱的勇敢少女\\n确定是我Rocket girl\\n傅菁：\\n从陌生变成热烈\\n成为彼此的特别\\n紫宁：\\n我为你加油打气\\n你对我热爱不减\\nSunnee：\\n你已经成为我心底语言\\n我的一切都和你有关联\\nSo 心的默契越来越了解\\n赖美云：\\n孤单 被你 燃烧成了温暖\\nOn love love love love yeah\\nIt's Our Show Time\\n吴宣仪：\\n噗通噗通\\n孟美岐：\\n心跳讯号 炙热我的微笑\\nSunnee：\\n我和你\\n杨超越：\\n再见拥抱 未来要更好\\n紫宁：\\n所有不凡的相遇\\n赖美云：\\n命中注定遇见你\\n段奥娟：\\nBabe I love I love us 限定的幸运\\n傅菁：\\n噗通噗通的心跳\\n合：\\nI love I love us\\n噗通噗通的心跳\\n李紫婷：\\n心永不分离\\n杨超越：\\n第三次心跳更确定\\n美丽化身成正义\\n现在起你挺我 Oh Yeah\\n徐梦洁：\\n从陌生变成热烈\\n融化彼此的世界\\n吴宣仪：\\n我为你加油打气\\n你对我热爱不减\\n赖美云：\\n热爱不减\\n段奥娟：\\n你已经成为我心底语言\\n我的一切都和你有关联\\nSo心的默契越来越了解\\nSunnee：\\n越了解\\n紫宁：\\n孤单 被你 燃烧成了温暖\\nOn love love love love yeah\\n孟美岐：\\nIt's Our Show Time\\n徐梦洁：\\n噗通噗通\\nSunnee：\\n心跳讯号 炙热我的微笑\\n杨超越：\\n我和你\\n李紫婷：\\n再见拥抱 未来要更好\\n傅菁：\\n所有不凡的相遇\\n命中注定遇见你\\nBabe I love I love us\\n孟美岐：\\n限定的幸运\\n段奥娟：\\n噗通噗通的心跳\\n合：\\nI love I love us\\n噗通噗通的心跳\\n徐梦洁：\\n心永不分离\\nYamy：\\n我的高冷被你改变360度\\n24小时在线分享你的mood\\n姐妹登场就是排面会飞的crew\\n时光会记得最美的心电图\\n李紫婷：\\nLa 心动的记忆\\n不管走多远保持初心\\nLa 美丽的相遇\\n一半的心跳 为你而跳\\n合：\\n噗通噗通 心跳讯号 炙热我的微笑\\n我和你 再见拥抱 未来要更好\\n所有不凡的相遇\\n命中注定遇见你\\nBabe I love I love us 限定的幸运\\n噗通噗通的心跳I love I love us\\n噗通噗通的心跳 心永不分离\\n噗通噗通的心跳I love I love us\\n噗通噗通的心跳 心永不分离\\nI love I love us\", 'duration_s': 92, 'lang': 'zh', 'audio_filepath': 's3://suno-data/datasets/harvest/genius_hq/audio/AY26g8oMP9Y.webm', 'original_id': 'AY26g8oMP9Y', 'genius_slug': '101-rocket-girls-101-11-times-heartbeat-11-lyrics', 'genius_views': 24, 'youtube_views': 3191}\n" ] } ], "source": [ "for m in metas:\n", " if m[\"lang\"] == \"zh\":\n", " print(m)\n", " break" ] }, { "cell_type": "markdown", "id": "346cc768", "metadata": {}, "source": [ "# Pack the data\n", "format to save to:\n", "{\"audio_filepath\": \"./an4/wav/an4_clstk/fash/an251-fash-b.wav\", \"duration\": 1.0, \"text\": \"yes\"}\n" ] }, { "cell_type": "code", "execution_count": 11, "id": "71df5d56", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:20.258192Z", "start_time": "2024-01-09T15:10:19.486945Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.207648Z", "iopub.status.busy": "2025-01-14T00:23:43.207505Z", "iopub.status.idle": "2025-01-14T00:23:43.244969Z", "shell.execute_reply": "2025-01-14T00:23:43.244585Z", "shell.execute_reply.started": "2025-01-14T00:23:43.207634Z" } }, "outputs": [], "source": [ "# # this will also take 2 min...\n", "# pre_download_val_metas = read_jsonl(\"/app/suno/data/audio_2ch_48khz_lg/metas_val.jsonl\")\n", "# pre_download_train_metas = read_jsonl(\n", "# \"/app/suno/data/audio_2ch_48khz_lg/metas_tr.jsonl\"\n", "# )\n", "# downloaded_ids_to_info = {}\n", "# for download_meta in pre_download_train_metas:\n", "# if download_meta[\"original_id\"] not in downloaded_ids_to_info:\n", "# downloaded_ids_to_info[download_meta[\"original_id\"]] = download_meta\n", "# for download_meta in pre_download_val_metas:\n", "# if download_meta[\"original_id\"] not in downloaded_ids_to_info:\n", "# downloaded_ids_to_info[download_meta[\"original_id\"]] = download_meta\n", "# downloaded_ids = set(downloaded_ids_to_info.keys())\n", "# print(len(downloaded_ids))" ] }, { "cell_type": "code", "execution_count": 12, "id": "9870b504", "metadata": { "execution": { "iopub.execute_input": "2025-01-14T00:23:43.245543Z", "iopub.status.busy": "2025-01-14T00:23:43.245415Z", "iopub.status.idle": "2025-01-14T00:23:43.278868Z", "shell.execute_reply": "2025-01-14T00:23:43.278489Z", "shell.execute_reply.started": "2025-01-14T00:23:43.245530Z" } }, "outputs": [], "source": [ "# sum(\n", "# download_meta[\"dataset\"] == \"genius_hq\"\n", "# for download_meta in pre_download_train_metas\n", "# )\n", "# subset_meta = [\n", "# m for m in metas if (m[\"id\"] in downloaded_ids)\n", "# ] # and m[\"lang\"] == \"en\")]\n", "# print(len(subset_meta))\n", "# 986464" ] }, { "cell_type": "code", "execution_count": 13, "id": "c8227733", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:20.872851Z", "start_time": "2024-01-09T15:10:20.261963Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.279446Z", "iopub.status.busy": "2025-01-14T00:23:43.279310Z", "iopub.status.idle": "2025-01-14T00:23:43.323282Z", "shell.execute_reply": "2025-01-14T00:23:43.322846Z", "shell.execute_reply.started": "2025-01-14T00:23:43.279433Z" } }, "outputs": [], "source": [ "subset_meta = metas.copy()" ] }, { "cell_type": "code", "execution_count": 14, "id": "2f9e4207", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:21.009129Z", "start_time": "2024-01-09T15:10:20.874879Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.324006Z", "iopub.status.busy": "2025-01-14T00:23:43.323874Z", "iopub.status.idle": "2025-01-14T00:23:43.360124Z", "shell.execute_reply": "2025-01-14T00:23:43.359705Z", "shell.execute_reply.started": "2025-01-14T00:23:43.323993Z" } }, "outputs": [], "source": [ "RE_EMOJI = re.compile(\"[\\U00010000-\\U0010ffff]\", flags=re.UNICODE)\n", "\n", "\n", "def clean_text(text):\n", " \"\"\"General text cleaning. A bit tight but makes the content very clean.\"\"\"\n", " text = \"\\n\" + text\n", " text = RE_EMOJI.sub(r\"\", text) # remove emojis\n", " text = text.replace(\"’\", \"'\").lower()\n", " text = text.replace('\"', \"\").lower()\n", " text = re.sub(r\"\\[.+?\\]\", \" \", text) # tags\n", " text = re.sub(r\"\\n.+?\\:\", \" \", text) # new line ends with :\n", " text = re.sub(r\"\\n.+?\\：\", \" \", text) # new line ends with :\n", " text = re.sub(r\"\\n\\(.+?\\)\", \" \", text) # new line with ()\n", " text = re.sub(r\"[\\d]\", \" \", text) # digits\n", " text = re.sub(r\"▁\", \"\", text) # special stuff\n", " text = remove_speakers(text)\n", " text = re.sub(r\"[^\\w\\'\\s]\", \" \", text) # keep only the words\n", " # text = re.sub(r\"[ \\t]+\", \" \", text) # remove multiple whitespace\n", " # text = re.sub(r\"\\n{2,}\", \"\\n\", text) # Merge multiple newlines into single newline\n", " text = normalize_whitespace(text)\n", " return text" ] }, { "cell_type": "code", "execution_count": 15, "id": "0d2555e3", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:21.070547Z", "start_time": "2024-01-09T15:10:21.011397Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.360753Z", "iopub.status.busy": "2025-01-14T00:23:43.360609Z", "iopub.status.idle": "2025-01-14T00:23:43.396751Z", "shell.execute_reply": "2025-01-14T00:23:43.396322Z", "shell.execute_reply.started": "2025-01-14T00:23:43.360738Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(1er Verse)\n", "C'est maintenant que j'mûris\n", "Que je m'aperçois dans quoi que j'vis\n", "J'vois que tout le monde paranoïe\n", "Savent même pas c'est quoi qu'y arrive\n", "Parfois je m'arrête\n", "En me disant c'est sur qu'un jour j'die, mais comment\n", "Personnellement, j'serais pas surpris\n", "Que ça serait une surprise\n", "Exemple, le temps\n", "La température fend\n", "La nature, elle nous haïs\n", "On a agit comme des p'tits mouks\n", "Fait que maintenant tout brise en les brisant\n", "L'échelle raccourcit\n", "J'vois qu' en haut, c'est pas loin\n", "Fuck, c'est tanant\n", "Y'a des choses que je me doute\n", "Quand je check le cadran\n", "Sur ma route, j'suis méfiant\n", "Tic tic tic\n", "J'avance en regardant\n", "J'suis tellement stressé\n", "Que mon tic, je le trouve stressant\n", "Quand j'me frotte le nez\n", "Depuis que j'suis enfant\n", "J'en ai enduré sérieusement\n", "On dit que le temps arrange les choses\n", "Mais le temps, je le trouve fucker moi\n", "(Refrain x2)\n", "J'vois la nature maltraitée\n", "Un autre ti-cul mal élevé\n", "Une autre prison bien remplie\n", "Un autre indice à cacher\n", "La vie, c'est un suspense\n", "Et moi, j'écris\n", "\n", "(2e Verse)\n", "Quand j'regarde ça, je m'inquiète pour demain\n", "Notre génération délire\n", "Nos enfants, dans quoi qui vont grandir hein?\n", "En tous cas, je les plaint\n", "Mais devant eux, je garde le sourire\n", "J'veux pas qu'ils prennent mon avion\n", "J'veux qu'ils sachent courir sans trébucher\n", "Mais c'est dur quand on regarde la société\n", "Dans n'importe quel quartier, y'a des mal élevés\n", "Y'a des filles que ça a douze ans\n", "Pis elles sont quasiment désabillées\n", "Le p'tit cul avec sa smoke plus grosse que lui\n", "Fréquente l'école tout crotté\n", "Déjà des jokes sur ma couleur\n", "Mais où est-ce qui es a pêché\n", "En plus d'être mal élevé, mal instruit\n", "Y sait même pas que j'suis un mélangé\n", "Éduquer un enfant de cette façon, c't'un péché\n", "Réalise bien qu'est-ce que lui en haut t'a donné\n", "Réalise bien qu'est-ce que lui en haut t'a donné\n", "Réalise bien qu'est-ce que la vie d'ici t'a donné\n", "Straight up\n", "\n", "\n", "(Refrain x2)\n", "J'vois la nature maltraitée\n", "Un autre ti-cul mal élevé\n", "Une autre prison bien remplie\n", "Un autre indice à cacher\n", "La vie, c'est un suspense\n", "Et moi, j'écris\n", "\n", "(3e Verse)\n", "Vu d'en haut de ma tour, j'vois que c'est pas beau\n", "Je m'aperçois que dehors, y fait pas chaud\n", "J'vois que tout le monde panique\n", "Veulent tous le plus gros morceau\n", "Détruire la nature ça c'est pas cool\n", "Aujourd'hui, elle est frustrée contre nous\n", "J'pense que l'être humain devrait commencer à prendre ça cool\n", "Parce que le temps qu'il faut qu'on rattrape\n", "J'vois qu'il s'écoule\n", "Faut évoluer, juste réalise dans quoi que t'avance\n", "C'est pas fini, c'est ta planète\n", "Human being\n", "\n", "(Refrain x2)\n", "J'vois la nature maltraitée\n", "Un autre ti-cul mal élevé\n", "Une autre prison bien remplie\n", "Un autre indice à cacher\n", "La vie, c'est un suspense\n", "Et moi, j'écris....\n", "-------\n", "c'est maintenant que j'mûris\n", "que je m'aperçois dans quoi que j'vis\n", "j'vois que tout le monde paranoïe\n", "savent même pas c'est quoi qu'y arrive\n", "parfois je m'arrête\n", "en me disant c'est sur qu'un jour j'die mais comment\n", "personnellement j'serais pas surpris\n", "que ça serait une surprise\n", "exemple  le temps\n", "la température fend\n", "la nature elle nous haïs\n", "on a agit comme des p'tits mouks\n", "fait que maintenant tout brise en les brisant\n", "l'échelle raccourcit\n", "j'vois qu' en haut c'est pas loin\n", "fuck c'est tanant\n", "y'a des choses que je me doute\n", "quand je check le cadran\n", "sur ma route j'suis méfiant\n", "tic tic tic\n", "j'avance en regardant\n", "j'suis tellement stressé\n", "que mon tic je le trouve stressant\n", "quand j'me frotte le nez\n", "depuis que j'suis enfant\n", "j'en ai enduré sérieusement\n", "on dit que le temps arrange les choses\n", "mais le temps je le trouve fucker moi \n", "j'vois la nature maltraitée\n", "un autre ti cul mal élevé\n", "une autre prison bien remplie\n", "un autre indice à cacher\n", "la vie c'est un suspense\n", "et moi j'écris\n", " \n", "quand j'regarde ça je m'inquiète pour demain\n", "notre génération délire\n", "nos enfants dans quoi qui vont grandir hein \n", "en tous cas je les plaint\n", "mais devant eux je garde le sourire\n", "j'veux pas qu'ils prennent mon avion\n", "j'veux qu'ils sachent courir sans trébucher\n", "mais c'est dur quand on regarde la société\n", "dans n'importe quel quartier y'a des mal élevés\n", "y'a des filles que ça a douze ans\n", "pis elles sont quasiment désabillées\n", "le p'tit cul avec sa smoke plus grosse que lui\n", "fréquente l'école tout crotté\n", "déjà des jokes sur ma couleur\n", "mais où est ce qui es a pêché\n", "en plus d'être mal élevé mal instruit\n", "y sait même pas que j'suis un mélangé\n", "éduquer un enfant de cette façon c't'un péché\n", "réalise bien qu'est ce que lui en haut t'a donné\n", "réalise bien qu'est ce que lui en haut t'a donné\n", "réalise bien qu'est ce que la vie d'ici t'a donné\n", "straight up\n", " \n", "j'vois la nature maltraitée\n", "un autre ti cul mal élevé\n", "une autre prison bien remplie\n", "un autre indice à cacher\n", "la vie c'est un suspense\n", "et moi j'écris\n", " \n", "vu d'en haut de ma tour j'vois que c'est pas beau\n", "je m'aperçois que dehors y fait pas chaud\n", "j'vois que tout le monde panique\n", "veulent tous le plus gros morceau\n", "détruire la nature ça c'est pas cool\n", "aujourd'hui elle est frustrée contre nous\n", "j'pense que l'être humain devrait commencer à prendre ça cool\n", "parce que le temps qu'il faut qu'on rattrape\n", "j'vois qu'il s'écoule\n", "faut évoluer juste réalise dans quoi que t'avance\n", "c'est pas fini c'est ta planète\n", "human being\n", " \n", "j'vois la nature maltraitée\n", "un autre ti cul mal élevé\n", "une autre prison bien remplie\n", "un autre indice à cacher\n", "la vie c'est un suspense\n", "et moi j'écris \n" ] } ], "source": [ "# Quality checks\n", "n_c = 0\n", "for m in metas:\n", " if m[\"lang\"] == \"fr\":\n", " n_c += 1\n", " if n_c == 1:\n", " text = m[\"lyrics\"]\n", " print(text)\n", " print(\"-------\")\n", " print(clean_text(text))\n", " break" ] }, { "cell_type": "code", "execution_count": 16, "id": "e8b679e7", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:21.167632Z", "start_time": "2024-01-09T15:10:21.072528Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.397353Z", "iopub.status.busy": "2025-01-14T00:23:43.397215Z", "iopub.status.idle": "2025-01-14T00:23:43.455604Z", "shell.execute_reply": "2025-01-14T00:23:43.455194Z", "shell.execute_reply.started": "2025-01-14T00:23:43.397338Z" } }, "outputs": [], "source": [ "# # TODO: swap to larger tokenizer\n", "# import sentencepiece\n", "\n", "# tokenizer = sentencepiece.SentencePieceProcessor()\n", "# tokenizer.load(\"tokenizer_spe_bpe_v20480/tokenizer.model\")" ] }, { "cell_type": "code", "execution_count": 17, "id": "4c0e6344", "metadata": {}, "outputs": [], "source": [ "from hoot.model import Tokenizer\n", "\n", "tokenizer = Tokenizer(filepath=\"/app/suno/data/dpo/models/tokenizer_60k.json\")" ] }, { "cell_type": "code", "execution_count": 18, "id": "feacb5f1", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T15:10:22.487284Z", "start_time": "2024-01-09T15:10:21.169512Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:43.456202Z", "iopub.status.busy": "2025-01-14T00:23:43.456070Z", "iopub.status.idle": "2025-01-14T00:23:44.817495Z", "shell.execute_reply": "2025-01-14T00:23:44.816908Z", "shell.execute_reply.started": "2025-01-14T00:23:43.456189Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1835250 1835250\n" ] } ], "source": [ "print(len(subset_meta), len(set(m[\"original_id\"] for m in subset_meta)))\n", "random.shuffle(subset_meta)" ] }, { "cell_type": "code", "execution_count": 19, "id": "8d3fa27d-c422-444d-8fbf-149274636627", "metadata": { "execution": { "iopub.status.busy": "2025-01-15T01:20:39.231764Z", "iopub.status.idle": "2025-01-15T01:20:39.231945Z", "shell.execute_reply": "2025-01-15T01:20:39.231859Z", "shell.execute_reply.started": "2025-01-15T01:20:39.231851Z" } }, "outputs": [], "source": [ "## two nodes 35 hrs to download" ] }, { "cell_type": "code", "execution_count": 20, "id": "3a3246ca", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:53.367719Z", "start_time": "2024-01-09T15:10:22.489702Z" }, "execution": { "iopub.execute_input": "2025-01-14T00:23:44.818238Z", "iopub.status.busy": "2025-01-14T00:23:44.818090Z", "iopub.status.idle": "2025-01-14T01:36:38.514831Z", "shell.execute_reply": "2025-01-14T01:36:38.514272Z", "shell.execute_reply.started": "2025-01-14T00:23:44.818222Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 1835250/1835250 [2:51:37<00:00, 178.22it/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ "total 1810350, too long: 19821, text too short: 514, text too long: 118, text bad: 4447\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "total_infos = []\n", "n_text_too_short = 0\n", "n_text_too_long = 0\n", "n_audio_too_long = 0\n", "n_lang = collections.defaultdict(int)\n", "n_repeat_ratio_too_high = 0\n", "n_lang_max_cut = 5_000_000 # 10k songs :D, 2_000_000 is max\n", "bad_meta_ids = []\n", "for meta in tqdm.tqdm(subset_meta):\n", " output_path = os.path.join(\n", " \"/app/suno/data/hoot/audios/genius\", f\"{meta['original_id']}.mp3\"\n", " )\n", " if not os.path.exists(output_path):\n", " continue\n", " # output_path = downloaded_ids_to_info[meta[\"id\"]][\"filepath\"]\n", " if (\n", " meta[\"duration_s\"] > 400\n", " ): # this cut is mainly for batching purposes...X.x, but...we probably shouldn't have this.\n", " n_audio_too_long += 1\n", " continue\n", " n_lang[meta[\"lang\"]] += 1\n", " # if n_lang[meta[\"lang\"]] > n_lang_max_cut:\n", " # continue\n", " cleaned_text = clean_text(meta[\"lyrics\"])\n", " # cleaned_text = meta[\"lyrics\"].lower() # clean_text(meta[\"lyrics\"])\n", " # cleaned_tokens = tokenizer.encode(cleaned_text)\n", " # decoded_tokens = tokenizer.decode(cleaned_tokens)\n", " # if decoded_tokens != cleaned_text:\n", " # print(cleaned_text)\n", " # print(decoded_tokens)\n", " if len(cleaned_text) < 20:\n", " n_text_too_short += 1\n", " continue\n", " # check the high freq token frequency\n", " encoded_array = np.array(tokenizer.encode(cleaned_text))\n", " if len(encoded_array) > meta[\"duration_s\"] * 12:\n", " n_text_too_long += 1\n", " continue\n", " repeat_ratio = max(np.bincount(encoded_array)) / (encoded_array.shape[0] + 1)\n", " if repeat_ratio > 0.3:\n", " n_repeat_ratio_too_high += 1\n", " bad_meta_ids.append(meta[\"id\"])\n", " continue\n", " total_infos.append(\n", " {\n", " \"audio_filepath\": output_path,\n", " \"duration\": meta[\"duration_s\"],\n", " \"text\": cleaned_text,\n", " \"lyrics\": meta[\"lyrics\"],\n", " \"lang\": meta[\"lang\"],\n", " \"id\": meta[\"id\"],\n", " \"genius_views\": meta.get(\"genius_views\", 0),\n", " \"youtube_views\": meta.get(\"youtube_views\", 0),\n", " }\n", " )\n", " if not os.path.exists(output_path):\n", " # print(\"WTF\", output_path)\n", " continue\n", "# audio = Audio.from_s3(meta[\"audio_filepath\"])\n", "# audio = audio.convert(16_000, audio.byte_width, n_channels=1)\n", "# audio.to_wav(output_path)\n", "print(\n", " f\"total {len(total_infos)}, too long: {n_audio_too_long}, text too short: {n_text_too_short}, text too long: {n_text_too_long}, text bad: {n_repeat_ratio_too_high}\"\n", ")" ] }, { "cell_type": "code", "execution_count": 21, "id": "618075a0", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:54.930054Z", "start_time": "2024-01-09T16:37:53.368988Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:38.515547Z", "iopub.status.busy": "2025-01-14T01:36:38.515398Z", "iopub.status.idle": "2025-01-14T01:36:40.077169Z", "shell.execute_reply": "2025-01-14T01:36:40.076608Z", "shell.execute_reply.started": "2025-01-14T01:36:38.515532Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total duration khr 104.30186805555556 60 400 20 8946\n" ] } ], "source": [ "print(\n", " \"total duration khr\",\n", " sum(m[\"duration\"] for m in total_infos) / 3600000,\n", " min(m[\"duration\"] for m in total_infos),\n", " max(m[\"duration\"] for m in total_infos),\n", " min(len(m[\"text\"]) for m in total_infos),\n", " max(len(m[\"text\"]) for m in total_infos),\n", ")\n", "# 29.15879888888889 120 100 2879\n", "# cut on durations 400: total duration khr 88.80962305555556 60 480 100 5616\n", "# cut on 240: total duration khr 54.709809444444446 60 240 100 2880" ] }, { "cell_type": "code", "execution_count": 22, "id": "76dcc2f9-777f-476e-b190-fddced599117", "metadata": { "execution": { "iopub.execute_input": "2025-01-15T01:20:50.635781Z", "iopub.status.busy": "2025-01-15T01:20:50.635598Z", "iopub.status.idle": "2025-01-15T01:20:50.638227Z", "shell.execute_reply": "2025-01-15T01:20:50.637805Z", "shell.execute_reply.started": "2025-01-15T01:20:50.635766Z" } }, "outputs": [], "source": [ "# exist_info = []\n", "# for test_input in test_info:\n", "# if os.path.exists(test_input[\"audio_filepath\"]):\n", "# exist_info.append(test_input)\n", "# print(len(info), len(exist_info))" ] }, { "cell_type": "code", "execution_count": 23, "id": "d8eb67a9", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:55.757800Z", "start_time": "2024-01-09T16:37:54.931303Z" }, "execution": { "iopub.execute_input": "2025-01-15T00:53:38.889667Z", "iopub.status.busy": "2025-01-15T00:53:38.889229Z", "iopub.status.idle": "2025-01-15T00:53:38.893159Z", "shell.execute_reply": "2025-01-15T00:53:38.892581Z", "shell.execute_reply.started": "2025-01-15T00:53:38.889646Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1810350 0\n" ] } ], "source": [ "# cutoff = -2000\n", "# random.shuffle(info)\n", "# train_info = info[:cutoff]\n", "# test_info = info[cutoff:]\n", "# print(len(train_info), len(test_info))\n", "# # 545838 1000\n", "train_info = total_infos.copy()\n", "test_info = []\n", "print(len(train_info), len(test_info))" ] }, { "cell_type": "code", "execution_count": 24, "id": "72bc9539", "metadata": { "execution": { "iopub.execute_input": "2025-01-14T01:36:40.832617Z", "iopub.status.busy": "2025-01-14T01:36:40.832218Z", "iopub.status.idle": "2025-01-14T01:36:40.834572Z", "shell.execute_reply": "2025-01-14T01:36:40.834176Z", "shell.execute_reply.started": "2025-01-14T01:36:40.832599Z" } }, "outputs": [], "source": [ "# BREAK" ] }, { "cell_type": "code", "execution_count": 25, "id": "a48b2d77", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:55.761119Z", "start_time": "2024-01-09T16:37:55.759035Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:40.835146Z", "iopub.status.busy": "2025-01-14T01:36:40.835018Z", "iopub.status.idle": "2025-01-14T01:36:40.870228Z", "shell.execute_reply": "2025-01-14T01:36:40.869844Z", "shell.execute_reply.started": "2025-01-14T01:36:40.835132Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/all_train_manifest_fast.json\", \"w\") as fp:\n", "# # for l in train_info[:80]:\n", "# for l in train_info[:80]:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')\n", "# with open(\"/home/tony/Data/Hoot/all_test_manifest_fast.json\", \"w\") as fp:\n", "# for l in test_info[:8]:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')" ] }, { "cell_type": "code", "execution_count": 26, "id": "4a7003ff", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:55.836785Z", "start_time": "2024-01-09T16:37:55.762070Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:40.870803Z", "iopub.status.busy": "2025-01-14T01:36:40.870669Z", "iopub.status.idle": "2025-01-14T01:36:40.906745Z", "shell.execute_reply": "2025-01-14T01:36:40.906369Z", "shell.execute_reply.started": "2025-01-14T01:36:40.870789Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/multi_all_104khr_filtered_train_manifest.json\", \"w\") as fp:\n", "# for l in train_info:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')\n", "# with open(\"/home/tony/Data/Hoot/multi_all_104khr_filtered_test_manifest.json\", \"w\") as fp:\n", "# for l in test_info:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')" ] }, { "cell_type": "code", "execution_count": 27, "id": "506b6b20", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:55.891610Z", "start_time": "2024-01-09T16:37:55.838219Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:40.907307Z", "iopub.status.busy": "2025-01-14T01:36:40.907178Z", "iopub.status.idle": "2025-01-14T01:36:40.944089Z", "shell.execute_reply": "2025-01-14T01:36:40.943708Z", "shell.execute_reply.started": "2025-01-14T01:36:40.907294Z" } }, "outputs": [], "source": [ "# with open(DataStoreObject(manifest).get(), 'r') as in_reader:\n", "# for line in in_reader:\n", "# item = json.loads(line)" ] }, { "cell_type": "markdown", "id": "5907d689", "metadata": {}, "source": [ "# Train the tokenizer" ] }, { "cell_type": "markdown", "id": "3a36ab23", "metadata": {}, "source": [ "\n", "\n", "- 0.998 has vocab size of 780\n", "- 0.9995 has vocab size of 1936\n", "- 0.9996 has vocab of 2176\n", "- 0.9999 has vocab of 3878\n", "- 1.0 has vocab of 12402\n", "------------------------\n", "python ./scripts/process_asr_text_tokenizer.py \\\n", " --manifest=\"/home/tony/Data/Hoot/all_train_manifest.json\" \\\n", " --data_root=\"./tokenizers/all/\" \\\n", " --vocab_size=1024 \\\n", " --tokenizer=\"spe\" \\\n", " --spe_character_coverage=0.998 \\\n", " --spe_type=\"bpe\" \\\n", " --log\n", "\n", "------------------------\n", "python ./scripts/process_asr_text_tokenizer.py \\\n", " --manifest=\"/home/tony/Data/Hoot/all_train_manifest.json\" \\\n", " --data_root=\"./tokenizers/multi/\" \\\n", " --vocab_size=5120 \\\n", " --tokenizer=\"spe\" \\\n", " --spe_character_coverage=0.9999 \\\n", " --spe_type=\"bpe\" \\\n", " --log\n", " \n", " ------------------------\n", "python ./scripts/process_asr_text_tokenizer.py \\\n", " --manifest=\"/home/tony/Data/Hoot/multi_filtered_train_manifest.json\" \\\n", " --data_root=\"./tokenizers/multi_filtered/\" \\\n", " --vocab_size=5120 \\\n", " --tokenizer=\"spe\" \\\n", " --spe_character_coverage=0.9999 \\\n", " --spe_type=\"bpe\" \\\n", " --log\n", " \n", " ------------------------\n", " python ./scripts/process_asr_text_tokenizer.py \\\n", " --manifest=\"/home/tony/Data/Hoot/metas.json\" \\\n", " --data_root=\"./tokenizers/full/\" \\\n", " --vocab_size=10240 \\\n", " --tokenizer=\"spe\" \\\n", " --spe_character_coverage=1.0 \\\n", " --spe_type=\"bpe\" \\\n", " --log\n", " \n", " ------------------------\n", " python ./scripts/process_asr_text_tokenizer.py \\\n", " --manifest=\"/home/tony/Data/Hoot/multi_all_104khr_filtered_train_manifest.json\" \\\n", " --data_root=\"./tokenizers/v3/\" \\\n", " --vocab_size=10241 \\\n", " --tokenizer=\"spe\" \\\n", " --spe_character_coverage=1.0 \\\n", " --spe_type=\"bpe\" \\\n", " --log\n", "\n", " ------------------------\n", " python ./scripts/process_asr_text_tokenizer.py \\\n", " --manifest=\"/home/tony/Data/Hoot/genius_metas.json,/home/tony/Data/Hoot/deezer_metas.json,/home/tony/Data/Hoot/discogs_metas.json\" \\\n", " --data_root=\"./tokenizers/v5/\" \\\n", " --vocab_size=20481 \\\n", " --tokenizer=\"spe\" \\\n", " --spe_character_coverage=0.99999 \\\n", " --spe_type=\"bpe\" \\\n", " --log" ] }, { "cell_type": "markdown", "id": "a0616448", "metadata": {}, "source": [ "# Check tokenzier" ] }, { "cell_type": "code", "execution_count": 28, "id": "253035d4", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:55.947077Z", "start_time": "2024-01-09T16:37:55.892947Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:40.944620Z", "iopub.status.busy": "2025-01-14T01:36:40.944495Z", "iopub.status.idle": "2025-01-14T01:36:40.981045Z", "shell.execute_reply": "2025-01-14T01:36:40.980637Z", "shell.execute_reply.started": "2025-01-14T01:36:40.944606Z" } }, "outputs": [], "source": [ "test_text = \"whoa oh oh whoa oh oh whoa oh oh oh oh oh oh whoa oh oh whoa oh oh whoa oh oh oh oh oh oh 취할 것 같은 불빛이 화려한 밤에 누구보다도 자유로워 지기를 바래 아름다운 이 순간이 끝나기 전에 바람소리에 이 음악이 실리길 원해 whoa oh oh whoa oh oh whoa oh oh oh oh oh oh whoa oh oh whoa oh oh whoa oh oh oh oh oh oh lets get it on tonight and set me free tonight 이 순간 즐겨봐 너와 나 소리 높여 lets groove it party time 두근거리는 느낌이 황홀한 밤에 푸른 바다에 달빛을 물들이길 바래 파도가 이 밤을 데려가기 전에 지금 이대로 시간이 멈추기를 원해 whoa oh oh whoa oh oh whoa oh oh oh oh oh oh whoa oh oh whoa oh oh whoa oh oh oh oh oh oh 이 순간 즐겨봐 너와 나 소리 높여 lets groove it party time whoa oh oh whoa oh oh whoa oh oh oh oh oh oh whoa oh oh whoa oh oh whoa oh oh oh oh oh oh 이 순간 즐겨봐 너와 나 소리 높여 lets groove it party time whoa whoa whoa yeah oh whoa oh whoa oh whoa\"" ] }, { "cell_type": "code", "execution_count": 29, "id": "d0599dc0", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.061972Z", "start_time": "2024-01-09T16:37:55.948411Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:40.981716Z", "iopub.status.busy": "2025-01-14T01:36:40.981584Z", "iopub.status.idle": "2025-01-14T01:36:41.081968Z", "shell.execute_reply": "2025-01-14T01:36:41.081553Z", "shell.execute_reply.started": "2025-01-14T01:36:40.981702Z" } }, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import sentencepiece\n", "\n", "tokenizer = sentencepiece.SentencePieceProcessor()\n", "tokenizer.load(\"tokenizer_spe_bpe_v20480/tokenizer.model\")" ] }, { "cell_type": "code", "execution_count": 30, "id": "6baf7076", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.122437Z", "start_time": "2024-01-09T16:37:56.063015Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.082560Z", "iopub.status.busy": "2025-01-14T01:36:41.082427Z", "iopub.status.idle": "2025-01-14T01:36:41.108656Z", "shell.execute_reply": "2025-01-14T01:36:41.108197Z", "shell.execute_reply.started": "2025-01-14T01:36:41.082546Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(312,)\n", "whoa oh oh whoa oh oh whoa oh oh oh oh oh oh whoa oh oh whoa oh oh whoa oh oh oh oh oh oh 취할 것 같은 불빛이 화려한 밤에 누구보다도 자유로워 지기를 바래 아름다운 이 순간이 끝나기 전에 바람소리에 이 음악이 실리길 원해 whoa oh oh whoa oh oh whoa oh oh oh oh oh oh whoa oh oh whoa oh oh whoa oh oh oh oh oh oh lets get it on tonight and set me free tonight 이 순간 즐겨봐 너와 나 소리 높여 lets groove it party time 두근거리는 느낌이 황홀한 밤에 푸른 바다에 달빛을 물들이길 바래 파도가 이 밤을 데려가기 전에 지금 이대로 시간이 멈추기를 원해 whoa oh oh whoa oh oh whoa oh oh oh oh oh oh whoa oh oh whoa oh oh whoa oh oh oh oh oh oh 이 순간 즐겨봐 너와 나 소리 높여 lets groove it party time whoa oh oh whoa oh oh whoa oh oh oh oh oh oh whoa oh oh whoa oh oh whoa oh oh oh oh oh oh 이 순간 즐겨봐 너와 나 소리 높여 lets groove it party time whoa whoa whoa yeah oh whoa oh whoa oh whoa\n" ] } ], "source": [ "print(np.array(tokenizer.encode(test_text)).shape)\n", "print(tokenizer.decode(tokenizer.encode(test_text)))" ] }, { "cell_type": "markdown", "id": "6d966633", "metadata": {}, "source": [ "# DEBUG" ] }, { "cell_type": "code", "execution_count": 31, "id": "353d4d0f", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.239480Z", "start_time": "2024-01-09T16:37:56.184175Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.109247Z", "iopub.status.busy": "2025-01-14T01:36:41.109114Z", "iopub.status.idle": "2025-01-14T01:36:41.202044Z", "shell.execute_reply": "2025-01-14T01:36:41.201540Z", "shell.execute_reply.started": "2025-01-14T01:36:41.109233Z" } }, "outputs": [], "source": [ "# x = []\n", "# with open(\"/home/tony/Data/Hoot/all_train_manifest.json\", \"r\") as fp:\n", "# for l in fp:\n", "# x.append(json.loads(l))\n", "\n", "# bad_indices = []\n", "# for i, meta in tqdm.tqdm(enumerate(x)):\n", "# encoded_array = np.array(tokenizer.encode(meta[\"text\"]))\n", "# repeat_ratio = max(np.bincount(encoded_array)) / (encoded_array.shape[0] + 1)\n", "# if repeat_ratio > 0.3:\n", "# # print(repeat_ratio, meta)\n", "# bad_indices.append(i)" ] }, { "cell_type": "code", "execution_count": 32, "id": "e5f54770", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.293353Z", "start_time": "2024-01-09T16:37:56.240396Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.202685Z", "iopub.status.busy": "2025-01-14T01:36:41.202544Z", "iopub.status.idle": "2025-01-14T01:36:41.239169Z", "shell.execute_reply": "2025-01-14T01:36:41.238783Z", "shell.execute_reply.started": "2025-01-14T01:36:41.202670Z" } }, "outputs": [], "source": [ "# len(bad_indices)\n", "\n", "# bad_ids = [\n", "# 36004, 91329, 772492\n", "# ]" ] }, { "cell_type": "code", "execution_count": 33, "id": "f65a307d", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.365226Z", "start_time": "2024-01-09T16:37:56.294287Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.239745Z", "iopub.status.busy": "2025-01-14T01:36:41.239620Z", "iopub.status.idle": "2025-01-14T01:36:41.273980Z", "shell.execute_reply": "2025-01-14T01:36:41.273608Z", "shell.execute_reply.started": "2025-01-14T01:36:41.239732Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/all_train_manifest_debug.json\", \"w\") as fp:\n", "# # for l in train_info[:80]:\n", "# for l in selected:\n", "# json.dump(x[l], fp, ensure_ascii=True)\n", "# fp.write('\\n')" ] }, { "cell_type": "code", "execution_count": 34, "id": "9c374d06", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.420690Z", "start_time": "2024-01-09T16:37:56.366689Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.274539Z", "iopub.status.busy": "2025-01-14T01:36:41.274414Z", "iopub.status.idle": "2025-01-14T01:36:41.312345Z", "shell.execute_reply": "2025-01-14T01:36:41.311969Z", "shell.execute_reply.started": "2025-01-14T01:36:41.274526Z" } }, "outputs": [], "source": [ "# for bad_id in bad_ids:\n", "# print(x[bad_id])\n", "\n", "# test_audio = Audio.from_file('/app/suno/data/hoot/2I8_OV4DtCs.wav')\n", "# test_audio.play(compress=False)\n", "\n", "# for meta in base_metas:\n", "# if meta[\"original_id\"] == \"JpkwJW92UcY\":\n", "# print(meta)" ] }, { "cell_type": "markdown", "id": "e7f8985c", "metadata": {}, "source": [ "# Load the hooted output" ] }, { "cell_type": "code", "execution_count": 35, "id": "a7be6e73", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.505275Z", "start_time": "2024-01-09T16:37:56.421982Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.312896Z", "iopub.status.busy": "2025-01-14T01:36:41.312777Z", "iopub.status.idle": "2025-01-14T01:36:41.350393Z", "shell.execute_reply": "2025-01-14T01:36:41.350007Z", "shell.execute_reply.started": "2025-01-14T01:36:41.312884Z" } }, "outputs": [], "source": [ "# total_infos = dict()\n", "# for i in tqdm.tqdm(range(0, 1836168, 100)):\n", "# if not os.path.exists(f\"/home/tony/Data/Hoot/alignments/outputs/batch_{i}.json\"):\n", "# # print(\"Missing,\", i)\n", "# continue\n", "# try:\n", "# with open(f\"/home/tony/Data/Hoot/alignments/outputs/batch_{i}.json\", \"r\") as fp:\n", "# infos = json.load(fp)\n", "# for x_info in infos:\n", "# if x_info[\"id\"] in total_infos:\n", "# break\n", "# total_infos[x_info[\"id\"]] = x_info[\"cer_val\"]\n", "# except:\n", "# # print(\"WTF,\", i)\n", "# pass\n", "\n", "# plt.hist(total_infos.values(), bins=np.linspace(0, 1, 50))\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": 36, "id": "552c1c73", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.570374Z", "start_time": "2024-01-09T16:37:56.506567Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.351050Z", "iopub.status.busy": "2025-01-14T01:36:41.350923Z", "iopub.status.idle": "2025-01-14T01:36:41.385902Z", "shell.execute_reply": "2025-01-14T01:36:41.385519Z", "shell.execute_reply.started": "2025-01-14T01:36:41.351037Z" } }, "outputs": [], "source": [ "# high_cer_infos = {k:v for k, v in total_infos.items() if v < 0.4}\n", "# print(len(high_cer_infos))\n", "\n", "# high_cer_train_info = []\n", "# for info in train_info:\n", "# if info[\"id\"] in high_cer_infos and info[\"duration\"] < 240:\n", "# high_cer_train_info.append(info)\n", "# print(len(high_cer_train_info))\n", "\n", "# print(\n", "# \"total duration khr\", sum(m[\"duration\"] for m in high_cer_train_info) / 3600000,\n", "# min(m[\"duration\"] for m in high_cer_train_info),\n", "# max(m[\"duration\"] for m in high_cer_train_info),\n", "# min(len(m[\"text\"]) for m in high_cer_train_info),\n", "# max(len(m[\"text\"]) for m in high_cer_train_info),\n", "# )\n", "\n", "# print(Counter(m[\"lang\"] for m in high_cer_train_info))\n", "\n", "# cutoff = -2000\n", "# random.shuffle(high_cer_train_info)\n", "# high_cer_train_info, high_cer_test_info = high_cer_train_info[:cutoff], high_cer_train_info[cutoff:]\n", "# print(len(high_cer_train_info), len(high_cer_test_info))" ] }, { "cell_type": "code", "execution_count": 37, "id": "417c0eb3", "metadata": { "ExecuteTime": { "end_time": "2024-01-09T16:37:56.651676Z", "start_time": "2024-01-09T16:37:56.571763Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.386487Z", "iopub.status.busy": "2025-01-14T01:36:41.386360Z", "iopub.status.idle": "2025-01-14T01:36:41.427365Z", "shell.execute_reply": "2025-01-14T01:36:41.426982Z", "shell.execute_reply.started": "2025-01-14T01:36:41.386474Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/multi_large_vocab_cer_04_train.json\", \"w\") as fp:\n", "# for l in high_cer_train_info:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')\n", "# with open(\"/home/tony/Data/Hoot/multi_large_vocab_cer_04_test.json\", \"w\") as fp:\n", "# for l in high_cer_test_info:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')" ] }, { "cell_type": "markdown", "id": "1642e3b5", "metadata": {}, "source": [ "# 2nd round of hoot data prep setup" ] }, { "cell_type": "code", "execution_count": 5, "id": "52f2a9a1", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:43:46.229977Z", "start_time": "2024-01-18T02:43:05.476371Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:41.427915Z", "iopub.status.busy": "2025-01-14T01:36:41.427793Z", "iopub.status.idle": "2025-01-14T01:36:42.450600Z", "shell.execute_reply": "2025-01-14T01:36:42.450006Z", "shell.execute_reply.started": "2025-01-14T01:36:41.427903Z" } }, "outputs": [], "source": [ "with open(\"/home/tony/Data/Hoot/genius_metas.json\", \"r\") as fp:\n", " metas = json.load(fp)\n", "metas_map = {m[\"id\"]: m for m in metas}" ] }, { "cell_type": "code", "execution_count": 6, "id": "274e39d7", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:57:19.260628Z", "start_time": "2024-01-18T02:43:49.821152Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:36:42.451326Z", "iopub.status.busy": "2025-01-14T01:36:42.451172Z", "iopub.status.idle": "2025-01-14T01:47:52.315467Z", "shell.execute_reply": "2025-01-14T01:47:52.314897Z", "shell.execute_reply.started": "2025-01-14T01:36:42.451310Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 18362/18362 [16:06<00:00, 18.99it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "WTF, 1835300\n", "WTF, 1835400\n", "WTF, 1835500\n", "WTF, 1835600\n", "WTF, 1835700\n", "WTF, 1835800\n", "WTF, 1835900\n", "WTF, 1836000\n", "WTF, 1836100\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "from collections import defaultdict\n", "\n", "total_infos_2 = dict()\n", "lang_cers = defaultdict(list)\n", "for i in tqdm.tqdm(range(0, 1836168, 100)):\n", " try:\n", " with open(\n", " f\"/app/suno/data/hoot/alignments/genius_v5_fix/batch_{i}.json\", \"r\"\n", " ) as fp:\n", " infos = json.load(fp)\n", " for x_info in infos:\n", " if x_info[\"id\"] in total_infos_2:\n", " break\n", " if x_info[\"id\"] not in metas_map:\n", " continue\n", " # if x_info[\"id\"] not in metas_map:\n", " # this is somehow already filtered out...?\n", " total_infos_2[x_info[\"id\"]] = x_info[\"cer_val\"]\n", " lang_cers[metas_map[x_info[\"id\"]][\"lang\"]].append(x_info[\"cer_val\"])\n", " except:\n", " print(\"WTF,\", i)" ] }, { "cell_type": "code", "execution_count": 7, "id": "033dc016", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:57:19.317468Z", "start_time": "2024-01-18T02:57:19.266816Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:47:52.316177Z", "iopub.status.busy": "2025-01-14T01:47:52.316027Z", "iopub.status.idle": "2025-01-14T01:47:52.319363Z", "shell.execute_reply": "2025-01-14T01:47:52.318967Z", "shell.execute_reply.started": "2025-01-14T01:47:52.316161Z" } }, "outputs": [ { "data": { "text/plain": [ "(1835017, 153)" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(total_infos_2), len(lang_cers)" ] }, { "cell_type": "code", "execution_count": 8, "id": "ca6fcca0", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T02:57:19.825960Z", "start_time": "2024-01-18T02:57:19.318687Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:47:52.320042Z", "iopub.status.busy": "2025-01-14T01:47:52.319914Z", "iopub.status.idle": "2025-01-14T01:47:52.753995Z", "shell.execute_reply": "2025-01-14T01:47:52.753453Z", "shell.execute_reply.started": "2025-01-14T01:47:52.320028Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "lang n_songs median cer\n", "en 1135948 0.213\n", "es 109115 0.176\n", "pt 78369 0.214\n", "fr 75527 0.262\n", "de 63395 0.216\n", "ru 57300 0.23\n", "pl 55302 0.205\n", "it 46153 0.156\n", "tr 25347 0.251\n", "ko 21881 0.264\n", "ja 18170 0.374\n", "iw 11926 0.3\n", "nl 11031 0.293\n", "el 8427 0.39\n", "ro 8018 0.265\n", "da 7446 0.348\n", "sv 7002 0.313\n", "ar 6498 0.413\n", "hr 5928 0.218\n", "sr 5687 0.231\n", "vi 5078 0.426\n", "no 5076 0.352\n", "fi 4441 0.29\n", "cs 4431 0.249\n", "romanization 4275 0.691\n", "tl 3879 0.268\n", "zh-Hant 3640 0.363\n", "az 3569 0.284\n", "id 3503 0.268\n", "hu 3449 0.332\n", "bs 2976 0.236\n", "fa 2852 0.304\n", "sk 2668 0.279\n", "bg 1709 0.334\n", "sq 1676 0.417\n", "et 1429 0.501\n", "uk 1412 0.338\n", "zh 1348 0.472\n", "ms 1326 0.285\n", "ca 1081 0.335\n", "lv 985 0.419\n", "lt 903 0.457\n", "hi 895 0.431\n", "th 856 0.504\n", "sw 765 0.424\n", "is 665 0.556\n", "ne 627 0.393\n", "eu 619 0.462\n", "zu 594 0.606\n", "sl 557 0.404\n", "af 552 0.352\n", "mk 547 0.386\n", "la 499 0.878\n", "co 493 0.467\n", "ht 465 0.507\n", "gl 464 0.41\n", "sco 425 0.343\n", "xh 294 0.637\n", "eo 269 0.419\n", "bn 247 0.403\n", "ka 246 0.646\n", "ta 225 0.521\n", "si 217 0.984\n", "ln 186 0.63\n", "yo 170 0.566\n", "sn 161 0.554\n", "rw 156 0.57\n", "ga 153 0.846\n", "hy 152 0.63\n", "cy 143 0.798\n", "ceb 138 0.41\n", "my 122 0.98\n", "mn 117 0.672\n", "wo 112 0.742\n", "sa 102 0.806\n", "jw 98 0.526\n", "pa 95 0.465\n", "st 94 0.652\n", "kk 93 0.645\n", "ak 91 0.626\n", "lb 90 0.57\n", "uz 86 0.651\n", "war 83 0.38\n", "mi 82 0.662\n", "be 80 0.45\n", "gd 77 0.848\n", "fo 70 0.615\n", "rm 69 0.611\n", "lg 69 0.583\n", "so 59 0.788\n", "aa 56 0.814\n", "mg 56 0.642\n", "ig 53 0.687\n", "sm 51 0.602\n", "tn 51 0.637\n", "oc 49 0.625\n", "te 46 1.0\n", "haw 44 0.608\n", "ky 41 0.529\n", "ny 40 0.534\n", "tk 37 0.613\n", "ia 37 0.375\n", "ha 37 0.666\n", "br 36 0.874\n", "ts 35 0.629\n", "am 35 0.533\n", "su 35 0.606\n", "ve 33 0.714\n", "qu 29 0.563\n", "ie 29 0.537\n", "ku 27 0.694\n", "mt 26 0.832\n", "om 26 0.783\n", "kha 25 0.824\n", "tlh 24 0.59\n", "tt 24 0.773\n", "ur 24 0.982\n", "ml 24 1.0\n", "bi 23 0.511\n", "bh 23 0.48\n", "crs 23 0.844\n", "gn 22 0.485\n", "kl 22 0.851\n", "to 21 0.786\n", "rn 20 0.6\n", "km 19 0.911\n", "ss 16 0.586\n", "fy 16 0.652\n", "mfe 16 0.65\n", "fj 14 0.799\n", "mr 13 0.47\n", "vo 12 0.669\n", "ba 10 0.766\n", "tg 9 0.576\n", "gv 9 0.743\n", "as 7 0.682\n", "kn 6 0.921\n", "yi 5 0.695\n", "ps 5 0.875\n", "hmn 5 0.892\n", "nso 3 0.616\n", "ay 3 0.639\n", "ks 3 0.977\n", "na 3 0.655\n", "syr 3 0.998\n", "jp 2 0.847\n", "iu 2 0.643\n", "ik 2 0.756\n" ] } ], "source": [ "langs = [(k, len(v)) for k, v in lang_cers.items()]\n", "langs.sort(key=lambda x: (-x[1]))\n", "print(\"lang n_songs median cer\")\n", "cer_percentage_threshold = 0.5\n", "langs_cer_cut = {}\n", "for k, l_v in langs:\n", " v = lang_cers[k]\n", " if l_v > 1:\n", " print(k, l_v, round(np.quantile(v, cer_percentage_threshold), 3))\n", " langs_cer_cut[k] = round(np.quantile(v, cer_percentage_threshold), 3)" ] }, { "cell_type": "code", "execution_count": 9, "id": "bf85525f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "lang n_songs median cer\n", "en 1135948 0.213\n", "es 109115 0.176\n", "pt 78369 0.214\n", "fr 75527 0.262\n", "de 63395 0.216\n", "ru 57300 0.23\n", "pl 55302 0.205\n", "it 46153 0.156\n", "tr 25347 0.251\n", "ko 21881 0.264\n", "ja 18170 0.374\n", "iw 11926 0.3\n", "nl 11031 0.293\n", "el 8427 0.39\n", "ro 8018 0.265\n", "da 7446 0.348\n", "sv 7002 0.313\n", "ar 6498 0.413\n", "hr 5928 0.218\n", "sr 5687 0.231\n", "vi 5078 0.426\n", "no 5076 0.352\n", "fi 4441 0.29\n", "cs 4431 0.249\n", "romanization 4275 0.691\n", "tl 3879 0.268\n", "zh-Hant 3640 0.363\n", "az 3569 0.284\n", "id 3503 0.268\n", "hu 3449 0.332\n", "bs 2976 0.236\n", "fa 2852 0.304\n", "sk 2668 0.279\n", "bg 1709 0.334\n", "sq 1676 0.417\n", "et 1429 0.501\n", "uk 1412 0.338\n", "zh 1348 0.472\n", "ms 1326 0.285\n", "ca 1081 0.335\n", "lv 985 0.419\n", "lt 903 0.457\n", "hi 895 0.431\n", "th 856 0.504\n", "sw 765 0.424\n", "is 665 0.556\n", "ne 627 0.393\n", "eu 619 0.462\n", "zu 594 0.606\n", "sl 557 0.404\n", "af 552 0.352\n", "mk 547 0.386\n", "la 499 0.878\n", "co 493 0.467\n", "ht 465 0.507\n", "gl 464 0.41\n", "sco 425 0.343\n", "xh 294 0.637\n", "eo 269 0.419\n", "bn 247 0.403\n", "ka 246 0.646\n", "ta 225 0.521\n", "si 217 0.984\n", "ln 186 0.63\n", "yo 170 0.566\n", "sn 161 0.554\n", "rw 156 0.57\n", "ga 153 0.846\n", "hy 152 0.63\n", "cy 143 0.798\n", "ceb 138 0.41\n", "my 122 0.98\n", "mn 117 0.672\n", "wo 112 0.742\n", "sa 102 0.806\n", "jw 98 0.526\n", "pa 95 0.465\n", "st 94 0.652\n", "kk 93 0.645\n", "ak 91 0.626\n", "lb 90 0.57\n", "uz 86 0.651\n", "war 83 0.38\n", "mi 82 0.662\n", "be 80 0.45\n", "gd 77 0.848\n", "fo 70 0.615\n", "rm 69 0.611\n", "lg 69 0.583\n", "so 59 0.788\n", "aa 56 0.814\n", "mg 56 0.642\n", "ig 53 0.687\n", "sm 51 0.602\n", "tn 51 0.637\n", "oc 49 0.625\n", "te 46 1.0\n", "haw 44 0.608\n", "ky 41 0.529\n", "ny 40 0.534\n", "tk 37 0.613\n", "ia 37 0.375\n", "ha 37 0.666\n", "br 36 0.874\n", "ts 35 0.629\n", "am 35 0.533\n", "su 35 0.606\n", "ve 33 0.714\n", "qu 29 0.563\n", "ie 29 0.537\n", "ku 27 0.694\n", "mt 26 0.832\n", "om 26 0.783\n", "kha 25 0.824\n", "tlh 24 0.59\n", "tt 24 0.773\n", "ur 24 0.982\n", "ml 24 1.0\n", "bi 23 0.511\n", "bh 23 0.48\n", "crs 23 0.844\n", "gn 22 0.485\n", "kl 22 0.851\n", "to 21 0.786\n", "rn 20 0.6\n", "km 19 0.911\n", "ss 16 0.586\n", "fy 16 0.652\n", "mfe 16 0.65\n", "fj 14 0.799\n", "mr 13 0.47\n", "vo 12 0.669\n", "ba 10 0.766\n", "tg 9 0.576\n", "gv 9 0.743\n", "as 7 0.682\n", "kn 6 0.921\n", "yi 5 0.695\n", "ps 5 0.875\n", "hmn 5 0.892\n", "nso 3 0.616\n", "ay 3 0.639\n", "ks 3 0.977\n", "na 3 0.655\n", "syr 3 0.998\n", "jp 2 0.847\n", "iu 2 0.643\n", "ik 2 0.756\n" ] } ], "source": [ "langs = [(k, len(v)) for k, v in lang_cers.items()]\n", "langs.sort(key=lambda x: (-x[1]))\n", "print(\"lang n_songs median cer\")\n", "cer_percentage_threshold = 0.5\n", "langs_cer_cut = {}\n", "for k, l_v in langs:\n", " v = lang_cers[k]\n", " if l_v > 1:\n", " print(k, l_v, round(np.quantile(v, cer_percentage_threshold), 3))\n", " langs_cer_cut[k] = round(np.quantile(v, cer_percentage_threshold), 3)" ] }, { "cell_type": "code", "execution_count": 10, "id": "768780a2", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T03:01:31.958520Z", "start_time": "2024-01-18T03:01:04.057834Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:47:52.754672Z", "iopub.status.busy": "2025-01-14T01:47:52.754529Z", "iopub.status.idle": "2025-01-14T01:48:24.789903Z", "shell.execute_reply": "2025-01-14T01:48:24.789336Z", "shell.execute_reply.started": "2025-01-14T01:47:52.754657Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 1835017/1835017 [00:29<00:00, 62411.28it/s]\n" ] } ], "source": [ "rates = []\n", "cers = []\n", "# youtube_views_log = []\n", "for k in tqdm.tqdm(total_infos_2):\n", " duration_s = metas_map[k][\"duration_s\"]\n", " n_lyrics = len(metas_map[k][\"lyrics\"].split())\n", " # print(n_lyrics / duration_s)\n", " cers.append(total_infos_2[k])\n", " rates.append(n_lyrics / duration_s)\n", " # youtube_views_log.append(np.log10(metas_map[k][\"youtube_views\"]))" ] }, { "cell_type": "code", "execution_count": 11, "id": "ddca3db4", "metadata": { "ExecuteTime": { "end_time": "2024-01-12T22:41:25.073971Z", "start_time": "2024-01-12T22:41:25.072066Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:24.790628Z", "iopub.status.busy": "2025-01-14T01:48:24.790474Z", "iopub.status.idle": "2025-01-14T01:48:24.792987Z", "shell.execute_reply": "2025-01-14T01:48:24.792598Z", "shell.execute_reply.started": "2025-01-14T01:48:24.790612Z" } }, "outputs": [], "source": [ "# plt.clf()\n", "# h = plt.hist2d(rates, youtube_views_log, bins=200)\n", "# plt.colorbar(h[3])\n", "# # plt.ylim(0, 4)\n", "# plt.xlim(0, 5)\n", "# plt.xlabel(\"n_words / sec\")\n", "# plt.ylabel(\"youtube_views, log10\")\n", "# plt.title(\"youtube views vs. num words per sec\")\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": 12, "id": "2e5eb800", "metadata": { "ExecuteTime": { "end_time": "2024-01-12T22:41:25.152543Z", "start_time": "2024-01-12T22:41:25.074881Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:24.793569Z", "iopub.status.busy": "2025-01-14T01:48:24.793443Z", "iopub.status.idle": "2025-01-14T01:48:24.832093Z", "shell.execute_reply": "2025-01-14T01:48:24.831718Z", "shell.execute_reply.started": "2025-01-14T01:48:24.793556Z" } }, "outputs": [], "source": [ "# plt.hist([r for r, v in zip(rates, youtube_views_log) if v < 6], bins=100, density=True, label=\"views < 1m\", alpha=0.5)\n", "# plt.hist([r for r, v in zip(rates, youtube_views_log) if v >= 6], bins=100, density=True, label=\"views >= 1m\", alpha=0.5)\n", "# plt.legend()\n", "# plt.xlabel(\"word rate / sec\")\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": 13, "id": "b19130f2", "metadata": { "ExecuteTime": { "end_time": "2024-01-12T22:41:25.212542Z", "start_time": "2024-01-12T22:41:25.153867Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:24.832726Z", "iopub.status.busy": "2025-01-14T01:48:24.832599Z", "iopub.status.idle": "2025-01-14T01:48:24.868630Z", "shell.execute_reply": "2025-01-14T01:48:24.868258Z", "shell.execute_reply.started": "2025-01-14T01:48:24.832713Z" } }, "outputs": [], "source": [ "# plt.clf()\n", "# h = plt.hist2d(cers, youtube_views_log, bins=100)\n", "# plt.colorbar(h[3])\n", "# # plt.ylim(0, 4)\n", "# # plt.xlim(0, 0.99)\n", "# plt.xlabel(\"cer\")\n", "# plt.ylabel(\"youtube_views\")\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": 14, "id": "416b615d", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T03:01:32.652951Z", "start_time": "2024-01-18T03:01:31.959970Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:24.869209Z", "iopub.status.busy": "2025-01-14T01:48:24.869078Z", "iopub.status.idle": "2025-01-14T01:48:25.558515Z", "shell.execute_reply": "2025-01-14T01:48:25.558027Z", "shell.execute_reply.started": "2025-01-14T01:48:24.869196Z" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAG2CAYAAABcYt1RAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABoVUlEQVR4nO3de1wU5f4H8M/swgIqu4jKTfFSdLzl3dLFUksSlVNa1snLEfN6LKyUjhodU483zDK1NMnMqF9yTDtmHTUVMfKoeImkvGVaJHpiMStZMbntzu8PY3OehV2GXWCFz/v12tfLmecy35kd5MszM89IsizLICIiIvJgmtoOgIiIiMgZJixERETk8ZiwEBERkcdjwkJEREQejwkLEREReTwmLEREROTxmLAQERGRx2PCQkRERB6PCQsRERF5PCYsRERE5PE8KmFZsmQJJEnCtGnTHNbbvHkz2rVrB19fX3Tq1Ak7duyomQCJiIioVnhMwnL06FG8+eab6Ny5s8N6Bw8exMiRIzFhwgQcO3YMw4YNw7Bhw3DixIkaipSIiIhqmuQJLz8sKChA9+7d8cYbb2DhwoXo2rUrVqxYUW7dxx9/HNeuXcO2bdts63r37o2uXbsiKSmphiImIiKimuRV2wEAQFxcHGJiYhAVFYWFCxc6rJuRkYH4+HjFuujoaGzdurXc+kVFRSgqKrItW61W/PLLL2jSpAkkSXI5diIiqrtkWcbVq1cRFhYGjab6LkoUFhaiuLjY5X50Oh18fX3dEJHnqfWEZePGjfjyyy9x9OjRStU3mUwIDg5WrAsODobJZCq3fmJiIv75z3+6HCcREdVfFy5cQIsWLaql78LCQrRp1QimSxaX+woJCUF2dnadTFpqNWG5cOECnn32WaSmplbbwU1ISFCMyOTn56Nly5a4B0PgBe9q2aZqkkZYlByW2zXXCuVie9XxCO3dfdXQKvQnxqu2/GZujlUWtyVb3dtfbXNxf4jqulKUYD92wN/fv9q2UVxcDNMlC85ntobev+qjOOarVrTq8QOKi4uZsLhbZmYmLl26hO7du9vWWSwW7Nu3D6tWrUJRURG0Wq2iTUhICPLy8hTr8vLyEBISUu42fHx84OPjY7feC97wkjw0YRETBmcJi6QVV7gYj9jezb9kJaE/u/1VWa7g3l/Ast22XExYHMZeG5iwEDn0+49sTdxC0MhfQiP/qm/Hirp9m0OtPiU0YMAAHD9+HFlZWbZPz549MXr0aGRlZdklKwBgNBqRlpamWJeamgqj0VhTYRMREbmdRba6/KnLanWExd/fH3feeadiXcOGDdGkSRPb+tjYWDRv3hyJiYkAgGeffRb9+vXDsmXLEBMTg40bN+KLL77A2rVrazx+IiIid7FChtWFEW1X2t4KPGYelork5OQgNzfXthwZGYmUlBSsXbsWXbp0wYcffoitW7faJT5ERERUd9T6U0Ki9PR0h8sA8Nhjj+Gxxx6rmYCIiIhqgBVWl+4qc6215/O4hIWIiKg+ssgyLC486ehK21sBE5aa4OSxZUm8uVh8KsjuMWcn7UXiZEfO7nZX8xgxYP8osauPRVuFvxLE9jftrizWFa9yqt62sr6kEft3fKydPbYsPtDl7seKVT827eQJtHI24N72RJ5IcV5r3P6gJFUNExYiIiIPwJtuHWPCQkRE5AGskGFhwlIhj39KiIiIiIgjLERERB6Al4QcY8JCRETkAfiUkGO8JEREREQejyMsREREHsAK115HWtcnDWDCQkRE5AEsLj4l5ErbWwETFiIiIg9gkW98XGlfl/EeFiIiIvJ4HGEhIiLyALyHxTEmLERERB7ACgkWOHnXm5P2dRkvCREREZHH4wgLERGRB7DKdi+MV92+LmPCQkRE5AEsLl4ScqXtrYCXhIiIiMjjcYSFiIjIA3CExTEmLNVB0giLypNI0mqV9b2UX4Pk7a1cFsoh9AeNxvGypPIkFus7e6GWWG61Oi4X+xfrixxdmLVYHG/Lri/HscmSWO7k2AmxScJXC9nxvslWFwc5hf7Fc815c5UXvYVz227/xHJX299c7qwt0S3OKkuwOvs/x0n7uoyXhIiIiMjjcYSFiIjIA/CSkGNMWIiIiDyABRpYXLjwYXFe5ZbGhIWIiMgDyC7ewyLzHhYiIiKi2sURFiIiIg/Ae1gc4wgLERGRB7DIGpc/qrZnseDFF19EmzZt4Ofnh9tvvx0LFiyAfNN0D7IsY86cOQgNDYWfnx+ioqJw9uxZRT+//PILRo8eDb1ej4CAAEyYMAEFBQWKOl9//TXuvfde+Pr6Ijw8HEuXLlV9fJiwEBER1UMvvfQS1qxZg1WrVuH06dN46aWXsHTpUrz++uu2OkuXLsVrr72GpKQkHD58GA0bNkR0dDQKCwttdUaPHo2TJ08iNTUV27Ztw759+zB58mRbudlsxsCBA9GqVStkZmbi5Zdfxrx587B27VpV8fKSEBERkQewQoLVhXEEK9RNBHnw4EEMHToUMTExAIDWrVvjX//6F44cOQLgxujKihUrMHv2bAwdOhQA8N577yE4OBhbt27FiBEjcPr0aezcuRNHjx5Fz549AQCvv/46hgwZgldeeQVhYWHYsGEDiouLsX79euh0OnTs2BFZWVl49dVXFYmNMxxhISIi8gBl97C48gFujGjc/CkqKip3e5GRkUhLS8O3334LAPjqq6+wf/9+DB48GACQnZ0Nk8mEqKgoWxuDwYBevXohIyMDAJCRkYGAgABbsgIAUVFR0Gg0OHz4sK1O3759odPpbHWio6Nx5swZ/Prrr5U+PkxYiIiI6pDw8HAYDAbbJzExsdx6zz//PEaMGIF27drB29sb3bp1w7Rp0zB69GgAgMlkAgAEBwcr2gUHB9vKTCYTgoKCFOVeXl4IDAxU1Cmvj5u3URm8JEREROQBqnLjrLL9jUtCFy5cgF6vt6338fEpt/6mTZuwYcMGpKSk2C7TTJs2DWFhYRg7dmyV46guTFiIiIg8wI17WFx4+eHvbfV6vSJhqciMGTNsoywA0KlTJ5w/fx6JiYkYO3YsQkJCAAB5eXkIDQ21tcvLy0PXrl0BACEhIbh06ZKi39LSUvzyyy+29iEhIcjLy1PUKVsuq1MZvCRERERUD/3222/QaJRpgFarhfX3t9q3adMGISEhSEtLs5WbzWYcPnwYRqMRAGA0GnHlyhVkZmba6uzduxdWqxW9evWy1dm3bx9KSkpsdVJTU9G2bVs0bty40vEyYSEiIvIA1t/fJVTVj9onjB588EEsWrQI27dvxw8//ICPPvoIr776Kh5++GEAgCRJmDZtGhYuXIhPPvkEx48fR2xsLMLCwjBs2DAAQPv27TFo0CBMmjQJR44cwYEDBzB16lSMGDECYWFhAIBRo0ZBp9NhwoQJOHnyJD744AOsXLkS8fHxquKt1YRlzZo16Ny5s234ymg04tNPP62wfnJyMiRJUnx8fX1rMGIiIqLqUdMTx73++ut49NFH8dRTT6F9+/b4+9//jr/97W9YsGCBrc7MmTPx9NNPY/LkybjrrrtQUFCAnTt3Kn73btiwAe3atcOAAQMwZMgQ3HPPPYo5VgwGA3bv3o3s7Gz06NEDzz33HObMmaPqkWYAkOSbp7SrYf/5z3+g1Wpxxx13QJZlvPvuu3j55Zdx7NgxdOzY0a5+cnIynn32WZw5c8a2TpIku7uPHTGbzTAYDOiPofCSvN2yH3YkjbCovCYpabXK+l7KW4kkb2VcklAOoT8IQ3p2y5LKa6JifWeniFj++3BiheVi/2J9kbXi7csW4f2kzmJ1EpvsLHYVsf3eoZNiF3/8nPTvvLmbty+p/BvIWfubyx2VEbnTTedaqVyCdPkj5OfnV+q+kKoo+72UknUnGvhrnTeowG9XLRjV9US1xlqbavWm2wcffFCxvGjRIqxZswaHDh0qN2EBbiQoam7SqRHOEhQvIQHxFg67TiwXEikxYfHRKZfFBMBLecLLYgIjkMRfynYJiJNykVheWuq4vsgi/CJykNDYxe4sgRH2RUx47FI7MXYnyZxdAgXH//lIknhsVSY4Ts49u/pC/2J9Z9uz719dgqK6vaMEyFlypDaZcpYAudqeiFziMfewWCwWbNy4EdeuXbPdzFOegoICtGrVCuHh4Rg6dChOnjzpsN+ioiK7SXSIiIg8jUWWXP7UZbWesBw/fhyNGjWCj48PpkyZgo8++ggdOnQot27btm2xfv16fPzxx3j//fdhtVoRGRmJixcvVth/YmKiYgKd8PDw6toVIiKiKnPlhtuyT11W63vXtm1bZGVl4fDhw3jyyScxduxYnDp1qty6RqMRsbGx6Nq1K/r164ctW7agWbNmePPNNyvsPyEhAfn5+bbPhQsXqmtXiIiIqJrU+sRxOp0OERERAIAePXrg6NGjWLlypcMkpEzZVMLnzp2rsI6Pj0+Fs/wRERF5CqusgdWFmW6ttfcMTY2o9REWkdVqrfBFTSKLxYLjx48rZuAjIiK6FfGSkGO1OsKSkJCAwYMHo2XLlrh69SpSUlKQnp6OXbt2AQBiY2PRvHlz24ub5s+fj969eyMiIgJXrlzByy+/jPPnz2PixIm1uRtERERUzWo1Ybl06RJiY2ORm5sLg8GAzp07Y9euXXjggQcAADk5OYppg3/99VdMmjQJJpMJjRs3Ro8ePXDw4MEKb9IlIiK6VVgBl570qesP1tdqwvL22287LE9PT1csL1++HMuXL6/GiIiIiGqHtQrT64vt67K6vXdERERUJ9T6U0JERESEKr0PSGxflzFhISIi8gBWSLDavyBEVfu6jAkLERGRB+AIi2N1e++IiIioTuAICxERkQdwdfI3ThxHRERE1c4qS7C6Mg8L39ZMREREVLs4wkJEROQBrC5eEqrrE8cxYXEDSaMchpO8vJXLOmHZV3h7tPg26YZ+ikXZR9le9lKelLK31mF8suRkmFAsFl74KVmVEz5LpcIE0E7eECpZhHKLRbksdCeVCuWOCLGhtFRYFvoSYpXEWARO330q9qcVvgsxPqG+LImTaQv/4Yj9S+rexipplP3LVnX/oYnt7cs9eAhacvE/b1fbq+1fruGJ1dVuv7bjrQdcf1tz3U5Y6vbeERERUZ3AERYiIiIPYIEEiwuTv7nS9lbAhIWIiMgD8JKQY3V774iIiKhO4AgLERGRB7DAtcs6Kh5XuCUxYSEiIvIAvCTkGBMWIiIiD8CXHzpWt/eOiIiI6gSOsBAREXkAGRKsLtzDIvOxZiIiIqpuvCTkWN3eOyIiIqoTOMJCRETkAayyBKtc9cs6rrS9FXCEhYiIyANYfn9bsysfNVq3bg1Jkuw+cXFxAIDCwkLExcWhSZMmaNSoEYYPH468vDxFHzk5OYiJiUGDBg0QFBSEGTNmoFR4CW16ejq6d+8OHx8fREREIDk5uUrHhwkLERFRPXT06FHk5ubaPqmpqQCAxx57DAAwffp0/Oc//8HmzZvx+eef48cff8Qjjzxia2+xWBATE4Pi4mIcPHgQ7777LpKTkzFnzhxbnezsbMTExOC+++5DVlYWpk2bhokTJ2LXrl2q4+UlISIiIg9Q05eEmjVrplhesmQJbr/9dvTr1w/5+fl4++23kZKSgvvvvx8A8M4776B9+/Y4dOgQevfujd27d+PUqVPYs2cPgoOD0bVrVyxYsACzZs3CvHnzoNPpkJSUhDZt2mDZsmUAgPbt22P//v1Yvnw5oqOjVcXLERYiIiIPYIXG5Q8AmM1mxaeoqMjptouLi/H+++9j/PjxkCQJmZmZKCkpQVRUlK1Ou3bt0LJlS2RkZAAAMjIy0KlTJwQHB9vqREdHw2w24+TJk7Y6N/dRVqesDzWYsBAREdUh4eHhMBgMtk9iYqLTNlu3bsWVK1fwxBNPAABMJhN0Oh0CAgIU9YKDg2EymWx1bk5WysrLyhzVMZvNuH79uqr94iUhIiIiD2CRJVhcuCRU1vbChQvQ6/W29T4+Pk7bvv322xg8eDDCwsKqvP3qxoSlKiTlwJTk5a1c9lYeVqmBn7J9wwaKRbmhr3JZp2xf2lDZv8VPWW43V5BGecJbvB3/AGiLrA7LJYus7L5ErqBmWX1lf5JVbO94exC2J1mtFZZLxcq70SUvrVBXaCv2JcQGWYhdIxxcsb0stBe2J8vOjq2T+IT+ZcnJ9kVW4VwVzwWLs/e7Ko+nLB4vZ5ztv8bxueloe2Jb1bHZd+jZ7Z0cS5f7Vxt/dcdbi24+tyRZqrHXILvrHha9Xq9IWJw5f/489uzZgy1bttjWhYSEoLi4GFeuXFGMsuTl5SEkJMRW58iRI4q+yp4iurmO+GRRXl4e9Ho9/PyE341O8JIQERGRB5B/f1tzVT9yFWe6feeddxAUFISYmBjbuh49esDb2xtpaWm2dWfOnEFOTg6MRiMAwGg04vjx47h06ZKtTmpqKvR6PTp06GCrc3MfZXXK+lBD1d6VlJTAy8sLJ06cUL0hIiIi8ixWqxXvvPMOxo4dCy+vP0bvDQYDJkyYgPj4eHz22WfIzMzEuHHjYDQa0bt3bwDAwIED0aFDB4wZMwZfffUVdu3ahdmzZyMuLs52GWrKlCn4/vvvMXPmTHzzzTd44403sGnTJkyfPl11rKouCXl7e6Nly5awOB1GJiIiIjUskGBx4QWGVWm7Z88e5OTkYPz48XZly5cvh0ajwfDhw1FUVITo6Gi88cYbtnKtVott27bhySefhNFoRMOGDTF27FjMnz/fVqdNmzbYvn07pk+fjpUrV6JFixZYt26d6keaAUCSZWcXwZXefvttbNmyBf/3f/+HwMBA1RusbWazGQaDAf0xFF6St/MG5RGu3Wp0OmWxeA+LcM8K72GpxntY7Oq6dg8Liksct3fxHhan8Yn3sDjbvkjcP/GeEZV/fLj7HhZXtuf2e1g8XXXfw+Jut/I9LNo/7t0qlUvwmWUL8vPzVd0XokbZ76Vx6X+BrpHOeYMKFBcU453+m6o11tqk+qbbVatW4dy5cwgLC0OrVq3QsGFDRfmXX37ptuCIiIiIgCokLMOGDauGMIiIiOq3sptnXWlfl6lOWObOneu2ja9ZswZr1qzBDz/8AADo2LEj5syZg8GDB1fYZvPmzXjxxRfxww8/4I477sBLL72EIUOGuC0mIiKi2mCFBKsL97C40vZWUKV07MqVK1i3bh0SEhLwyy+/ALhxKeh///ufqn5atGiBJUuWIDMzE1988QXuv/9+DB061Dalr+jgwYMYOXIkJkyYgGPHjmHYsGEYNmwYn1oiIiKq41SPsHz99deIioqCwWDADz/8gEmTJiEwMBBbtmxBTk4O3nvvvUr39eCDDyqWFy1ahDVr1uDQoUPo2LGjXf2VK1di0KBBmDFjBgBgwYIFSE1NxapVq5CUlKR2V4iIiDyGu2a6ratUj7DEx8fjiSeewNmzZ+Hr+8fTLUOGDMG+ffuqHIjFYsHGjRtx7dq1CieUcedLlIiIiDyJK5PGuXr/y61A9QjL0aNH8eabb9qtb968ue1lR2ocP34cRqMRhYWFaNSoET766CPbDHmiil6i5Gi7RUVFijdVms1m1TESERFR7VKdjvn4+JT7S//bb79Fs2bNVAfQtm1bZGVl4fDhw3jyyScxduxYnDp1SnU/FUlMTFS8tTI8PNxtfRMREbmLFZLtfUJV+vCmW6WHHnoI8+fPR0nJjQm1JElCTk4OZs2aheHDh6sOQKfTISIiAj169EBiYiK6dOmClStXllu3opcolb1kqTwJCQnIz8+3fS5cuKA6RiIiouom//6UUFU/MhMWpWXLlqGgoABBQUG4fv06+vXrh4iICPj7+2PRokUuB2S1WhWXcG5WlZco+fj42N5cqfYNlkRERDXFpdEVF9/0fCtQfQ+LwWBAamoqDhw4gK+++goFBQXo3r273c2wlZGQkIDBgwejZcuWuHr1KlJSUpCeno5du3YBAGJjY9G8eXMkJiYCAJ599ln069cPy5YtQ0xMDDZu3IgvvvgCa9euVb1tIiIiunWoTljee+89PP744+jTpw/69OljW19cXIyNGzciNja20n1dunQJsbGxyM3NhcFgQOfOnbFr1y488MADAICcnBxoNH8MAkVGRiIlJQWzZ8/GCy+8gDvuuANbt27FnXfeqXY3iIiIPApnunVM9csPtVotcnNzERQUpFj/888/IygoyOPf5MyXH9rjyw9vwpcfOsSXH9YivvywxtTWyw+H7h4P74ZVf/lhybVifDxwPV9+WEaWZUiS/S/AixcvwmAwuCUojyP8oN98MgPlJCh+ygQEfn6KRbmRcrlU76NYLvEXEhRf5fYtOuXxL/VVLkvi72Dh6xLLxYxHLBcTFGcJi6ZUTDiE7QvlzhMmZbmi/wbKY6UpVv4ClkqEX8jCL3ixXNYKx/KqsK/iuS8mDKXC9kuVCZXYXoaQEGmFXypi8iece3YJh9i/k+3b/RJzktCISYLILmkQf3acJRnCLzln23MptltdTScgzogJirP4PDmhUcTuYce5Hqt0wtKtWzdIkgRJkjBgwAB4ef3R1GKxIDs7G4MGDaqWIImIiOo6vkvIsUonLGVvac7KykJ0dDQaNWpkK9PpdGjdunWVHmsmIiIiuPykD58S+l3ZW5pbt26NESNGwMfHx0kLIiIiIvdQfXHu/vvvx08//WRbPnLkCKZNm8ZHi4mIiFzAeVgcU52wjBo1Cp999hmAG+/2iYqKwpEjR/CPf/wD8+fPd3uARERE9QETFsdUJywnTpzA3XffDQDYtGkTOnXqhIMHD2LDhg1ITk52d3xERERE6h9rLikpsd2/smfPHjz00EMAgHbt2iE3N9e90REREdUTvOnWMdUjLB07dkRSUhL++9//IjU11fYo848//ogmTZq4PUAiIqL6QAZcfPlh3aY6YXnppZfw5ptvon///hg5ciS6dOkCAPjkk09sl4qIiIhIHd7D4pjqS0L9+/fH5cuXYTab0bhxY9v6yZMno0GDBg5aEhEREVWN6oQFuPE+oZuTFeDG/CxERERUNbyHxbEqJSwffvghNm3ahJycHBQXFyvKvvzyS7cERkREVJ8wYXFM9T0sr732GsaNG4fg4GAcO3YMd999N5o0aYLvv/8egwcPro4YiYiIqJ5TnbC88cYbWLt2LV5//XXodDrMnDkTqampeOaZZ5Cfn18dMRIREdV5vOnWMdUJS05ODiIjIwEAfn5+uHr1KgBgzJgx+Ne//uXe6IiIiOoJWZZc/tRlqhOWkJAQ/PLLLwCAli1b4tChQwCA7OxsyHJdfwqciIiIakOVXn74ySefAADGjRuH6dOn44EHHsDjjz+Ohx9+2O0BEhER1QeuTBpX9lHrf//7H/7617+iSZMm8PPzQ6dOnfDFF1/YymVZxpw5cxAaGgo/Pz9ERUXh7Nmzij5++eUXjB49Gnq9HgEBAZgwYQIKCgoUdb7++mvce++98PX1RXh4OJYuXao6VtVPCa1duxZWqxUAEBcXhyZNmuDgwYN46KGH8Le//U11ALcCSaM8CSStVlnBSziMfr6KRbmRcrkkQLlcFKhTLJc2UOaRJX7K7VuV1VHqIwSsMg3VlCiXJatQXirsv0VZri1WjqyJ7SWLslzcnsVPGbBdfYtwvG8q15YoNyZrhR9YP+V3oylUBi95CQdLI8RSKu6M0H+xePCEcm/h3BBGIZ3+96IVtm9Vtpe1Gofldv2Lo6DCuSxbhC9XPLedjKJKklAui8dPOL4a4fuzqv4bqmLCtsWfY+fNhWPp5vZiuVOSG49NdVAbn1jf0bni5DyyK3fRzd+VVIOXWWr6KaFff/0Vffr0wX333YdPP/0UzZo1w9mzZxXTlixduhSvvfYa3n33XbRp0wYvvvgioqOjcerUKfj63vhdNnr0aOTm5iI1NRUlJSUYN24cJk+ejJSUFACA2WzGwIEDERUVhaSkJBw/fhzjx49HQEAAJk+eXOl4K5WwPPLII0hOToZer8f777+Pxx9/HF6//0c2YsQIjBgxotIbJCIiotr30ksvITw8HO+8845tXZs2bWz/lmUZK1aswOzZszF06FAAwHvvvYfg4GBs3boVI0aMwOnTp7Fz504cPXoUPXv2BAC8/vrrGDJkCF555RWEhYVhw4YNKC4uxvr166HT6dCxY0dkZWXh1VdfVZWwVCol3rZtG65duwbgxmUgPg1ERETkXu666dZsNis+RUVF5W7vk08+Qc+ePfHYY48hKCgI3bp1w1tvvWUrz87OhslkQlRUlG2dwWBAr169kJGRAQDIyMhAQECALVkBgKioKGg0Ghw+fNhWp2/fvtDp/rg8EB0djTNnzuDXX3+t9PGp1AhLu3btkJCQgPvuuw+yLGPTpk3Q6/Xl1o2Nja30xomIiOgGd10SCg8PV6yfO3cu5s2bZ1f/+++/x5o1axAfH48XXngBR48exTPPPAOdToexY8fCZDIBAIKDgxXtgoODbWUmkwlBQUGKci8vLwQGBirq3Dxyc3OfJpPJbub8ilQqYUlKSkJ8fDy2b98OSZIwe/ZsSOK1egCSJDFhISIiqgJXH00ua3vhwgXFoIKPj3ij4w1WqxU9e/bE4sWLAQDdunXDiRMnkJSUhLFjx1Y5jupSqYQlMjLS9viyRqPBt99+a5dRERERUe3T6/UVXgW5WWhoKDp06KBY1759e/z73/8GcGMaEwDIy8tDaGiorU5eXh66du1qq3Pp0iVFH6Wlpfjll19s7UNCQpCXl6eoU7ZcVqcyVN92np2djWbNmqltRkRERA7ILs5yq3Z0pk+fPjhz5oxi3bfffotWrVoBuHEDbkhICNLS0mzlZrMZhw8fhtFoBAAYjUZcuXIFmZmZtjp79+6F1WpFr169bHX27duHkpI/nqpMTU1F27ZtK305CKhCwtKqVatyLwcRERFR1cm4MXNAlT8qtzd9+nQcOnQIixcvxrlz55CSkoK1a9ciLi4OwI3bPKZNm4aFCxfik08+wfHjxxEbG4uwsDAMGzYMwI0RmUGDBmHSpEk4cuQIDhw4gKlTp2LEiBEICwsDAIwaNQo6nQ4TJkzAyZMn8cEHH2DlypWIj49XFW+V3tZMREREt7a77roLH330ERISEjB//ny0adMGK1aswOjRo211Zs6ciWvXrmHy5Mm4cuUK7rnnHuzcudM2BwsAbNiwAVOnTsWAAQOg0WgwfPhwvPbaa7Zyg8GA3bt3Iy4uDj169EDTpk0xZ84cVY80A4Ak17P59M1mMwwGA/pjKLwk70q1ESeKk7yU7SRhojhJ30ixLOsbKpZLAhsolj1/4jih3M0Tx2mEcvuJ45T1HU0cpykSKwvbEieOEydKEyaO0xQUKsudTRwn9icSf9yKih3XtzqZOE6cMEucjEyMR9y+sD92E8eJnP13IW5f5YRfqidTc8TFycQ8buK4us6DJo7T3PT4balcgr3Fm5Gfn1+p+0Kqouz3UpcPn4O2Qfk3yFaG5bcifPXosmqNtTap+tUmyzJycnJQWFjovDIRERFVGl9+6JjqhCUiIgIXLlyorniIiIiI7KhKWDQaDe644w78/PPP1RUPERFRveTKE0KuTjp3K1D9lNCSJUswY8YMnDhxojriISIiqpdcekJIdn6L2a1O9VNCsbGx+O2339ClSxfodDr4+fkpyn/55Re3BUdEREQEVCFhWbFiRTWEQUREVL+5a2r+ukp1wuKJ7xcgIiK61TFhcUz1PSwA8N1332H27NkYOXKk7R0Cn376KU6ePOnW4IiIiOoL3nTrmOoRls8//xyDBw9Gnz59sG/fPixatAhBQUH46quv8Pbbb+PDDz+sdF+JiYnYsmULvvnmG/j5+SEyMhIvvfQS2rZtW2Gb5ORkjBs3TrHOx8eneueGEScp0iqXJZ0wAZ0wkZylkXK5tIHysJcKE8MVN1IulyjnnYNV2JxFeRsRZCFccaI3cf5mSVx2MlGcl3CoS32V8WqFudAki7JcEu4M05Qoy8WJ4jTCxHTa4j8miZK9hUn9fJXLmmJhYjLhu9OUaIVyIdYSZTCyMNGapBN+hIT6kpOJ3+yWxcnJLEJ74dhJpcKXJUxyKF8Xviwn/UvCxHl2E9cJ25fFcrs/gZTx2E28Vypu34W/LsVjKf7cqiRpnExWZh+AUN3xvtS1ieRcnVjP4fF18btU3f7mnyPZyWSKVGNUnwXPP/88Fi5ciNTUVOhumg3w/vvvt73RubI+//xzxMXF4dChQ0hNTUVJSQkGDhyIa9euOWyn1+uRm5tr+5w/f17tbhAREXkUPiXkmOoRluPHjyMlJcVufVBQEC5fvqyqr507dyqWk5OTERQUhMzMTPTt27fCdpIkqXolNRERkae7kXS4cg+LG4PxQKpHWAICApCbm2u3/tixY2jevLlLweTn5wMAAgMDHdYrKChAq1atEB4ejqFDhzq8d6aoqAhms1nxISIioluL6oRlxIgRmDVrFkwmEyRJgtVqxYEDB/D3v/8dsbGxVQ7EarVi2rRp6NOnD+68884K67Vt2xbr16/Hxx9/jPfffx9WqxWRkZG4ePFiufUTExNhMBhsn/Dw8CrHSEREVF34LiHHVCcsixcvRrt27RAeHo6CggJ06NABffv2RWRkJGbPnl3lQOLi4nDixAls3LjRYT2j0YjY2Fh07doV/fr1w5YtW9CsWTO8+eab5dZPSEhAfn6+7cP3IBERkSeS3fCpy1Tfw6LT6fDWW2/hxRdfxIkTJ1BQUIBu3brhjjvuqHIQU6dOxbZt27Bv3z60aNFCVVtvb29069YN586dK7fcx8cHPj5Vf103ERER1T7VCUuZli1bomXLli5tXJZlPP300/joo4+Qnp6ONm3aqO7DYrHg+PHjGDJkiEuxEBER1SZOHOeY6oTFYrEgOTkZaWlpuHTpEqzCPAx79+6tdF9xcXFISUnBxx9/DH9/f5hMJgCAwWCwvaMoNjYWzZs3R2JiIgBg/vz56N27NyIiInDlyhW8/PLLOH/+PCZOnKh2V4iIiDyHq9d16vg1IdUJy7PPPovk5GTExMTgzjvvhCROBKXCmjVrAAD9+/dXrH/nnXfwxBNPAABycnKguWkyq19//RWTJk2CyWRC48aN0aNHDxw8eBAdOnSochxERES1ztUbZznCorRx40Zs2rTJLZdg5Eo8NJ6enq5YXr58OZYvX+7ytomIiOjWUaWbbiMiIqojFiIionrL1dlqOXGc4LnnnsPKlSsrNTpCRERElcN5WByr1AjLI488oljeu3cvPv30U3Ts2BHe3so38W3ZssV90RERERGhkgmLwWBQLD/88MPVEgwREVG9JUuu3TjLEZYbT+0QERFR9eE9LI6pvoeFiIiIqKapfkqoW7du5c69IkkSfH19ERERgSeeeAL33XefWwIkIiKqFzhxnEOqE5ZBgwZhzZo16NSpE+6++24AwNGjR/H111/jiSeewKlTpxAVFYUtW7Zg6NChbg+4RkiOB54k4UZjCMtWX51i2eKnPMwljbSK5aIA5fYsyuaw+CmXrcLmrcK3KGuFZfFVSsJJrS1WLjvZfbv+xfbiuJ1kEZatyoTXKvSnKVUGqNUo68tef2xAUyJuS9nWqlUGo9Eq+9J6CX0L29IUCwdbHHO1iAdTSOZLxJ1XlkvCTNF2/Ys/oVahXPzjQezf28thObRCf0L/skWIXyCJ8TrbH4Gs1TosV/btuC9JY3VYbrdtJ/2JPwiSeB7afRfCiS+ri0fsXy2n++Nke662d0bs35X9dXrsXaX4Oam5+0I4Nb9jqhOWy5cv47nnnsOLL76oWL9w4UKcP38eu3fvxty5c7FgwYJbN2EhIiIij6I6Ld20aRNGjhxpt37EiBHYtGkTAGDkyJE4c+aM69ERERHVJ7ILnzpOdcLi6+uLgwcP2q0/ePAgfH19AQBWq9X2byIiInKOE8c5pvqS0NNPP40pU6YgMzMTd911F4Ab97CsW7cOL7zwAgBg165d6Nq1q1sDJSIiqtN4061DqkdYZs+ejbfeegtHjhzBM888g2eeeQZHjhzBW2+9hX/84x8AgClTpuA///mP24MlIiIi95g3bx4kSVJ82rVrZysvLCxEXFwcmjRpgkaNGmH48OHIy8tT9JGTk4OYmBg0aNAAQUFBmDFjBkpLSxV10tPT0b17d/j4+CAiIgLJyclVilf1CAsAjB49GqNHj66w3M/Pr8IyIiIiKo8E155KUt+2Y8eO2LNnj23Zy+uPtGD69OnYvn07Nm/eDIPBgKlTp+KRRx7BgQMHAAAWiwUxMTEICQnBwYMHkZubi9jYWHh7e2Px4sUAgOzsbMTExGDKlCnYsGED0tLSMHHiRISGhiI6OlpVrFVKWIiIiMjNauGSkJeXF0JCQuzW5+fn4+2330ZKSgruv/9+ADdmvW/fvj0OHTqE3r17Y/fu3Th16hT27NmD4OBgdO3aFQsWLMCsWbMwb9486HQ6JCUloU2bNli2bBkAoH379ti/fz+WL1+uOmGp1CWhwMBAXL58GQDQuHFjBAYGVvghIiKi2mM2mxWfoqKiCuuePXsWYWFhuO222zB69Gjk5OQAADIzM1FSUoKoqChb3Xbt2qFly5bIyMgAAGRkZKBTp04IDg621YmOjobZbMbJkydtdW7uo6xOWR9qVGqEZfny5fD39wcArFixQvVGiIiIyAk3jbCEh4crVs+dOxfz5s2zq96rVy8kJyejbdu2yM3NxT//+U/ce++9OHHiBEwmE3Q6HQICAhRtgoODYTKZAAAmk0mRrJSVl5U5qmM2m3H9+nVVt5BUKmEZO3Zsuf8mIiIiN3HT25ovXLgAvV5vW+3jI053fsPgwYNt/+7cuTN69eqFVq1aYdOmTR55L2qV7mGxWq04d+4cLl26BKswFXffvn3dEhgRERGpp9frFQlLZQUEBOBPf/oTzp07hwceeADFxcW4cuWKYpQlLy/Pds9LSEgIjhw5ouij7Cmim+uITxbl5eVBr9erTopUJyyHDh3CqFGjcP78ecjCe0IkSYLFybtHiIiIyJ4sO339ltP2rigoKMB3332HMWPGoEePHvD29kZaWhqGDx8OADhz5gxycnJgNBoBAEajEYsWLcKlS5cQFBQEAEhNTYVer0eHDh1sdXbs2KHYTmpqqq0PNVQnLFOmTEHPnj2xfft2hIaGlvvmZiIiIlKphp8S+vvf/44HH3wQrVq1wo8//oi5c+dCq9Vi5MiRMBgMmDBhAuLj4xEYGAi9Xo+nn34aRqMRvXv3BgAMHDgQHTp0wJgxY7B06VKYTCbMnj0bcXFxtstQU6ZMwapVqzBz5kyMHz8ee/fuxaZNm7B9+3bVu6c6YTl79iw+/PBDREREqN4YEREReYaLFy9i5MiR+Pnnn9GsWTPcc889OHToEJo1awbgxgM3Go0Gw4cPR1FREaKjo/HGG2/Y2mu1Wmzbtg1PPvkkjEYjGjZsiLFjx2L+/Pm2Om3atMH27dsxffp0rFy5Ei1atMC6detUP9IMVCFh6dWrF86dO8eEhYiIyJ3cdNNtZW3cuNFhua+vL1avXo3Vq1dXWKdVq1Z2l3xE/fv3x7Fjx1TFVp5KJSxff/217d9PP/00nnvuOZhMJnTq1Ane3t6Kup07d3Y5KCIiovpGkm98XGlfl1UqYenatSskSVLcZDt+/Hjbv8vKeNMtERFRFfHlhw5VKmHJzs6u7jg8mqTVKpd1ylElNFA+mmVtoCwv0SsPc2kD5bBdqa+yO4uwbBU2V9JIeDpL+WQ5ZGfzFwsntdVbGY8ktNeUCO2Fcquzs0h5+OBVKIQjlIuvSLcKt77LGummfwuhCbFqi5XLVp0Ym3Lj4oiq1lu5gZu3DQCaYuXBl+wCEtoLN6lrSywOyyXxtv9S4Q8CjZMhYGHaAafEJ/8sGoflsvgHingTvhi/uH9qQrM70cW+vRyXW4V906g8NsIPhthetorxOPlBlJ1sX2zvpL7k7FxwwtX2TvvXij/oKvZfqOssVvG7EOvbfVd2m/5j23Y/01RrKpWwtGrVqrrjICIiqt9q+B6WWw1ffkhEROQJeEnIIY51ERERkcfjCAsREZEn4AiLQ0xYiIiIPAETFodcuiT01FNP4fLly+6KhYiIiKhcLiUs77//Psxms7tiISIiqr/KnhJy5VOHuXRJSHxbMxEREVUNZ7p1jE8JERERkcdzKWG5evUqbrvttiq3T0xMxF133QV/f38EBQVh2LBhOHPmjNN2mzdvRrt27eDr64tOnTo5ffESERGRx5Pd8KnDXEpYYmJikJubW+X2n3/+OeLi4nDo0CGkpqaipKQEAwcOxLVr1ypsc/DgQYwcORITJkzAsWPHMGzYMAwbNgwnTpyochxERETk2Vy6h2Xfvn24fv16ldvv3LlTsZycnIygoCBkZmaib9++5bZZuXIlBg0ahBkzZgAAFixYgNTUVKxatQpJSUlVjoWIiKg2SXDxHha3ReKZPOoelvz8fABAYGBghXUyMjIQFRWlWBcdHY2MjIxy6xcVFcFsNis+REREdGtxKWFp1aoVvL29nVesBKvVimnTpqFPnz648847K6xnMpkQHBysWBccHAyTyVRu/cTERBgMBtsnPDzcLfESERG5FR9rdsilhOXEiRNuSwDi4uJw4sQJbNy40S39lUlISEB+fr7tc+HCBbf2T0RE5Ba86dYhj5iaf+rUqdi2bRv27duHFi1aOKwbEhKCvLw8xbq8vDyEhISUW9/Hxwc+Pj5ui5WIiIhqXq3ewyLLMqZOnYqPPvoIe/fuRZs2bZy2MRqNSEtLU6xLTU2F0WisrjCJiIiqH0dYHKrVEZa4uDikpKTg448/hr+/v+0+FIPBAD8/PwBAbGwsmjdvjsTERADAs88+i379+mHZsmWIiYnBxo0b8cUXX2Dt2rVui0vSKK8DSt7CYdLpFItWf1/FcrFBWV7cSCMsK/u3KJujtIFy2eKjPAtlIc20W/ZS1pcsyu1pSoT6WuEsF6+DSo6L7dJeoTuNRblsFQ6nplRoLu6P43Ac9i021hbLjorttg1JWUH2VlawCuWSxapY1oizQYv9eWkclztpL5UKB1cknrvOZqcW4odGiK9EefJIWq2y3Es8oI63J1uE+MXtWf+IR9yWXVuRcKygdXLi2u27k/sBrMr2kkZoLynLxXjtjp1AtorfvZO/L2XH23dKbC9y9/bFckfbV1MX9v+Hqy2H100/N86OixtxplvHanWEZc2aNcjPz0f//v0RGhpq+3zwwQe2Ojk5OYq5XiIjI5GSkoK1a9eiS5cu+PDDD7F161aHN+oSERHRra1WR1gq8y6i9PR0u3WPPfYYHnvssWqIiIiIqJa4elmnjo+weMRNt0RERPUeExaHPGriOCIiIqLycISFiIjIA/CmW8eYsBAREXkCV2er5Uy3Ss888wxee+01u/WrVq3CtGnT3BETERFR/cN5WBxSnbD8+9//Rp8+fezWR0ZG4sMPP3RLUEREREQ3U31J6Oeff4bBYLBbr9frcfnyZbcERUREVN/wHhbHVI+wREREYOfOnXbrP/30U9x2221uCYqIiKjeqeVLQkuWLIEkSYrbOwoLCxEXF4cmTZqgUaNGGD58uN37/HJychATE4MGDRogKCgIM2bMQGmpcgrz9PR0dO/eHT4+PoiIiEBycrLq+FSPsMTHx2Pq1Kn46aefcP/99wMA0tLSsGzZMqxYsUJ1AERERFS7jh49ijfffBOdO3dWrJ8+fTq2b9+OzZs3w2AwYOrUqXjkkUdw4MABAIDFYkFMTAxCQkJw8OBB5ObmIjY2Ft7e3li8eDEAIDs7GzExMZgyZQo2bNiAtLQ0TJw4EaGhoYiOjq50jKoTlvHjx6OoqAiLFi3CggULAACtW7fGmjVrEBsbq7Y7IiIiAgAXLwlVdYSloKAAo0ePxltvvYWFCxfa1ufn5+Ptt99GSkqKbYDinXfeQfv27XHo0CH07t0bu3fvxqlTp7Bnzx4EBweja9euWLBgAWbNmoV58+ZBp9MhKSkJbdq0wbJlywAA7du3x/79+7F8+XJVCUuVJo578skncfHiReTl5cFsNuP7779nskJEROSKWrokFBcXh5iYGERFRSnWZ2ZmoqSkRLG+Xbt2aNmyJTIyMgAAGRkZ6NSpE4KDg211oqOjYTabcfLkSVsdse/o6GhbH5VVpXlYSktLkZ6eju+++w6jRo0CAPz444/Q6/Vo1KhRVbokIiIiNzCbzYplHx8f+Pj4lFt348aN+PLLL3H06FG7MpPJBJ1Oh4CAAMX64OBgmEwmW52bk5Wy8rIyR3XMZjOuX78OPz+/Su2X6oTl/PnzGDRoEHJyclBUVIQHHngA/v7+eOmll1BUVISkpCS1XRIREZGb3iUUHh6uWD137lzMmzfPrvqFCxfw7LPPIjU1Fb6+vi5suGaoviT07LPPomfPnvj1118VWdHDDz+MtLQ0twZHRERUX5Q91uzKB7iRiOTn59s+CQkJ5W4vMzMTly5dQvfu3eHl5QUvLy98/vnneO211+Dl5YXg4GAUFxfjypUrinZ5eXkICQkBAISEhNg9NVS27KyOXq+v9OgKUIURlv/+9784ePAgdDqdYn3r1q3xv//9T213RERE5EZ6vR56vd5pvQEDBuD48eOKdePGjUO7du0wa9YshIeHw9vbG2lpaRg+fDgA4MyZM8jJyYHRaAQAGI1GLFq0CJcuXUJQUBAAIDU1FXq9Hh06dLDV2bFjh2I7qamptj4qS3XCYrVaYbFY7NZfvHgR/v7+arvzDJJyoEkSkjGpgTIDlAOU9+mUNFYOpRUHaBXL15sq3+9QItzmU9pAuSx7KccErb6yw3JYHL8/QhK/Lifvm3B2l7pYLlmVyxrl4/d2Q5x2/Yvl9qdXheXirmgs4rLYubjvjnfW6iWcG6VCfVm5LAnbkzXCIKZG2L4Yj118ImH7zvoTaZTnJmThy3PW3ttbuSzMtSAeDzti/1plPJKwLJfzf42truMtQRZis+vbKuy71vGAsyR8lzKE2KxO2mudHHu77Sn3ULbKTsqr9AzFH/3ZHR+V1ybE/0erM37JSV0nx9ZZf5L3H78aJfE8qUP8/f1x5513KtY1bNgQTZo0sa2fMGEC4uPjERgYCL1ej6effhpGoxG9e/cGAAwcOBAdOnTAmDFjsHTpUphMJsyePRtxcXG2+2amTJmCVatWYebMmRg/fjz27t2LTZs2Yfv27ariVX2GDxw4UDHfiiRJKCgowNy5czFkyBC13RERERFQ6xPHlWf58uX485//jOHDh6Nv374ICQnBli1bbOVarRbbtm2DVquF0WjEX//6V8TGxmL+/Pm2Om3atMH27duRmpqKLl26YNmyZVi3bp2qR5oBQJJlZ38SKV28eBHR0dGQZRlnz55Fz549cfbsWTRt2hT79u2zDQl5KrPZDIPBgP4YCi/p978Whexa46u8m1rjrxwSkZs1ViwXhyhHlq43U/4Vei1Y2b/aERaLn2sjLJoSZbmm2El94Y9mTYnjZUmo73Vd6NDJCIs4omLXv/CXmbYYFdKUinWFv+rEAQW7ERFlufdVZXBiLOKIgqZE2ID40yX8Vam9WigEJHw34o9nqbJ/u7/+xPbXxf6FHRT/ErU4+WvSLh4nIyzishCfXKz8stWMsMBRGaowwuKE3QiLuH21IxIqRwGcj1C49tuqpvtz9/aEztTVF34utI3/eP1MqbUYey6vR35+fqUus1RF2e+liOcXQ+vCza+WwkKcW/JCtcZam1RfEmrRogW++uorfPDBB/jqq69QUFCACRMmYPTo0apuniEiIiKqrCrNw+Ll5YXRo0dj9OjR7o6HiIio/qrjLzB0hep7WN59913FjTIzZ85EQEAAIiMjcf78ebcGR0REVG944D0snkR1wrJ48WLbpZ+MjAysWrUKS5cuRdOmTTF9+nS3B0hERESk+pLQhQsXEBERAQDYunUrHn30UUyePBl9+vRB//793R0fERFRvXDz5G9VbV+XqR5hadSoEX7++WcAwO7du/HAAw8AAHx9fXH9uvh4CBEREVUKLwk5pHqE5YEHHsDEiRPRrVs3fPvtt7a5V06ePInWrVu7Oz4iIiIi9SMsq1evhtFoxE8//YR///vfaNKkCYAb7yQYOXKk2wMkIiKqD9z1LqG6SvUIS0BAAFatWmW3/p///KdbAiIiIqqXXL2sU8cTFtdePkFERERUA6o0cRwRERG5GUdYHGLCQkRE5AH4WLNjlb4k9Ntvv1VnHERERPUbH2t2qNIJS9OmTfHnP/8Za9euhclkqs6YiIiIiBQqnbB88803iI6OxqZNm9C6dWv06tULixYtwvHjx6szPiIiovqBIywOVfoelpYtW+Lpp5/G008/jfz8fOzYsQMff/wxXn75ZQQGBuKhhx7CQw89hH79+kGr1VZnzG4nCfFKOp2yQoMGisXSxsrl6029Fcu/BSnzwKJAZXcleuVZZfW1KitolOWyxslZqNw8pCLl9mXh67AK9TWlQn+yJCwL5UK4krAMoblkEZZlZ+XKChqx3Crf9G9lmf2+iMvKFVKpcKx1yuA1pcoNWHXKYyuJ25OEnRcDkMXvVuhPFg+O0J+X8DdGidCft3AuFwr1tcKycGwh/uiK8QjHA15O/gspKRH6FzZg94UJNOLxvDk2YV+EYyUJxxZWZeziz73dvorE/oX2sngyOvsuxZ8zq1Bf2HdJ/LIk4dzRiD+I7iU5+i7c0N6V/mXx2ElO/haXnRwrH58//l29h1WB97A4VqXHmg0GA0aOHImNGzfip59+wptvvgmLxYJx48ahWbNm2LBhg7vjJCIionrM5XlYvL298cADD+D111/H+fPnkZaWhj/96U+Vartv3z48+OCDCAsLgyRJ2Lp1q8P66enpkCTJ7sN7aoiI6JbHS0IOuf2x5m7dulW67rVr19ClSxeMHz8ejzzySKXbnTlzBnq93rYcFBSkKkYiIiJPw0tCjtXqPCyDBw/G4MGDVbcLCgpCQECA+wMiIiIij3RLTs3ftWtXhIaG4oEHHsCBAwcc1i0qKoLZbFZ8iIiIPA4vCTl0SyUsoaGhSEpKwr///W/8+9//Rnh4OPr3748vv/yywjaJiYkwGAy2T3h4eA1GTEREVElMWBxSfUno+vXrkGUZDX5/1Pf8+fP46KOP0KFDBwwcONDtAd6sbdu2aNu2rW05MjIS3333HZYvX47/+7//K7dNQkIC4uPjbctms5lJCxER0S1G9QjL0KFD8d577wEArly5gl69emHZsmUYOnQo1qxZ4/YAnbn77rtx7ty5Cst9fHyg1+sVHyIiIk8jueFTl6lOWL788kvce++9AIAPP/wQwcHBOH/+PN577z289tprbg/QmaysLISGhtb4domIiNyKl4QcUn1J6LfffoO/vz8AYPfu3XjkkUeg0WjQu3dvnD9/XlVfBQUFitGR7OxsZGVlITAwEC1btkRCQgL+97//2UZ0VqxYgTZt2qBjx44oLCzEunXrsHfvXuzevVvtbhAREXkUPtbsmOoRloiICGzduhUXLlzArl27bPetXLp0SfXlli+++ALdunWzzd0SHx+Pbt26Yc6cOQCA3Nxc5OTk2OoXFxfjueeeQ6dOndCvXz989dVX2LNnDwYMGKB2N4iIiOgWonqEZc6cORg1ahSmT5+OAQMGwGg0Argx2qJm0jgA6N+/P2QH7+9ITk5WLM+cORMzZ85UGzIREZHnc/WyTh0fYVGdsDz66KO45557kJubiy5dutjWDxgwAA8//LBbgyMiIqpX6njS4YoqzXQbEhKCkJAQxbq7777bLQERERERiSqVsKh5z8+WLVuqHAwREVF9xZtuHavUTbc3zxSr1+uRlpaGL774wlaemZmJtLQ0GAyGaguUiIioTqvhx5rXrFmDzp072+YoMxqN+PTTT23lhYWFiIuLQ5MmTdCoUSMMHz4ceXl5ij5ycnIQExODBg0aICgoCDNmzEBpaamiTnp6Orp37w4fHx9ERETY3Z9aWZUaYXnnnXds/541axb+8pe/ICkpCVqtFgBgsVjw1FNPcVI2IiKiW0SLFi2wZMkS3HHHHZBlGe+++y6GDh2KY8eOoWPHjpg+fTq2b9+OzZs3w2AwYOrUqXjkkUds7/CzWCyIiYlBSEgIDh48iNzcXMTGxsLb2xuLFy8GcGO6kpiYGEyZMgUbNmxAWloaJk6ciNDQUERHR6uKV5IdPaZTjmbNmmH//v2KKfIB4MyZM4iMjMTPP/+sKoCaZjabYTAY0F96GF6SNwBAo9Mp6mgClImXtUWQYrmgVUNlny21iuXrytt7UNK0RLEseVsVyzo/ZblWqywvLVX2LyopFPJOizDfYYlyIE3zm9CfMI6o/U1ZX3tdWV1bLASgDBeaEmHZIiyXCstCfcmqjEesL93Un7ZYFsqEZSE2bYlwugv1oVUeO2+zMjhZoywXY4Xw4ySVKpdlL2V7ryvKgyt7K78bcX9gFXZIJG7/WpGy3Ev47kuFL0dkFcrF/VUZDyTx3BRPFmHQ96b+ZYsyFklS1pVlIRYxVmexicR9E2MT4pGF+pJQ31m5XXgWJ9+NyNn+qiUeT+F425WLxPpq+1fT1s2k21vb/l1qKULamWXIz8+vtj/Ky34vdZq4GFqdb5X7sRQX4vi6F1yKNTAwEC+//DIeffRRNGvWDCkpKXj00UcBAN988w3at2+PjIwM9O7dG59++in+/Oc/48cff0RwcDAAICkpCbNmzcJPP/0EnU6HWbNmYfv27Thx4oRtGyNGjMCVK1ewc+dOVbGpnoeltLQU33zzjd36b775BlZn/3kRERFR+dx0SchsNis+RUXCHy7lsFgs2LhxI65duwaj0YjMzEyUlJQgKirKVqddu3Zo2bIlMjIyAAAZGRno1KmTLVkBgOjoaJjNZpw8edJW5+Y+yuqU9aGG6qeExo0bhwkTJuC7776zPRl0+PBhLFmyBOPGjVMdABEREbmP+ILfuXPnYt68eeXWPX78OIxGIwoLC9GoUSPby4yzsrKg0+kQEBCgqB8cHAyTyQQAMJlMimSlrLyszFEds9mM69evw8/Pr9L7pTpheeWVVxASEoJly5YhNzcXABAaGooZM2bgueeeU9sdERERwX1PCV24cEFxScjHx6fCNm3btkVWVhby8/Px4YcfYuzYsfj888+rHkQ1UpWwlJaWIiUlBWPHjsXMmTNhNpsBgDfbEhERucpNM92WPfVTGTqdDhEREQCAHj164OjRo1i5ciUef/xxFBcX48qVK4pRlry8PNs8bCEhIThy5Iiiv7KniG6uIz5ZlJeXB71er2p0BVB5D4uXlxemTJmCwsJCAOoOChERETngAW9rtlqtKCoqQo8ePeDt7Y20tDRb2ZkzZ5CTk2N7JY/RaMTx48dx6dIlW53U1FTo9Xp06NDBVufmPsrqlPWhhupLQnfffTeOHTuGVq1aqd4YEREReYaEhAQMHjwYLVu2xNWrV5GSkoL09HTs2rULBoMBEyZMQHx8PAIDA6HX6/H000/DaDSid+/eAICBAweiQ4cOGDNmDJYuXQqTyYTZs2cjLi7OdhlqypQpWLVqFWbOnInx48dj79692LRpE7Zv3646XtUJy1NPPYXnnnsOFy9eRI8ePdCwofIR386dO6sOgoiIqL6r6ZluL126hNjYWOTm5sJgMKBz587YtWsXHnjgAQDA8uXLodFoMHz4cBQVFSE6OhpvvPGGrb1Wq8W2bdvw5JNPwmg0omHDhhg7dizmz59vq9OmTRts374d06dPx8qVK9GiRQusW7dO9RwsN/ZP5TwsmnLmDpAkCbIsQ5IkWNTOHVDDOA8L52FR4DwsjvvjPCwVx8Z5WBy35zwslVb2e6lLrOvzsHz1nmvzsHgy1SMs2dnZ1REHERERUYVUJyy8d4WIiMj9JFmGpO6ih137ukx1wgIA3333HVasWIHTp08DADp06IBnn30Wt99+u1uDIyIiqjfc9FhzXaV6av5du3ahQ4cOOHLkCDp37ozOnTvj8OHD6NixI1JTU6sjRiIiIqrnVI+wPP/885g+fTqWLFlit37WrFm2u4uJiIio8mr6KaFbjeoRltOnT2PChAl268ePH49Tp065JSgiIqJ6xwMmjvNkqhOWZs2aISsry259VlYWgoKC7BsQERERuUj1JaFJkyZh8uTJ+P777xEZGQkAOHDgAF566SXEx8e7PUAiIqL6gJeEHFOdsLz44ovw9/fHsmXLkJCQAAAICwvDvHnz8Mwzz7g9QCIionqBTwk5pDphkSQJ06dPx/Tp03H16lUAgL+/v9sDq0mSzlu5Qq/cn6KmyjdK/hYkzGwbrGxe0kQ5e6ePQTnbaANf5VSx3uJUsILCEmV8FqtytlCrRXllzyJe6RNmprXqlLNEaoqFGTmFyUitwuERs3htoXLZbmZacTJYoX/72WKFcgeHRxYnARY7FzqzCvXFQ2/f3PFMrXb7Iv6HYbevQrk426k4E6zdwRDKS4UOvYT+NGJ/AnHmW3HmWXF7Yn1xxlGLsOxsXghx/x3Ul7SOZ3yWoCyX4WRmXHEmWfFY2X0XAi/lf5+SXX/K7Ulif06OjbP9FWfOFWdpdnrsxXjE706ceVY8PlYndxQ4O/fE9s7q30wSZ/dWOfOtk1l2S5v+MZu5s5nG3YkjLI6pvodl/fr1ttlu/f39b/lkhYiIiDyf6oQlMTERERERaNmyJcaMGYN169bh3Llz1REbERFR/cGnhBxSnbCcPXsWOTk5SExMRIMGDfDKK6+gbdu2aNGiBf76179WR4xERET1Qtlloap86jrVCQsANG/eHKNHj8by5cuxcuVKjBkzBnl5edi4caO74yMiIiJSf9Pt7t27kZ6ejvT0dBw7dgzt27dHv3798OGHH6Jv377VESMREVHdJ8vOb5Z21r4OU52wDBo0CM2aNcNzzz2HHTt2ICAgoBrCIiIiql/4lJBjqi8Jvfrqq+jTpw+WLl2Kjh07YtSoUVi7di2+/fbb6oiPiIiISH3CMm3aNGzZsgWXL1/Gzp07ERkZiZ07d+LOO+9EixYtqiNGIiKiuo9PCTmk+pIQAMiyjGPHjiE9PR2fffYZ9u/fD6vVimbNmrk7PiIionpBspYzuaTK9nWZ6oTlwQcfxIEDB2A2m9GlSxf0798fkyZNQt++fXk/CxEREVUL1QlLu3bt8Le//Q333nsvDAZDdcRERERU//BdQg6pvofl5Zdfxp///GenyUqnTp1w4cIFh3X27duHBx98EGFhYZAkCVu3bnW6/fT0dHTv3h0+Pj6IiIhAcnKyiuiJiIg8kyuTxtWHyeOqNHFcZfzwww8oEV+kJrh27Rq6dOmC1atXV6rP7OxsxMTE4L777kNWVhamTZuGiRMnYteuXe4ImYiIqPaUzcPiyqcOq9JNt+4yePBgDB48uNL1k5KS0KZNGyxbtgwA0L59e+zfvx/Lly9HdHR0dYVJREREtazaRliqQ0ZGBqKiohTroqOjkZGRUWGboqIimM1mxYeIiMjT8JKQY7dUwmIymRAcHKxYFxwcDLPZjOvXr5fbJjExEQaDwfYJDw+viVCJiIjU4TwsDt1SCUtVJCQkID8/3/ZxdiMwEREReZ5avYdFrZCQEOTl5SnW5eXlQa/Xw8/Pr9w2Pj4+8PHxsVsvabWQJO2Nf+t0ijLZT7ls1UnKZfGoScJiseM8sLDIW7FcotUqlkstyvayVdi+sGwpVrbHdWG5VFlfUyIsF4nlyubaYjgslyxwSBaOj9ifVaus4FXk+M+Em4c97WKxysKy49hEshCLrJEqqFlJktCf2L9WPHmEci/hXLII+6cVjpXQHhrhXPJWnrySeJOe2F5kEb5sqxCfcOrZ3QSoESpYhS9I3P7N29OIx8LxiSdpxWDEcuXPoV2s4vi6eC4I55ostteKx0aMX9h38Vg42V9JLBePnZMbMGW77Tn+7sXjKUM4/s62b3duihtw4WdNPLZOz2vH34W5te8fRcUADlY9NDX4LiHHqi1hefPNN+0u37jKaDRix44dinWpqakwGo1u3Q4REVGN49uaHapSwpKWloa0tDRcunQJViFLX79+PQBg1KhRTvspKCjAuXPnbMvZ2dnIyspCYGAgWrZsiYSEBPzvf//De++9BwCYMmUKVq1ahZkzZ2L8+PHYu3cvNm3ahO3bt1dlN4iIiOgWofoeln/+858YOHAg0tLScPnyZfz666+KjxpffPEFunXrhm7dugEA4uPj0a1bN8yZMwcAkJubi5ycHFv9Nm3aYPv27UhNTUWXLl2wbNkyrFu3jo80ExHRLa+mnxJKTEzEXXfdBX9/fwQFBWHYsGE4c+aMok5hYSHi4uLQpEkTNGrUCMOHD7e7NSMnJwcxMTFo0KABgoKCMGPGDJSWlirquGPSV9UjLElJSUhOTsaYMWNUb0zUv39/++u+Nylvh/r3749jx465vG0iIiKPUsNT83/++eeIi4vDXXfdhdLSUrzwwgsYOHAgTp06hYYNGwIApk+fju3bt2Pz5s0wGAyYOnUqHnnkERw4cAAAYLFYEBMTg5CQEBw8eBC5ubmIjY2Ft7c3Fi9eDOCPSV+nTJmCDRs2IC0tDRMnTkRoaKiqAQfVCUtxcTEiIyPVNiMiIiIPsnPnTsVycnIygoKCkJmZib59+yI/Px9vv/02UlJScP/99wMA3nnnHbRv3x6HDh1C7969sXv3bpw6dQp79uxBcHAwunbtigULFmDWrFmYN28edDqd2yZ9VX1JaOLEiUhJSVHbjIiIiByo7Ynj8vPzAQCBgYEAgMzMTJSUlCgmbG3Xrh1atmxpm7A1IyMDnTp1UjxkEx0dDbPZjJMnT9rqqJ30tTyqR1gKCwuxdu1a7NmzB507d4a3t/LRwFdffVVtl0RERGSV7R6XV90esJvRvaLpPRRNrVZMmzYNffr0wZ133gngxmStOp0OAQEBirrBwcEwmUy2OuVN6FpW5qhO2aSvFU1LIlKdsHz99dfo2rUrAODEiROKMsmV5+iJiIjqMzfdwyLO6D537lzMmzfPYdO4uDicOHEC+/fvdyGA6qU6Yfnss8+qIw4iIiJygwsXLkCv19uWnY2uTJ06Fdu2bcO+ffvQokUL2/qQkBAUFxfjypUrilGWvLw8hISE2OocOXJE0V/ZU0Q311E76Wt56vzU/ERERLcCCS7ew/J7P3q9XvGpKGGRZRlTp07FRx99hL1796JNmzaK8h49esDb2xtpaWm2dWfOnEFOTo5twlaj0Yjjx4/j0qVLtjqpqanQ6/Xo0KGDrc7NfZTVUTvp6y01NT8REVGdVcMz3cbFxSElJQUff/wx/P39bfecGAwG+Pn5wWAwYMKECYiPj0dgYCD0ej2efvppGI1G9O7dGwAwcOBAdOjQAWPGjMHSpUthMpkwe/ZsxMXF2RIld036yhEWIiKiemjNmjXIz89H//79ERoaavt88MEHtjrLly/Hn//8ZwwfPhx9+/ZFSEgItmzZYivXarXYtm0btFotjEYj/vrXvyI2Nhbz58+31XHXpK8cYSEiIvIANf3yQ0cTt5bx9fXF6tWrsXr16grrtGrVyu49fyJ3TPrKhIWIiMgT1PBMt7caXhIiIiIij8cRFiIiIg8gyTIkF266daXtrYAJCxERkSew/v5xpX0dVm8TFkkjVTwzr8WiWPS+qnxNtt/PwpU0Sblc0lCrWC79uZGy3FeZBRcrq0MSTjpNiTJOjVDuowzXrr14EmsLheViob3Qn0bsX1jWFjvO6jXKwwfJoqwv3igm9i9el9WW3LRC+IvCbt8FUqmyvkaIBaWOYxP3VFMibFA4pySLslzWCl+2M87+YhK2J4vb1wrnqlaoL363GuFnolQ8GcRz3+6IKBct4sknxuPkeHjdVC72Jcai9q9Lsb5V7F+oL25P+H8CsrJccrJvdtGKx8ZZuTiFu7P9F/ZPEl6rYrf/TvqTvBz/+pCF4yOJx8+Zm89lMRYns6qr3rZQfunuP7ZnvS4DGx03p5pRbxMWIiIiT8JLQo4xYSEiIvIEfErIISYsREREnqCGZ7q91fCxZiIiIvJ4HGEhIiLyADU90+2thgkLERGRJ+AlIYd4SYiIiIg8HkdYiIiIPIBkdT6XlLP2dRkTFiIiIk/AS0IO8ZIQEREReTyOsBAREXkCThznEBMWIiIiD8Cp+R3jJSEiIiLyeBxhISIi8gS86dYhJixERESeQAbgyqPJdTtfYcJCRETkCXgPi2O8h4WIiIg8HkdYAMjFxYpl6RezYllXVKpY9sr3VSw39PdRLFt9tIrlUj9lXihrJGFZGY/4AitZEsqdDBlqSoUOpPLr2YpLlPUlq3JZY1GZtTuJT+xfXBaPj6bEKpTf1NbiuC87wl8gYnurt/K70/6mPDdkL+WXJZVYhNiEcnF7Qn3N9RJle0m576r/YtIKB1881sK5bHfN2+rky7OoHK+WhfoWoVzcnnD8FNsTY5WcnNhq64vEWOxiU+6MJAnl4rkonNeSVnmuyRbx4Aic9Of0UoKwPTvi/onfjXg8nZybktifePzdORog/pyp3bZwbB6954jt30UFJVjlcoCVJMPFe1jcFolHYsJCRETkCXjTrUO8JEREREQejyMsREREnsAKp5fwnbavwzxihGX16tVo3bo1fH190atXLxw5cqTCusnJyZAkSfHx9fWtsD4REdGtoOwpIVc+dVmtJywffPAB4uPjMXfuXHz55Zfo0qULoqOjcenSpQrb6PV65Obm2j7nz5+vwYiJiIioptV6wvLqq69i0qRJGDduHDp06ICkpCQ0aNAA69evr7CNJEkICQmxfYKDg2swYiIiompQdtOtK586rFYTluLiYmRmZiIqKsq2TqPRICoqChkZGRW2KygoQKtWrRAeHo6hQ4fi5MmTNREuERFR9WHC4lCtJiyXL1+GxWKxGyEJDg6GyWQqt03btm2xfv16fPzxx3j//fdhtVoRGRmJixcvllu/qKgIZrNZ8SEiIqJbS61fElLLaDQiNjYWXbt2Rb9+/bBlyxY0a9YMb775Zrn1ExMTYTAYbJ/w8PAajpiIiKgSamGEZd++fXjwwQcRFhYGSZKwdetWISQZc+bMQWhoKPz8/BAVFYWzZ88q6vzyyy8YPXo09Ho9AgICMGHCBBQUFCjqfP3117j33nvh6+uL8PBwLF26VHWstZqwNG3aFFqtFnl5eYr1eXl5CAkJqVQf3t7e6NatG86dO1dueUJCAvLz822fCxcuuBw3ERGR21nd8FHp2rVr6NKlC1avXl1u+dKlS/Haa68hKSkJhw8fRsOGDREdHY3CwkJbndGjR+PkyZNITU3Ftm3bsG/fPkyePNlWbjabMXDgQLRq1QqZmZl4+eWXMW/ePKxdu1ZVrLU6D4tOp0OPHj2QlpaGYcOGAQCsVivS0tIwderUSvVhsVhw/PhxDBkypNxyHx8f+Pj4lFtGRETkKWrj5YeDBw/G4MGDyy2TZRkrVqzA7NmzMXToUADAe++9h+DgYGzduhUjRozA6dOnsXPnThw9ehQ9e/YEALz++usYMmQIXnnlFYSFhWHDhg0oLi7G+vXrodPp0LFjR2RlZeHVV19VJDbO1Polofj4eLz11lt49913cfr0aTz55JO4du0axo0bBwCIjY1FQkKCrf78+fOxe/dufP/99/jyyy/x17/+FefPn8fEiRNraxeIiIg8hnjfZlFRUZX6yc7OhslkUjwYYzAY0KtXL9uDMRkZGQgICLAlKwAQFRUFjUaDw4cP2+r07dsXOp3OVic6OhpnzpzBr7/+Wul4an2m28cffxw//fQT5syZA5PJhK5du2Lnzp22G3FzcnKguelFVr/++ismTZoEk8mExo0bo0ePHjh48CA6dOhQW7tARETkOje9S0i8V3Pu3LmYN2+e6u7KHn5x9GCMyWRCUFCQotzLywuBgYGKOm3atLHro6yscePGlYqn1hMWAJg6dWqFl4DS09MVy8uXL8fy5ctrICoiIqIaZJUByYWE5fc3el+4cAF6vd62uq7cFlHrl4SIiIjIffR6veJT1YSl7OEXRw/GhISE2M1MX1pail9++UVRp7w+bt5GZTBhISIi8gQeNnFcmzZtEBISgrS0NNs6s9mMw4cPw2g0Argx1ciVK1eQmZlpq7N3715YrVb06tXLVmffvn0oKSmx1UlNTUXbtm0rfTkIYMJCRETkIVxNVtQnLAUFBcjKykJWVhaAGzfaZmVlIScnB5IkYdq0aVi4cCE++eQTHD9+HLGxsQgLC7M92du+fXsMGjQIkyZNwpEjR3DgwAFMnToVI0aMQFhYGABg1KhR0Ol0mDBhAk6ePIkPPvgAK1euRHx8vKpYPeIeltpmvV6oWJaKS5TLV5UT4Eh5yvd/33znc7m0Ql4oCe8P12odtxfrOyNuzyo7Lne2fbG9u2mE/XNle1aLclk8duJfIMK2NOKxEc8FjeMc39k3JYn9l5Qqy8V4LcL+iNu3KidekMTvskQZv93xcHZuOfuLzVm5k3NPLhX3v+LjK8tVmGTCUSz2G1Aui8dGPE8tQjzCd+Hsu3LKxb+WZbXbc7J9u3PL2facxe/sZ/PmciexqN1Xu59j4edsafAx27/NDSxYpar3W8sXX3yB++67z7ZclkSMHTsWycnJmDlzJq5du4bJkyfjypUruOeee7Bz5074+vra2mzYsAFTp07FgAEDoNFoMHz4cLz22mu2coPBgN27dyMuLg49evRA06ZNMWfOHFWPNAOAJMt1/OUDArPZDIPBgPt1j8FL8gYAyFbHPwySt5DXCT9oEhMW13hQwmJ3bISExe6XkFpOEha7eFUmLHbfJROWimOx34BymQmLYrFOJyyCHd8esP3bfNWCxn/6Hvn5+YobWd2p7PdSVJun4aWp+g2ypdYi7Ml+vVpjrU0cYSEiIvIE1qpd1lG2r7t4DwsRERF5PI6wEBEReQLZeuPjSvs6jAkLERGRJ3DTTLd1FRMWIiIiT8B7WBziPSxERETk8TjCQkRE5Al4ScghJixERESeQIaLCYvbIvFIvCREREREHo8jLERERJ6Al4QcYsJCRETkCaxWAC7MpeLq6xg8HC8JERERkcfjCAsREZEn4CUhh5iwEBEReQImLA7xkhARERF5PI6wEBEReQJOze9QvU1YJD9fSJIOAGDNz1eUyRaLsnJxsZPerin71kguBqcRFlX2p9U6LhdPaq0w0CYOK0oqt+/qsKQYn6P9F78rycmgoYtvM5WF2CTxWKvsX+zPGfFcsGsvbt/Z8XBVdW/v5v7V9u3su1DZn8vH3tVyZ9y8v3bdi/8Puhqfmp9Voa5cUurStpz91G0uMNj+/VuBxUFN95JlK2QX/o9ype2toN4mLERERB5Fll0bJeE9LERERES1iyMsREREnkB28R6WOj7CwoSFiIjIE1itgOTCfSh1/B4WXhIiIiIij8cRFiIiIk/AS0IOMWEhIiLyALLVCtmFS0J1/bFmXhIiIiIij8cRFiIiIk/AS0IOMWEhIiLyBFYZkJiwVISXhIiIiMjjcYSFiIjIE8gyAFfmYanbIyxMWIiIiDyAbJUhu3BJSGbCQkRERNVOtsK1ERY+1lztVq9ejdatW8PX1xe9evXCkSNHHNbfvHkz2rVrB19fX3Tq1Ak7duyooUiJiIjqFrW/g2tLrScsH3zwAeLj4zF37lx8+eWX6NKlC6Kjo3Hp0qVy6x88eBAjR47EhAkTcOzYMQwbNgzDhg3DiRMnajhyIiIi95GtsssftdT+Dq5NtZ6wvPrqq5g0aRLGjRuHDh06ICkpCQ0aNMD69evLrb9y5UoMGjQIM2bMQPv27bFgwQJ0794dq1atquHIiYiI3Ei2uv5RSe3v4NpUq/ewFBcXIzMzEwkJCbZ1Go0GUVFRyMjIKLdNRkYG4uPjFeuio6OxdevWcusXFRWhqKjItpyfnw8ASD6xGHq93sU9ICKiuszsawYQXiM3tJaixKV540pRAgAwm82K9T4+PvDx8bGrX5XfwbWpVhOWy5cvw2KxIDg4WLE+ODgY33zzTbltTCZTufVNJlO59RMTE/HPf/7Tbn14eHgVoyYiovrm559/hsFgqJa+dTodQkJCsN/k+v2YjRo1svv9NnfuXMybN8+ublV+B9emOv+UUEJCgmJE5sqVK2jVqhVycnKq7eSry8xmM8LDw3HhwgWOUKnEY1d1PHau4fGruvz8fLRs2RKBgYHVtg1fX19kZ2ejuLjY5b5kWYYkSYp15Y2u3IpqNWFp2rQptFot8vLyFOvz8vIQEhJSbpuQkBBV9SsaCjMYDPzBdYFer+fxqyIeu6rjsXMNj1/VaTTVe8unr68vfH19q3Uboqr8Dq5NtXrTrU6nQ48ePZCWlmZbZ7VakZaWBqPRWG4bo9GoqA8AqampFdYnIiIie1X5HVybav2SUHx8PMaOHYuePXvi7rvvxooVK3Dt2jWMGzcOABAbG4vmzZsjMTERAPDss8+iX79+WLZsGWJiYrBx40Z88cUXWLt2bW3uBhER0S3H2e9gT1LrCcvjjz+On376CXPmzIHJZELXrl2xc+dO201AOTk5iqG4yMhIpKSkYPbs2XjhhRdwxx13YOvWrbjzzjsrtT0fHx/MnTu3zlzTq2k8flXHY1d1PHau4fGrurp+7Jz9DvYkklzXXz5AREREt7xanziOiIiIyBkmLEREROTxmLAQERGRx2PCQkRERB6vTiYsal+VvXnzZrRr1w6+vr7o1KkTduxwfXrkW5ma4/fWW2/h3nvvRePGjdG4cWNERUV57KvJa0JVX9O+ceNGSJKEYcOGVW+AHkztsbty5Qri4uIQGhoKHx8f/OlPf6rXP7tqj9+KFSvQtm1b+Pn5ITw8HNOnT0dhYWENRes59u3bhwcffBBhYWGQJKnC99LdLD09Hd27d4ePjw8iIiKQnJxc7XESALmO2bhxo6zT6eT169fLJ0+elCdNmiQHBATIeXl55dY/cOCArNVq5aVLl8qnTp2SZ8+eLXt7e8vHjx+v4cg9g9rjN2rUKHn16tXysWPH5NOnT8tPPPGEbDAY5IsXL9Zw5LVP7bErk52dLTdv3ly+99575aFDh9ZMsB5G7bErKiqSe/bsKQ8ZMkTev3+/nJ2dLaenp8tZWVk1HLlnUHv8NmzYIPv4+MgbNmyQs7Oz5V27dsmhoaHy9OnTazjy2rdjxw75H//4h7xlyxYZgPzRRx85rP/999/LDRo0kOPj4+VTp07Jr7/+uqzVauWdO3fWTMD1WJ1LWO6++245Li7OtmyxWOSwsDA5MTGx3Pp/+ctf5JiYGMW6Xr16yX/729+qNU5Ppfb4iUpLS2V/f3/53Xffra4QPVZVjl1paakcGRkpr1u3Th47dmy9TVjUHrs1a9bIt912m1xcXFxTIXo0tccvLi5Ovv/++xXr4uPj5T59+lRrnJ6uMgnLzJkz5Y4dOyrWPf7443J0dHQ1RkayLMt16pJQ2auyo6KibOucvSo7IyNDUR8AoqOjPfLV2tWtKsdP9Ntvv6GkpKRaXxTmiap67ObPn4+goCBMmDChJsL0SFU5dp988gmMRiPi4uIQHByMO++8E4sXL4bFYqmpsD1GVY5fZGQkMjMzbZeNvv/+e+zYsQNDhgypkZhvZfydUXtqfaZbd6rKq7JNJlO59U0mU7XF6anc8arxWbNmISwszO4Huq6ryrHbv38/3n77bWRlZdVAhJ6rKsfu+++/x969ezF69Gjs2LED586dw1NPPYWSkhLMnTu3JsL2GFU5fqNGjcLly5dxzz33QJZllJaWYsqUKXjhhRdqIuRbWkW/M8xmM65fvw4/P79aiqzuq1MjLFS7lixZgo0bN+Kjjz6q8beO3mquXr2KMWPG4K233kLTpk1rO5xbjtVqRVBQENauXYsePXrg8ccfxz/+8Q8kJSXVdmi3hPT0dCxevBhvvPEGvvzyS2zZsgXbt2/HggULajs0ogrVqRGWqrwqOyQk5JZ5tXZ1c+VV46+88gqWLFmCPXv2oHPnztUZpkdSe+y+++47/PDDD3jwwQdt66xWKwDAy8sLZ86cwe233169QXuIqpx3oaGh8Pb2hlarta1r3749TCYTiouLodPpqjVmT1KV4/fiiy9izJgxmDhxIgCgU6dOuHbtGiZPnox//OMfive3kVJFvzP0ej1HV6pZnTorq/KqbKPRqKgPAKmpqR75au3qVtVXjS9duhQLFizAzp070bNnz5oI1eOoPXbt2rXD8ePHkZWVZfs89NBDuO+++5CVlYXw8PCaDL9WVeW869OnD86dO2dL8gDg22+/RWhoaL1KVoCqHb/ffvvNLikpS/5kvl7OIf7OqEW1fdevu23cuFH28fGRk5OT5VOnTsmTJ0+WAwICZJPJJMuyLI8ZM0Z+/vnnbfUPHDgge3l5ya+88op8+vRpee7cufX+sWY1x2/JkiWyTqeTP/zwQzk3N9f2uXr1am3tQq1Re+xE9fkpIbXHLicnR/b395enTp0qnzlzRt62bZscFBQkL1y4sLZ2oVapPX5z586V/f395X/961/y999/L+/evVu+/fbb5b/85S+1tQu15urVq/KxY8fkY8eOyQDkV199VT527Jh8/vx5WZZl+fnnn5fHjBljq1/2WPOMGTPk06dPy6tXr+ZjzTWkziUssizLr7/+utyyZUtZp9PJd999t3zo0CFbWb9+/eSxY8cq6m/atEn+05/+JOt0Orljx47y9u3bazhiz6Lm+LVq1UoGYPeZO3duzQfuAdSeezerzwmLLKs/dgcPHpR79eol+/j4yLfddpu8aNEiubS0tIaj9hxqjl9JSYk8b948+fbbb5d9fX3l8PBw+amnnpJ//fXXmg+8ln322Wfl/h9WdrzGjh0r9+vXz65N165dZZ1OJ992223yO++8U+Nx10eSLHP8j4iIiDxbnbqHhYiIiOomJixERETk8ZiwEBERkcdjwkJEREQejwkLEREReTwmLEREROTxmLAQERGRx2PCQkRERB6PCQsRERF5PCYsRKRacXFxbYdARPUMExaiesJqtWLp0qWIiIiAj48PWrZsiUWLFgEALly4gL/85S8ICAhAYGAghg4dih9++MHW9oknnsCwYcOwaNEihIWFoW3btrW0F0RUX3nVdgBEVDMSEhLw1ltvYfny5bjnnnuQm5uLb775BiUlJYiOjobRaMR///tfeHl5YeHChRg0aBC+/vpr6HQ6AEBaWhr0ej1SU1NreU+IqD7iyw+J6oGrV6+iWbNmWLVqFSZOnKgoe//997Fw4UKcPn0akiQBuHHJJyAgAFu3bsXAgQPxxBNPYOfOncjJybElMERENYkjLET1wOnTp1FUVIQBAwbYlX311Vc4d+4c/P39FesLCwvx3Xff2ZY7derEZIWIag0TFqJ6wM/Pr8KygoIC9OjRAxs2bLAra9asme3fDRs2rJbYiIgqgzfdEtUDd9xxB/z8/JCWlmZX1r17d5w9exZBQUGIiIhQfAwGQy1ES0RkjwkLUT3g6+uLWbNmYebMmXjvvffw3Xff4dChQ3j77bcxevRoNG3aFEOHDsV///tfZGdnIz09Hc888wwuXrxY26ETEQHgJSGieuPFF1+El5cX5syZgx9//BGhoaGYMmUKGjRogH379mHWrFl45JFHcPXqVTRv3hwDBgyAXq+v7bCJiADwKSEiIiK6BfCSEBEREXk8JixERETk8ZiwEBERkcdjwkJEREQejwkLEREReTwmLEREROTxmLAQERGRx2PCQkRERB6PCQsRERF5PCYsRERE5PGYsBAREZHHY8JCREREHu//ASoBuMa+aXABAAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.clf()\n", "h = plt.hist2d(cers, rates, bins=100)\n", "plt.colorbar(h[3])\n", "plt.ylim(0, 4)\n", "# plt.xlim(0, 0.99)\n", "plt.xlabel(\"cer\")\n", "plt.ylabel(\"n_words / sec -- higher faster\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 15, "id": "2172be9d", "metadata": { "ExecuteTime": { "end_time": "2024-01-18T03:01:36.428046Z", "start_time": "2024-01-18T03:01:32.654154Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:25.559235Z", "iopub.status.busy": "2025-01-14T01:48:25.559090Z", "iopub.status.idle": "2025-01-14T01:48:30.078970Z", "shell.execute_reply": "2025-01-14T01:48:30.078473Z", "shell.execute_reply.started": "2025-01-14T01:48:25.559220Z" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAGdCAYAAAD+JxxnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAtTUlEQVR4nO3df1xVdZ7H8Tc/4kfmBX8MP+5GSj/VkYlRE7Fspo2HtFqz7NgmyRrTkE4FbUqlmIZWlg79GLVM1poZfDxWV3M3XUOjGBxjU0JF2ZSEatO0dS/aQ+EqJSCc/aMHZ7xqCcy9EHxfz8fj/sH5fu45n/Od6r7ne88518+yLEsAAAAG8u/uBgAAALoLQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYKzA7m7gh6y1tVVHjx5V37595efn193tAACAdrAsS6dOnZLT6ZS///ev+RCEvsfRo0cVExPT3W0AAIBOOHLkiK688srvrSEIfY++fftK+nYiHQ5HN3cDAADaw+12KyYmxv4c/z4Eoe/R9nWYw+EgCAEA0MO057IWLpYGAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMFZgdzcAcw3O2dyuukOLJ/q4EwCAqVgRAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwVoeDUGlpqe666y45nU75+flp48aN9lhzc7Nmz56tuLg49enTR06nU/fdd5+OHj3qsY8TJ04oLS1NDodD4eHhysjI0OnTpz1qPvroI40bN04hISGKiYlRXl7eBb2sX79eQ4YMUUhIiOLi4rRlyxaPccuylJubq+joaIWGhiopKUmffvppR08ZAAD0Uh0OQg0NDbrxxhu1fPnyC8a+/vpr7dmzR0899ZT27Nmjt956SzU1NfrFL37hUZeWlqaqqioVFxersLBQpaWlmj59uj3udrs1fvx4DRo0SBUVFXrhhRe0YMECrVy50q7ZsWOH7r33XmVkZGjv3r1KSUlRSkqK9u/fb9fk5eVp2bJlys/PV3l5ufr06aPk5GSdOXOmo6cNAAB6IT/LsqxOv9nPTxs2bFBKSsp31uzatUujR4/WF198oauuukoHDhzQsGHDtGvXLo0aNUqSVFRUpAkTJujLL7+U0+nUihUrNHfuXLlcLgUFBUmScnJytHHjRlVXV0uSJk+erIaGBhUWFtrHGjNmjOLj45Wfny/LsuR0OvXYY4/p8ccflyTV19crMjJSBQUFSk1NveT5ud1uhYWFqb6+Xg6Ho7PThO/AAxUBAL7Qkc9vn18jVF9fLz8/P4WHh0uSysrKFB4ebocgSUpKSpK/v7/Ky8vtmltvvdUOQZKUnJysmpoanTx50q5JSkryOFZycrLKysokSQcPHpTL5fKoCQsLU0JCgl0DAADM5tOf2Dhz5oxmz56te++9105kLpdLERERnk0EBqp///5yuVx2TWxsrEdNZGSkPdavXz+5XC5727k15+7j3PddrOZ8jY2NamxstP92u90dOl8AANCz+CwINTc365577pFlWVqxYoWvDuNVixYt0tNPP93dbfQK7f3aCwCA7uSTr8baQtAXX3yh4uJij+/noqKidOzYMY/6s2fP6sSJE4qKirJramtrPWra/r5Uzbnj577vYjXnmzNnjurr6+3XkSNHOnTeAACgZ/F6EGoLQZ9++qn+9Kc/acCAAR7jiYmJqqurU0VFhb1t69atam1tVUJCgl1TWlqq5uZmu6a4uFg33HCD+vXrZ9eUlJR47Lu4uFiJiYmSpNjYWEVFRXnUuN1ulZeX2zXnCw4OlsPh8HgBAIDeq8NB6PTp06qsrFRlZaWkby9Krqys1OHDh9Xc3Ky7775bu3fv1urVq9XS0iKXyyWXy6WmpiZJ0tChQ3XHHXdo2rRp2rlzp7Zv366srCylpqbK6XRKkqZMmaKgoCBlZGSoqqpK69at09KlS5WdnW338eijj6qoqEgvvfSSqqurtWDBAu3evVtZWVmSvr2jbcaMGVq4cKE2bdqkffv26b777pPT6fzeu9wAAIA5Onz7/LZt23TbbbddsD09PV0LFiy44CLnNn/+85/185//XNK3D1TMysrS22+/LX9/f02aNEnLli3TFVdcYdd/9NFHyszM1K5duzRw4EA98sgjmj17tsc+169fr3nz5unQoUO67rrrlJeXpwkTJtjjlmVp/vz5Wrlyperq6nTLLbfotdde0/XXX9+uc+X2+c7z5jVC3D4PAOiIjnx+/1XPEertCEKdRxACAHSXH9RzhAAAAH6oCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLECu7sB9DzefGo0AADdiRUhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYPFARP3jteYDjocUTu6ATAEBvw4oQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLn9hAr8DPcAAAOoMVIQAAYCyCEAAAMBZBCAAAGItrhOChPdfaAADQW7AiBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxOhyESktLddddd8npdMrPz08bN270GLcsS7m5uYqOjlZoaKiSkpL06aefetScOHFCaWlpcjgcCg8PV0ZGhk6fPu1R89FHH2ncuHEKCQlRTEyM8vLyLuhl/fr1GjJkiEJCQhQXF6ctW7Z0uBcAAGCuDgehhoYG3XjjjVq+fPlFx/Py8rRs2TLl5+ervLxcffr0UXJyss6cOWPXpKWlqaqqSsXFxSosLFRpaammT59uj7vdbo0fP16DBg1SRUWFXnjhBS1YsEArV660a3bs2KF7771XGRkZ2rt3r1JSUpSSkqL9+/d3qBcAAGAuP8uyrE6/2c9PGzZsUEpKiqRvV2CcTqcee+wxPf7445Kk+vp6RUZGqqCgQKmpqTpw4ICGDRumXbt2adSoUZKkoqIiTZgwQV9++aWcTqdWrFihuXPnyuVyKSgoSJKUk5OjjRs3qrq6WpI0efJkNTQ0qLCw0O5nzJgxio+PV35+frt6uRS3262wsDDV19fL4XB0dpp6lME5m7u7BZ85tHhid7cAAOgCHfn89uo1QgcPHpTL5VJSUpK9LSwsTAkJCSorK5MklZWVKTw83A5BkpSUlCR/f3+Vl5fbNbfeeqsdgiQpOTlZNTU1OnnypF1z7nHaatqO055eztfY2Ci32+3xAgAAvZdXg5DL5ZIkRUZGemyPjIy0x1wulyIiIjzGAwMD1b9/f4+ai+3j3GN8V82545fq5XyLFi1SWFiY/YqJiWnHWQMAgJ6Ku8bOMWfOHNXX19uvI0eOdHdLAADAh7wahKKioiRJtbW1Httra2vtsaioKB07dsxj/OzZszpx4oRHzcX2ce4xvqvm3PFL9XK+4OBgORwOjxcAAOi9vBqEYmNjFRUVpZKSEnub2+1WeXm5EhMTJUmJiYmqq6tTRUWFXbN161a1trYqISHBriktLVVzc7NdU1xcrBtuuEH9+vWza849TltN23Ha0wsAADBbh4PQ6dOnVVlZqcrKSknfXpRcWVmpw4cPy8/PTzNmzNDChQu1adMm7du3T/fdd5+cTqd9Z9nQoUN1xx13aNq0adq5c6e2b9+urKwspaamyul0SpKmTJmioKAgZWRkqKqqSuvWrdPSpUuVnZ1t9/Hoo4+qqKhIL730kqqrq7VgwQLt3r1bWVlZktSuXgAAgNkCO/qG3bt367bbbrP/bgsn6enpKigo0KxZs9TQ0KDp06errq5Ot9xyi4qKihQSEmK/Z/Xq1crKytLtt98uf39/TZo0ScuWLbPHw8LC9N577ykzM1MjR47UwIEDlZub6/GsobFjx2rNmjWaN2+ennzySV133XXauHGjhg8fbte0pxcAAGCuv+o5Qr0dzxHqXXiOEACYodueIwQAANCTEIQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYKwO//o80FO15wdl+WFWADALK0IAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMFdjdDQA/JINzNl+y5tDiiV3QCQCgK7AiBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWNw1BnRQe+4sk7i7DAB6AlaEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYy+tBqKWlRU899ZRiY2MVGhqqa665Rs8++6wsy7JrLMtSbm6uoqOjFRoaqqSkJH366ace+zlx4oTS0tLkcDgUHh6ujIwMnT592qPmo48+0rhx4xQSEqKYmBjl5eVd0M/69es1ZMgQhYSEKC4uTlu2bPH2KQMAgB7K60Hot7/9rVasWKFXX31VBw4c0G9/+1vl5eXplVdesWvy8vK0bNky5efnq7y8XH369FFycrLOnDlj16SlpamqqkrFxcUqLCxUaWmppk+fbo+73W6NHz9egwYNUkVFhV544QUtWLBAK1eutGt27Nihe++9VxkZGdq7d69SUlKUkpKi/fv3e/u0AQBAD+RnnbtU4wV33nmnIiMj9fvf/97eNmnSJIWGhupf//VfZVmWnE6nHnvsMT3++OOSpPr6ekVGRqqgoECpqak6cOCAhg0bpl27dmnUqFGSpKKiIk2YMEFffvmlnE6nVqxYoblz58rlcikoKEiSlJOTo40bN6q6ulqSNHnyZDU0NKiwsNDuZcyYMYqPj1d+fv4lz8XtdissLEz19fVyOBxem6MfsvY+IweXxnOEAKB7dOTz2+srQmPHjlVJSYk++eQTSdJ///d/64MPPtDf/d3fSZIOHjwol8ulpKQk+z1hYWFKSEhQWVmZJKmsrEzh4eF2CJKkpKQk+fv7q7y83K659dZb7RAkScnJyaqpqdHJkyftmnOP01bTdpzzNTY2yu12e7wAAEDv5fUnS+fk5MjtdmvIkCEKCAhQS0uLnnvuOaWlpUmSXC6XJCkyMtLjfZGRkfaYy+VSRESEZ6OBgerfv79HTWxs7AX7aBvr16+fXC7X9x7nfIsWLdLTTz/dmdMGAAA9kNdXhN58802tXr1aa9as0Z49e7Rq1Sq9+OKLWrVqlbcP5XVz5sxRfX29/Tpy5Eh3twQAAHzI6ytCTzzxhHJycpSamipJiouL0xdffKFFixYpPT1dUVFRkqTa2lpFR0fb76utrVV8fLwkKSoqSseOHfPY79mzZ3XixAn7/VFRUaqtrfWoafv7UjVt4+cLDg5WcHBwZ04bAAD0QF5fEfr666/l7++524CAALW2tkqSYmNjFRUVpZKSEnvc7XarvLxciYmJkqTExETV1dWpoqLCrtm6dataW1uVkJBg15SWlqq5udmuKS4u1g033KB+/frZNecep62m7TgAAMBsXg9Cd911l5577jlt3rxZhw4d0oYNG/Tyyy/rH/7hHyRJfn5+mjFjhhYuXKhNmzZp3759uu++++R0OpWSkiJJGjp0qO644w5NmzZNO3fu1Pbt25WVlaXU1FQ5nU5J0pQpUxQUFKSMjAxVVVVp3bp1Wrp0qbKzs+1eHn30URUVFemll15SdXW1FixYoN27dysrK8vbpw0AAHogr3819sorr+ipp57Sww8/rGPHjsnpdOo3v/mNcnNz7ZpZs2apoaFB06dPV11dnW655RYVFRUpJCTErlm9erWysrJ0++23y9/fX5MmTdKyZcvs8bCwML333nvKzMzUyJEjNXDgQOXm5no8a2js2LFas2aN5s2bpyeffFLXXXedNm7cqOHDh3v7tAEAQA/k9ecI9SY8Rwh/DZ4jBADdo1ufIwQAANBTEIQAAICxvH6NEH6Y+MoLAIALsSIEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMbiJzYAH2nPz5rwC/UA0L1YEQIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsfitMaAb8XtkANC9WBECAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWD4JQv/7v/+rf/qnf9KAAQMUGhqquLg47d692x63LEu5ubmKjo5WaGiokpKS9Omnn3rs48SJE0pLS5PD4VB4eLgyMjJ0+vRpj5qPPvpI48aNU0hIiGJiYpSXl3dBL+vXr9eQIUMUEhKiuLg4bdmyxRenDAAAeiCvB6GTJ0/q5ptv1mWXXaZ33nlHH3/8sV566SX169fPrsnLy9OyZcuUn5+v8vJy9enTR8nJyTpz5oxdk5aWpqqqKhUXF6uwsFClpaWaPn26Pe52uzV+/HgNGjRIFRUVeuGFF7RgwQKtXLnSrtmxY4fuvfdeZWRkaO/evUpJSVFKSor279/v7dMGAAA9kJ9lWZY3d5iTk6Pt27frv/7rvy46blmWnE6nHnvsMT3++OOSpPr6ekVGRqqgoECpqak6cOCAhg0bpl27dmnUqFGSpKKiIk2YMEFffvmlnE6nVqxYoblz58rlcikoKMg+9saNG1VdXS1Jmjx5shoaGlRYWGgff8yYMYqPj1d+fv4lz8XtdissLEz19fVyOBx/1bx0t8E5m7u7BXTSocUTu7sFAOhROvL57fUVoU2bNmnUqFH6x3/8R0VEROinP/2pXn/9dXv84MGDcrlcSkpKsreFhYUpISFBZWVlkqSysjKFh4fbIUiSkpKS5O/vr/Lycrvm1ltvtUOQJCUnJ6umpkYnT560a849TltN23HO19jYKLfb7fECAAC9l9eD0Oeff64VK1bouuuu07vvvquHHnpI//zP/6xVq1ZJklwulyQpMjLS432RkZH2mMvlUkREhMd4YGCg+vfv71FzsX2ce4zvqmkbP9+iRYsUFhZmv2JiYjp8/gAAoOfwehBqbW3ViBEj9Pzzz+unP/2ppk+frmnTprXrq6juNmfOHNXX19uvI0eOdHdLAADAhwK9vcPo6GgNGzbMY9vQoUP1H//xH5KkqKgoSVJtba2io6PtmtraWsXHx9s1x44d89jH2bNndeLECfv9UVFRqq2t9ahp+/tSNW3j5wsODlZwcHC7zxXoCu25vovriACgc7y+InTzzTerpqbGY9snn3yiQYMGSZJiY2MVFRWlkpISe9ztdqu8vFyJiYmSpMTERNXV1amiosKu2bp1q1pbW5WQkGDXlJaWqrm52a4pLi7WDTfcYN+hlpiY6HGctpq24wAAALN5PQjNnDlTH374oZ5//nl99tlnWrNmjVauXKnMzExJkp+fn2bMmKGFCxdq06ZN2rdvn+677z45nU6lpKRI+nYF6Y477tC0adO0c+dObd++XVlZWUpNTZXT6ZQkTZkyRUFBQcrIyFBVVZXWrVunpUuXKjs72+7l0UcfVVFRkV566SVVV1drwYIF2r17t7Kysrx92gAAoAfy+ldjN910kzZs2KA5c+bomWeeUWxsrJYsWaK0tDS7ZtasWWpoaND06dNVV1enW265RUVFRQoJCbFrVq9eraysLN1+++3y9/fXpEmTtGzZMns8LCxM7733njIzMzVy5EgNHDhQubm5Hs8aGjt2rNasWaN58+bpySef1HXXXaeNGzdq+PDh3j5tAADQA3n9OUK9Cc8RQk/BNUIA8Bfd+hwhAACAnoIgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYgd3dAIC/3uCczZesObR4Yhd0AgA9CytCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYPFkaMER7nj4t8QRqAGZhRQgAABiLIAQAAIxFEAIAAMbiGqFeoL3XfgAAAE+sCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABj+TwILV68WH5+fpoxY4a97cyZM8rMzNSAAQN0xRVXaNKkSaqtrfV43+HDhzVx4kRdfvnlioiI0BNPPKGzZ8961Gzbtk0jRoxQcHCwrr32WhUUFFxw/OXLl2vw4MEKCQlRQkKCdu7c6YvTBHqNwTmbL/kCgN7Cp0Fo165d+pd/+Rf95Cc/8dg+c+ZMvf3221q/fr3ef/99HT16VL/85S/t8ZaWFk2cOFFNTU3asWOHVq1apYKCAuXm5to1Bw8e1MSJE3XbbbepsrJSM2bM0AMPPKB3333Xrlm3bp2ys7M1f/587dmzRzfeeKOSk5N17NgxX542AADoIfwsy7J8sePTp09rxIgReu2117Rw4ULFx8dryZIlqq+v149+9COtWbNGd999tySpurpaQ4cOVVlZmcaMGaN33nlHd955p44eParIyEhJUn5+vmbPnq3jx48rKChIs2fP1ubNm7V//377mKmpqaqrq1NRUZEkKSEhQTfddJNeffVVSVJra6tiYmL0yCOPKCcn55Ln4Ha7FRYWpvr6ejkcDm9Pkdfw/9DR1Q4tntjdLQDAd+rI57fPVoQyMzM1ceJEJSUleWyvqKhQc3Ozx/YhQ4boqquuUllZmSSprKxMcXFxdgiSpOTkZLndblVVVdk15+87OTnZ3kdTU5MqKio8avz9/ZWUlGTXnK+xsVFut9vjBQAAeq9AX+x07dq12rNnj3bt2nXBmMvlUlBQkMLDwz22R0ZGyuVy2TXnhqC28bax76txu9365ptvdPLkSbW0tFy0prq6+qJ9L1q0SE8//XT7TxQAAPRoXl8ROnLkiB599FGtXr1aISEh3t69T82ZM0f19fX268iRI93dEgAA8CGvB6GKigodO3ZMI0aMUGBgoAIDA/X+++9r2bJlCgwMVGRkpJqamlRXV+fxvtraWkVFRUmSoqKiLriLrO3vS9U4HA6FhoZq4MCBCggIuGhN2z7OFxwcLIfD4fECAAC9l9e/Grv99tu1b98+j23333+/hgwZotmzZysmJkaXXXaZSkpKNGnSJElSTU2NDh8+rMTERElSYmKinnvuOR07dkwRERGSpOLiYjkcDg0bNsyu2bJli8dxiouL7X0EBQVp5MiRKikpUUpKiqRvL5YuKSlRVlaWt08bAACcpz0383T3zRdeD0J9+/bV8OHDPbb16dNHAwYMsLdnZGQoOztb/fv3l8Ph0COPPKLExESNGTNGkjR+/HgNGzZMU6dOVV5enlwul+bNm6fMzEwFBwdLkh588EG9+uqrmjVrln79619r69atevPNN7V5818mPTs7W+np6Ro1apRGjx6tJUuWqKGhQffff7+3TxswSk/4jxsAtIdPLpa+lN/97nfy9/fXpEmT1NjYqOTkZL322mv2eEBAgAoLC/XQQw8pMTFRffr0UXp6up555hm7JjY2Vps3b9bMmTO1dOlSXXnllXrjjTeUnJxs10yePFnHjx9Xbm6uXC6X4uPjVVRUdMEF1AAAwEw+e45Qb8BzhIDOY0UIQHetHv8gniMEAADwQ0cQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGCswO5uAEDvNDhn8yVrDi2e2AWdAMB3Y0UIAAAYiyAEAACMRRACAADGIggBAABjEYQAAICxCEIAAMBYBCEAAGAsghAAADAWQQgAABiLIAQAAIzFT2wA6Db8DAeA7saKEAAAMBZBCAAAGIsgBAAAjEUQAgAAxiIIAQAAY3HXGIAftPbcWSZxdxmAzmFFCAAAGIsgBAAAjEUQAgAAxiIIAQAAYxGEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMxU9sAOgV2vNTHPwMB4DzsSIEAACMRRACAADG8noQWrRokW666Sb17dtXERERSklJUU1NjUfNmTNnlJmZqQEDBuiKK67QpEmTVFtb61Fz+PBhTZw4UZdffrkiIiL0xBNP6OzZsx4127Zt04gRIxQcHKxrr71WBQUFF/SzfPlyDR48WCEhIUpISNDOnTu9fcoAAKCH8noQev/995WZmakPP/xQxcXFam5u1vjx49XQ0GDXzJw5U2+//bbWr1+v999/X0ePHtUvf/lLe7ylpUUTJ05UU1OTduzYoVWrVqmgoEC5ubl2zcGDBzVx4kTddtttqqys1IwZM/TAAw/o3XfftWvWrVun7OxszZ8/X3v27NGNN96o5ORkHTt2zNunDQAAeiA/y7IsXx7g+PHjioiI0Pvvv69bb71V9fX1+tGPfqQ1a9bo7rvvliRVV1dr6NChKisr05gxY/TOO+/ozjvv1NGjRxUZGSlJys/P1+zZs3X8+HEFBQVp9uzZ2rx5s/bv328fKzU1VXV1dSoqKpIkJSQk6KabbtKrr74qSWptbVVMTIweeeQR5eTkXLJ3t9utsLAw1dfXy+FweHtqvKY9F4kCaB8uqAa8p7tuYujI57fPrxGqr6+XJPXv31+SVFFRoebmZiUlJdk1Q4YM0VVXXaWysjJJUllZmeLi4uwQJEnJyclyu92qqqqya87dR1tN2z6amppUUVHhUePv76+kpCS75nyNjY1yu90eLwAA0Hv5NAi1trZqxowZuvnmmzV8+HBJksvlUlBQkMLDwz1qIyMj5XK57JpzQ1DbeNvY99W43W598803+uqrr9TS0nLRmrZ9nG/RokUKCwuzXzExMZ07cQAA0CP4NAhlZmZq//79Wrt2rS8P4zVz5sxRfX29/Tpy5Eh3twQAAHzIZw9UzMrKUmFhoUpLS3XllVfa26OiotTU1KS6ujqPVaHa2lpFRUXZNeff3dV2V9m5NeffaVZbWyuHw6HQ0FAFBAQoICDgojVt+zhfcHCwgoODO3fCAACgx/H6ipBlWcrKytKGDRu0detWxcbGeoyPHDlSl112mUpKSuxtNTU1Onz4sBITEyVJiYmJ2rdvn8fdXcXFxXI4HBo2bJhdc+4+2mra9hEUFKSRI0d61LS2tqqkpMSuAQAAZvP6ilBmZqbWrFmj//zP/1Tfvn3t63HCwsIUGhqqsLAwZWRkKDs7W/3795fD4dAjjzyixMREjRkzRpI0fvx4DRs2TFOnTlVeXp5cLpfmzZunzMxMe8XmwQcf1KuvvqpZs2bp17/+tbZu3ao333xTmzf/5Qr17Oxspaena9SoURo9erSWLFmihoYG3X///d4+bQAA0AN5PQitWLFCkvTzn//cY/sf//hH/epXv5Ik/e53v5O/v78mTZqkxsZGJScn67XXXrNrAwICVFhYqIceekiJiYnq06eP0tPT9cwzz9g1sbGx2rx5s2bOnKmlS5fqyiuv1BtvvKHk5GS7ZvLkyTp+/Lhyc3PlcrkUHx+voqKiCy6gBgAAZvL5c4R6Mp4jBJiH5wgB3tMTniPEr88DwDn4FXvALAShHzhWewAA8B1+fR4AABiLFSEA6CC+PgN6D1aEAACAsQhCAADAWAQhAABgLIIQAAAwFkEIAAAYi7vGAMAH2vsMMO4uA7oXK0IAAMBYBCEAAGAsghAAADAWQQgAABiLi6UBoBvxcx1A92JFCAAAGIsVIQD4gWPVCPAdVoQAAICxCEIAAMBYBCEAAGAsrhECgF6A64iAziEIAYAhCEvAhfhqDAAAGIsgBAAAjEUQAgAAxuIaIQCArT3XEUlcS4TegyAEAOgwLrxGb0EQAgD4BGEJPQHXCAEAAGMRhAAAgLH4agwA0G34+gzdjSAEAPhBIyzBl/hqDAAAGIsVIQBAj8fzj9BZBCEAgDH4mg3nIwgBAHAOb4UlQlfPQBACAKCD2vtVXFftp70IXhciCAEAYAhvBa/eFKgIQgAAoEO6eiXLl7h9HgAAGIsVoW7UmxI1AAA9EStCAADAWAQhAABgLIIQAAAwFkEIAAAYiyAEAACMZUQQWr58uQYPHqyQkBAlJCRo586d3d0SAAD4Aej1QWjdunXKzs7W/PnztWfPHt14441KTk7WsWPHurs1AADQzXp9EHr55Zc1bdo03X///Ro2bJjy8/N1+eWX6w9/+EN3twYAALpZr36gYlNTkyoqKjRnzhx7m7+/v5KSklRWVnZBfWNjoxobG+2/6+vrJUlut9sn/bU2fu2T/QIA0FP44jO2bZ+WZV2ytlcHoa+++kotLS2KjIz02B4ZGanq6uoL6hctWqSnn376gu0xMTE+6xEAAJOFLfHdvk+dOqWwsLDvrenVQaij5syZo+zsbPvv1tZWnThxQgMGDJCfn59Xj+V2uxUTE6MjR47I4XB4dd/4C+a5azDPXYN57jrMddfw1TxblqVTp07J6XResrZXB6GBAwcqICBAtbW1Httra2sVFRV1QX1wcLCCg4M9toWHh/uyRTkcDv4l6wLMc9dgnrsG89x1mOuu4Yt5vtRKUJtefbF0UFCQRo4cqZKSEntba2urSkpKlJiY2I2dAQCAH4JevSIkSdnZ2UpPT9eoUaM0evRoLVmyRA0NDbr//vu7uzUAANDNen0Qmjx5so4fP67c3Fy5XC7Fx8erqKjogguou1pwcLDmz59/wVdx8C7muWswz12Dee46zHXX+CHMs5/VnnvLAAAAeqFefY0QAADA9yEIAQAAYxGEAACAsQhCAADAWAQhH1q+fLkGDx6skJAQJSQkaOfOnd9bv379eg0ZMkQhISGKi4vTli1buqjTnq0j8/z6669r3Lhx6tevn/r166ekpKRL/u+Cb3X0n+c2a9eulZ+fn1JSUnzbYC/R0Xmuq6tTZmamoqOjFRwcrOuvv57/drRDR+d5yZIluuGGGxQaGqqYmBjNnDlTZ86c6aJue6bS0lLdddddcjqd8vPz08aNGy/5nm3btmnEiBEKDg7Wtddeq4KCAp/3KQs+sXbtWisoKMj6wx/+YFVVVVnTpk2zwsPDrdra2ovWb9++3QoICLDy8vKsjz/+2Jo3b5512WWXWfv27eviznuWjs7zlClTrOXLl1t79+61Dhw4YP3qV7+ywsLCrC+//LKLO+9ZOjrPbQ4ePGj9zd/8jTVu3Djr7//+77um2R6so/Pc2NhojRo1ypowYYL1wQcfWAcPHrS2bdtmVVZWdnHnPUtH53n16tVWcHCwtXr1auvgwYPWu+++a0VHR1szZ87s4s57li1btlhz58613nrrLUuStWHDhu+t//zzz63LL7/cys7Otj7++GPrlVdesQICAqyioiKf9kkQ8pHRo0dbmZmZ9t8tLS2W0+m0Fi1adNH6e+65x5o4caLHtoSEBOs3v/mNT/vs6To6z+c7e/as1bdvX2vVqlW+arFX6Mw8nz171ho7dqz1xhtvWOnp6QShdujoPK9YscK6+uqrraampq5qsVfo6DxnZmZaf/u3f+uxLTs727r55pt92mdv0p4gNGvWLOvHP/6xx7bJkydbycnJPuzMsvhqzAeamppUUVGhpKQke5u/v7+SkpJUVlZ20feUlZV51EtScnLyd9ajc/N8vq+//lrNzc3q37+/r9rs8To7z88884wiIiKUkZHRFW32eJ2Z502bNikxMVGZmZmKjIzU8OHD9fzzz6ulpaWr2u5xOjPPY8eOVUVFhf312eeff64tW7ZowoQJXdKzKbrrc7DXP1m6O3z11VdqaWm54OnVkZGRqq6uvuh7XC7XRetdLpfP+uzpOjPP55s9e7acTucF//LhLzozzx988IF+//vfq7Kysgs67B06M8+ff/65tm7dqrS0NG3ZskWfffaZHn74YTU3N2v+/Pld0XaP05l5njJlir766ivdcsstsixLZ8+e1YMPPqgnn3yyK1o2xnd9Drrdbn3zzTcKDQ31yXFZEYKxFi9erLVr12rDhg0KCQnp7nZ6jVOnTmnq1Kl6/fXXNXDgwO5up1drbW1VRESEVq5cqZEjR2ry5MmaO3eu8vPzu7u1XmXbtm16/vnn9dprr2nPnj166623tHnzZj377LPd3Rq8gBUhHxg4cKACAgJUW1vrsb22tlZRUVEXfU9UVFSH6tG5eW7z4osvavHixfrTn/6kn/zkJ75ss8fr6Dz/z//8jw4dOqS77rrL3tba2ipJCgwMVE1Nja655hrfNt0Ddeaf5+joaF122WUKCAiwtw0dOlQul0tNTU0KCgryac89UWfm+amnntLUqVP1wAMPSJLi4uLU0NCg6dOna+7cufL3Z03BG77rc9DhcPhsNUhiRcgngoKCNHLkSJWUlNjbWltbVVJSosTExIu+JzEx0aNekoqLi7+zHp2bZ0nKy8vTs88+q6KiIo0aNaorWu3ROjrPQ4YM0b59+1RZWWm/fvGLX+i2225TZWWlYmJiurL9HqMz/zzffPPN+uyzz+ygKUmffPKJoqOjCUHfoTPz/PXXX18QdtrCp8XPdXpNt30O+vRSbIOtXbvWCg4OtgoKCqyPP/7Ymj59uhUeHm65XC7Lsixr6tSpVk5Ojl2/fft2KzAw0HrxxRetAwcOWPPnz+f2+Xbo6DwvXrzYCgoKsv793//d+r//+z/7derUqe46hR6ho/N8Pu4aa5+OzvPhw4etvn37WllZWVZNTY1VWFhoRUREWAsXLuyuU+gROjrP8+fPt/r27Wv927/9m/X5559b7733nnXNNddY99xzT3edQo9w6tQpa+/evdbevXstSdbLL79s7d271/riiy8sy7KsnJwca+rUqXZ92+3zTzzxhHXgwAFr+fLl3D7f073yyivWVVddZQUFBVmjR4+2PvzwQ3vsZz/7mZWenu5R/+abb1rXX3+9FRQUZP34xz+2Nm/e3MUd90wdmedBgwZZki54zZ8/v+sb72E6+s/zuQhC7dfRed6xY4eVkJBgBQcHW1dffbX13HPPWWfPnu3irnuejsxzc3OztWDBAuuaa66xQkJCrJiYGOvhhx+2Tp482fWN9yB//vOfL/rf27a5TU9Pt372s59d8J74+HgrKCjIuvrqq60//vGPPu/Tz7JY1wMAAGbiGiEAAGAsghAAADAWQQgAABiLIAQAAIxFEAIAAMYiCAEAAGMRhAAAgLEIQgAAwFgEIQAAYCyCEAAAMBZBCAAAGIsgBAAAjPX/PmJWNkBVgrcAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.hist(total_infos_2.values(), bins=np.linspace(0, 1, 50))\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 16, "id": "097e26a4", "metadata": { "ExecuteTime": { "end_time": "2024-01-12T22:41:30.694878Z", "start_time": "2024-01-12T22:41:30.692934Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:30.079677Z", "iopub.status.busy": "2025-01-14T01:48:30.079522Z", "iopub.status.idle": "2025-01-14T01:48:30.082164Z", "shell.execute_reply": "2025-01-14T01:48:30.081768Z", "shell.execute_reply.started": "2025-01-14T01:48:30.079662Z" } }, "outputs": [], "source": [ "# sub_total_infos = {k: v for k, v in total_infos.items() if k in total_infos_2}\n", "\n", "# len(sub_total_infos)\n", "\n", "# plt.hist(sub_total_infos.values(), bins=np.linspace(0, 1, 50))\n", "# plt.show()\n", "\n", "# np.median(list(sub_total_infos.values())), np.median(list(total_infos_2.values()))\n", "\n", "# interesting_diffs = []\n", "# for k in total_infos_2:\n", "# if k in sub_total_infos and total_infos_2[k] - sub_total_infos[k] < 0.3 and sub_total_infos[k] > 0.6:\n", "# interesting_diffs.append(k)\n", "# print(len(interesting_diffs))\n", "\n", "# for idx in interesting_diffs:\n", "# if idx in metas_map and metas_map[idx][\"lang\"] == \"zh\":\n", "# print(metas_map[idx])\n", "# break" ] }, { "cell_type": "markdown", "id": "42dcbcc8", "metadata": {}, "source": [ "# prepare the 2nd hoot dataset" ] }, { "cell_type": "code", "execution_count": 50, "id": "326e1b84", "metadata": { "ExecuteTime": { "end_time": "2024-01-15T18:49:58.544826Z", "start_time": "2024-01-15T18:49:56.487590Z" }, "execution": { "iopub.execute_input": "2025-01-14T15:34:40.602741Z", "iopub.status.busy": "2025-01-14T15:34:40.602572Z", "iopub.status.idle": "2025-01-14T15:34:43.196382Z", "shell.execute_reply": "2025-01-14T15:34:43.195791Z", "shell.execute_reply.started": "2025-01-14T15:34:40.602725Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "918380\n" ] } ], "source": [ "high_cer_infos_2 = {}\n", "for k, v in total_infos_2.items():\n", " k_lang = metas_map[k][\"lang\"]\n", " # should probably also play it safe\n", " cut_threshold = min(langs_cer_cut.get(k_lang, 0.5), 0.8)\n", " if v <= cut_threshold:\n", " high_cer_infos_2[k] = v\n", "print(len(high_cer_infos_2))" ] }, { "cell_type": "code", "execution_count": 51, "id": "636c3311", "metadata": { "ExecuteTime": { "end_time": "2024-01-15T18:51:44.029594Z", "start_time": "2024-01-15T18:51:42.398978Z" }, "execution": { "iopub.execute_input": "2025-01-14T15:37:30.232542Z", "iopub.status.busy": "2025-01-14T15:37:30.232210Z", "iopub.status.idle": "2025-01-14T15:37:31.992066Z", "shell.execute_reply": "2025-01-14T15:37:31.991479Z", "shell.execute_reply.started": "2025-01-14T15:37:30.232525Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "903751 7060.5546875\n" ] } ], "source": [ "high_cer_train_info_2 = []\n", "for info in train_info:\n", " if (\n", " info[\"id\"] in high_cer_infos_2 and info[\"duration\"] < 360\n", " ): # --> this is how v3 t9 data is prepared\n", " # if info[\"id\"] in high_cer_infos_2 and 210 < info[\"duration\"] < 300:\n", " high_cer_train_info_2.append(info)\n", "print(len(high_cer_train_info_2), len(high_cer_train_info_2) / 2 / 8 / 8)" ] }, { "cell_type": "code", "execution_count": 52, "id": "bf451a42", "metadata": { "ExecuteTime": { "end_time": "2024-01-15T18:52:00.685027Z", "start_time": "2024-01-15T18:51:59.480509Z" }, "execution": { "iopub.execute_input": "2025-01-14T15:37:43.551574Z", "iopub.status.busy": "2025-01-14T15:37:43.551251Z", "iopub.status.idle": "2025-01-14T15:37:46.406410Z", "shell.execute_reply": "2025-01-14T15:37:46.405823Z", "shell.execute_reply.started": "2025-01-14T15:37:43.551558Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total duration khr 50.90579694444445 60 359 22 8124\n", "Counter({'en': 559077, 'es': 53968, 'pt': 38660, 'fr': 37395, 'de': 31485, 'ru': 28524, 'pl': 27476, 'it': 22982, 'tr': 12566, 'ko': 10893, 'ja': 8828, 'iw': 5911, 'nl': 5446, 'el': 4183, 'ro': 4004, 'da': 3683, 'sv': 3493, 'ar': 3181, 'hr': 2948, 'sr': 2827, 'no': 2536, 'vi': 2436, 'cs': 2212, 'fi': 2206, 'romanization': 2128, 'tl': 1875, 'zh-Hant': 1794, 'az': 1761, 'id': 1724, 'hu': 1715, 'bs': 1473, 'fa': 1392, 'sk': 1331, 'bg': 852, 'sq': 836, 'et': 707, 'uk': 706, 'zh': 667, 'ms': 642, 'ca': 540, 'lv': 492, 'lt': 449, 'th': 386, 'sw': 380, 'is': 329, 'eu': 301, 'sl': 279, 'af': 276, 'zu': 274, 'mk': 266, 'co': 242, 'ht': 234, 'gl': 230, 'sco': 211, 'la': 177, 'xh': 144, 'eo': 133, 'yo': 83, 'sn': 79, 'rw': 78, 'ln': 74, 'cy': 70, 'ceb': 69, 'hi': 59, 'wo': 56, 'jw': 48, 'st': 47, 'ak': 46, 'sa': 44, 'lb': 44, 'ne': 43, 'uz': 43, 'ga': 43, 'war': 42, 'mi': 40, 'be': 40, 'rm': 35, 'gd': 34, 'fo': 34, 'lg': 33, 'aa': 29, 'mg': 28, 'so': 28, 'tn': 26, 'sm': 26, 'ig': 26, 'my': 25, 'oc': 24, 'ka': 23, 'ny': 21, 'haw': 21, 'ha': 19, 'ia': 19, 'tk': 19, 'ts': 18, 'su': 17, 'bn': 16, 've': 16, 'pa': 15, 'ie': 15, 'qu': 15, 'ku': 14, 'mt': 13, 'bi': 12, 'crs': 12, 'tlh': 12, 'gn': 11, 'hy': 11, 'br': 11, 'om': 11, 'kha': 11, 'to': 11, 'rn': 10, 'kl': 9, 'tt': 9, 'mn': 9, 'ss': 8, 'fy': 8, 'mfe': 7, 'km': 7, 'si': 7, 'vo': 6, 'fj': 6, 'kk': 5, 'tg': 5, 'ta': 4, 'ky': 4, 'gv': 4, 'ml': 3, 'te': 3, 'ur': 2, 'ay': 2, 'hmn': 2, 'nso': 1, 'iu': 1, 'yi': 1, 'na': 1, 'kn': 1, 'ik': 1})\n" ] } ], "source": [ "print(\n", " \"total duration khr\",\n", " sum(m[\"duration\"] for m in high_cer_train_info_2) / 3600000,\n", " min(m[\"duration\"] for m in high_cer_train_info_2),\n", " max(m[\"duration\"] for m in high_cer_train_info_2),\n", " min(len(m[\"text\"]) for m in high_cer_train_info_2),\n", " max(len(m[\"text\"]) for m in high_cer_train_info_2),\n", ")\n", "print(Counter(m[\"lang\"] for m in high_cer_train_info_2))" ] }, { "cell_type": "code", "execution_count": 53, "id": "063513d5-ddf4-47c5-aa3c-6382d63f35e2", "metadata": { "execution": { "iopub.execute_input": "2025-01-14T16:00:32.470361Z", "iopub.status.busy": "2025-01-14T16:00:32.469783Z", "iopub.status.idle": "2025-01-14T16:00:32.699485Z", "shell.execute_reply": "2025-01-14T16:00:32.698913Z", "shell.execute_reply.started": "2025-01-14T16:00:32.470342Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3870\n", "3000\n" ] } ], "source": [ "selected_multi_validation_set = read_jsonl(\n", " \"/home/tony/Data/Hoot/v4_multi_validation_set.json\"\n", ")\n", "print(len(selected_multi_validation_set))\n", "selected_en_validation_set = read_jsonl(\n", " \"/home/tony/Data/Hoot/v4_en_validation_set.json\"\n", ")\n", "print(len(selected_en_validation_set))" ] }, { "cell_type": "code", "execution_count": 54, "id": "26c6eac1-736a-4f93-9a19-40e1c0152dc5", "metadata": { "execution": { "iopub.execute_input": "2025-01-14T16:00:33.396095Z", "iopub.status.busy": "2025-01-14T16:00:33.395767Z", "iopub.status.idle": "2025-01-14T16:00:33.400562Z", "shell.execute_reply": "2025-01-14T16:00:33.400011Z", "shell.execute_reply.started": "2025-01-14T16:00:33.396079Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "6870\n" ] } ], "source": [ "selected_validation_set_ids = set(x[\"id\"] for x in selected_multi_validation_set).union(\n", " set(x[\"id\"] for x in selected_en_validation_set)\n", ")\n", "print(len(selected_validation_set_ids))" ] }, { "cell_type": "code", "execution_count": 55, "id": "5492f723-0ac1-45bd-8539-bfd093a5335b", "metadata": { "execution": { "iopub.execute_input": "2025-01-14T15:59:56.367179Z", "iopub.status.busy": "2025-01-14T15:59:56.366634Z", "iopub.status.idle": "2025-01-14T15:59:57.063118Z", "shell.execute_reply": "2025-01-14T15:59:57.062469Z", "shell.execute_reply.started": "2025-01-14T15:59:56.367125Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "903751\n", "897604\n" ] } ], "source": [ "print(len(high_cer_train_info_2))\n", "high_cer_train_info_2 = [\n", " x for x in high_cer_train_info_2 if x[\"id\"] not in selected_validation_set_ids\n", "]\n", "print(len(high_cer_train_info_2))" ] }, { "cell_type": "code", "execution_count": 56, "id": "90466614", "metadata": { "ExecuteTime": { "end_time": "2024-01-15T18:52:12.098221Z", "start_time": "2024-01-15T18:52:11.939584Z" }, "execution": { "iopub.execute_input": "2025-01-14T15:44:14.123946Z", "iopub.status.busy": "2025-01-14T15:44:14.123554Z", "iopub.status.idle": "2025-01-14T15:44:14.517921Z", "shell.execute_reply": "2025-01-14T15:44:14.517273Z", "shell.execute_reply.started": "2025-01-14T15:44:14.123925Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "893604 4000\n" ] } ], "source": [ "cutoff = -4000\n", "random.shuffle(high_cer_train_info_2)\n", "high_cer_train_info_2, high_cer_test_info_2 = (\n", " high_cer_train_info_2[:cutoff],\n", " high_cer_train_info_2[cutoff:],\n", ")\n", "print(len(high_cer_train_info_2), len(high_cer_test_info_2))" ] }, { "cell_type": "code", "execution_count": 57, "id": "74ef0ed9", "metadata": { "ExecuteTime": { "end_time": "2024-01-15T18:52:34.087188Z", "start_time": "2024-01-15T18:52:23.399643Z" }, "execution": { "iopub.execute_input": "2025-01-14T15:44:18.027449Z", "iopub.status.busy": "2025-01-14T15:44:18.027087Z", "iopub.status.idle": "2025-01-14T15:44:53.231893Z", "shell.execute_reply": "2025-01-14T15:44:53.231193Z", "shell.execute_reply.started": "2025-01-14T15:44:18.027431Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/v5_t1_cer_50_long_train.json\", \"w\") as fp:\n", "# for l in high_cer_train_info_2:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')\n", "# with open(\"/home/tony/Data/Hoot/v5_t1_cer_50_long_test.json\", \"w\") as fp:\n", "# for l in high_cer_test_info_2:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write('\\n')" ] }, { "cell_type": "code", "execution_count": 58, "id": "61409fbd", "metadata": { "ExecuteTime": { "end_time": "2024-01-15T18:52:34.091372Z", "start_time": "2024-01-15T18:52:34.089444Z" }, "execution": { "iopub.execute_input": "2025-01-14T01:48:35.505118Z", "iopub.status.busy": "2025-01-14T01:48:35.504994Z", "iopub.status.idle": "2025-01-14T01:48:35.539824Z", "shell.execute_reply": "2025-01-14T01:48:35.539404Z", "shell.execute_reply.started": "2025-01-14T01:48:35.505106Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Done\n" ] } ], "source": [ "print(\"Done\")" ] }, { "cell_type": "markdown", "id": "535130a0", "metadata": {}, "source": [ "# Curate a seperate list of long audios, for just evaluation" ] }, { "cell_type": "code", "execution_count": 59, "id": "13fa3abf", "metadata": { "ExecuteTime": { "end_time": "2024-01-11T19:10:11.366040Z", "start_time": "2024-01-11T19:10:10.100230Z" }, "execution": { "iopub.execute_input": "2025-01-14T16:00:15.047699Z", "iopub.status.busy": "2025-01-14T16:00:15.047401Z", "iopub.status.idle": "2025-01-14T16:00:17.115621Z", "shell.execute_reply": "2025-01-14T16:00:17.114871Z", "shell.execute_reply.started": "2025-01-14T16:00:15.047681Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "863864\n" ] } ], "source": [ "high_cer_train_info_2_long = []\n", "for info in train_info:\n", " if info[\"id\"] in high_cer_infos_2 and info[\"duration\"] < 300:\n", " high_cer_train_info_2_long.append(info)\n", "print(len(high_cer_train_info_2_long))" ] }, { "cell_type": "code", "execution_count": 60, "id": "5c7f0742", "metadata": { "ExecuteTime": { "end_time": "2024-01-11T19:10:11.424285Z", "start_time": "2024-01-11T19:10:11.367473Z" }, "execution": { "iopub.execute_input": "2025-01-14T16:00:17.116825Z", "iopub.status.busy": "2025-01-14T16:00:17.116651Z", "iopub.status.idle": "2025-01-14T16:00:17.480047Z", "shell.execute_reply": "2025-01-14T16:00:17.479393Z", "shell.execute_reply.started": "2025-01-14T16:00:17.116808Z" } }, "outputs": [], "source": [ "random.shuffle(high_cer_train_info_2_long)" ] }, { "cell_type": "code", "execution_count": 61, "id": "cfbf0a99", "metadata": { "ExecuteTime": { "end_time": "2024-01-11T19:10:12.582481Z", "start_time": "2024-01-11T19:10:12.457526Z" }, "execution": { "iopub.execute_input": "2025-01-14T16:00:17.480831Z", "iopub.status.busy": "2025-01-14T16:00:17.480681Z", "iopub.status.idle": "2025-01-14T16:00:18.515817Z", "shell.execute_reply": "2025-01-14T16:00:18.515175Z", "shell.execute_reply.started": "2025-01-14T16:00:17.480815Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3907\n" ] } ], "source": [ "selected_validation_set = []\n", "c = Counter()\n", "for m in high_cer_train_info_2_long:\n", " if c[m[\"lang\"]] < min(len(lang_cers.get(m[\"lang\"], [])) // 4, 50):\n", " selected_validation_set.append(m)\n", " c[m[\"lang\"]] += 1\n", "print(len(selected_validation_set))" ] }, { "cell_type": "code", "execution_count": 62, "id": "d3c6afd4-c44a-495d-b311-bb8042a9bfe1", "metadata": { "execution": { "iopub.execute_input": "2025-01-15T01:23:44.489159Z", "iopub.status.busy": "2025-01-15T01:23:44.488728Z", "iopub.status.idle": "2025-01-15T01:23:44.789090Z", "shell.execute_reply": "2025-01-15T01:23:44.788560Z", "shell.execute_reply.started": "2025-01-15T01:23:44.489140Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/v4_multi_validation_set.json\", \"w\") as fp:\n", "# for l in selected_validation_set:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write(\"\\n\")" ] }, { "cell_type": "code", "execution_count": 63, "id": "f8d88911", "metadata": { "ExecuteTime": { "end_time": "2024-01-11T19:10:43.816910Z", "start_time": "2024-01-11T19:10:43.660712Z" }, "execution": { "iopub.execute_input": "2025-01-14T16:00:18.763232Z", "iopub.status.busy": "2025-01-14T16:00:18.762767Z", "iopub.status.idle": "2025-01-14T16:00:19.678028Z", "shell.execute_reply": "2025-01-14T16:00:19.677390Z", "shell.execute_reply.started": "2025-01-14T16:00:18.763214Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3000\n" ] } ], "source": [ "selected_validation_set_ids = set(x[\"id\"] for x in selected_validation_set)\n", "selected_en_validation_set = []\n", "c = Counter()\n", "for m in high_cer_train_info_2_long:\n", " if (\n", " m[\"lang\"] == \"en\"\n", " and m[\"id\"] not in selected_validation_set_ids\n", " and c[m[\"lang\"]] < 3000\n", " ):\n", " selected_en_validation_set.append(m)\n", " c[m[\"lang\"]] += 1\n", "print(len(selected_en_validation_set))" ] }, { "cell_type": "code", "execution_count": 64, "id": "c1e7ac97", "metadata": { "execution": { "iopub.execute_input": "2025-01-15T01:23:52.018137Z", "iopub.status.busy": "2025-01-15T01:23:52.017514Z", "iopub.status.idle": "2025-01-15T01:23:52.217614Z", "shell.execute_reply": "2025-01-15T01:23:52.217124Z", "shell.execute_reply.started": "2025-01-15T01:23:52.018119Z" } }, "outputs": [], "source": [ "# with open(\"/home/tony/Data/Hoot/v4_en_validation_set.json\", \"w\") as fp:\n", "# for l in selected_en_validation_set:\n", "# json.dump(l, fp, ensure_ascii=True)\n", "# fp.write(\"\\n\")" ] }, { "cell_type": "markdown", "id": "2854610a", "metadata": {}, "source": [ "# Validations and checks" ] }, { "cell_type": "code", "execution_count": 65, "id": "a0e31899-aeca-457d-bafa-c313f22180b7", "metadata": {}, "outputs": [], "source": [ "train_infos = read_jsonl(\"/home/tony/Data/Hoot/v4_t1_cer_50_long_train.json\")" ] }, { "cell_type": "code", "execution_count": 66, "id": "094c8548", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "882484\n" ] } ], "source": [ "print(len(train_infos))" ] }, { "cell_type": "code", "execution_count": 67, "id": "df1ffb98", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "399\n", "{'audio_filepath': '/app/suno/data/hoot/audios/genius/RX2YPLZfGaI.mp3', 'duration': 225, 'text': 'бля я не стремился быть чистеньким я не стремился быть нужным вам все что хотел это высверлить свое имя в их в головах загонялся писал заливался искал себе выход из ёбаной жизни амебы город за городом стопка за стопкой все льется рекой из окна небоскреба окурком сигары размажь себя вся твоя жизнь жидкая кашица пятиэтажная жидкая кашица полная злости и полная зависти нахуй всех нахуй всех нахуй всех нахуй всех нахуй всех нахуй всех нахуй всех нахуй всех', 'lyrics': '[Стих: СД]\\nБля...\\nЯ не стремился быть чистеньким, я не стремился быть нужным вам\\nВсе что хотел это высверлить,\\u2005свое\\u2005имя в их\\u2005в головах\\nЗагонялся, писал\\nЗаливался, искал себе выход\\u2005из ёбаной жизни амебы\\nГород за городом стопка за стопкой\\nВсе льется рекой из окна небоскреба\\nОкурком сигары размажь себя, вся твоя жизнь жидкая кашица Пятиэтажная жидкая кашица, полная злости и полная зависти\\nНахуй всех, нахуй всех\\nНахуй всех, нахуй всех\\nНахуй всех, нахуй всех\\nНахуй всех, нахуй всех', 'lang': 'ru', 'id': 'e16485e4-2241-45c9-b73d-9ec4782c23cf', 'genius_views': 784, 'youtube_views': 560}\n", "{'audio_filepath': '/app/suno/data/hoot/audios/genius/jSJmaKevLDY.mp3', 'duration': 219, 'text': 'lá dentro das trevas um homem que foge órfão da graça órfão de deus deus que me ajude não vou descansar ele há de ser meu ele há de ser meu na escuridão ele vai enquanto eu sigo o senhor quem caminha do lado dos justos terá o seu valor e quem cair o inferno verá terá o horror ah as estrelas são são incontáveis enchem as trevas de ordem e luz e sentinelas são mudas e sós céu que a todos conduz deus que a todos conduz e não se soltam do céu e nunca vão se soltar girando e girando e voltando outra vez para o mesmo lugar pois quem cair o inferno verá terá o horror e assim há de ser e assim foi escrito nas estrelas e páginas e quem for fraco e sair da luz irá pagar deus me dê forças hei de buscá lo há de ser meu eu não vou ceder eu juro na lei das estrelas do céu o inspetor se acha muito mas quem manda aqui sou eu meu teatro nunca fecha se a cortina não desceu sou gavroche creia em mim eu tô sempre aqui assim', 'lyrics': '[JAVERT]\\nLá, dentro das trevas\\nUm homem que foge\\nÓrfão da graça, órfão de Deus\\nDeus que me ajude\\nNão vou descansar\\nEle há de ser meu\\nEle há de ser meu\\n\\nNa escuridão, ele vai\\nEnquanto eu sigo o senhor\\nQuem caminha do lado dos justos\\nTerá o seu valor\\nE quem cair o inferno verá\\nTerá o horror\\nAh, as estrelas são\\nSão incontáveis\\nEnchem as trevas de ordem e luz\\nE sentinelas são\\nMudas e sós\\nCéu que a todos conduz\\nDeus que a todos conduz\\n\\nE não se soltam do céu\\nE nunca vão se soltar\\nGirando e girando e voltando outra vez\\nPara o mesmo lugar\\nPois quem cair o inferno verá\\nTerá o horror\\n\\nE assim há de ser\\nE assim foi escrito\\nNas estrelas e páginas\\nE quem for fraco e sair da luz\\nIrá pagar\\n\\nDeus me dê forças\\nHei de buscá-lo\\nHá de ser meu\\nEu não vou ceder\\nEu juro na lei\\nDas estrelas do céu\\n\\n\\n\\n[GAVROCHE]\\nO inspetor se acha muito\\nMas quem manda aqui sou eu\\nMeu teatro nunca fecha\\nSe a cortina não desceu\\nSou Gavroche, creia em mim\\nEu tô sempre aqui assim', 'lang': 'pt', 'id': '002e40aa-6183-40a1-b135-c3989403c77e', 'genius_views': 798, 'youtube_views': 83291}\n", "{'audio_filepath': '/app/suno/data/hoot/audios/genius/W9aOK2v5s5s.mp3', 'duration': 193, 'text': \"your sundress and bleach blonde hair adolescent vacant stare my peace of mind cure the thoughts swimming in my head imprompt decisions late nights in love forgone conclusions tell me what you're thinking of just leave me out to dry been lost here for the longest time i can't even win i'll just breathe you in it's the end of an era it's a fall from grace you said you needed time you said you needed space and no simple lie it's wishful thinking you and i felt like dreaming but you left me out to dry i've been living off of borrowed time i can't even win i'll just breathe you in it's the end of an era it's a fall from grace you said you needed time you said you needed space you can leave if you wanna someone will take my place so in the meantime can we fade away i used to know who you were now i know who you're not forgetting all that i learned such a freeing thought do i miss you when i have nothing else can i win when i'm losing myself was what we had even real so i keep looking back and i refuse to believe that it's the end of an era it's a fall from grace you said you needed time you said you needed space you can leave if you wanna someone will take my place so in the meantime can we fade away can we fade away\", 'lyrics': \"[Verse 1]\\nYour sundress and bleach blonde hair\\nAdolescent vacant stare\\nMy peace of mind\\nCure the thoughts swimming in my head\\nImprompt decisions\\nLate nights in love\\nForgone conclusions\\nTell me what you're thinking of\\n\\n[Pre-Chorus]\\nJust leave me out to dry\\nBeen lost here for the longest time\\nI can't even win\\nI'll just breathe you in\\n[Chorus]\\nIt's the end of an era\\nIt's a fall from grace\\nYou said you needed time\\nYou said you needed space\\n\\n[Verse 2]\\nAnd no simple lie\\nIt's wishful thinking\\nYou and I\\nFelt like dreaming\\n\\n[Pre-Chorus]\\nBut you left me out to dry\\nI've been living off of borrowed time\\nI can't even win\\nI'll just breathe you in\\n\\n[Chorus]\\nIt's the end of an era\\nIt's a fall from grace\\nYou said you needed time\\nYou said you needed space\\nYou can leave if you wanna\\nSomeone will take my place\\nSo in the meantime\\nCan we fade away?\\n\\n\\n[Bridge]\\nI used to know who you were\\nNow I know who you're not\\nForgetting all that I learned\\nSuch a freeing thought\\nDo I miss you when I have nothing else?\\nCan I win when I'm losing myself?\\nWas what we had even real?\\nSo I keep looking back\\nAnd I refuse to believe that...\\n\\n[Chorus]\\nIt's the end of an era\\nIt's a fall from grace\\nYou said you needed time\\nYou said you needed space\\nYou can leave if you wanna\\nSomeone will take my place\\nSo in the meantime\\nCan we fade away?\\nCan we fade away?\", 'lang': 'en', 'id': '024b610a-164d-4f00-b63d-5ea6be4434d9', 'genius_views': 1375, 'youtube_views': 46811}\n", "{'audio_filepath': '/app/suno/data/hoot/audios/genius/qEN97bJxulM.mp3', 'duration': 213, 'text': 'refrão sei que tu quer me dar muito amor e prazer dividir uma casinha e fazer um bebê se tu quiser me dar vou te satisfazer então promete ser minha que eu me entrego a você eu e ela os altos e baixos nós sustenta ela pequena maleável eu quase com ela é marrenta eu vou na dela ela me atenta e aqui dentro o bagulho esquenta ela senta em câmera lenta não vai faltar língua vai achar que eu sou poliglota essa roupa tá boa em você mas pelada tu é mais gostosa fiquei a mercê dessa moça mas sem pressa vou chegar dessa bossa que ela esboça com esse jogo de olhar te sinto recito repito esse rito não é difícil eu sou perito é instinto quando te excito se eu beber mais uma dose eu não sei qual vai ser o fim foi só falar levantou e pegou a garrafa de gim olhou pra mim e encheu meu copo perguntei quer ser feliz ela me disse topo vou te levar prum estrelas uisque arregado mentira vou fazer tu ver estrelas no meu quarto condição não tem mas gata curte o momento sei que tá faltando a cota mas tá sobrando talento eu e você o bagulho é doido nem a sombra acompanha é o magrinho malandreado e a gostosa piranha refrão sei que tu quer me dar muito amor e prazer dividir uma casinha e fazer um bebê se tu quiser me dar vou te satisfazer então promete ser minha que eu me entrego a você quando eu tô duro ela fortalece no din quando eu tô duro ela vem sentar em mim algo me diz que você foi feita pra mim tamo juntão bebê é nós até depois do fim ligação de alma inexplicável nosso vínculo vida me dá porrada mas você é o meu estímulo e eu que não sei nadar no teu mar eu mergulho nossa história é de outras vidas esse é mais um capítulo então vem vamos escrever sem pensar no amanhã a manha que só você tem pique mocinha e vilã tipo doce como hortelã e bolada que nem o islã só você me deixa louco e mantém minha mente sã se tu é bonnie e eu sou clyde tu é pé no chão e eu sou viagem é sensatez ato pensado e eu sou inconsequente selvagem eu sou movido a sacanagem e tu chega na carruagem salto alto e maquiagem refrão sei que tu quer me dar muito amor e prazer dividir uma casinha e fazer um bebê se tu quiser me dar vou te satisfazer então promete ser minha que eu me entrego a você', 'lyrics': 'Refrão\\n(Ian Matheus)\\nSei que tu quer me dar muito amor e prazer\\nDividir uma casinha e fazer um bebê\\n\\nSe tu quiser me dar, vou te satisfazer\\nEntão promete ser minha que eu me entrego a você\\n\\n\\n(Tchelo)\\nEu e ela, os altos e baixos nós sustenta\\nEla pequena maleável, eu quase com 1,90\\nEla é marrenta, eu vou na dela, ela me atenta\\nE aqui dentro o bagulho esquenta, ela senta em câmera lenta\\nNão vai faltar língua, vai achar que eu sou poliglota\\nEssa roupa tá boa em você, mas pelada tu é mais gostosa\\nFiquei a mercê dessa moça, mas sem pressa vou chegar\\nDessa bossa que ela esboça com esse jogo de olhar\\n\\nTe sinto, recito, repito esse rito\\nNão é difícil, eu sou perito, é instinto quando te excito\\nSe eu beber mais uma dose eu não sei qual vai ser o fim\\nFoi só falar, levantou e pegou a garrafa de gim\\n\\nOlhou pra mim e encheu meu copo\\nPerguntei \"quer ser feliz?\" ela me disse \"topo\"\\nVou te levar prum 5 estrelas, uisque arregado\\nMentira, vou fazer tu ver estrelas no meu quarto\\n\\nCondição não tem, mas gata curte o momento\\nSei que tá faltando a cota, mas tá sobrando talento\\nEu e você o bagulho é doido, nem a sombra acompanha\\nÉ o magrinho malandreado e a gostosa piranha\\n\\n\\nRefrão\\n(Ian Matheus)\\nSei que tu quer me dar muito amor e prazer\\nDividir uma casinha e fazer um bebê\\n\\n\\nSe tu quiser me dar, vou te satisfazer\\nEntão promete ser minha que eu me entrego a você\\n\\n\\n(Tchelo)\\nQuando eu tô duro ela fortalece no din\\nQuando eu tô duro ela vem sentar em mim\\nAlgo me diz que você foi feita pra mim\\nTamo juntão bebê, é nós até depois do fim\\n\\nLigação de alma, inexplicável nosso vínculo\\nVida me dá porrada, mas você é o meu estímulo\\nE eu que não sei nadar, no teu mar eu mergulho\\nNossa história é de outras vidas, esse é mais um capítulo (então vem)\\n\\nVamos escrever sem pensar no amanhã\\nA manha que só você tem, pique mocinha e vilã\\nTipo doce como hortelã e bolada que nem o islã\\nSó você me deixa louco e mantém minha mente sã\\n\\nSe tu é Bonnie e eu sou Clyde, tu é pé no chão e eu sou viagem\\nÉ sensatez, ato pensado e eu sou inconsequente, selvagem\\nEu sou movido a sacanagem e tu chega na carruagem, salto alto e maquiagem\\n\\n\\nRefrão\\n(Ian Matheus)\\nSei que tu quer me dar muito amor e prazer\\nDividir uma casinha e fazer um bebê\\n\\n\\nSe tu quiser me dar, vou te satisfazer\\nEntão promete ser minha que eu me entrego a você', 'lang': 'pt', 'id': '94a15a27-0a3b-4368-b5e2-a46de7275799', 'genius_views': 24, 'youtube_views': 1223}\n", "{'audio_filepath': '/app/suno/data/hoot/audios/genius/JSHq1dLEn6I.mp3', 'duration': 202, 'text': 'neste lugar colecionei todas as coisas que eu encontrei quem olha pode pensar que eu conquistei o que eu quis tudo o que achei dentro do mar é um tesouro com algo a contar sim ela é feliz tenho coisas por todos os cantos traquitanas e trecos demais eu parei de contar tenho tantos e daí tanto faz quero mais quero saber como os homens são quero aprender como eles dançam como é pular num só como eles dizem ah pé caudas são feitas só pra nadar mas quem tem pernas caminha dança e sempre corre atrás do que o coração quer onde tem sol onde tem ar onde é normal correr e dançar sinto que lá posso encontrar o meu lugar quero saber como é viver onde o sol brilha topo pagar o que custar pra acontecer lá pode crer vão me entender lá um pai sabe ver que a filha é ousada e cansada de obedecer lá eu vou saber tudo o que eu quiser minhas perguntas terão respostas como o fogo que traz calor traz também dor faço o que for pra me tornar parte do mundo acima do mar quem sabe lá posso encontrar o meu lugar', 'lyrics': '[ARIEL]\\nNeste lugar colecionei\\nTodas as coisas que eu encontrei\\nQuem olha pode pensar\\nQue eu conquistei o que eu quis\\n\\nTudo o que achei dentro do mar\\nÉ um tesouro com algo a contar\\nQuem olha em volta supõe:\\n\"Sim, ela é feliz!\"\\nTenho coisas por todos os cantos\\nTraquitanas e trecos demais\\nEu parei de contar, tenho tantos\\nE daí? Tanto faz, quero mais!\\n\\nQuero saber como os homens são\\nQuero aprender como eles dançam\\nComo é pular num só…\\nComo eles dizem?\\nAh, pé!\\n\\nCaudas são feitas só pra nadar\\nMas quem tem pernas caminha, dança\\nE sempre corre atrás\\nDo que o coração quer\\n\\nOnde tem sol\\nOnde tem ar\\nOnde é normal\\nCorrer e dançar\\nSinto que lá posso encontrar\\nO meu lugar\\n\\nQuero saber como é viver\\nOnde o sol brilha\\nTopo pagar o que custar pra acontecer\\n\\n\\nLá, pode crer, vão me entender\\nLá um pai sabe ver que a filha\\nÉ ousada e cansada de obedecer\\n\\nLá eu vou saber tudo o que eu quiser\\nMinhas perguntas terão respostas\\n\\nComo o fogo\\nQue traz calor\\nTraz também dor\\n\\nFaço o que for pra me tornar\\nParte do mundo acima do mar\\n\\nQuem sabe lá\\nPosso encontrar\\nO meu lugar', 'lang': 'pt', 'id': 'eb71c444-a557-41ab-b4e3-9de533a19746', 'genius_views': 113, 'youtube_views': 258703}\n", "{'audio_filepath': '/app/suno/data/hoot/audios/genius/-JouZ2iUSLI.mp3', 'duration': 197, 'text': 'podziurawiłem parę nike czasem tak się gubię mordo że się kurwa nie poznaję całe życie na przypale to przeze mnie bo policja nie próbuje mnie odnaleźć piszę to kiedy mam fazę potem wrzucamy do sieci i czekamy co się stanie ziomek mówi mi że siedzi wiem że wszyscy tu koledzy już czekają co się stanie wszyscy mamy chorą banie ale co trzeba jakoś iść żeby być tu myślę o tym kiedy palę przy stoliku myślę o tym kiedy wracam do warszawy ryju całe noce siedzę w pracy tamta miała być ostatnio z takim hajsem iść w pizdu nie wiesz co to dla mnie znaczy ale nigdy niе zostanę na chodniku czy na winklu wypuszczam dym chmura wygląda jak my kolejna jesień przychodzi razеm z nią zupełnie mordo nie dziś ona mi mówi żebym w końcu żył i chyba rozumie jak nikt ja sam nie rozumiem jak wszyscy ja sam nie rozumiem jak nikt wypuszczam dym chmura wygląda jak my kolejna jesień przychodzi razem z nią zupełnie mordo nie dziś ona mi mówi żebym w końcu żył i chyba rozumie jak nikt ja sam nie rozumiem jak wszyscy ja sam nie rozumiem jak ty gdzie te lepsze dni powiedziałem parę rzeczy które nie znaczyły nic obiecałem parę rzeczy które potem miałem w chuju i do dzisiaj jest mi za to wstyd wszystko dookoła mówi idź wszystko mówi mi bądźmy dorośli stawiam flachę na stolik jak przyjmuje gości ciekawe co będę pamiętał z tej nocy zajebane ale jestem mam stówę w kieszeni nie setkę mam lufę w kieszeni i cichą nadzieję że przede mną tylko najlepsze nie wiem gdzie moje miejsce ale kiedyś tam będę studiuje wzrokiem przestrzeń ciekawe co życie przyniesie wypuszczam dym chmura wygląda jak my kolejna jesień przychodzi razem z nią zupełnie mordo nie dziś ona mi mówi żebym w końcu żył i chyba rozumie jak nikt ja sam nie rozumiem jak wszyscy ja sam nie rozumiem jak nikt wypuszczam dym chmura wygląda jak my kolejna jesień przychodzi razem z nią zupełnie mordo nie dziś ona mi mówi żebym w końcu żył i chyba rozumie jak nikt ja sam nie rozumiem jak wszyscy ja sam nie rozumiem jak ty', 'lyrics': '[Zwrotka 1]\\nPodziurawiłem parę Nike\\nCzasem tak się gubię mordo\\nŻe się kurwa nie poznaję\\nCałe życie na przypale\\nTo przeze mnie\\nBo policja nie próbuje, mnie odnaleźć\\nPiszę to kiedy mam fazę\\nPotem wrzucamy do sieci, i czekamy co się stanie\\nZiomek mówi mi, że siedzi\\nWiem, że wszyscy tu koledzy już czekają co się stanie\\nWszyscy mamy chorą banie, ale co\\nTrzeba jakoś iść, żeby być tu\\nMyślę o tym, kiedy palę przy stoliku\\nMyślę o tym, kiedy wracam do Warszawy, ryju\\nCałe noce siedzę w pracy\\nTamta miała być ostatnio\\nZ takim hajsem iść w pizdu\\nNie wiesz co to dla mnie znaczy\\nAle nigdy niе zostanę na chodniku, czy na winklu\\n\\n[Refren]\\nWypuszczam dym, chmura wygląda jak my\\nKolejna jesień przychodzi, razеm z nią zupełnie\\nMordo nie dziś\\nOna mi mówi, żebym w końcu żył\\nI chyba rozumie jak nikt\\nJa sam nie rozumiem jak wszyscy\\nJa sam nie rozumiem jak (nikt)\\nWypuszczam dym, chmura wygląda jak my\\nKolejna jesień przychodzi, razem z nią zupełnie\\nMordo nie dziś\\nOna mi mówi, żebym w końcu żył\\nI chyba rozumie jak nikt\\nJa sam nie rozumiem jak wszyscy\\nJa sam nie rozumiem jak ty\\n\\n\\n[Zwrotka 2]\\nGdzie te lepsze dni?\\nPowiedziałem parę rzeczy\\nKtóre nie znaczyły nic\\nObiecałem parę rzeczy\\nKtóre potem miałem w chuju\\nI do dzisiaj jest mi za to wstyd\\nWszystko dookoła mówi idź\\nWszystko mówi mi, bądźmy dorośli\\nStawiam flachę na stolik, jak przyjmuje gości\\nCiekawe co będę pamiętał z tej nocy\\nZajebane, ale jestem\\nMam stówę w kieszeni, nie setkę\\nMam lufę w kieszeni, i cichą nadzieję\\nŻe przede mną tylko najlepsze\\nNie wiem gdzie moje miejsce\\nAle kiedyś tam będę\\nStudiuje wzrokiem przestrzeń\\nCiekawe co życie przyniesie\\n\\n\\n\\n[Refren]\\nWypuszczam dym, chmura wygląda jak my\\nKolejna jesień przychodzi, razem z nią zupełnie\\nMordo nie dziś\\nOna mi mówi, żebym w końcu żył\\nI chyba rozumie jak nikt\\nJa sam nie rozumiem jak wszyscy\\nJa sam nie rozumiem jak (nikt)\\nWypuszczam dym, chmura wygląda jak my\\nKolejna jesień przychodzi, razem z nią zupełnie\\nMordo nie dziś\\nOna mi mówi, żebym w końcu żył\\nI chyba rozumie jak nikt\\nJa sam nie rozumiem jak wszyscy\\nJa sam nie rozumiem jak ty', 'lang': 'pl', 'id': '981dfdff-ffbf-4f72-80fa-758e64470ff0', 'genius_views': 33, 'youtube_views': 23853}\n", "{'audio_filepath': '/app/suno/data/hoot/audios/genius/JPDaGyAvkOM.mp3', 'duration': 292, 'text': \"no wait before coppa strikes you need to make another cover of megalovania i'm talking like this we need a proper lyrical adaptation of this okay let me just follow that link real quick hmm this might be tricky but i think i can make something out of this\", 'lyrics': \"[Intro]\\nNo, wait\\nBefore coppa strikes, you NEED to make another cover of Megalovania\\nI'm talking like this\\nWe need a proper lyrical adaptation of this\\nOkay, let me just follow that link real quick\\nHmm, this might be tricky but I think I can make something out of this\", 'lang': 'en', 'id': '282aff77-4967-464f-bb20-2519c098df97', 'genius_views': 83, 'youtube_views': 92126}\n", "{'audio_filepath': '/app/suno/data/hoot/audios/genius/M90sp9OpSFw.mp3', 'duration': 285, 'text': \"okay dann battlen wir platz aus grinchi wenn das hier mal kein fest wird ich gehör' ins finale und du bist auch nicht zurecht hier konstante leistung in den battles mein freund du hast in vier battles dreimal auf die fresse bekomm'n\", 'lyrics': \"[Part]\\nOkay, dann battlen wir Platz 3 aus Grinchi, wenn das hier mal kein Fest wird\\nIch gehör' ins Finale und du bist auch nicht zurecht hier\\nKonstante Leistung in den Battles, mein Freund\\nDu hast in vier Battles dreimal auf die Fresse bekomm'n\", 'lang': 'de', 'id': '221e7f8d-5773-4f13-9da4-71f1f24a8c4a', 'genius_views': 427, 'youtube_views': 219192}\n", "{'audio_filepath': '/app/suno/data/hoot/audios/genius/xGtgaOJ6lo4.mp3', 'duration': 207, 'text': \"tu tá me diminuindo tu não soma assim você sempre me diz não e eu sou mais sim eu querendo falar de amor mas eu não sou marcinho eu disse para de fumar e ela disse só um macinho e ela me transformou no seu cigarro me usou até que eu virasse fumaça e eu que já não tinha nenhum filtro fui nos levando direto pra essa desgraça e a gente pode ser o que a gente quer quando mais novo sonhava em ser astronauta e como cê me levava pra outro mundo esse realizar de sonho ainda vai me fazer falta e eu te transformei na minha bebida te bebi até perder todos reflexos eu que só dizia coisas sem sentido fiquei só pra contar essa história sem nexo e a gente pode ser o que gente quer quando mais nova sonhava em ser bailarina como eu sempre dancei conforme a música ainda posso realizar todos esses teus sonhos mina ainda faço teu café bem forte com muito açúcar dessa vez chego mais cedo eu tô perdendo na sinuca cansei de ver teu choro virar maço de malboro depois da última briga quando eu vi tava no soro linda bailarina eu sei seguir teus passos sabe que eu não 'cinto' usando um cadarço vai ficar redondo feito um compasso conheço os seus limites por isso ultrapasso eu conheço seus limites por isso ultrapasso por isso ultrapasso eu não cinto usando cadarço conheço os limites por isso ultrapasso por isso ultrapasso e ela vem com carinho e eu agi igual um calo não sei porque mas sempre que tem que falar me calo o menos dela sempre mais e o meu mais sempre menos quis transformar minha loucura nos seus dias amenos ela as vezes ta que nem dalit cheia de não me toque repetido eu sou seu card vê se não me troque a minha vida é toda torta e ela tem toque as vezes me enche tanto me sinto bangkok falei que orgulho é opção ela diz que eu sou doente falei se orgulho é solução saí pra esfriar a mente menino prepotente já dizia minha coroa a vida é tão simples e nós que complica atoa de joão de barro à beija flor nós fomos papagaio eu sempre tinha que repetir e tu falava pra caralho já cansei de c'est la vie enquanto isso eu tô aqui não me apegando em crenças só pensando se ela ouvir se não surtir efeito assim eu me calo mas se fizer efeito vai ser igual a falta do roberto carlos e eu vou ser rei igual roberto carlos eu vou ser rei eu vou ser rei refrão então me diz quem é você onde tu tá agora o que tu tá fazendo essa hora a mina que eu conheci dizia que ela não chora me diz cadê você ou melhor quem você é agora o que tu tá fazendo essa hora a mina que eu conheci dizia que ela não chora a mina que eu conheci dizia que ela não chora o que tu tá fazendo essa hora quem você é agora a mina que eu conheci dizia que ela não chora\", 'lyrics': '(Gudi Tupa)\\nTu tá me diminuindo, tu não soma assim\\nVocê sempre me diz não e eu sou mais sim\\nEu querendo falar de amor, mas eu não sou Marcinho\\nEu disse \"para de fumar\" e ela disse \"só um macinho\"\\n\\nE ela me transformou no seu cigarro\\nMe usou até que eu virasse fumaça\\nE eu que já não tinha nenhum filtro\\nFui nos levando direto pra essa desgraça\\nE a gente pode ser o que a gente quer\\nQuando mais novo sonhava em ser astronauta\\nE como cê me levava pra outro mundo\\nEsse realizar de sonho ainda vai me fazer falta\\n\\nE eu te transformei na minha bebida\\nTe bebi até perder todos reflexos\\nEu que só dizia coisas sem sentido\\nFiquei só pra contar essa história sem nexo\\n\\nE a gente pode ser o que gente quer\\nQuando mais nova sonhava em ser bailarina\\nComo eu sempre dancei conforme a música\\nAinda posso realizar todos esses teus sonhos mina\\n\\nAinda faço teu café bem forte\\nCom muito açúcar\\nDessa vez chego mais cedo\\nEu tô perdendo na sinuca\\n\\nCansei de ver teu choro\\nVirar maço de malboro\\nDepois da última briga\\nQuando eu vi tava no soro\\n\\nLinda bailarina, eu sei seguir teus passos\\nSabe que eu não \\'cinto\\' usando um cadarço\\nVai ficar redondo feito um compasso\\nConheço os seus limites por isso ultrapasso\\n\\n\\nEu conheço seus limites por isso ultrapasso\\nPor isso ultrapasso\\nEu não \"cinto\" usando cadarço\\nConheço os limites por isso ultrapasso, por isso ultrapasso\\n\\n\\nE ela vem com carinho e eu agi igual um calo\\nNão sei porque, mas sempre que tem que falar me calo\\nO menos dela sempre mais e o meu mais sempre menos\\nQuis transformar minha loucura nos seus dias amenos\\n\\nEla as vezes ta que nem dalit, cheia de não me toque\\nRepetido eu sou seu card, vê se não me troque\\nA minha vida é toda torta e ela tem toque\\nAs vezes me enche tanto me sinto Bangkok\\n\\nFalei que orgulho é opção\\nEla diz que eu sou doente\\nFalei se orgulho é solução\\nSaí pra esfriar a mente\\n\\nMenino prepotente\\nJá dizia minha coroa\\n\"A vida é tão simples e nós que complica atoa\"\\n\\nDe João de barro à beija flor\\nNós fomos papagaio\\nEu sempre tinha que repetir\\nE tu falava pra caralho\\n\\n\\nJá cansei de C\\'est La Vie\\nEnquanto isso eu tô aqui\\nNão me apegando em crenças\\nSó pensando se ela ouvir\\n\\nSe não surtir efeito assim eu me calo\\nMas se fizer efeito vai ser igual a falta do Roberto Carlos\\nE eu vou ser rei igual Roberto Carlos\\nEu vou ser rei, eu vou ser rei\\n\\n\\nRefrão\\nEntão me diz quem é você, onde tu tá agora?\\nO que tu tá fazendo essa hora?\\nA mina que eu conheci dizia que ela não chora\\n\\nMe diz cadê você? Ou melhor\\nQuem você é agora?\\nO que tu tá fazendo essa hora?\\n\\nA mina que eu conheci dizia que ela não chora\\nA mina que eu conheci dizia que ela não chora\\nO que tu tá fazendo essa hora?\\n\\nQuem você é agora?\\nA mina que eu conheci dizia que ela não chora', 'lang': 'pt', 'id': 'd38c6f50-3f2a-4717-acb3-0f3b9fedb3b9', 'genius_views': 17, 'youtube_views': 1292}\n", "{'audio_filepath': '/app/suno/data/hoot/audios/genius/si6wZ00c5yw.mp3', 'duration': 260, 'text': \"i wasn't always like this uncommitted to my embrace but lately i feel like there's nothing else for me these days 'cause everybody's movin' with the seasons while i hold the door for all my demons that i can't resist no i wasn't meant for this so i'm trying to keep my head above the water but the waves bear more than life promised to offer me but maybe someday i'll resurface on my own yeah maybe someday where it hurts is where i'll grow oh tell me someday i won't fall apart like i used to but it's only monday can someday come soon i can never understand every thought inside my head and trying to unravel it was the reason why she left 'cause everybody wants someone to fight for but no one ever stays when they can't afford to rebuild and repair a story not like the fairytales so i'm trying to keep my head above the water but the waves bear more than life promised to offer me but maybe someday i'll resurface on my own yeah maybe someday where it hurts is where i'll grow oh tell me someday i won't fall apart like i used to but it's only monday can someday come soon ooh they make it look so easy to hold the stars maybe i'm too scarred ooh i've tried and tried but how did it get so hard to love my heart but maybe someday i'll resurface on my own yeah maybe someday where it hurts is where i'll grow oh tell me someday i won't fall apart like i used to but it's only monday can someday come soon it's only monday can sunday come soon\", 'lyrics': \"[Verse 1]\\nI wasn't always like this\\nUncommitted to my embrace\\nBut lately, I feel like\\nThere's nothing else for\\u2005me\\u2005these days\\n'Cause everybody's\\u2005movin' with the seasons\\nWhile I hold\\u2005the door for all my demons\\nThat I can't resist\\nNo, I\\u205fwasn't\\u205fmeant\\u205ffor this\\n\\n[Pre-Chorus]\\nSo I'm\\u205ftrying to keep\\u205fmy head above the water\\nBut the waves bear more than life promised to offer me\\n[Chorus]\\nBut maybe someday\\nI'll resurface on my own\\nYeah maybe someday\\nWhere it hurts is where I'll grow\\nOh tell me, someday\\nI won't fall apart like I used to\\nBut it's only Monday\\nCan someday come soon?\\n\\n[Verse 2]\\nI can never understand\\nEvery thought inside my head\\nAnd trying to unravel it\\nWas the reason why she left\\n'Cause everybody wants someone to fight for\\nBut no one ever stays when they can't afford\\nTo rebuild and repair a story not like the fairytales\\n\\n[Pre-Chorus]\\nSo I'm trying to keep my head above the water\\nBut the waves bear more than life promised to offer me\\n\\n[Chorus]\\nBut maybe someday\\nI'll resurface on my own\\nYeah maybe someday\\nWhere it hurts is where I'll grow\\nOh tell me, someday\\nI won't fall apart like I used to\\nBut it's only Monday\\nCan someday come soon?\\n\\n\\n[Bridge]\\nOoh, they make it look so easy\\nTo hold the stars\\nMaybe I'm too scarred\\nOoh, I've tried and tried\\nBut how did it get so hard\\nTo love my heart?\\n\\n[Chorus]\\nBut maybe someday\\nI'll resurface on my own\\nYeah maybe someday\\nWhere it hurts is where I'll grow\\nOh tell me, someday\\nI won't fall apart like I used to\\nBut it's only Monday\\nCan someday come soon?\\n\\n[Outro]\\nIt's only Monday, can Sunday come soon?\", 'lang': 'en', 'id': '82eae2f1-c6a5-47b4-a827-f676b27602f4', 'genius_views': 72, 'youtube_views': 344}\n", "{'audio_filepath': '/app/suno/data/hoot/audios/genius/N8rASZnK-ls.mp3', 'duration': 223, 'text': \"las jotas fam diguem que n'és de la teva ubicació on estàs no et veig fa temps diuen que ara tens una casa prop de japó que ja no preferies recordar tots els moments t'enrollo com uramaki que el ritme amb tu no freni estic calculant la money més al sud que nagasaki el ritme vull que em porti celebrem que ja no em porti nena y si tú quieres recorrer tokio en mi carro bebemo' par de trago' de eso' sé que es amargo toy esperando a que te quites el quimono y honey yo te como honey yo sé que si me voy contigo a japón atrás lo dejo to' por esta ocasión baby sé dónde te escondes tú solo dime entonces tu ubicación vamos pal' cuarto pa' prender un philie en la maleta me he traído ese papel de blunt tengo a mis enemigos haciéndose el harakiri porque pa' colmo me tenío que echar una novia recuerda soy el de la foto quien recorría contigo el monte fuji en moto a mí me gustas tú bebé y en eso hago hincapié iré y te buscaré todo ha cambiado no sé cómo estarás yo ahora me siento más centrado no te niego que lo que sentí por ti es fuerte está claro no nos quedemos con lo malo aposté por ti y apostar por ti es como apostar por el jugando a los dados nena me has dejao' tocado mirando hacia mi fin deseando que esto vuelva por sus manos nena y si tú quieres recorrer tokio en mi carro bebemo' par de trago' de eso' sé que es amargo toy esperando a que te quites el quimono y honey yo te como honey yo sé que si me voy contigo a japón atrás lo dejo to' por esta ocasión baby sé dónde te escondes tú solo dime entonces tu ubicación baby que saps que tu ets el yin i jo el yang que jo amb tu nena vull anar a japan i mirar te aquest somriure tots aquests petons al dia fins que ens fem grans i viure en un temple d'amor a mi no em torni a robar el cor perquè tu ets la meva geisha no vull viure amb la mateixa per menjar me el món nena vente al templo templo tú eres del mismo cuento cuento y yo tú samurai seré por ti yo cambiaré bésame y si te busco yo me pierdo en tu cama y luego me reencuentro cuando prendemo' en llamas el techo en la tana trinchao' con kattana y el pecho me lo he pateao' ya ma' diguem que n'és de la teva ubicació on estàs no et veig fa temps diuen que ara tens una casa prop de japó que ja no preferies recordar tots els moments nena y si tú quieres recorrer tokio en mi carro bebemo' par de trago' de eso' sé que es amargo toy esperando a que te quites el quimono y honey yo te como honey yo sé que si me voy contigo a japón atrás lo dejo to' por esta ocasión baby sé dónde te escondes tú solo dime entonces tu ubicación\", 'lyrics': '[Letra de \"Japan\" ft. 31 FAM]\\n\\n[Intro]\\nLas Jotas\\n31 FAM\\n\\n[Verso 1]\\nDiguem que n\\'és de la teva ubicació\\nOn estàs no et veig fa temps\\nDiuen que ara tens una casa prop de Japó\\nQue ja no preferies recordar tots els moments\\nT\\'enrollo com uramaki\\nQue el ritme amb tu no freni\\nEstic calculant la money\\nMés al sud que Nagasaki\\nEl ritme vull que em porti\\nCelebrem que ja no em porti\\n\\n[Estribillo]\\nNena, y si tú quieres recorrer Tokio en mi carro\\nBebemo\\' par de trago\\' de eso\\', sé que es amargo\\nToy esperando a que te quites el Quimono\\nY, honey, yo te como\\n\\n[Post-Estribillo]\\nHoney, yo sé que si me voy contigo a Japón\\nAtrás lo dejo to\\' por esta ocasión\\nBaby, sé dónde te escondes\\nTú solo dime entonces tu ubicación\\n\\n[Verso 2]\\nVamos pal\\' cuarto pa\\' prender un Philie\\nEn la maleta me he traído ese papel de blunt\\nTengo a mis enemigos haciéndose el Harakiri\\nPorque pa\\' colmo me tenío que echar una novia\\n\\nRecuerda, soy el de la foto\\nQuien recorría contigo el monte Fuji en moto\\nA mí me gustas tú, bebé\\nY en eso hago hincapié, iré y te buscaré\\n\\n\\nTodo ha cambiado\\nNo sé cómo estarás, yo ahora me siento más centrado\\nNo te niego que lo que sentí por ti es fuerte, está claro\\nNo nos quedemos con lo malo\\n\\nAposté por ti\\nY apostar por ti es como apostar por el 0 jugando a los dados\\nNena, me has dejao\\' tocado\\nMirando hacia mi fin, deseando que esto vuelva por sus manos\\n\\n[Estribillo]\\nNena, y si tú quieres recorrer Tokio en mi carro\\nBebemo\\' par de trago\\' de eso\\', sé que es amargo\\nToy esperando a que te quites el Quimono\\nY, honey, yo te como\\n\\n[Post-Estribillo]\\nHoney, yo sé que si me voy contigo a Japón\\nAtrás lo dejo to\\' por esta ocasión\\nBaby, sé dónde te escondes\\nTú solo dime entonces tu ubicación\\n\\n[Verso 3]\\nBaby, que saps que tu ets el Yin i jo el Yang\\nQue jo amb tu nena vull anar a Japan\\nI mirar-te aquest somriure tots aquests petons al dia\\nFins que ens fem grans\\n\\n\\nI viure en un temple d\\'amor\\nA mi no em torni a robar el cor\\nPerquè tu ets la meva Geisha\\nNo vull viure amb la mateixa per menjar-me el món\\n\\nNena, vente al templo, templo\\nTú eres del mismo cuento, cuento\\nY yo, tú samurai seré\\nPor ti yo cambiaré\\n\\nBésame, y si te busco, yo me pierdo en tu cama\\nY luego me reencuentro cuando prendemo\\' en llamas\\nEl techo en la tana, trinchao\\' con kattana\\nY el pecho me lo he pateao\\' ya, ma\\'\\n\\n[Puente]\\nDiguem que n\\'és de la teva ubicació\\nOn estàs no et veig fa temps\\nDiuen que ara tens una casa prop de Japó\\nQue ja no preferies recordar tots els moments\\n\\n[Estribillo]\\nNena, y si tú quieres recorrer Tokio en mi carro\\nBebemo\\' par de trago\\' de eso\\', sé que es amargo\\nToy esperando a que te quites el Quimono\\nY, honey, yo te como\\n\\n[Post-Estribillo]\\nHoney, yo sé que si me voy contigo a Japón\\nAtrás lo dejo to\\' por esta ocasión\\nBaby, sé dónde te escondes\\nTú solo dime entonces tu ubicación', 'lang': 'ca', 'id': '03a03aa6-fb6d-44e4-ab4d-5a3555424e26', 'genius_views': 117, 'youtube_views': 293258}\n", "11\n" ] } ], "source": [ "with open(\"tmp/genius_hq_silent_80_ids.txt\", \"r\") as fp:\n", " ids = set(fp.read().splitlines())\n", "print(len(ids))\n", "n_silent_in_train = 0\n", "for train_info in train_infos:\n", " if train_info[\"id\"] in ids:\n", " n_silent_in_train += 1\n", " print(train_info)\n", "print(n_silent_in_train)" ] }, { "cell_type": "code", "execution_count": null, "id": "755afe1f", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_env_dev", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }