{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d31d8850",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-25T20:32:05.165284Z",
     "start_time": "2024-03-25T20:32:05.163846Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7cd4c0dd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-25T20:32:07.681564Z",
     "start_time": "2024-03-25T20:32:05.166118Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tony/anaconda3/envs/suno_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "import math\n",
    "import torch\n",
    "import random\n",
    "import funcy\n",
    "import copy\n",
    "import gc\n",
    "import re\n",
    "import json\n",
    "import tempfile\n",
    "import collections\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import fasttext\n",
    "from joblib import Parallel, delayed\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from suno_utils.audio import Audio\n",
    "from suno_utils.utils.text import (\n",
    "    write_jsonl,\n",
    "    read_jsonl,\n",
    "    write_json,\n",
    "    read_json,\n",
    "    normalize_whitespace,\n",
    ")\n",
    "from suno_utils.utils.s3 import read_from_s3, check_s3_file_exists, open_from_s3\n",
    "from suno_utils.utils.tokenizers import tokenize\n",
    "from suno_utils.harvest.youtube.constants.text_lang import BASE_TO_FASTTEXT_REMAP\n",
    "\n",
    "TMP_DIR = os.path.join(os.getcwd(), \"tmp\")\n",
    "os.makedirs(TMP_DIR, exist_ok=True)\n",
    "\n",
    "# setup autoload\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9d50793",
   "metadata": {},
   "source": [
    "## genius_hq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8f428b8f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-25T20:32:07.801217Z",
     "start_time": "2024-03-25T20:32:07.683777Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tony/anaconda3/envs/suno_env/lib/python3.10/site-packages/Bio/pairwise2.py:278: BiopythonDeprecationWarning: Bio.pairwise2 has been deprecated, and we intend to remove it in a future release of Biopython. As an alternative, please consider using Bio.Align.PairwiseAligner as a replacement, and contact the Biopython developers if you still need the Bio.pairwise2 module.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from Bio import pairwise2\n",
    "\n",
    "from suno_utils.utils.metrics import get_cer\n",
    "from suno_utils.tasks.hoot import (\n",
    "    parse_lyrics,\n",
    "    legacy_parse_lyrics,\n",
    "    EMBEDDING_RATE as HOOT_EMBEDDING_RATE,\n",
    ")\n",
    "from suno_utils.utils.lyrics import remove_speakers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1cc806d",
   "metadata": {},
   "source": [
    "#### do alignment (will be cache in TMP_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0deec392",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-25T20:32:07.804005Z",
     "start_time": "2024-03-25T20:32:07.802247Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_name = \"ytm\"\n",
    "version_number = \"t30_v1\"\n",
    "aligned_output_folder = (\n",
    "    f\"/home/tony/Data/Hoot/alignments/{dataset_name}_{version_number}\"\n",
    ")\n",
    "output_large_jsonl_name = f\"{dataset_name}_hq_alignments_{version_number}.jsonl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3028a60c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-25T20:33:17.280611Z",
     "start_time": "2024-03-25T20:32:07.804977Z"
    }
   },
   "outputs": [],
   "source": [
    "# for n_step, jsonl_name in tqdm.tqdm(\n",
    "#     enumerate(sorted(os.listdir(aligned_output_folder)))\n",
    "# ):\n",
    "#     try:\n",
    "#         flat_aligned_lyrics = read_jsonl(\n",
    "#             os.path.join(aligned_output_folder, jsonl_name)\n",
    "#         )\n",
    "#         write_jsonl(\n",
    "#             flat_aligned_lyrics,\n",
    "#             os.path.join(TMP_DIR, output_large_jsonl_name),\n",
    "#             do_append=bool(n_step != 0),\n",
    "#         )\n",
    "#     except Exception as e:\n",
    "#         print(e)\n",
    "#         print(n_step, jsonl_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c43d1c8",
   "metadata": {},
   "source": [
    "#### Verify some stats after saving cache file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "23000343",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-25T20:33:27.442159Z",
     "start_time": "2024-03-25T20:33:17.281819Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/95 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.1 hours\n",
      "0.0 hours silence\n",
      "tokens median 59.0, 99% 77.4\n",
      "durations median 30.0, 90% 30.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# get summary (sample) stats\n",
    "with open(os.path.join(TMP_DIR, output_large_jsonl_name)) as f:\n",
    "    n_tot_rows = sum(1 for _ in f)\n",
    "\n",
    "test_tokenize = True\n",
    "n_max_sample = n_tot_rows\n",
    "if test_tokenize:\n",
    "    n_max_sample = int(round(n_max_sample / 180))\n",
    "sample_data = []\n",
    "tot_duration_s = 0\n",
    "tot_duration_silence_s = 0\n",
    "n_tokens_list = []\n",
    "durations_list = []\n",
    "unique_ids = set()\n",
    "n = 0\n",
    "with open(os.path.join(TMP_DIR, output_large_jsonl_name)) as f:\n",
    "    for line in tqdm.tqdm(f, total=n_tot_rows):\n",
    "        line = line.strip()\n",
    "        if len(line) == 0:\n",
    "            continue\n",
    "        k, v = json.loads(line)\n",
    "        if k in unique_ids:\n",
    "            continue\n",
    "        unique_ids.add(k)\n",
    "        for e in v:\n",
    "            if e[\"text\"] != \"\":\n",
    "                tot_duration_s += e[\"end_s\"] - e[\"start_s\"]\n",
    "                durations_list.append(e[\"end_s\"] - e[\"start_s\"])\n",
    "            if e[\"text\"] == \"\":\n",
    "                tot_duration_silence_s += e[\"end_s\"] - e[\"start_s\"]\n",
    "            if test_tokenize:\n",
    "                n_tokens_list.append(len(tokenize(e[\"text\"], max_tokens=512 * 8)))\n",
    "        if n % 500 == 0:\n",
    "            sample_data.append((k, v))\n",
    "        n += 1\n",
    "        if n == n_max_sample:\n",
    "            break\n",
    "# print(int(len(unique_ids)*n_tot_rows/n_max_sample), \"items\")\n",
    "print(round(tot_duration_s * n_tot_rows / n_max_sample / 60 / 60, 1), \"hours\")\n",
    "print(\n",
    "    round(tot_duration_silence_s * n_tot_rows / n_max_sample / 60 / 60, 1),\n",
    "    \"hours silence\",\n",
    ")\n",
    "print(\n",
    "    f\"tokens median {np.median(n_tokens_list):.1f}, 99% {np.quantile(n_tokens_list, 0.99):.1f}\"\n",
    ")\n",
    "print(\n",
    "    f\"durations median {np.median(durations_list):.1f}, 90% {np.quantile(durations_list, 0.90):.1f}\"\n",
    ")\n",
    "## last round\n",
    "# v3\n",
    "# 89881.3 hours\n",
    "# 0.0 hours silence\n",
    "# tokens median 174.0, 99% 764.5\n",
    "# durations median 77.9, 90% 120.0\n",
    "##################\n",
    "# v5 test\n",
    "# 87786.4 hours\n",
    "# 0.0 hours silence\n",
    "# tokens median 174.0, 99% 756.6\n",
    "# durations median 78.0, 90% 120.0\n",
    "##################\n",
    "# v6\n",
    "# genius\n",
    "# 87788.2 hours\n",
    "# 0.0 hours silence\n",
    "# tokens median 174.0, 99% 756.6\n",
    "# durations median 78.0, 90% 120.0\n",
    "# ytm\n",
    "# 37248.1 hours\n",
    "# 0.0 hours silence\n",
    "# tokens median 144.0, 99% 670.0\n",
    "# durations median 78.7, 90% 120.0\n",
    "# deezer\n",
    "# 26311.2 hours\n",
    "# 0.0 hours silence\n",
    "# tokens median 232.0, 99% 816.0\n",
    "# durations median 115.0, 90% 120.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8cdd3fd1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-25T20:33:27.603864Z",
     "start_time": "2024-03-25T20:33:27.443844Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tokens median 59.0, 99% 77.4\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiUAAAGdCAYAAADNHANuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkX0lEQVR4nO3dfXBU1eH/8c8mJBsyGFAjCcRg8BEReZCUNFBHHQORMlj7NCgUMCodhIzgVpGgJOHLKLRWik5Rxge0M5qCOhWtWDBGQ2tJQYJR6SiIoHHEBCoDwUQ3W/b8/uhvV9bsZvdCIGez79cMQ3PvuXvP/XB2/TR3k3UZY4wAAAC6WVJ3TwAAAECilAAAAEtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArNCruycQC7/fr/379+uMM86Qy+Xq7ukAAIAYGGN09OhRDRw4UElJ0b8PEhelZP/+/crNze3uaQAAgBPw+eef69xzz406Li5KyRlnnCHpfxeVkZFx2s7r8/n0+uuva8KECUpJSTlt541X5OUcmTlDXs6RmTPk5Uy0vFpaWpSbmxv873g0cVFKArdsMjIyTnspSU9PV0ZGBoszBuTlHJk5Q17OkZkz5OVMrHnF+tYL3ugKAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGAFSgkAALACpQQAAFiBUgIAAKxAKQEAAFZwXEr+/ve/a/LkyRo4cKBcLpfWr18f9Zja2lpdccUVcrvduvDCC/XMM8+cwFQBAEBP5riUtLa2asSIEVq1alVM4/ft26dJkybpmmuuUUNDg+bPn6/bbrtNmzZtcjxZAADQczn+QL6JEydq4sSJMY9fvXq1Bg8erIceekiSdOmll+rtt9/WH/7wBxUXFzs9PQAA6KFO+acE19XVqaioKGRbcXGx5s+fH/EYr9crr9cb/LqlpUXS/z6N0OfznZJ5hhM41+k8ZzwjL+fIzBnyco7MnCEvZ6Ll5TTHU15KmpqalJWVFbItKytLLS0t+uabb9S7d+8OxyxbtkxLlizpsP31119Xenr6KZtrJNXV1af9nPGMvJwjM2fIyzkyc4a8nImUV1tbm6PHOeWl5ESUlZXJ4/EEv25paVFubq4mTJigjIyMLj3XsMrI721xJxktzfdr/PjxSklJ6dLz9iSBDAN5Ld6eJK/fFTJmZ2Vi36qLtM6Oz6y+/LrTPKv44/P5VF1dnXDPyc5epwIiPcfiPbNYrj0Wsb4GxXte4ZzM+okmWl6BOx2xOuWlJDs7W83NzSHbmpublZGREfa7JJLkdrvldrs7bE9JSenyReI95oo65lSctyf5foZev6vDtkTPL9o68/pdCZ+RE4n2nIz1dSra/njMLJZrj4XTa4/XvMLpivUTy/HhHsPp457y31NSWFiompqakG3V1dUqLCw81acGAABxxHEp+frrr9XQ0KCGhgZJ//uR34aGBjU2Nkr6362XGTNmBMfPnj1be/fu1YIFC/TRRx/p0Ucf1fPPP68777yza64AAAD0CI5Lyfbt2zVq1CiNGjVKkuTxeDRq1CiVl5dLkr788stgQZGkwYMHa8OGDaqurtaIESP00EMP6cknn+THgQEAQAjH7ym5+uqrZYyJuD/cb2u9+uqr9e677zo9FQAASCB89g0AALACpQQAAFiBUgIAAKxAKQEAAFaglAAAACtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArEApAQAAVqCUAAAAK1BKAACAFSglAADACpQSAABgBUoJAACwAqUEAABYgVICAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGAFSgkAALACpQQAAFiBUgIAAKxAKQEAAFaglAAAACtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArEApAQAAVqCUAAAAK1BKAACAFSglAADACpQSAABgBUoJAACwAqUEAABYgVICAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGAFSgkAALACpQQAAFiBUgIAAKxAKQEAAFaglAAAACtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArEApAQAAVjihUrJq1Srl5eUpLS1NBQUF2rZtW6fjV65cqUsuuUS9e/dWbm6u7rzzTn377bcnNGEAANAzOS4l69atk8fjUUVFhXbs2KERI0aouLhYBw4cCDu+qqpKCxcuVEVFhT788EM99dRTWrdunRYtWnTSkwcAAD2H41KyYsUKzZo1SyUlJRo6dKhWr16t9PR0rVmzJuz4LVu2aNy4cZo6dary8vI0YcIE3XTTTVG/uwIAABJLLyeD29vbVV9fr7KysuC2pKQkFRUVqa6uLuwxY8eO1bPPPqtt27ZpzJgx2rt3r1577TVNnz494nm8Xq+8Xm/w65aWFkmSz+eTz+dzMuWo3Mkm8r4kEzwvIgtkGMgr8PfxEj3DSOvs+MwSPaNYBDJKtKw6e50KiJRJvGcWy7XHItbrj/e8wjmZ9RPrcdHWX6xcxpiY/8X379+vnJwcbdmyRYWFhcHtCxYs0ObNm7V169awxz3yyCO66667ZIzRf//7X82ePVuPPfZYxPNUVlZqyZIlHbZXVVUpPT091ukCAIBu1NbWpqlTp+rIkSPKyMiIOt7Rd0pORG1trR544AE9+uijKigo0J49ezRv3jwtXbpUixcvDntMWVmZPB5P8OuWlhbl5uZqwoQJMV2UE8MqN0Xc504yWprv1/jx45WSktKl5+1JAhkG8lq8PUlevytkzM7K4u6YmjUirbPjM6svv+40zyr++Hw+VVdXJ9xzsrPXqYBIz7F4zyyWa49FrK9B8Z5XOCezfqKJllfgTkesHJWSzMxMJScnq7m5OWR7c3OzsrOzwx6zePFiTZ8+Xbfddpsk6fLLL1dra6t+/etf695771VSUse3tbjdbrnd7g7bU1JSunyReI+5oo45FeftSb6fodfv6rAt0fOLts68flfCZ+REoj0nY32dirY/HjOL5dpj4fTa4zWvcLpi/cRyfLjHcPq4jt7ompqaqtGjR6umpia4ze/3q6amJuR2zvHa2to6FI/k5GRJkoM7RwAAoIdzfPvG4/Fo5syZys/P15gxY7Ry5Uq1traqpKREkjRjxgzl5ORo2bJlkqTJkydrxYoVGjVqVPD2zeLFizV58uRgOQEAAHBcSqZMmaKDBw+qvLxcTU1NGjlypDZu3KisrCxJUmNjY8h3Ru677z65XC7dd999+uKLL3TOOedo8uTJuv/++7vuKgAAQNw7oTe6lpaWqrS0NOy+2tra0BP06qWKigpVVFScyKkAAECC4LNvAACAFSglAADACpQSAABgBUoJAACwAqUEAABYgVICAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGAFSgkAALACpQQAAFiBUgIAAKxAKQEAAFaglAAAACtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArEApAQAAVqCUAAAAK1BKAACAFSglAADACpQSAABgBUoJAACwAqUEAABYgVICAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGAFSgkAALACpQQAAFiBUgIAAKxAKQEAAFaglAAAACtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArEApAQAAVqCUAAAAK1BKAACAFSglAADACpQSAABgBUoJAACwAqUEAABYgVICAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGCFEyolq1atUl5entLS0lRQUKBt27Z1Ov7w4cOaO3euBgwYILfbrYsvvlivvfbaCU0YAAD0TL2cHrBu3Tp5PB6tXr1aBQUFWrlypYqLi7Vr1y7179+/w/j29naNHz9e/fv314svvqicnBx99tln6tevX1fMHwAA9BCOS8mKFSs0a9YslZSUSJJWr16tDRs2aM2aNVq4cGGH8WvWrNGhQ4e0ZcsWpaSkSJLy8vJObtYAAKDHcVRK2tvbVV9fr7KysuC2pKQkFRUVqa6uLuwxr7zyigoLCzV37ly9/PLLOuecczR16lTdc889Sk5ODnuM1+uV1+sNft3S0iJJ8vl88vl8TqYclTvZRN6XZILnRWSBDAN5Bf4+XqJnGGmdHZ9ZomcUi0BGiZZVZ69TAZEyiffMYrn2WMR6/fGeVzgns35iPS7a+ouVyxgT87/4/v37lZOToy1btqiwsDC4fcGCBdq8ebO2bt3a4ZghQ4bo008/1bRp0zRnzhzt2bNHc+bM0R133KGKioqw56msrNSSJUs6bK+qqlJ6enqs0wUAAN2ora1NU6dO1ZEjR5SRkRF1vOPbN075/X71799fjz/+uJKTkzV69Gh98cUXevDBByOWkrKyMnk8nuDXLS0tys3N1YQJE2K6KCeGVW6KuM+dZLQ036/x48cHbz2ho0CGgbwWb0+S1+8KGbOzsrg7pmaNSOvs+Mzqy687zbOKPz6fT9XV1Qn3nOzsdSog0nMs3jOL5dpjEetrULznFc7JrJ9oouUVuNMRK0elJDMzU8nJyWpubg7Z3tzcrOzs7LDHDBgwQCkpKSG3ai699FI1NTWpvb1dqampHY5xu91yu90dtqekpHT5IvEec0UdcyrO25N8P0Ov39VhW6LnF22def2uhM/IiUR7Tsb6OhVtfzxmFsu1x8LptcdrXuF0xfqJ5fhwj+H0cR39SHBqaqpGjx6tmpqa4Da/36+ampqQ2znHGzdunPbs2SO/3x/ctnv3bg0YMCBsIQEAAInJ8e8p8Xg8euKJJ/SnP/1JH374oW6//Xa1trYGfxpnxowZIW+Evf3223Xo0CHNmzdPu3fv1oYNG/TAAw9o7ty5XXcVAAAg7jl+T8mUKVN08OBBlZeXq6mpSSNHjtTGjRuVlZUlSWpsbFRS0nddJzc3V5s2bdKdd96p4cOHKycnR/PmzdM999zTdVcBAADi3gm90bW0tFSlpaVh99XW1nbYVlhYqH/9618ncioAAJAg+OwbAABgBUoJAACwAqUEAABYgVICAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGAFSgkAALACpQQAAFiBUgIAAKxAKQEAAFaglAAAACtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArEApAQAAVqCUAAAAK1BKAACAFSglAADACpQSAABgBUoJAACwAqUEAABYgVICAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGAFSgkAALACpQQAAFiBUgIAAKxAKQEAAFaglAAAACtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArEApAQAAVqCUAAAAK1BKAACAFSglAADACpQSAABgBUoJAACwAqUEAABYgVICAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGAFSgkAALACpQQAAFjhhErJqlWrlJeXp7S0NBUUFGjbtm0xHbd27Vq5XC7dcMMNJ3JaAADQgzkuJevWrZPH41FFRYV27NihESNGqLi4WAcOHOj0uE8//VR33XWXrrzyyhOeLAAA6Lkcl5IVK1Zo1qxZKikp0dChQ7V69Wqlp6drzZo1EY85duyYpk2bpiVLluj8888/qQkDAICeqZeTwe3t7aqvr1dZWVlwW1JSkoqKilRXVxfxuP/7v/9T//79deutt+of//hH1PN4vV55vd7g1y0tLZIkn88nn8/nZMpRuZNN5H1JJnheRBbIMJBX4O/jJXqGkdbZ8ZklekaxCGSUaFl19joVECmTeM8slmuPRazXH+95hXMy6yfW46Ktv1i5jDEx/4vv379fOTk52rJliwoLC4PbFyxYoM2bN2vr1q0djnn77bd14403qqGhQZmZmbr55pt1+PBhrV+/PuJ5KisrtWTJkg7bq6qqlJ6eHut0AQBAN2pra9PUqVN15MgRZWRkRB3v6DslTh09elTTp0/XE088oczMzJiPKysrk8fjCX7d0tKi3NxcTZgwIaaLcmJY5aaI+9xJRkvz/Ro/frxSUlK69Lw9SSDDQF6LtyfJ63eFjNlZWdwdU7NGpHV2fGb15ded5lnFH5/Pp+rq6oR7Tnb2OhUQ6TkW75nFcu2xiPU1KN7zCudk1k800fIK3OmIlaNSkpmZqeTkZDU3N4dsb25uVnZ2dofxn3zyiT799FNNnjw5uM3v9//vxL16adeuXbrgggs6HOd2u+V2uztsT0lJ6fJF4j3mijrmVJy3J/l+hl6/q8O2RM8v2jrz+l0Jn5ETifacjPV1Ktr+eMwslmuPhdNrj9e8wumK9RPL8eEew+njOnqja2pqqkaPHq2amprgNr/fr5qampDbOQFDhgzRBx98oIaGhuCf66+/Xtdcc40aGhqUm5vraLIAAKDncnz7xuPxaObMmcrPz9eYMWO0cuVKtba2qqSkRJI0Y8YM5eTkaNmyZUpLS9OwYcNCju/Xr58kddgOAAASm+NSMmXKFB08eFDl5eVqamrSyJEjtXHjRmVlZUmSGhsblZTEL4oFAADOnNAbXUtLS1VaWhp2X21tbafHPvPMMydySgAA0MPxLQ0AAGAFSgkAALACpQQAAFiBUgIAAKxAKQEAAFaglAAAACtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArEApAQAAVqCUAAAAK1BKAACAFSglAADACpQSAABgBUoJAACwAqUEAABYgVICAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGAFSgkAALACpQQAAFiBUgIAAKxAKQEAAFaglAAAACtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArEApAQAAVqCUAAAAK1BKAACAFSglAADACpQSAABgBUoJAACwAqUEAABYgVICAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGAFSgkAALACpQQAAFiBUgIAAKxAKQEAAFaglAAAACtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArHBCpWTVqlXKy8tTWlqaCgoKtG3btohjn3jiCV155ZU688wzdeaZZ6qoqKjT8QAAIDE5LiXr1q2Tx+NRRUWFduzYoREjRqi4uFgHDhwIO762tlY33XST3nrrLdXV1Sk3N1cTJkzQF198cdKTBwAAPYfjUrJixQrNmjVLJSUlGjp0qFavXq309HStWbMm7PjnnntOc+bM0ciRIzVkyBA9+eST8vv9qqmpOenJAwCAnqOXk8Ht7e2qr69XWVlZcFtSUpKKiopUV1cX02O0tbXJ5/PprLPOijjG6/XK6/UGv25paZEk+Xw++Xw+J1OOyp1sIu9LMsHzIrJAhoG8An8fL9EzjLTOjs8s0TOKRSCjRMuqs9epgEiZxHtmsVx7LGK9/njPK5yTWT+xHhdt/cXKZYyJ+V98//79ysnJ0ZYtW1RYWBjcvmDBAm3evFlbt26N+hhz5szRpk2b9O9//1tpaWlhx1RWVmrJkiUdtldVVSk9PT3W6QIAgG7U1tamqVOn6siRI8rIyIg63tF3Sk7W8uXLtXbtWtXW1kYsJJJUVlYmj8cT/LqlpSX4XpRYLsqJYZWbIu5zJxktzfdr/PjxSklJ6dLz9iSBDAN5Ld6eJK/fFTJmZ2Vxd0zNGpHW2fGZ1Zdfd5pnFX98Pp+qq6sT7jnZ2etUQKTnWLxnFsu1xyLW16B4zyuck1k/0UTLK3CnI1aOSklmZqaSk5PV3Nwcsr25uVnZ2dmdHvv73/9ey5cv1xtvvKHhw4d3OtbtdsvtdnfYnpKS0uWLxHvMFXXMqThvT/L9DL1+V4dtiZ5ftHXm9bsSPiMnEu05GevrVLT98ZhZLNceC6fXHq95hdMV6yeW48M9htPHdfRG19TUVI0ePTrkTaqBN60efzvn+373u99p6dKl2rhxo/Lz8x1NEAAAJAbHt288Ho9mzpyp/Px8jRkzRitXrlRra6tKSkokSTNmzFBOTo6WLVsmSfrtb3+r8vJyVVVVKS8vT01NTZKkPn36qE+fPl14KQAAIJ45LiVTpkzRwYMHVV5erqamJo0cOVIbN25UVlaWJKmxsVFJSd99A+axxx5Te3u7fvGLX4Q8TkVFhSorK09u9gAAoMc4oTe6lpaWqrS0NOy+2trakK8//fTTEzkFAABIMHz2DQAAsAKlBAAAWIFSAgAArEApAQAAVqCUAAAAK1BKAACAFSglAADACpQSAABgBUoJAACwAqUEAABYgVICAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGAFSgkAALACpQQAAFiBUgIAAKxAKQEAAFaglAAAACtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArEApAQAAVqCUAAAAK1BKAACAFSglAADACpQSAABgBUoJAACwAqUEAABYgVICAACsQCkBAABWoJQAAAArUEoAAIAVKCUAAMAKlBIAAGAFSgkAALACpQQAAFiBUgIAAKxAKQEAAFaglAAAACtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArEApAQAAVqCUAAAAK1BKAACAFSglAADACpQSAABgBUoJAACwAqUEAABYgVICAACscEKlZNWqVcrLy1NaWpoKCgq0bdu2Tse/8MILGjJkiNLS0nT55ZfrtddeO6HJAgCAnstxKVm3bp08Ho8qKiq0Y8cOjRgxQsXFxTpw4EDY8Vu2bNFNN92kW2+9Ve+++65uuOEG3XDDDdq5c+dJTx4AAPQcjkvJihUrNGvWLJWUlGjo0KFavXq10tPTtWbNmrDjH374YV133XW6++67demll2rp0qW64oor9Mc//vGkJw8AAHqOXk4Gt7e3q76+XmVlZcFtSUlJKioqUl1dXdhj6urq5PF4QrYVFxdr/fr1Ec/j9Xrl9XqDXx85ckSSdOjQIfl8PidTjqrXf1sj7/MbtbX59dVXXyklJaVLz9uTBDIM5NXLl6RjflfImK+++qo7pmaNSOvs+MwSPaNY+Hw+tbW1JdxzsrPXqYBI6yfeM4vl2mMR6/Mr3vMK52TWTzTR8jp69KgkyRgT0+M5KiX/+c9/dOzYMWVlZYVsz8rK0kcffRT2mKamprDjm5qaIp5n2bJlWrJkSYftgwcPdjLdLjH1tJ8xvkXKK/Oh0zqNuBLILPPBbp0G4hzPsc6RT+dOdT5Hjx5V3759o45zVEpOl7KyspDvrvj9fh06dEhnn322XC5XJ0d2rZaWFuXm5urzzz9XRkbGaTtvvCIv58jMGfJyjsycIS9nouVljNHRo0c1cODAmB7PUSnJzMxUcnKympubQ7Y3NzcrOzs77DHZ2dmOxkuS2+2W2+0O2davXz8nU+1SGRkZLE4HyMs5MnOGvJwjM2fIy5nO8orlOyQBjt7ompqaqtGjR6umpia4ze/3q6amRoWFhWGPKSwsDBkvSdXV1RHHAwCAxOT49o3H49HMmTOVn5+vMWPGaOXKlWptbVVJSYkkacaMGcrJydGyZcskSfPmzdNVV12lhx56SJMmTdLatWu1fft2Pf744117JQAAIK45LiVTpkzRwYMHVV5erqamJo0cOVIbN24Mvpm1sbFRSUnffQNm7Nixqqqq0n333adFixbpoosu0vr16zVs2LCuu4pTxO12q6KiosOtJIRHXs6RmTPk5RyZOUNeznR1Xi4T68/pAAAAnEJ89g0AALACpQQAAFiBUgIAAKxAKQEAAFZI+FKybNky/eAHP9AZZ5yh/v3764YbbtCuXbtCxnz77beaO3euzj77bPXp00c///nPO/xCuETy2GOPafjw4cFfllNYWKi//e1vwf3k1bnly5fL5XJp/vz5wW1k9p3Kykq5XK6QP0OGDAnuJ6vwvvjiC/3qV7/S2Wefrd69e+vyyy/X9u3bg/uNMSovL9eAAQPUu3dvFRUV6eOPP+7GGXefvLy8DmvM5XJp7ty5klhj4Rw7dkyLFy/W4MGD1bt3b11wwQVaunRpyGfadMkaMwmuuLjYPP3002bnzp2moaHB/PjHPzaDBg0yX3/9dXDM7NmzTW5urqmpqTHbt283P/zhD83YsWO7cdbd65VXXjEbNmwwu3fvNrt27TKLFi0yKSkpZufOncYY8urMtm3bTF5enhk+fLiZN29ecDuZfaeiosJcdtll5ssvvwz+OXjwYHA/WXV06NAhc95555mbb77ZbN261ezdu9ds2rTJ7NmzJzhm+fLlpm/fvmb9+vXmvffeM9dff70ZPHiw+eabb7px5t3jwIEDIeururraSDJvvfWWMYY1Fs79999vzj77bPPqq6+affv2mRdeeMH06dPHPPzww8ExXbHGEr6UfN+BAweMJLN582ZjjDGHDx82KSkp5oUXXgiO+fDDD40kU1dX113TtM6ZZ55pnnzySfLqxNGjR81FF11kqqurzVVXXRUsJWQWqqKiwowYMSLsPrIK75577jE/+tGPIu73+/0mOzvbPPjgg8Fthw8fNm632/z5z38+HVO02rx588wFF1xg/H4/ayyCSZMmmVtuuSVk289+9jMzbdo0Y0zXrbGEv33zfUeOHJEknXXWWZKk+vp6+Xw+FRUVBccMGTJEgwYNUl1dXbfM0SbHjh3T2rVr1draqsLCQvLqxNy5czVp0qSQbCTWWDgff/yxBg4cqPPPP1/Tpk1TY2OjJLKK5JVXXlF+fr5++ctfqn///ho1apSeeOKJ4P59+/apqakpJLe+ffuqoKAgoXOTpPb2dj377LO65ZZb5HK5WGMRjB07VjU1Ndq9e7ck6b333tPbb7+tiRMnSuq6NWblpwR3F7/fr/nz52vcuHHB3zjb1NSk1NTUDh8ImJWVpaampm6YpR0++OADFRYW6ttvv1WfPn300ksvaejQoWpoaCCvMNauXasdO3bonXfe6bCPNRaqoKBAzzzzjC655BJ9+eWXWrJkia688krt3LmTrCLYu3evHnvsMXk8Hi1atEjvvPOO7rjjDqWmpmrmzJnBbAK/eTsg0XOTpPXr1+vw4cO6+eabJfF8jGThwoVqaWnRkCFDlJycrGPHjun+++/XtGnTJKnL1hil5Dhz587Vzp079fbbb3f3VKx3ySWXqKGhQUeOHNGLL76omTNnavPmzd09LSt9/vnnmjdvnqqrq5WWltbd07Fe4P95SdLw4cNVUFCg8847T88//7x69+7djTOzl9/vV35+vh544AFJ0qhRo7Rz506tXr1aM2fO7ObZ2e2pp57SxIkTNXDgwO6eitWef/55Pffcc6qqqtJll12mhoYGzZ8/XwMHDuzSNcbtm/+vtLRUr776qt566y2de+65we3Z2dlqb2/X4cOHQ8Y3NzcrOzv7NM/SHqmpqbrwwgs1evRoLVu2TCNGjNDDDz9MXmHU19frwIEDuuKKK9SrVy/16tVLmzdv1iOPPKJevXopKyuLzDrRr18/XXzxxdqzZw/rK4IBAwZo6NChIdsuvfTS4G2vQDbf/wmSRM/ts88+0xtvvKHbbrstuI01Ft7dd9+thQsX6sYbb9Tll1+u6dOn68477wx++G5XrbGELyXGGJWWluqll17Sm2++qcGDB4fsHz16tFJSUlRTUxPctmvXLjU2NqqwsPB0T9dafr9fXq+XvMK49tpr9cEHH6ihoSH4Jz8/X9OmTQv+bzKL7Ouvv9Ynn3yiAQMGsL4iGDduXIdfZbB7926dd955kqTBgwcrOzs7JLeWlhZt3bo1oXN7+umn1b9/f02aNCm4jTUWXltbW8iH7UpScnKy/H6/pC5cY13yttw4dvvtt5u+ffua2trakB8Ra2trC46ZPXu2GTRokHnzzTfN9u3bTWFhoSksLOzGWXevhQsXms2bN5t9+/aZ999/3yxcuNC4XC7z+uuvG2PIKxbH//SNMWR2vN/85jemtrbW7Nu3z/zzn/80RUVFJjMz0xw4cMAYQ1bhbNu2zfTq1cvcf//95uOPPzbPPfecSU9PN88++2xwzPLly02/fv3Myy+/bN5//33zk5/8JGF/JNgYY44dO2YGDRpk7rnnng77WGMdzZw50+Tk5AR/JPgvf/mLyczMNAsWLAiO6Yo1lvClRFLYP08//XRwzDfffGPmzJljzjzzTJOenm5++tOfmi+//LL7Jt3NbrnlFnPeeeeZ1NRUc84555hrr702WEiMIa9YfL+UkNl3pkyZYgYMGGBSU1NNTk6OmTJlSsjv2yCr8P7617+aYcOGGbfbbYYMGWIef/zxkP1+v98sXrzYZGVlGbfbba699lqza9eubppt99u0aZORFDYD1lhHLS0tZt68eWbQoEEmLS3NnH/++ebee+81Xq83OKYr1pjLmON+HRsAAEA3Sfj3lAAAADtQSgAAgBUoJQAAwAqUEgAAYAVKCQAAsAKlBAAAWIFSAgAArEApAQAAVqCUAAAAK1BKAACAFSglAADACpQSAABghf8HcdrK+0xWXtcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# max is prob around 1000\n",
    "pd.Series(n_tokens_list).hist(bins=50)\n",
    "print(\n",
    "    f\"tokens median {np.median(n_tokens_list):.1f}, 99% {np.quantile(n_tokens_list, 0.99):.1f}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f1611437",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-25T20:33:27.741763Z",
     "start_time": "2024-03-25T20:33:27.604938Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAHHCAYAAACRAnNyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6kklEQVR4nO3df3zOdf////ux2Y79YBobNpsfUea3ijTq9Hs4FZ394EOdqZzpzMopFfppkuiH6K2iVFQoIaWckuRXfhSiUn7TiczJxMY222F7fv/wdZyONtvMsR3Pze16ueyS1/P1PF6vx+PYOnbf6/U6jpfDGGMEAABgIT9fFwAAAHA+BBUAAGAtggoAALAWQQUAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLUIKgAAwFoEFcByDodDSUlJ7uXp06fL4XDot99+81lNAFBaCCoAiuz555/Xddddp8jISAUFBemKK67QkCFDdOTIkTxzc3Nz9eKLL6pu3boKCgpSs2bN9OGHHxZ5X8ePH9fAgQMVGRmp0NBQdejQQT/88MNF1b9x40Z169ZNYWFhqlSpkhISErR58+Z8565Zs0bXX3+9QkJCVKNGDQ0ePFgnT570mPP777+rR48eCgsLU6NGjfT555/n2c4nn3yiatWqKTU1tUg1rly5Uj179lRsbKyCgoJUo0YNdevWTatXry52nQV555131LBhQ/f3c9KkSUV+LFAqDACrSTIjR450L58+fdpkZmaa3NzcUq/llltuMffff7+ZMGGCefvtt80jjzxiwsLCTP369c3Jkyc95o4YMcJIMvfdd5956623TI8ePYwk8+GHHxa6n5ycHNOmTRsTGhpqkpKSzGuvvWYaNWpkKlWqZHbs2FGs2jdu3GiCgoLMFVdcYV5++WXz4osvmjp16piwsDCzbds2j7mbNm0yQUFB5qqrrjKTJ082Tz75pHE6naZbt24e8zp16mTi4uLMG2+8Ye644w7jdDrN3r173eszMzNN3bp1zZtvvlnkOqdOnWp69eplnnvuOfP222+bl156yTRv3tz4+fmZRYsWFavO85kyZYqRZG699Vbz1ltvmb///e9Gkhk3blyR6wVKGkEFsNyfg4pt5s6dmyeAHDhwwAQEBJjExET3WG5urrnhhhtMTEyMOX36dIHbnD17tpFk5syZ4x47fPiwueyyy0zfvn2LVedf//pXEx4eblJSUtxjBw8eNBUrVjS33HKLx9zu3bubqKgok5qa6h6bOnWqkWQWL15sjDEmIyPDOBwOs2LFCnd/devWNVOmTHE/ZvTo0aZFixYmJyenWDWflZ6ebqpXr266du16wXWeT0ZGhqlatarp0aOHx/gdd9xhQkNDzR9//HFRNQPewqkfoABJSUlyOBzasWOH7rzzTlWuXFmRkZF6+umnZYzR/v371atXL4WFhalGjRoaP358nm1kZWVp5MiRql+/vpxOp2JjYzVs2DBlZWXlmffwww8rMjJSlSpVUs+ePXXgwIE828vvGpXPPvtMPXr0UHR0tJxOp+rVq6fRo0crJyfH47Ht27dXkyZN9Ouvv6pDhw4KCQlRzZo19eKLLxb7OapTp46kM6dqzq3H5XJp0KBB7jGHw6EHHnhABw4c0Nq1awvc5ty5c1W9enXdcsst7rHIyEj17t1bn332WZ7nrihWrVqlzp07q2rVqu6xqKgotWvXTl988YX7dElaWpqWLFmiO++8U2FhYe65d911lypWrKiPP/5YknTq1CkZYxQeHu7u77LLLlNGRoakM6eFxo0bp1dffVV+fhf3UhsSEqLIyEiP57iodZ7PsmXLdPToUY/vkSQlJiYqPT1dCxcuvKiaAW8hqABF0KdPH+Xm5mrcuHFq3bq1nnvuOU2cOFFdunRRzZo19cILL6h+/fp69NFHtXLlSvfjcnNz1bNnT7388su66aabNGnSJN18882aMGGC+vTp47GPf/zjH5o4caISEhI0btw4BQQEqEePHkWqb/r06apYsaKGDh2qV199Vddcc42eeeYZjRgxIs/cY8eOqVu3bmrevLnGjx+vuLg4DR8+XIsWLSrSvowxSklJ0aFDh7Rq1SoNHjxY/v7+at++vXvOpk2bFBoaqoYNG3o89tprr3WvL8imTZt09dVX5/kFf+211yojI0M7duwoUq3nysrKUnBwcJ7xkJAQZWdna8uWLZKkn3/+WadPn1bLli095gUGBqpFixbu2sPDw1WvXj09//zz2rt3r2bOnKnNmze7exw2bJi6d++uv/zlLxdcq3QmiKSkpGjbtm164okntGXLFnXq1Mm9vqh1ns/Z9X9+/DXXXCM/P79CHw+UGh8f0QGsNnLkSCPJDBw40D12+vRpExMTYxwOh8e5/GPHjpng4GDTv39/99gHH3xg/Pz8zKpVqzy2e/bagNWrVxtjjNm8ebORZAYNGuQxr1+/fnlO/UybNs1I8rgWIiMjI0/t999/vwkJCTGnTp1yj7Vr185IMu+//757LCsry9SoUcPceuutRXpOkpOTjST3V0xMjJk9e7bHnB49epjLL788z2PT09ONJDNixIgC9xEaGmruvffePOMLFy40ksyXX35ZpFrP1bRpU3PllVd6nHbKysoytWrVMpLM3LlzjTHGzJkzx0gyK1euzLON22+/3dSoUcO9vHTpUhMeHu5+LoYMGWKMMWb16tUmODjY/Pbbbxdc51ldu3Z1bzcwMNDcf//9JjMz073+QurMT2JiovH39893XWRkpPl//+//Fbt2wJs4ogIUwT/+8Q/3v/39/dWyZUsZYzRgwAD3+GWXXaYGDRpoz5497rE5c+aoYcOGiouLU0pKivurY8eOks4cfpekf//735KkwYMHe+x3yJAhRarv3CMFJ06cUEpKim644QZlZGRo27ZtHnMrVqyoO++8070cGBioa6+91qPuglSpUkVLlizR559/rmeffVYRERF53mWSmZkpp9OZ57FBQUHu9QW52MfnZ9CgQdqxY4cGDBigX3/9VVu2bNFdd92l5ORkj22e/e/59n/uvjt27Kh9+/Zp3bp12rdvnyZMmKDc3FwNHjxYjzzyiGrXrq3JkycrLi5ODRo00JQpU4pc77hx4/TVV1/pnXfe0XXXXafs7GydPn3avf5C6sxPZmamAgMD811XlMcDpaWCrwsAyoJatWp5LFeuXFlBQUGKiIjIM3706FH38s6dO7V161ZFRkbmu93Dhw9Lkv7zn//Iz89P9erV81jfoEGDItX3yy+/6KmnntI333yjtLQ0j3V/fltsTEyMHA6Hx1h4eLh++umnIu0rMDBQnTt3liTdeOON6tSpk9q2batq1arpxhtvlHQmOOV3HcmpU6fc6wtysY/Pzz//+U/t379fL730kt577z1JZ057DBs2TGPGjFHFihU9tn2+/f953xUrVlTr1q3dy9OmTdOhQ4c0YsQIff3113rsscc0Y8YMORwO9evXTw0aNFCHDh0KrbdFixbuf9955526+uqrdffdd2vu3LnFqvPPgoODlZ2dne+6ojweKC0EFaAI/P39izQmnbmG46zc3Fw1bdpUr7zySr5zY2NjL7q248ePq127dgoLC9Ozzz6revXqKSgoSD/88IOGDx+u3NzcC677QrRp00ZRUVGaOXOmO6hERUVp2bJlMsZ4hKKzRy+io6ML3GZUVJR77rmK+vjzGTNmjB599FH98ssvqly5spo2baonnnhCknTllVe6933uvv68/4L2nZaWpieffFIvv/yyQkND9eGHH+q2227TzTffLEm67bbbNHPmzCIFlXMFBgaqZ8+eGjdunDIzMxUcHHxRdUpn+szJydHhw4dVrVo193h2draOHj1a7OcY8DaCClCC6tWrpx9//FGdOnXKcxTjXLVr11Zubq52797tcRRl+/bthe5j+fLlOnr0qD755BOPCzf37t17ccVfgFOnTnkcuWnRooXefvttbd26VY0aNXKPf/fdd+71BWnRooVWrVql3Nxcjwtqv/vuO4WEhLhDRXGEh4fr+uuvdy9//fXXiomJUVxcnCSpSZMmqlChgjZs2KDevXu752VnZ2vz5s0eY3/27LPPqm7durrjjjskSQcPHtRVV13lXh8dHX3eD5grTGZmpowxOnHihIKDgy+qTul/34MNGzbor3/9q3t8w4YNys3NLfR7BJQWrlEBSlDv3r31+++/a+rUqXnWZWZmKj09XZLUvXt3SdL//d//ecyZOHFiofs4e4Tk3CMi2dnZeuONN4pbdr7S09Pdb70917x583Ts2DGPd4/06tVLAQEBHjUYYzRlyhTVrFlTbdq0cY8nJydr27Ztcrlc7rHbbrtN//3vf/XJJ5+4x1JSUjRnzhzddNNN+V6XURyzZ8/W+vXrNWTIEHcgqly5sjp37qwZM2boxIkT7rkffPCBTp48qdtvvz3fbe3YsUOvvfaaXn31VXcorV69usc1Qlu3blWNGjUKrOns6cBzHT9+XPPmzVNsbKz76MeF1Hn2WqWUlBT3WMeOHVWlShVNnjzZY1+TJ09WSEhIkd9xBpQ0jqgAJejvf/+7Pv74Y/3zn//UsmXL1LZtW+Xk5Gjbtm36+OOPtXjxYrVs2VItWrRQ37599cYbbyg1NVVt2rTR0qVLtWvXrkL30aZNG4WHh6t///4aPHiwHA6HPvjgg2KfyjmfnTt3qnPnzurTp4/i4uLk5+enDRs2aMaMGapTp47+9a9/uefGxMRoyJAheumll+RyudSqVSt9+umnWrVqlWbOnOlx+unxxx/Xe++9p71797o/k+W2227Tddddp3vuuUe//vqrIiIi9MYbbygnJ0ejRo3yqOvuu+/O8/j8rFy5Us8++6wSEhJUtWpVrVu3TtOmTVO3bt08apfOnCJq06aN2rVrp4EDB+rAgQMaP368EhIS1K1bt3y3//DDD6tPnz7utyef7aNXr17u00uff/65vvjiiwKf5+7duysmJkatW7dWtWrVtG/fPk2bNk0HDx7U7Nmzi1Xn999/rw4dOmjkyJHu+0YFBwdr9OjRSkxM1O23366uXbtq1apVmjFjhsaMGaMqVaoUWCdQanz3hiPAfmffnnzkyBGP8f79+5vQ0NA889u1a2caN27sMZadnW1eeOEF07hxY+N0Ok14eLi55pprzKhRozw+UTQzM9MMHjzYVK1a1YSGhpqbbrrJ7N+/v0hvT169erW57rrrTHBwsImOjjbDhg0zixcvNpLMsmXLCqzvbD+1a9cu8Lk4cuSIGThwoImLizOhoaEmMDDQXHHFFWbIkCF5nh9jznwM/vPPP29q165tAgMDTePGjc2MGTPy3fef+zHGmD/++MMMGDDAVK1a1YSEhJh27dqZ9evX53n8rbfeaoKDg82xY8cKrH/Xrl0mISHBREREGKfTaeLi4szYsWNNVlZWvvNXrVpl2rRpY4KCgkxkZKRJTEw0aWlp+c5duHChqVixojl48GCedWPHjjXR0dEmKirKvPDCCwXWaIwxr732mrn++utNRESEqVChgomMjDQ33XRTvm9DLmqdy5YtO+8nHL/11lumQYMGJjAw0NSrV89MmDDBJ7dnAM7HYYyX/+wCgFJUvXp13XXXXXrppZd8XQqAEkBQAVBm/fLLL4qPj9eePXvyvFUcQPlAUAEAANbiXT8AAMBaBBUAAGAtggoAALAWQQUAAFirTH/gW25urg4ePKhKlSoV+PHkAADAHub/vx1EdHS0x20y8lOmg8rBgwe9clM3AABQ+vbv36+YmJgC55TpoFKpUiVJZxoNCwvzcTXe4XK59NVXXykhIUEBAQG+LqfE0W/5Rr/lG/2WfyXVc1pammJjY92/xwtSpoPK2dM9YWFh5SqohISEKCws7JL4H4F+yzf6Ld/ot/wr6Z6LctkGF9MCAABrEVQAAIC1CCoAAMBaBBUAAGAtggoAALAWQQUAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLV8HlR+//133XnnnapataqCg4PVtGlTbdiwwddlAQAAC/j0Xj/Hjh1T27Zt1aFDBy1atEiRkZHauXOnwsPDfVkWAACwhE+DygsvvKDY2FhNmzbNPVa3bl0fVgQAAGzi01M/CxYsUMuWLXX77berWrVquuqqqzR16lRflgQAACzi0yMqe/bs0eTJkzV06FA98cQTWr9+vQYPHqzAwED1798/z/ysrCxlZWW5l9PS0iSduQ21y+UqtbpL0tk+yks/haHf8o1+yzf6Lf9KqucL2Z7DGGO8uvcLEBgYqJYtW2rNmjXuscGDB2v9+vVau3ZtnvlJSUkaNWpUnvFZs2YpJCSkRGsFAADekZGRoX79+ik1NVVhYWEFzvXpEZWoqCg1atTIY6xhw4aaN29evvMff/xxDR061L2clpam2NhYJSQkFNpoWeFyubRkyRJ16dJFAQEBvi6nxNFv+Ua/5Rv9ln1NkhYXuN7pZzS6Za7Xez57RqQofBpU2rZtq+3bt3uM7dixQ7Vr1853vtPplNPpzDMeEBBQbn5oziqPPRWEfss3+i3f6LfsyspxFGmet3u+kG359GLahx9+WOvWrdPzzz+vXbt2adasWXrrrbeUmJjoy7IAAIAlfBpUWrVqpfnz5+vDDz9UkyZNNHr0aE2cOFF33HGHL8sCAACW8OmpH0m68cYbdeONN/q6DAAAYCGff4Q+AADA+RBUAACAtQgqAADAWgQVAABgLYIKAACwFkEFAABYi6ACAACsRVABAADWIqgAAABrEVQAAIC1CCoAAMBaBBUAAGAtggoAALAWQQUAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLUIKgAAwFoEFQAAYC2CCgAAsBZBBQAAWIugAgAArEVQAQAA1iKoAAAAaxFUAACAtQgqAADAWgQVAABgLYIKAACwFkEFAABYi6ACAACsRVABAADWIqgAAABrEVQAAIC1CCoAAMBaBBUAAGAtggoAALAWQQUAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLUIKgAAwFoEFQAAYC2CCgAAsBZBBQAAWIugAgAArEVQAQAA1vJpUElKSpLD4fD4iouL82VJAADAIhV8XUDjxo319ddfu5crVPB5SQAAwBI+TwUVKlRQjRo1fF0GAACwkM+Dys6dOxUdHa2goCDFx8dr7NixqlWrVr5zs7KylJWV5V5OS0uTJLlcLrlcrlKpt6Sd7aO89FMY+i3f6Ld8o9+yz+lvCl7vd2a9t3u+kO05jDEFV1mCFi1apJMnT6pBgwZKTk7WqFGj9Pvvv2vLli2qVKlSnvlJSUkaNWpUnvFZs2YpJCSkNEoGAAAXKSMjQ/369VNqaqrCwsIKnOvToPJnx48fV+3atfXKK69owIABedbnd0QlNjZWKSkphTZaVrhcLi1ZskRdunRRQECAr8spcfRbvtFv+Ua/ZV+TpMUFrnf6GY1umev1ntPS0hQREVGkoOLzUz/nuuyyy3TllVdq165d+a53Op1yOp15xgMCAsrND81Z5bGngtBv+Ua/5Rv9ll1ZOY4izfN2zxeyLas+R+XkyZPavXu3oqKifF0KAACwgE+DyqOPPqoVK1bot99+05o1a/S3v/1N/v7+6tu3ry/LAgAAlvDpqZ8DBw6ob9++Onr0qCIjI3X99ddr3bp1ioyM9GVZAADAEj4NKh999JEvdw8AACxn1TUqAAAA5yKoAAAAaxFUAACAtQgqAADAWgQVAABgLYIKAACwFkEFAABYi6ACAACsRVABAADWIqgAAABrEVQAAIC1CCoAAMBaBBUAAGAtggoAALAWQQUAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLUIKgAAwFoEFQAAYC2CCgAAsBZBBQAAWIugAgAArEVQAQAA1iKoAAAAaxFUAACAtQgqAADAWgQVAABgLYIKAACwFkEFAABYi6ACAACsRVABAADWIqgAAABrEVQAAIC1CCoAAMBaBBUAAGAtggoAALAWQQUAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLUIKgAAwFoEFQAAYC2CCgAAsBZBBQAAWIugAgAArGVNUBk3bpwcDoeGDBni61IAAIAlrAgq69ev15tvvqlmzZr5uhQAAGARnweVkydP6o477tDUqVMVHh7u63IAAIBFKvi6gMTERPXo0UOdO3fWc889V+DcrKwsZWVluZfT0tIkSS6XSy6Xq0TrLC1n+ygv/RSGfss3+i3f6Lfsc/qbgtf7nVnv7Z4vZHsOY0zBVZagjz76SGPGjNH69esVFBSk9u3bq0WLFpo4cWK+85OSkjRq1Kg847NmzVJISEgJVwsAALwhIyND/fr1U2pqqsLCwgqc67Ogsn//frVs2VJLlixxX5tSWFDJ74hKbGysUlJSCm20rHC5XFqyZIm6dOmigIAAX5dT4ui3fKPf8o1+y74mSYsLXO/0MxrdMtfrPaelpSkiIqJIQcVnp342btyow4cP6+qrr3aP5eTkaOXKlXrttdeUlZUlf39/j8c4nU45nc482woICCg3PzRnlceeCkK/5Rv9lm/0W3Zl5TiKNM/bPV/ItnwWVDp16qSff/7ZY+yee+5RXFychg8fniekAACAS4/PgkqlSpXUpEkTj7HQ0FBVrVo1zzgAALg0+fztyQAAAOfj87cnn2v58uW+LgEAAFiEIyoAAMBaBBUAAGAtggoAALAWQQUAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLUIKgAAwFoEFQAAYC2CCgAAsBZBBQAAWIugAgAArEVQAQAA1iKoAAAAaxFUAACAtQgqAADAWgQVAABgLYIKAACwFkEFAABYi6ACAACsRVABAADWKlZQ2b9/vw4cOOBe/v777zVkyBC99dZbXisMAACgWEGlX79+WrZsmSTp0KFD6tKli77//ns9+eSTevbZZ71aIAAAuHQVK6hs2bJF1157rSTp448/VpMmTbRmzRrNnDlT06dP92Z9AADgElasoOJyueR0OiVJX3/9tXr27ClJiouLU3JysveqAwAAl7RiBZXGjRtrypQpWrVqlZYsWaJu3bpJkg4ePKiqVat6tUAAAHDpKlZQeeGFF/Tmm2+qffv26tu3r5o3by5JWrBggfuUEAAAwMWqUJwHtW/fXikpKUpLS1N4eLh7fODAgQoNDfVacQAA4NJWrCMqHTt21IkTJzxCiiRVqVJFffr08UphAAAAxQoqy5cvV3Z2dp7xU6dOadWqVRddFAAAgHSBp35++ukn979//fVXHTp0yL2ck5OjL7/8UjVr1vRedQAA4JJ2QUGlRYsWcjgccjgc6tixY571wcHBmjRpkteKAwAAl7YLCip79+6VMUaXX365vv/+e0VGRrrXBQYGqlq1avL39/d6kQAA4NJ0QUGldu3akqTc3NwSKQYAAOBcxXp7siTt3LlTy5Yt0+HDh/MEl2eeeeaiCwMAAChWUJk6daoeeOABRUREqEaNGnI4HO51DoeDoAIAALyiWEHlueee05gxYzR8+HBv1wMAAOBWrM9ROXbsmG6//XZv1wIAAOChWEHl9ttv11dffeXtWgAAADwU69RP/fr19fTTT2vdunVq2rSpAgICPNYPHjzYK8UBAIBLW7GCyltvvaWKFStqxYoVWrFihcc6h8NBUAEAAF5RrKCyd+9eb9cBAACQR7GuUQEAACgNxTqicu+99xa4/t133y1WMQAAAOcqVlA5duyYx7LL5dKWLVt0/PjxfG9WCAAAUBzFCirz58/PM5abm6sHHnhA9erVu+iiAAAAJC9eo+Ln56ehQ4dqwoQJRX7M5MmT1axZM4WFhSksLEzx8fFatGiRt0oCAABlnFcvpt29e7dOnz5d5PkxMTEaN26cNm7cqA0bNqhjx47q1auXfvnlF2+WBQAAyqhinfoZOnSox7IxRsnJyVq4cKH69+9f5O3cdNNNHstjxozR5MmTtW7dOjVu3Lg4pQEAgHKkWEFl06ZNHst+fn6KjIzU+PHjC31H0Pnk5ORozpw5Sk9PV3x8fLG2AQAAypdiBZVly5Z5rYCff/5Z8fHxOnXqlCpWrKj58+erUaNG+c7NyspSVlaWezktLU3SmXcduVwur9XkS2f7KC/9FIZ+yzf6Ld/ot+xz+puC1/udWe/tni9kew5jTMFVFuDIkSPavn27JKlBgwaKjIy84G1kZ2dr3759Sk1N1dy5c/X2229rxYoV+YaVpKQkjRo1Ks/4rFmzFBIScuENAACAUpeRkaF+/fopNTVVYWFhBc4tVlBJT0/XQw89pPfff1+5ubmSJH9/f911112aNGnSRYWGzp07q169enrzzTfzrMvviEpsbKxSUlIKbbSscLlcWrJkibp06ZLnZo/lEf2Wb/RbvtFv2dckaXGB651+RqNb5nq957S0NEVERBQpqBT7YtoVK1bo888/V9u2bSVJ3377rQYPHqxHHnlEkydPLs5mJZ35PJZzw8i5nE6nnE5nnvGAgIBy80NzVnnsqSD0W77Rb/lGv2VXVo6jSPO83fOFbKtYQWXevHmaO3eu2rdv7x7761//quDgYPXu3bvIQeXxxx9X9+7dVatWLZ04cUKzZs3S8uXLtXhxwQkPAABcGooVVDIyMlS9evU849WqVVNGRkaRt3P48GHdddddSk5OVuXKldWsWTMtXrxYXbp0KU5ZAACgnClWUImPj9fIkSP1/vvvKygoSJKUmZmpUaNGXdBbi995553i7B4AAFwiihVUJk6cqG7duikmJkbNmzeXJP34449yOp366quvvFogAAC4dBUrqDRt2lQ7d+7UzJkztW3bNklS3759dccddyg4ONirBQIAgEtXsYLK2LFjVb16dd13330e4++++66OHDmi4cOHe6U4AABwaSvWTQnffPNNxcXF5Rlv3LixpkyZctFFAQAASMUMKocOHVJUVFSe8cjISCUnJ190UQAAAFIxg0psbKxWr16dZ3z16tWKjo6+6KIAAACkYl6jct9992nIkCFyuVzq2LGjJGnp0qUaNmyYHnnkEa8WCAAALl3FCiqPPfaYjh49qkGDBik7O1uSFBQUpOHDh+vxxx/3aoEAAODSVayg4nA49MILL+jpp5/W1q1bFRwcrCuuuCLf+/AAAAAUV7GCylkVK1ZUq1atvFULAACAh2JdTAsAAFAaCCoAAMBaBBUAAGAtggoAALAWQQUAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLUIKgAAwFoEFQAAYC2CCgAAsBZBBQAAWIugAgAArEVQAQAA1iKoAAAAaxFUAACAtQgqAADAWgQVAABgLYIKAACwFkEFAABYi6ACAACsRVABAADWIqgAAABrEVQAAIC1CCoAAMBaBBUAAGAtggoAALAWQQUAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLUIKgAAwFoEFQAAYC2CCgAAsBZBBQAAWIugAgAArOXToDJ27Fi1atVKlSpVUrVq1XTzzTdr+/btviwJAABYxKdBZcWKFUpMTNS6deu0ZMkSuVwuJSQkKD093ZdlAQAAS1Tw5c6//PJLj+Xp06erWrVq2rhxo/7yl7/4qCoAAGALnwaVP0tNTZUkValSJd/1WVlZysrKci+npaVJklwul1wuV8kXWArO9lFe+ikM/ZZv9Fu+0W/Z5/Q3Ba/3O7Pe2z1fyPYcxpiCqywlubm56tmzp44fP65vv/023zlJSUkaNWpUnvFZs2YpJCSkpEsEAABekJGRoX79+ik1NVVhYWEFzrUmqDzwwANatGiRvv32W8XExOQ7J78jKrGxsUpJSSm00bLC5XJpyZIl6tKliwICAnxdTomj3/KNfss3+i37miQtLnC9089odMtcr/eclpamiIiIIgUVK079PPjgg/riiy+0cuXK84YUSXI6nXI6nXnGAwICys0PzVnlsaeC0G/5Rr/lG/2WXVk5jiLN83bPF7ItnwYVY4weeughzZ8/X8uXL1fdunV9WQ4AALCMT4NKYmKiZs2apc8++0yVKlXSoUOHJEmVK1dWcHCwL0sDAAAW8OnnqEyePFmpqalq3769oqKi3F+zZ8/2ZVkAAMASPj/1AwAAcD7c6wcAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLUIKgAAwFoEFQAAYC2CCgAAsBZBBQAAWIugAgAArEVQAQAA1iKoAAAAaxFUAACAtQgqAADAWgQVAABgLYIKAACwFkEFAABYi6ACAACsRVABAADWIqgAAABrEVQAAIC1CCoAAMBaBBUAAGAtggoAALAWQQUAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLUIKgAAwFoEFQAAYC2CCgAAsBZBBQAAWIugAgAArEVQAQAA1iKoAAAAaxFUAACAtQgqAADAWgQVAABgLYIKAACwFkEFAABYi6ACAACsRVABAADWIqgAAABrEVQAAIC1CCoAAMBaBBUAAGAtggoAALCWT4PKypUrddNNNyk6OloOh0OffvqpL8sBAACW8WlQSU9PV/PmzfX666/7sgwAAGCpCr7ceffu3dW9e3dflgAAACzm06ByobKyspSVleVeTktLkyS5XC65XC5fleVVZ/soL/0Uhn7LN/ot3+i37HP6m4LX+51Z7+2eL2R7DmNMwVWWEofDofnz5+vmm28+75ykpCSNGjUqz/isWbMUEhJSgtUBAABvycjIUL9+/ZSamqqwsLAC55apoJLfEZXY2FilpKQU2mhxNElaXOicLUldvbpPl8ulJUuWqEuXLgoICPDqtm1Ev+Ub/ZZv9Fv2FfZ7zulnNLplrtd7TktLU0RERJGCSpk69eN0OuV0OvOMBwQElMgPTVaOo9A5JfXDWlI92Yp+yzf6Ld/ot+wqyu85yfs9X8i2+BwVAABgLZ8eUTl58qR27drlXt67d682b96sKlWqqFatWj6sDAAA2MCnQWXDhg3q0KGDe3no0KGSpP79+2v69Ok+qgoAANjCp0Glffv2suRaXgAAYCGuUQEAANYiqAAAAGsRVAAAgLUIKgAAwFoEFQAAYC2CCgAAsBZBBQAAWIugAgAArEVQAQAA1iKoAAAAaxFUAACAtQgqAADAWgQVAABgLYIKAACwFkEFAABYi6ACAACsRVABAADWIqgAAABrEVQAAIC1CCoAAMBaBBUAAGAtggoAALAWQQUAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLUIKgAAwFoEFQAAYC2CCgAAsBZBBQAAWIugAgAArEVQAQAA1iKoAAAAaxFUAACAtQgqAADAWgQVAABgLYIKAACwFkEFAABYi6ACAACsRVABAADWIqgAAABrEVQAAIC1CCoAAMBaBBUAAGAtggoAALAWQQUAAFjLiqDy+uuvq06dOgoKClLr1q31/fff+7okAABgAZ8HldmzZ2vo0KEaOXKkfvjhBzVv3lxdu3bV4cOHfV0aAADwMZ8HlVdeeUX33Xef7rnnHjVq1EhTpkxRSEiI3n33XV+XBgAAfMynQSU7O1sbN25U586d3WN+fn7q3Lmz1q5d68PKAACADSr4cucpKSnKyclR9erVPcarV6+ubdu25ZmflZWlrKws93Jqaqok6Y8//pDL5fJ6fRVOpxc65+jRo17dp8vlUkZGho4ePaqAgACvbttG9Fu+0W/5Rr9lX2G/5yrkGmVk5Hq95xMnTkiSjDGFzvVpULlQY8eO1ahRo/KM161b1wfVnBEx3me7BgCgxPUrwW2fOHFClStXLnCOT4NKRESE/P399d///tdj/L///a9q1KiRZ/7jjz+uoUOHupdzc3P1xx9/qGrVqnI4HCVeb2lIS0tTbGys9u/fr7CwMF+XU+Lot3yj3/KNfsu/kurZGKMTJ04oOjq60Lk+DSqBgYG65pprtHTpUt18882SzoSPpUuX6sEHH8wz3+l0yul0eoxddtllpVBp6QsLC7tk/keQ6Le8o9/yjX7Lv5LoubAjKWf5/NTP0KFD1b9/f7Vs2VLXXnutJk6cqPT0dN1zzz2+Lg0AAPiYz4NKnz59dOTIET3zzDM6dOiQWrRooS+//DLPBbYAAODS4/OgIkkPPvhgvqd6LkVOp1MjR47Mc4qrvKLf8o1+yzf6Lf9s6NlhivLeIAAAAB/w+SfTAgAAnA9BBQAAWIugAgAArEVQAQAA1iKo+MDKlSt10003KTo6Wg6HQ59++mmeOVu3blXPnj1VuXJlhYaGqlWrVtq3b1/pF+slhfV88uRJPfjgg4qJiVFwcLD7Ttpl0dixY9WqVStVqlRJ1apV080336zt27d7zDl16pQSExNVtWpVVaxYUbfeemueT2guKwrr948//tBDDz2kBg0aKDg4WLVq1dLgwYPd9+oqa4ry/T3LGKPu3buf9//zsqKoPa9du1YdO3ZUaGiowsLC9Je//EWZmZk+qPjiFKXfQ4cO6e9//7tq1Kih0NBQXX311Zo3b56PKr44kydPVrNmzdwf6hYfH69Fixa51/v69Yqg4gPp6elq3ry5Xn/99XzX7969W9dff73i4uK0fPly/fTTT3r66acVFBRUypV6T2E9Dx06VF9++aVmzJihrVu3asiQIXrwwQe1YMGCUq704q1YsUKJiYlat26dlixZIpfLpYSEBKWn/+/mXw8//LA+//xzzZkzRytWrNDBgwd1yy23+LDq4ius34MHD+rgwYN6+eWXtWXLFk2fPl1ffvmlBgwY4OPKi6co39+zJk6cWC5u71GUnteuXatu3bopISFB33//vdavX68HH3xQfn5l79dMUfq96667tH37di1YsEA///yzbrnlFvXu3VubNm3yYeXFExMTo3Hjxmnjxo3asGGDOnbsqF69eumXX36RZMHrlYFPSTLz58/3GOvTp4+58847fVNQKciv58aNG5tnn33WY+zqq682Tz75ZClWVjIOHz5sJJkVK1YYY4w5fvy4CQgIMHPmzHHP2bp1q5Fk1q5d66syvebP/ebn448/NoGBgcblcpViZSXjfP1u2rTJ1KxZ0yQnJ+f7M1+W5ddz69atzVNPPeXDqkpOfv2Ghoaa999/32NelSpVzNSpU0u7vBIRHh5u3n77bSter8pe1C3ncnNztXDhQl155ZXq2rWrqlWrptatW5fpw8ZF0aZNGy1YsEC///67jDFatmyZduzYoYSEBF+XdtHOnuKoUqWKJGnjxo1yuVzq3Lmze05cXJxq1aqltWvX+qRGb/pzv+ebExYWpgoVrPjMyYuSX78ZGRnq16+fXn/99XxvsFrW/bnnw4cP67vvvlO1atXUpk0bVa9eXe3atdO3337ryzK9Jr/vcZs2bTR79mz98ccfys3N1UcffaRTp06pffv2PqrSO3JycvTRRx8pPT1d8fHxdrxelUocwnnpT39pnf3rKyQkxLzyyitm06ZNZuzYscbhcJjly5f7rlAv+nPPxhhz6tQpc9dddxlJpkKFCiYwMNC89957vinQi3JyckyPHj1M27Zt3WMzZ840gYGBeea2atXKDBs2rDTL87r8+v2zI0eOmFq1apknnniiFCsrGefrd+DAgWbAgAHu5fx+5suq/Hpeu3atkWSqVKli3n33XfPDDz+YIUOGmMDAQLNjxw4fVnvxzvc9PnbsmElISHC/ZoWFhZnFixf7qMqL99NPP5nQ0FDj7+9vKleubBYuXGiMseP1quz/OVPO5ObmSpJ69eqlhx9+WJLUokULrVmzRlOmTFG7du18WV6JmTRpktatW6cFCxaodu3aWrlypRITExUdHe2R5MuaxMREbdmypdz8ZVmYwvpNS0tTjx491KhRIyUlJZVucSUgv34XLFigb775pkxeq1AU+fV89nXr/vvvd99Q9qqrrtLSpUv17rvvauzYsT6p1RvO9zP99NNP6/jx4/r6668VERGhTz/9VL1799aqVavUtGlTH1VbfA0aNNDmzZuVmpqquXPnqn///lqxYoWvyzqjVOIQzkt/+ksrKyvLVKhQwYwePdpj3rBhw0ybNm1KubqS8eeeMzIyTEBAgPniiy885g0YMMB07dq1lKvznsTERBMTE2P27NnjMb506VIjyRw7dsxjvFatWuaVV14pxQq963z9npWWlmbi4+NNp06dTGZmZilX533n6/df//qXcTgcxt/f3/0lyfj5+Zl27dr5plgvOV/Pe/bsMZLMBx984DHeu3dv069fv9Is0avO1++uXbuMJLNlyxaP8U6dOpn777+/NEssMZ06dTIDBw604vWKa1QsExgYqFatWuV5K9yOHTtUu3ZtH1VVslwul1wuV553B/j7+7v/UitLjDF68MEHNX/+fH3zzTeqW7eux/prrrlGAQEBWrp0qXts+/bt2rdvn+Lj40u73ItWWL/SmSMpCQkJCgwM1IIFC8r0O9gK63fEiBH66aeftHnzZveXJE2YMEHTpk3zQcUXr7Ce69Spo+jo6HLzulVYvxkZGZJUbl6z8pObm6usrCw7Xq9KJQ7Bw4kTJ8ymTZvMpk2bjCT3tSj/+c9/jDHGfPLJJyYgIMC89dZbZufOnWbSpEnG39/frFq1yseVF19hPbdr1840btzYLFu2zOzZs8dMmzbNBAUFmTfeeMPHlV+4Bx54wFSuXNksX77cJCcnu78yMjLcc/75z3+aWrVqmW+++cZs2LDBxMfHm/j4eB9WXXyF9Zuammpat25tmjZtanbt2uUx5/Tp0z6u/sIV5fv7Zyrj16gUpecJEyaYsLAwM2fOHLNz507z1FNPmaCgILNr1y4fVl48hfWbnZ1t6tevb2644Qbz3XffmV27dpmXX37ZOBwO97UdZcmIESPMihUrzN69e81PP/1kRowYYRwOh/nqq6+MMb5/vSKo+MCyZcuMpDxf/fv3d8955513TP369U1QUJBp3ry5+fTTT31XsBcU1nNycrK5++67TXR0tAkKCjINGjQw48ePN7m5ub4tvBjy61OSmTZtmntOZmamGTRokAkPDzchISHmb3/7m0lOTvZd0RehsH7P972XZPbu3evT2oujKN/f/B5TloNKUXseO3asiYmJMSEhISY+Pr7M/nFVlH537NhhbrnlFlOtWjUTEhJimjVrluftymXFvffea2rXrm0CAwNNZGSk6dSpkzukGOP71yuHMcZ4+ygNAACAN3CNCgAAsBZBBQAAWIugAgAArEVQAQAA1iKoAAAAaxFUAACAtQgqAADAWgQVAGXa3XffrZtvvtln+//tt9/kcDjcH5UPwLsIKgB8LikpSS1atPB1GYXKLxTFxsYqOTlZTZo08U1RQDlHUAFwyXO5XMV+rL+/v2rUqKEKFSp4sSIAZxFUgDJi7ty5atq0qYKDg1W1alV17txZ6enp7vVvv/22GjZsqKCgIMXFxemNN97wePyaNWvUokULBQUFqWXLlvr00089TlksX75cDodDixcv1lVXXaXg4GB17NhRhw8f1qJFi9SwYUOFhYWpX79+7rvHSmfusjp27FjVrVtXwcHBat68uebOnetef3a7S5cuVcuWLRUSEqI2bdq477Q7ffp0jRo1Sj/++KMcDoccDoemT5+e73OQk5OjoUOH6rLLLlPVqlU1bNgw/fkuIHXq1NHEiRM9xlq0aKGkpCT3ssPh0OTJk9WzZ0+FhoZqzJgxysnJ0YABA9x9NGjQQK+++qr7MUlJSXrvvff02Wefuetcvnx5vqd+VqxYoWuvvVZOp1NRUVEaMWKETp8+7V7fvn17DR48WMOGDVOVKlVUo0YNj/oAnKPU7ioEoNgOHjxoKlSoYF555RX3HU5ff/11c+LECWOMMTNmzDBRUVFm3rx5Zs+ePWbevHmmSpUqZvr06caYM3cwrlKlirnzzjvNL7/8Yv7973+bK6+80kgymzZtMsb87+aB1113nfn222/NDz/8YOrXr2/atWtnEhISzA8//GBWrlxpqlatasaNG+eu7bnnnjNxcXHmyy+/NLt37zbTpk0zTqfTLF++3GO7rVu3NsuXLze//PKLueGGG0ybNm2MMcZkZGSYRx55xDRu3LjQOxG/8MILJjw83MybN8/8+uuvZsCAAaZSpUqmV69e7jm1a9c2EyZM8Hhc8+bNzciRI93Lkky1atXMu+++a3bv3m3+85//mOzsbPPMM8+Y9evXmz179pgZM2aYkJAQM3v2bGPMmTuA9+7d23Tr1s1dZ1ZWltm7d6/H83jgwAETEhJiBg0aZLZu3Wrmz59vIiIiPPbfrl07ExYWZpKSksyOHTvMe++953G3WgD/Q1AByoCNGzcaSea3337Ld329evXMrFmzPMZGjx7tvhX75MmTTdWqVU1mZqZ7/dSpU/MNKl9//bV7ztixY40ks3v3bvfY/fffb7p27WqMMebUqVMmJCTErFmzxmPfAwYMMH379j3vdhcuXGgkuesZOXKkad68eaHPQ1RUlHnxxRfdyy6Xy8TExBQrqAwZMqTQ/SUmJppbb73Vvdy/f3+PfRlj8gSVJ554wjRo0MDjzt+vv/66qVixosnJyTHGnAkq119/vcd2WrVqZYYPH15oTcClhpOqQBnQvHlzderUSU2bNlXXrl2VkJCg2267TeHh4UpPT9fu3bs1YMAA3Xfffe7HnD59WpUrV5Ykbd++Xc2aNVNQUJB7/bXXXpvvvpo1a+b+d/Xq1RUSEqLLL7/cY+z777+XJO3atUsZGRnq0qWLxzays7N11VVXnXe7UVFRkqTDhw+rVq1aRXoOUlNTlZycrNatW7vHKlSooJYtW+Y5/VMULVu2zDP2+uuv691339W+ffuUmZmp7OzsC77Id+vWrYqPj5fD4XCPtW3bVidPntSBAwfc/Z77fEhnnpPDhw9fcB9AeUdQAcoAf39/LVmyRGvWrNFXX32lSZMm6cknn9R3332nkJAQSdLUqVM9fomffdyFCggIcP/b4XB4LJ8dy83NlSSdPHlSkrRw4ULVrFnTY57T6Sxwu5Lc2/EmPz+/PMElv4tlQ0NDPZY/+ugjPfrooxo/frzi4+NVqVIlvfTSS/ruu++8XqOkAp9XAP/DxbRAGeFwONS2bVuNGjVKmzZtUmBgoObPn6/q1asrOjpae/bsUf369T2+6tatK0lq0KCBfv75Z2VlZbm3t379+ouuqVGjRnI6ndq3b1+efcfGxhZ5O4GBgcrJySlwTuXKlRUVFeURHE6fPq2NGzd6zIuMjFRycrJ7OS0tTXv37i20htWrV6tNmzYaNGiQrrrqKtWvX1+7d+++4DobNmyotWvXeoSl1atXq1KlSoqJiSm0DgCeCCpAGfDdd9/p+eef14YNG7Rv3z598sknOnLkiBo2bChJGjVqlMaOHav/+7//044dO/Tzzz9r2rRpeuWVVyRJ/fr1U25urgYOHKitW7dq8eLFevnllyXJ4xTFhapUqZIeffRRPfzww3rvvfe0e/du/fDDD5o0aZLee++9Im+nTp062rt3rzZv3qyUlBSPQHWuf/3rXxo3bpw+/fRTbdu2TYMGDdLx48c95nTs2FEffPCBVq1apZ9//ln9+/cv0pGlK664Qhs2bNDixYu1Y8cOPf3003nCXJ06dfTTTz9p+/btSklJyfdIzaBBg7R//3499NBD2rZtmz777DONHDlSQ4cOlZ8fL7nAheLUD1AGhIWFaeXKlZo4caLS0tJUu3ZtjR8/Xt27d5ck/eMf/1BISIheeuklPfbYYwoNDVXTpk01ZMgQ9+M///xzPfDAA2rRooWaNm2qZ555Rv369fO4bqU4Ro8ercjISI0dO1Z79uzRZZddpquvvlpPPPFEkbdx66236pNPPlGHDh10/PhxTZs2TXfffXeeeY888oiSk5PVv39/+fn56d5779Xf/vY3paamuuc8/vjj2rt3r2688UZVrlxZo0ePLtIRlfvvv1+bNm1Snz595HA41LdvXw0aNEiLFi1yz7nvvvu0fPlytWzZUidPntSyZctUp04dj+3UrFlT//73v/XYY4+pefPmqlKligYMGKCnnnqqyM8HgP9xmOJchQagzJs5c6buuecepaamKjg42NflAEC+OKICXCLef/99XX755apZs6Z+/PFHDR8+XL179yakALAaQQW4RBw6dEjPPPOMDh06pKioKN1+++0aM2aMr8sCgAJx6gcAAFiLS9ABAIC1CCoAAMBaBBUAAGAtggoAALAWQQUAAFiLoAIAAKxFUAEAANYiqAAAAGsRVAAAgLX+P6DfsbmRNtFHAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "median 30.0, 90% 30.0\n"
     ]
    }
   ],
   "source": [
    "# check the durations distribution\n",
    "pd.Series(durations_list).hist(bins=50)\n",
    "plt.title(\n",
    "    f\"median {np.median(durations_list):.1f}, 90% {np.quantile(durations_list, 0.90):.1f}\"\n",
    ")\n",
    "plt.xlabel(\"segment duration\")\n",
    "plt.ylabel(\"counts\")\n",
    "plt.show()\n",
    "print(\n",
    "    f\"median {np.median(durations_list):.1f}, 90% {np.quantile(durations_list, 0.90):.1f}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "52357e46",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-25T20:41:53.526111Z",
     "start_time": "2024-03-25T20:41:41.606560Z"
    }
   },
   "outputs": [],
   "source": [
    "# if dataset_name == \"genius\":\n",
    "#     INPUT_PATH = \"/home/tony/Data/Hoot/metas.json\"\n",
    "# elif dataset_name == \"ytm\":\n",
    "#     INPUT_PATH = \"/home/tony/Data/Hoot/ytm_metas.json\"\n",
    "# elif dataset_name == \"deezer\":\n",
    "#     INPUT_PATH = \"/home/tony/Data/Hoot/deezer_metas.json\"\n",
    "# with open(INPUT_PATH, \"r\") as fp:\n",
    "#     metas = json.load(fp)\n",
    "# meta_id_map = {meta[\"id\"]: meta for meta in metas}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "71969894",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-25T20:45:36.378521Z",
     "start_time": "2024-03-25T20:45:34.011992Z"
    }
   },
   "outputs": [],
   "source": [
    "# # # listen to some\n",
    "# # there should be some leftover here\n",
    "# k, segments = sample_data[-1]\n",
    "# #meta = meta_id_map[k]\n",
    "# # audio = Audio.from_s3(meta[\"audio_filepath\"])\n",
    "# # audio.play(compress=False)\n",
    "# # print(f\"{meta['id']}\")\n",
    "# # print(meta[\"genius_slug\"])\n",
    "# print(\"-\"*10)\n",
    "# for s in segments:\n",
    "#     print(s[\"text\"], s[\"start_s\"], s[\"end_s\"], s[\"vocal_start_s\"], s[\"vocal_end_s\"])\n",
    "#     # audio.get_segment(s[\"start_s\"], s[\"end_s\"]).play(compress=False)\n",
    "#     print(\"-\"*10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2db28860",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from suno_utils.tasks.hoot import get_aligned_lyrics\n",
    "\n",
    "# with open(\"/app/suno/data/hoot/ytm_3/batch_0.json\", \"r\") as fp:\n",
    "#     data = json.load(fp)\n",
    "\n",
    "# test_data = data[1]\n",
    "# print(test_data)\n",
    "# test_output = get_aligned_lyrics(\n",
    "#     [test_data],\n",
    "#     silent=True,\n",
    "#     min_cer=0.8,\n",
    "#     min_p_align=0.0,\n",
    "#     aligned_text_frac=0.9,\n",
    "#     return_only_valid=True,)\n",
    "# for segment in list(test_output.values())[0]:\n",
    "#     print(segment)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67256edf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}