{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.040680Z", "start_time": "2024-05-16T13:58:19.777010Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:18:53.620084Z", "iopub.status.busy": "2024-10-03T02:18:53.619747Z", "iopub.status.idle": "2024-10-03T02:18:56.925496Z", "shell.execute_reply": "2024-10-03T02:18:56.924979Z", "shell.execute_reply.started": "2024-10-03T02:18:53.620065Z" } }, "outputs": [], "source": [ "import ast\n", "import os\n", "import shutil\n", "import sys\n", "from collections import defaultdict\n", "\n", "import numpy as np\n", "import pandas as pd\n", "from preference_data_preparation_4min_30b_task import *\n", "from preference_helper import *\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n", "from tqdm import tqdm\n", "\n", "pd.set_option(\"display.max_rows\", 500)\n", "pd.set_option(\"display.max_columns\", 500)\n", "pd.set_option(\"display.width\", 1000)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.082172Z", "start_time": "2024-05-16T13:58:21.041926Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:18:56.926571Z", "iopub.status.busy": "2024-10-03T02:18:56.926364Z", "iopub.status.idle": "2024-10-03T02:18:56.971933Z", "shell.execute_reply": "2024-10-03T02:18:56.971476Z", "shell.execute_reply.started": "2024-10-03T02:18:56.926553Z" } }, "outputs": [], "source": [ "OUT_DATA_DIR = \"/app/suno/data/dpo/30b_t4_v21\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "shutil.copyfile(\n", " \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", " os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", ")\n", "NPZ_DIR = \"/app/suno/data/dpo/30b_npz\"" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:53.962528Z", "start_time": "2024-05-16T13:58:21.105919Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:19:04.403275Z", "iopub.status.busy": "2024-10-03T02:19:04.402829Z", "iopub.status.idle": "2024-10-03T02:19:07.785909Z", "shell.execute_reply": "2024-10-03T02:19:07.785332Z", "shell.execute_reply.started": "2024-10-03T02:19:04.403256Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Preference data shape (80036, 71)\n" ] } ], "source": [ "df = pd.read_pickle(\n", " \"/home/tony/Data/Preference/30b_v3/interesting_clips_v4_t_4_20241002_full_with_cer.pkl\"\n", " # \"/home/tony/Data/Preference/30b_v3/interesting_clips_v4_t_4_20241001_full_with_sem_distance_and_similarity.pkl\"\n", " # \"/home/tony/Data/Preference/30b_v3/interesting_clips_v4_t_4_20240923_full_with_sem_distance_and_similarity.pkl\"\n", " # \"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_20240919_full_l10_with_cer.pkl\"\n", ") # , engine='python')\n", "# df = pd.read_csv(\n", "# \"/home/tony/Data/Preference/30b_v0/interesting_clips_v4_t_1_20240808.csv\"\n", "# ) # , engine='python')\n", "print(\"Preference data shape\", df.shape)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:19:16.687188Z", "iopub.status.busy": "2024-10-03T02:19:16.686852Z", "iopub.status.idle": "2024-10-03T02:19:16.791196Z", "shell.execute_reply": "2024-10-03T02:19:16.790647Z", "shell.execute_reply.started": "2024-10-03T02:19:16.687170Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(80036, 71)\n", "(80036, 66)\n" ] } ], "source": [ "print(df.shape)\n", "df = df.dropna(axis=1, how=\"all\")\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.199480Z", "start_time": "2024-05-16T13:58:53.963687Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:19:18.362834Z", "iopub.status.busy": "2024-10-03T02:19:18.362475Z", "iopub.status.idle": "2024-10-03T02:20:35.427844Z", "shell.execute_reply": "2024-10-03T02:20:35.427253Z", "shell.execute_reply.started": "2024-10-03T02:19:18.362817Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1261543\n", "1261543\n", "pre-downloaded df (80036, 66)\n", "downloaded df (80036, 66)\n" ] } ], "source": [ "converted_paths = os.listdir(NPZ_DIR)\n", "print(len(converted_paths))\n", "\n", "converted_paths = set([f.replace(\".npz\", \"\") for f in converted_paths])\n", "print(len(converted_paths))\n", "\n", "print(\"pre-downloaded df\", df.shape)\n", "df[df[\"s3_id\"].isin(converted_paths)].shape\n", "df = df[df[\"s3_id\"].isin(converted_paths)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.467253Z", "start_time": "2024-05-16T13:58:56.207647Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:20:35.428935Z", "iopub.status.busy": "2024-10-03T02:20:35.428775Z", "iopub.status.idle": "2024-10-03T02:20:35.505264Z", "shell.execute_reply": "2024-10-03T02:20:35.504795Z", "shell.execute_reply.started": "2024-10-03T02:20:35.428919Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "is_30b\n", "True 80036\n", "Name: count, dtype: int64\n" ] }, { "data": { "text/plain": [ "task\n", "cover 40442\n", " 32424\n", "extend 7144\n", "artist_consistency 26\n", "Name: count, dtype: int64" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"is_30b\"] = df[\"model_name\"].str.contains(\"-t\")\n", "print(df[\"is_30b\"].value_counts())\n", "df[\"task\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.592883Z", "start_time": "2024-05-16T13:58:56.470781Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:21:06.890960Z", "iopub.status.busy": "2024-10-03T02:21:06.890643Z", "iopub.status.idle": "2024-10-03T02:21:06.922226Z", "shell.execute_reply": "2024-10-03T02:21:06.921713Z", "shell.execute_reply.started": "2024-10-03T02:21:06.890944Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "preference model_name \n", "False chirp-v3p5-engine-t-3 40018\n", "True chirp-v3p5-engine-t-3 40018\n", "Name: count, dtype: int64\n", "(80036, 67)\n", "(80036, 67)\n" ] } ], "source": [ "## for 13b this is easy for now\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "print(df.shape)\n", "df = df[df[\"model_name\"].isin([\"chirp-v3p5-engine-t-3\"])]\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.909539Z", "start_time": "2024-05-16T13:58:56.595736Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:21:08.363164Z", "iopub.status.busy": "2024-10-03T02:21:08.362846Z", "iopub.status.idle": "2024-10-03T02:21:08.543723Z", "shell.execute_reply": "2024-10-03T02:21:08.543172Z", "shell.execute_reply.started": "2024-10-03T02:21:08.363146Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(80036, 67)\n", "(80036, 67)\n", "preference model_name \n", "False chirp-v3p5-engine-t-3 40018\n", "True chirp-v3p5-engine-t-3 40018\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(df.shape)\n", "df = df[\n", " df[\"request_id\"].isin(\n", " df[\"request_id\"].value_counts().index[df[\"request_id\"].value_counts() == 2]\n", " )\n", "]\n", "print(df.shape)\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "assert df.shape[0] == df[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:21:10.262743Z", "iopub.status.busy": "2024-10-03T02:21:10.262469Z", "iopub.status.idle": "2024-10-03T02:21:10.265331Z", "shell.execute_reply": "2024-10-03T02:21:10.264898Z", "shell.execute_reply.started": "2024-10-03T02:21:10.262727Z" } }, "outputs": [], "source": [ "import json\n", "\n", "def custom_parse(x):\n", " try:\n", " return json.loads(x)\n", " except:\n", " return {}" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:21:13.154573Z", "iopub.status.busy": "2024-10-03T02:21:13.154276Z", "iopub.status.idle": "2024-10-03T02:21:32.762385Z", "shell.execute_reply": "2024-10-03T02:21:32.761833Z", "shell.execute_reply.started": "2024-10-03T02:21:13.154558Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "unique_requests 40018\n" ] } ], "source": [ "# Let's use the old selection for now -- for quality assurance\n", "# expand the metadata columns -- this takes forever...~ 6 mins\n", "test_slice = df[\"metadata\"].apply(lambda x: ast.literal_eval(str(x)))\n", "# test_slice = df[\"metadata\"].apply(lambda x: custom_parse(x))\n", "test_slice_series = test_slice.apply(pd.Series)\n", "df = pd.concat([df, test_slice_series], axis=1, join=\"inner\")\n", "print(\"unique_requests\", df[\"request_id\"].nunique())" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:21:35.762438Z", "iopub.status.busy": "2024-10-03T02:21:35.762134Z", "iopub.status.idle": "2024-10-03T02:21:35.989696Z", "shell.execute_reply": "2024-10-03T02:21:35.989106Z", "shell.execute_reply.started": "2024-10-03T02:21:35.762421Z" } }, "outputs": [], "source": [ "df = df.loc[:, ~df.columns.duplicated()].copy()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:21:36.674330Z", "iopub.status.busy": "2024-10-03T02:21:36.674161Z", "iopub.status.idle": "2024-10-03T02:21:36.681852Z", "shell.execute_reply": "2024-10-03T02:21:36.681418Z", "shell.execute_reply.started": "2024-10-03T02:21:36.674314Z" } }, "outputs": [ { "data": { "text/plain": [ "task\n", "cover 40442\n", " 32424\n", "extend 7144\n", "artist_consistency 26\n", "Name: count, dtype: int64" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"task\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:21:40.710896Z", "iopub.status.busy": "2024-10-03T02:21:40.710383Z", "iopub.status.idle": "2024-10-03T02:21:40.889110Z", "shell.execute_reply": "2024-10-03T02:21:40.888618Z", "shell.execute_reply.started": "2024-10-03T02:21:40.710879Z" } }, "outputs": [ { "data": { "text/plain": [ "pos_diff_preference\n", "1.0 25481\n", "2.0 14537\n", "Name: count, dtype: int64" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df.sort_values(by=[\"request_id\", \"preference\", \"diff_preference\"])\n", "df[\"pos_diff_preference\"] = df[\"diff_preference\"].diff()\n", "df[\"cer_diff_preference\"] = df[\"cer\"].diff()\n", "df[df[\"preference\"]][\"pos_diff_preference\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:21:41.734302Z", "iopub.status.busy": "2024-10-03T02:21:41.733868Z", "iopub.status.idle": "2024-10-03T02:21:41.736269Z", "shell.execute_reply": "2024-10-03T02:21:41.735876Z", "shell.execute_reply.started": "2024-10-03T02:21:41.734284Z" } }, "outputs": [], "source": [ "# df[[\"preference\", \"diff_preference\", \"pos_diff_preference\", \"cer\", \"cer_diff_preference\"]].tail(n=6)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:21:42.962393Z", "iopub.status.busy": "2024-10-03T02:21:42.962246Z", "iopub.status.idle": "2024-10-03T02:21:43.003605Z", "shell.execute_reply": "2024-10-03T02:21:43.003105Z", "shell.execute_reply.started": "2024-10-03T02:21:42.962379Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive param_experiment\n", "max_cfg_12_ntag_3 4887\n", "s_max_t_12_tag_3 1862\n", "s_30_t_12_tag_3 1591\n", "s_max_t_12_tag_4 1106\n", "s_max_t_13_tag_3 976\n", "s_30_t_10_tag_3 802\n", "s_30_t_12_tag_4 736\n", "s_30_t_10_tag_4 714\n", "s_max_t_14_tag_3 659\n", "text_cfg_12_250 526\n", "max_cfg_12 477\n", "cfg_12 432\n", "n_repeat_tags_3 389\n", "s_30_t_13_tag_3 227\n", "s_30_t_14_tag_3 206\n", "s_tag_3 146\n", "s_30_t_11_tag_3 98\n", "s_t_13_tag_3 53\n", "s_t_09 27\n", "s_30_t_12_tag_2 16\n", "s_max_t_12_tag_2 11\n", "Name: count, dtype: int64\n" ] } ], "source": [ "try:\n", " print(\"positive\", df[df[\"preference\"]][\"param_experiment\"].value_counts())\n", "except:\n", " pass" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:21:46.651467Z", "iopub.status.busy": "2024-10-03T02:21:46.650941Z", "iopub.status.idle": "2024-10-03T02:21:47.251543Z", "shell.execute_reply": "2024-10-03T02:21:47.251057Z", "shell.execute_reply.started": "2024-10-03T02:21:46.651450Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.35860000000000003\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1kUlEQVR4nO3de3RU1f3//9dMLogSYkyCFEN/2mACkjtxpaRDscrFGrENavEK2FQURbRF0WK4U0Aa/QBihQZMDdIiH0CrBfxUbLV0BVqlCWkgAkFAKK0k4RLCNZfz+4PmfB0IJBNmkszm+ViLtcg5e3b2e/bM5DV7zjnjsCzLEgAAgGGcbT0AAAAAXyDkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjBbb1ANqDyspj8uaXWzgcUnh4iNf7bU9Mr5H6/J/pNVKf/zO9Rl/W19B3Uwg5kixLPnmA+arf9sT0GqnP/5leI/X5P9NrbMv6+LgKAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJH4FnIARgsIuPh7ufp6S/X1Bn8FNHAZI+QAMJLT6VBdvaWwsKsu2q62rl5Hj5wg6AAGIuQAMJLD4VCA06Gnlxeq7GB1o216dOmkefcly+l0EHIAAxFyABit7GC1th6oauthAGgDHHgMAACMRMgBAABGIuQAAAAjEXIAAICRPA45n376qR5//HG5XC7FxsZq/fr19r6amhr98pe/1JAhQ5SUlCSXy6Xx48frq6++cuvjyJEjGjdunFJSUpSamqoJEybo+PHjbm0+//xzPfDAA4qPj1f//v2Vm5t73ljWrVun22+/XfHx8RoyZIg++eQTT8sBAACG8jjknDhxQrGxsZo8efJ5+06dOqVt27Zp9OjRWr16tRYsWKDdu3dr9OjRbu2effZZlZWVKS8vTwsXLtRnn32mSZMm2furq6uVlZWlbt26afXq1Ro/frwWLFigt99+227zj3/8Q+PGjdM999yjd999V7fddpuefPJJ7dixw9OSAACAgTw+hbx///7q379/o/tCQkKUl5fntm3ixIm69957deDAAXXr1k27du3Shg0btHLlSsXHx0uSsrOzNWrUKI0fP17XXnut3nvvPdXU1GjmzJkKDg7WjTfeqNLSUuXl5WnYsGGSpPz8fPXr108/+clPJEnPPPOMCgoK9NZbb2natGmelgUAAAzj8+vkVFdXy+FwqHPnzpKkwsJCde7c2Q44kpSeni6n06ni4mINHDhQRUVFSk1NVXBwsN3G5XIpNzdXR48eVWhoqIqKijRy5Ei33+Vyudw+Pmsuh6NltTXVn7f7bU9Mr5H6/J+ntfnbfWH6HJpen2R+jb6sr7l9+jTknD59Wjk5OcrIyFCnTp0kSRUVFbrmmmvcBxEYqNDQUJWXl9ttoqKi3NpERETY+0JDQ1VRUWFvaxAeHq6KigqPxxkeHuLxbdqy3/bE9Bqp7/LQ1Fc/tGemz6Hp9Unm19iW9fks5NTU1Ojpp5+WZVmaOnWqr36NV1RWHpPlxSu6OxxnJ9Xb/bYnptdIff4vMNCpq69uXng5fPi46urqfTwi7zJ9Dk2vTzK/Rl/W19B3U3wScmpqavTMM8/owIEDevPNN+1VHOnsisyhQ4fc2tfW1uro0aOKjIy025y7ItPwc8PqTWNtKisrz1vdaQ7Lkk8eYL7qtz0xvUbq81+e1uWv94PJcyiZX59kfo1tWZ/Xr5PTEHD27t2r3/zmNwoLC3Pbn5ycrKqqKpWUlNjbNm3apPr6eiUkJEiSkpKS9Nlnn6mmpsZuU1BQoBtuuEGhoaF2m02bNrn1XVBQoKSkJG+XBAAA/JDHIef48eMqLS1VaWmpJGn//v0qLS3VgQMHVFNTo7Fjx6qkpEQ5OTmqq6tTeXm5ysvLdebMGUlSdHS0+vXrp4kTJ6q4uFibN2/W9OnTlZGRoWuvvVaSNGTIEAUFBenFF1/Uzp07tXbtWuXn5+uRRx6xxzF8+HBt2LBBb7zxhnbt2qVXX31VJSUleuihh7xxvwAAAD/n8cdVJSUlGj58uP3zrFmzJEmZmZkaM2aM/vSnP0mSfvCDH7jdLj8/X2lpaZKknJwcTZ8+XSNGjJDT6dSgQYOUnZ1ttw0JCdGSJUs0bdo0DR06VGFhYXriiSfs08clKSUlRTk5OZo7d65eeeUVXX/99XrttdcUExPjaUkAAMBAHoectLQ0bd++/YL7L7avwdVXX62XX375om169uyp3/72txdt8/3vf1/f//73m/x9AADg8sN3VwEAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADCSxyHn008/1eOPPy6Xy6XY2FitX7/ebb9lWZo3b55cLpcSEhI0cuRI7dmzx63NkSNHNG7cOKWkpCg1NVUTJkzQ8ePH3dp8/vnneuCBBxQfH6/+/fsrNzf3vLGsW7dOt99+u+Lj4zVkyBB98sknnpYDAAAM5XHIOXHihGJjYzV58uRG9+fm5mrp0qWaMmWKVqxYoY4dOyorK0unT5+22zz77LMqKytTXl6eFi5cqM8++0yTJk2y91dXVysrK0vdunXT6tWrNX78eC1YsEBvv/223eYf//iHxo0bp3vuuUfvvvuubrvtNj355JPasWOHpyUBAAADeRxy+vfvr5/+9KcaOHDgefssy1J+fr5Gjx6tAQMGqGfPnpozZ44OHjxor/js2rVLGzZs0IwZM5SYmKjU1FRlZ2drzZo1+uqrryRJ7733nmpqajRz5kzdeOONysjI0MMPP6y8vDz7d+Xn56tfv376yU9+oujoaD3zzDO66aab9NZbb7X0vgAAAAbx6jE5+/fvV3l5udLT0+1tISEhSkxMVGFhoSSpsLBQnTt3Vnx8vN0mPT1dTqdTxcXFkqSioiKlpqYqODjYbuNyubR7924dPXrUbtO3b1+33+9yuVRUVOTNkgAAgJ8K9GZn5eXlkqTw8HC37eHh4aqoqJAkVVRU6JprrnEfRGCgQkND7dtXVFQoKirKrU1ERIS9LzQ0VBUVFfa2xn6PJxwOj2/SrP683W97YnqN1Of/PK3N3+4L0+fQ9Pok82v0ZX3N7dOrIcdfhYeH+FW/7YnpNVLf5SEs7Kq2HkKLmT6HptcnmV9jW9bn1ZATGRkpSaqsrFSXLl3s7ZWVlerZs6eksysyhw4dcrtdbW2tjh49at8+IiLivBWZhp8bVm8aa1NZWXne6k5zVFYek2V5fLMLcjjOTqq3+21PTK+R+vxfYKBTV1/dvPBy+PBx1dXV+3hE3mX6HJpen2R+jb6sr6Hvpng15ERFRSkyMlIbN25Ur169JJ09U2rLli26//77JUnJycmqqqpSSUmJ4uLiJEmbNm1SfX29EhISJElJSUmaO3euampqFBQUJEkqKCjQDTfcoNDQULvNpk2bNHLkSPv3FxQUKCkpyeNxW5Z88gDzVb/tiek1Up//8rQuf70fTJ5Dyfz6JPNrbMv6PD7w+Pjx4yotLVVpaamkswcbl5aW6sCBA3I4HBo+fLhef/11ffTRR9q+fbvGjx+vLl26aMCAAZKk6Oho9evXTxMnTlRxcbE2b96s6dOnKyMjQ9dee60kaciQIQoKCtKLL76onTt3au3atcrPz9cjjzxij2P48OHasGGD3njjDe3atUuvvvqqSkpK9NBDD3njfgEAAH7O45WckpISDR8+3P551qxZkqTMzEzNnj1bjz76qE6ePKlJkyapqqpKffr00eLFi9WhQwf7Njk5OZo+fbpGjBghp9OpQYMGKTs7294fEhKiJUuWaNq0aRo6dKjCwsL0xBNPaNiwYXablJQU5eTkaO7cuXrllVd0/fXX67XXXlNMTEyL7ggAAGAWh2WZvEjWPBUV3j8mJyIixOv9tiem10h9/i8w0KmwsKuUMX+Dth6oarRN726dtWZsPx0+fFy1tf53TI7Jc2h6fZL5Nfqyvoa+m8J3VwEAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABjJ6yGnrq5Oc+fO1a233qqEhAQNGDBAr732mizLsttYlqV58+bJ5XIpISFBI0eO1J49e9z6OXLkiMaNG6eUlBSlpqZqwoQJOn78uFubzz//XA888IDi4+PVv39/5ebmerscAADgp7wecnJzc/W73/1OkyZN0tq1a/Xss89q8eLFWrp0qVubpUuXasqUKVqxYoU6duyorKwsnT592m7z7LPPqqysTHl5eVq4cKE+++wzTZo0yd5fXV2trKwsdevWTatXr9b48eO1YMECvf32294uCQAA+CGvh5zCwkLddtttuuWWWxQVFaXbb79dLpdLxcXFks6u4uTn52v06NEaMGCAevbsqTlz5ujgwYNav369JGnXrl3asGGDZsyYocTERKWmpio7O1tr1qzRV199JUl67733VFNTo5kzZ+rGG29URkaGHn74YeXl5Xm7JAAA4Ie8HnKSk5O1adMm7d69W9LZj5Q2b96s7373u5Kk/fv3q7y8XOnp6fZtQkJClJiYqMLCQklng1Lnzp0VHx9vt0lPT5fT6bTDUlFRkVJTUxUcHGy3cblc2r17t44ePertsgAAgJ8J9HaHo0aNUnV1tb7//e8rICBAdXV1+ulPf6q77rpLklReXi5JCg8Pd7tdeHi4KioqJEkVFRW65ppr3AcaGKjQ0FD79hUVFYqKinJrExERYe8LDQ1t9pgdDg8K9KA/b/fbnpheI/X5P09r87f7wvQ5NL0+yfwafVlfc/v0eshZt26d3n//fb388svq0aOHSktLNWvWLHXp0kWZmZne/nVeER4e4lf9tiem10h9l4ewsKvaeggtZvocml6fZH6NbVmf10POnDlzNGrUKGVkZEiSYmNjdeDAAS1atEiZmZmKjIyUJFVWVqpLly727SorK9WzZ09JZ1dkDh065NZvbW2tjh49at8+IiLCXvlp0PBzw4pOc1VWHtPXTv66ZA7H2Un1dr/tiek1Up//Cwx06uqrmxdeDh8+rrq6eh+PyLtMn0PT65PMr9GX9TX03RSvh5xTp07Jcc46UkBAgH0KeVRUlCIjI7Vx40b16tVL0tkzpbZs2aL7779f0tnjeqqqqlRSUqK4uDhJ0qZNm1RfX6+EhARJUlJSkubOnauamhoFBQVJkgoKCnTDDTd49FGVJFmWfPIA81W/7YnpNVKf//K0Ln+9H0yeQ8n8+iTza2zL+rx+4PH3vvc9LVy4UB9//LH279+vDz/8UHl5eRowYIAkyeFwaPjw4Xr99df10Ucfafv27Ro/fry6dOlit4mOjla/fv00ceJEFRcXa/PmzZo+fboyMjJ07bXXSpKGDBmioKAgvfjii9q5c6fWrl2r/Px8PfLII94uCQAA+CGvr+RkZ2dr3rx5mjp1qv2R1LBhw/Tkk0/abR599FGdPHlSkyZNUlVVlfr06aPFixerQ4cOdpucnBxNnz5dI0aMkNPp1KBBg5SdnW3vDwkJ0ZIlSzRt2jQNHTpUYWFheuKJJzRs2DBvlwQAAPyQw7JMXiRrnooK7x+TExER4vV+2xPTa6Q+/xcY6FRY2FXKmL9BWw9UNdqmd7fOWjO2nw4fPq7aWv87JsfkOTS9Psn8Gn1ZX0PfTeG7qwAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIzkk5Dz1Vdf6dlnn1VaWpoSEhI0ZMgQ/fOf/7T3W5alefPmyeVyKSEhQSNHjtSePXvc+jhy5IjGjRunlJQUpaamasKECTp+/Lhbm88//1wPPPCA4uPj1b9/f+Xm5vqiHAAA4Ie8HnKOHj2q+++/X0FBQcrNzdWaNWv0/PPPKzQ01G6Tm5urpUuXasqUKVqxYoU6duyorKwsnT592m7z7LPPqqysTHl5eVq4cKE+++wzTZo0yd5fXV2trKwsdevWTatXr9b48eO1YMECvf32294uCQAA+KFAb3eYm5urrl27atasWfa27t272/+3LEv5+fkaPXq0BgwYIEmaM2eO0tPTtX79emVkZGjXrl3asGGDVq5cqfj4eElSdna2Ro0apfHjx+vaa6/Ve++9p5qaGs2cOVPBwcG68cYbVVpaqry8PA0bNszbZQEAAD/j9ZWcP/3pT4qLi9PYsWPVt29f/fCHP9SKFSvs/fv371d5ebnS09PtbSEhIUpMTFRhYaEkqbCwUJ07d7YDjiSlp6fL6XSquLhYklRUVKTU1FQFBwfbbVwul3bv3q2jR496uywAAOBnvL6Ss2/fPv3ud7/TI488oscff1z//Oc/NWPGDAUFBSkzM1Pl5eWSpPDwcLfbhYeHq6KiQpJUUVGha665xn2ggYEKDQ21b19RUaGoqCi3NhEREfa+r3881hSHw7Mam9uft/ttT0yvkfr8n6e1+dt9Yfocml6fZH6NvqyvuX16PeRYlqW4uDj97Gc/kyTddNNN2rlzp5YvX67MzExv/zqvCA8P8at+2xPTa6S+y0NY2FVtPYQWM30OTa9PMr/GtqzP6yEnMjJS0dHRbtu+9a1v6f/+7//s/ZJUWVmpLl262G0qKyvVs2dPSWdXZA4dOuTWR21trY4ePWrfPiIiwl75adDwc8OKTnNVVh6TZXl0k4tyOM5Oqrf7bU9Mr5H6/F9goFNXX9288HL48HHV1dX7eETeZfocml6fZH6Nvqyvoe+meD3kpKSkaPfu3W7b9uzZo+uuu06SFBUVpcjISG3cuFG9evWSdPZMqS1btuj++++XJCUnJ6uqqkolJSWKi4uTJG3atEn19fVKSEiQJCUlJWnu3LmqqalRUFCQJKmgoEA33HCDRx9VSZJlyScPMF/1256YXiP1+S9P6/LX+8HkOZTMr08yv8a2rM/rBx6PGDFCW7Zs0cKFC7V37169//77WrFihR544AFJksPh0PDhw/X666/ro48+0vbt2zV+/Hh16dLFPtsqOjpa/fr108SJE1VcXKzNmzdr+vTpysjI0LXXXitJGjJkiIKCgvTiiy9q586dWrt2rfLz8/XII494uyQAAOCHvL6Sk5CQoAULFuiVV17Ra6+9pqioKE2YMEF33XWX3ebRRx/VyZMnNWnSJFVVValPnz5avHixOnToYLfJycnR9OnTNWLECDmdTg0aNEjZ2dn2/pCQEC1ZskTTpk3T0KFDFRYWpieeeILTxwEAgCTJYVkmL5I1T0WF94/JiYgI8Xq/7YnpNVKf/wsMdCos7CplzN+grQeqGm3Tu1tnrRnbT4cPH1dtrf8dk2PyHJpen2R+jb6sr6HvpvDdVQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjBbb1AACgJZxOh5xOxwX3BwTwHg643BFyAPgdp9Oh0KuvVCBBBsBFEHIA+B2n06HAAKeeXl6osoPVjba5JTZSzw3u2cojA9CeEHIA+K2yg9XaeqCq0X3RkVe18mgAtDes9QIAACMRcgAAgJEIOQAAwEg+Dzm//vWvFRsbq1/84hf2ttOnT2vq1KlKS0tTcnKynnrqKVVUVLjd7sCBAxo1apQSExPVt29fvfTSS6qtrXVr87e//U2ZmZmKi4vTwIEDtXr1al+XAwAA/IRPQ05xcbGWL1+u2NhYt+0zZ87Un//8Z82dO1dLly7VwYMHNWbMGHt/XV2dHnvsMdXU1Gj58uWaPXu23nnnHc2fP99us2/fPj322GNKS0vT73//e40YMULZ2dnasGGDL0sCAAB+wmch5/jx43ruuec0Y8YMhYaG2tuPHTumVatW6YUXXlDfvn0VFxenmTNnqrCwUEVFRZKkv/71ryorK9Mvf/lL9erVS/3799fTTz+tZcuW6cyZM5Kk5cuXKyoqSi+88IKio6P10EMPafDgwfrNb37jq5IAAIAf8VnImTZtmvr376/09HS37SUlJaqpqXHbHh0drW7dutkhp6ioSDExMYqIiLDbuFwuVVdXq6yszG7Tt29ft75dLpfdhyccDu//81W/7emf6TVSX/v95wttXdPlNofUd3nU6Mv6msMn18lZs2aNtm3bppUrV563r6KiQkFBQercubPb9vDwcJWXl9ttvh5wJNk/N9Wmurpap06d0hVXXNHs8YaHhzS7rSd81W97YnqN1Hd5CAvz32vqmD6HptcnmV9jW9bn9ZDz73//W7/4xS/0xhtvqEOHDt7u3icqK4/JsrzXn8NxdlK93W97YnqN1Ne+BQQ4vRpMDh8+rrq6eq/11xr8fQ6bYnp9kvk1+rK+hr6b4vWQs3XrVlVWVmro0KH2trq6On366adatmyZlixZopqaGlVVVbmt5lRWVioyMlLS2RWZ4uJit34bzr76eptzz8iqqKhQp06dPFrFkSTLkk8eYL7qtz0xvUbqu3z46/1g+hyaXp9kfo1tWZ/XQ863v/1tvf/++27bfv7zn+tb3/qWHn30UX3jG99QUFCQNm7cqMGDB0uSvvjiCx04cEBJSUmSpKSkJC1cuFCVlZUKDw+XJBUUFKhTp07q0aOH3eYvf/mL2+8pKCiw+wAAAJc3r4ecTp06KSYmxm3blVdeqauvvtrefvfdd2v27NkKDQ1Vp06dNGPGDCUnJ9sBxeVyqUePHho/fryee+45lZeXa+7cuXrwwQcVHBwsSbrvvvu0bNkyzZkzR3fffbc2bdqkdevWadGiRd4uCQAA+KE2+YLOCRMmyOl0auzYsTpz5oxcLpcmT55s7w8ICNDChQs1ZcoUDRs2TB07dlRmZqbGjh1rt+nevbsWLVqkWbNmKT8/X127dtWMGTPUr1+/tigJAAC0M60ScpYuXer2c4cOHTR58mS3YHOu6667Trm5uRftNy0tTe+++643hggAAAzDd1cBAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABgpsK0HAADncjodcjodF9wfEMD7MwBNI+QAaFecTodCr75SgQQZAJeIkAOgXXE6HQoMcOrp5YUqO1jdaJtbYiP13OCerTwyAP6GkAOgXSo7WK2tB6oa3RcdeVUrjwaAP2I9GAAAGImQAwAAjETIAQAARuKYHACXvaZOSa+vt1Rfb7XSaAB4CyEHwGUrslMH1dVb6ty540Xb1dbV6+iREwQdwM8QcgBctjp3DFSA03HR09V7dOmkefcly+l0EHIAP0PIAXDZu9jp6gD8FwceAwAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwktdDzqJFi3T33XcrOTlZffv21RNPPKEvvvjCrc3p06c1depUpaWlKTk5WU899ZQqKirc2hw4cECjRo1SYmKi+vbtq5deekm1tbVubf72t78pMzNTcXFxGjhwoFavXu3tcgAAgJ/yesj5+9//rgcffFArVqxQXl6eamtrlZWVpRMnTthtZs6cqT//+c+aO3euli5dqoMHD2rMmDH2/rq6Oj322GOqqanR8uXLNXv2bL3zzjuaP3++3Wbfvn167LHHlJaWpt///vcaMWKEsrOztWHDBm+XBAAA/JDXr5OzZMkSt59nz56tvn37auvWrbr55pt17NgxrVq1Sjk5Oerbt6+ks6HnjjvuUFFRkZKSkvTXv/5VZWVlysvLU0REhHr16qWnn35aOTk5GjNmjIKDg7V8+XJFRUXphRdekCRFR0dr8+bN+s1vfqN+/fp5uywAAOBnfH4xwGPHjkmSQkNDJUklJSWqqalRenq63SY6OlrdunWzQ05RUZFiYmIUERFht3G5XJoyZYrKysp00003qaioyA5JX28zc+ZMj8focLSksqb783a/7YnpNVIfGtOe7i/T59D0+iTza/Rlfc3t06chp76+XjNnzlRKSopiYmIkSRUVFQoKClLnzp3d2oaHh6u8vNxu8/WAI8n+uak21dXVOnXqlK644opmjzM8PMSzwtq43/bE9BqpDw3Cwq5q6yE0yvQ5NL0+yfwa27I+n4acqVOnaufOnfrtb3/ry19zySorj8ny4lfSOBxnJ9Xb/bYnptdIfb7hdDrUOfRKBTbxrd/t0eHDx1VXV9/Ww7DxGPV/ptfoy/oa+m6Kz0LOtGnT9PHHH+utt95S165d7e0RERGqqalRVVWV22pOZWWlIiMj7TbFxcVu/TWcffX1NueekVVRUaFOnTp5tIojSZYlnzzAfNVve2J6jdTnXQ6HQ4EBzot+IeYtsZF6bnDP1huUB9rjY4HHqP8zvca2rM/rb6csy9K0adP04Ycf6s0331T37t3d9sfFxSkoKEgbN260t33xxRc6cOCAkpKSJElJSUnasWOHKisr7TYFBQXq1KmTevToYbfZtGmTW98FBQV2HwDar4YvxGzs375DJ5ruAACaweshZ+rUqXrvvff08ssv66qrrlJ5ebnKy8t16tQpSVJISIjuvvtuzZ49W5s2bVJJSYkmTJig5ORkO6C4XC716NFD48eP1+eff64NGzZo7ty5evDBBxUcHCxJuu+++7Rv3z7NmTNHu3bt0rJly7Ru3TqNHDnS2yUBAAA/5PWPq373u99Jkh5++GG37bNmzdLQoUMlSRMmTJDT6dTYsWN15swZuVwuTZ482W4bEBCghQsXasqUKRo2bJg6duyozMxMjR071m7TvXt3LVq0SLNmzVJ+fr66du2qGTNmcPo4AACQ5IOQs3379ibbdOjQQZMnT3YLNue67rrrlJube9F+0tLS9O6773o6RAAAcBnwv1McAAAAmoGQAwAAjETIAQAARiLkAAAAI/n8u6sAwAQBTVylub7eUn29wVd0A/wQIQcALiKyUwfV1Vvq3LnjRdvV1tXr6JETBB2gHSHkAMBFdO4YqACn46JfRdGjSyfNuy9ZTqeDkAO0I4QcAGiGhq+iAOA/CDkAvMbpdMjpdFxwf1PHtQCANxFyAHiF0+lQ6NVXKpAgA6CdIOQA8Aqn06HAAOdFj125JTZSzw3u2cojA3C5IuQA8KqLHbsSHXlVK48GwOWMdWUAAGAkQg4AADASH1cBgJdwVWSgfSHkAMAl4qrIQPtEyAHQLFwD58K4KjLQPhFyADSJa+A0D1dFBtoXQg6AJnENHAD+iJADoNm4Bg4Af8LaMwAAMBIhBwAAGImQAwAAjMQxOQDQirhgINB6CDnAZa6p699Il/c1cLyFCwYCrY+QA1zGnE6HOody/ZvWwAUDgdZHyAEuYw5H09e/kbgGjjdxwUCg9RByADT5h5dr4ADwR4QcwGB83xSAyxkhBzBUc75vqqmDYAHAnxFyAEPxfVMALneEHMBwfN+U/2nOtXQsi7OvgKYQcgA/xfE25vHkWjpVR0+00qgA/0XIAfxQc463gf/x5Fo6gYEBki4cZrlyMkDIAfwSx9uY7WIfMZ672hMW1vhHjlw5GSDkAH6N420uP1w5GWg+Qg4A+CGunAw0jZADtEMcVAxv4BvPcbkj5ACtrKkA43A41CnkCg4qRovxjefAWYQcwIu8GWA4qBgt5clxO0FBAaqrq79gX6z2wJ8RcgAv8eS07uYEGA4qxqXy5CytC2G1B/6MkAM0U3OOk2nuad0EGLQ1VntwOSDkwHhNhZOLaThw05OPmQgw8Ces9sBkhBz4NW8dA1NXbymgkX7OvdAax8ngcuLN1Z7mYEUI3kbIQZtozupKUy943j4Gho+ZgMZ5Y7XnQm8kvq62rl7Vx065ffnouafBE4TgCUKOD12u16jw1upKYy94X+ftY2AIMIDnmrPa05w3EjdfH6aJd/bW1Vdf6bb93NXUpl4XmsvU11+48/uQs2zZMi1ZskTl5eXq2bOnJk6cqISEhDYdk9PpUF29dcHvlGngrSdrczTnCd2SY1fODXLeOkX6Qi94jSGcAG3PG28kmgpLzX1daOmqUUtczmHpUo53bC1+HXLWrl2rWbNmaerUqUpMTNSbb76prKwsffDBBwoPD2+zcTkcjlZ9snrjCd3SY1cuFOS8sbrS3HeHAMzhrdcFfwpL3ryCubfe0HrrcIG6euvsG/+6tgmCfh1y8vLy9KMf/Uh33323JGnq1Kn6+OOPtWrVKo0aNaqNR9c6T1ZvPqGl9nfsCqs0AM7VGqtGrRmWwsKuandvaL1xuEDDQekOh0MSIccjZ86c0datW/XYY4/Z25xOp9LT01VYWOhRX06n5M1PjBz/fQz27tZZHYMDGm0THdlJktQh0HnBNsH/fRBeapuwK4MV4HRo4ce7dODoyUbbxF8XqntTu3ttPM2pnTb+0aY9jok2tPFGm9Z8bW2NNjd26aQH0v6/Zr2h9VY/F7t/OgQ2XILj7N9Zb3I081Myh9UaB4T4wFdffaXvfve7Wr58uZKTk+3tc+bM0aeffqr//d//bcPRAQCAtsY3AAIAACP5bcgJCwtTQECAKisr3bZXVlYqIiKijUYFAADaC78NOcHBwerdu7c2btxob6uvr9fGjRvdPr4CAACXJ7898FiSHnnkET3//POKi4tTQkKC3nzzTZ08eVJDhw5t66EBAIA25tch54477tChQ4c0f/58lZeXq1evXlq8eDEfVwEAAP89uwoAAOBi/PaYHAAAgIsh5AAAACMRcgAAgJEIOQAAwEiEnBZ4/fXXdd999ykxMVGpqanNuo1lWZo3b55cLpcSEhI0cuRI7dmzx63NkSNHNG7cOKWkpCg1NVUTJkzQ8ePHfVBB0zwdy/79+xUbG9vov3Xr1tntGtu/Zs2a1ijJTUvu64cffvi8sU+aNMmtzYEDBzRq1CglJiaqb9++eumll1RbW+vLUi7I0xqPHDmi6dOna/DgwUpISNAtt9yiGTNm6NixY27t2moOly1bpltvvVXx8fG69957VVxcfNH269at0+233674+HgNGTJEn3zyidv+5jwnW5snNa5YsUIPPPCAbr75Zt18880aOXLkee1feOGF8+YqKyvL12VckCf1rV69+ryxx8fHu7Vpb3PoSX2NvZ7Exsa6fbl0e5q/Tz/9VI8//rhcLpdiY2O1fv36Jm/zt7/9TZmZmYqLi9PAgQO1evXq89p4+rz2mAWPzZs3z8rLy7NmzZpl9enTp1m3WbRokdWnTx/rww8/tEpLS63HH3/cuvXWW61Tp07ZbbKysqy77rrLKioqsj799FNr4MCB1s9+9jNflXFRno6ltrbWOnjwoNu/V1991UpKSrKqq6vtdjExMdaqVavc2n39PmgtLbmvH3roISs7O9tt7MeOHbP319bWWnfeeac1cuRIa9u2bdbHH39spaWlWS+//LKvy2mUpzVu377dGjNmjPXRRx9Ze/futQoKCqxBgwZZTz31lFu7tpjDNWvWWL1797ZWrlxp7dy508rOzrZSU1OtioqKRttv3rzZ6tWrl5Wbm2uVlZVZ//M//2P17t3b2r59u92mOc/J1uRpjT/72c+st956y9q2bZtVVlZmvfDCC1afPn2s//znP3ab559/3srKynKbqyNHjrRWSW48rW/VqlVWSkqK29jLy8vd2rSnOfS0vsOHD7vVtmPHDqtXr17WqlWr7Dbtaf4+/vhj65VXXrH++Mc/WjExMdaHH3540fZffvmllZiYaM2aNcsqKyuzli5davXq1cv6y1/+Yrfx9D5rCULOJVi1alWzQk59fb31ne98x1q8eLG9raqqyoqLi7P+8Ic/WJZlWWVlZVZMTIxVXFxst/nkk0+s2NhYtxet1uCtsfzgBz+wfv7zn7tta86Tw9daWt9DDz1kzZgx44L7P/74Y6tnz55uL8S//e1vrZSUFOv06dPeGXwzeWsO165da/Xu3duqqamxt7XFHN5zzz3W1KlT7Z/r6uosl8tlLVq0qNH2Tz/9tDVq1Ci3bffee681ceJEy7Ka95xsbZ7WeK7a2lorOTnZeuedd+xtzz//vDV69GhvD7VFPK2vqdfX9jaHlzp/eXl5VnJysnX8+HF7W3uav69rzmvAnDlzrIyMDLdtzzzzjPXjH//Y/vlS77Pm4OOqVrB//36Vl5crPT3d3hYSEqLExEQVFhZKkgoLC9W5c2e35dj09HQ5nU7vL981wRtjKSkpUWlpqe65557z9k2dOlVpaWm65557tHLlSlmtfKmmS6nv/fffV1pamu688069/PLLOnnypL2vqKhIMTExbhejdLlcqq6uVllZmfcLuQhvPZ6qq6vVqVMnBQa6Xze0NefwzJkz2rp1q9vzx+l0Kj093X7+nKuoqEh9+/Z12+ZyuVRUVCSpec/J1tSSGs918uRJ1dbWKjQ01G373//+d/Xt21eDBw/W5MmTdfjwYa+OvTlaWt+JEyf0ve99T/3799fo0aO1c+dOe197mkNvzN+qVauUkZGhK6+80m17e5i/lmjqOeiN+6w5/PqKx/6ivLxckhQeHu62PTw8XBUVFZKkiooKXXPNNW77AwMDFRoaat++tXhjLCtXrlR0dLRSUlLcto8dO1bf/va31bFjR/31r3/V1KlTdeLECQ0fPtxr429KS+u788471a1bN3Xp0kXbt29XTk6Odu/erQULFtj9nnu17Yaf/XEODx06pF/96lcaNmyY2/bWnsPDhw+rrq6u0efPF1980ehtGpuLrz/fmvOcbE0tqfFcOTk56tKli9sfjX79+mngwIGKiorSvn379Morr+jRRx/V22+/rYCAAK/WcDEtqe+GG27QzJkzFRsbq2PHjumNN97QfffdpzVr1qhr167tag4vdf6Ki4u1Y8cO/eIXv3Db3l7mryUu9HpYXV2tU6dO6ejRo5f8mG8OQs5/5eTkKDc396Jt1q5dq+jo6FYakfc1t8ZLderUKf3hD3/QE088cd6+J5980v7/TTfdpJMnT2rJkiVe+QPp6/q+/sc+NjZWkZGRGjlypL788kt985vfbHG/nmitOayurtZjjz2m6OhojRkzxm2fL+cQLfPrX/9aa9euVX5+vjp06GBvz8jIsP/fcODqgAED7NWB9iw5Odnty5aTk5N1xx13aPny5XrmmWfabmA+sHLlSsXExCghIcFtuz/PX3tByPmvH//4x8rMzLxom+7du7eo78jISElSZWWlunTpYm+vrKxUz549JZ1NuIcOHXK7XW1trY4ePWrf/lI1t8ZLHcsHH3ygU6dO6Yc//GGTbRMTE/WrX/1KZ86cUXBwcJPtL6a16muQmJgoSdq7d6+++c1vKiIi4ryPghreUfrTHFZXV+snP/mJrrrqKr322msKCgq6aHtvzmFjwsLCFBAQoMrKSrftlZWVF/yeuoiIiPPezX+9fXOek62pJTU2WLJkiX79618rLy+vybF3795dYWFh2rt3b6v+kbyU+hoEBQWpV69e+vLLLyW1rzm8lPpOnDihNWvWaOzYsU3+nraav5Zo7DlYUVGhTp066YorrpDT6bzkx0RzcEzOf11zzTWKjo6+6L+WvoBHRUUpMjJSGzdutLdVV1dry5Yt9juV5ORkVVVVqaSkxG6zadMm1dfXn5fuW6q5NV7qWFatWqVbb731vI9LGlNaWqrQ0FCv/HFsrfq+Pnbp/73YJiUlaceOHW5P2oKCAnXq1Ek9evS45Ppao8bq6mplZWUpKChIr7/+utuqwIV4cw4bExwcrN69e7s9f+rr67Vx40a3d/pfl5SUpE2bNrltKygoUFJSkqTmPSdbU0tqlKTc3Fz96le/0uLFi887vbox//nPf3TkyBGvhe7maml9X1dXV6cdO3bYY29Pc3gp9X3wwQc6c+aM7rrrriZ/T1vNX0s09Rz0xmOiWbx2CPNl5F//+pe1bds2+xTpbdu2Wdu2bXM7VXrw4MHWH//4R/vnRYsWWampqdb69eutzz//3Bo9enSjp5D/8Ic/tLZs2WJ99tln1qBBg9r0FPKLjeU///mPNXjwYGvLli1ut9uzZ48VGxtrffLJJ+f1+dFHH1krVqywtm/fbu3Zs8datmyZlZiYaM2bN8/n9ZzL0/r27t1rLViwwPrnP/9p7du3z1q/fr112223WQ8++KB9m4ZTyH/84x9bpaWl1l/+8hfr29/+dpueQu5JjceOHbPuvfde684777T27t3rdtpqbW2tZVltN4dr1qyx4uLirNWrV1tlZWXWxIkTrdTUVPtMtueee87Kycmx22/evNm66aabrCVLllhlZWXW/PnzGz2FvKnnZGvytMZFixZZvXv3tj744AO3uWp4HaqurrZmz55tFRYWWvv27bMKCgqszMxMa9CgQa1+tl9L6nv11VetDRs2WF9++aVVUlJi/fSnP7Xi4+OtnTt32m3a0xx6Wl+D+++/33rmmWfO297e5q+6utr+WxcTE2Pl5eVZ27Zts/71r39ZlmVZOTk51nPPPWe3bziF/KWXXrLKysqst956q9FTyC92n3kDH1e1wPz58/XOO+/YPzd8LJOfn6+0tDRJ0u7du90uovboo4/q5MmTmjRpkqqqqtSnTx8tXrzY7Z1yTk6Opk+frhEjRsjpdGrQoEHKzs5unaLO0dRYampqtHv3brezi6Szqzhdu3aVy+U6r8/AwEAtW7ZMM2fOlCR985vf1AsvvKAf/ehHvi2mEZ7WFxQUpI0bNyo/P18nTpzQN77xDQ0aNMjtuKOAgAAtXLhQU6ZM0bBhw9SxY0dlZmY2axnaFzytcevWrdqyZYskaeDAgW59ffTRR4qKimqzObzjjjt06NAhzZ8/X+Xl5erVq5cWL15sL2v/+9//ltP5/xamU1JSlJOTo7lz5+qVV17R9ddfr9dee00xMTF2m+Y8J1uTpzUuX75cNTU15z2+xowZo6eeekoBAQHasWOH3n33XR07dkxdunTRd77zHT399NM+W3W7GE/rq6qq0sSJE1VeXq7Q0FD17t1by5cvd1sVbU9z6Gl9kvTFF19o8+bNeuONN87rr73NX0lJidtxd7NmzZIkZWZmavbs2SovL9e///1ve3/37t21aNEizZo1S/n5+eratatmzJihfv362W2aus+8wWFZrXz+LgAAQCvgmBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjPT/A+Xhjvh7vxAAAAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Axes(0.125,0.11;0.775x0.77)\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxiklEQVR4nO3df3hU5Z3//9dMfgCSACEToAitmsjPAAnqJ00aSmuluFC8GlgXlC7gcgFiQ9haBYooBGgCbHQjlTaIbCiIphQi4iJ269YiXAmoLBGDQgnUEo2VzIiGoJBf5/sH30wZxSSTTE5yJ8/HdXFdzpl77nOf9yHk5X3uc8ZhWZYlAAAAgzjbegAAAAD+IsAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMEt/UAWpvHc0GB/LIEh0OKjAwPeL/4KmptD+psD+psD+psj9asc33fjenwAcay1Cp/iVurX3wVtbYHdbYHdbYHdbZHW9aZS0gAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGMevAHP77bdr8ODBX/mTnp4uSbp8+bLS09OVkJCg+Ph4LViwQG6326ePsrIyzZ07V6NGjVJiYqLWrl2rmpoanzaHDx9WSkqKYmNjNW7cOOXn57fwMAEAQEfi17dR79y5U7W1td7Xp06d0n333ac777xTkpSRkaH9+/crOztb4eHhWrVqlVJTU5WXlydJqq2t1bx58+RyuZSXl6dz585p8eLFCgkJ0YMPPihJKi0t1bx58zRt2jRlZWWpsLBQy5YtU1RUlMaMGROo426xoKCGs19dnaW6Or4KFQCA1uBXgOndu7fP66efflrf/OY39f/+3//ThQsXtGvXLmVlZSkxMVHSlUAzYcIEFRUVKS4uTgcPHlRJSYlyc3Plcrk0dOhQLVy4UFlZWUpNTVVoaKjy8vI0YMAALVmyRJIUHR2tI0eOaMuWLe0iwDidDtXWWYqI6N5gu5raOn326eeEGAAAWoFfAeZqVVVV2rNnj+677z45HA4VFxerurpaSUlJ3jbR0dHq37+/N8AUFRVp0KBBcrlc3jbJyclasWKFSkpKNGzYMBUVFXkD0NVtMjIymjvUgHI4HApyOrQw76hKzlVes01MnzA9OS1eTqeDAAMAQCtodoB59dVXdeHCBaWkpEiS3G63QkJC1KNHD592kZGRKi8v97a5OrxI8r5urE1lZaUuXbqkrl27+jVOh8Ov5k3ur+RcpY6XVdi+/86kvnbUsHVRZ3tQZ3tQZ3u0Zp2b2mezA8yuXbv03e9+V3379m1uF7aIjAxvs303dpkJTdOW57Azoc72oM72oM72aMs6NyvAfPjhhyooKNCvfvUr7zaXy6Xq6mpVVFT4zMJ4PB5FRUV52xw7dsynr/q7lK5u8+U7l9xut8LCwvyefbmy/wuyAngVJzjYqV69mhZMzp+/qNrausDtvJNxOK78cAT6HMIXdbYHdbYHdbZHa9a5vu/GNCvA5OfnKzIyUt/73ve822JjYxUSEqLCwkKNHz9eknTmzBmVlZUpLi5OkhQXF6ecnBx5PB5FRkZKkgoKChQWFqaYmBhvm9dff91nfwUFBd4+/GVZCmhx/e2LH6CWC/Q5xLVRZ3tQZ3tQZ3u0ZZ39fpBdXV2d8vPz9eMf/1jBwf/IP+Hh4ZoyZYrWrFmjQ4cOqbi4WEuXLlV8fLw3fCQnJysmJkaLFi3SiRMndODAAWVnZ2v69OkKDQ2VJE2bNk2lpaVat26dTp8+re3bt2vfvn2aNWtWQA4YAACYz+8ZmIKCApWVlWnKlClfeW/p0qVyOp1KS0tTVVWVkpOTtXz5cu/7QUFBysnJ0YoVKzR16lR169ZNKSkpSktL87YZOHCgNm7cqMzMTG3dulX9+vXT6tWr28Ut1AAAoH1wWFbHnmRzuwO/BiYiorsmrj/wtXchDe/fQ3vTxuj8+YuqqWENTHM5HJLLFR7wcwhf1Nke1Nke1NkerVnn+r4bw3chAQAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIzjd4D5+OOP9dBDDykhIUEjR47UpEmT9M4773jftyxLTz75pJKTkzVy5EjNmjVL77//vk8fn376qX7+859r9OjRuvXWW7V06VJdvHjRp82JEyd07733asSIERo7dqw2bdrUvCMEAAAdjl8B5rPPPtM999yjkJAQbdq0SXv37tXixYvVs2dPb5tNmzZp27ZtWrFihXbs2KFu3bpp9uzZunz5srfNQw89pJKSEuXm5ionJ0dvvfWWHnvsMe/7lZWVmj17tvr376/8/HwtWrRITz31lH73u98F4JABAIDpgv1pvGnTJvXr10+ZmZnebQMHDvT+t2VZ2rp1q+bPn6877rhDkrRu3TolJSXp1Vdf1cSJE3X69GkdOHBAO3fu1IgRIyRJy5Yt09y5c7Vo0SL17dtXe/bsUXV1tTIyMhQaGqqbb75Z7733nnJzczV16tRAHDcAADCYXzMwf/rTnxQbG6u0tDQlJibqxz/+sXbs2OF9/4MPPlB5ebmSkpK828LDwzVq1CgdPXpUknT06FH16NHDG14kKSkpSU6nU8eOHZMkFRUV6dZbb1VoaKi3TXJysv7617/qs88+a96RAgCADsOvGZjS0lI9//zzuu+++3T//ffrnXfe0erVqxUSEqKUlBSVl5dLkiIjI30+FxkZKbfbLUlyu93q3bu37yCCg9WzZ0/v591utwYMGODTxuVyed+7+pJVYxwOf44w8P0Fev+dSX3tqGHros72oM72oM72aM06N7VPvwKMZVmKjY3Vgw8+KEkaNmyYTp06pby8PKWkpPg9SDtERoa32b4jIrq32b47krY8h50JdbYHdbYHdbZHW9bZrwATFRWl6Ohon2033XST/vCHP3jflySPx6M+ffp423g8Hg0ZMkTSlZmUTz75xKePmpoaffbZZ97Pu1wu74xNvfrX9TMxTeXxXJBl+fWRBgUHO9WrV9OCyfnzF1VbWxe4nXcyDseVH45An0P4os72oM72oM72aM061/fdGL8CzOjRo/XXv/7VZ9v777+v66+/XpI0YMAARUVFqbCwUEOHDpV05Y6it99+W/fcc48kKT4+XhUVFSouLlZsbKwk6dChQ6qrq9PIkSMlSXFxccrOzlZ1dbVCQkIkSQUFBbrxxhv9unwkSZalgBbX3774AWq5QJ9DXBt1tgd1tgd1tkdb1tmvRbwzZ87U22+/rZycHP3tb3/TSy+9pB07dujee++VJDkcDs2YMUO/+c1v9L//+786efKkFi1apD59+njvSoqOjtaYMWP06KOP6tixYzpy5IhWrVqliRMnqm/fvpKkSZMmKSQkRI888ohOnTqll19+WVu3btV9990X4MMHAAAm8msGZuTIkXrqqaf0xBNPaMOGDRowYICWLl2qu+66y9tmzpw5+uKLL/TYY4+poqJCt9xyi5555hl16dLF2yYrK0urVq3SzJkz5XQ69cMf/lDLli3zvh8eHq7Nmzdr5cqVmjx5siIiIvTAAw9wCzUAAJAkOSyrY0+yud2BXwMTEdFdE9cf0PGyimu2Gd6/h/amjdH58xdVU8MamOZyOCSXKzzg5xC+qLM9qLM9qLM9WrPO9X03hu9CAgAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABjHrwDzq1/9SoMHD/b5c+edd3rfv3z5stLT05WQkKD4+HgtWLBAbrfbp4+ysjLNnTtXo0aNUmJiotauXauamhqfNocPH1ZKSopiY2M1btw45efnt+AQAQBARxPs7wduvvlm5ebmel8HBQV5/zsjI0P79+9Xdna2wsPDtWrVKqWmpiovL0+SVFtbq3nz5snlcikvL0/nzp3T4sWLFRISogcffFCSVFpaqnnz5mnatGnKyspSYWGhli1bpqioKI0ZM6alxwsAADoAvwNMUFCQoqKivrL9woUL2rVrl7KyspSYmCjpSqCZMGGCioqKFBcXp4MHD6qkpES5ublyuVwaOnSoFi5cqKysLKWmpio0NFR5eXkaMGCAlixZIkmKjo7WkSNHtGXLFgIMAACQ1Iw1MH/729+UnJysH/zgB/r5z3+usrIySVJxcbGqq6uVlJTkbRsdHa3+/furqKhIklRUVKRBgwbJ5XJ52yQnJ6uyslIlJSXeNvUB6Oo29X0AAAD4NQMzcuRIZWZm6sYbb1R5ebk2bNig6dOn66WXXpLb7VZISIh69Ojh85nIyEiVl5dLktxut094keR93VibyspKXbp0SV27dvXrAB0Ov5oHvL9A778zqa8dNWxd1Nke1Nke1NkerVnnpvbpV4AZO3as97+HDBmiUaNG6fvf/7727dvnd7CwS2RkeJvtOyKie5vtuyNpy3PYmVBne1Bne1Bne7Rlnf1eA3O1Hj166IYbbtDZs2eVlJSk6upqVVRU+MzCeDwe75oZl8ulY8eO+fRRf5fS1W2+fOeS2+1WWFhYs0KSx3NBluX3x75WcLBTvXo1LZicP39RtbV1gdt5J+NwXPnhCPQ5hC/qbA/qbA/qbI/WrHN9341pUYC5ePGiSktLFRUVpdjYWIWEhKiwsFDjx4+XJJ05c0ZlZWWKi4uTJMXFxSknJ0cej0eRkZGSpIKCAoWFhSkmJsbb5vXXX/fZT0FBgbcPf1mWAlpcf/viB6jlAn0OcW3U2R7U2R7U2R5tWWe/FvGuXbtWb7zxhj744AP93//9n1JTU+V0OvWjH/1I4eHhmjJlitasWaNDhw6puLhYS5cuVXx8vDd8JCcnKyYmRosWLdKJEyd04MABZWdna/r06QoNDZUkTZs2TaWlpVq3bp1Onz6t7du3a9++fZo1a1agjx0AABjKrxmYv//973rwwQf16aefqnfv3rrlllu0Y8cO9e7dW5K0dOlSOZ1OpaWlqaqqSsnJyVq+fLn380FBQcrJydGKFSs0depUdevWTSkpKUpLS/O2GThwoDZu3KjMzExt3bpV/fr10+rVq7mFGgAAeDksq2NPsrndgV8DExHRXRPXH9Dxsoprthnev4f2po3R+fMXVVPDGpjmcjgklys84OcQvqizPaizPaizPVqzzvV9N4bvQgIAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYp0UB5umnn9bgwYP1y1/+0rvt8uXLSk9PV0JCguLj47VgwQK53W6fz5WVlWnu3LkaNWqUEhMTtXbtWtXU1Pi0OXz4sFJSUhQbG6tx48YpPz+/JUMFAAAdSLMDzLFjx5SXl6fBgwf7bM/IyNBrr72m7Oxsbdu2TefOnVNqaqr3/draWs2bN0/V1dXKy8vTmjVr9MILL2j9+vXeNqWlpZo3b54SEhL04osvaubMmVq2bJkOHDjQ3OECAIAOpFkB5uLFi3r44Ye1evVq9ezZ07v9woUL2rVrl5YsWaLExETFxsYqIyNDR48eVVFRkSTp4MGDKikp0X/8x39o6NChGjt2rBYuXKjt27erqqpKkpSXl6cBAwZoyZIlio6O1k9+8hONHz9eW7ZsafEBAwAA8zUrwKxcuVJjx45VUlKSz/bi4mJVV1f7bI+Ojlb//v29AaaoqEiDBg2Sy+XytklOTlZlZaVKSkq8bRITE336Tk5O9vYBAAA6t2B/P7B37169++672rlz51fec7vdCgkJUY8ePXy2R0ZGqry83Nvm6vAiyfu6sTaVlZW6dOmSunbt2uTxOhxNbtoq/QV6/51Jfe2oYeuizvagzvagzvZozTo3tU+/AsxHH32kX/7yl/qv//ovdenSpTnjsl1kZHib7Tsionub7bsjactz2JlQZ3tQZ3tQZ3u0ZZ39CjDHjx+Xx+PR5MmTvdtqa2v15ptvavv27dq8ebOqq6tVUVHhMwvj8XgUFRUl6cpMyrFjx3z6rb9L6eo2X75zye12KywszK/Zlyv7viDL8usjDQoOdqpXr6YFk/PnL6q2ti5wO+9kHI4rPxyBPofwRZ3tQZ3tQZ3t0Zp1ru+7MX4FmG9/+9t66aWXfLb94he/0E033aQ5c+boG9/4hkJCQlRYWKjx48dLks6cOaOysjLFxcVJkuLi4pSTkyOPx6PIyEhJUkFBgcLCwhQTE+Nt8/rrr/vsp6CgwNuHPyxLAS2uv33xA9RygT6HuDbqbA/qbA/qbI+2rLNfASYsLEyDBg3y2XbdddepV69e3u1TpkzRmjVr1LNnT4WFhWn16tWKj4/3ho/k5GTFxMRo0aJFevjhh1VeXq7s7GxNnz5doaGhkqRp06Zp+/btWrdunaZMmaJDhw5p37592rhxYwAOGQAAmM7vRbyNWbp0qZxOp9LS0lRVVaXk5GQtX77c+35QUJBycnK0YsUKTZ06Vd26dVNKSorS0tK8bQYOHKiNGzcqMzNTW7duVb9+/bR69WqNGTMm0MMFAAAGclhWx55kc7sDvwYmIqK7Jq4/oONlFddsM7x/D+1NG6Pz5y+qpoY1MM3lcEguV3jAzyF8UWd7UGd7UGd7tGad6/tuDN+FBAAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADCOXwHmueee06RJkzR69GiNHj1aU6dO1f79+73vX758Wenp6UpISFB8fLwWLFggt9vt00dZWZnmzp2rUaNGKTExUWvXrlVNTY1Pm8OHDyslJUWxsbEaN26c8vPzW3CIAACgo/ErwPTr108PPfSQ8vPztWvXLn3729/WT3/6U506dUqSlJGRoddee03Z2dnatm2bzp07p9TUVO/na2trNW/ePFVXVysvL09r1qzRCy+8oPXr13vblJaWat68eUpISNCLL76omTNnatmyZTpw4ECADhkAAJgu2J/Gt99+u8/rn/3sZ3r++edVVFSkfv36adeuXcrKylJiYqKkK4FmwoQJKioqUlxcnA4ePKiSkhLl5ubK5XJp6NChWrhwobKyspSamqrQ0FDl5eVpwIABWrJkiSQpOjpaR44c0ZYtWzRmzJgAHTYAAPg6TqdDTqejrYfRIL8CzNVqa2v1yiuv6PPPP1d8fLyKi4tVXV2tpKQkb5vo6Gj179/fG2CKioo0aNAguVwub5vk5GStWLFCJSUlGjZsmIqKirwB6Oo2GRkZzRqnI8D197e/QO+/M6mvHTVsXdTZHtTZHtS55ZxOh3r0vE7BQQ1fpKmtsxQU5FBtrRXQ/Tf13PkdYE6ePKlp06bp8uXLuu6667RhwwbFxMTovffeU0hIiHr06OHTPjIyUuXl5ZIkt9vtE14keV831qayslKXLl1S165d/RpvZGS4X+0DKSKie5vtuyNpy3PYmVBne1Bne1DnlluYd1Ql5yqv+V5MnzA9OS1eERFhNo/qH/wOMDfeeKN2796tCxcu6A9/+IMWL16sZ599tjXGFhAezwVZAQyHwcFO9erVtGBy/vxF1dbWBW7nnYzDceUfoUCfQ/iizvagzvagzi0XFORURER3lZyr1PGyigbbfvrpRdXUBPb3XP05bIzfASY0NFTf+ta3JEmxsbF65513tHXrVv3TP/2TqqurVVFR4TML4/F4FBUVJenKTMqxY8d8+qu/S+nqNl++c8ntdissLMzv2RdJsiwF9C+xv33xA9RygT6HuDbqbA/qbA/qbI+2rHOLnwNTV1enqqoqxcbGKiQkRIWFhd73zpw5o7KyMsXFxUmS4uLi9Je//EUej8fbpqCgQGFhYYqJifG2OXTokM8+CgoKvH0AAAD4FWAef/xxvfnmm/rggw908uRJPf7443rjjTc0adIkhYeHa8qUKVqzZo0OHTqk4uJiLV26VPHx8d7wkZycrJiYGC1atEgnTpzQgQMHlJ2drenTpys0NFSSNG3aNJWWlmrdunU6ffq0tm/frn379mnWrFmBPnYAAGAovy4heTweLV68WOfOnVN4eLgGDx6szZs36zvf+Y4kaenSpXI6nUpLS1NVVZWSk5O1fPly7+eDgoKUk5OjFStWaOrUqerWrZtSUlKUlpbmbTNw4EBt3LhRmZmZ2rp1q/r166fVq1dzCzUAAPDyK8A0ditzly5dtHz5cp/Q8mXXX3+9Nm3a1GA/CQkJ2r17tz9DAwAAnQjfhQQAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwjl8BZuPGjZoyZYri4+OVmJioBx54QGfOnPFpc/nyZaWnpyshIUHx8fFasGCB3G63T5uysjLNnTtXo0aNUmJiotauXauamhqfNocPH1ZKSopiY2M1btw45efnN/MQAQBAR+NXgHnjjTc0ffp07dixQ7m5uaqpqdHs2bP1+eefe9tkZGTotddeU3Z2trZt26Zz584pNTXV+35tba3mzZun6upq5eXlac2aNXrhhRe0fv16b5vS0lLNmzdPCQkJevHFFzVz5kwtW7ZMBw4cCMAhAwAA0wX703jz5s0+r9esWaPExEQdP35ct912my5cuKBdu3YpKytLiYmJkq4EmgkTJqioqEhxcXE6ePCgSkpKlJubK5fLpaFDh2rhwoXKyspSamqqQkNDlZeXpwEDBmjJkiWSpOjoaB05ckRbtmzRmDFjAnToAADAVH4FmC+7cOGCJKlnz56SpOLiYlVXVyspKcnbJjo6Wv379/cGmKKiIg0aNEgul8vbJjk5WStWrFBJSYmGDRumoqIibwC6uk1GRobfY3Q4mnNkgesv0PvvTOprRw1bF3W2B3W2B3W2l8PRdr9nmx1g6urqlJGRodGjR2vQoEGSJLfbrZCQEPXo0cOnbWRkpMrLy71trg4vkryvG2tTWVmpS5cuqWvXrk0eZ2RkuH8HFkAREd3bbN8dSVuew86EOtuDOtuDOtujV6+2+z3X7ACTnp6uU6dO6bnnngvkeALO47kgywpcf8HBziafsPPnL6q2ti5wO+9kHI4r/wgF+hzCF3W2B3W2B3VuuaAgZ5P/B/zTTy+qpiawv+fqz2FjmhVgVq5cqT//+c969tln1a9fP+92l8ul6upqVVRU+MzCeDweRUVFedscO3bMp7/6u5SubvPlO5fcbrfCwsL8mn2RJMtSQP8S+9sXP0AtF+hziGujzvagzvagzvZoyzr7dReSZVlauXKl/vjHP+q3v/2tBg4c6PN+bGysQkJCVFhY6N125swZlZWVKS4uTpIUFxenv/zlL/J4PN42BQUFCgsLU0xMjLfNoUOHfPouKCjw9gEAADo3vwJMenq69uzZo8cff1zdu3dXeXm5ysvLdenSJUlSeHi4pkyZojVr1ujQoUMqLi7W0qVLFR8f7w0fycnJiomJ0aJFi3TixAkdOHBA2dnZmj59ukJDQyVJ06ZNU2lpqdatW6fTp09r+/bt2rdvn2bNmhXQgwcAAGby6xLS888/L0n613/9V5/tmZmZmjx5siRp6dKlcjqdSktLU1VVlZKTk7V8+XJv26CgIOXk5GjFihWaOnWqunXrppSUFKWlpXnbDBw4UBs3blRmZqa2bt2qfv36afXq1dxCDQAAJPkZYE6ePNlomy5dumj58uU+oeXLrr/+em3atKnBfhISErR7925/hgcAADoJvgsJAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHH8+jZq2M/pdMjpdDTYpq7OUl2dZdOIAABoewSYdszpdKhnr+sUHNTwRFlNbZ0++/RzQgwAoNMgwLRjTqdDwUFOLcw7qpJzlddsE9MnTE9Oi5fT6SDAAAA6DQKMAUrOVep4WUVbDwMAgHaDANOGGlvfEtTIpSMAADorAkwbaer6lqZqLOyw0BcA0JEQYNpIU9a3fG9wlB4eP6TBfqLCuqi2zlKPHt0abMdCXwBAR0KAaWMNrW+Jjure6Od7dAtWkNPBQl8AQKdCgOkgWOgLAOhMWCUKAACMwwxMJ9KUu5pY7AsAMAEBphNo6kJficW+AAAzEGA6gaYs9JX+sdg3JCRItbV1X9uOWRoAQFsjwHQijS305ZZsAIApCDDw4pZsAIApCDD4iqbcks2TfwEAbYkAA79wmQkA0B4QYOAXLjMBANoDAgyahctMAIC2RIBpRQ39Am/KQ+VM5c9lpsoLl2RZhBgAgH8IMK3AnwfHdURNucx02w0RevRHw9Wr13UN9lVbZ8npdKi2lpADAPgHAkwraMov8O8NjtLD44fYPDJ7NfZN201dS+NwOCQRYAAA/0CAaUWN/QIHa2kAAM1DgEG7xS3bAICvQ4BBu+XPLdt8fxMAdC4EGLR7DV1mYpYGADonvwPMm2++qc2bN6u4uFjl5eXasGGD7rjjDu/7lmVp/fr1+v3vf6+KigqNHj1aK1as0A033OBt8+mnn2rVqlV67bXX5HQ69cMf/lCPPPKIunf/x7qQEydOaOXKlXrnnXfUu3dv/eQnP9GcOXNadrTocHiwHgB0Tn4/jOTzzz/X4MGDtXz58mu+v2nTJm3btk0rVqzQjh071K1bN82ePVuXL1/2tnnooYdUUlKi3Nxc5eTk6K233tJjjz3mfb+yslKzZ89W//79lZ+fr0WLFumpp57S7373u2YcIjqD+lmaa/35umADADCX3zMwY8eO1dixY6/5nmVZ2rp1q+bPn++dlVm3bp2SkpL06quvauLEiTp9+rQOHDignTt3asSIEZKkZcuWae7cuVq0aJH69u2rPXv2qLq6WhkZGQoNDdXNN9+s9957T7m5uZo6dWoLDhcAAHQEAV0D88EHH6i8vFxJSUnebeHh4Ro1apSOHj2qiRMn6ujRo+rRo4c3vEhSUlKSnE6njh07pnHjxqmoqEi33nqrQkNDvW2Sk5O1adMmffbZZ+rZs2eTx+RwBObYWqs/2Ivz91X1NaE2rYs624M628vhaLvfswENMOXl5ZKkyMhIn+2RkZFyu92SJLfbrd69e/sOIjhYPXv29H7e7XZrwIABPm1cLpf3PX8CTGRkuH8HgQ4rIoJn7zSEnxV7UGd7UGd79OrVdv+udvi7kDyeCwrkV+0EBzvb9ITBf/V3KgU5G471NbV1qvis892p5HBc+cc+0D8r8EWd7UGdWy4oyNnk/+H79NOLqqn5+kdYNEf9OWxMQANMVFSUJMnj8ahPnz7e7R6PR0OGXHlsvsvl0ieffOLzuZqaGn322Wfez7tcLu+MTb361/UzMU1lWQroX2J+IMzjz51KwcGd93kygf5ZwbVRZ3tQZ3u0ZZ0DGmAGDBigqKgoFRYWaujQoZKu3FH09ttv65577pEkxcfHq6KiQsXFxYqNjZUkHTp0SHV1dRo5cqQkKS4uTtnZ2aqurlZISIgkqaCgQDfeeKNfl4+Aq/E8GQDoOPwOMBcvXtTZs2e9rz/44AO999576tmzp/r3768ZM2boN7/5jb71rW9pwIABevLJJ9WnTx/vXUnR0dEaM2aMHn30UaWnp6u6ulqrVq3SxIkT1bdvX0nSpEmTtGHDBj3yyCOaM2eOTp06pa1bt+oXv/hFgA4b8MXzZADALH4HmOLiYs2YMcP7OjMzU5KUkpKiNWvWaM6cOfriiy/02GOPqaKiQrfccoueeeYZdenSxfuZrKwsrVq1SjNnzvQ+yG7ZsmXe98PDw7V582atXLlSkydPVkREhB544AFuoUar48slAcAMfgeYhIQEnTx58mvfdzgcWrhwoRYuXPi1bXr16qXHH3+8wf0MGTJEzz33nL/DA1oNl5kAoP3o8HchAYHCZSYAaD8IMICfuMwEAG2PAAMEEJeZAMAeBBgggPy5zBQS0vAzZ5qCmRwAnRUBBmgFgXjmTFOfHsxMDoDOiAAD2KwpszTfGxylh8cPCchMDrM0ADoiAgzQRhqapYmO6t5oG9bbAOjMCDCAobitG0BnRoABDNeU27oBoKNp+GEVAAAA7RAzMEAn0NiD9QDANAQYoAPz55btkJCgBtfJcDcTgPaEAAN0YE1Z6HvbDRF69EfD1bPndQ32xd1MANoTAgzQCTR2y3agnh7MLA0AuxBgAEgK3DNnKi9ckmVxKQpA6yLAAGiUP5eievXiUhSA1keAAdBkXIoC0F4QYAAEFF9/AMAOBBgAtgnk1x84nQ45G/m2bmZygI6LAAPAdi39+gOn06Geva5TcCMP6GMmB+i4CDAA2qWGnh4cFORUcJCT9TZAJ0aAAdCuNHWdjMR6G6AzI8AAaFeask7me4Oj9PD4IS3uh1kawFwEGADtUmO3bAeiH2ZpAHMRYAB0WoGcpQFgLwIMgE4vELM0tXWWghq5rbu2zpLT6VBtLTM5QEsRYACgAf6syWnKTE5wcJAcDtbbAC1FgAGAJmjKmhy7vhCzKQhC6OgIMABgg0B+IWZTLlex8BgdHQEGAGzU0i/E9OdyFV/HgI6MAAMA7UxLL1fVa+hpxg6HQ2HhXfk6BhiLAAMAHYw/TzPmQX8wFQEGADoYf+6csmvhMSEHgUaAAYAOqqVPMw7kwmO7765q6PJZIPdl5zqipuwrEAJROzsQYAAADWrpwuNA313VUBByOByqrbMUEdFwQAvUvpq6jqil4a2p+2rqAxUba2MCAgwAoMXsuLuqqUFIanhtj537CmR4a2xfTTkuf9q0dwQYAIAtWnp3lT9BqD3tKxDhLZDH1dQ27R0BBgBgFDt/+drxregdKVTYyYyVOgAAAFdp1wFm+/btuv322zVixAjdfffdOnbsWFsPCQAAtAPtNsC8/PLLyszM1E9/+lO98MILGjJkiGbPni2Px9PWQwMAAG2s3QaY3Nxc/cu//IumTJmimJgYpaenq2vXrtq1a1dbDw0AALSxdrmIt6qqSsePH9e8efO825xOp5KSknT06FG/+nI6pRY+N8mH4/+/y214/x7qFhp0zTbRUWHGtWmPY6INbWhDG9q0zzY3ua4sKnY4rvyeDSRHEx9R47Ba+ljEVvDxxx/ru9/9rvLy8hQfH+/dvm7dOr355pv6/e9/34ajAwAAba3dXkICAAD4Ou0ywERERCgoKOgrC3Y9Ho9cLlcbjQoAALQX7TLAhIaGavjw4SosLPRuq6urU2Fhoc8lJQAA0Dm1y0W8knTfffdp8eLFio2N1ciRI/Xb3/5WX3zxhSZPntzWQwMAAG2s3QaYCRMm6JNPPtH69etVXl6uoUOH6plnnuESEgAAaJ93IQEAADSkXa6BAQAAaAgBBgAAGIcAAwAAjEOAAQAAxiHAXMP27dt1++23a8SIEbr77rt17NixBtvv27dPd955p0aMGKFJkyZp//79No3UfP7UeseOHbr33nt122236bbbbtOsWbMaPTe4wt+/0/X27t2rwYMH64EHHmjlEXYM/ta5oqJC6enpSk5OVmxsrMaPH8+/H03gb523bNmi8ePHa+TIkRo7dqwyMjJ0+fJlm0ZrpjfffFP333+/kpOTNXjwYL366quNfubw4cNKSUlRbGysxo0bp/z8/NYdpAUfe/futYYPH27t3LnTOnXqlLVs2TLr1ltvtdxu9zXbHzlyxBo6dKi1adMmq6SkxPrP//xPa/jw4dbJkydtHrl5/K31gw8+aD377LPWu+++a5WUlFhLliyxbrnlFuvvf/+7zSM3i791rldaWmqNGTPGuvfee6358+fbNFpz+Vvny5cvW5MnT7bmzJljvfXWW1Zpaal1+PBh67333rN55Gbxt8579uyxYmNjrT179lilpaXWgQMHrO985ztWRkaGzSM3y5///GfriSeesP7nf/7HGjRokPXHP/6xwfZnz561Ro0aZWVmZlolJSXWtm3brKFDh1qvv/56q42RAPMl//zP/2ylp6d7X9fW1lrJycnWxo0br9l+4cKF1ty5c3223X333dajjz7aquPsCPyt9ZfV1NRY8fHx1gsvvNBKI+wYmlPnmpoaa+rUqdaOHTusxYsXE2CawN86P/fcc9YPfvADq6qqyq4hdgj+1jk9Pd2aMWOGz7bMzExr2rRprTrOjqQpAWbdunXWxIkTfbb9+7//u/Vv//ZvrTYuLiFdpaqqSsePH1dSUpJ3m9PpVFJSko4ePXrNzxQVFSkxMdFnW3JysoqKilpzqMZrTq2/7IsvvlBNTY169uzZWsM0XnPrvGHDBkVGRuruu++2Y5jGa06d//SnPykuLk4rV65UUlKSfvSjHyknJ0e1tbV2Dds4zalzfHy8jh8/7r3MVFpaqv3792vs2LG2jLmzaIvfhe32Sbxt4fz586qtrVVkZKTP9sjISJ05c+aan3G73V95OnBkZKTcbnerjbMjaE6tvywrK0t9+vTx+ccMvppT57feeks7d+7U7t27bRhhx9CcOpeWlurQoUOaNGmSnn76aZ09e1bp6emqqalRamqqHcM2TnPqPGnSJJ0/f1733nuvLMtSTU2Npk2bpvvvv9+OIXca1/pd6HK5VFlZqUuXLqlr164B3yczMDDS008/rZdffllPPfWUunTp0tbD6TAqKyu1aNEirVq1Sr17927r4XRolmUpMjJSq1atUmxsrCZMmKD7779feXl5bT20DuXw4cPauHGjli9frvz8fD311FPav3+/NmzY0NZDQwsxA3OViIgIBQUFyePx+Gz3eDxf+x1MLpfrK7MtDbXHFc2pdb3Nmzfr6aefVm5uroYMGdKawzSev3UuLS3Vhx9+qPnz53u31dXVSZKGDRumV155Rd/85jdbd9AGas7f56ioKAUHBysoKMi77aabblJ5ebmqqqoUGhraqmM2UXPq/OSTT+quu+7yXg4dPHiwPv/8cz322GOaP3++nE7+Pz4QrvW70O12KywsrFVmXyRmYHyEhoZq+PDhKiws9G6rq6tTYWGh4uPjr/mZuLg4HTp0yGdbQUGB4uLiWnOoxmtOrSVp06ZN+vWvf61nnnlGI0aMsGOoRvO3zjfddJNeeukl7d692/vn9ttvV0JCgnbv3q1+/frZOXxjNOfv8+jRo3X27FlvQJSk999/X1FRUYSXr9GcOl+6dOkrIaU+NFp8FWDAtMnvwlZbHmyovXv3WrGxsVZ+fr5VUlJiPfroo9att95qlZeXW5ZlWQ8//LCVlZXlbX/kyBFr2LBh1ubNm62SkhJr/fr13EbdRP7WeuPGjdbw4cOtV155xTp37pz3T2VlZVsdghH8rfOXcRdS0/hb57KyMis+Pt5auXKldebMGeu1116zEhMTrV//+tdtdQhG8LfO69evt+Lj463//u//ts6ePWsdPHjQuuOOO6yFCxe20RGYobKy0nr33Xetd9991xo0aJCVm5trvfvuu9aHH35oWZZlZWVlWQ8//LC3ff1t1GvXrrVKSkqsZ599ttVvo+YS0pdMmDBBn3zyidavX6/y8nINHTpUzzzzjHd68qOPPvJJ86NHj1ZWVpays7P1xBNP6IYbbtCGDRs0aNCgtjoEY/hb67y8PFVXVystLc2nn9TUVC1YsMDWsZvE3zqjefyt8ze+8Q1t3rxZmZmZuuuuu9S3b1/NmDFDc+bMaatDMIK/dZ4/f74cDoeys7P18ccfq3fv3vr+97+vn/3sZ211CEYoLi7WjBkzvK8zMzMlSSkpKVqzZo3Ky8v10Ucfed8fOHCgNm7cqMzMTG3dulX9+vXT6tWrNWbMmFYbo8OymEMDAABm4X+7AACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADDO/wcTEe2PYMQkswAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "df[df[\"preference\"]][\"cer_diff_preference\"].hist(bins=50)\n", "print(df[df[\"preference\"]][\"cer_diff_preference\"].quantile(0.95))\n", "plt.show()\n", "print(df[df[\"preference\"]][\"cer\"].hist(bins=50))\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:21:53.575397Z", "iopub.status.busy": "2024-10-03T02:21:53.574894Z", "iopub.status.idle": "2024-10-03T02:21:53.577418Z", "shell.execute_reply": "2024-10-03T02:21:53.577010Z", "shell.execute_reply.started": "2024-10-03T02:21:53.575379Z" } }, "outputs": [], "source": [ "# df[df[\"preference\"]][\"similarity\"].hist(bins=200)\n", "# plt.show()\n", "# df[\"similarity_diff\"] = df[\"similarity\"].diff()\n", "# df[df[\"preference\"]][\"similarity_diff\"].hist(bins=200)\n", "# plt.show()\n", "# df[\"continued_parent\"] = None\n", "# df[\"continue_at\"] = -1\n", "# df[df[\"preference\"]][\"similarity\"].describe()" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:40.799375Z", "start_time": "2024-05-16T13:59:36.394236Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:21:58.342574Z", "iopub.status.busy": "2024-10-03T02:21:58.342148Z", "iopub.status.idle": "2024-10-03T02:21:58.830717Z", "shell.execute_reply": "2024-10-03T02:21:58.830162Z", "shell.execute_reply.started": "2024-10-03T02:21:58.342558Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3036\n", "good_continue_at\n", "True 79787\n", "False 249\n", "Name: count, dtype: int64\n", "\n", " Check some basics... \n", " preference\n", "False 40018\n", "True 40018\n", "Name: count, dtype: int64 is_30b\n", "True 80036\n", "Name: count, dtype: int64 model_name\n", "chirp-v3p5-engine-t-3 80036\n", "Name: count, dtype: int64 preference model_name \n", "False chirp-v3p5-engine-t-3 40018\n", "True chirp-v3p5-engine-t-3 40018\n", "Name: count, dtype: int64\n", "task\n", "cover 40442\n", " 32424\n", "extend 7144\n", "artist_consistency 26\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df[\"id\"] = df[\"str_id\"]\n", "# get the original duration of the clips, if they are concacted\n", "df[\"original_duration_s\"] = df[\"total_start_s\"] + df[\"duration\"]\n", "# classify the continue at behavoirs by the duration choice\n", "audio_prompt_id_to_continue_at = {}\n", "\n", "for _, row in df[~df[\"continued_parent\"].isna()].iterrows():\n", " audio_prompt_id = row[\"continued_parent\"]\n", " if audio_prompt_id not in audio_prompt_id_to_continue_at:\n", " audio_prompt_id_to_continue_at[audio_prompt_id] = row[\"continue_at\"]\n", " else:\n", " # pick the max\n", " audio_prompt_id = max(\n", " audio_prompt_id_to_continue_at[audio_prompt_id], row[\"continue_at\"]\n", " )\n", "print(len(audio_prompt_id_to_continue_at))\n", "df[\"has_continue_and_start_continue_at\"] = df[\"id\"].apply(\n", " lambda x: audio_prompt_id_to_continue_at.get(x)\n", ")\n", "# we want continue at to be at most of the clip...\n", "df[\"good_continue_at\"] = (\n", " (df[\"has_continue_and_start_continue_at\"] / df[\"duration\"]) > 0.9\n", ") | df[\"has_continue_and_start_continue_at\"].isna()\n", "print(df[\"good_continue_at\"].value_counts())\n", "\n", "\n", "print(\n", " \"\\n Check some basics... \\n\",\n", " df[\"preference\"].value_counts(),\n", " df[\"is_30b\"].value_counts(),\n", " df[\"model_name\"].value_counts(),\n", " df.groupby([\"preference\"])[\"model_name\"].value_counts(),\n", ")\n", "\n", "df = df.sort_values(by=[\"request_id\", \"preference\"])\n", "df[\"duration_rel_diff\"] = df[\"duration\"].diff()\n", "df[\"play_rel_diff\"] = df[\"reaction_play_count\"].diff()\n", "print(df[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.035167Z", "start_time": "2024-05-16T13:59:40.801098Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:22:40.695318Z", "iopub.status.busy": "2024-10-03T02:22:40.695025Z", "iopub.status.idle": "2024-10-03T02:22:40.874388Z", "shell.execute_reply": "2024-10-03T02:22:40.873829Z", "shell.execute_reply.started": "2024-10-03T02:22:40.695301Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "negative 39367 positive 35707\n", "total pair requests 40018 --> selected pair requests 35171 frac 0.879\n" ] } ], "source": [ "normal_pos_play_count = 3\n", "# this is lower, cause a concat is probably already ensuring that it is good\n", "concat_pos_play_count = 1\n", "# this is a filter on the concated clip\n", "concat_total_play_count = 3\n", "\n", "neg_filter_selection_mask = (\n", " (~df[\"preference\"]) # get basics aligned\n", " & (df[\"reaction_play_count\"] >= 1) # has to be played once\n", " # & (df[\"play_count\"] <= 3) # if it is actually bad, shouldn't be listened often\n", " & (df[\"duration\"] >= 10) # can't be too short, otherwise it is obvious\n", " & (df[\"duration\"] <= 240) # can't be badly long\n", " & (df[\"has_continue_and_start_continue_at\"].isna()) # won't have any continues\n", " & (df[\"upvote_count\"] == 0) # won't have any likes\n", " & ((df[\"norm_play_frac\"] <= 3.1))\n", " # & (df[\"dislike_count\"] >= 1) # this is kinda strict\n", " # & (\n", " # (df_slice[\"is_in_playlist\"] == False)\n", " # & (df_slice[\"concat_in_playlist\"] == False)\n", " # ) # can't be part of a playlist -- otherwise there are some like signal in it?\n", ")\n", "pos_filter_selectin_mask = (\n", " (df[\"preference\"]) # get basics aligned\n", " & (\n", " df[\"good_continue_at\"]\n", " ) # if continue, needs to continue off a certain percentage\n", " & (df[\"reaction_play_count\"] >= 1)\n", " & (df[\"play_rel_diff\"] >= 0) # this is more like quality assurance\n", " & (df[\"duration\"] >= 10) # can't be too short, otherwise it is obvious\n", " & (df[\"duration\"] <= 240) # can't be badly long\n", " & (df[\"dislike_count\"] == 0) # can't have dislikes\n", " & (df[\"flag_count\"] == 0) # can't have issues\n", " & (\n", " (\n", " (df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= concat_pos_play_count)\n", " & (df[\"concat_play_counts\"] >= concat_total_play_count)\n", " )\n", " | (\n", " (~df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= normal_pos_play_count)\n", " # & (df[\"norm_play_frac\"] >= 1.25)\n", " )\n", " )\n", " # & (df[\"norm_play_frac\"] >= 1.9)\n", " # & (df[\"user_n_clips\"] >= 40) # user needs to have genereated at least 20\n", " # & (df[\"duration_rel_diff\"] < 10) # positive isn't just longer\n", " # & (df[\"task\"] == \"cover\")\n", " & ((df[\"upvote_count\"] >= 1) | (df[\"reaction_play_count\"] >= 10)| (df[\"concat_play_counts\"] >= 10))\n", " # & (df[\"pos_diff_preference\"] == 2)\n", " # & ((0 < df[\"similarity\"]) & (df[\"similarity\"] <= 0.99))\n", " & ((df[\"cer_diff_preference\"] < 0.5) & (df[\"cer\"] < 0.99)) # cut on hoot cer difference and abs cer\n", ")\n", "print(\n", " \"negative\",\n", " sum(neg_filter_selection_mask),\n", " \"positive\",\n", " sum(pos_filter_selectin_mask),\n", ")\n", "\n", "neg_filter_requests = df[neg_filter_selection_mask][\"request_id\"].unique()\n", "pos_filter_requests = df[pos_filter_selectin_mask][\"request_id\"].unique()\n", "# looking for very strong signal here:\n", "# listen to the positive/negative more than once\n", "# disliked one of the clips\n", "unique_requests = set(pos_filter_requests).intersection(neg_filter_requests)\n", "print(\n", " \"total pair requests\",\n", " df[\"request_id\"].nunique(),\n", " \" --> selected pair requests\",\n", " len(unique_requests),\n", " f\"frac {len(unique_requests) / df['request_id'].nunique():.3f}\",\n", ")" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.250737Z", "start_time": "2024-05-16T13:59:41.036434Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:22:42.006892Z", "iopub.status.busy": "2024-10-03T02:22:42.006497Z", "iopub.status.idle": "2024-10-03T02:22:42.278575Z", "shell.execute_reply": "2024-10-03T02:22:42.277998Z", "shell.execute_reply.started": "2024-10-03T02:22:42.006876Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "30b_t4_v21 requests 35171 clips 70342 total khrs 3.535; N gpus for 1000 iters 4.396; 4 gpus for x iters 1099.094; n unique users 19027 n pro users 12301\n" ] } ], "source": [ "df_slice = df[df[\"request_id\"].isin(set(unique_requests))].copy()\n", "print(\n", " f\"{os.path.basename(OUT_DATA_DIR)} requests\",\n", " df_slice[\"request_id\"].nunique(),\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"4 gpus for x iters {df_slice.shape[0] / 8 / 2 / 4:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", " f\"n pro users {df_slice[df_slice['is_pro_user']]['user_id'].nunique()}\",\n", ")\n", "# 76171 152342 total khrs 2.880 n gpus for 1250 iters 3.809\n", "# v10 has 78866\n", "# v14 has 110402\n", "# 30b_t4_v12 requests 47755 clips 95510 total khrs 4.555; N gpus for 1000 iters 5.969; 4 gpus for x iters 1492.344; n unique users 13790 n pro users 13526" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:22:43.410185Z", "iopub.status.busy": "2024-10-03T02:22:43.410012Z", "iopub.status.idle": "2024-10-03T02:22:43.412395Z", "shell.execute_reply": "2024-10-03T02:22:43.411993Z", "shell.execute_reply.started": "2024-10-03T02:22:43.410170Z" } }, "outputs": [], "source": [ "# v1 requests 17681 clips 35362 total khrs 1.815; N gpus for 1500 iters 1.473; 4 gpus for x iters 552.531; n unique users 15179 n pro users 5983" ] }, { "cell_type": "code", "execution_count": 39, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.277006Z", "start_time": "2024-05-16T13:59:41.252105Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:22:44.006291Z", "iopub.status.busy": "2024-10-03T02:22:44.006008Z", "iopub.status.idle": "2024-10-03T02:22:44.023842Z", "shell.execute_reply": "2024-10-03T02:22:44.023365Z", "shell.execute_reply.started": "2024-10-03T02:22:44.006275Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive in playlist (10422, 116)\n" ] } ], "source": [ "test_mask = (df_slice[\"preference\"]) & (\n", " (df_slice[\"is_in_playlist\"]) | (df_slice[\"concat_in_playlist\"])\n", ")\n", "print(\"positive in playlist\", df_slice[test_mask].shape)" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.323409Z", "start_time": "2024-05-16T13:59:41.278278Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:22:44.550504Z", "iopub.status.busy": "2024-10-03T02:22:44.550204Z", "iopub.status.idle": "2024-10-03T02:22:44.552484Z", "shell.execute_reply": "2024-10-03T02:22:44.552101Z", "shell.execute_reply.started": "2024-10-03T02:22:44.550489Z" } }, "outputs": [], "source": [ "# interesting_clips_must_be_positive_mask = (\n", "# (df_slice[\"upvoted\"] == True)\n", "# | (df_slice[\"has_action\"] == True)\n", "# | (df_slice[\"part_of_concat\"] == True)\n", "# )\n", "# interesting_clips_must_be_not_negative_mask = (df_slice[\"downvoted\"] == False) # & (df_slice[\"dislike_count\"] < 1)\n", "# interesting_clips_mask = interesting_clips_must_be_positive_mask & interesting_clips_must_be_not_negative_mask\n", "# assert interesting_clips_mask.eq(df_slice[\"preference\"]).all()" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.392244Z", "start_time": "2024-05-16T13:59:41.324472Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:22:44.833717Z", "iopub.status.busy": "2024-10-03T02:22:44.833449Z", "iopub.status.idle": "2024-10-03T02:22:44.835548Z", "shell.execute_reply": "2024-10-03T02:22:44.835168Z", "shell.execute_reply.started": "2024-10-03T02:22:44.833702Z" } }, "outputs": [], "source": [ "# save positive ids\n", "# positive_preference_ids = df_slice[df_slice[\"preference\"] == False][\"s3_id\"].to_json(orient='values')\n", "# with open('/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id_negative.json', 'w') as file:\n", "# file.write(positive_preference_ids)" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:22:45.170009Z", "iopub.status.busy": "2024-10-03T02:22:45.169865Z", "iopub.status.idle": "2024-10-03T02:22:45.308803Z", "shell.execute_reply": "2024-10-03T02:22:45.308422Z", "shell.execute_reply.started": "2024-10-03T02:22:45.169994Z" } }, "outputs": [], "source": [ "# df_slice_2 = pd.read_csv(\"/home/tony/Data/Preference/30b_v1/interesting_clips_v4_t_1_20240808_slice.csv\")\n", "# df_total = pd.concat([df_slice, df_slice_2])\n", "# print(df_total.shape)" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:22:45.365960Z", "iopub.status.busy": "2024-10-03T02:22:45.365826Z", "iopub.status.idle": "2024-10-03T02:22:45.367852Z", "shell.execute_reply": "2024-10-03T02:22:45.367454Z", "shell.execute_reply.started": "2024-10-03T02:22:45.365946Z" } }, "outputs": [], "source": [ "# print(df_slices_2.shape, df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 44, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:22:45.545909Z", "iopub.status.busy": "2024-10-03T02:22:45.545767Z", "iopub.status.idle": "2024-10-03T02:22:45.547718Z", "shell.execute_reply": "2024-10-03T02:22:45.547347Z", "shell.execute_reply.started": "2024-10-03T02:22:45.545895Z" } }, "outputs": [], "source": [ "# print(df_total.shape)\n", "# df_slice = df_total.copy()" ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:22:45.726231Z", "iopub.status.busy": "2024-10-03T02:22:45.725956Z", "iopub.status.idle": "2024-10-03T02:22:45.966985Z", "shell.execute_reply": "2024-10-03T02:22:45.966600Z", "shell.execute_reply.started": "2024-10-03T02:22:45.726216Z" } }, "outputs": [], "source": [ "# df_slice_prev = pd.read_pickle(\"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_v15_20240902_slice.pkl\")\n", "# df_slice_prev = df_slice_prev[((df_slice_prev[\"task\"] != \"infill\") & (df_slice_prev[\"task\"] != \"cover\") & (df_slice_prev[\"task\"] != \"artist_consistency\"))].copy()\n", "# df_slice = pd.concat([df_slice, df_slice_prev])\n", "# print(df_slice.shape)\n", "# print(df_slice[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T14:00:20.866354Z", "start_time": "2024-05-16T14:00:12.443344Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:22:45.967818Z", "iopub.status.busy": "2024-10-03T02:22:45.967657Z", "iopub.status.idle": "2024-10-03T02:22:46.229771Z", "shell.execute_reply": "2024-10-03T02:22:46.229227Z", "shell.execute_reply.started": "2024-10-03T02:22:45.967803Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(70342, 116)\n", "task\n", "cover 36056\n", " 28904\n", "extend 5362\n", "artist_consistency 20\n", "Name: count, dtype: int64\n" ] }, { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[46], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mvalue_counts())\n\u001b[0;32m----> 4\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "# df_slice.to_pickle(\"/home/tony/Data/Preference/30b_v3/interesting_clips_v4_t_4_v3_20240916_full_with_cer.pkl\")\n", "print(df_slice.shape)\n", "print(df_slice[\"task\"].value_counts())\n", "BREAK" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932296Z", "start_time": "2024-05-16T13:59:41.932287Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:22:49.374361Z", "iopub.status.busy": "2024-10-03T02:22:49.373962Z", "iopub.status.idle": "2024-10-03T02:22:49.436665Z", "shell.execute_reply": "2024-10-03T02:22:49.436218Z", "shell.execute_reply.started": "2024-10-03T02:22:49.374345Z" } }, "outputs": [], "source": [ "# don't have continue at\n", "df_slice[\"request_id\"] = df_slice[\"request_id\"].astype(str)\n", "# df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique(), df_slice[\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:22:50.254375Z", "iopub.status.busy": "2024-10-03T02:22:50.254222Z", "iopub.status.idle": "2024-10-03T02:22:50.382433Z", "shell.execute_reply": "2024-10-03T02:22:50.381865Z", "shell.execute_reply.started": "2024-10-03T02:22:50.254360Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(70342, 116)\n", "(70342, 116)\n", "(70342, 116)\n" ] } ], "source": [ "print(df_slice.shape)\n", "df_slice = df_slice[df_slice[\"request_id\"].apply(lambda x: len(x) > 3)]\n", "print(df_slice.shape)\n", "# df_slice = df_slice[df_slice[\"is_pro_user\"]].copy()\n", "print(df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 49, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932966Z", "start_time": "2024-05-16T13:59:41.932957Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:22:54.823813Z", "iopub.status.busy": "2024-10-03T02:22:54.823513Z", "iopub.status.idle": "2024-10-03T02:22:54.833743Z", "shell.execute_reply": "2024-10-03T02:22:54.833224Z", "shell.execute_reply.started": "2024-10-03T02:22:54.823796Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "35171\n" ] } ], "source": [ "final_filtered_requests = df_slice[\"request_id\"].astype(str).unique()\n", "# final_filtered_requests = df_slice[df_slice[\"is_pro_user\"]][\"request_id\"].astype(str).unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": 50, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.933558Z", "start_time": "2024-05-16T13:59:41.933550Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:22:54.834801Z", "iopub.status.busy": "2024-10-03T02:22:54.834651Z", "iopub.status.idle": "2024-10-03T02:22:54.875908Z", "shell.execute_reply": "2024-10-03T02:22:54.875513Z", "shell.execute_reply.started": "2024-10-03T02:22:54.834786Z" } }, "outputs": [], "source": [ "# df_slice.to_csv(\"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_20240902_slice.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T02:22:54.876568Z", "iopub.status.busy": "2024-10-03T02:22:54.876436Z", "iopub.status.idle": "2024-10-03T02:22:54.915154Z", "shell.execute_reply": "2024-10-03T02:22:54.914778Z", "shell.execute_reply.started": "2024-10-03T02:22:54.876555Z" } }, "outputs": [], "source": [ "# df_slice[\"continue_at\"] = -1" ] }, { "cell_type": "code", "execution_count": 52, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934277Z", "start_time": "2024-05-16T13:59:41.934268Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:22:54.915787Z", "iopub.status.busy": "2024-10-03T02:22:54.915650Z", "iopub.status.idle": "2024-10-03T02:22:55.138960Z", "shell.execute_reply": "2024-10-03T02:22:55.138405Z", "shell.execute_reply.started": "2024-10-03T02:22:54.915774Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "34819 352\n", "(69638, 116) (704, 116)\n" ] } ], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df# .reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df# .reset_index()\n", "\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": 53, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934954Z", "start_time": "2024-05-16T13:59:41.934946Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:22:55.140562Z", "iopub.status.busy": "2024-10-03T02:22:55.140253Z", "iopub.status.idle": "2024-10-03T02:22:55.142529Z", "shell.execute_reply": "2024-10-03T02:22:55.142133Z", "shell.execute_reply.started": "2024-10-03T02:22:55.140546Z" } }, "outputs": [], "source": [ "# val_df[[\"request_id\", \"metadata\", \"updated_at\", \"user_id\", \"preference\"]].head()" ] }, { "cell_type": "code", "execution_count": 56, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.935620Z", "start_time": "2024-05-16T13:59:41.935613Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:23:12.619942Z", "iopub.status.busy": "2024-10-03T02:23:12.619613Z", "iopub.status.idle": "2024-10-03T02:23:14.971062Z", "shell.execute_reply": "2024-10-03T02:23:14.970499Z", "shell.execute_reply.started": "2024-10-03T02:23:12.619924Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 69638/69638 [00:02<00:00, 29683.71it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "3,500 hours of 69638 clips, 4.352375 nodes, 1088.09375 iters\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "total_duration = 0\n", "for i, row in tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " total_duration += row[\"duration\"]\n", " except Exception as E:\n", " print(i, row)\n", " print(E)\n", " raise ValueError()\n", " \n", "print(\n", " f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1000} nodes, {train_df.shape[0] / 8 / 2 / 4} iters\"\n", ")" ] }, { "cell_type": "code", "execution_count": 57, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936268Z", "start_time": "2024-05-16T13:59:41.936260Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:23:14.972250Z", "iopub.status.busy": "2024-10-03T02:23:14.972084Z", "iopub.status.idle": "2024-10-03T02:23:48.264575Z", "shell.execute_reply": "2024-10-03T02:23:48.264046Z", "shell.execute_reply.started": "2024-10-03T02:23:14.972234Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████| 704/704 [00:33<00:00, 21.19it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 704 clips, 1 different prompts\n", "21 hours of False\n", "21 hours of True\n", "Done\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "make_dataset(val_df, OUT_DATA_DIR, is_val=True, npz_dir=NPZ_DIR)" ] }, { "cell_type": "code", "execution_count": 58, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936964Z", "start_time": "2024-05-16T13:59:41.936957Z" }, "execution": { "iopub.execute_input": "2024-10-03T02:23:48.265403Z", "iopub.status.busy": "2024-10-03T02:23:48.265248Z", "iopub.status.idle": "2024-10-03T03:25:12.619798Z", "shell.execute_reply": "2024-10-03T03:25:12.619250Z", "shell.execute_reply.started": "2024-10-03T02:23:48.265388Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 49%|███████████████████████████████████████████████████▉ | 34447/69638 [30:40<19:25, 30.19it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "weird, /app/suno/data/dpo/30b_npz/84eca617-5737-416f-b87f-344e8ce43aec.npz, with only v3.5\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 69638/69638 [1:01:24<00:00, 18.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 69638 clips, 14 different prompts\n", "2,077 hours of False\n", "2,064 hours of True\n", "Done\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "make_dataset(train_df, OUT_DATA_DIR, is_val=False, npz_dir=NPZ_DIR)" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": 59, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.937879Z", "start_time": "2024-05-16T13:59:41.937870Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:12.620658Z", "iopub.status.busy": "2024-10-03T03:25:12.620499Z", "iopub.status.idle": "2024-10-03T03:25:12.644841Z", "shell.execute_reply": "2024-10-03T03:25:12.644380Z", "shell.execute_reply.started": "2024-10-03T03:25:12.620641Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, 6016, 13)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000\n", "assert mm[:100, :, 1:].min() >= 0\n", "assert mm[:100, :, 1:].max() <= 2048" ] }, { "cell_type": "code", "execution_count": 60, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.938629Z", "start_time": "2024-05-16T13:59:41.938621Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:12.646363Z", "iopub.status.busy": "2024-10-03T03:25:12.646071Z", "iopub.status.idle": "2024-10-03T03:25:12.686217Z", "shell.execute_reply": "2024-10-03T03:25:12.685809Z", "shell.execute_reply.started": "2024-10-03T03:25:12.646347Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as codec_encode,\n", "# decode_stream_to_full_audio as codec_decode,\n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# decode as decode\n", "# )\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "# assert len(test_metas) == len(mm)\n", "# idx_list = list(range(len(test_metas)))\n", "# # random.shuffle(idx_list)\n", "# # idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": 61, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939205Z", "start_time": "2024-05-16T13:59:41.939198Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:12.686896Z", "iopub.status.busy": "2024-10-03T03:25:12.686761Z", "iopub.status.idle": "2024-10-03T03:25:12.732064Z", "shell.execute_reply": "2024-10-03T03:25:12.731680Z", "shell.execute_reply.started": "2024-10-03T03:25:12.686882Z" } }, "outputs": [], "source": [ "# import random\n", "# idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "# assert \"original_duration_s\" in test_metas[idx]\n", "# # positive index should be shifted by 1\n", "# pos_idx = idx + 1\n", "# print(\n", "# \"tags:\",\n", "# test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", "# test_metas[idx].get(\"tags\"),\n", "# )\n", "# arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[: pad_idx_arr[0], :]\n", "# pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pos_pad_idx_arr) > 0:\n", "# pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "# a = decode(arr)\n", "# print(\"\\n negative example \\n\", test_metas[idx])\n", "# a.play(compress=False)\n", "# pos_a = decode(pos_arr)\n", "# print(\"\\n positive example \\n\", test_metas[pos_idx])\n", "# pos_a.play(compress=False)\n", "# print(\n", "# \"text:\",\n", "# test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", "# test_metas[idx].get(\"text\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": 62, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939977Z", "start_time": "2024-05-16T13:59:41.939969Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:12.732897Z", "iopub.status.busy": "2024-10-03T03:25:12.732753Z", "iopub.status.idle": "2024-10-03T03:25:12.777692Z", "shell.execute_reply": "2024-10-03T03:25:12.777300Z", "shell.execute_reply.started": "2024-10-03T03:25:12.732882Z" } }, "outputs": [], "source": [ "# val_df[val_df[\"tags\"] == 'a vibrant blend of experimental jazz fusion, drum-and-bass and swagger fuzzed-out guitars']" ] }, { "cell_type": "code", "execution_count": 63, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.940610Z", "start_time": "2024-05-16T13:59:41.940603Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:12.778360Z", "iopub.status.busy": "2024-10-03T03:25:12.778225Z", "iopub.status.idle": "2024-10-03T03:25:12.822110Z", "shell.execute_reply": "2024-10-03T03:25:12.821728Z", "shell.execute_reply.started": "2024-10-03T03:25:12.778346Z" } }, "outputs": [], "source": [ "# from collections import Counter\n", "# c = Counter()\n", "# for _, row in df_slice.iterrows():\n", "# # print(row[\"metadata\"])\n", "# for k in ast.literal_eval(row[\"metadata\"]).keys():\n", "# c[k] += 1" ] }, { "cell_type": "code", "execution_count": 64, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941167Z", "start_time": "2024-05-16T13:59:41.941159Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:12.822750Z", "iopub.status.busy": "2024-10-03T03:25:12.822624Z", "iopub.status.idle": "2024-10-03T03:25:13.299040Z", "shell.execute_reply": "2024-10-03T03:25:13.298621Z", "shell.execute_reply.started": "2024-10-03T03:25:12.822736Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": 65, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941801Z", "start_time": "2024-05-16T13:59:41.941793Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:13.299778Z", "iopub.status.busy": "2024-10-03T03:25:13.299637Z", "iopub.status.idle": "2024-10-03T03:25:13.345734Z", "shell.execute_reply": "2024-10-03T03:25:13.345281Z", "shell.execute_reply.started": "2024-10-03T03:25:13.299764Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "352 0\n" ] } ], "source": [ "def validation_on_metas(input_metas):\n", " total_bad = 0\n", " total_good = 0\n", " for idx in range(len(input_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if input_metas[idx].get(\"tags\") != input_metas[pos_idx].get(\"tags\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", " print(total_good, total_bad)\n", " return\n", "\n", "\n", "validation_on_metas(test_metas)" ] }, { "cell_type": "code", "execution_count": 66, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.942520Z", "start_time": "2024-05-16T13:59:41.942511Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:13.346446Z", "iopub.status.busy": "2024-10-03T03:25:13.346310Z", "iopub.status.idle": "2024-10-03T03:25:13.392086Z", "shell.execute_reply": "2024-10-03T03:25:13.391644Z", "shell.execute_reply.started": "2024-10-03T03:25:13.346432Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": 67, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.943072Z", "start_time": "2024-05-16T13:59:41.943065Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:13.392891Z", "iopub.status.busy": "2024-10-03T03:25:13.392754Z", "iopub.status.idle": "2024-10-03T03:25:13.431649Z", "shell.execute_reply": "2024-10-03T03:25:13.431240Z", "shell.execute_reply.started": "2024-10-03T03:25:13.392878Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)" ] }, { "cell_type": "code", "execution_count": 68, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.944246Z", "start_time": "2024-05-16T13:59:41.944237Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:13.432349Z", "iopub.status.busy": "2024-10-03T03:25:13.432210Z", "iopub.status.idle": "2024-10-03T03:25:13.830852Z", "shell.execute_reply": "2024-10-03T03:25:13.830433Z", "shell.execute_reply.started": "2024-10-03T03:25:13.432335Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total samples 69638 (69638, 116)\n" ] } ], "source": [ "total_iters = len(n_neg_tr) + len(n_pos_tr)\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": 69, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945249Z", "start_time": "2024-05-16T13:59:41.945241Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:13.832489Z", "iopub.status.busy": "2024-10-03T03:25:13.832330Z", "iopub.status.idle": "2024-10-03T03:25:13.870260Z", "shell.execute_reply": "2024-10-03T03:25:13.869833Z", "shell.execute_reply.started": "2024-10-03T03:25:13.832474Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 epoch per batch 4, total 1088.09375\n" ] } ], "source": [ "print(\"1 epoch per batch 4, total\", total_iters / 8 / 2 / 4)" ] }, { "cell_type": "code", "execution_count": 70, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945972Z", "start_time": "2024-05-16T13:59:41.945964Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:13.870955Z", "iopub.status.busy": "2024-10-03T03:25:13.870817Z", "iopub.status.idle": "2024-10-03T03:25:14.782800Z", "shell.execute_reply": "2024-10-03T03:25:14.782202Z", "shell.execute_reply.started": "2024-10-03T03:25:13.870941Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Submitted batch job 3028\n" ] } ], "source": [ "!cd /home/tony/Work/tony/slurm/30b_dpo && sbatch sbatch_ipo_30b" ] }, { "cell_type": "code", "execution_count": 71, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T03:25:14.783810Z", "iopub.status.busy": "2024-10-03T03:25:14.783632Z", "iopub.status.idle": "2024-10-03T03:25:14.968146Z", "shell.execute_reply": "2024-10-03T03:25:14.967689Z", "shell.execute_reply.started": "2024-10-03T03:25:14.783795Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cache kept!\n" ] } ], "source": [ "import shutil\n", "\n", "# Basic file copy\n", "shutil.copy('/home/tony/Work/tony/Preference/make_dataset_13b_v4_t4_all.ipynb', os.path.join(OUT_DATA_DIR, \"make_dataset.ipynb\"))\n", "print(\"Cache kept!\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# some gymathtics loading prev data" ] }, { "cell_type": "code", "execution_count": 72, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.946562Z", "start_time": "2024-05-16T13:59:41.946555Z" }, "execution": { "iopub.execute_input": "2024-10-03T03:25:14.968912Z", "iopub.status.busy": "2024-10-03T03:25:14.968765Z", "iopub.status.idle": "2024-10-03T03:25:15.050571Z", "shell.execute_reply": "2024-10-03T03:25:15.050165Z", "shell.execute_reply.started": "2024-10-03T03:25:14.968898Z" } }, "outputs": [], "source": [ "# prev_v3_data = \"/app/suno/data/dpo/7v_v20_full/\"\n", "\n", "# test_val_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_val.jsonl\"))\n", "# test_tr_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_tr.jsonl\"))\n", "\n", "# all_ids = set()\n", "# for meta in test_val_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# for meta in test_tr_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# print(len(all_ids), len(test_val_metas) + len(test_tr_metas))\n", "\n", "# all_ids = list(all_ids)\n", "# with open(\"/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id.json\", \"w\") as fp:\n", "# json.dump(all_ids, fp)" ] }, { "cell_type": "code", "execution_count": 73, "metadata": { "execution": { "iopub.execute_input": "2024-10-03T03:25:15.051390Z", "iopub.status.busy": "2024-10-03T03:25:15.051260Z", "iopub.status.idle": "2024-10-03T03:25:15.209298Z", "shell.execute_reply": "2024-10-03T03:25:15.208711Z", "shell.execute_reply.started": "2024-10-03T03:25:15.051377Z" } }, "outputs": [ { "ename": "KeyError", "evalue": "\"['similarity'] not in index\"", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[73], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mval_df\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtail\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43ms3_id\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mprompt\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msimilarity\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpreference\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\n", "File \u001b[0;32m~/anaconda3/envs/suno_env/lib/python3.10/site-packages/pandas/core/frame.py:4108\u001b[0m, in \u001b[0;36mDataFrame.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 4106\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_iterator(key):\n\u001b[1;32m 4107\u001b[0m key \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(key)\n\u001b[0;32m-> 4108\u001b[0m indexer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_indexer_strict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcolumns\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m[\u001b[38;5;241m1\u001b[39m]\n\u001b[1;32m 4110\u001b[0m \u001b[38;5;66;03m# take() does not accept boolean indexers\u001b[39;00m\n\u001b[1;32m 4111\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(indexer, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdtype\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mbool\u001b[39m:\n", "File \u001b[0;32m~/anaconda3/envs/suno_env/lib/python3.10/site-packages/pandas/core/indexes/base.py:6200\u001b[0m, in \u001b[0;36mIndex._get_indexer_strict\u001b[0;34m(self, key, axis_name)\u001b[0m\n\u001b[1;32m 6197\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 6198\u001b[0m keyarr, indexer, new_indexer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reindex_non_unique(keyarr)\n\u001b[0;32m-> 6200\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_raise_if_missing\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkeyarr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis_name\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 6202\u001b[0m keyarr \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtake(indexer)\n\u001b[1;32m 6203\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(key, Index):\n\u001b[1;32m 6204\u001b[0m \u001b[38;5;66;03m# GH 42790 - Preserve name from an Index\u001b[39;00m\n", "File \u001b[0;32m~/anaconda3/envs/suno_env/lib/python3.10/site-packages/pandas/core/indexes/base.py:6252\u001b[0m, in \u001b[0;36mIndex._raise_if_missing\u001b[0;34m(self, key, indexer, axis_name)\u001b[0m\n\u001b[1;32m 6249\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNone of [\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mkey\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m] are in the [\u001b[39m\u001b[38;5;132;01m{\u001b[39;00maxis_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m]\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 6251\u001b[0m not_found \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(ensure_index(key)[missing_mask\u001b[38;5;241m.\u001b[39mnonzero()[\u001b[38;5;241m0\u001b[39m]]\u001b[38;5;241m.\u001b[39munique())\n\u001b[0;32m-> 6252\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mnot_found\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m not in index\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", "\u001b[0;31mKeyError\u001b[0m: \"['similarity'] not in index\"" ] } ], "source": [ "val_df.tail(n=10)[[\"s3_id\", \"prompt\", \"similarity\", \"preference\"]]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2024-10-03T03:25:15.209782Z", "iopub.status.idle": "2024-10-03T03:25:15.209961Z", "shell.execute_reply": "2024-10-03T03:25:15.209879Z", "shell.execute_reply.started": "2024-10-03T03:25:15.209870Z" } }, "outputs": [], "source": [ "x_data = train_df[train_df[\"preference\"]][\"similarity\"]\n", "y_data = train_df[~train_df[\"preference\"]][\"similarity\"]\n", "from matplotlib.colors import LogNorm\n", "\n", "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 10))\n", "\n", "# 2D Histogram\n", "h = ax1.hist2d(\n", " x_data,\n", " y_data,\n", " bins=(50, 50),\n", " cmap=\"coolwarm\",\n", " range=[[0, 1], [0, 1]],\n", " norm=LogNorm(),\n", ")\n", "\n", "ax1.set_xlabel(\"Semantic Distance (Preferred)\")\n", "ax1.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "ax1.set_title(\n", " \"2D Histogram of Semantic Distances: Preferred vs Non-Preferred (Log Scale)\"\n", ")\n", "\n", "cbar1 = plt.colorbar(h[3], ax=ax1)\n", "cbar1.set_label(\"Number of Request IDs (Log Scale)\")\n", "\n", "# Scatter plot\n", "ax2.scatter(x_data, y_data, alpha=0.1, s=1)\n", "ax2.set_xlabel(\"Semantic Distance (Preferred)\")\n", "ax2.set_ylabel(\"Semantic Distance (Non-Preferred)\")\n", "ax2.set_title(\"Scatter Plot of Semantic Distances: Preferred vs Non-Preferred\")\n", "ax2.set_xlim(0, 1)\n", "ax2.set_ylim(0, 1)\n", "\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }