{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.040680Z", "start_time": "2024-05-16T13:58:19.777010Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:11:17.966505Z", "iopub.status.busy": "2024-09-04T00:11:17.966365Z", "iopub.status.idle": "2024-09-04T00:11:19.358103Z", "shell.execute_reply": "2024-09-04T00:11:19.357590Z", "shell.execute_reply.started": "2024-09-04T00:11:17.966490Z" } }, "outputs": [], "source": [ "import ast\n", "import os\n", "import shutil\n", "import sys\n", "from collections import defaultdict\n", "\n", "import numpy as np\n", "import pandas as pd\n", "from preference_data_preparation_4min_30b import *\n", "from preference_helper import *\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n", "from tqdm import tqdm\n", "\n", "pd.set_option(\"display.max_rows\", 500)\n", "pd.set_option(\"display.max_columns\", 500)\n", "pd.set_option(\"display.width\", 1000)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.082172Z", "start_time": "2024-05-16T13:58:21.041926Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:11:19.358946Z", "iopub.status.busy": "2024-09-04T00:11:19.358754Z", "iopub.status.idle": "2024-09-04T00:11:19.384475Z", "shell.execute_reply": "2024-09-04T00:11:19.384028Z", "shell.execute_reply.started": "2024-09-04T00:11:19.358931Z" } }, "outputs": [], "source": [ "OUT_DATA_DIR = \"/app/suno/data/dpo/30b_t2_v16\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "shutil.copyfile(\n", " \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", " os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", ")\n", "NPZ_DIR = \"/app/suno/data/dpo/30b_npz\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:53.962528Z", "start_time": "2024-05-16T13:58:21.105919Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:11:19.385292Z", "iopub.status.busy": "2024-09-04T00:11:19.385158Z", "iopub.status.idle": "2024-09-04T00:11:21.589922Z", "shell.execute_reply": "2024-09-04T00:11:21.589342Z", "shell.execute_reply.started": "2024-09-04T00:11:19.385279Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Preference data shape (90094, 101)\n" ] } ], "source": [ "# df = pd.read_pickle(\n", "# \"/home/tony/Data/Preference/30b_v1/interesting_clips_v4_t_2_20240830_full.pkl\"\n", "# )\n", "df = pd.read_pickle(\n", " \"/home/tony/Data/Preference/30b_v1/interesting_clips_v4_t_2_20240830_v16_slice_with_cer.pkl\"\n", ")\n", "# df = pd.read_csv(\n", "# \"/home/tony/Data/Preference/30b_v0/interesting_clips_v4_t_1_20240808.csv\"\n", "# ) # , engine='python')\n", "print(\"Preference data shape\", df.shape)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2024-09-04T00:11:21.590757Z", "iopub.status.busy": "2024-09-04T00:11:21.590596Z", "iopub.status.idle": "2024-09-04T00:11:21.758876Z", "shell.execute_reply": "2024-09-04T00:11:21.758306Z", "shell.execute_reply.started": "2024-09-04T00:11:21.590742Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(90094, 101)\n", "(90094, 97)\n" ] } ], "source": [ "print(df.shape)\n", "df = df.dropna(axis=1, how=\"all\")\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.199480Z", "start_time": "2024-05-16T13:58:53.963687Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:11:21.759709Z", "iopub.status.busy": "2024-09-04T00:11:21.759555Z", "iopub.status.idle": "2024-09-04T00:11:39.882680Z", "shell.execute_reply": "2024-09-04T00:11:39.882104Z", "shell.execute_reply.started": "2024-09-04T00:11:21.759693Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "897311\n", "897311\n", "pre-downloaded df (90094, 97)\n", "downloaded df (90094, 97)\n" ] } ], "source": [ "converted_paths = os.listdir(NPZ_DIR)\n", "print(len(converted_paths))\n", "\n", "converted_paths = set([f.replace(\".npz\", \"\") for f in converted_paths])\n", "print(len(converted_paths))\n", "\n", "print(\"pre-downloaded df\", df.shape)\n", "df[df[\"s3_id\"].isin(converted_paths)].shape\n", "df = df[df[\"s3_id\"].isin(converted_paths)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.467253Z", "start_time": "2024-05-16T13:58:56.207647Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:11:39.884494Z", "iopub.status.busy": "2024-09-04T00:11:39.884251Z", "iopub.status.idle": "2024-09-04T00:11:39.905307Z", "shell.execute_reply": "2024-09-04T00:11:39.904874Z", "shell.execute_reply.started": "2024-09-04T00:11:39.884478Z" } }, "outputs": [ { "data": { "text/plain": [ "is_30b\n", "True 90094\n", "Name: count, dtype: int64" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"is_30b\"] = df[\"model_name\"].str.contains(\"-t\")\n", "df[\"is_30b\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.592883Z", "start_time": "2024-05-16T13:58:56.470781Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:11:39.906015Z", "iopub.status.busy": "2024-09-04T00:11:39.905879Z", "iopub.status.idle": "2024-09-04T00:11:40.002832Z", "shell.execute_reply": "2024-09-04T00:11:40.002325Z", "shell.execute_reply.started": "2024-09-04T00:11:39.906001Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "preference model_name \n", "False chirp-v3p5-engine-t-2 45047\n", "True chirp-v3p5-engine-t-2 45047\n", "Name: count, dtype: int64\n", "(90094, 97)\n", "(90094, 97)\n" ] } ], "source": [ "## for 13b this is easy for now\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "print(df.shape)\n", "df = df[df[\"model_name\"].isin([\"chirp-v3p5-engine-t-2\"])]\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.909539Z", "start_time": "2024-05-16T13:58:56.595736Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:11:40.003610Z", "iopub.status.busy": "2024-09-04T00:11:40.003464Z", "iopub.status.idle": "2024-09-04T00:11:40.223532Z", "shell.execute_reply": "2024-09-04T00:11:40.222980Z", "shell.execute_reply.started": "2024-09-04T00:11:40.003595Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(90094, 97)\n", "(90094, 97)\n", "preference model_name \n", "False chirp-v3p5-engine-t-2 45047\n", "True chirp-v3p5-engine-t-2 45047\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(df.shape)\n", "df = df[\n", " df[\"request_id\"].isin(\n", " df[\"request_id\"].value_counts().index[df[\"request_id\"].value_counts() == 2]\n", " )\n", "]\n", "print(df.shape)\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "assert df.shape[0] == df[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2024-09-04T00:11:40.224326Z", "iopub.status.busy": "2024-09-04T00:11:40.224180Z", "iopub.status.idle": "2024-09-04T00:12:01.327888Z", "shell.execute_reply": "2024-09-04T00:12:01.327309Z", "shell.execute_reply.started": "2024-09-04T00:11:40.224311Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "unique_requests 45047\n" ] } ], "source": [ "# Let's use the old selection for now -- for quality assurance\n", "# expand the metadata columns -- this takes forever...~ 6 mins\n", "test_slice = df[\"metadata\"].apply(lambda x: ast.literal_eval(str(x)))\n", "test_slice_series = test_slice.apply(pd.Series)\n", "df = pd.concat([df, test_slice_series], axis=1, join=\"inner\")\n", "print(\"unique_requests\", df[\"request_id\"].nunique())" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "execution": { "iopub.execute_input": "2024-09-04T00:12:01.328713Z", "iopub.status.busy": "2024-09-04T00:12:01.328564Z", "iopub.status.idle": "2024-09-04T00:12:01.552005Z", "shell.execute_reply": "2024-09-04T00:12:01.551408Z", "shell.execute_reply.started": "2024-09-04T00:12:01.328698Z" } }, "outputs": [], "source": [ "df = df.loc[:, ~df.columns.duplicated()].copy()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "execution": { "iopub.execute_input": "2024-09-04T00:12:01.552896Z", "iopub.status.busy": "2024-09-04T00:12:01.552742Z", "iopub.status.idle": "2024-09-04T00:12:01.732682Z", "shell.execute_reply": "2024-09-04T00:12:01.732207Z", "shell.execute_reply.started": "2024-09-04T00:12:01.552881Z" } }, "outputs": [ { "data": { "text/plain": [ "pos_diff_preference\n", "1.0 31470\n", "2.0 13577\n", "Name: count, dtype: int64" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df.sort_values(by=[\"request_id\", \"preference\", \"diff_preference\"])\n", "df[\"pos_diff_preference\"] = df[\"diff_preference\"].diff()\n", "df[\"cer_diff_preference\"] = df[\"cer\"].diff()\n", "df[df[\"preference\"]][\"pos_diff_preference\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "execution": { "iopub.execute_input": "2024-09-04T00:12:01.733482Z", "iopub.status.busy": "2024-09-04T00:12:01.733331Z", "iopub.status.idle": "2024-09-04T00:12:01.772088Z", "shell.execute_reply": "2024-09-04T00:12:01.771565Z", "shell.execute_reply.started": "2024-09-04T00:12:01.733467Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive param_experiment\n", "text_cfg_12 2824\n", "tag_cfg_steps_max_text_cfg_13 2788\n", "tag_cfg_steps_max_text_cfg_12 2735\n", "text_cfg_13 2600\n", "n_repeat_tags_3 2234\n", "max_cfg_vs_n_repeat_tags_3 2033\n", "max_cfg_vs_sem_85 1921\n", "max_cfg_vs_sem_95 1872\n", "max_cfg_vs_neg_tag_20 1870\n", "max_cfg_vs_default 1509\n", "text_cfg_11 366\n", "tag_cfg_steps_max_text_cfg_11 365\n", "tag_cfg_steps_max 351\n", "Name: count, dtype: int64\n" ] } ], "source": [ "try:\n", " print(\"positive\", df[df[\"preference\"]][\"param_experiment\"].value_counts())\n", "except:\n", " pass" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "execution": { "iopub.execute_input": "2024-09-04T00:12:01.772862Z", "iopub.status.busy": "2024-09-04T00:12:01.772717Z", "iopub.status.idle": "2024-09-04T00:12:02.149714Z", "shell.execute_reply": "2024-09-04T00:12:02.149212Z", "shell.execute_reply.started": "2024-09-04T00:12:01.772847Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.20659999999999998\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGhCAYAAACDNqXeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1RElEQVR4nO3de3RU5b3/8c9MLoiExJgEKIYubSgBITeIi5IOB6tcPEY8jTcsKmBTURTBFkGL4U4BOdEDiAUaMDVIRRag1QKeihXFFbBCCTFyDQJCaSUTICFcc9m/PzjZP0dikgmTyzy8X2uxltn7O0/2d55M/GRfHZZlWQIAADCMs7k3AAAAoDEQcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkbwOOZ9//rmeeOIJuVwuxcbGauPGjfa68vJy/fd//7cGDx6sxMREuVwuTZgwQd98843HGKdOndK4cePUs2dPJScna+LEiTpz5oxHzZ49ezR06FDFxcWpX79+ysrKumxbNmzYoDvuuENxcXEaPHiwPv74Y2/bAQAAhvI65Jw9e1axsbGaMmXKZevOnz+vXbt2adSoUVq7dq0WLlyogwcPatSoUR51zz77rAoLC5Wdna3Fixdr27Ztmjx5sr2+rKxM6enp6tixo9auXasJEyZo4cKFeuutt+yaf/zjHxo3bpzuu+8+vfPOO7r99tv11FNPad++fd62BAAADOS4kgd0xsbG6tVXX1X//v2/tyY/P1/333+/PvroI3Xs2FEHDhzQnXfeqdWrVysuLk6S9Mknn2jkyJH6+OOP1b59e/3pT3/SvHnz9Omnnyo4OFiSlJmZqY0bN+r999+XJD3zzDM6d+6clixZYn+vBx54QF27dtX06dMb2hIAADBEo5+TU1ZWJofDodDQUEnSjh07FBoaagccSUpJSZHT6VR+fr4kKS8vT8nJyXbAkSSXy6WDBw+qpKTErunTp4/H93K5XMrLy2vkjgAAgD9o1JBz4cIFZWZmKjU1VSEhIZIkt9ut66+/3qMuMDBQYWFhKioqsmsiIyM9aqq/drvd31sTERFhrwcAAFe3wMYauLy8XGPHjpVlWZo2bVpjfRufKC4+rYYftLucwyFFRLT1+bgtiek90p//M71H+vN/pvfYmP1Vj12XRgk55eXleuaZZ3Ts2DG9/vrr9l4c6dIemRMnTnjUV1RUqKSkRFFRUXbNd/fIVH9dvfemppri4uLL9u7Uh2WpUX7AGmvclsT0HunP/5neI/35P9N7bM7+fH64qjrgHD58WH/84x8VHh7usT4pKUmlpaUqKCiwl23dulVVVVWKj4+XJCUmJmrbtm0qLy+3a3Jzc3XTTTcpLCzMrtm6davH2Lm5uUpMTPR1SwAAwA95HXLOnDmj3bt3a/fu3ZKko0ePavfu3Tp27JjKy8s1ZswYFRQUKDMzU5WVlSoqKlJRUZEuXrwoSYqJiVHfvn01adIk5efna/v27ZoxY4ZSU1PVvn17SdLgwYMVFBSkF154Qfv379f69euVk5OjRx991N6OYcOGafPmzXrttdd04MABvfLKKyooKNDDDz/si/cFAAD4Oa8vIf/ss880bNiwy5anpaVp9OjRuv3222t8XU5Ojnr37i3p0s0AZ8yYob/97W9yOp0aOHCgMjIy1KZNG7t+z549mj59ur744guFh4fr4Ycf1siRIz3G3LBhg+bNm6d//vOfuvHGGzV+/Hj169fPm3YkSW6378/JiYxs6/NxWxLTe6Q//2d6j/Tn/0zvsTH7qx67zroruU+OKQg53jO9R/rzf6b3SH/+z/QeW0LI4dlVAADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRGu0p5ADQEgQE1P63XFWVpaoqA+/EBoCQA8BMTqdDlVWWwsPb1FpXUVmlklNnCTqAgQg5AIzkcDgU4HRo7ModKjxeVmNN53Yhmv9gkpxOByEHMBAhB4DRCo+X6ctjpc29GQCaASceAwAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABjJ65Dz+eef64knnpDL5VJsbKw2btzosd6yLM2fP18ul0vx8fEaMWKEDh065FFz6tQpjRs3Tj179lRycrImTpyoM2fOeNTs2bNHQ4cOVVxcnPr166esrKzLtmXDhg264447FBcXp8GDB+vjjz/2th0AAGAor0PO2bNnFRsbqylTptS4PisrS8uXL9fUqVO1atUqtW7dWunp6bpw4YJd8+yzz6qwsFDZ2dlavHixtm3bpsmTJ9vry8rKlJ6ero4dO2rt2rWaMGGCFi5cqLfeesuu+cc//qFx48bpvvvu0zvvvKPbb79dTz31lPbt2+dtSwAAwEBeh5x+/frp17/+tQYMGHDZOsuylJOTo1GjRql///7q2rWr5s6dq+PHj9t7fA4cOKDNmzdr5syZSkhIUHJysjIyMrRu3Tp98803kqR3331X5eXlmjVrln784x8rNTVVjzzyiLKzs+3vlZOTo759++pXv/qVYmJi9Mwzz+jmm2/WG2+80dD3AgAAGCTQl4MdPXpURUVFSklJsZe1bdtWCQkJ2rFjh1JTU7Vjxw6FhoYqLi7OrklJSZHT6VR+fr4GDBigvLw8JScnKzg42K5xuVzKyspSSUmJwsLClJeXpxEjRnh8f5fLddnhs/pwOLzvtT7j+XrclsT0HunP/3nbm7+9F6bPoen9Seb32Jj91XdMn4acoqIiSVJERITH8oiICLndbkmS2+3W9ddf77kRgYEKCwuzX+92uxUdHe1RExkZaa8LCwuT2+22l9X0fbwREdHW69c057gtiek90t/VITy8TXNvQoOZPoem9yeZ32Nz9ufTkOOviotPy7J8N57DcWlSfT1uS2J6j/Tn/wIDnbruuvqFl5Mnz6iysqqRt8i3TJ9D0/uTzO+xMfurHrsuPg05UVFRkqTi4mK1a9fOXl5cXKyuXbtKurRH5sSJEx6vq6ioUElJif36yMjIy/bIVH9dvfemppri4uLL9u7Uh2WpUX7AGmvclsT0HunPf3nbl7++DybPoWR+f5L5PTZnfz69T050dLSioqK0ZcsWe1lZWZl27typpKQkSVJSUpJKS0tVUFBg12zdulVVVVWKj4+XJCUmJmrbtm0qLy+3a3Jzc3XTTTcpLCzMrtm6davH98/NzVViYqIvWwIAAH7K65Bz5swZ7d69W7t375Z06WTj3bt369ixY3I4HBo2bJgWLVqkDz/8UHv37tWECRPUrl079e/fX5IUExOjvn37atKkScrPz9f27ds1Y8YMpaamqn379pKkwYMHKygoSC+88IL279+v9evXKycnR48++qi9HcOGDdPmzZv12muv6cCBA3rllVdUUFCghx9+2BfvCwAA8HNeH64qKCjQsGHD7K9nz54tSUpLS9OcOXP02GOP6dy5c5o8ebJKS0vVq1cvLV26VK1atbJfk5mZqRkzZmj48OFyOp0aOHCgMjIy7PVt27bVsmXLNH36dN1zzz0KDw/Xk08+qSFDhtg1PXv2VGZmpubNm6eXX35ZN954o1599VV16dKlQW8EAAAwi8OyTD4SWD9ut+9PPI6MbOvzcVsS03ukP/8XGOhUeHgbpS7YrC+PldZY071jqNaN6auTJ8+oosL/Tjw2eQ5N708yv8fG7K967Lrw7CoAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABjJ5yGnsrJS8+bN02233ab4+Hj1799fr776qizLsmssy9L8+fPlcrkUHx+vESNG6NChQx7jnDp1SuPGjVPPnj2VnJysiRMn6syZMx41e/bs0dChQxUXF6d+/fopKyvL1+0AAAA/5fOQk5WVpTfffFOTJ0/W+vXr9eyzz2rp0qVavny5R83y5cs1depUrVq1Sq1bt1Z6erouXLhg1zz77LMqLCxUdna2Fi9erG3btmny5Mn2+rKyMqWnp6tjx45au3atJkyYoIULF+qtt97ydUsAAMAP+Tzk7NixQ7fffrtuvfVWRUdH64477pDL5VJ+fr6kS3txcnJyNGrUKPXv319du3bV3Llzdfz4cW3cuFGSdODAAW3evFkzZ85UQkKCkpOTlZGRoXXr1umbb76RJL377rsqLy/XrFmz9OMf/1ipqal65JFHlJ2d7euWAACAH/J5yElKStLWrVt18OBBSZcOKW3fvl3/8R//IUk6evSoioqKlJKSYr+mbdu2SkhI0I4dOyRdCkqhoaGKi4uza1JSUuR0Ou2wlJeXp+TkZAUHB9s1LpdLBw8eVElJiVfb7HD4/l9jjduS/pneI/35/7/m/j3AHNLf1d5jY/ZXH4He/Rqo28iRI1VWVqb//M//VEBAgCorK/XrX/9ad999tySpqKhIkhQREeHxuoiICLndbkmS2+3W9ddf77mhgYEKCwuzX+92uxUdHe1RExkZaa8LCwur9zZHRLT1osP6a6xxWxLTe6S/q0N4eJvm3oQGM30OTe9PMr/H5uzP5yFnw4YNeu+99/TSSy+pc+fO2r17t2bPnq127dopLS3N19/OJ4qLT+tb50VfMYfj0qT6etyWxPQe6c//BQY6dd119QsvJ0+eUWVlVSNvkW+ZPoem9yeZ32Nj9lc9dl18HnLmzp2rkSNHKjU1VZIUGxurY8eOacmSJUpLS1NUVJQkqbi4WO3atbNfV1xcrK5du0q6tEfmxIkTHuNWVFSopKTEfn1kZKS956da9dfVe3Tqy7LUKD9gjTVuS2J6j/Tnv7zty1/fB5PnUDK/P8n8HpuzP5+fk3P+/Hk5vnOwLCAgwL6EPDo6WlFRUdqyZYu9vqysTDt37lRSUpKkS+f1lJaWqqCgwK7ZunWrqqqqFB8fL0lKTEzUtm3bVF5ebtfk5ubqpptu8upQFQAAMJPPQ87PfvYzLV68WJs2bdLRo0f1wQcfKDs7W/3795ckORwODRs2TIsWLdKHH36ovXv3asKECWrXrp1dExMTo759+2rSpEnKz8/X9u3bNWPGDKWmpqp9+/aSpMGDBysoKEgvvPCC9u/fr/Xr1ysnJ0ePPvqor1sCAAB+yOeHqzIyMjR//nxNmzbNPiQ1ZMgQPfXUU3bNY489pnPnzmny5MkqLS1Vr169tHTpUrVq1cquyczM1IwZMzR8+HA5nU4NHDhQGRkZ9vq2bdtq2bJlmj59uu655x6Fh4frySef1JAhQ3zdEgAA8EMOyzL5SGD9uN2+P/E4MrKtz8dtSUzvkf78X2CgU+HhbZS6YLO+PFZaY033jqFaN6avTp48o4oK/zvx2OQ5NL0/yfweG7O/6rHrwrOrAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASI0Scr755hs9++yz6t27t+Lj4zV48GB98cUX9nrLsjR//ny5XC7Fx8drxIgROnTokMcYp06d0rhx49SzZ08lJydr4sSJOnPmjEfNnj17NHToUMXFxalfv37KyspqjHYAAIAf8nnIKSkp0S9+8QsFBQUpKytL69at03PPPaewsDC7JisrS8uXL9fUqVO1atUqtW7dWunp6bpw4YJd8+yzz6qwsFDZ2dlavHixtm3bpsmTJ9vry8rKlJ6ero4dO2rt2rWaMGGCFi5cqLfeesvXLQEAAD8U6OsBs7Ky1KFDB82ePdte1qlTJ/u/LctSTk6ORo0apf79+0uS5s6dq5SUFG3cuFGpqak6cOCANm/erNWrVysuLk6SlJGRoZEjR2rChAlq37693n33XZWXl2vWrFkKDg7Wj3/8Y+3evVvZ2dkaMmSIr9sCAAB+xuch529/+5tcLpfGjBmjzz//XO3bt9fQoUP1wAMPSJKOHj2qoqIipaSk2K9p27atEhIStGPHDqWmpmrHjh0KDQ21A44kpaSkyOl0Kj8/XwMGDFBeXp6Sk5MVHBxs17hcLmVlZamkpMRjz1FdHA4fNF7DeL4etyUxvUf683/e9uZv74Xpc2h6f5L5PTZmf/Ud0+ch58iRI3rzzTf16KOP6oknntAXX3yhmTNnKigoSGlpaSoqKpIkRUREeLwuIiJCbrdbkuR2u3X99dd7bmhgoMLCwuzXu91uRUdHe9RERkba67wJORERbb1rspnHbUlM75H+rg7h4W2aexMazPQ5NL0/yfwem7M/n4ccy7LUo0cP/eY3v5Ek3Xzzzdq/f79WrlyptLQ0X387nyguPi3L8t14DselSfX1uC2J6T3Sn/8LDHTquuvqF15OnjyjysqqRt4i3zJ9Dk3vTzK/x8bsr3rsuvg85ERFRSkmJsZj2Y9+9CP97//+r71ekoqLi9WuXTu7pri4WF27dpV0aY/MiRMnPMaoqKhQSUmJ/frIyEh7z0+16q+r9+jUl2WpUX7AGmvclsT0HunPf3nbl7++DybPoWR+f5L5PTZnfz6/uqpnz546ePCgx7JDhw7phhtukCRFR0crKipKW7ZssdeXlZVp586dSkpKkiQlJSWptLRUBQUFds3WrVtVVVWl+Ph4SVJiYqK2bdum8vJyuyY3N1c33XSTV4eqAACAmXwecoYPH66dO3dq8eLFOnz4sN577z2tWrVKQ4cOlSQ5HA4NGzZMixYt0ocffqi9e/dqwoQJateunX21VUxMjPr27atJkyYpPz9f27dv14wZM5Samqr27dtLkgYPHqygoCC98MIL2r9/v9avX6+cnBw9+uijvm4JAAD4IZ8froqPj9fChQv18ssv69VXX1V0dLQmTpyou+++26557LHHdO7cOU2ePFmlpaXq1auXli5dqlatWtk1mZmZmjFjhoYPHy6n06mBAwcqIyPDXt+2bVstW7ZM06dP1z333KPw8HA9+eSTXD4OAAAkSQ7LMvlIYP243b4/8Tgysq3Px21JTO+R/vxfYKBT4eFtlLpgs748VlpjTfeOoVo3pq9Onjyjigr/O/HY5Dk0vT/J/B4bs7/qsevCs6sAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIzk8/vkAEBTcDodcjq//1HEAQH8DQdc7Qg5APyO0+lQ2HXXKpAgA6AWhBwAfsfpdCgwwKmxK3eo8HhZjTW3xkZp/KCuTbxlAFoSQg4Av1V4vOx772YcE9WmibcGQEvDvl4AAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgpEYPOX/4wx8UGxur3/3ud/ayCxcuaNq0aerdu7eSkpL09NNPy+12e7zu2LFjGjlypBISEtSnTx+9+OKLqqio8Kj57LPPlJaWph49emjAgAFau3ZtY7cDAAD8RKOGnPz8fK1cuVKxsbEey2fNmqWPPvpI8+bN0/Lly3X8+HGNHj3aXl9ZWanHH39c5eXlWrlypebMmaO3335bCxYssGuOHDmixx9/XL1799af//xnDR8+XBkZGdq8eXNjtgQAAPxEYGMNfObMGY0fP14zZ87UokWL7OWnT5/WmjVrlJmZqT59+ki6FHruvPNO5eXlKTExUZ9++qkKCwuVnZ2tyMhIdevWTWPHjlVmZqZGjx6t4OBgrVy5UtHR0Xr++eclSTExMdq+fbv++Mc/qm/fvl5tq8Phu76/PZ6vx21JTO+R/q4+/vZemD6Hpvcnmd9jY/ZX3zEbLeRMnz5d/fr1U0pKikfIKSgoUHl5uVJSUuxlMTEx6tixox1y8vLy1KVLF0VGRto1LpdLU6dOVWFhoW6++Wbl5eXZIenbNbNmzfJ6WyMi2jagw+YbtyUxvUf6uzqEh7dp7k1oMNPn0PT+JPN7bM7+GiXkrFu3Trt27dLq1asvW+d2uxUUFKTQ0FCP5RERESoqKrJrvh1wJNlf11VTVlam8+fP65prrqn39hYXn5Zl1bu8Tg7HpUn19bgtiek90l/LFhDg9GkwOXnyjCorq3w2XlPw9zmsi+n9Seb32Jj9VY9dF5+HnH/961/63e9+p9dee02tWrXy9fCNwrLUKD9gjTVuS2J6j/R39fDX98H0OTS9P8n8HpuzP5+HnC+//FLFxcW655577GWVlZX6/PPPtWLFCi1btkzl5eUqLS312JtTXFysqKgoSZf2yOTn53uMW3311bdrvntFltvtVkhIiFd7cQAAgJl8HnJ+8pOf6L333vNY9tvf/lY/+tGP9Nhjj+kHP/iBgoKCtGXLFg0aNEiS9NVXX+nYsWNKTEyUJCUmJmrx4sUqLi5WRESEJCk3N1chISHq3LmzXfPJJ594fJ/c3Fx7DAAAcHXzecgJCQlRly5dPJZde+21uu666+zl9957r+bMmaOwsDCFhIRo5syZSkpKsgOKy+VS586dNWHCBI0fP15FRUWaN2+eHnroIQUHB0uSHnzwQa1YsUJz587Vvffeq61bt2rDhg1asmSJr1sCAAB+qNGurqrNxIkT5XQ6NWbMGF28eFEul0tTpkyx1wcEBGjx4sWaOnWqhgwZotatWystLU1jxoyxazp16qQlS5Zo9uzZysnJUYcOHTRz5kyvLx8HAABmapKQs3z5co+vW7VqpSlTpngEm++64YYblJWVVeu4vXv31jvvvOOLTQQAAIbh2VUAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEjN8lgHAKiN0+mQ0+n43vUBAfx9BqBuhBwALYrT6VDYddcqkCAD4AoRcgC0KE6nQ4EBTo1duUOFx8tqrLk1NkrjB3Vt4i0D4G8IOQBapMLjZfryWGmN62Ki2jTx1gDwR+wPBgAARiLkAAAAIxFyAACAkQg5AADASJx4DOCqV9d9d6qqLFVVWU20NQB8hZAD4KoVFdJKlVWWQkNb11pXUVmlklNnCTqAnyHkALhqhbYOVIDTUes9eTq3C9H8B5PkdDoIOYCfIeQAuOrVdk8eAP6LE48BAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgpMDm3gAAVxen0yGn0/G96wMC+NsLgG8QcgA0GafTobDrrlUgQQZAEyDkAGgyTqdDgQFOjV25Q4XHy2qsuTU2SuMHdW3iLQNgIkIOgCZXeLxMXx4rrXFdTFSbJt4aAKZinzEAADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJF8HnKWLFmie++9V0lJSerTp4+efPJJffXVVx41Fy5c0LRp09S7d28lJSXp6aefltvt9qg5duyYRo4cqYSEBPXp00cvvviiKioqPGo+++wzpaWlqUePHhowYIDWrl3r63YAAICf8nnI+fvf/66HHnpIq1atUnZ2tioqKpSenq6zZ8/aNbNmzdJHH32kefPmafny5Tp+/LhGjx5tr6+srNTjjz+u8vJyrVy5UnPmzNHbb7+tBQsW2DVHjhzR448/rt69e+vPf/6zhg8froyMDG3evNnXLQGAAgKcCgz8/n+1PaoCQPPw+c0Aly1b5vH1nDlz1KdPH3355Ze65ZZbdPr0aa1Zs0aZmZnq06ePpEuh584771ReXp4SExP16aefqrCwUNnZ2YqMjFS3bt00duxYZWZmavTo0QoODtbKlSsVHR2t559/XpIUExOj7du3649//KP69u3r67YAXKWiQlqpsspSaGjrWusqKqtUcuqsqqqsJtoyAHVp9Dsenz59WpIUFhYmSSooKFB5eblSUlLsmpiYGHXs2NEOOXl5eerSpYsiIyPtGpfLpalTp6qwsFA333yz8vLy7JD07ZpZs2Z5vY0OH/8BVj2er8dtSUzvkf5QLbR1oAKcjlofRdG5XYjmP5gkp9Mhy2qakGP6HJren2R+j43ZX33HbNSQU1VVpVmzZqlnz57q0qWLJMntdisoKEihoaEetRERESoqKrJrvh1wJNlf11VTVlam8+fP65prrqn3dkZEtPWusWYetyUxvUf6Q7XaHkVRLTy86R9JYfocmt6fZH6Pzdlfo4acadOmaf/+/frTn/7UmN/mihUXn5Yv//hyOC5Nqq/HbUlM75H+GkdAgLNZgkBTOXnyjCorq5rke/Ez6v9M77Ex+6seuy6NFnKmT5+uTZs26Y033lCHDh3s5ZGRkSovL1dpaanH3pzi4mJFRUXZNfn5+R7jVV999e2a716R5Xa7FRIS4tVeHEmyLDXKD1hjjduSmN4j/cFbTf1+mj6Hpvcnmd9jc/bn86urLMvS9OnT9cEHH+j1119Xp06dPNb36NFDQUFB2rJli73sq6++0rFjx5SYmChJSkxM1L59+1RcXGzX5ObmKiQkRJ07d7Zrtm7d6jF2bm6uPQYAALi6+TzkTJs2Te+++65eeukltWnTRkVFRSoqKtL58+clSW3bttW9996rOXPmaOvWrSooKNDEiROVlJRkBxSXy6XOnTtrwoQJ2rNnjzZv3qx58+bpoYceUnBwsCTpwQcf1JEjRzR37lwdOHBAK1as0IYNGzRixAhftwQAAPyQzw9Xvfnmm5KkRx55xGP57Nmzdc8990iSJk6cKKfTqTFjxujixYtyuVyaMmWKXRsQEKDFixdr6tSpGjJkiFq3bq20tDSNGTPGrunUqZOWLFmi2bNnKycnRx06dNDMmTO5fBwAAEhqhJCzd+/eOmtatWqlKVOmeASb77rhhhuUlZVV6zi9e/fWO++84+0mAgCAqwDPrgIAAEYi5AAAACM1+h2PAVw9nE5Hrc9wCgjg7yoATYeQA8AnnE6Hwq67VoEEGQAtBCEHgE84nQ4FBjhrfcbTrbFRGj+oaxNvGYCrFSEHgE/V9oynmChzH+kAoOVhvzIAADASe3IAwEfqOrG6qspSVZXBDykCWhhCDgBcoaiQVqqsshQa2rrWuorKKpWcOkvQAZoIIQcArlBo60AFOB21nnTduV2I5j+YJKfTQcgBmgghBwB8pLaTrgE0PU48BgAARiLkAAAAI3G4CkC98MgGAP6GkAOgTjyyAYA/IuQAqBOPbADgjwg5AOqNRzYA8CfsewYAAEZiTw4ANCEe/QA0HUIOADQBHv0AND1CDgA0AR79ADQ9Qg5wlavr/jcS98DxJR79ADQdQg5wFXM6HQoN4/43AMxEyAGuYg5H3fe/kbgHDgD/RMgBUOchFO6BA8AfsY8aAAAYiZADAACMxOEqwGA8Odw/1eeGgZbFJeZAXQg5gKHq8+Twum5Mh6blzQ0DS0vONtFWAf6LkAMYiieH+x9vbhjocNR+byMAhBzAeDw53P9ww0DANzggDwAAjETIAQAARuJwFeCnuHIKAGpHyAH8UH2unILZqkPs94XZqiqLJ5njqkfIAfwQV05dvb57mXl4eM0nj1dUVqnk1FmCDq5qhBygBarvoSiunLr6eHOZudPpIOTgqkbIAVoYDkWhPupzmXl97pxMCILJCDlAC8OhKFwpb+6czCEtmIyQA7RQHIpCQ3FIC7iEkAP4UF3n0tQHl37DV7hzMq52hBzAR+p7Lk1llaWAKwxCgK9w3g5MRsgBfMSbc2k43wbNjfN2cDUg5AA+Vp9zaTjfBs2N83ZwNSDkAPr+c2m8OT+Gc2ngj7gUHSYj5OCqV9u5NN++myzn0uBq480hrbLT52VZ3x90CEJoDoQcGK8+dw/mXBrgcvU5pHXLjeGadFd3XXfdtbWOxbk9aA6EHDQLX1xqXR8Oh0Mhba+p192DOZcGqFldP/f1PbcnKChAlZVVXn3v7x4qY48QvEHIaeF8FQbq84uhId+rIeeh1Dd41OfwUH0PIbEHBmhctQWh+h72qunz/N0HkHJoDN4g5DSjukKFL8NAXb8YGvq9anoCsi+Dh69q2AMDNJ/6HPaqz+fZm0NjdQUhXyFQtWyEnEZU214Obw6jNNUvBl98L18HDw4hAebwxefZV+cI+WpPsS8CVUCAk7DUSPw+5KxYsULLli1TUVGRunbtqkmTJik+Pr5Zt8npdKiyyqpxL8d3NVUYqO9fUQQPAC2dr37fNdUfkHWFpfDwNk269+lq4tchZ/369Zo9e7amTZumhIQEvf7660pPT9f777+viIiIZtsuh8Ph01DhKwQPAFeLpvqDrSkPw/lq71NT1zidDlVWNk948+uQk52drQceeED33nuvJGnatGnatGmT1qxZo5EjR9Z7HKdT8mV4dvzfnLcKdKp1cECNNcH/d5iqe8fQ762JiQqhhppmr2mJ20QNNS2ppj6/62urCb82WAFOhxZvOqBjJedqrIm7IUz3J3fyq5qOYa31xK0xcjodPt9D5ajnNTIOy0/3jV28eFGJiYlasGCB+vfvby9/7rnnVFpaqkWLFjXj1gEAgObmt/ehP3nypCorKy87LBURESG3291MWwUAAFoKvw05AAAAtfHbkBMeHq6AgAAVFxd7LC8uLlZkZGQzbRUAAGgp/DbkBAcHq3v37tqyZYu9rKqqSlu2bFFSUlIzbhkAAGgJ/PrqqkcffVTPPfecevToofj4eL3++us6d+6c7rnnnubeNAAA0Mz8OuTceeedOnHihBYsWKCioiJ169ZNS5cu5XAVAADw30vIAQAAauO35+QAAADUhpADAACMRMgBAABGIuQAAAAjEXIaYNGiRXrwwQeVkJCg5OTker3GsizNnz9fLpdL8fHxGjFihA4dOuRRc+rUKY0bN049e/ZUcnKyJk6cqDNnzjRCB3XzdluOHj2q2NjYGv9t2LDBrqtp/bp165qiJQ8Nea8feeSRy7Z98uTJHjXHjh3TyJEjlZCQoD59+ujFF19URUVFY7byvbzt8dSpU5oxY4YGDRqk+Ph43XrrrZo5c6ZOnz7tUddcc7hixQrddtttiouL0/3336/8/Pxa6zds2KA77rhDcXFxGjx4sD7++GOP9fX5TDY1b3pctWqVhg4dqltuuUW33HKLRowYcVn9888/f9lcpaenN3Yb38ub/tauXXvZtsfFxXnUtLQ59Ka/mn6fxMbGejxcuiXN3+eff64nnnhCLpdLsbGx2rhxY52v+eyzz5SWlqYePXpowIABWrt27WU13n6uvWbBa/Pnz7eys7Ot2bNnW7169arXa5YsWWL16tXL+uCDD6zdu3dbTzzxhHXbbbdZ58+ft2vS09Otu+++28rLy7M+//xza8CAAdZvfvObxmqjVt5uS0VFhXX8+HGPf6+88oqVmJholZWV2XVdunSx1qxZ41H37fegqTTkvX744YetjIwMj20/ffq0vb6iosK66667rBEjRli7du2yNm3aZPXu3dt66aWXGrudGnnb4969e63Ro0dbH374oXX48GErNzfXGjhwoPX000971DXHHK5bt87q3r27tXr1amv//v1WRkaGlZycbLnd7hrrt2/fbnXr1s3KysqyCgsLrf/5n/+xunfvbu3du9euqc9nsil52+NvfvMb64033rB27dplFRYWWs8//7zVq1cv69///rdd89xzz1np6ekec3Xq1KmmasmDt/2tWbPG6tmzp8e2FxUVedS0pDn0tr+TJ0969LZv3z6rW7du1po1a+yaljR/mzZtsl5++WXrr3/9q9WlSxfrgw8+qLX+66+/thISEqzZs2dbhYWF1vLly61u3bpZn3zyiV3j7XvWEIScK7BmzZp6hZyqqirrpz/9qbV06VJ7WWlpqdWjRw/rL3/5i2VZllVYWGh16dLFys/Pt2s+/vhjKzY21uOXVlPw1bb813/9l/Xb3/7WY1l9PhyNraH9Pfzww9bMmTO/d/2mTZusrl27evwi/tOf/mT17NnTunDhgm82vp58NYfr16+3unfvbpWXl9vLmmMO77vvPmvatGn215WVlZbL5bKWLFlSY/3YsWOtkSNHeiy7//77rUmTJlmWVb/PZFPztsfvqqiosJKSkqy3337bXvbcc89Zo0aN8vWmNoi3/dX1+7WlzeGVzl92draVlJRknTlzxl7Wkubv2+rzO2Du3LlWamqqx7JnnnnG+uUvf2l/faXvWX1wuKoJHD16VEVFRUpJSbGXtW3bVgkJCdqxY4ckaceOHQoNDfXYHZuSkiKn0+n73Xd18MW2FBQUaPfu3brvvvsuWzdt2jT17t1b9913n1avXi2riW/VdCX9vffee+rdu7fuuusuvfTSSzp37py9Li8vT126dPG4GaXL5VJZWZkKCwt930gtfPXzVFZWppCQEAUGet43tCnn8OLFi/ryyy89Pj9Op1MpKSn25+e78vLy1KdPH49lLpdLeXl5kur3mWxKDenxu86dO6eKigqFhYV5LP/73/+uPn36aNCgQZoyZYpOnjzp022vj4b2d/bsWf3sZz9Tv379NGrUKO3fv99e15Lm0Bfzt2bNGqWmpuraa6/1WN4S5q8h6voM+uI9qw+/vuOxvygqKpIkRUREeCyPiIiQ2+2WJLndbl1//fUe6wMDAxUWFma/vqn4YltWr16tmJgY9ezZ02P5mDFj9JOf/EStW7fWp59+qmnTpuns2bMaNmyYz7a/Lg3t76677lLHjh3Vrl077d27V5mZmTp48KAWLlxoj/vdu21Xf+2Pc3jixAn9/ve/15AhQzyWN/Ucnjx5UpWVlTV+fr766qsaX1PTXHz781afz2RTakiP35WZmal27dp5/E+jb9++GjBggKKjo3XkyBG9/PLLeuyxx/TWW28pICDApz3UpiH93XTTTZo1a5ZiY2N1+vRpvfbaa3rwwQe1bt06dejQoUXN4ZXOX35+vvbt26ff/e53Hstbyvw1xPf9PiwrK9P58+dVUlJyxT/z9UHI+T+ZmZnKysqqtWb9+vWKiYlpoi3yvfr2eKXOnz+vv/zlL3ryyScvW/fUU0/Z/33zzTfr3LlzWrZsmU/+B9nY/X37f/axsbGKiorSiBEj9PXXX+uHP/xhg8f1RlPNYVlZmR5//HHFxMRo9OjRHusacw7RMH/4wx+0fv165eTkqFWrVvby1NRU+7+rT1zt37+/vXegJUtKSvJ42HJSUpLuvPNOrVy5Us8880zzbVgjWL16tbp06aL4+HiP5f48fy0FIef//PKXv1RaWlqtNZ06dWrQ2FFRUZKk4uJitWvXzl5eXFysrl27SrqUcE+cOOHxuoqKCpWUlNivv1L17fFKt+X999/X+fPn9fOf/7zO2oSEBP3+97/XxYsXFRwcXGd9bZqqv2oJCQmSpMOHD+uHP/yhIiMjLzsUVP0XpT/NYVlZmX71q1+pTZs2evXVVxUUFFRrvS/nsCbh4eEKCAhQcXGxx/Li4uLvfU5dZGTkZX/Nf7u+Pp/JptSQHqstW7ZMf/jDH5SdnV3ntnfq1Enh4eE6fPhwk/5P8kr6qxYUFKRu3brp66+/ltSy5vBK+jt79qzWrVunMWPG1Pl9mmv+GqKmz6Db7VZISIiuueYaOZ3OK/6ZqA/Oyfk/119/vWJiYmr919Bf4NHR0YqKitKWLVvsZWVlZdq5c6f9l0pSUpJKS0tVUFBg12zdulVVVVWXpfuGqm+PV7ota9as0W233XbZ4ZKa7N69W2FhYT75n2NT9fftbZf+/y/bxMRE7du3z+NDm5ubq5CQEHXu3PmK+2uKHsvKypSenq6goCAtWrTIY6/A9/HlHNYkODhY3bt39/j8VFVVacuWLR5/6X9bYmKitm7d6rEsNzdXiYmJkur3mWxKDelRkrKysvT73/9eS5cuvezy6pr8+9//1qlTp3wWuuurof19W2Vlpfbt22dve0uawyvp7/3339fFixd199131/l9mmv+GqKuz6AvfibqxWenMF9F/vnPf1q7du2yL5HetWuXtWvXLo9LpQcNGmT99a9/tb9esmSJlZycbG3cuNHas2ePNWrUqBovIf/5z39u7dy509q2bZs1cODAZr2EvLZt+fe//20NGjTI2rlzp8frDh06ZMXGxloff/zxZWN++OGH1qpVq6y9e/dahw4dslasWGElJCRY8+fPb/R+vsvb/g4fPmwtXLjQ+uKLL6wjR45YGzdutG6//XbroYcesl9TfQn5L3/5S2v37t3WJ598Yv3kJz9p1kvIvenx9OnT1v3332/ddddd1uHDhz0uW62oqLAsq/nmcN26dVaPHj2stWvXWoWFhdakSZOs5ORk+0q28ePHW5mZmXb99u3brZtvvtlatmyZVVhYaC1YsKDGS8jr+kw2JW97XLJkidW9e3fr/fff95ir6t9DZWVl1pw5c6wdO3ZYR44csXJzc620tDRr4MCBTX61X0P6e+WVV6zNmzdbX3/9tVVQUGD9+te/tuLi4qz9+/fbNS1pDr3tr9ovfvEL65lnnrlseUubv7KyMvv/dV26dLGys7OtXbt2Wf/85z8ty7KszMxMa/z48XZ99SXkL774olVYWGi98cYbNV5CXtt75gscrmqABQsW6O2337a/rj4sk5OTo969e0uSDh486HETtccee0znzp3T5MmTVVpaql69emnp0qUefylnZmZqxowZGj58uJxOpwYOHKiMjIymaeo76tqW8vJyHTx40OPqIunSXpwOHTrI5XJdNmZgYKBWrFihWbNmSZJ++MMf6vnnn9cDDzzQuM3UwNv+goKCtGXLFuXk5Ojs2bP6wQ9+oIEDB3qcdxQQEKDFixdr6tSpGjJkiFq3bq20tLR67YZuDN72+OWXX2rnzp2SpAEDBniM9eGHHyo6OrrZ5vDOO+/UiRMntGDBAhUVFalbt25aunSpvVv7X//6l5zO/79jumfPnsrMzNS8efP08ssv68Ybb9Srr76qLl262DX1+Uw2JW97XLlypcrLyy/7+Ro9erSefvppBQQEaN++fXrnnXd0+vRptWvXTj/96U81duzYRtvrVhtv+ystLdWkSZNUVFSksLAwde/eXStXrvTYK9qS5tDb/iTpq6++0vbt2/Xaa69dNl5Lm7+CggKP8+5mz54tSUpLS9OcOXNUVFSkf/3rX/b6Tp06acmSJZo9e7ZycnLUoUMHzZw5U3379rVr6nrPfMFhWU18/S4AAEAT4JwcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABjp/wEYXZzScCFbqgAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Axes(0.125,0.11;0.775x0.77)\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAqlUlEQVR4nO3df1TU153/8dfMAP4CFIHEurpNAhV/oIJJDoWOS5uta9fUnoNuVjZ2Tbo50ZgibNP6Y10TRS2gJV3i1qzEuFqNWU42/khSY3ab3dTqAYx1ZV1/xIS4XUlJKzMxUTQKDJ/vH36dk0kUZmD4DBeej3M4J3zmzmfu5z0gr9y5934clmVZAgAAMIgz0h0AAAAIFQEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABgnKtId6Gle7yWF82YJDoeUmBgX9vPii6i1PaizPaizPaizPXqyzjfO3Zk+H2AsSz3yQ9xT58UXUWt7UGd7UGd7UGd7RLLOfIQEAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDh9/m7UPcXl6jj7tbdbam/nVqgAAPQEAkyInE6HfO2WEhKGdNiuzdeuTz6+QogBAKAHEGBC5HA45HI6VFR1TPXnm2/aJvW2WD2Tnymn00GAAQCgBxBguqj+fLNONl6MdDcAAOiXmMQLAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgnJACzD/+4z8qLS0t4Otb3/qW//Fr166puLhYWVlZyszM1KJFi+TxeALO0djYqPnz52vy5MnKzs7WunXr1NbWFtDm8OHDysvLU3p6uqZNm6bdu3d34xIBAEBfExXqE77yla9o69at/u9dLpf/v0tKSnTgwAFVVFQoLi5Oa9asUUFBgaqqqiRJPp9PCxYsUFJSkqqqqnT+/HktXbpU0dHReuKJJyRJDQ0NWrBggfLz81VeXq6amhqtWLFCycnJmjp1anevFwAA9AEhBxiXy6Xk5OQvHL906ZJ27dql8vJyZWdnS7oeaGbMmKG6ujplZGTo0KFDqq+v19atW5WUlKRx48apqKhI5eXlKigoUExMjKqqqjRq1CgtW7ZMkpSSkqKjR49q27ZtBBgAACCpCwHm//7v/+R2uzVgwABlZGTohz/8oUaOHKkTJ06otbVVOTk5/rYpKSkaOXKkP8DU1dVpzJgxSkpK8rdxu91atWqV6uvrNX78eNXV1fkD0GfblJSUdOkCHY4uPS1s5wv36/cnN2pHDXsWdbYHdbYHdbZHT9Y52HOGFGAmTZqk0tJS3XnnnWpqatLGjRs1d+5cvfbaa/J4PIqOjlZ8fHzAcxITE9XU1CRJ8ng8AeFFkv/7zto0Nzfr6tWrGjhwYChdVmJiXEjtwykhYUjEXrsvieR72J9QZ3tQZ3tQZ3tEss4hBZjc3Fz/f48dO1aTJ0/WN77xDe3fvz/kYGEXr/eSLCt854uKcmrYsOCCyYULl+XztYfvxfsZh+P6L0e430MEos72oM72oM726Mk63zh3Z0L+COmz4uPjdccdd+jcuXPKyclRa2urLl68GDAK4/V6/XNmkpKSdPz48YBz3Fil9Nk2n1+55PF4FBsb26WQZFkKa3FDPRe/QN0X7vcQN0ed7UGd7UGd7RHJOndrH5jLly+roaFBycnJSk9PV3R0tGpqavyPnz17Vo2NjcrIyJAkZWRk6N1335XX6/W3qa6uVmxsrFJTU/1tamtrA16nurrafw4AAICQAsy6dev09ttv64MPPtB//dd/qaCgQE6nU9/+9rcVFxen2bNnq6ysTLW1tTpx4oSWL1+uzMxMf/hwu91KTU3VkiVL9M477+jgwYOqqKjQ3LlzFRMTI0nKz89XQ0OD1q9fr/fff187d+7U/v379fDDD4f72gEAgKFC+gjp97//vZ544gl9/PHHGj58uO6++2699NJLGj58uCRp+fLlcjqdKiwsVEtLi9xut1auXOl/vsvl0qZNm7Rq1SrNmTNHgwYNUl5engoLC/1tRo8ercrKSpWWlmr79u0aMWKE1q5dyxJqAADg57Csvv0poccT/km8CQlDdP+GgzrZePGmbSaMjNe+wqm6cOGy2tqYxNtVDoeUlBQX9vcQgaizPaizPaizPXqyzjfO3RnuhQQAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDjdCjDPPfec0tLS9OMf/9h/7Nq1ayouLlZWVpYyMzO1aNEieTyegOc1NjZq/vz5mjx5srKzs7Vu3Tq1tbUFtDl8+LDy8vKUnp6uadOmaffu3d3pKgAA6EO6HGCOHz+uqqoqpaWlBRwvKSnRW2+9pYqKCu3YsUPnz59XQUGB/3Gfz6cFCxaotbVVVVVVKisr0549e7RhwwZ/m4aGBi1YsEBZWVl65ZVX9NBDD2nFihU6ePBgV7sLAAD6kC4FmMuXL2vx4sVau3athg4d6j9+6dIl7dq1S8uWLVN2drbS09NVUlKiY8eOqa6uTpJ06NAh1dfX6yc/+YnGjRun3NxcFRUVaefOnWppaZEkVVVVadSoUVq2bJlSUlL03e9+V9OnT9e2bdu6fcEAAMB8XQowq1evVm5urnJycgKOnzhxQq2trQHHU1JSNHLkSH+Aqaur05gxY5SUlORv43a71dzcrPr6en+b7OzsgHO73W7/OULhcIT/K9Kv35++qCF17ktf1Jk696WvnqxzMKJC+3Ms7du3T6dOndLLL7/8hcc8Ho+io6MVHx8fcDwxMVFNTU3+Np8NL5L833fWprm5WVevXtXAgQOD7m9iYlzQbcMtIWFIxF67L4nke9ifUGd7UGd7UGd7RLLOIQWYDz/8UD/+8Y/1z//8zxowYEBP9SmsvN5LsqzwnS8qyqlhw4ILJhcuXJbP1x6+F+9nHI7rvxzhfg8RiDrbgzrbgzrboyfrfOPcnQkpwJw8eVJer1ezZs3yH/P5fDpy5Ih27typLVu2qLW1VRcvXgwYhfF6vUpOTpZ0fSTl+PHjAee9sUrps20+v3LJ4/EoNjY2pNEXSbIshbW4oZ6LX6DuC/d7iJujzvagzvagzvaIZJ1DCjBf/epX9dprrwUc+7u/+zvdddddevTRR/WlL31J0dHRqqmp0fTp0yVJZ8+eVWNjozIyMiRJGRkZ2rRpk7xerxITEyVJ1dXVio2NVWpqqr/Nr3/964DXqa6u9p8DAAD0byEFmNjYWI0ZMybg2ODBgzVs2DD/8dmzZ6usrExDhw5VbGys1q5dq8zMTH/4cLvdSk1N1ZIlS7R48WI1NTWpoqJCc+fOVUxMjCQpPz9fO3fu1Pr16zV79mzV1tZq//79qqysDMMlAwAA04U8ibczy5cvl9PpVGFhoVpaWuR2u7Vy5Ur/4y6XS5s2bdKqVas0Z84cDRo0SHl5eSosLPS3GT16tCorK1VaWqrt27drxIgRWrt2raZOnRru7gIAAAM5LKtvf0ro8YR/Em9CwhDdv+GgTjZevGmbCSPjta9wqi5cuKy2NibxdpXDISUlxYX9PUQg6mwP6mwP6myPnqzzjXN3hnshAQAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYJ6QA8+KLL2rmzJmaMmWKpkyZojlz5ujAgQP+x69du6bi4mJlZWUpMzNTixYtksfjCThHY2Oj5s+fr8mTJys7O1vr1q1TW1tbQJvDhw8rLy9P6enpmjZtmnbv3t2NSwQAAH1NSAFmxIgR+tGPfqTdu3dr165d+upXv6rvf//7eu+99yRJJSUleuutt1RRUaEdO3bo/PnzKigo8D/f5/NpwYIFam1tVVVVlcrKyrRnzx5t2LDB36ahoUELFixQVlaWXnnlFT300ENasWKFDh48GKZLBgAApgspwNx3333Kzc3VHXfcoTvvvFM/+MEPNHjwYNXV1enSpUvatWuXli1bpuzsbKWnp6ukpETHjh1TXV2dJOnQoUOqr6/XT37yE40bN065ubkqKirSzp071dLSIkmqqqrSqFGjtGzZMqWkpOi73/2upk+frm3btoX72gEAgKGiuvpEn8+nN954Q1euXFFmZqZOnDih1tZW5eTk+NukpKRo5MiRqqurU0ZGhurq6jRmzBglJSX527jdbq1atUr19fUaP3686urqlJ2dHfBabrdbJSUlXeqnw9G16wvX+cL9+v3JjdpRw55Fne1Bne1Bne3Rk3UO9pwhB5gzZ84oPz9f165d0+DBg7Vx40alpqbq9OnTio6OVnx8fED7xMRENTU1SZI8Hk9AeJHk/76zNs3Nzbp69aoGDhwYUn8TE+NCah9OCQlDIvbafUkk38P+hDrbgzrbgzrbI5J1DjnA3Hnnndq7d68uXbqkf/u3f9PSpUv1wgsv9ETfwsLrvSTLCt/5oqKcGjYsuGBy4cJl+Xzt4XvxfsbhuP7LEe73EIGosz2osz2osz16ss43zt2ZkANMTEyMvvzlL0uS0tPT9T//8z/avn27/vzP/1ytra26ePFiwCiM1+tVcnKypOsjKcePHw84341VSp9t8/mVSx6PR7GxsSGPvkiSZSmsxQ31XPwCdV+430PcHHW2B3W2B3W2RyTr3O19YNrb29XS0qL09HRFR0erpqbG/9jZs2fV2NiojIwMSVJGRobeffddeb1ef5vq6mrFxsYqNTXV36a2tjbgNaqrq/3nAAAACCnAPP300zpy5Ig++OADnTlzRk8//bTefvttzZw5U3FxcZo9e7bKyspUW1urEydOaPny5crMzPSHD7fbrdTUVC1ZskTvvPOODh48qIqKCs2dO1cxMTGSpPz8fDU0NGj9+vV6//33tXPnTu3fv18PP/xwuK8dAAAYKqSPkLxer5YuXarz588rLi5OaWlp2rJli772ta9JkpYvXy6n06nCwkK1tLTI7XZr5cqV/ue7XC5t2rRJq1at0pw5czRo0CDl5eWpsLDQ32b06NGqrKxUaWmptm/frhEjRmjt2rWaOnVqmC4ZAACYzmFZfftTQo8n/JN4ExKG6P4NB3Wy8eJN20wYGa99hVN14cJltbUxiberHA4pKSku7O8hAlFne1Bne1Bne/RknW+cuzPcCwkAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxgnpZo4AAKDvczodcjodke5GhwgwAADAz+l0aOiwwYpydfwhja/dktPpkM8XmbtmEmAAAICf0+lQlMupoqpjqj/ffNM2qbfF6pn8TDkcDkkEGAAA0EvUn2/WycaLke7GLTGJFwAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYJyoSHcAHXM6HXI6HR22aW+31N5u2dQjAAAijwDTizmdDg0dNlhRro4Hytp87frk4yuEGABAv0GA6cWcToeiXE4VVR1T/fnmm7ZJvS1Wz+Rnyul0EGAAAP0GAcYA9eebdbLxYqS7AQBAr0GAiaDO5re4OvnoCACA/ooAEyHBzm8BAABfRICJkGDmt3w9LVmLp4+1uWcAAPR+BJgI62h+S0ryEJt7AwCAGQgwfURn82XYKwYA0JcQYAyXHDtAvnZL8fGDOmzHXjEAgL6EAGO4+EFRcjkd7BUDAOhXCDB9BHvFAAD6E9bwAgAA4xBgAACAcQgwAADAOAQYAABgnJACTGVlpWbPnq3MzExlZ2fr8ccf19mzZwPaXLt2TcXFxcrKylJmZqYWLVokj8cT0KaxsVHz58/X5MmTlZ2drXXr1qmtrS2gzeHDh5WXl6f09HRNmzZNu3fv7uIlAgCAviakAPP2229r7ty5eumll7R161a1tbXpkUce0ZUrV/xtSkpK9NZbb6miokI7duzQ+fPnVVBQ4H/c5/NpwYIFam1tVVVVlcrKyrRnzx5t2LDB36ahoUELFixQVlaWXnnlFT300ENasWKFDh48GIZLBgAApgtpGfWWLVsCvi8rK1N2drZOnjype++9V5cuXdKuXbtUXl6u7OxsSdcDzYwZM1RXV6eMjAwdOnRI9fX12rp1q5KSkjRu3DgVFRWpvLxcBQUFiomJUVVVlUaNGqVly5ZJklJSUnT06FFt27ZNU6dODdOlAwAAU3VrH5hLly5JkoYOHSpJOnHihFpbW5WTk+Nvk5KSopEjR/oDTF1dncaMGaOkpCR/G7fbrVWrVqm+vl7jx49XXV2dPwB9tk1JSUnIfXQ4unJl4TtfuF+/u3pbfzpyo68m9dlE1Nke1Nke1NleDkfk/s52OcC0t7erpKREU6ZM0ZgxYyRJHo9H0dHRio+PD2ibmJiopqYmf5vPhhdJ/u87a9Pc3KyrV69q4MCBQfczMTEutAsLo4SE3nUzxt7Wn2BF8j3sT6izPaizPaizPYYNi9zflS4HmOLiYr333nt68cUXw9mfsPN6L8kK4+75UVHOoN+wCxcuy+drv+ljLpfT9kDRUX96I4fj+j9C4X4PEYg624M624M6d18of58+/viy2trC+3flxnvYmS4FmNWrV+tXv/qVXnjhBY0YMcJ/PCkpSa2trbp48WLAKIzX61VycrK/zfHjxwPOd2OV0mfbfH7lksfjUWxsbEijL5JkWQrrD3Go5+ptv0C9rT/BCPd7iJujzvagzvagzvaIZJ1DWoVkWZZWr16tX/7yl/r5z3+u0aNHBzyenp6u6Oho1dTU+I+dPXtWjY2NysjIkCRlZGTo3Xffldfr9beprq5WbGysUlNT/W1qa2sDzl1dXe0/B7rG5XIqKqrjL6eTD44BAL1fSCMwxcXF+sUvfqFnn31WQ4YM8c9ZiYuL08CBAxUXF6fZs2errKxMQ4cOVWxsrNauXavMzEx/+HC73UpNTdWSJUu0ePFiNTU1qaKiQnPnzlVMTIwkKT8/Xzt37tT69es1e/Zs1dbWav/+/aqsrAzv1fcTybED5Gu3FB8/qNO2bb52ffLxFe5aDQDo1UIKMP/yL/8iSfrrv/7rgOOlpaWaNWuWJGn58uVyOp0qLCxUS0uL3G63Vq5c6W/rcrm0adMmrVq1SnPmzNGgQYOUl5enwsJCf5vRo0ersrJSpaWl2r59u0aMGKG1a9eyhLqL4gdFyeV0qKjqmOrPN9+yXeptsXomP1NOp4MAAwDo1UIKMGfOnOm0zYABA7Ry5cqA0PJ5f/RHf6TNmzd3eJ6srCzt3bs3lO6hE/Xnm3Wy8WKkuwEAQLdxLyQAAGAcAgwAADAOAQYAABiHAAMAAIzTrXshoWMu163zYUePAQCAjhFgekAo+64AAIDQEWB6QDD7rnw9LVmLp4+1uWcAAPQNBJge1NG+KynJZt4ZGgCA3oCJGAAAwDgEGAAAYBw+QsIXdLZCqr3d4l5JAICIIsDAL9jVU9yxGgAQaQQY+AWzeurGHaujo13y+dpveS5GaQAAPYkAgy/oaPUUozQAgN6AAIOQhDJK43Q6CDAAgB5BgEGXdDRKAwBAT2MZNQAAMA4BBgAAGIcAAwAAjMMcGPQYNsQDAPQUAgzCjqXWAICeRoBB2IV7qTUjOQCAzyPAoMd0d6m10+mQr91SQsKQDtsxkgMA/Q8BBr2Ww+Fg0zwAwE0RYNDrsWkeAODzCDCIqI7mt3Q29wUA0H8RYBARwa5UAgDgZggwiIhgVip9PS1Zi6ePtblnAAATEGAQUR3Nb0lJ7nj1EQCg/2KSAQAAMA4jMOgT2OwOAPoXAgyMxm0LAKB/IsDAaOG+bQEAwAwEGPQJbHYHAP0Lk3gBAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMZhHxj0G53dbiAY3JIAAHoHAgz6vGBvN+Brt+RyOjpswy0JAKB3IMCgzwvmdgNfT0vW4uljuSUBABiCAIN+o6PbDaQkD+m0DQCg92ASLwAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOOwDA4Sos1sScLsBAOh5IY/AHDlyRI899pjcbrfS0tL05ptvBjxuWZaeeeYZud1uTZo0SQ8//LB++9vfBrT5+OOP9cMf/lBTpkzRPffco+XLl+vy5csBbd555x09+OCDmjhxonJzc7V58+bQrw4Io8/ekiAhYcgtv4YOGyxnJ7ckAAB0T8gjMFeuXFFaWppmz56tgoKCLzy+efNm7dixQ2VlZRo1apSeeeYZPfLII3r99dc1YMAASdKPfvQjNTU1aevWrWptbdXy5cv11FNP6emnn5YkNTc365FHHlF2draKi4v17rvvavny5YqPj9ecOXO6eclA1wRzSwJuNwAA9gg5wOTm5io3N/emj1mWpe3bt2vhwoX65je/KUlav369cnJy9Oabb+r+++/X+++/r4MHD+rll1/WxIkTJUkrVqzQ/PnztWTJEt1+++169dVX1draqpKSEsXExOgrX/mKTp8+ra1btxJgEHHcbgAAIi+sk3g/+OADNTU1KScnx38sLi5OkydP1rFjxyRJx44dU3x8vD+8SFJOTo6cTqeOHz8uSaqrq9M999yjmJgYfxu3263//d//1SeffBLOLgMAAAOFdRJvU1OTJCkxMTHgeGJiojwejyTJ4/Fo+PDhgZ2IitLQoUP9z/d4PBo1alRAm6SkJP9jQ4cODbpPjjBPRQj3+dB3mfKzcqOfpvTXVNTZHtTZXg5H5P7O9vlVSImJcZHuAvqhhIQhke5CyPhdsQd1tgd1tsewYZH7ty6sASY5OVmS5PV6ddttt/mPe71ejR07VtL1kZSPPvoo4HltbW365JNP/M9PSkryj9jccOP7GyMxwfJ6L8kK41zKqChnRN8wmOHChcvy+doj3Y2gOBzX/7EP9+8KAlFne1Dn7nO5nEH/T9jHH19WW1t4/6278R52JqwBZtSoUUpOTlZNTY3GjRsn6fqKov/+7//WX/3VX0mSMjMzdfHiRZ04cULp6emSpNraWrW3t2vSpEmSpIyMDFVUVKi1tVXR0dGSpOrqat15550hfXwkSZalsP4Q8wuBYJn2sxLu3xXcHHW2B3W2RyTrHPIk3suXL+v06dM6ffq0pOsTd0+fPq3GxkY5HA7NmzdP//RP/6T/+I//0JkzZ7RkyRLddttt/lVJKSkpmjp1qp588kkdP35cR48e1Zo1a3T//ffr9ttvlyTNnDlT0dHR+vu//3u99957ev3117V9+3Z973vfC+OlAwAAU4U8AnPixAnNmzfP/31paakkKS8vT2VlZXr00Uf16aef6qmnntLFixd199136/nnn/fvASNJ5eXlWrNmjR566CE5nU792Z/9mVasWOF/PC4uTlu2bNHq1as1a9YsJSQk6PHHH2cJNYzR2W694cKuvwD6q5ADTFZWls6cOXPLxx0Oh4qKilRUVHTLNsOGDfNvWncrY8eO1Ysvvhhq94CI+uxuvR3xtVtydbJbbzBt2nzt+uTjK4QYAP1On1+FBNgpmN16v56WrMXTx3a7Dbv+AujPCDBAD+hot96U5CFhaQMA/Zk9H9QDAACEESMwgOE6mzDMRF8AfREBBjBUsBOGmegLoC8iwACGCmbCMBN9AfRVBBjAcEz0BdAfMYkXAAAYhwADAACMQ4ABAADGYQ4M0A/YdW8mALALAQbow0K5N5PT6ZDPx0olAGYgwAB9WChLraOiXHI42m95LjbEA9CbEGCAfqCjpdZsiAfARAQYoJ9jQzwAJiLAAJDEhngAzMLSBAAAYBxGYACEjdPpkNPp6LANk4EBhAMBBkDQOtpPxuFwKDZuoKI62XOGycAAwoEAA6BTwa5UksRkYAC2IMAA6FQwK5W+npasxdPHMhkYgC0IMACC1lE4SUkeYnNvAPRnrEICAADGYQQGgO06u7kkK5UAdIYAA8A23LYAQLgQYADYJpy3LWDPGaB/I8AAsF0wK5XYcwZARwgwAHoV9pwBEAwCDIBeJdx7zjBhGOibCDAAeqXu7jnDhGGgbyPAAOiTQpkwHB3tks/XfstzMUoD9D4EGAB9WkcjOYzSAOYiwADot8K5rBuAvQgwAPo9bkAJmIcAAwBB6Gw1EwB7EWAAoAPBzpPxtVtydbIzsK/dktPpkM/HR1FAdxFgAKADoexLE8xcmqgolxwOVjwB3UWAAYAgBLMvjV0rnrgPFECAAQBbhGtfGu4DBVxHgAEAG4VjlEYK7j5QbNCHvowAAwC9RLjuA8UGfegPCDAA0Mt09z5Q4byNQjAYyUEkEGAAoI8KxyhNMMvDGclBJBBgAKAfCvfycObbmKOzVWymbNpIgAGAfszO5eHNl67Ksm4dYgg5Pc/pdGjosMGdrmIzAQEGANBlwYzk3HtHgp789gQNGza4w3P1tpATzH47wehNfXa5nIpyOYMaeevtCDAAgG7rbCTHzpATDsGOVIRrjlA4wlKwewRJ3Z8o3hsQYAAAtrAr5PjaLUVHuzoMDJ2NijidjqBHKuzanDCYsCR1vEeQKaMrwSDAAAB6jXCFnKFDuzeSc2Miq11zhKTggkd39wgyZXQlGAQYAIBR7BrJCYdwbU4YTFjqS+EkGAQYAECf092QE+6PWgge4UeAAQD0S4QKs5m/EBwAAPQ7BBgAAGCcXh1gdu7cqfvuu08TJ07UAw88oOPHj0e6SwAAoBfotQHm9ddfV2lpqb7//e9rz549Gjt2rB555BF5vd5Idw0AAERYrw0wW7du1V/+5V9q9uzZSk1NVXFxsQYOHKhdu3ZFumsAACDCeuUqpJaWFp08eVILFizwH3M6ncrJydGxY8dCOpfTKYVzx2nH/98EccLIeA2Kcd20TUpyrHFtemOfaEMb2tCGNr2zzV1J11dpORzX/86GkyPIOyo4rJ6+oUQX/OEPf9Cf/MmfqKqqSpmZmf7j69ev15EjR/Sv//qvEewdAACItF77ERIAAMCt9MoAk5CQIJfL9YUJu16vV0lJSRHqFQAA6C16ZYCJiYnRhAkTVFNT4z/W3t6umpqagI+UAABA/9QrJ/FK0ve+9z0tXbpU6enpmjRpkn7+85/r008/1axZsyLdNQAAEGG9NsDMmDFDH330kTZs2KCmpiaNGzdOzz//PB8hAQCA3rkKCQAAoCO9cg4MAABARwgwAADAOAQYAABgHAIMAAAwDgHmJnbu3Kn77rtPEydO1AMPPKDjx4932H7//v361re+pYkTJ2rmzJk6cOCATT01Xyi1fumll/Tggw/q3nvv1b333quHH3640/cG14X6M33Dvn37lJaWpscff7yHe9g3hFrnixcvqri4WG63W+np6Zo+fTr/fgQh1Dpv27ZN06dP16RJk5Sbm6uSkhJdu3bNpt6a6ciRI3rsscfkdruVlpamN998s9PnHD58WHl5eUpPT9e0adO0e/funu2khQD79u2zJkyYYL388svWe++9Z61YscK65557LI/Hc9P2R48etcaNG2dt3rzZqq+vt/7hH/7BmjBhgnXmzBmbe26eUGv9xBNPWC+88IJ16tQpq76+3lq2bJl19913W7///e9t7rlZQq3zDQ0NDdbUqVOtBx980Fq4cKFNvTVXqHW+du2aNWvWLOvRRx+1fvOb31gNDQ3W4cOHrdOnT9vcc7OEWudXX33VSk9Pt1599VWroaHBOnjwoPW1r33NKikpsbnnZvnVr35l/fSnP7X+/d//3RozZoz1y1/+ssP2586dsyZPnmyVlpZa9fX11o4dO6xx48ZZv/71r3usjwSYz/mLv/gLq7i42P+9z+ez3G63VVlZedP2RUVF1vz58wOOPfDAA9aTTz7Zo/3sC0Kt9ee1tbVZmZmZ1p49e3qoh31DV+rc1tZmzZkzx3rppZespUuXEmCCEGqdX3zxRetP//RPrZaWFru62CeEWufi4mJr3rx5AcdKS0ut/Pz8Hu1nXxJMgFm/fr11//33Bxz727/9W+tv/uZveqxffIT0GS0tLTp58qRycnL8x5xOp3JycnTs2LGbPqeurk7Z2dkBx9xut+rq6nqyq8brSq0/79NPP1VbW5uGDh3aU900XlfrvHHjRiUmJuqBBx6wo5vG60qd//M//1MZGRlavXq1cnJy9O1vf1ubNm2Sz+ezq9vG6UqdMzMzdfLkSf/HTA0NDTpw4IByc3Nt6XN/EYm/hb12J95IuHDhgnw+nxITEwOOJyYm6uzZszd9jsfj+cLuwImJifJ4PD3Wz76gK7X+vPLyct12220B/5ghUFfq/Jvf/EYvv/yy9u7da0MP+4au1LmhoUG1tbWaOXOmnnvuOZ07d07FxcVqa2tTQUGBHd02TlfqPHPmTF24cEEPPvigLMtSW1ub8vPz9dhjj9nR5X7jZn8Lk5KS1NzcrKtXr2rgwIFhf01GYGCk5557Tq+//rp+9rOfacCAAZHuTp/R3NysJUuWaM2aNRo+fHiku9OnWZalxMRErVmzRunp6ZoxY4Yee+wxVVVVRbprfcrhw4dVWVmplStXavfu3frZz36mAwcOaOPGjZHuGrqJEZjPSEhIkMvlktfrDTju9XpveQ+mpKSkL4y2dNQe13Wl1jds2bJFzz33nLZu3aqxY8f2ZDeNF2qdGxoa9Lvf/U4LFy70H2tvb5ckjR8/Xm+88Yb++I//uGc7baCu/DwnJycrKipKLpfLf+yuu+5SU1OTWlpaFBMT06N9NlFX6vzMM8/oO9/5jv/j0LS0NF25ckVPPfWUFi5cKKeT/48Ph5v9LfR4PIqNje2R0ReJEZgAMTExmjBhgmpqavzH2tvbVVNTo8zMzJs+JyMjQ7W1tQHHqqurlZGR0ZNdNV5Xai1Jmzdv1rPPPqvnn39eEydOtKOrRgu1znfddZdee+017d271/913333KSsrS3v37tWIESPs7L4xuvLzPGXKFJ07d84fECXpt7/9rZKTkwkvt9CVOl+9evULIeVGaLS4FWDYRORvYY9NDzbUvn37rPT0dGv37t1WfX299eSTT1r33HOP1dTUZFmWZS1evNgqLy/3tz969Kg1fvx4a8uWLVZ9fb21YcMGllEHKdRaV1ZWWhMmTLDeeOMN6/z58/6v5ubmSF2CEUKt8+exCik4oda5sbHRyszMtFavXm2dPXvWeuutt6zs7Gzr2WefjdQlGCHUOm/YsMHKzMy0fvGLX1jnzp2zDh06ZH3zm9+0ioqKInQFZmhubrZOnTplnTp1yhozZoy1detW69SpU9bvfvc7y7Isq7y83Fq8eLG//Y1l1OvWrbPq6+utF154oceXUfMR0ufMmDFDH330kTZs2KCmpiaNGzdOzz//vH948sMPPwxI81OmTFF5ebkqKir005/+VHfccYc2btyoMWPGROoSjBFqrauqqtTa2qrCwsKA8xQUFGjRokW29t0kodYZXRNqnb/0pS9py5YtKi0t1Xe+8x3dfvvtmjdvnh599NFIXYIRQq3zwoUL5XA4VFFRoT/84Q8aPny4vvGNb+gHP/hBpC7BCCdOnNC8efP835eWlkqS8vLyVFZWpqamJn344Yf+x0ePHq3KykqVlpZq+/btGjFihNauXaupU6f2WB8dlsUYGgAAMAv/2wUAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcf4fGOyikZBV0q8AAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "df[df[\"preference\"]][\"cer_diff_preference\"].hist(bins=50)\n", "print(df[df[\"preference\"]][\"cer_diff_preference\"].quantile(0.9))\n", "plt.show()\n", "print(df[df[\"preference\"]][\"cer\"].hist(bins=50))\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:40.799375Z", "start_time": "2024-05-16T13:59:36.394236Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:12:02.150566Z", "iopub.status.busy": "2024-09-04T00:12:02.150414Z", "iopub.status.idle": "2024-09-04T00:12:02.741764Z", "shell.execute_reply": "2024-09-04T00:12:02.741195Z", "shell.execute_reply.started": "2024-09-04T00:12:02.150551Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5406\n", "good_continue_at\n", "True 89891\n", "False 203\n", "Name: count, dtype: int64\n", "\n", " Check some basics... \n", " preference\n", "False 45047\n", "True 45047\n", "Name: count, dtype: int64 is_30b\n", "True 90094\n", "Name: count, dtype: int64 model_name\n", "chirp-v3p5-engine-t-2 90094\n", "Name: count, dtype: int64 preference model_name \n", "False chirp-v3p5-engine-t-2 45047\n", "True chirp-v3p5-engine-t-2 45047\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df[\"id\"] = df[\"str_id\"]\n", "# get the original duration of the clips, if they are concacted\n", "df[\"original_duration_s\"] = df[\"total_start_s\"] + df[\"duration\"]\n", "# classify the continue at behavoirs by the duration choice\n", "audio_prompt_id_to_continue_at = {}\n", "for _, row in df[~df[\"audio_prompt_id\"].isna()].iterrows():\n", " audio_prompt_id = row[\"audio_prompt_id\"]\n", " if audio_prompt_id not in audio_prompt_id_to_continue_at:\n", " audio_prompt_id_to_continue_at[audio_prompt_id] = row[\"continue_at\"]\n", " else:\n", " # pick the max\n", " audio_prompt_id = max(\n", " audio_prompt_id_to_continue_at[audio_prompt_id], row[\"continue_at\"]\n", " )\n", "print(len(audio_prompt_id_to_continue_at))\n", "df[\"has_continue_and_start_continue_at\"] = df[\"id\"].apply(\n", " lambda x: audio_prompt_id_to_continue_at.get(x)\n", ")\n", "# we want continue at to be at most of the clip...\n", "df[\"good_continue_at\"] = (\n", " (df[\"has_continue_and_start_continue_at\"] / df[\"duration\"]) > 0.9\n", ") | df[\"has_continue_and_start_continue_at\"].isna()\n", "print(df[\"good_continue_at\"].value_counts())\n", "\n", "\n", "print(\n", " \"\\n Check some basics... \\n\",\n", " df[\"preference\"].value_counts(),\n", " df[\"is_30b\"].value_counts(),\n", " df[\"model_name\"].value_counts(),\n", " df.groupby([\"preference\"])[\"model_name\"].value_counts(),\n", ")\n", "\n", "df = df.sort_values(by=[\"request_id\", \"preference\"])\n", "df[\"duration_rel_diff\"] = df[\"duration\"].diff()\n", "df[\"play_rel_diff\"] = df[\"reaction_play_count\"].diff()" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.035167Z", "start_time": "2024-05-16T13:59:40.801098Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:12:02.742643Z", "iopub.status.busy": "2024-09-04T00:12:02.742488Z", "iopub.status.idle": "2024-09-04T00:12:02.919062Z", "shell.execute_reply": "2024-09-04T00:12:02.918487Z", "shell.execute_reply.started": "2024-09-04T00:12:02.742628Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "negative 45047 positive 34464\n", "total pair requests 45047 --> selected pair requests 34464 frac 0.765\n" ] } ], "source": [ "normal_pos_play_count = 2\n", "# this is lower, cause a concat is probably already ensuring that it is good\n", "concat_pos_play_count = 1\n", "# this is a filter on the concated clip\n", "concat_total_play_count = 2\n", "\n", "neg_filter_selection_mask = (\n", " (~df[\"preference\"]) # get basics aligned\n", " & (df[\"reaction_play_count\"] >= 1) # has to be played once\n", " # & (df[\"play_count\"] <= 3) # if it is actually bad, shouldn't be listened often\n", " & (df[\"duration\"] >= 10) # can't be too short, otherwise it is obvious\n", " & (df[\"duration\"] <= 240) # can't be badly long\n", " # & (df[\"has_continue_and_start_continue_at\"].isna()) # won't have any continues\n", " & (df[\"upvote_count\"] == 0) # won't have any likes\n", " & ((df[\"norm_play_frac\"] <= 3.1))\n", " # & (df[\"dislike_count\"] >= 1) # this is kinda strict\n", " # & (\n", " # (df_slice[\"is_in_playlist\"] == False)\n", " # & (df_slice[\"concat_in_playlist\"] == False)\n", " # ) # can't be part of a playlist -- otherwise there are some like signal in it?\n", ")\n", "pos_filter_selectin_mask = (\n", " (df[\"preference\"]) # get basics aligned\n", " # & (\n", " # df[\"good_continue_at\"]\n", " # ) # if continue, needs to continue off a certain percentage\n", " & (df[\"reaction_play_count\"] >= 1)\n", " & (df[\"play_rel_diff\"] >= 0) # this is more like quality assurance\n", " & (df[\"duration\"] >= 10) # can't be too short, otherwise it is obvious\n", " & (df[\"duration\"] <= 240) # can't be badly long\n", " & (df[\"dislike_count\"] == 0) # can't have dislikes\n", " & (df[\"flag_count\"] == 0) # can't have issues\n", " & (\n", " (\n", " (df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= concat_pos_play_count)\n", " & (df[\"concat_play_counts\"] >= concat_total_play_count)\n", " )\n", " | (\n", " (~df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= normal_pos_play_count)\n", " & (df[\"norm_play_frac\"] >= 1.5)\n", " )\n", " )\n", " # & (df[\"norm_play_frac\"] >= 1.9)\n", " & (df[\"user_n_clips\"] >= 40) # user needs to have genereated at least 20\n", " # & (df[\"duration_rel_diff\"] < 10) # positive isn't just longer\n", " & ((df[\"upvote_count\"] >= 1) | (df[\"reaction_play_count\"] >= 5)| (df[\"concat_play_counts\"] >= 5))\n", " # & ((df[\"upvote_count\"] >= 1) | (df[\"norm_play_frac\"] >= 1.9))\n", " # & (df[\"pos_diff_preference\"] == 2)\n", " & ((df[\"cer_diff_preference\"] < 0.2) & (df[\"cer\"] < 0.5)) # cut on hoot cer difference and abs cer\n", ")\n", "print(\n", " \"negative\",\n", " sum(neg_filter_selection_mask),\n", " \"positive\",\n", " sum(pos_filter_selectin_mask),\n", ")\n", "\n", "neg_filter_requests = df[neg_filter_selection_mask][\"request_id\"].unique()\n", "pos_filter_requests = df[pos_filter_selectin_mask][\"request_id\"].unique()\n", "# looking for very strong signal here:\n", "# listen to the positive/negative more than once\n", "# disliked one of the clips\n", "unique_requests = set(pos_filter_requests).intersection(neg_filter_requests)\n", "print(\n", " \"total pair requests\",\n", " df[\"request_id\"].nunique(),\n", " \" --> selected pair requests\",\n", " len(unique_requests),\n", " f\"frac {len(unique_requests) / df['request_id'].nunique():.3f}\",\n", ")" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.250737Z", "start_time": "2024-05-16T13:59:41.036434Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:12:02.919903Z", "iopub.status.busy": "2024-09-04T00:12:02.919746Z", "iopub.status.idle": "2024-09-04T00:12:03.131145Z", "shell.execute_reply": "2024-09-04T00:12:03.130567Z", "shell.execute_reply.started": "2024-09-04T00:12:02.919888Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "30b_t2_v16 requests 34464 clips 68928 total khrs 3.495; N gpus for 1000 iters 4.308; 4 gpus for x iters 1077.000; n unique users 25794 n pro users 13377\n" ] } ], "source": [ "df_slice = df[df[\"request_id\"].isin(set(unique_requests))].copy()\n", "print(\n", " f\"{os.path.basename(OUT_DATA_DIR)} requests\",\n", " df_slice[\"request_id\"].nunique(),\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"4 gpus for x iters {df_slice.shape[0] / 8 / 2 / 4:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", " f\"n pro users {df_slice[df_slice['is_pro_user']]['user_id'].nunique()}\",\n", ")\n", "# 76171 152342 total khrs 2.880 n gpus for 1250 iters 3.809\n", "# v10 has 78866\n", "# v14 has 110402\n", "# before cer: 30b_t2_v16 requests 45047 clips 90094 total khrs 4.560; N gpus for 1000 iters 5.631; 4 gpus for x iters 1407.719; n unique users 32119 n pro users 16178\n", "# after cer: 30b_t2_v16 requests 34464 clips 68928 total khrs 3.495; N gpus for 1000 iters 4.308; 4 gpus for x iters 1077.000; n unique users 25794 n pro users 13377" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "execution": { "iopub.execute_input": "2024-09-04T00:12:03.131991Z", "iopub.status.busy": "2024-09-04T00:12:03.131836Z", "iopub.status.idle": "2024-09-04T00:12:03.134122Z", "shell.execute_reply": "2024-09-04T00:12:03.133727Z", "shell.execute_reply.started": "2024-09-04T00:12:03.131976Z" } }, "outputs": [], "source": [ "# v1 requests 17681 clips 35362 total khrs 1.815; N gpus for 1500 iters 1.473; 4 gpus for x iters 552.531; n unique users 15179 n pro users 5983" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.277006Z", "start_time": "2024-05-16T13:59:41.252105Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:12:03.134809Z", "iopub.status.busy": "2024-09-04T00:12:03.134676Z", "iopub.status.idle": "2024-09-04T00:12:03.180809Z", "shell.execute_reply": "2024-09-04T00:12:03.180344Z", "shell.execute_reply.started": "2024-09-04T00:12:03.134796Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive in playlist (5735, 101)\n" ] } ], "source": [ "test_mask = (df_slice[\"preference\"]) & (\n", " (df_slice[\"is_in_playlist\"]) | (df_slice[\"concat_in_playlist\"])\n", ")\n", "print(\"positive in playlist\", df_slice[test_mask].shape)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.323409Z", "start_time": "2024-05-16T13:59:41.278278Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:12:03.181527Z", "iopub.status.busy": "2024-09-04T00:12:03.181384Z", "iopub.status.idle": "2024-09-04T00:12:03.212351Z", "shell.execute_reply": "2024-09-04T00:12:03.211973Z", "shell.execute_reply.started": "2024-09-04T00:12:03.181513Z" } }, "outputs": [], "source": [ "# interesting_clips_must_be_positive_mask = (\n", "# (df_slice[\"upvoted\"] == True)\n", "# | (df_slice[\"has_action\"] == True)\n", "# | (df_slice[\"part_of_concat\"] == True)\n", "# )\n", "# interesting_clips_must_be_not_negative_mask = (df_slice[\"downvoted\"] == False) # & (df_slice[\"dislike_count\"] < 1)\n", "# interesting_clips_mask = interesting_clips_must_be_positive_mask & interesting_clips_must_be_not_negative_mask\n", "# assert interesting_clips_mask.eq(df_slice[\"preference\"]).all()" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.392244Z", "start_time": "2024-05-16T13:59:41.324472Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:12:03.213022Z", "iopub.status.busy": "2024-09-04T00:12:03.212883Z", "iopub.status.idle": "2024-09-04T00:12:03.246284Z", "shell.execute_reply": "2024-09-04T00:12:03.245904Z", "shell.execute_reply.started": "2024-09-04T00:12:03.213010Z" } }, "outputs": [], "source": [ "# save positive ids\n", "# positive_preference_ids = df_slice[df_slice[\"preference\"] == False][\"s3_id\"].to_json(orient='values')\n", "# with open('/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id_negative.json', 'w') as file:\n", "# file.write(positive_preference_ids)" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "execution": { "iopub.execute_input": "2024-09-04T00:12:03.249130Z", "iopub.status.busy": "2024-09-04T00:12:03.248791Z", "iopub.status.idle": "2024-09-04T00:12:03.283030Z", "shell.execute_reply": "2024-09-04T00:12:03.282651Z", "shell.execute_reply.started": "2024-09-04T00:12:03.249115Z" } }, "outputs": [], "source": [ "# df_slice_2 = pd.read_csv(\"/home/tony/Data/Preference/30b_v1/interesting_clips_v4_t_1_20240808_slice.csv\")\n", "# df_total = pd.concat([df_slice, df_slice_2])\n", "# print(df_total.shape)" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "execution": { "iopub.execute_input": "2024-09-04T00:12:03.283704Z", "iopub.status.busy": "2024-09-04T00:12:03.283568Z", "iopub.status.idle": "2024-09-04T00:12:03.318107Z", "shell.execute_reply": "2024-09-04T00:12:03.317729Z", "shell.execute_reply.started": "2024-09-04T00:12:03.283690Z" } }, "outputs": [], "source": [ "# print(df_slices_2.shape, df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "execution": { "iopub.execute_input": "2024-09-04T00:12:03.318734Z", "iopub.status.busy": "2024-09-04T00:12:03.318609Z", "iopub.status.idle": "2024-09-04T00:12:03.353297Z", "shell.execute_reply": "2024-09-04T00:12:03.352923Z", "shell.execute_reply.started": "2024-09-04T00:12:03.318721Z" } }, "outputs": [], "source": [ "# print(df_total.shape)\n", "# df_slice = df_total.copy()" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T14:00:20.866354Z", "start_time": "2024-05-16T14:00:12.443344Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:13:35.727951Z", "iopub.status.busy": "2024-09-04T00:13:35.727460Z", "iopub.status.idle": "2024-09-04T00:13:35.730020Z", "shell.execute_reply": "2024-09-04T00:13:35.729608Z", "shell.execute_reply.started": "2024-09-04T00:13:35.727932Z" } }, "outputs": [], "source": [ "# df_slice.to_pickle(\"/home/tony/Data/Preference/30b_v1/interesting_clips_v4_t_2_20240830_v16_slice.pkl\")\n", "# BREAK" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932296Z", "start_time": "2024-05-16T13:59:41.932287Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:13:35.731030Z", "iopub.status.busy": "2024-09-04T00:13:35.730883Z", "iopub.status.idle": "2024-09-04T00:13:35.893470Z", "shell.execute_reply": "2024-09-04T00:13:35.893026Z", "shell.execute_reply.started": "2024-09-04T00:13:35.731016Z" } }, "outputs": [ { "data": { "text/plain": [ "(30175, 34464)" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# don't have continue at\n", "df_slice[\"request_id\"] = df_slice[\"request_id\"].astype(str)\n", "df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique(), df_slice[\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "execution": { "iopub.execute_input": "2024-09-04T00:13:35.894213Z", "iopub.status.busy": "2024-09-04T00:13:35.894065Z", "iopub.status.idle": "2024-09-04T00:13:35.994457Z", "shell.execute_reply": "2024-09-04T00:13:35.993887Z", "shell.execute_reply.started": "2024-09-04T00:13:35.894199Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(68928, 101)\n", "(68928, 101)\n", "(68928, 101)\n" ] } ], "source": [ "print(df_slice.shape)\n", "df_slice = df_slice[df_slice[\"request_id\"].apply(lambda x: len(x) > 3)]\n", "print(df_slice.shape)\n", "# df_slice = df_slice[df_slice[\"is_pro_user\"]].copy()\n", "print(df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932966Z", "start_time": "2024-05-16T13:59:41.932957Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:13:35.995298Z", "iopub.status.busy": "2024-09-04T00:13:35.995143Z", "iopub.status.idle": "2024-09-04T00:13:36.004295Z", "shell.execute_reply": "2024-09-04T00:13:36.003810Z", "shell.execute_reply.started": "2024-09-04T00:13:35.995283Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "34464\n" ] } ], "source": [ "final_filtered_requests = df_slice[\"request_id\"].astype(str).unique()\n", "# final_filtered_requests = df_slice[df_slice[\"is_pro_user\"]][\"request_id\"].astype(str).unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.933558Z", "start_time": "2024-05-16T13:59:41.933550Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:13:36.005657Z", "iopub.status.busy": "2024-09-04T00:13:36.005503Z", "iopub.status.idle": "2024-09-04T00:13:36.044542Z", "shell.execute_reply": "2024-09-04T00:13:36.044147Z", "shell.execute_reply.started": "2024-09-04T00:13:36.005642Z" } }, "outputs": [], "source": [ "# df_slice.to_csv(\"/home/tony/Data/Preference/7b_v2/7b_before_recode_20240412\", index=False)" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934277Z", "start_time": "2024-05-16T13:59:41.934268Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:13:36.045230Z", "iopub.status.busy": "2024-09-04T00:13:36.045096Z", "iopub.status.idle": "2024-09-04T00:13:36.252891Z", "shell.execute_reply": "2024-09-04T00:13:36.252322Z", "shell.execute_reply.started": "2024-09-04T00:13:36.045217Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "34119 345\n", "(68238, 102) (690, 102)\n" ] } ], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df.reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df.reset_index()\n", "\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934954Z", "start_time": "2024-05-16T13:59:41.934946Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:13:36.253733Z", "iopub.status.busy": "2024-09-04T00:13:36.253582Z", "iopub.status.idle": "2024-09-04T00:13:36.255833Z", "shell.execute_reply": "2024-09-04T00:13:36.255432Z", "shell.execute_reply.started": "2024-09-04T00:13:36.253718Z" } }, "outputs": [], "source": [ "# val_df[[\"request_id\", \"metadata\", \"updated_at\", \"user_id\", \"preference\"]].head()" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.935620Z", "start_time": "2024-05-16T13:59:41.935613Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:13:36.256495Z", "iopub.status.busy": "2024-09-04T00:13:36.256366Z", "iopub.status.idle": "2024-09-04T00:13:38.786586Z", "shell.execute_reply": "2024-09-04T00:13:38.786025Z", "shell.execute_reply.started": "2024-09-04T00:13:36.256482Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 68238/68238 [00:02<00:00, 27437.04it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "3,460 hours of 68238 clips, 4.264875 nodes, 710.8125 iters\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "total_duration = 0\n", "for i, row in tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " total_duration += row[\"duration\"]\n", " except Exception as E:\n", " print(i, row)\n", " print(E)\n", " raise ValueError()\n", " \n", "print(\n", " f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1000} nodes, {train_df.shape[0] / 8 / 2 / 6} iters\"\n", ")" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936268Z", "start_time": "2024-05-16T13:59:41.936260Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:13:38.787412Z", "iopub.status.busy": "2024-09-04T00:13:38.787257Z", "iopub.status.idle": "2024-09-04T00:13:49.617906Z", "shell.execute_reply": "2024-09-04T00:13:49.617375Z", "shell.execute_reply.started": "2024-09-04T00:13:38.787397Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 690/690 [00:10<00:00, 63.78it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 690 clips, 6 different prompts\n", "18 hours of False\n", "18 hours of True\n", "Done\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "make_dataset(val_df, OUT_DATA_DIR, is_val=True, npz_dir=NPZ_DIR)" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936964Z", "start_time": "2024-05-16T13:59:41.936957Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:13:49.618721Z", "iopub.status.busy": "2024-09-04T00:13:49.618571Z", "iopub.status.idle": "2024-09-04T00:31:20.560053Z", "shell.execute_reply": "2024-09-04T00:31:20.559347Z", "shell.execute_reply.started": "2024-09-04T00:13:49.618705Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 68238/68238 [17:30<00:00, 64.94it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 68238 clips, 529 different prompts\n", "1,763 hours of False\n", "1,697 hours of True\n", "Done\n" ] } ], "source": [ "make_dataset(train_df, OUT_DATA_DIR, is_val=False, npz_dir=NPZ_DIR)" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.937879Z", "start_time": "2024-05-16T13:59:41.937870Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:31:20.561135Z", "iopub.status.busy": "2024-09-04T00:31:20.560962Z", "iopub.status.idle": "2024-09-04T00:31:20.585073Z", "shell.execute_reply": "2024-09-04T00:31:20.584511Z", "shell.execute_reply.started": "2024-09-04T00:31:20.561118Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, 6016, 13)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000\n", "assert mm[:100, :, 1:].min() >= 0\n", "assert mm[:100, :, 1:].max() <= 2048" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.938629Z", "start_time": "2024-05-16T13:59:41.938621Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:31:20.585938Z", "iopub.status.busy": "2024-09-04T00:31:20.585790Z", "iopub.status.idle": "2024-09-04T00:31:20.607019Z", "shell.execute_reply": "2024-09-04T00:31:20.606528Z", "shell.execute_reply.started": "2024-09-04T00:31:20.585923Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as codec_encode,\n", "# decode_stream_to_full_audio as codec_decode,\n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# decode as decode\n", "# )\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "# assert len(test_metas) == len(mm)\n", "# idx_list = list(range(len(test_metas)))\n", "# # random.shuffle(idx_list)\n", "# # idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939205Z", "start_time": "2024-05-16T13:59:41.939198Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:31:20.607754Z", "iopub.status.busy": "2024-09-04T00:31:20.607618Z", "iopub.status.idle": "2024-09-04T00:31:20.643415Z", "shell.execute_reply": "2024-09-04T00:31:20.642931Z", "shell.execute_reply.started": "2024-09-04T00:31:20.607741Z" } }, "outputs": [], "source": [ "# import random\n", "# idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "# assert \"original_duration_s\" in test_metas[idx]\n", "# # positive index should be shifted by 1\n", "# pos_idx = idx + 1\n", "# print(\n", "# \"tags:\",\n", "# test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", "# test_metas[idx].get(\"tags\"),\n", "# )\n", "# arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[: pad_idx_arr[0], :]\n", "# pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pos_pad_idx_arr) > 0:\n", "# pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "# a = decode(arr)\n", "# print(\"\\n negative example \\n\", test_metas[idx])\n", "# a.play(compress=False)\n", "# pos_a = decode(pos_arr)\n", "# print(\"\\n positive example \\n\", test_metas[pos_idx])\n", "# pos_a.play(compress=False)\n", "# print(\n", "# \"text:\",\n", "# test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", "# test_metas[idx].get(\"text\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939977Z", "start_time": "2024-05-16T13:59:41.939969Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:31:20.645591Z", "iopub.status.busy": "2024-09-04T00:31:20.645266Z", "iopub.status.idle": "2024-09-04T00:31:20.677626Z", "shell.execute_reply": "2024-09-04T00:31:20.677136Z", "shell.execute_reply.started": "2024-09-04T00:31:20.645575Z" } }, "outputs": [], "source": [ "# val_df[val_df[\"tags\"] == 'a vibrant blend of experimental jazz fusion, drum-and-bass and swagger fuzzed-out guitars']" ] }, { "cell_type": "code", "execution_count": 39, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.940610Z", "start_time": "2024-05-16T13:59:41.940603Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:31:20.678642Z", "iopub.status.busy": "2024-09-04T00:31:20.678280Z", "iopub.status.idle": "2024-09-04T00:31:20.713084Z", "shell.execute_reply": "2024-09-04T00:31:20.712606Z", "shell.execute_reply.started": "2024-09-04T00:31:20.678627Z" } }, "outputs": [], "source": [ "# from collections import Counter\n", "# c = Counter()\n", "# for _, row in df_slice.iterrows():\n", "# # print(row[\"metadata\"])\n", "# for k in ast.literal_eval(row[\"metadata\"]).keys():\n", "# c[k] += 1" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941167Z", "start_time": "2024-05-16T13:59:41.941159Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:31:20.713809Z", "iopub.status.busy": "2024-09-04T00:31:20.713671Z", "iopub.status.idle": "2024-09-04T00:31:20.748337Z", "shell.execute_reply": "2024-09-04T00:31:20.747855Z", "shell.execute_reply.started": "2024-09-04T00:31:20.713796Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941801Z", "start_time": "2024-05-16T13:59:41.941793Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:31:20.749127Z", "iopub.status.busy": "2024-09-04T00:31:20.748993Z", "iopub.status.idle": "2024-09-04T00:31:20.786472Z", "shell.execute_reply": "2024-09-04T00:31:20.785935Z", "shell.execute_reply.started": "2024-09-04T00:31:20.749115Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "345 0\n" ] } ], "source": [ "def validation_on_metas(input_metas):\n", " total_bad = 0\n", " total_good = 0\n", " for idx in range(len(input_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if input_metas[idx].get(\"tags\") != input_metas[pos_idx].get(\"tags\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", " print(total_good, total_bad)\n", " return\n", "\n", "\n", "validation_on_metas(test_metas)" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.942520Z", "start_time": "2024-05-16T13:59:41.942511Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:31:20.787240Z", "iopub.status.busy": "2024-09-04T00:31:20.787096Z", "iopub.status.idle": "2024-09-04T00:31:20.831207Z", "shell.execute_reply": "2024-09-04T00:31:20.830683Z", "shell.execute_reply.started": "2024-09-04T00:31:20.787226Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.943072Z", "start_time": "2024-05-16T13:59:41.943065Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:31:20.831992Z", "iopub.status.busy": "2024-09-04T00:31:20.831853Z", "iopub.status.idle": "2024-09-04T00:31:20.863685Z", "shell.execute_reply": "2024-09-04T00:31:20.863166Z", "shell.execute_reply.started": "2024-09-04T00:31:20.831979Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)" ] }, { "cell_type": "code", "execution_count": 44, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.944246Z", "start_time": "2024-05-16T13:59:41.944237Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:31:20.864588Z", "iopub.status.busy": "2024-09-04T00:31:20.864451Z", "iopub.status.idle": "2024-09-04T00:31:20.898769Z", "shell.execute_reply": "2024-09-04T00:31:20.898235Z", "shell.execute_reply.started": "2024-09-04T00:31:20.864575Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total samples 68238 (68238, 102)\n" ] } ], "source": [ "total_iters = len(n_neg_tr) + len(n_pos_tr)\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945249Z", "start_time": "2024-05-16T13:59:41.945241Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:31:20.899554Z", "iopub.status.busy": "2024-09-04T00:31:20.899413Z", "iopub.status.idle": "2024-09-04T00:31:20.931582Z", "shell.execute_reply": "2024-09-04T00:31:20.931043Z", "shell.execute_reply.started": "2024-09-04T00:31:20.899541Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 epoch per batch 2, total 1066.21875\n" ] } ], "source": [ "print(\"1 epoch per batch 2, total\", total_iters / 8 / 2 / 4)" ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945972Z", "start_time": "2024-05-16T13:59:41.945964Z" }, "execution": { "iopub.execute_input": "2024-09-04T00:31:20.932618Z", "iopub.status.busy": "2024-09-04T00:31:20.932235Z", "iopub.status.idle": "2024-09-04T00:31:21.203110Z", "shell.execute_reply": "2024-09-04T00:31:21.202416Z", "shell.execute_reply.started": "2024-09-04T00:31:20.932603Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Submitted batch job 2250\n" ] } ], "source": [ "!cd /home/tony/Work/tony/slurm/30b_dpo && sbatch sbatch_ipo_30b_t2" ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "execution": { "iopub.execute_input": "2024-09-04T00:31:36.426314Z", "iopub.status.busy": "2024-09-04T00:31:36.426002Z", "iopub.status.idle": "2024-09-04T00:31:36.437323Z", "shell.execute_reply": "2024-09-04T00:31:36.436769Z", "shell.execute_reply.started": "2024-09-04T00:31:36.426297Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cache kept!\n" ] } ], "source": [ "import shutil\n", "\n", "# Basic file copy\n", "shutil.copy('/home/tony/Work/tony/Preference/make_dataset_13b_v4_t2.ipynb', os.path.join(OUT_DATA_DIR, \"make_dataset.ipynb\"))\n", "print(\"Cache kept!\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# some gymathtics loading prev data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.946562Z", "start_time": "2024-05-16T13:59:41.946555Z" }, "execution": { "iopub.status.busy": "2024-09-04T00:31:21.301219Z", "iopub.status.idle": "2024-09-04T00:31:21.301414Z", "shell.execute_reply": "2024-09-04T00:31:21.301324Z", "shell.execute_reply.started": "2024-09-04T00:31:21.301316Z" } }, "outputs": [], "source": [ "# prev_v3_data = \"/app/suno/data/dpo/7v_v20_full/\"\n", "\n", "# test_val_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_val.jsonl\"))\n", "# test_tr_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_tr.jsonl\"))\n", "\n", "# all_ids = set()\n", "# for meta in test_val_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# for meta in test_tr_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# print(len(all_ids), len(test_val_metas) + len(test_tr_metas))\n", "\n", "# all_ids = list(all_ids)\n", "# with open(\"/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id.json\", \"w\") as fp:\n", "# json.dump(all_ids, fp)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2024-09-04T00:31:21.302240Z", "iopub.status.idle": "2024-09-04T00:31:21.302418Z", "shell.execute_reply": "2024-09-04T00:31:21.302332Z", "shell.execute_reply.started": "2024-09-04T00:31:21.302324Z" } }, "outputs": [], "source": [ "val_df.head(n=10)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }