{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.040680Z", "start_time": "2024-05-16T13:58:19.777010Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:17:42.034654Z", "iopub.status.busy": "2024-09-17T00:17:42.034515Z", "iopub.status.idle": "2024-09-17T00:17:43.477854Z", "shell.execute_reply": "2024-09-17T00:17:43.477344Z", "shell.execute_reply.started": "2024-09-17T00:17:42.034639Z" } }, "outputs": [], "source": [ "import ast\n", "import os\n", "import shutil\n", "import sys\n", "from collections import defaultdict\n", "\n", "import numpy as np\n", "import pandas as pd\n", "from preference_data_preparation_4min_30b import *\n", "from preference_helper import *\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n", "from tqdm import tqdm\n", "\n", "pd.set_option(\"display.max_rows\", 500)\n", "pd.set_option(\"display.max_columns\", 500)\n", "pd.set_option(\"display.width\", 1000)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.082172Z", "start_time": "2024-05-16T13:58:21.041926Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:17:43.478692Z", "iopub.status.busy": "2024-09-17T00:17:43.478501Z", "iopub.status.idle": "2024-09-17T00:17:43.507452Z", "shell.execute_reply": "2024-09-17T00:17:43.507012Z", "shell.execute_reply.started": "2024-09-17T00:17:43.478676Z" } }, "outputs": [], "source": [ "OUT_DATA_DIR = \"/app/suno/data/dpo/30b_t3_v15\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "shutil.copyfile(\n", " \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", " os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", ")\n", "NPZ_DIR = \"/app/suno/data/dpo/30b_npz\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:53.962528Z", "start_time": "2024-05-16T13:58:21.105919Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:17:43.508171Z", "iopub.status.busy": "2024-09-17T00:17:43.508030Z", "iopub.status.idle": "2024-09-17T00:17:49.559073Z", "shell.execute_reply": "2024-09-17T00:17:49.558507Z", "shell.execute_reply.started": "2024-09-17T00:17:43.508157Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Preference data shape (181264, 68)\n" ] } ], "source": [ "df = pd.read_pickle(\n", " \"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_20240912_full_with_cer.pkl\"\n", ") # , engine='python')\n", "# df = pd.read_csv(\n", "# \"/home/tony/Data/Preference/30b_v0/interesting_clips_v4_t_1_20240808.csv\"\n", "# ) # , engine='python')\n", "print(\"Preference data shape\", df.shape)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:17:49.559920Z", "iopub.status.busy": "2024-09-17T00:17:49.559767Z", "iopub.status.idle": "2024-09-17T00:17:49.771016Z", "shell.execute_reply": "2024-09-17T00:17:49.770458Z", "shell.execute_reply.started": "2024-09-17T00:17:49.559904Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(181264, 68)\n", "(181264, 64)\n" ] } ], "source": [ "print(df.shape)\n", "df = df.dropna(axis=1, how=\"all\")\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.199480Z", "start_time": "2024-05-16T13:58:53.963687Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:17:49.771850Z", "iopub.status.busy": "2024-09-17T00:17:49.771700Z", "iopub.status.idle": "2024-09-17T00:17:52.186292Z", "shell.execute_reply": "2024-09-17T00:17:52.185719Z", "shell.execute_reply.started": "2024-09-17T00:17:49.771835Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1054349\n", "1054349\n", "pre-downloaded df (181264, 64)\n", "downloaded df (181264, 64)\n" ] } ], "source": [ "converted_paths = os.listdir(NPZ_DIR)\n", "print(len(converted_paths))\n", "\n", "converted_paths = set([f.replace(\".npz\", \"\") for f in converted_paths])\n", "print(len(converted_paths))\n", "\n", "print(\"pre-downloaded df\", df.shape)\n", "df[df[\"s3_id\"].isin(converted_paths)].shape\n", "df = df[df[\"s3_id\"].isin(converted_paths)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.467253Z", "start_time": "2024-05-16T13:58:56.207647Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:17:52.188036Z", "iopub.status.busy": "2024-09-17T00:17:52.187870Z", "iopub.status.idle": "2024-09-17T00:17:52.231493Z", "shell.execute_reply": "2024-09-17T00:17:52.231054Z", "shell.execute_reply.started": "2024-09-17T00:17:52.188020Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "is_30b\n", "True 181264\n", "Name: count, dtype: int64\n" ] }, { "data": { "text/plain": [ "task\n", " 141798\n", "extend 39216\n", "cover 232\n", "infill 12\n", "artist_consistency 6\n", "Name: count, dtype: int64" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"is_30b\"] = df[\"model_name\"].str.contains(\"-t\")\n", "print(df[\"is_30b\"].value_counts())\n", "df[\"task\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.592883Z", "start_time": "2024-05-16T13:58:56.470781Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:17:52.232259Z", "iopub.status.busy": "2024-09-17T00:17:52.232118Z", "iopub.status.idle": "2024-09-17T00:17:52.327263Z", "shell.execute_reply": "2024-09-17T00:17:52.326741Z", "shell.execute_reply.started": "2024-09-17T00:17:52.232245Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "preference model_name \n", "False chirp-v3p5-engine-t-3 90632\n", "True chirp-v3p5-engine-t-3 90632\n", "Name: count, dtype: int64\n", "(181264, 65)\n", "(181264, 65)\n" ] } ], "source": [ "## for 13b this is easy for now\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "print(df.shape)\n", "df = df[df[\"model_name\"].isin([\"chirp-v3p5-engine-t-3\"])]\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.909539Z", "start_time": "2024-05-16T13:58:56.595736Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:17:52.328055Z", "iopub.status.busy": "2024-09-17T00:17:52.327911Z", "iopub.status.idle": "2024-09-17T00:17:52.745445Z", "shell.execute_reply": "2024-09-17T00:17:52.744909Z", "shell.execute_reply.started": "2024-09-17T00:17:52.328040Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(181264, 65)\n", "(181264, 65)\n", "preference model_name \n", "False chirp-v3p5-engine-t-3 90632\n", "True chirp-v3p5-engine-t-3 90632\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(df.shape)\n", "df = df[\n", " df[\"request_id\"].isin(\n", " df[\"request_id\"].value_counts().index[df[\"request_id\"].value_counts() == 2]\n", " )\n", "]\n", "print(df.shape)\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "assert df.shape[0] == df[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:17:52.746237Z", "iopub.status.busy": "2024-09-17T00:17:52.746092Z", "iopub.status.idle": "2024-09-17T00:18:38.255241Z", "shell.execute_reply": "2024-09-17T00:18:38.254654Z", "shell.execute_reply.started": "2024-09-17T00:17:52.746222Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "unique_requests 90632\n" ] } ], "source": [ "# Let's use the old selection for now -- for quality assurance\n", "# expand the metadata columns -- this takes forever...~ 6 mins\n", "test_slice = df[\"metadata\"].apply(lambda x: ast.literal_eval(str(x)))\n", "test_slice_series = test_slice.apply(pd.Series)\n", "df = pd.concat([df, test_slice_series], axis=1, join=\"inner\")\n", "print(\"unique_requests\", df[\"request_id\"].nunique())" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:18:38.256213Z", "iopub.status.busy": "2024-09-17T00:18:38.256063Z", "iopub.status.idle": "2024-09-17T00:18:38.737348Z", "shell.execute_reply": "2024-09-17T00:18:38.736757Z", "shell.execute_reply.started": "2024-09-17T00:18:38.256198Z" } }, "outputs": [], "source": [ "df = df.loc[:, ~df.columns.duplicated()].copy()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:18:38.738236Z", "iopub.status.busy": "2024-09-17T00:18:38.738088Z", "iopub.status.idle": "2024-09-17T00:18:38.748871Z", "shell.execute_reply": "2024-09-17T00:18:38.748445Z", "shell.execute_reply.started": "2024-09-17T00:18:38.738222Z" } }, "outputs": [ { "data": { "text/plain": [ "task\n", " 141798\n", "extend 39216\n", "cover 232\n", "infill 12\n", "artist_consistency 6\n", "Name: count, dtype: int64" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"task\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:18:38.749621Z", "iopub.status.busy": "2024-09-17T00:18:38.749480Z", "iopub.status.idle": "2024-09-17T00:18:39.169970Z", "shell.execute_reply": "2024-09-17T00:18:39.169494Z", "shell.execute_reply.started": "2024-09-17T00:18:38.749607Z" } }, "outputs": [ { "data": { "text/plain": [ "pos_diff_preference\n", "1.0 68384\n", "2.0 22248\n", "Name: count, dtype: int64" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df.sort_values(by=[\"request_id\", \"preference\", \"diff_preference\"])\n", "df[\"pos_diff_preference\"] = df[\"diff_preference\"].diff()\n", "df[\"cer_diff_preference\"] = df[\"cer\"].diff()\n", "df[df[\"preference\"]][\"pos_diff_preference\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:18:39.170787Z", "iopub.status.busy": "2024-09-17T00:18:39.170644Z", "iopub.status.idle": "2024-09-17T00:18:39.172880Z", "shell.execute_reply": "2024-09-17T00:18:39.172497Z", "shell.execute_reply.started": "2024-09-17T00:18:39.170774Z" } }, "outputs": [], "source": [ "# df[[\"preference\", \"diff_preference\", \"pos_diff_preference\", \"cer\", \"cer_diff_preference\"]].tail(n=6)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:18:39.173544Z", "iopub.status.busy": "2024-09-17T00:18:39.173416Z", "iopub.status.idle": "2024-09-17T00:18:39.294971Z", "shell.execute_reply": "2024-09-17T00:18:39.294457Z", "shell.execute_reply.started": "2024-09-17T00:18:39.173531Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive param_experiment\n", "max_cfg_12_ntag_3 23349\n", "max_cfg_12 2350\n", "text_cfg_12_250 2322\n", "cfg_12 2135\n", "n_repeat_tags_3 1991\n", "s_30_t_12_tag_3 90\n", "s_max_t_12_tag_2 88\n", "s_max_t_12_tag_4 84\n", "s_30_t_12_tag_4 81\n", "s_max_t_12_tag_3 79\n", "s_30_t_12_tag_2 77\n", "Name: count, dtype: int64\n" ] } ], "source": [ "try:\n", " print(\"positive\", df[df[\"preference\"]][\"param_experiment\"].value_counts())\n", "except:\n", " pass" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:18:39.295826Z", "iopub.status.busy": "2024-09-17T00:18:39.295684Z", "iopub.status.idle": "2024-09-17T00:18:39.796236Z", "shell.execute_reply": "2024-09-17T00:18:39.795750Z", "shell.execute_reply.started": "2024-09-17T00:18:39.295813Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.3706449999999997\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyeklEQVR4nO3de1TVZb7H8c/eXNK8EALmeLSphYGmCKgtw4HjdFEbtTmDVnqyzKK0zNTJSR3HG+oR9ZCjlqWjRpE25Ent1KitqdbRbIFdTPLgHS0v4zQC3sBLcvmdPxp+py0Ie+MG9n58v9ZiLfntZz883/2wtx+e381hWZYlAAAAwzgbewAAAAD1gZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIwU2NgD8AVFRcXy5s0tHA4pLKyF1/v1JabXSH3+z/Qaqc//mV5jfdZX2XdtCDmSLEv18gtWX/36EtNrpD7/Z3qN1Of/TK+xMetjdxUAADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAI3EXcgBGCwio+W+5igpLFRUG3wIauI55tJKzfPlyDR48WPHx8UpISNDo0aN1+PBhlzaPPfaYoqOjXb6mT5/u0ubEiRMaOXKkYmNjlZCQoPnz56usrMylzeeff67k5GR16dJFffr00fr166uMZ82aNbrnnnsUExOjhx56SLt27fKkHAAGczodKq+wFBrarMavkJtulNPpaOzhAqgHHq3kfPHFFxo2bJhiYmJUXl6uhQsXKiUlRRs3btSNN95ot3v44Yc1duxY+/umTZva/y4vL9eoUaMUHh6urKwsnTx5UpMmTVJQUJBeeOEFSdKxY8c0atQoDR06VOnp6crJydHUqVMVERGhpKQkSdKmTZuUlpam1NRUxcbG6s0331RKSoo+/PBDhYWFXdOLAsD/ORwOBTgdGpe1U/knS6pt06F1cy0eGi+n08FqDmAgj0LOqlWrXL6fN2+eEhIStHv3bt1555329iZNmigiIqLaPj777DPl5+crIyND4eHh6tSpk8aNG6f09HSNGTNGwcHBysrKUrt27TR58mRJUmRkpHbs2KE33njDDjkZGRl6+OGHNXjwYElSamqqtmzZonXr1mnkyJGelAXAYPknS7T7xLnGHgaARnBNx+QUFxdLkkJCQly2f/DBB3r//fcVERGhu+++W6NHj7ZXc3JzcxUVFaXw8HC7fWJiombOnKn8/Hzdcccdys3NVUJCgkufiYmJmjt3riTp8uXL2r17t0aNGmU/7nQ61atXL+3cudPjOhxeXqmu7M/b/foS02ukPv/naW3+9lqYPoem1yeZX2N91udun3UOORUVFZo7d666deumqKgoe/vAgQPVtm1btW7dWvv371d6erq+/fZbvfLKK5KkwsJCl4Ajyf6+oKCgxjYlJSW6dOmSzp49q/Ly8iq7pcLCwqocI+SOsLAWHj+nMfv1JabXSH3Xh9DQZo09hDozfQ5Nr08yv8bGrK/OISc1NVUHDx7U22+/7bJ9yJAh9r+jo6MVERGhESNG6OjRo7rlllvqPtJ6VFRULMuLu+Mdjh8n1dv9+hLTa6Q+/xcY6NRNN7kXXk6fPq/y8op6HpF3mT6HptcnmV9jfdZX2Xdt6hRyZs2apS1btmj16tVq06ZNjW1jY2MlSUeOHNEtt9yi8PDwKmdBFRYWSpJ9HE94eLi97adtmjdvriZNmsjpdCogIEBFRUUubYqKiqqsALnDslQvv2D11a8vMb1G6vNfntblr6+DyXMomV+fZH6NjVmfR6eQW5alWbNm6aOPPtKbb76p9u3b1/qcvXv3Svr/ABMXF6cDBw64BJTs7Gw1b95cHTp0sNts377dpZ/s7GzFxcVJkoKDg9W5c2fl5OTYj1dUVCgnJ0fx8fGelAQAAAzlUchJTU3V+++/r5deeknNmjVTQUGBCgoKdOnSJUnS0aNHtXTpUuXl5en48eP65JNPNGnSJN15553q2LGjpB8PIO7QoYMmTpyoffv2adu2bVq0aJGGDRum4OBgSdLQoUN17NgxLViwQIcOHdKaNWu0efNmjRgxwh7LE088obVr12rDhg06dOiQZs6cqYsXL2rQoEFeemkAAIA/82h31Z///GdJP17w76fS0tI0aNAgBQUFKScnR5mZmbpw4YJ+9rOfqW/fvho9erTdNiAgQMuWLdPMmTM1ZMgQNW3aVMnJyS7X1Wnfvr2WL1+utLQ0ZWZmqk2bNpozZ459+rgk9e/fX6dOndKSJUtUUFCgTp06aeXKlXXaXQUAAMzjsCyT9wS6p7DQ+wceh4e38Hq/vsT0GqnP/wUGOhUa2kwDlmy76nVyOrdtqY1jk3T69HmVlfnfgccmz6Hp9Unm11if9VX2XRtu0AkAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARvIo5CxfvlyDBw9WfHy8EhISNHr0aB0+fNilzQ8//KDU1FT17NlT8fHxev7551VYWOjS5sSJExo5cqRiY2OVkJCg+fPnq6yszKXN559/ruTkZHXp0kV9+vTR+vXrq4xnzZo1uueeexQTE6OHHnpIu3bt8qQcAABgMI9CzhdffKFhw4Zp7dq1ysjIUFlZmVJSUnThwgW7zdy5c/U///M/WrRokd566y2dPHlSY8aMsR8vLy/XqFGjVFpaqqysLM2bN08bNmzQkiVL7DbHjh3TqFGj1LNnT/33f/+3Hn/8cU2dOlXbtm2z22zatElpaWl67rnntGHDBnXs2FEpKSkqKiq6ltcDAAAYwqOQs2rVKg0aNEi33367OnbsqHnz5unEiRPavXu3JKm4uFjr1q3T5MmTlZCQoC5dumju3LnauXOncnNzJUmfffaZ8vPz9Z//+Z/q1KmTevfurXHjxmnNmjW6fPmyJCkrK0vt2rXT5MmTFRkZqUcffVT9+vXTG2+8YY8lIyNDDz/8sAYPHqwOHTooNTVVTZo00bp167zzygAAAL92TcfkFBcXS5JCQkIkSXl5eSotLVWvXr3sNpGRkWrbtq0dcnJzcxUVFaXw8HC7TWJiokpKSpSfn2+3SUhIcPlZiYmJdh+XL1/W7t27XX6O0+lUr169tHPnTo/rcDi8/1Vf/frSl+k1Up//fzX25wBzSH3Xe431WZ87Aj37GPh/FRUVmjt3rrp166aoqChJUmFhoYKCgtSyZUuXtmFhYSooKLDb/DTgSLK/r61NSUmJLl26pLNnz6q8vFxhYWFVfs6Vxwi5IyyshcfPacx+fYnpNVLf9SE0tFljD6HOTJ9D0+uTzK+xMeurc8hJTU3VwYMH9fbbb3tzPI2iqKhYluW9/hyOHyfV2/36EtNrpD7/Fxjo1E03uRdeTp8+r/LyinoekXeZPoem1yeZX2N91lfZd23qFHJmzZqlLVu2aPXq1WrTpo29PTw8XKWlpTp37pzLak5RUZEiIiLsNleeBVV59tVP21x5RlZhYaGaN2+uJk2ayOl0KiAgoMpBxkVFRVVWgNxhWaqXX7D66teXmF4j9fkvT+vy19fB5DmUzK9PMr/GxqzPo2NyLMvSrFmz9NFHH+nNN99U+/btXR7v0qWLgoKClJOTY287fPiwTpw4obi4OElSXFycDhw44BJQsrOz1bx5c3Xo0MFus337dpe+s7Oz7T6Cg4PVuXNnl59TUVGhnJwcxcfHe1ISAAAwlEchJzU1Ve+//75eeuklNWvWTAUFBSooKNClS5ckSS1atNDgwYM1b948bd++XXl5eZoyZYri4+PtgJKYmKgOHTpo4sSJ2rdvn7Zt26ZFixZp2LBhCg4OliQNHTpUx44d04IFC3To0CGtWbNGmzdv1ogRI+yxPPHEE1q7dq02bNigQ4cOaebMmbp48aIGDRrknVcGAAD4NY92V/35z3+WJD322GMu29PS0uxwMWXKFDmdTo0dO1aXL19WYmKiZsyYYbcNCAjQsmXLNHPmTA0ZMkRNmzZVcnKyxo4da7dp3769li9frrS0NGVmZqpNmzaaM2eOkpKS7Db9+/fXqVOntGTJEhUUFKhTp05auXJlnXZXAQAA8zgsy+Q9ge4pLPT+gcfh4S283q8vMb1G6vN/gYFOhYY204Al27T7xLlq23Ru21Ibxybp9OnzKivzvwOPTZ5D0+uTzK+xPuur7Ls23LsKAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEbyOOR8+eWXeuaZZ5SYmKjo6Gh9/PHHLo9PnjxZ0dHRLl8pKSkubc6cOaMJEyaoW7du6tGjh6ZMmaLz58+7tNm3b58eeeQRxcTEqHfv3lqxYkWVsWzevFn333+/YmJi9MADD2jr1q2elgMAAAzlcci5cOGCoqOjNWPGjKu2SUpK0meffWZ/LVy40OXx3/3ud8rPz1dGRoaWLVumr776StOnT7cfLykpUUpKitq2bav169dr4sSJeuWVV/TOO+/Ybb7++mtNmDBBDz74oN577z3de++9eu6553TgwAFPSwIAAAYK9PQJvXv3Vu/evWtsExwcrIiIiGofO3TokLZt26Z3331XMTExkqSpU6dq5MiRmjhxom6++Wa9//77Ki0t1dy5cxUcHKzbb79de/fuVUZGhoYMGSJJyszMVFJSkp566ilJ0vjx45Wdna3Vq1dr1qxZnpYFAAAM43HIcccXX3yhhIQEtWzZUnfddZfGjx+v0NBQSdLOnTvVsmVLO+BIUq9eveR0OrVr1y716dNHubm56tGjh4KDg+02iYmJWrFihc6ePauQkBDl5uZqxIgRLj83MTGxyu4zdzgcdauztv683a8vMb1G6vN/ntbmb6+F6XNoen2S+TXWZ33u9un1kJOUlKQ+ffqoXbt2OnbsmBYuXKinn35a77zzjgICAlRYWKhWrVq5DiIwUCEhISooKJAkFRYWql27di5twsPD7cdCQkJUWFhob6sUFhamwsJCj8ccFtbC4+c0Zr++xPQaqe/6EBrarLGHUGemz6Hp9Unm19iY9Xk95AwYMMD+d+WBx/fdd5+9uuOLioqKZVne68/h+HFSvd2vLzG9Rurzf4GBTt10k3vh5fTp8yovr6jnEXmX6XNoen2S+TXWZ32VfdemXnZX/VT79u0VGhqqI0eOKCEhQeHh4Tp16pRLm7KyMp09e9Y+jic8PLzKikzl95WrN9W1KSoqqrK64w7LUr38gtVXv77E9Bqpz395Wpe/vg4mz6Fkfn2S+TU2Zn31fp2c77//XmfOnLEDTHx8vM6dO6e8vDy7zfbt21VRUaGuXbtKkuLi4vTVV1+ptLTUbpOdna3bbrtNISEhdpvt27e7/Kzs7GzFxcXVc0UAAMAfeBxyzp8/r71792rv3r2SpOPHj2vv3r06ceKEzp8/r/nz5ys3N1fHjx9XTk6ORo8erZ///OdKSkqSJEVGRiopKUnTpk3Trl27tGPHDs2ePVsDBgzQzTffLEl64IEHFBQUpD/84Q86ePCgNm3apMzMTD3xxBP2OIYPH65t27bp9ddf16FDh/Tyyy8rLy9Pjz76qDdeFwAA4Oc83l2Vl5en4cOH29+npaVJkpKTkzVz5kwdOHBA7733noqLi9W6dWv94he/0Lhx41zOlEpPT9fs2bP1+OOPy+l0qm/fvpo6dar9eIsWLbRq1SrNmjVLgwYNUmhoqEaPHm2fPi5J3bp1U3p6uhYtWqSFCxfq1ltv1dKlSxUVFVWnFwIAAJjFYVkm7wl0T2Gh9w88Dg9v4fV+fYnpNVKf/wsMdCo0tJkGLNmm3SfOVdumc9uW2jg2SadPn1dZmf8deGzyHJpen2R+jfVZX2XfteHeVQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwksch58svv9QzzzyjxMRERUdH6+OPP3Z53LIsLV68WImJieratatGjBih7777zqXNmTNnNGHCBHXr1k09evTQlClTdP78eZc2+/bt0yOPPKKYmBj17t1bK1asqDKWzZs36/7771dMTIweeOABbd261dNyAACAoTwOORcuXFB0dLRmzJhR7eMrVqzQW2+9pZkzZ2rt2rVq2rSpUlJS9MMPP9htfve73yk/P18ZGRlatmyZvvrqK02fPt1+vKSkRCkpKWrbtq3Wr1+viRMn6pVXXtE777xjt/n66681YcIEPfjgg3rvvfd077336rnnntOBAwc8LQkAABjI45DTu3dv/fa3v1WfPn2qPGZZljIzM/Xss8/qvvvuU8eOHbVgwQKdPHnSXvE5dOiQtm3bpjlz5ig2NlY9evTQ1KlTtXHjRv3jH/+QJL3//vsqLS3V3Llzdfvtt2vAgAF67LHHlJGRYf+szMxMJSUl6amnnlJkZKTGjx+vO+64Q6tXr67rawEAAAwS6M3Ojh8/roKCAvXq1cve1qJFC8XGxmrnzp0aMGCAdu7cqZYtWyomJsZu06tXLzmdTu3atUt9+vRRbm6uevTooeDgYLtNYmKiVqxYobNnzyokJES5ubkaMWKEy89PTEyssvvMHQ6H57W605+3+/UlptdIff7P09r87bUwfQ5Nr08yv8b6rM/dPr0acgoKCiRJYWFhLtvDwsJUWFgoSSosLFSrVq1cBxEYqJCQEPv5hYWFateunUub8PBw+7GQkBAVFhba26r7OZ4IC2vh8XMas19fYnqN1Hd9CA1t1thDqDPT59D0+iTza2zM+rwacvxVUVGxLMt7/TkcP06qt/v1JabXSH3+LzDQqZtuci+8nD59XuXlFfU8Iu8yfQ5Nr08yv8b6rK+y79p4NeRERERIkoqKitS6dWt7e1FRkTp27CjpxxWZU6dOuTyvrKxMZ8+etZ8fHh5eZUWm8vvK1Zvq2hQVFVVZ3XGHZalefsHqq19fYnqN1Oe7nE6HnM6rr1k7nZ4dcuivr4M/z6E7TK9PMr/GxqzPq9fJadeunSIiIpSTk2NvKykp0TfffKP4+HhJUnx8vM6dO6e8vDy7zfbt21VRUaGuXbtKkuLi4vTVV1+ptLTUbpOdna3bbrtNISEhdpvt27e7/Pzs7GzFxcV5syQAPsjpdCjkphsVGtrsql8tWzZt7GECaGQer+ScP39eR48etb8/fvy49u7dq5CQELVt21bDhw/Xa6+9pp///Odq166dFi9erNatW+u+++6TJEVGRiopKUnTpk1TamqqSktLNXv2bA0YMEA333yzJOmBBx7Q0qVL9Yc//EFPP/20Dh48qMzMTP3+97+3f+7w4cP12GOP6fXXX1fv3r21adMm5eXladasWdf6mgDwcU6nQ4EBTo3L2qn8kyXVtvlldIRe7NexgUcGwJd4HHLy8vI0fPhw+/u0tDRJUnJysubNm6enn35aFy9e1PTp03Xu3Dl1795dK1eu1A033GA/Jz09XbNnz9bjjz8up9Opvn37aurUqfbjLVq00KpVqzRr1iwNGjRIoaGhGj16tIYMGWK36datm9LT07Vo0SItXLhQt956q5YuXaqoqKg6vRAA/E/+yRLtPnGu2sciI/z3YGIA3uFxyOnZs6f2799/1ccdDofGjRuncePGXbXNTTfdpJdeeqnGn9OxY0e9/fbbNbb51a9+pV/96lc1DxgAAFyXuHcVAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGCmwsQcAAFdyOh1yOh1XfTwggL/PANSOkAPApzidDoXcdKMCCTIArhEhB4BPcTodCgxwalzWTuWfLKm2zS+jI/Riv44NPDIA/oaQA8An5Z8s0e4T56p9LDKiWQOPBoA/Yj0YAAAYiZADAACMRMgBAABG4pgcANe92k5Jr6iwVFFhNdBoAHgLIQfAdSui+Q0qr7DUsmXTGtuVlVfo7JkLBB3AzxByAFy3WjYNVIDTUePp6h1aN9fiofFyOh2EHMDPEHIAXPdqOl0dgP/iwGMAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEbyesh5+eWXFR0d7fJ1//3324//8MMPSk1NVc+ePRUfH6/nn39ehYWFLn2cOHFCI0eOVGxsrBISEjR//nyVlZW5tPn888+VnJysLl26qE+fPlq/fr23SwEAAH6sXk4hv/3225WRkWF/HxAQYP977ty52rp1qxYtWqQWLVpo9uzZGjNmjLKysiRJ5eXlGjVqlMLDw5WVlaWTJ09q0qRJCgoK0gsvvCBJOnbsmEaNGqWhQ4cqPT1dOTk5mjp1qiIiIpSUlFQfJQEAAD9TLyEnICBAERERVbYXFxdr3bp1Sk9PV0JCgqQfQ0///v2Vm5uruLg4ffbZZ8rPz1dGRobCw8PVqVMnjRs3Tunp6RozZoyCg4OVlZWldu3aafLkyZKkyMhI7dixQ2+88QYhBwAASKqnY3KOHDmixMRE3XvvvZowYYJOnDghScrLy1Npaal69eplt42MjFTbtm2Vm5srScrNzVVUVJTCw8PtNomJiSopKVF+fr7dpjIk/bRNZR8AAABeX8np2rWr0tLSdNttt6mgoEBLly7VsGHD9MEHH6iwsFBBQUFq2bKly3PCwsJUUFAgSSosLHQJOJLs72trU1JSokuXLqlJkyYejdnh8Ki52/15u19fYnqN1Ifq+NLrZfocml6fZH6N9Vmfu316PeT07t3b/nfHjh0VGxuru+++W5s3b/Y4fDSUsLAWftWvLzG9RupDpdDQZo09hGqZPoem1yeZX2Nj1lfv965q2bKlbr31Vh09elS9evVSaWmpzp0757KaU1RUZB/DEx4erl27drn0UXn21U/bXHlGVmFhoZo3b16nIFVUVCzLi/fdczh+nFRv9+tLTK+R+hpPQIDTJwPF6dPnVV5e0djDsPnyHHqD6fVJ5tdYn/VV9l2beg8558+f17FjxxQREaEuXbooKChIOTk56tevnyTp8OHDOnHihOLi4iRJcXFxWrZsmYqKihQWFiZJys7OVvPmzdWhQwe7zaeffuryc7Kzs+0+PGVZqpdfsPrq15eYXiP14ad88bUyfQ5Nr08yv8bGrM/rBx7Pnz9fX3zxhY4fP66vv/5aY8aMkdPp1MCBA9WiRQsNHjxY8+bN0/bt25WXl6cpU6YoPj7eDiiJiYnq0KGDJk6cqH379mnbtm1atGiRhg0bpuDgYEnS0KFDdezYMS1YsECHDh3SmjVrtHnzZo0YMcLb5QAAAD/l9ZWc77//Xi+88ILOnDmjVq1aqXv37lq7dq1atWolSZoyZYqcTqfGjh2ry5cvKzExUTNmzLCfHxAQoGXLlmnmzJkaMmSImjZtquTkZI0dO9Zu0759ey1fvlxpaWnKzMxUmzZtNGfOHE4fBwAANq+HnD/+8Y81Pn7DDTdoxowZLsHmSv/yL/+iFStW1NhPz5499d5779VliAAakdPpkNN59VMjAgK42wwA76j3Y3IAoJLT6VDITTcqkCADoAEQcgA0GKfTocAAp8Zl7VT+yZJq2/wyOkIv9uvYwCMDYCJCDoAGl3+yRLtPnKv2scgI3zt9HIB/Ys0YAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBInEIOAG6o7UrMFRWWKioMvssi4IcIOQBQg4jmN6i8wlLLlk1rbFdWXqGzZy4QdAAfQsgBgBq0bBqoAKejxqs0d2jdXIuHxsvpdBByAB9CyAEAN9R0lWYAvomQA8BruMM4AF9CyAHgFdxhHICvIeQA8AruMA7A1xByAHgVdxgH4CtYVwYAAEYi5AAAACMRcgAAgJE4JgcAvIRbPwC+hZADANeIWz8AvomQAwDXiFs/AL6JkAMAXsKtHwDfwoHHAADASKzkAHAL96UC4G8IOQBqxX2pAPgjQg6AWnFfKgD+iJADwG3clwqAP2HtGQAAGImVHABoQFwVGWg4hBwAaABcFRloeIQcAGgAXBUZaHiEHOA6V9v1bySugeNNXBUZaDiEHOA65nQ61DKE698AMBMhB7iOORy1X/9G4ho4APwTIQdArbtQuAYOAH/EGjUAADASKzmAwbipJoDrGSEHMJQ7N9Ws7ZotaBzuXDDQsjjFHKgNIQcwFDfV9D+eXDDw3NkLDTQqwH8RcgDDcVNN/+HJBQMdjpqvbQSAkAMAPocLBgLewVGHAADASKzkAH6KM6eub5Xze7V55m7mACEH8EvunDkFM115cHJoaPXHVXE3c4CQA/glzpy6fnE3c8B9hBzAB7m7K4ozp65fHJwM1I6QA/gYdkXBW9y5qCArPTAZIQdoYO6s0rArCtfCk4sKctwOTEbIARqQJ6s07IpCXXly3E5QUIDKyyuu2herPfBnhBzAi1ilgS+pKSiz2oPrASEH8BJWaeBPOEsL1wNCDqCrr8B4ckE9Vmngj9w5S4sDmOGvCDkwXm27kBwOh5q3aFLtCsxPL7RWXmEpoIZ+KrFKA1OwSwv+jpADv3YtAeZK7qzAsEqD6wkHMMPfEXLqEUu8V1dbOHGHtwOMOyswrNLgeuStA5hLii/Jsjz7zLvyc/R6/tyE5wg59cDpdKi8wrrqPWUq+eMS75XhpC43gXQ3nLi7e8hbAQaA59xZ7bnz1lBNG9hZN910Y419Vfeev/Jz1J2wRBBCJUJOPXA4HF5b4vUl1YWT6oKcN8MJAQbwD7W9D2v7THTnPe9uWCIIoZLfh5w1a9Zo1apVKigoUMeOHTVt2jR17dq1sYclyTtLvO4EhoZsIzVsOCHAAObwxnveW6tGdd19dqXrOSx547CD+ubXIWfTpk1KS0tTamqqYmNj9eabbyolJUUffvihwsLCGnt4NXJnideTwNCQbQgnABpTQwUhd/7w80ZYqstu/8bmyWEHTqdD5eWNEwT9OuRkZGTo4Ycf1uDBgyVJqamp2rJli9atW6eRI0c28ujc463A0JBtAMDX+dLus9rCUmhoM59btffGyn7lYRkOh0MSIccjly9f1u7duzVq1Ch7m9PpVK9evbRz506P+nI6pWtcsXTh+OfvRee2LdU0OKDaNpERzWlDG79o44tjog1tvNHmhkDnVdsE/3OFoqY2oTcGK8Dp0LIth3Ti7MVq28T8S4ge6tHe2DY1vT43BP74GjocP/4/600ON/eSOaxr3SHZSP7xj3/oX//1X5WVlaX4+Hh7+4IFC/Tll1/qv/7rvxpxdAAAoLH5345AAAAAN/htyAkNDVVAQICKiopcthcVFSk8PLyRRgUAAHyF34ac4OBgde7cWTk5Ofa2iooK5eTkuOy+AgAA1ye/PfBYkp544glNmjRJXbp0UdeuXfXmm2/q4sWLGjRoUGMPDQAANDK/Djn9+/fXqVOntGTJEhUUFKhTp05auXIlu6sAAID/nl0FAABQE789JgcAAKAmhBwAAGAkQg4AADASIQcAABiJkFMHr732moYOHarY2Fj16NHDredYlqXFixcrMTFRXbt21YgRI/Tdd9+5tDlz5owmTJigbt26qUePHpoyZYrOnz9fDxXUztOxHD9+XNHR0dV+bd682W5X3eMbN25siJJc1OW1fuyxx6qMffr06S5tTpw4oZEjRyo2NlYJCQmaP3++ysrK6rOUq/K0xjNnzmj27Nnq16+funbtql/+8peaM2eOiouLXdo11hyuWbNG99xzj2JiYvTQQw9p165dNbbfvHmz7r//fsXExOiBBx7Q1q1bXR535z3Z0Dypce3atXrkkUd055136s4779SIESOqtJ88eXKVuUpJSanvMq7Kk/rWr19fZewxMTEubXxtDj2pr7rPk+joaJebS/vS/H355Zd65plnlJiYqOjoaH388ce1Pufzzz9XcnKyunTpoj59+mj9+vVV2nj6vvaYBY8tXrzYysjIsNLS0qzu3bu79Zzly5db3bt3tz766CNr79691jPPPGPdc8891qVLl+w2KSkp1q9//WsrNzfX+vLLL60+ffpYL7zwQn2VUSNPx1JWVmadPHnS5evll1+24uLirJKSErtdVFSUtW7dOpd2P30NGkpdXutHH33Umjp1qsvYi4uL7cfLysqsgQMHWiNGjLD27NljbdmyxerZs6f10ksv1Xc51fK0xv3791tjxoyxPvnkE+vIkSNWdna21bdvX+v55593adcYc7hx40arc+fO1rvvvmsdPHjQmjp1qtWjRw+rsLCw2vY7duywOnXqZK1YscLKz8+3/vjHP1qdO3e29u/fb7dx5z3ZkDyt8YUXXrBWr15t7dmzx8rPz7cmT55sde/e3fr+++/tNpMmTbJSUlJc5urMmTMNVZILT+tbt26d1a1bN5exFxQUuLTxpTn0tL7Tp0+71HbgwAGrU6dO1rp16+w2vjR/W7ZssRYuXGj99a9/taKioqyPPvqoxvZHjx61YmNjrbS0NCs/P9966623rE6dOlmffvqp3cbT16wuCDnXYN26dW6FnIqKCusXv/iFtXLlSnvbuXPnrC5dulh/+ctfLMuyrPz8fCsqKsratWuX3Wbr1q1WdHS0y4dWQ/DWWP7t3/7N+v3vf++yzZ03R32ra32PPvqoNWfOnKs+vmXLFqtjx44uH8Rvv/221a1bN+uHH37wzuDd5K053LRpk9W5c2ertLTU3tYYc/jggw9aqamp9vfl5eVWYmKitXz58mrbjxs3zho5cqTLtoceesiaNm2aZVnuvScbmqc1XqmsrMyKj4+3NmzYYG+bNGmS9eyzz3p7qHXiaX21fb762hxe6/xlZGRY8fHx1vnz5+1tvjR/P+XOZ8CCBQusAQMGuGwbP3689eSTT9rfX+tr5g52VzWA48ePq6CgQL169bK3tWjRQrGxsdq5c6ckaefOnWrZsqXLcmyvXr3kdDq9v3xXC2+MJS8vT3v37tWDDz5Y5bHU1FT17NlTDz74oN59911ZDXyppmup74MPPlDPnj01cOBAvfTSS7p48aL9WG5urqKiolwuRpmYmKiSkhLl5+d7v5AaeOv3qaSkRM2bN1dgoOt1QxtyDi9fvqzdu3e7vH+cTqd69eplv3+ulJubq4SEBJdtiYmJys3NleTee7Ih1aXGK128eFFlZWUKCQlx2f7FF18oISFB/fr104wZM3T69Gmvjt0dda3vwoULuvvuu9W7d289++yzOnjwoP2YL82hN+Zv3bp1GjBggG688UaX7b4wf3VR23vQG6+ZO/z6isf+oqCgQJIUFhbmsj0sLEyFhYWSpMLCQrVq1crl8cDAQIWEhNjPbyjeGMu7776ryMhIdevWzWX72LFjddddd6lp06b67LPPlJqaqgsXLmj48OFeG39t6lrfwIED1bZtW7Vu3Vr79+9Xenq6vv32W73yyit2v1debbvye3+cw1OnTunVV1/VkCFDXLY39ByePn1a5eXl1b5/Dh8+XO1zqpuLn77f3HlPNqS61Hil9PR0tW7d2uU/jaSkJPXp00ft2rXTsWPHtHDhQj399NN65513FBAQ4NUaalKX+m677TbNnTtX0dHRKi4u1uuvv66hQ4dq48aNatOmjU/N4bXO365du3TgwAH9x3/8h8t2X5m/urja52FJSYkuXbqks2fPXvPvvDsIOf+Unp6uFStW1Nhm06ZNioyMbKAReZ+7NV6rS5cu6S9/+YtGjx5d5bHnnnvO/vcdd9yhixcvatWqVV75D7K+6/vpf/bR0dGKiIjQiBEjdPToUd1yyy117tcTDTWHJSUlGjVqlCIjIzVmzBiXx+pzDlE3f/rTn7Rp0yZlZmbqhhtusLcPGDDA/nflgav33XefvTrgy+Lj411uthwfH6/+/fsrKytL48ePb7yB1YN3331XUVFR6tq1q8t2f54/X0HI+acnn3xSycnJNbZp3759nfqOiIiQJBUVFal169b29qKiInXs2FHSjwn31KlTLs8rKyvT2bNn7edfK3drvNaxfPjhh7p06ZJ+85vf1No2NjZWr776qi5fvqzg4OBa29ekoeqrFBsbK0k6cuSIbrnlFoWHh1fZFVT5F6U/zWFJSYmeeuopNWvWTEuXLlVQUFCN7b05h9UJDQ1VQECAioqKXLYXFRVd9T514eHhVf6a/2l7d96TDakuNVZatWqV/vSnPykjI6PWsbdv316hoaE6cuRIg/4neS31VQoKClKnTp109OhRSb41h9dS34ULF7Rx40aNHTu21p/TWPNXF9W9BwsLC9W8eXM1adJETqfzmn8n3MExOf/UqlUrRUZG1vhV1w/wdu3aKSIiQjk5Ofa2kpISffPNN/ZfKvHx8Tp37pzy8vLsNtu3b1dFRUWVdF9X7tZ4rWNZt26d7rnnniq7S6qzd+9ehYSEeOU/x4aq76djl/7/wzYuLk4HDhxwedNmZ2erefPm6tChwzXX1xA1lpSUKCUlRUFBQXrttddcVgWuxptzWJ3g4GB17tzZ5f1TUVGhnJwcl7/0fyouLk7bt2932Zadna24uDhJ7r0nG1JdapSkFStW6NVXX9XKlSurnF5dne+//15nzpzxWuh2V13r+6ny8nIdOHDAHrsvzeG11Pfhhx/q8uXL+vWvf13rz2ms+auL2t6D3vidcIvXDmG+jvztb3+z9uzZY58ivWfPHmvPnj0up0r369fP+utf/2p/v3z5cqtHjx7Wxx9/bO3bt8969tlnqz2F/De/+Y31zTffWF999ZXVt2/fRj2FvKaxfP/991a/fv2sb775xuV53333nRUdHW1t3bq1Sp+ffPKJtXbtWmv//v3Wd999Z61Zs8aKjY21Fi9eXO/1XMnT+o4cOWK98sor1v/+7/9ax44dsz7++GPr3nvvtYYNG2Y/p/IU8ieffNLau3ev9emnn1p33XVXo55C7kmNxcXF1kMPPWQNHDjQOnLkiMtpq2VlZZZlNd4cbty40erSpYu1fv16Kz8/35o2bZrVo0cP+0y2F1980UpPT7fb79ixw7rjjjusVatWWfn5+daSJUuqPYW8tvdkQ/K0xuXLl1udO3e2PvzwQ5e5qvwcKikpsebNm2ft3LnTOnbsmJWdnW0lJydbffv2bfCz/epS38svv2xt27bNOnr0qJWXl2f99re/tWJiYqyDBw/abXxpDj2tr9K///u/W+PHj6+y3dfmr6SkxP6/LioqysrIyLD27Nlj/e1vf7Msy7LS09OtF1980W5feQr5/Pnzrfz8fGv16tXVnkJe02vmDeyuqoMlS5Zow4YN9veVu2UyMzPVs2dPSdK3337rchG1p59+WhcvXtT06dN17tw5de/eXStXrnT5Szk9PV2zZ8/W448/LqfTqb59+2rq1KkNU9QVahtLaWmpvv32W5ezi6QfV3HatGmjxMTEKn0GBgZqzZo1mjt3riTplltu0eTJk/Xwww/XbzHV8LS+oKAg5eTkKDMzUxcuXNDPfvYz9e3b1+W4o4CAAC1btkwzZ87UkCFD1LRpUyUnJ7u1DF0fPK1x9+7d+uabbyRJffr0cenrk08+Ubt27RptDvv3769Tp05pyZIlKigoUKdOnbRy5Up7Wfvvf/+7nM7/X5ju1q2b0tPTtWjRIi1cuFC33nqrli5dqqioKLuNO+/JhuRpjVlZWSotLa3y+zVmzBg9//zzCggI0IEDB/Tee++puLhYrVu31i9+8QuNGzeu3lbdauJpfefOndO0adNUUFCgkJAQde7cWVlZWS6ror40h57WJ0mHDx/Wjh079Prrr1fpz9fmLy8vz+W4u7S0NElScnKy5s2bp4KCAv3973+3H2/fvr2WL1+utLQ0ZWZmqk2bNpozZ46SkpLsNrW9Zt7gsKwGPn8XAACgAXBMDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABG+j/V0TZPaOXJ5AAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Axes(0.125,0.11;0.775x0.77)\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzg0lEQVR4nO3df1RV9Z7/8dc5/FBTRAS0jL6rRMVUEIyWI3O4XEuziaE7al0bK9MhTcurlUpdw594wbzUaGlpaHRRy5oyx6565/Y7W4ijJhGKJVY3jUY5+IOwVH7s7x8OZ+5JBY5uDpzN87EWa8Xen/PZn/0+IK8++7P3sRmGYQgAAMBi7C09AAAAgOZAyAEAAJZEyAEAAJZEyAEAAJZEyAEAAJZEyAEAAJZEyAEAAJZEyAEAAJZEyAEAAJZEyAEAAJbk7+kLdu3apTVr1qi4uFjl5eVasWKFhg0bdtG2c+fO1euvv67f//73Gj9+vGv7yZMnlZGRoQ8//FB2u1233XabnnrqKXXs2NHV5sCBA1q4cKG++OILde3aVffdd58mTpzo1v+2bdu0bNkyff/997r++us1c+ZMJSUleXpKqqj4UWZ+uIXNJoWGBpneL9xRZ++h1t5Bnb2DOntHc9a5vu/GeBxyfvrpJ0VFRWn06NGaOnXqJdu9++67+vzzz9WtW7cL9s2cOVPl5eXKzc1VdXW1Zs+erblz5+qZZ56RJFVVVSk1NVVDhgzRggUL9NVXX2n27Nnq3LmzxowZI0n67LPPNGPGDD3++OMaOnSo3nnnHT3yyCPauHGj+vTp49E5GYaa5Qe9ufqFO+rsPdTaO6izd1Bn72jJOnt8uSopKUmPPfaYhg8ffsk2R48eVUZGhrKzsxUQEOC279ChQ9q+fbsWLVqkgQMHKj4+Xunp6dqyZYuOHj0qSdq8ebOqq6uVmZmp3r17Kzk5Wffff79yc3Nd/eTl5SkxMVEPPvigIiMj9eijj6pfv35at26dp6cEAAAsyOOZnMbU1dVp1qxZSk1NVe/evS/Yv3fvXnXu3FnR0dGubQkJCbLb7SoqKtLw4cNVWFio+Ph4BQYGuto4HA7l5OTo1KlTCg4OVmFhodslsPo27733nsdjttk8fkmT+jO7X7ijzt5Drb2DOnsHdfaO5qxzU/s0PeTk5OTI399f48aNu+h+p9Oprl27ug/C31/BwcEqLy93tYmIiHBrExYW5toXHBwsp9Pp2lYvNDRUTqfT4zE35bre5WiufuGOOnsPtfYO6uwd1Nk7WrLOpoac4uJi5eXlaePGjbL5UERm4bFvos7eQ629gzp7B3X2Dp9ceNyQ3bt3q6KiQkOHDnVtq62t1dNPP628vDx98MEHCgsL0/Hjx91eV1NTo1OnTik8PFzS+VmbX87I1H9fP3tzsTYVFRUXzO40BQuPfRt19h5q7R3U2Tuos3e0ZJ1NDTm/+c1vlJCQ4LYtNTVVv/nNbzRq1ChJUlxcnCorK1VcXKwBAwZIkgoKClRXV6eYmBhJUmxsrJYuXarq6mrXwuX8/HzdcMMNCg4OdrUpKChwW5eTn5+v2NhYM08JAAD4KI/vrjp9+rRKSkpUUlIiSTpy5IhKSkpUVlamkJAQ9enTx+0rICBAYWFh6tmzpyQpMjJSiYmJmjNnjoqKirRnzx5lZGQoOTlZ3bt3lySlpKQoICBATz31lA4ePKitW7cqLy9PEyZMcI1j3Lhx2r59u15++WUdOnRIzz//vIqLi3XfffeZURcAAODjPJ7JKS4udltUnJWVJUkaOXKkFi9e3KQ+srOzlZGRoQceeMD1MMD09HTX/qCgIK1Zs0YLFy7UqFGjFBISoocfftj1jBxJGjRokLKzs7V06VI9++yzuv7667VixQqPn5EDAACsyWYYXJF0Os1feBwWFmR6v3BHnb2HWnsHdfYO6uwdzVnn+r4bw2dXAQAASyLkAAAASyLkAAAASyLkAAAASzL9Yx3wf/z8Gs6QdXWG6upY9QYAQHMg5DQDu92m2jpDISEdG2xXU1unUyd/IugAANAMCDnNwGazyc9u0/QNe1V6rOqibXp166Rl98TJbrcRcgAAaAaEnGZUeqxK+8oqW3oYAAC0SSw8BgAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAlkTIAQAAluRxyNm1a5cmT54sh8OhqKgovffee6591dXV+uMf/6iUlBTFxsbK4XAoLS1NR48edevj5MmTmjFjhgYNGqT4+HjNnj1bp0+fdmtz4MABjR07VtHR0UpKSlJOTs4FY9m2bZtuv/12RUdHKyUlRR9//LGnpwMAACzK45Dz008/KSoqSvPmzbtg35kzZ7R//35NmTJFGzdu1PLly/XNN99oypQpbu1mzpyp0tJS5ebmauXKldq9e7fmzp3r2l9VVaXU1FT16NFDGzduVFpampYvX67XX3/d1eazzz7TjBkzdNddd2nTpk269dZb9cgjj+irr77y9JQAAIAF+Xv6gqSkJCUlJV10X1BQkHJzc922zZkzR3fffbfKysrUo0cPHTp0SNu3b9ebb76p6OhoSVJ6eromTZqktLQ0de/eXZs3b1Z1dbUyMzMVGBio3r17q6SkRLm5uRozZowkKS8vT4mJiXrwwQclSY8++qjy8/O1bt06LVy40NPTAgAAFuNxyPFUVVWVbDabOnfuLEnau3evOnfu7Ao4kpSQkCC73a6ioiINHz5chYWFio+PV2BgoKuNw+FQTk6OTp06peDgYBUWFmr8+PFux3I4HG6Xz5rKZru8czOrP7OP31bU1436NT9q7R3U2Tuos3c0Z52b2mezhpyzZ88qOztbycnJ6tSpkyTJ6XSqa9eu7oPw91dwcLDKy8tdbSIiItzahIWFufYFBwfL6XS6ttULDQ2V0+n0eJyhoUEev8YsISEdW+zYVtGS719bQ629gzp7B3X2jpasc7OFnOrqak2fPl2GYWjBggXNdRhTVFT8KMMwrz9/f7u6dGlaeDlx4rRqa+vMO3gbYrOd/+Ux+/3Dhai1d1Bn76DO3tGcda7vuzHNEnKqq6v16KOPqqysTH/6059cszjS+RmZ48ePu7WvqanRqVOnFB4e7mrzyxmZ+u/rZ28u1qaiouKC2Z2mMAyZ+gZ42he/ZFfG7PcPl0atvYM6ewd19o6WrLPpz8mpDzh/+9vf9MorrygkJMRtf1xcnCorK1VcXOzaVlBQoLq6OsXExEiSYmNjtXv3blVXV7va5Ofn64YbblBwcLCrTUFBgVvf+fn5io2NNfuUAACAD/I45Jw+fVolJSUqKSmRJB05ckQlJSUqKytTdXW1pk2bpuLiYmVnZ6u2tlbl5eUqLy/XuXPnJEmRkZFKTEzUnDlzVFRUpD179igjI0PJycnq3r27JCklJUUBAQF66qmndPDgQW3dulV5eXmaMGGCaxzjxo3T9u3b9fLLL+vQoUN6/vnnVVxcrPvuu8+MugAAAB9nMwzPJpF27typcePGXbB95MiRmjp1qm699daLvi4vL0+DBw+WdP5hgBkZGfrggw9kt9t12223KT09XR07/t86lgMHDmjhwoX64osvFBISovvuu0+TJk1y63Pbtm1aunSpvv/+e11//fWaNWvWJW9vb4jTaf6anJCQjkp+brv2lVVetE3/Hp21ZVqiTpw4rZoa1uRcDptNCgsLMv39w4WotXdQZ++gzt7RnHWu77vRdp6GHCsi5Pgm/qHyHmrtHdTZO6izd7SGkMNnVwEAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEvyOOTs2rVLkydPlsPhUFRUlN577z23/YZhaNmyZXI4HIqJidH48eP17bffurU5efKkZsyYoUGDBik+Pl6zZ8/W6dOn3docOHBAY8eOVXR0tJKSkpSTk3PBWLZt26bbb79d0dHRSklJ0ccff+zp6QAAAIvyOOT89NNPioqK0rx58y66PycnR2vXrtX8+fP1xhtvqEOHDkpNTdXZs2ddbWbOnKnS0lLl5uZq5cqV2r17t+bOnevaX1VVpdTUVPXo0UMbN25UWlqali9frtdff93V5rPPPtOMGTN01113adOmTbr11lv1yCOP6KuvvvL0lAAAgAV5HHKSkpL02GOPafjw4RfsMwxDeXl5mjJlioYNG6a+fftqyZIlOnbsmGvG59ChQ9q+fbsWLVqkgQMHKj4+Xunp6dqyZYuOHj0qSdq8ebOqq6uVmZmp3r17Kzk5Wffff79yc3Ndx8rLy1NiYqIefPBBRUZG6tFHH1W/fv20bt26y60FAACwEH8zOzty5IjKy8uVkJDg2hYUFKSBAwdq7969Sk5O1t69e9W5c2dFR0e72iQkJMhut6uoqEjDhw9XYWGh4uPjFRgY6GrjcDiUk5OjU6dOKTg4WIWFhRo/frzb8R0OxwWXz5rCZvP8XM3sz+zjtxX1daN+zY9aewd19g7q7B3NWeem9mlqyCkvL5ckhYaGum0PDQ2V0+mUJDmdTnXt2tV9EP7+Cg4Odr3e6XQqIiLCrU1YWJhrX3BwsJxOp2vbxY7jidDQII9fY5aQkI4tdmyraMn3r62h1t5Bnb2DOntHS9bZ1JDjqyoqfpRhmNefv79dXbo0LbycOHFatbV15h28DbHZzv/ymP3+4ULU2juos3dQZ+9ozjrX990YU0NOeHi4JKmiokLdunVzba+oqFDfvn0lnZ+ROX78uNvrampqdOrUKdfrw8LCLpiRqf++fvbmYm0qKioumN1pCsOQqW+Ap33xS3ZlzH7/cGnU2juos3dQZ+9oyTqb+pyciIgIhYeHa8eOHa5tVVVV+vzzzxUXFydJiouLU2VlpYqLi11tCgoKVFdXp5iYGElSbGysdu/ererqaleb/Px83XDDDQoODna1KSgocDt+fn6+YmNjzTwlAADgozwOOadPn1ZJSYlKSkoknV9sXFJSorKyMtlsNo0bN04vvvii3n//fX355ZdKS0tTt27dNGzYMElSZGSkEhMTNWfOHBUVFWnPnj3KyMhQcnKyunfvLklKSUlRQECAnnrqKR08eFBbt25VXl6eJkyY4BrHuHHjtH37dr388ss6dOiQnn/+eRUXF+u+++4zoy4AAMDHeXy5qri4WOPGjXN9n5WVJUkaOXKkFi9erIkTJ+rnn3/W3LlzVVlZqZtuukmrV69Wu3btXK/Jzs5WRkaGHnjgAdntdt12221KT0937Q8KCtKaNWu0cOFCjRo1SiEhIXr44Yc1ZswYV5tBgwYpOztbS5cu1bPPPqvrr79eK1asUJ8+fS6rEAAAwFpshsEVSafT/IXHISEdlfzcdu0rq7xom/49OmvLtESdOHFaNTUsPL4cNpsUFhZk+vuHC1Fr76DO3kGdvaM561zfd2P47CoAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJhBwAAGBJpoec2tpaLV26VLfccotiYmI0bNgwrVixQoZhuNoYhqFly5bJ4XAoJiZG48eP17fffuvWz8mTJzVjxgwNGjRI8fHxmj17tk6fPu3W5sCBAxo7dqyio6OVlJSknJwcs08HAAD4KNNDTk5Ojl577TXNnTtXW7du1cyZM7V69WqtXbvWrc3atWs1f/58vfHGG+rQoYNSU1N19uxZV5uZM2eqtLRUubm5WrlypXbv3q25c+e69ldVVSk1NVU9evTQxo0blZaWpuXLl+v11183+5QAAIAPMj3k7N27V7feeqt+/etfKyIiQrfffrscDoeKiooknZ/FycvL05QpUzRs2DD17dtXS5Ys0bFjx/Tee+9Jkg4dOqTt27dr0aJFGjhwoOLj45Wenq4tW7bo6NGjkqTNmzerurpamZmZ6t27t5KTk3X//fcrNzfX7FMCAAA+yPSQExcXp4KCAn3zzTeSzl9S2rNnj371q19Jko4cOaLy8nIlJCS4XhMUFKSBAwdq7969ks4Hpc6dOys6OtrVJiEhQXa73RWWCgsLFR8fr8DAQFcbh8Ohb775RqdOnfJozDab+V8tffy28kX9qLXVvqgzdbbSV3PWuSn8Pftz3LhJkyapqqpK//RP/yQ/Pz/V1tbqscce05133ilJKi8vlySFhoa6vS40NFROp1OS5HQ61bVrV/eB+vsrODjY9Xqn06mIiAi3NmFhYa59wcHBTR5zaGiQB2dorpCQji12bKtoyfevraHW3kGdvYM6e0dL1tn0kLNt2za98847euaZZ9SrVy+VlJQoKytL3bp108iRI80+nCkqKn7U362LvmL+/nZ16dK08HLixGnV1taZd/A2xGY7/8tj9vuHC1Fr76DO3kGdvaM561zfd2NMDzlLlizRpEmTlJycLEmKiopSWVmZVq1apZEjRyo8PFySVFFRoW7durleV1FRob59+0o6PyNz/Phxt35ramp06tQp1+vDwsJcMz/16r+vn9FpKsOQqW+Ap33xS3ZlzH7/cGnU2juos3dQZ+9oyTqbvibnzJkzsv3iYpmfn5/rFvKIiAiFh4drx44drv1VVVX6/PPPFRcXJ+n8up7KykoVFxe72hQUFKiurk4xMTGSpNjYWO3evVvV1dWuNvn5+brhhhs8ulQFAACsyfSQM3ToUK1cuVIfffSRjhw5onfffVe5ubkaNmyYJMlms2ncuHF68cUX9f777+vLL79UWlqaunXr5moTGRmpxMREzZkzR0VFRdqzZ48yMjKUnJys7t27S5JSUlIUEBCgp556SgcPHtTWrVuVl5enCRMmmH1KAADAB5l+uSo9PV3Lli3TggULXJekxowZo0ceecTVZuLEifr55581d+5cVVZW6qabbtLq1avVrl07V5vs7GxlZGTogQcekN1u12233ab09HTX/qCgIK1Zs0YLFy7UqFGjFBISoocfflhjxowx+5QAAIAPshkGVySdTvMXHoeEdFTyc9u1r6zyom369+isLdMSdeLEadXUsPD4cthsUlhYkOnvHy5Erb2DOnsHdfaO5qxzfd+N4bOrAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJRFyAACAJfm39AAAAIDvsdttstttLT2MBhFyAACAR+x2m4K7XCV/v4YvCNXWGbLbbaqtNbw0MneEHAAA4BG73SZ/P7umb9ir0mNVF23Tq1snLbsnTjabTRIhBwAA+JDSY1XaV1bZ0sO4JBYeAwAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAAS2qWkHP06FHNnDlTgwcPVkxMjFJSUvTFF1+49huGoWXLlsnhcCgmJkbjx4/Xt99+69bHyZMnNWPGDA0aNEjx8fGaPXu2Tp8+7dbmwIEDGjt2rKKjo5WUlKScnJzmOB0AAOCDTA85p06d0r/+678qICBAOTk52rJli5544gkFBwe72uTk5Gjt2rWaP3++3njjDXXo0EGpqak6e/asq83MmTNVWlqq3NxcrVy5Urt379bcuXNd+6uqqpSamqoePXpo48aNSktL0/Lly/X666+bfUoAAMAH+ZvdYU5Ojq6++mplZWW5tl133XWu/zYMQ3l5eZoyZYqGDRsmSVqyZIkSEhL03nvvKTk5WYcOHdL27dv15ptvKjo6WpKUnp6uSZMmKS0tTd27d9fmzZtVXV2tzMxMBQYGqnfv3iopKVFubq7GjBlj9mkBAAAfY3rI+eCDD+RwODRt2jTt2rVL3bt319ixY/Xb3/5WknTkyBGVl5crISHB9ZqgoCANHDhQe/fuVXJysvbu3avOnTu7Ao4kJSQkyG63q6ioSMOHD1dhYaHi4+MVGBjoauNwOJSTk6NTp065zRw1xmYz4cSvoD+zj99W1NeN+jU/au0d1Nk7qLN32Wwt93fW9JBz+PBhvfbaa5owYYImT56sL774QosWLVJAQIBGjhyp8vJySVJoaKjb60JDQ+V0OiVJTqdTXbt2dR+ov7+Cg4Ndr3c6nYqIiHBrExYW5trnScgJDQ3y7CRNFBLSscWObRUt+f61NdTaO6izd1Bn7+jSpeX+zpkecgzD0IABA/T4449Lkvr166eDBw9qw4YNGjlypNmHM0VFxY8yDPP68/e3N/lNPXHitGpr68w7eBtis53/R8rs9w8XotbeQZ29gzpfOT8/e5P/J/3kydOqqTH371z9e9gY00NOeHi4IiMj3bb17NlT//Vf/+XaL0kVFRXq1q2bq01FRYX69u0r6fyMzPHjx936qKmp0alTp1yvDwsLc8381Kv/vn5Gp6kMQ6b+oHvaF79kV8bs9w+XRq29gzp7B3X2jpass+l3Vw0aNEjffPON27Zvv/1W1157rSQpIiJC4eHh2rFjh2t/VVWVPv/8c8XFxUmS4uLiVFlZqeLiYlebgoIC1dXVKSYmRpIUGxur3bt3q7q62tUmPz9fN9xwg0eXqgAAgDWZHnIeeOABff7551q5cqX+9re/6Z133tEbb7yhsWPHSpJsNpvGjRunF198Ue+//76+/PJLpaWlqVu3bq67rSIjI5WYmKg5c+aoqKhIe/bsUUZGhpKTk9W9e3dJUkpKigICAvTUU0/p4MGD2rp1q/Ly8jRhwgSzTwkAAPgg0y9XxcTEaPny5Xr22We1YsUKRUREaPbs2brzzjtdbSZOnKiff/5Zc+fOVWVlpW666SatXr1a7dq1c7XJzs5WRkaGHnjgAdntdt12221KT0937Q8KCtKaNWu0cOFCjRo1SiEhIXr44Ye5fRwAAEhqhpAjSUOHDtXQoUMvud9ms2n69OmaPn36Jdt06dJFzzzzTIPH6du3r1599dXLHicAALAuPrsKAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYUrOHnJdeeklRUVH6wx/+4Np29uxZLViwQIMHD1ZcXJx+97vfyel0ur2urKxMkyZN0sCBAzVkyBA9/fTTqqmpcWuzc+dOjRw5UgMGDNDw4cO1cePG5j4dAADgI5o15BQVFWnDhg2Kiopy256ZmakPP/xQS5cu1dq1a3Xs2DFNnTrVtb+2tlYPPfSQqqurtWHDBi1evFhvv/22nnvuOVebw4cP66GHHtLgwYP1n//5n3rggQeUnp6u7du3N+cpAQAAH9FsIef06dOaNWuWFi1apODgYNf2H3/8UW+99ZaefPJJDRkyRAMGDFBmZqb27t2rwsJCSdKnn36q0tJS/fGPf9SNN96opKQkTZ8+XevXr9e5c+ckSRs2bFBERISefPJJRUZG6r777tOIESP0yiuvNNcpAQAAH9JsIWfhwoVKSkpSQkKC2/bi4mJVV1e7bY+MjFSPHj1cIaewsFB9+vRRWFiYq43D4VBVVZVKS0tdbYYMGeLWt8PhcPXhCZvN/K+WPn5b+aJ+1NpqX9SZOvvCV2v4G9cU/p4NtWm2bNmi/fv3680337xgn9PpVEBAgDp37uy2PTQ0VOXl5a42fx9wJLm+b6xNVVWVzpw5o/bt2zd5vKGhQU1ua7aQkI4tdmyraMn3r62h1t5Bnb2DOntHly4t93fO9JDzww8/6A9/+INefvlltWvXzuzum0VFxY8yDPP68/e3N/lNPXHitGpr68w7eBtis53/R8rs9w8XotbeQZ29gzpfOT8/e5P/J/3kydOqqTH371z9e9gY00POvn37VFFRoVGjRrm21dbWateuXVq/fr3WrFmj6upqVVZWus3mVFRUKDw8XNL5GZmioiK3fuvvvvr7Nr+8I8vpdKpTp04ezeJIkmHI1B90T/vil+zKmP3+4dKotXdQZ++gzt7RknU2PeT8wz/8g9555x23bb///e/Vs2dPTZw4Uddcc40CAgK0Y8cOjRgxQpL09ddfq6ysTLGxsZKk2NhYrVy5UhUVFQoNDZUk5efnq1OnTurVq5erzSeffOJ2nPz8fFcfAACgbTM95HTq1El9+vRx23bVVVepS5curu2jR4/W4sWLFRwcrE6dOmnRokWKi4tzBRSHw6FevXopLS1Ns2bNUnl5uZYuXap7771XgYGBkqR77rlH69ev15IlSzR69GgVFBRo27ZtWrVqldmnBAAAfFCzLDxuzOzZs2W32zVt2jSdO3dODodD8+bNc+338/PTypUrNX/+fI0ZM0YdOnTQyJEjNW3aNFeb6667TqtWrVJWVpby8vJ09dVXa9GiRUpMTGyJU2pRdrtNdnvDS83r6gzV1TEvCwBoO7wSctauXev2fbt27TRv3jy3YPNL1157rXJychrsd/Dgwdq0aZMZQ2y1GgswNptNnYLay9+v4acB1NTW6dTJnwg6AIA2o0VmctA0drtNwV2uajTASNL0DXtVeqzqovt6deukZffEyW63EXIAAG0GIacVs9tt8vezNxhgfh0Vrlkj+qr0WJX2lVV6eYQAALRehBwf0FCAiQznYYIAAFwMIacN8WvksheLkwEAVkLIaQPCO7VTbZ2hzp07NNiOxckAACsh5LQBnTv4y89uY3EyAKBNIeS0ISxOBgC0JYQcuGHdDgDAKgg5kMS6HQCA9RByIIl1OwAA6yHkwA3rdgAAVtH45wUAAAD4IEIOAACwJC5XwWPcgQUA8AWEHDQZd2ABAHwJIQdNxh1YAABfQsiBx5pyBxaXtAAALY2QA1NxSQsA0FoQcmAqTy5pBQT4qba27pJ9MdsDALgShBw0i4YuaTHbAwDwBkIOvM7M2R4AAC6FkIMWY8ZsT22dIbvdptpaZnsAAO4IOWiVPJntsdlskgg5AAB3hJwW1tCt1o3dht0W8IGhAIDLRchpIU29HAMAAC4PIaeFNOVyzK+jwjVrRF8vj8z38OBBAMDFEHJaWEOXYyLDO3p5NL6FW9EBAA0h5MBn8VlaAICGEHLg81icDAC4GG7fAQAAlkTIAQAAlsTlKrQJ3IEFAG0PIQeWxh1YANB2EXJgadyBBQBtFyEHbQJ3YAFA20PIAf4X63YAwFoIOWjzWLcDANZEyEGbx7odALAmQg7wv1i3AwDWQsgBPMC6HQDwHYQcoAlYtwMAvsf0kLNq1Sr99a9/1ddff6327dsrLi5OM2fOVM+ePV1tzp49q8WLF2vr1q06d+6cHA6H5s2bp7CwMFebsrIyzZ8/Xzt37tRVV12lf/mXf9GMGTPk7/9/Q965c6cWL16sgwcP6pprrtGUKVM0atQos08JYN0OAPgg00POf//3f+vee+9VdHS0amtr9eyzzyo1NVVbtmzRVVddJUnKzMzUxx9/rKVLlyooKEgZGRmaOnWqNmzYIEmqra3VQw89pLCwMG3YsEHHjh3TE088oYCAAD3++OOSpMOHD+uhhx7SPffco+zsbO3YsUPp6ekKDw9XYmKi2acFSGrauh0uaQFA62B6yFmzZo3b94sXL9aQIUO0b98+3Xzzzfrxxx/11ltvKTs7W0OGDJF0PvTccccdKiwsVGxsrD799FOVlpYqNzdXYWFhuvHGGzV9+nRlZ2dr6tSpCgwM1IYNGxQREaEnn3xSkhQZGak9e/bolVdeIeSgRXBJCwBal2Zfk/Pjjz9KkoKDgyVJxcXFqq6uVkJCgqtNZGSkevTo4Qo5hYWF6tOnj9vlK4fDofnz56u0tFT9+vVTYWGhKyT9fZvMzEyPx2izXc6Zea8/+AZPL2kZhu+EnPqfaX62mxd19g7q7F02W8v9nW3WkFNXV6fMzEwNGjRIffr0kSQ5nU4FBASoc+fObm1DQ0NVXl7uavP3AUeS6/vG2lRVVenMmTNq3759k8cZGhrk2YkBDWjKJa2QkI5eGo25+F3xDursHdTZO7p0abl/75o15CxYsEAHDx7Uq6++2pyHuWIVFT/KzP+p9ve3t+ibitavsvJn1dbWXXK/YbSudTs22/k/CGb/rsAddfYO6nzl/PzsTf6ftZMnT6um5tL/3l2O+vewMc0WchYuXKiPPvpI69at09VXX+3aHhYWpurqalVWVrrN5lRUVCg8PNzVpqioyK0/p9MpSW5t6rf9fZtOnTp5NIsjSYYhU3/Q+aXBpfj6uh2zf1dwcdTZO6izd7RknU0POYZhKCMjQ++++67Wrl2r6667zm3/gAEDFBAQoB07dmjEiBGSpK+//lplZWWKjY2VJMXGxmrlypWqqKhQaGioJCk/P1+dOnVSr169XG0++eQTt77z8/NdfQCtkSfrdgIC/Bqc7WkK7uQC0JaZHnIWLFigP//5z3rhhRfUsWNH1xqaoKAgtW/fXkFBQRo9erQWL16s4OBgderUSYsWLVJcXJwroDgcDvXq1UtpaWmaNWuWysvLtXTpUt17770KDAyUJN1zzz1av369lixZotGjR6ugoEDbtm3TqlWrzD4lwHQNrdtp6mxPbZ0hP3vDq+9a64wQAHiD6SHntddekyTdf//9btuzsrJcD+qbPXu27Ha7pk2b5vYwwHp+fn5auXKl5s+frzFjxqhDhw4aOXKkpk2b5mpz3XXXadWqVcrKylJeXp6uvvpqLVq0iNvH4fOaMtvz66hwzRrRl4cTAkADTA85X375ZaNt2rVrp3nz5rkFm1+69tprlZOT02A/gwcP1qZNmzwdIuATGprtiQzv2GgbAGjrGn40KwAAgI8i5AAAAEsi5AAAAEtq9o91ANCy+MBQAG0VIQewKF9/8CAAXClCDmBR3n7wIAC0NoQcwOLMePCgdP7hg3a7TbW1zPgA8A2EHKANa8psj/R/Mz7+/n6y2S4948P6HgCtCSEHQKMPFWR9DwBfRMgB0ChP1vfwMRIAWgtCDoAma8rHSHDLOoDWgpADwBSeXNKq+vGMDOPSQYcgBMAMhBwApmjKJa2brw/RnH/ury5drmqwL9b2ADADIQeAqRr79HTW9gDwFkIOAK9jbQ8AbyDkAGhVuF0dgFkIOQBaFW9/HAUzQoB1EXIAtEpmfBxFbZ0hP7utwTbMCAHWRcgB4HOaMtvz66hwzRrRl0XOQBtGyAHgsxq7k6uxNvVY5AxYEyEHQJvFImfA2gg5ANosMxc5M9sDtD6EHABtnhmLnJntAVofQg4ANMDbt7QDMA8hBwCawFu3tNfWGQoI8GtwRohLY0DTEHIA4AqZdUt7/QeYBgfzAaaAGQg5AGCSK72lnQ8wBcxFyAGAVoZn+wDmIOQAgA/x5G6vqh/PyDCaP+gQqNBaEXIAwIc0Zf1P/dqeLl0aXtvT1IXQfP4XfBUhBwB80JWu7WnKQmhPPv+LhyWiNSLkAIBFmbEQurE2Zl4+IwjBbIQcAMBlM/PyGZe9Wg+73SZ7A5cpG1v43loQcgAAV8ysW+PNeGp0U2eEuEPt4ux2m4K7XCV/HwkyDSHkAAC8wltPjW7s0pjNZlNtnaGQkI5X1I+ZmhKoGptdMetYdrtN/n72Jq3Xau0IOQCAFmf2U6MbuzQmqdXdodZYMOsU1L7R2RUzjlU/w9WU9VqtHSEHANBqeOOp0fVhqbXcoWZWMDP7WFZAyAEAWI5ZsxDeuEPNzGBm5rGsgJADAEAr4M3LQ1a4FNUUvr90GgAA4CIIOQAAwJJ8PuSsX79et9xyi6Kjo3X33XerqKiopYcEAABaAZ8OOVu3blVWVpYeeeQRvf322+rbt69SU1NVUVHR0kMDAAAtzKdDTm5urn77299q9OjR6tWrlxYsWKD27dvrrbfeaumhAQCAFuazd1edO3dO+/bt00MPPeTaZrfblZCQoL1793rUl90umflAS9v/Poepf4/O6hDod9E2keGdaEMbn2jTGsdEG9rQpvW36Rl2/i4tm+3831kz2Zr44Geb4Y3nVTeDo0eP6le/+pU2bNiguLg41/YlS5Zo165d+o//+I8WHB0AAGhpPn25CgAA4FJ8NuSEhITIz8/vgkXGFRUVCgsLa6FRAQCA1sJnQ05gYKD69++vHTt2uLbV1dVpx44dbpevAABA2+SzC48lacKECXriiSc0YMAAxcTE6E9/+pN+/vlnjRo1qqWHBgAAWphPh5w77rhDx48f13PPPafy8nLdeOONWr16NZerAACA795dBQAA0BCfXZMDAADQEEIOAACwJEIOAACwJEIOAACwJELOZVq/fr1uueUWRUdH6+6771ZRUVGD7bdt26bbb79d0dHRSklJ0ccff+ylkfo2T+r8xhtvaOzYsbr55pt18803a/z48Y2+LzjP05/nelu2bFFUVJQefvjhZh6hdXha68rKSi1YsEAOh0MDBgzQiBEj+PejCTyt8yuvvKIRI0YoJiZGSUlJyszM1NmzZ700Wt+0a9cuTZ48WQ6HQ1FRUXrvvfcafc3OnTs1cuRIDRgwQMOHD9fGjRubd5AGPLZlyxajf//+xptvvmkcPHjQSE9PN+Lj4w2n03nR9nv27DFuvPFGIycnxygtLTX+/d//3ejfv7/x5ZdfennkvsXTOj/++OPGunXrjP379xulpaXGk08+adx0003G//zP/3h55L7F0zrXO3z4sJGYmGiMHTvWmDJlipdG69s8rfXZs2eNUaNGGRMnTjR2795tHD582Ni5c6dRUlLi5ZH7Fk/rvHnzZmPAgAHG5s2bjcOHDxvbt283/vEf/9HIzMz08sh9y0cffWQ8++yzxl//+lejT58+xrvvvttg+++++84YOHCgkZWVZZSWlhpr1641brzxRuOTTz5ptjESci7DXXfdZSxYsMD1fW1treFwOIxVq1ZdtP306dONSZMmuW27++67jTlz5jTrOH2dp3X+pZqaGiMuLs54++23m2mE1nA5da6pqTHGjBljvPHGG8YTTzxByGkiT2v96quvGrfeeqtx7tw5bw3REjyt84IFC4xx48a5bcvKyjLuueeeZh2nlTQl5CxZssRITk522/boo48a//Zv/9Zs4+JylYfOnTunffv2KSEhwbXNbrcrISFBe/fuvehrCgsLNWTIELdtDodDhYWFzTlUn3Y5df6ln3/+WTU1NQoODm6uYfq8y63zihUrFBoaqrvvvtsbw7SEy6n1Bx98oNjYWC1cuFAJCQn653/+Z61cuVK1tbXeGrbPuZw6x8XFad++fa5LWocPH9bHH3+spKQkr4y5rWiJv4U+/cTjlnDixAnV1tYqNDTUbXtoaKi+/vrri77G6XRe8BTm0NBQOZ3OZhunr7ucOv9Sdna2unXr5vaPHdxdTp13796tN998U5s2bfLCCK3jcmp9+PBhFRQUKCUlRS+99JK+++47LViwQDU1NZo6dao3hu1zLqfOKSkpOnHihMaOHSvDMFRTU6N77rlHkydP9saQ24yL/S0MCwtTVVWVzpw5o/bt25t+TGZyYEkvvfSStm7dquXLl6tdu3YtPRzLqKqqUlpamjIyMtS1a9eWHo7lGYah0NBQZWRkaMCAAbrjjjs0efJkbdiwoaWHZik7d+7UqlWrNG/ePG3cuFHLly/Xxx9/rBUrVrT00HCFmMnxUEhIiPz8/FRRUeG2vaKi4pKfmRUWFnbBrE1D7XF5da63Zs0avfTSS8rNzVXfvn2bc5g+z9M6Hz58WN9//72mTJni2lZXVydJ6tevn/7yl7/o//2//9e8g/ZRl/MzHR4eLn9/f/n5+bm29ezZU+Xl5Tp37pwCAwObdcy+6HLqvGzZMt15552uy69RUVH66aefNHfuXE2ZMkV2O/MBZrjY30Kn06lOnTo1yyyOxEyOxwIDA9W/f3/t2LHDta2urk47duxQXFzcRV8TGxurgoICt235+fmKjY1tzqH6tMupsyTl5OTohRde0OrVqxUdHe2Nofo0T+vcs2dPvfPOO9q0aZPr65ZbbtHgwYO1adMmXX311d4cvk+5nJ/pQYMG6bvvvnMFSUn69ttvFR4eTsC5hMup85kzZy4IMvXB0uDjHU3TIn8Lm21Js4Vt2bLFGDBggLFx40ajtLTUmDNnjhEfH2+Ul5cbhmEYs2bNMrKzs13t9+zZY/Tr189Ys2aNUVpaajz33HPcQt4EntZ51apVRv/+/Y2//OUvxrFjx1xfVVVVLXUKPsHTOv8Sd1c1nae1LisrM+Li4oyFCxcaX3/9tfHhhx8aQ4YMMV544YWWOgWf4Gmdn3vuOSMuLs7485//bHz33XfGp59+agwbNsyYPn16C52Bb6iqqjL2799v7N+/3+jTp4+Rm5tr7N+/3/j+++8NwzCM7OxsY9asWa729beQP/3000Zpaamxbt26Zr+FnMtVl+GOO+7Q8ePH9dxzz6m8vFw33nijVq9e7ZoK/eGHH9z+r2DQoEHKzs7W0qVL9eyzz+r666/XihUr1KdPn5Y6BZ/gaZ03bNig6upqTZs2za2fqVOn6ne/+51Xx+5LPK0zLp+ntb7mmmu0Zs0aZWVl6c4771T37t01btw4TZw4saVOwSd4WucpU6bIZrNp6dKlOnr0qLp27aqhQ4fqsccea6lT8AnFxcUaN26c6/usrCxJ0siRI7V48WKVl5frhx9+cO2/7rrrtGrVKmVlZSkvL09XX321Fi1apMTExGYbo80wmIsDAADWw/+eAQAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAAS/r/1lNlgc+tCFMAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "df[df[\"preference\"]][\"cer_diff_preference\"].hist(bins=50)\n", "print(df[df[\"preference\"]][\"cer_diff_preference\"].quantile(0.95))\n", "plt.show()\n", "print(df[df[\"preference\"]][\"cer\"].hist(bins=50))\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:40.799375Z", "start_time": "2024-05-16T13:59:36.394236Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:18:39.797087Z", "iopub.status.busy": "2024-09-17T00:18:39.796924Z", "iopub.status.idle": "2024-09-17T00:18:41.566600Z", "shell.execute_reply": "2024-09-17T00:18:41.566042Z", "shell.execute_reply.started": "2024-09-17T00:18:39.797072Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "16429\n", "good_continue_at\n", "True 180424\n", "False 840\n", "Name: count, dtype: int64\n", "\n", " Check some basics... \n", " preference\n", "False 90632\n", "True 90632\n", "Name: count, dtype: int64 is_30b\n", "True 181264\n", "Name: count, dtype: int64 model_name\n", "chirp-v3p5-engine-t-3 181264\n", "Name: count, dtype: int64 preference model_name \n", "False chirp-v3p5-engine-t-3 90632\n", "True chirp-v3p5-engine-t-3 90632\n", "Name: count, dtype: int64\n", "task\n", " 141798\n", "extend 39216\n", "cover 232\n", "infill 12\n", "artist_consistency 6\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df[\"id\"] = df[\"str_id\"]\n", "# get the original duration of the clips, if they are concacted\n", "df[\"original_duration_s\"] = df[\"total_start_s\"] + df[\"duration\"]\n", "# classify the continue at behavoirs by the duration choice\n", "audio_prompt_id_to_continue_at = {}\n", "for _, row in df[~df[\"continued_parent\"].isna()].iterrows():\n", " audio_prompt_id = row[\"continued_parent\"]\n", " if audio_prompt_id not in audio_prompt_id_to_continue_at:\n", " audio_prompt_id_to_continue_at[audio_prompt_id] = row[\"continue_at\"]\n", " else:\n", " # pick the max\n", " audio_prompt_id = max(\n", " audio_prompt_id_to_continue_at[audio_prompt_id], row[\"continue_at\"]\n", " )\n", "print(len(audio_prompt_id_to_continue_at))\n", "df[\"has_continue_and_start_continue_at\"] = df[\"id\"].apply(\n", " lambda x: audio_prompt_id_to_continue_at.get(x)\n", ")\n", "# we want continue at to be at most of the clip...\n", "df[\"good_continue_at\"] = (\n", " (df[\"has_continue_and_start_continue_at\"] / df[\"duration\"]) > 0.9\n", ") | df[\"has_continue_and_start_continue_at\"].isna()\n", "print(df[\"good_continue_at\"].value_counts())\n", "\n", "\n", "print(\n", " \"\\n Check some basics... \\n\",\n", " df[\"preference\"].value_counts(),\n", " df[\"is_30b\"].value_counts(),\n", " df[\"model_name\"].value_counts(),\n", " df.groupby([\"preference\"])[\"model_name\"].value_counts(),\n", ")\n", "\n", "df = df.sort_values(by=[\"request_id\", \"preference\"])\n", "df[\"duration_rel_diff\"] = df[\"duration\"].diff()\n", "df[\"play_rel_diff\"] = df[\"reaction_play_count\"].diff()\n", "print(df[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.035167Z", "start_time": "2024-05-16T13:59:40.801098Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:20:31.915227Z", "iopub.status.busy": "2024-09-17T00:20:31.914685Z", "iopub.status.idle": "2024-09-17T00:20:32.213875Z", "shell.execute_reply": "2024-09-17T00:20:32.213314Z", "shell.execute_reply.started": "2024-09-17T00:20:31.915209Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "negative 89703 positive 13987\n", "total pair requests 90632 --> selected pair requests 13537 frac 0.149\n" ] } ], "source": [ "normal_pos_play_count = 3\n", "# this is lower, cause a concat is probably already ensuring that it is good\n", "concat_pos_play_count = 1\n", "# this is a filter on the concated clip\n", "concat_total_play_count = 3\n", "\n", "neg_filter_selection_mask = (\n", " (~df[\"preference\"]) # get basics aligned\n", " & (df[\"reaction_play_count\"] >= 1) # has to be played once\n", " # & (df[\"play_count\"] <= 3) # if it is actually bad, shouldn't be listened often\n", " & (df[\"duration\"] >= 10) # can't be too short, otherwise it is obvious\n", " & (df[\"duration\"] <= 240) # can't be badly long\n", " & (df[\"has_continue_and_start_continue_at\"].isna()) # won't have any continues\n", " & (df[\"upvote_count\"] == 0) # won't have any likes\n", " & ((df[\"norm_play_frac\"] <= 3.1))\n", " # & (df[\"dislike_count\"] >= 1) # this is kinda strict\n", " # & (\n", " # (df_slice[\"is_in_playlist\"] == False)\n", " # & (df_slice[\"concat_in_playlist\"] == False)\n", " # ) # can't be part of a playlist -- otherwise there are some like signal in it?\n", ")\n", "pos_filter_selectin_mask = (\n", " (df[\"preference\"]) # get basics aligned\n", " & (\n", " df[\"good_continue_at\"]\n", " ) # if continue, needs to continue off a certain percentage\n", " & (df[\"reaction_play_count\"] >= 1)\n", " & (df[\"play_rel_diff\"] >= 0) # this is more like quality assurance\n", " & (df[\"duration\"] >= 10) # can't be too short, otherwise it is obvious\n", " & (df[\"duration\"] <= 240) # can't be badly long\n", " & (df[\"dislike_count\"] == 0) # can't have dislikes\n", " & (df[\"flag_count\"] == 0) # can't have issues\n", " & (\n", " (\n", " (df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= concat_pos_play_count)\n", " & (df[\"concat_play_counts\"] >= concat_total_play_count)\n", " )\n", " | (\n", " (~df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= normal_pos_play_count)\n", " # & (df[\"norm_play_frac\"] >= 1.5)\n", " )\n", " \n", " )\n", " # & (df[\"norm_play_frac\"] >= 1.9)\n", " & (df[\"user_n_clips\"] >= 40) # user needs to have genereated at least 20\n", " # & (df[\"duration_rel_diff\"] < 10) # positive isn't just longer\n", " & ((df[\"upvote_count\"] >= 1) | (df[\"reaction_play_count\"] >= 3)| (df[\"concat_play_counts\"] >= 3))\n", " & (df[\"pos_diff_preference\"] == 2)\n", " # & ((df[\"task\"] != \"infill\") & (df[\"task\"] != \"cover\") & (df[\"task\"] != \"artist_consistency\"))\n", " & ((df[\"cer_diff_preference\"] < 0.5) & (df[\"cer\"] < 0.95)) # cut on hoot cer difference and abs cer\n", ")\n", "print(\n", " \"negative\",\n", " sum(neg_filter_selection_mask),\n", " \"positive\",\n", " sum(pos_filter_selectin_mask),\n", ")\n", "\n", "neg_filter_requests = df[neg_filter_selection_mask][\"request_id\"].unique()\n", "pos_filter_requests = df[pos_filter_selectin_mask][\"request_id\"].unique()\n", "# looking for very strong signal here:\n", "# listen to the positive/negative more than once\n", "# disliked one of the clips\n", "unique_requests = set(pos_filter_requests).intersection(neg_filter_requests)\n", "print(\n", " \"total pair requests\",\n", " df[\"request_id\"].nunique(),\n", " \" --> selected pair requests\",\n", " len(unique_requests),\n", " f\"frac {len(unique_requests) / df['request_id'].nunique():.3f}\",\n", ")" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.250737Z", "start_time": "2024-05-16T13:59:41.036434Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:20:33.481565Z", "iopub.status.busy": "2024-09-17T00:20:33.481148Z", "iopub.status.idle": "2024-09-17T00:20:33.612779Z", "shell.execute_reply": "2024-09-17T00:20:33.612226Z", "shell.execute_reply.started": "2024-09-17T00:20:33.481548Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "30b_t3_v15 requests 13537 clips 27074 total khrs 1.467; N gpus for 1000 iters 1.692; 4 gpus for x iters 423.031; n unique users 9950 n pro users 6661\n" ] } ], "source": [ "df_slice = df[df[\"request_id\"].isin(set(unique_requests))].copy()\n", "print(\n", " f\"{os.path.basename(OUT_DATA_DIR)} requests\",\n", " df_slice[\"request_id\"].nunique(),\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"4 gpus for x iters {df_slice.shape[0] / 8 / 2 / 4:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", " f\"n pro users {df_slice[df_slice['is_pro_user']]['user_id'].nunique()}\",\n", ")\n", "# 76171 152342 total khrs 2.880 n gpus for 1250 iters 3.809\n", "# v10 has 78866\n", "# v14 has 110402" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:20:34.622832Z", "iopub.status.busy": "2024-09-17T00:20:34.622493Z", "iopub.status.idle": "2024-09-17T00:20:34.624897Z", "shell.execute_reply": "2024-09-17T00:20:34.624501Z", "shell.execute_reply.started": "2024-09-17T00:20:34.622815Z" } }, "outputs": [], "source": [ "# v1 requests 17681 clips 35362 total khrs 1.815; N gpus for 1500 iters 1.473; 4 gpus for x iters 552.531; n unique users 15179 n pro users 5983" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.277006Z", "start_time": "2024-05-16T13:59:41.252105Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:20:35.062766Z", "iopub.status.busy": "2024-09-17T00:20:35.062423Z", "iopub.status.idle": "2024-09-17T00:20:35.069797Z", "shell.execute_reply": "2024-09-17T00:20:35.069336Z", "shell.execute_reply.started": "2024-09-17T00:20:35.062751Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive in playlist (2807, 115)\n" ] } ], "source": [ "test_mask = (df_slice[\"preference\"]) & (\n", " (df_slice[\"is_in_playlist\"]) | (df_slice[\"concat_in_playlist\"])\n", ")\n", "print(\"positive in playlist\", df_slice[test_mask].shape)" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.323409Z", "start_time": "2024-05-16T13:59:41.278278Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:20:35.526260Z", "iopub.status.busy": "2024-09-17T00:20:35.526115Z", "iopub.status.idle": "2024-09-17T00:20:35.528284Z", "shell.execute_reply": "2024-09-17T00:20:35.527891Z", "shell.execute_reply.started": "2024-09-17T00:20:35.526246Z" } }, "outputs": [], "source": [ "# interesting_clips_must_be_positive_mask = (\n", "# (df_slice[\"upvoted\"] == True)\n", "# | (df_slice[\"has_action\"] == True)\n", "# | (df_slice[\"part_of_concat\"] == True)\n", "# )\n", "# interesting_clips_must_be_not_negative_mask = (df_slice[\"downvoted\"] == False) # & (df_slice[\"dislike_count\"] < 1)\n", "# interesting_clips_mask = interesting_clips_must_be_positive_mask & interesting_clips_must_be_not_negative_mask\n", "# assert interesting_clips_mask.eq(df_slice[\"preference\"]).all()" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.392244Z", "start_time": "2024-05-16T13:59:41.324472Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:20:35.862462Z", "iopub.status.busy": "2024-09-17T00:20:35.862199Z", "iopub.status.idle": "2024-09-17T00:20:35.864302Z", "shell.execute_reply": "2024-09-17T00:20:35.863919Z", "shell.execute_reply.started": "2024-09-17T00:20:35.862448Z" } }, "outputs": [], "source": [ "# save positive ids\n", "# positive_preference_ids = df_slice[df_slice[\"preference\"] == False][\"s3_id\"].to_json(orient='values')\n", "# with open('/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id_negative.json', 'w') as file:\n", "# file.write(positive_preference_ids)" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:20:36.402614Z", "iopub.status.busy": "2024-09-17T00:20:36.402326Z", "iopub.status.idle": "2024-09-17T00:20:36.404476Z", "shell.execute_reply": "2024-09-17T00:20:36.404084Z", "shell.execute_reply.started": "2024-09-17T00:20:36.402599Z" } }, "outputs": [], "source": [ "# df_slice_2 = pd.read_csv(\"/home/tony/Data/Preference/30b_v1/interesting_clips_v4_t_1_20240808_slice.csv\")\n", "# df_total = pd.concat([df_slice, df_slice_2])\n", "# print(df_total.shape)" ] }, { "cell_type": "code", "execution_count": 39, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:20:36.617976Z", "iopub.status.busy": "2024-09-17T00:20:36.617736Z", "iopub.status.idle": "2024-09-17T00:20:36.619882Z", "shell.execute_reply": "2024-09-17T00:20:36.619490Z", "shell.execute_reply.started": "2024-09-17T00:20:36.617961Z" } }, "outputs": [], "source": [ "# print(df_slices_2.shape, df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:20:37.098521Z", "iopub.status.busy": "2024-09-17T00:20:37.098219Z", "iopub.status.idle": "2024-09-17T00:20:37.100289Z", "shell.execute_reply": "2024-09-17T00:20:37.099902Z", "shell.execute_reply.started": "2024-09-17T00:20:37.098506Z" } }, "outputs": [], "source": [ "# print(df_total.shape)\n", "# df_slice = df_total.copy()" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "execution": { "iopub.execute_input": "2024-09-17T00:20:37.526145Z", "iopub.status.busy": "2024-09-17T00:20:37.526011Z", "iopub.status.idle": "2024-09-17T00:20:37.531091Z", "shell.execute_reply": "2024-09-17T00:20:37.530681Z", "shell.execute_reply.started": "2024-09-17T00:20:37.526132Z" } }, "outputs": [ { "data": { "text/plain": [ "task\n", " 22770\n", "extend 4262\n", "cover 38\n", "infill 2\n", "artist_consistency 2\n", "Name: count, dtype: int64" ] }, "execution_count": 41, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_slice[\"task\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T14:00:20.866354Z", "start_time": "2024-05-16T14:00:12.443344Z" }, "execution": { "iopub.execute_input": "2024-09-17T00:21:23.767393Z", "iopub.status.busy": "2024-09-17T00:21:23.767035Z", "iopub.status.idle": "2024-09-17T00:21:24.562635Z", "shell.execute_reply": "2024-09-17T00:21:24.562021Z", "shell.execute_reply.started": "2024-09-17T00:21:23.767376Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(27074, 115)\n" ] }, { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[42], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m df_slice\u001b[38;5;241m.\u001b[39mto_pickle(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_v15_20240902_slice.pkl\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28mprint\u001b[39m(df_slice\u001b[38;5;241m.\u001b[39mshape)\n\u001b[0;32m----> 3\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "df_slice.to_pickle(\"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_v15_20240902_slice.pkl\")\n", "print(df_slice.shape)\n", "BREAK" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932296Z", "start_time": "2024-05-16T13:59:41.932287Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.677720Z", "iopub.status.idle": "2024-09-17T00:18:42.677896Z", "shell.execute_reply": "2024-09-17T00:18:42.677812Z", "shell.execute_reply.started": "2024-09-17T00:18:42.677803Z" } }, "outputs": [], "source": [ "# don't have continue at\n", "df_slice[\"request_id\"] = df_slice[\"request_id\"].astype(str)\n", "df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique(), df_slice[\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2024-09-17T00:18:42.678657Z", "iopub.status.idle": "2024-09-17T00:18:42.678826Z", "shell.execute_reply": "2024-09-17T00:18:42.678743Z", "shell.execute_reply.started": "2024-09-17T00:18:42.678735Z" } }, "outputs": [], "source": [ "print(df_slice.shape)\n", "df_slice = df_slice[df_slice[\"request_id\"].apply(lambda x: len(x) > 3)]\n", "print(df_slice.shape)\n", "# df_slice = df_slice[df_slice[\"is_pro_user\"]].copy()\n", "print(df_slice.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932966Z", "start_time": "2024-05-16T13:59:41.932957Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.679400Z", "iopub.status.idle": "2024-09-17T00:18:42.679562Z", "shell.execute_reply": "2024-09-17T00:18:42.679483Z", "shell.execute_reply.started": "2024-09-17T00:18:42.679475Z" } }, "outputs": [], "source": [ "final_filtered_requests = df_slice[\"request_id\"].astype(str).unique()\n", "# final_filtered_requests = df_slice[df_slice[\"is_pro_user\"]][\"request_id\"].astype(str).unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.933558Z", "start_time": "2024-05-16T13:59:41.933550Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.679934Z", "iopub.status.idle": "2024-09-17T00:18:42.680103Z", "shell.execute_reply": "2024-09-17T00:18:42.680024Z", "shell.execute_reply.started": "2024-09-17T00:18:42.680016Z" } }, "outputs": [], "source": [ "# df_slice.to_csv(\"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_20240902_slice.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934277Z", "start_time": "2024-05-16T13:59:41.934268Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.680717Z", "iopub.status.idle": "2024-09-17T00:18:42.680882Z", "shell.execute_reply": "2024-09-17T00:18:42.680804Z", "shell.execute_reply.started": "2024-09-17T00:18:42.680796Z" } }, "outputs": [], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df.reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df.reset_index()\n", "\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934954Z", "start_time": "2024-05-16T13:59:41.934946Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.681402Z", "iopub.status.idle": "2024-09-17T00:18:42.681557Z", "shell.execute_reply": "2024-09-17T00:18:42.681484Z", "shell.execute_reply.started": "2024-09-17T00:18:42.681477Z" } }, "outputs": [], "source": [ "# val_df[[\"request_id\", \"metadata\", \"updated_at\", \"user_id\", \"preference\"]].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.935620Z", "start_time": "2024-05-16T13:59:41.935613Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.682057Z", "iopub.status.idle": "2024-09-17T00:18:42.682207Z", "shell.execute_reply": "2024-09-17T00:18:42.682137Z", "shell.execute_reply.started": "2024-09-17T00:18:42.682129Z" } }, "outputs": [], "source": [ "total_duration = 0\n", "for i, row in tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " total_duration += row[\"duration\"]\n", " except Exception as E:\n", " print(i, row)\n", " print(E)\n", " raise ValueError()\n", " \n", "print(\n", " f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1000} nodes, {train_df.shape[0] / 8 / 2 / 6} iters\"\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936268Z", "start_time": "2024-05-16T13:59:41.936260Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.682593Z", "iopub.status.idle": "2024-09-17T00:18:42.682739Z", "shell.execute_reply": "2024-09-17T00:18:42.682672Z", "shell.execute_reply.started": "2024-09-17T00:18:42.682664Z" } }, "outputs": [], "source": [ "make_dataset(val_df, OUT_DATA_DIR, is_val=True, npz_dir=NPZ_DIR)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936964Z", "start_time": "2024-05-16T13:59:41.936957Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.683341Z", "iopub.status.idle": "2024-09-17T00:18:42.683513Z", "shell.execute_reply": "2024-09-17T00:18:42.683434Z", "shell.execute_reply.started": "2024-09-17T00:18:42.683426Z" } }, "outputs": [], "source": [ "make_dataset(train_df, OUT_DATA_DIR, is_val=False, npz_dir=NPZ_DIR)" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.937879Z", "start_time": "2024-05-16T13:59:41.937870Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.683994Z", "iopub.status.idle": "2024-09-17T00:18:42.684151Z", "shell.execute_reply": "2024-09-17T00:18:42.684077Z", "shell.execute_reply.started": "2024-09-17T00:18:42.684069Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, 6016, 13)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000\n", "assert mm[:100, :, 1:].min() >= 0\n", "assert mm[:100, :, 1:].max() <= 2048" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.938629Z", "start_time": "2024-05-16T13:59:41.938621Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.684673Z", "iopub.status.idle": "2024-09-17T00:18:42.684825Z", "shell.execute_reply": "2024-09-17T00:18:42.684755Z", "shell.execute_reply.started": "2024-09-17T00:18:42.684747Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as codec_encode,\n", "# decode_stream_to_full_audio as codec_decode,\n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# decode as decode\n", "# )\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "# assert len(test_metas) == len(mm)\n", "# idx_list = list(range(len(test_metas)))\n", "# # random.shuffle(idx_list)\n", "# # idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939205Z", "start_time": "2024-05-16T13:59:41.939198Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.685302Z", "iopub.status.idle": "2024-09-17T00:18:42.685451Z", "shell.execute_reply": "2024-09-17T00:18:42.685380Z", "shell.execute_reply.started": "2024-09-17T00:18:42.685373Z" } }, "outputs": [], "source": [ "# import random\n", "# idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "# assert \"original_duration_s\" in test_metas[idx]\n", "# # positive index should be shifted by 1\n", "# pos_idx = idx + 1\n", "# print(\n", "# \"tags:\",\n", "# test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", "# test_metas[idx].get(\"tags\"),\n", "# )\n", "# arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[: pad_idx_arr[0], :]\n", "# pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pos_pad_idx_arr) > 0:\n", "# pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "# a = decode(arr)\n", "# print(\"\\n negative example \\n\", test_metas[idx])\n", "# a.play(compress=False)\n", "# pos_a = decode(pos_arr)\n", "# print(\"\\n positive example \\n\", test_metas[pos_idx])\n", "# pos_a.play(compress=False)\n", "# print(\n", "# \"text:\",\n", "# test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", "# test_metas[idx].get(\"text\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939977Z", "start_time": "2024-05-16T13:59:41.939969Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.685997Z", "iopub.status.idle": "2024-09-17T00:18:42.686144Z", "shell.execute_reply": "2024-09-17T00:18:42.686074Z", "shell.execute_reply.started": "2024-09-17T00:18:42.686067Z" } }, "outputs": [], "source": [ "# val_df[val_df[\"tags\"] == 'a vibrant blend of experimental jazz fusion, drum-and-bass and swagger fuzzed-out guitars']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.940610Z", "start_time": "2024-05-16T13:59:41.940603Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.686522Z", "iopub.status.idle": "2024-09-17T00:18:42.686681Z", "shell.execute_reply": "2024-09-17T00:18:42.686610Z", "shell.execute_reply.started": "2024-09-17T00:18:42.686602Z" } }, "outputs": [], "source": [ "# from collections import Counter\n", "# c = Counter()\n", "# for _, row in df_slice.iterrows():\n", "# # print(row[\"metadata\"])\n", "# for k in ast.literal_eval(row[\"metadata\"]).keys():\n", "# c[k] += 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941167Z", "start_time": "2024-05-16T13:59:41.941159Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.687142Z", "iopub.status.idle": "2024-09-17T00:18:42.687295Z", "shell.execute_reply": "2024-09-17T00:18:42.687225Z", "shell.execute_reply.started": "2024-09-17T00:18:42.687217Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941801Z", "start_time": "2024-05-16T13:59:41.941793Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.687920Z", "iopub.status.idle": "2024-09-17T00:18:42.688069Z", "shell.execute_reply": "2024-09-17T00:18:42.688000Z", "shell.execute_reply.started": "2024-09-17T00:18:42.687993Z" } }, "outputs": [], "source": [ "def validation_on_metas(input_metas):\n", " total_bad = 0\n", " total_good = 0\n", " for idx in range(len(input_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if input_metas[idx].get(\"tags\") != input_metas[pos_idx].get(\"tags\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", " print(total_good, total_bad)\n", " return\n", "\n", "\n", "validation_on_metas(test_metas)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.942520Z", "start_time": "2024-05-16T13:59:41.942511Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.688655Z", "iopub.status.idle": "2024-09-17T00:18:42.688803Z", "shell.execute_reply": "2024-09-17T00:18:42.688734Z", "shell.execute_reply.started": "2024-09-17T00:18:42.688726Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2024-09-17T00:18:42.689310Z", "iopub.status.idle": "2024-09-17T00:18:42.689468Z", "shell.execute_reply": "2024-09-17T00:18:42.689393Z", "shell.execute_reply.started": "2024-09-17T00:18:42.689386Z" } }, "outputs": [], "source": [ "t = 0\n", "for k in train_info.keys():\n", " print(k)\n", " print(len(train_info[k][\"idx_list\"]))\n", " t += len(train_info[k][\"idx_list\"])\n", " print(t)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.943072Z", "start_time": "2024-05-16T13:59:41.943065Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.689981Z", "iopub.status.idle": "2024-09-17T00:18:42.690131Z", "shell.execute_reply": "2024-09-17T00:18:42.690060Z", "shell.execute_reply.started": "2024-09-17T00:18:42.690053Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.944246Z", "start_time": "2024-05-16T13:59:41.944237Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.690616Z", "iopub.status.idle": "2024-09-17T00:18:42.690764Z", "shell.execute_reply": "2024-09-17T00:18:42.690694Z", "shell.execute_reply.started": "2024-09-17T00:18:42.690687Z" } }, "outputs": [], "source": [ "total_iters = len(n_neg_tr) + len(n_pos_tr)\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945249Z", "start_time": "2024-05-16T13:59:41.945241Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.691234Z", "iopub.status.idle": "2024-09-17T00:18:42.691401Z", "shell.execute_reply": "2024-09-17T00:18:42.691329Z", "shell.execute_reply.started": "2024-09-17T00:18:42.691320Z" } }, "outputs": [], "source": [ "print(\"1 epoch per batch 2, total\", total_iters / 8 / 2 / 4)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945972Z", "start_time": "2024-05-16T13:59:41.945964Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.691824Z", "iopub.status.idle": "2024-09-17T00:18:42.692474Z", "shell.execute_reply": "2024-09-17T00:18:42.691905Z", "shell.execute_reply.started": "2024-09-17T00:18:42.691897Z" } }, "outputs": [], "source": [ "!cd /home/tony/Work/tony/slurm/30b_dpo && sbatch sbatch_ipo_30b" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2024-09-17T00:18:42.693125Z", "iopub.status.idle": "2024-09-17T00:18:42.693292Z", "shell.execute_reply": "2024-09-17T00:18:42.693216Z", "shell.execute_reply.started": "2024-09-17T00:18:42.693208Z" } }, "outputs": [], "source": [ "import shutil\n", "\n", "# Basic file copy\n", "shutil.copy('/home/tony/Work/tony/Preference/make_dataset_13b_v4_t3.ipynb', os.path.join(OUT_DATA_DIR, \"make_dataset.ipynb\"))\n", "print(\"Cache kept!\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# some gymathtics loading prev data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.946562Z", "start_time": "2024-05-16T13:59:41.946555Z" }, "execution": { "iopub.status.busy": "2024-09-17T00:18:42.693741Z", "iopub.status.idle": "2024-09-17T00:18:42.693889Z", "shell.execute_reply": "2024-09-17T00:18:42.693819Z", "shell.execute_reply.started": "2024-09-17T00:18:42.693811Z" } }, "outputs": [], "source": [ "# prev_v3_data = \"/app/suno/data/dpo/7v_v20_full/\"\n", "\n", "# test_val_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_val.jsonl\"))\n", "# test_tr_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_tr.jsonl\"))\n", "\n", "# all_ids = set()\n", "# for meta in test_val_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# for meta in test_tr_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# print(len(all_ids), len(test_val_metas) + len(test_tr_metas))\n", "\n", "# all_ids = list(all_ids)\n", "# with open(\"/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id.json\", \"w\") as fp:\n", "# json.dump(all_ids, fp)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2024-09-17T00:18:42.694234Z", "iopub.status.idle": "2024-09-17T00:18:42.694398Z", "shell.execute_reply": "2024-09-17T00:18:42.694321Z", "shell.execute_reply.started": "2024-09-17T00:18:42.694313Z" } }, "outputs": [], "source": [ "val_df.head(n=10)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }