{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.040680Z", "start_time": "2024-05-16T13:58:19.777010Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:03:49.394433Z", "iopub.status.busy": "2024-10-05T17:03:49.394296Z", "iopub.status.idle": "2024-10-05T17:03:50.696747Z", "shell.execute_reply": "2024-10-05T17:03:50.696241Z", "shell.execute_reply.started": "2024-10-05T17:03:49.394418Z" } }, "outputs": [], "source": [ "import ast\n", "import os\n", "import shutil\n", "import sys\n", "from collections import defaultdict\n", "\n", "import numpy as np\n", "import pandas as pd\n", "from preference_data_preparation_4min_30b import *\n", "from preference_helper import *\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n", "from tqdm import tqdm\n", "\n", "pd.set_option(\"display.max_rows\", 500)\n", "pd.set_option(\"display.max_columns\", 500)\n", "pd.set_option(\"display.width\", 1000)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.082172Z", "start_time": "2024-05-16T13:58:21.041926Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:03:51.410662Z", "iopub.status.busy": "2024-10-05T17:03:51.410218Z", "iopub.status.idle": "2024-10-05T17:03:51.441010Z", "shell.execute_reply": "2024-10-05T17:03:51.440564Z", "shell.execute_reply.started": "2024-10-05T17:03:51.410646Z" } }, "outputs": [], "source": [ "OUT_DATA_DIR = \"/app/suno/data/dpo/30b_t3_v13\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "shutil.copyfile(\n", " \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", " os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", ")\n", "NPZ_DIR = \"/app/suno/data/dpo/30b_npz\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:53.962528Z", "start_time": "2024-05-16T13:58:21.105919Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:03:52.526157Z", "iopub.status.busy": "2024-10-05T17:03:52.526008Z", "iopub.status.idle": "2024-10-05T17:03:57.243891Z", "shell.execute_reply": "2024-10-05T17:03:57.243325Z", "shell.execute_reply.started": "2024-10-05T17:03:52.526143Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Preference data shape (140152, 68)\n" ] } ], "source": [ "df = pd.read_pickle(\n", " \"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_20240909_full_with_cer.pkl\"\n", ") # , engine='python')\n", "# df = pd.read_csv(\n", "# \"/home/tony/Data/Preference/30b_v0/interesting_clips_v4_t_1_20240808.csv\"\n", "# ) # , engine='python')\n", "print(\"Preference data shape\", df.shape)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2024-10-05T17:03:57.244945Z", "iopub.status.busy": "2024-10-05T17:03:57.244787Z", "iopub.status.idle": "2024-10-05T17:03:57.402432Z", "shell.execute_reply": "2024-10-05T17:03:57.401893Z", "shell.execute_reply.started": "2024-10-05T17:03:57.244930Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(140152, 68)\n", "(140152, 64)\n" ] } ], "source": [ "print(df.shape)\n", "df = df.dropna(axis=1, how=\"all\")\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.199480Z", "start_time": "2024-05-16T13:58:53.963687Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:03:57.403260Z", "iopub.status.busy": "2024-10-05T17:03:57.403106Z", "iopub.status.idle": "2024-10-05T17:04:34.770087Z", "shell.execute_reply": "2024-10-05T17:04:34.769530Z", "shell.execute_reply.started": "2024-10-05T17:03:57.403245Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1293229\n", "1293229\n", "pre-downloaded df (140152, 64)\n", "downloaded df (140152, 64)\n" ] } ], "source": [ "converted_paths = os.listdir(NPZ_DIR)\n", "print(len(converted_paths))\n", "\n", "converted_paths = set([f.replace(\".npz\", \"\") for f in converted_paths])\n", "print(len(converted_paths))\n", "\n", "print(\"pre-downloaded df\", df.shape)\n", "df[df[\"s3_id\"].isin(converted_paths)].shape\n", "df = df[df[\"s3_id\"].isin(converted_paths)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.467253Z", "start_time": "2024-05-16T13:58:56.207647Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:04:34.771419Z", "iopub.status.busy": "2024-10-05T17:04:34.771251Z", "iopub.status.idle": "2024-10-05T17:04:34.806086Z", "shell.execute_reply": "2024-10-05T17:04:34.805662Z", "shell.execute_reply.started": "2024-10-05T17:04:34.771403Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "is_30b\n", "True 140152\n", "Name: count, dtype: int64\n" ] }, { "data": { "text/plain": [ "task\n", " 113536\n", "extend 26490\n", "cover 114\n", "infill 12\n", "Name: count, dtype: int64" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"is_30b\"] = df[\"model_name\"].str.contains(\"-t\")\n", "print(df[\"is_30b\"].value_counts())\n", "df[\"task\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.592883Z", "start_time": "2024-05-16T13:58:56.470781Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:04:34.806808Z", "iopub.status.busy": "2024-10-05T17:04:34.806673Z", "iopub.status.idle": "2024-10-05T17:04:34.879309Z", "shell.execute_reply": "2024-10-05T17:04:34.878826Z", "shell.execute_reply.started": "2024-10-05T17:04:34.806795Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "preference model_name \n", "False chirp-v3p5-engine-t-3 70076\n", "True chirp-v3p5-engine-t-3 70076\n", "Name: count, dtype: int64\n", "(140152, 65)\n", "(140152, 65)\n" ] } ], "source": [ "## for 13b this is easy for now\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "print(df.shape)\n", "df = df[df[\"model_name\"].isin([\"chirp-v3p5-engine-t-3\"])]\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.909539Z", "start_time": "2024-05-16T13:58:56.595736Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:04:34.880180Z", "iopub.status.busy": "2024-10-05T17:04:34.880039Z", "iopub.status.idle": "2024-10-05T17:04:35.188910Z", "shell.execute_reply": "2024-10-05T17:04:35.188369Z", "shell.execute_reply.started": "2024-10-05T17:04:34.880166Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(140152, 65)\n", "(140152, 65)\n", "preference model_name \n", "False chirp-v3p5-engine-t-3 70076\n", "True chirp-v3p5-engine-t-3 70076\n", "Name: count, dtype: int64\n" ] } ], "source": [ "print(df.shape)\n", "df = df[\n", " df[\"request_id\"].isin(\n", " df[\"request_id\"].value_counts().index[df[\"request_id\"].value_counts() == 2]\n", " )\n", "]\n", "print(df.shape)\n", "print(df.groupby([\"preference\"])[\"model_name\"].value_counts())\n", "assert df.shape[0] == df[\"request_id\"].nunique() * 2" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2024-10-05T17:04:35.189723Z", "iopub.status.busy": "2024-10-05T17:04:35.189578Z", "iopub.status.idle": "2024-10-05T17:05:09.487562Z", "shell.execute_reply": "2024-10-05T17:05:09.487000Z", "shell.execute_reply.started": "2024-10-05T17:04:35.189709Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "unique_requests 70076\n" ] } ], "source": [ "# Let's use the old selection for now -- for quality assurance\n", "# expand the metadata columns -- this takes forever...~ 6 mins\n", "test_slice = df[\"metadata\"].apply(lambda x: ast.literal_eval(str(x)))\n", "test_slice_series = test_slice.apply(pd.Series)\n", "df = pd.concat([df, test_slice_series], axis=1, join=\"inner\")\n", "print(\"unique_requests\", df[\"request_id\"].nunique())" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "execution": { "iopub.execute_input": "2024-10-05T17:05:09.488416Z", "iopub.status.busy": "2024-10-05T17:05:09.488258Z", "iopub.status.idle": "2024-10-05T17:05:09.846696Z", "shell.execute_reply": "2024-10-05T17:05:09.846134Z", "shell.execute_reply.started": "2024-10-05T17:05:09.488401Z" } }, "outputs": [], "source": [ "df = df.loc[:, ~df.columns.duplicated()].copy()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "execution": { "iopub.execute_input": "2024-10-05T17:05:09.847605Z", "iopub.status.busy": "2024-10-05T17:05:09.847447Z", "iopub.status.idle": "2024-10-05T17:05:09.856718Z", "shell.execute_reply": "2024-10-05T17:05:09.856287Z", "shell.execute_reply.started": "2024-10-05T17:05:09.847590Z" } }, "outputs": [ { "data": { "text/plain": [ "task\n", " 113536\n", "extend 26490\n", "cover 114\n", "infill 12\n", "Name: count, dtype: int64" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"task\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "execution": { "iopub.execute_input": "2024-10-05T17:05:09.858416Z", "iopub.status.busy": "2024-10-05T17:05:09.858258Z", "iopub.status.idle": "2024-10-05T17:05:10.177471Z", "shell.execute_reply": "2024-10-05T17:05:10.176991Z", "shell.execute_reply.started": "2024-10-05T17:05:09.858401Z" } }, "outputs": [ { "data": { "text/plain": [ "pos_diff_preference\n", "1.0 52831\n", "2.0 17245\n", "Name: count, dtype: int64" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df.sort_values(by=[\"request_id\", \"preference\", \"diff_preference\"])\n", "df[\"pos_diff_preference\"] = df[\"diff_preference\"].diff()\n", "df[\"cer_diff_preference\"] = df[\"cer\"].diff()\n", "df[df[\"preference\"]][\"pos_diff_preference\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "execution": { "iopub.execute_input": "2024-10-05T17:05:10.178270Z", "iopub.status.busy": "2024-10-05T17:05:10.178123Z", "iopub.status.idle": "2024-10-05T17:05:10.180347Z", "shell.execute_reply": "2024-10-05T17:05:10.179961Z", "shell.execute_reply.started": "2024-10-05T17:05:10.178256Z" } }, "outputs": [], "source": [ "# df[[\"preference\", \"diff_preference\", \"pos_diff_preference\", \"cer\", \"cer_diff_preference\"]].tail(n=6)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "execution": { "iopub.execute_input": "2024-10-05T17:05:10.181033Z", "iopub.status.busy": "2024-10-05T17:05:10.180895Z", "iopub.status.idle": "2024-10-05T17:05:10.277842Z", "shell.execute_reply": "2024-10-05T17:05:10.277349Z", "shell.execute_reply.started": "2024-10-05T17:05:10.181020Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive param_experiment\n", "max_cfg_12_ntag_3 18255\n", "max_cfg_12 2311\n", "cfg_12 2111\n", "n_repeat_tags_3 1950\n", "Name: count, dtype: int64\n" ] } ], "source": [ "try:\n", " print(\"positive\", df[df[\"preference\"]][\"param_experiment\"].value_counts())\n", "except:\n", " pass" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "execution": { "iopub.execute_input": "2024-10-05T17:05:10.278593Z", "iopub.status.busy": "2024-10-05T17:05:10.278452Z", "iopub.status.idle": "2024-10-05T17:05:10.727599Z", "shell.execute_reply": "2024-10-05T17:05:10.727128Z", "shell.execute_reply.started": "2024-10-05T17:05:10.278579Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.36949999999999994\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGgCAYAAABIanZ7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA9UklEQVR4nO3de1TVdb7/8RebS5oiIWDqaKdGA28ImB0HZ/Ojm9akNmGWnizDLC0z9eSkjWmKdsQ6jGOmpaHRkDbWUvPYqK3JzslsgZUmeSi8YOVlmI5svOIluXx/fzR8px0IbNwb2B+fj7VYi/39fr6f/XnvD3v78nvbAZZlWQIAADCMo6kHAAAA4AuEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJI9CzrJly3TPPfcoISFBiYmJGj9+vL755hu3Nj/88IPS0tLUr18/JSQk6Mknn5TL5XJrU1RUpLFjxyouLk6JiYl64YUXVF5e7tbm008/VUpKinr16qUBAwZo3bp11cazatUq3XLLLYqNjdW9996r3bt3e1IOAAAwmEch57PPPtPIkSP1zjvvKCsrS+Xl5RozZozOnj1rt5k3b57+53/+RwsXLtSbb76po0ePasKECfb6iooKjRs3TmVlZVq9erXmz5+vd999V4sWLbLbHD58WOPGjVO/fv30X//1X3rooYc0Y8YMbdu2zW6zadMmpaen64knntC7776rbt26acyYMSopKbmU1wMAAJjCugQlJSVWdHS09dlnn1mWZVmnTp2yevbsaW3evNluU1hYaEVHR1u7du2yLMuyPvroI6tbt25WcXGx3eatt96y+vTpY/3www+WZVnWiy++aA0aNMjtuSZPnmw9/PDD9uNhw4ZZaWlp9uOKigrL6XRay5Ytu5SSAACAIYIuJSCdPn1akhQWFiZJys/PV1lZmfr372+36dKlizp27Ki8vDzFx8crLy9P0dHRioyMtNs4nU7Nnj1bhYWF6tGjh/Ly8pSYmOj2XE6nU/PmzZMkXbhwQV999ZXGjRtnr3c4HOrfv7927drlcR3Hjp2WN7+LPSBAats21Ov9Niem10h9/s/0GqnP/5leoy/rq+q7Lg0OOZWVlZo3b5769Omj6OhoSZLL5VJwcLDatGnj1jYiIkLFxcV2m58GHEn247ralJaW6vz58zp58qQqKioUERFR7Xl+fo5QfdTnhWoIX/XbnJheI/X5P9NrpD7/Z3qNTVlfg0NOWlqa9u/fr7feesub42kSJSXe35MTERHq9X6bE9NrpD7/Z3qN1Of/TK/Rl/VV9V2XBoWcOXPm6KOPPtLKlSvVvn17e3lkZKTKysp06tQpt705JSUlioqKstv8/Cqoqquvftrm51dkuVwutW7dWi1atJDD4VBgYGC1k4xLSkqq7QGqD8uST/7AfNVvc2J6jdTn/0yvkfr8n+k1NmV9Hl1dZVmW5syZow8++EB/+tOf1LlzZ7f1vXr1UnBwsHJzc+1l33zzjYqKihQfHy9Jio+P1759+9wCSk5Ojlq3bq2uXbvabbZv3+7Wd05Ojt1HSEiIevbs6fY8lZWVys3NVUJCgiclAQAAQ3kUctLS0rRhwwb94Q9/UKtWrVRcXKzi4mKdP39ekhQaGqp77rlH8+fP1/bt25Wfn6/p06crISHBDihOp1Ndu3bV1KlTtWfPHm3btk0LFy7UyJEjFRISIkkaMWKEDh8+rBdffFEHDhzQqlWrtHnzZqWmptpjGT16tN555x29++67OnDggGbPnq1z585p6NCh3nllAACAX/PocNWf//xnSdKDDz7otjw9Pd0OF9OnT5fD4dDEiRN14cIFOZ1OzZo1y24bGBiopUuXavbs2Ro+fLhatmyplJQUTZw40W7TuXNnLVu2TOnp6crOzlb79u31/PPPKykpyW5z55136tixY1q0aJGKi4vVvXt3LV++vEGHqwAAgHkCLMvkI4H143J5/8TjyMhQr/fbnJheI/X5P9NrpD7/Z3qNvqyvqu+68N1VAADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjNfgLOgHAHwQG1v5/ucpKS5WVBt6kBAAhB4CZHI4AVVRaCg9vVWu78opKnTxxlqADGIiQA8BIAQEBCnQEaNLqXSo8Wlpjm67tWuulEQlyOAIIOYCBCDkAjFZ4tFRfFZ1q6mEAaAKceAwAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABjJ45Dz+eef67HHHpPT6VRMTIy2bNnitj4mJqbGn+XLl9ttbrnllmrrX3vtNbd+9uzZo/vvv1+xsbFKTk5WZmZmtbFs3rxZd9xxh2JjYzVkyBBt3brV03IAAIChgjzd4OzZs4qJidE999yjCRMmVFv/ySefuD3++OOP9eyzz+r22293Wz5x4kTdd9999uNWrVrZv5eWlmrMmDFKTExUWlqa9u3bp+nTp6tNmzYaPny4JOmLL77QlClT9NRTT+nmm2/We++9pyeeeELr1q1TdHS0p2UBAADDeBxykpOTlZycfNH1UVFRbo8//PBD9evXT507d3Zb3qpVq2ptq2zYsEFlZWWaN2+eQkJCdP3116ugoEBZWVl2yMnOzlZSUpIeeeQRSdLkyZOVk5OjlStXas6cOZ6WBQAADONxyPGEy+XS1q1bNX/+/GrrMjMz9eqrr6pDhw4aPHiwUlNTFRT043Dy8vLUt29fhYSE2O2dTqcyMzN18uRJhYWFKS8vT6mpqW59Op3OaofP6iMgwONN6tWft/ttTkyvkfr8n6e1+dtrYfocml6fZH6Nvqyvvn36NOS8++67atWqlQYOHOi2/MEHH1SPHj0UFhamXbt2acGCBSouLtbvf/97ST+Go06dOrltExkZaa8LCwuTy+Wyl1WJiIiQy+XyeJwREaEeb9OU/TYnptdIfZeH8PBWdTdqpkyfQ9Prk8yvsSnr82nIWbt2rYYMGaIrrrjCbfno0aPt37t166bg4GDNmjVLU6ZMcdt701hKSk7LsrzXX0DAj5Pq7X6bE9NrpD7/FxTk0FVX1S+8HD9+RhUVlT4ekXeZPoem1yeZX6Mv66vquy4+Czk7duzQt99+q4ULF9bZNi4uTuXl5Tpy5Ih++ctfKjIystoemarHVXtvampTUlJSbe9OfViWfPIH5qt+mxPTa6Q+/+VpXf76Opg8h5L59Unm19iU9fnsPjlr1qxRz5491a1btzrbFhQUyOFwKCIiQpIUHx+vHTt2qKyszG6Tk5Oj6667TmFhYXab7du3u/WTk5Oj+Ph47xUBAAD8lsch58yZMyooKFBBQYEk6ciRIyooKFBRUZHdprS0VO+//77uvffeatvv2rVLb7zxhvbs2aPDhw9rw4YNSk9P11133WUHmCFDhig4OFjPPvus9u/fr02bNik7O9vtMNeoUaO0bds2vf766zpw4IBefvll5efn64EHHvD4RQAAAObx+HBVfn6+Ro0aZT9OT0+XJKWkpNhXUW3cuFGWZWnw4MHVtg8JCdGmTZu0ePFiXbhwQZ06dVJqaqpbgAkNDdWKFSs0Z84cDR06VOHh4Ro/frx9+bgk9enTRxkZGVq4cKEWLFiga6+9VkuWLOEeOQAAQJIUYFkmHwmsH5fL+yceR0aGer3f5sT0GqnP/wUFORQe3kqDFm3TV0WnamzTs2MbbZyYpOPHz6i83P9OPDZ5Dk2vTzK/Rl/WV9V3XfjuKgAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYyeOQ8/nnn+uxxx6T0+lUTEyMtmzZ4rb+mWeeUUxMjNvPmDFj3NqcOHFCU6ZMUZ8+fdS3b19Nnz5dZ86ccWuzZ88e3X///YqNjVVycrIyMzOrjWXz5s264447FBsbqyFDhmjr1q2elgMAAAzlccg5e/asYmJiNGvWrIu2SUpK0ieffGL/LFiwwG397373OxUWFiorK0tLly7Vjh079Nxzz9nrS0tLNWbMGHXs2FHr1q3T1KlTtXjxYr399tt2my+++EJTpkzRsGHDtH79et1666164okntG/fPk9LAgAABgrydIPk5GQlJyfX2iYkJERRUVE1rjtw4IC2bdumNWvWKDY2VpI0Y8YMjR07VlOnTtXVV1+tDRs2qKysTPPmzVNISIiuv/56FRQUKCsrS8OHD5ckZWdnKykpSY888ogkafLkycrJydHKlSs1Z84cT8sCAACG8Tjk1Mdnn32mxMREtWnTRr/61a80efJkhYeHS5J27dqlNm3a2AFHkvr37y+Hw6Hdu3drwIABysvLU9++fRUSEmK3cTqdyszM1MmTJxUWFqa8vDylpqa6Pa/T6ax2+Kw+AgIaVmdd/Xm73+bE9Bqpz/95Wpu/vRamz6Hp9Unm1+jL+urbp9dDTlJSkgYMGKBOnTrp8OHDWrBggR599FG9/fbbCgwMlMvlUtu2bd0HERSksLAwFRcXS5JcLpc6derk1iYyMtJeFxYWJpfLZS+rEhERIZfL5fGYIyJCPd6mKfttTkyvkfouD+HhrZp6CA1m+hyaXp9kfo1NWZ/XQ86gQYPs36tOPL7tttvsvTvNUUnJaVmW9/oLCPhxUr3db3Nieo3U5/+Cghy66qr6hZfjx8+ooqLSxyPyLtPn0PT6JPNr9GV9VX3XxSeHq36qc+fOCg8P18GDB5WYmKjIyEgdO3bMrU15eblOnjxpn8cTGRlZbY9M1eOqvTc1tSkpKam2d6c+LEs++QPzVb/Niek1Up//8rQuf30dTJ5Dyfz6JPNrbMr6fH6fnO+//14nTpywA0xCQoJOnTql/Px8u8327dtVWVmp3r17S5Li4+O1Y8cOlZWV2W1ycnJ03XXXKSwszG6zfft2t+fKyclRfHy8jysCAAD+wOOQc+bMGRUUFKigoECSdOTIERUUFKioqEhnzpzRCy+8oLy8PB05ckS5ubkaP368/uVf/kVJSUmSpC5duigpKUkzZ87U7t27tXPnTs2dO1eDBg3S1VdfLUkaMmSIgoOD9eyzz2r//v3atGmTsrOzNXr0aHsco0aN0rZt2/T666/rwIEDevnll5Wfn68HHnjAG68LAADwcx4frsrPz9eoUaPsx+np6ZKklJQUzZ49W/v27dP69et1+vRptWvXTr/+9a81adIktyulMjIyNHfuXD300ENyOBwaOHCgZsyYYa8PDQ3VihUrNGfOHA0dOlTh4eEaP368ffm4JPXp00cZGRlauHChFixYoGuvvVZLlixRdHR0g14IAABglgDLMvlIYP24XN4/8TgyMtTr/TYnptdIff4vKMih8PBWGrRom74qOlVjm54d22jjxCQdP35G5eX+d+KxyXNoen2S+TX6sr6qvuvCd1cBAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEgeh5zPP/9cjz32mJxOp2JiYrRlyxZ7XVlZmf7zP/9TQ4YMUXx8vJxOp6ZOnar/+7//c+vjlltuUUxMjNvPa6+95tZmz549uv/++xUbG6vk5GRlZmZWG8vmzZt1xx13KDY2VkOGDNHWrVs9LQcAABjK45Bz9uxZxcTEaNasWdXWnT9/Xl9//bUef/xxrVu3TosXL9a3336rxx9/vFrbiRMn6pNPPrF/HnjgAXtdaWmpxowZo44dO2rdunWaOnWqFi9erLfffttu88UXX2jKlCkaNmyY1q9fr1tvvVVPPPGE9u3b52lJAADAQEGebpCcnKzk5OQa14WGhiorK8tt2cyZM3XvvfeqqKhIHTt2tJe3atVKUVFRNfazYcMGlZWVad68eQoJCdH111+vgoICZWVlafjw4ZKk7OxsJSUl6ZFHHpEkTZ48WTk5OVq5cqXmzJnjUU0BAR41r3d/3u63OTG9Rurzf57W5m+vhelzaHp9kvk1+rK++vbpccjxVGlpqQICAtSmTRu35ZmZmXr11VfVoUMHDR48WKmpqQoK+nE4eXl56tu3r0JCQuz2TqdTmZmZOnnypMLCwpSXl6fU1FS3Pp1Op9vhs/qKiAj1vLAm7Lc5Mb1G6rs8hIe3auohNJjpc2h6fZL5NTZlfT4NOT/88IMyMjI0aNAgtW7d2l7+4IMPqkePHgoLC9OuXbu0YMECFRcX6/e//70kyeVyqVOnTm59RUZG2uvCwsLkcrnsZVUiIiLkcrk8HmdJyWlZlsebXVRAwI+T6u1+mxPTa6Q+/xcU5NBVV9UvvBw/fkYVFZU+HpF3mT6HptcnmV+jL+ur6rsuPgs5ZWVlmjRpkizLUlpamtu60aNH279369ZNwcHBmjVrlqZMmeK296axWJZ88gfmq36bE9NrpD7/5Wld/vo6mDyHkvn1SebX2JT1+eQS8rKyMk2ePFlFRUV6/fXX3fbi1CQuLk7l5eU6cuSIpB/32vx8j0zV46q9NzW1KSkpqbZ3BwAAXJ68HnKqAs7Bgwf1xhtvKDw8vM5tCgoK5HA4FBERIUmKj4/Xjh07VFZWZrfJycnRddddp7CwMLvN9u3b3frJyclRfHy894oBAAB+y+OQc+bMGRUUFKigoECSdOTIERUUFKioqEhlZWWaOHGi8vPzlZGRoYqKChUXF6u4uFgXLlyQJO3atUtvvPGG9uzZo8OHD2vDhg1KT0/XXXfdZQeYIUOGKDg4WM8++6z279+vTZs2KTs72+0w16hRo7Rt2za9/vrrOnDggF5++WXl5+e7XYoOAAAuXx6fk5Ofn69Ro0bZj9PT0yVJKSkpmjBhgv77v/9bkvTb3/7Wbbvs7Gz169dPISEh2rRpkxYvXqwLFy6oU6dOSk1NdQswoaGhWrFihebMmaOhQ4cqPDxc48ePty8fl6Q+ffooIyNDCxcu1IIFC3TttddqyZIlio6O9rQkAABgII9DTr9+/bR3796Lrq9tnST17NlT77zzTp3P061bN7311lu1tvnNb36j3/zmN3X2BQAALj98dxUAADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjORxyPn888/12GOPyel0KiYmRlu2bHFbb1mWXnrpJTmdTvXu3Vupqan67rvv3NqcOHFCU6ZMUZ8+fdS3b19Nnz5dZ86ccWuzZ88e3X///YqNjVVycrIyMzOrjWXz5s264447FBsbqyFDhmjr1q2elgMAAAzlccg5e/asYmJiNGvWrBrXZ2Zm6s0339Ts2bP1zjvvqGXLlhozZox++OEHu83vfvc7FRYWKisrS0uXLtWOHTv03HPP2etLS0s1ZswYdezYUevWrdPUqVO1ePFivf3223abL774QlOmTNGwYcO0fv163XrrrXriiSe0b98+T0sCAAAGCvJ0g+TkZCUnJ9e4zrIsZWdn6/HHH9dtt90mSXrxxRfVv39/bdmyRYMGDdKBAwe0bds2rVmzRrGxsZKkGTNmaOzYsZo6daquvvpqbdiwQWVlZZo3b55CQkJ0/fXXq6CgQFlZWRo+fLgkKTs7W0lJSXrkkUckSZMnT1ZOTo5WrlypOXPmeFRTQICnr0L9+vN2v82J6TVSn//ztDZ/ey1Mn0PT65PMr9GX9dW3T49DTm2OHDmi4uJi9e/f314WGhqquLg47dq1S4MGDdKuXbvUpk0bO+BIUv/+/eVwOLR7924NGDBAeXl56tu3r0JCQuw2TqdTmZmZOnnypMLCwpSXl6fU1FS353c6ndUOn9VHRESo58U2Yb/Niek1Ut/lITy8VVMPocFMn0PT65PMr7Ep6/NqyCkuLpYkRUREuC2PiIiQy+WSJLlcLrVt29Z9EEFBCgsLs7d3uVzq1KmTW5vIyEh7XVhYmFwul72spufxREnJaVmWx5tdVEDAj5Pq7X6bE9NrpD7/FxTk0FVX1S+8HD9+RhUVlT4ekXeZPoem1yeZX6Mv66vquy5eDTn+yrLkkz8wX/XbnJheI/X5L0/r8tfXweQ5lMyvTzK/xqasz6uXkEdFRUmSSkpK3JaXlJTYe10iIyN17Ngxt/Xl5eU6efKkvX1kZGS1PTJVj3/az8/b/PR5AADA5c2rIadTp06KiopSbm6uvay0tFRffvmlEhISJEkJCQk6deqU8vPz7Tbbt29XZWWlevfuLUmKj4/Xjh07VFZWZrfJycnRddddp7CwMLvN9u3b3Z4/JydH8fHx3iwJAAD4KY9DzpkzZ1RQUKCCggJJP55sXFBQoKKiIgUEBGjUqFF69dVX9eGHH2rv3r2aOnWq2rVrZ19t1aVLFyUlJWnmzJnavXu3du7cqblz52rQoEG6+uqrJUlDhgxRcHCwnn32We3fv1+bNm1Sdna2Ro8ebY9j1KhR2rZtm15//XUdOHBAL7/8svLz8/XAAw9443UBAAB+zuNzcvLz8zVq1Cj7cXp6uiQpJSVF8+fP16OPPqpz587pueee06lTp3TDDTdo+fLluuKKK+xtMjIyNHfuXD300ENyOBwaOHCgZsyYYa8PDQ3VihUrNGfOHA0dOlTh4eEaP368ffm4JPXp00cZGRlauHChFixYoGuvvVZLlixRdHR0g14IAABglgDLMvl0p/pxubx/dVVkZKjX+21OTK+R+vxfUJBD4eGtNGjRNn1VdKrGNj07ttHGiUk6fvyMysv97+oqk+fQ9Pok82v0ZX1VfdeF764CAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJG8HnJuueUWxcTEVPtJS0uTJD344IPV1j333HNufRQVFWns2LGKi4tTYmKiXnjhBZWXl7u1+fTTT5WSkqJevXppwIABWrdunbdLAQAAfizI2x2uWbNGFRUV9uP9+/dr9OjRuuOOO+xl9913nyZOnGg/btmypf17RUWFxo0bp8jISK1evVpHjx7VtGnTFBwcrKeeekqSdPjwYY0bN04jRoxQRkaGcnNzNWPGDEVFRSkpKcnbJQFohhyOADkcARddHxjIjmrgcuf1kNO2bVu3x6+99pquueYa/eu//qu9rEWLFoqKiqpx+08++USFhYXKyspSZGSkunfvrkmTJikjI0MTJkxQSEiIVq9erU6dOumZZ56RJHXp0kU7d+7UG2+80aCQE3Dxz8kGqerP2/02J6bXSH3Nm8MRoDZhVyrIi0HG314Lf5/Duphen2R+jb6sr759ej3k/NSFCxe0YcMGjR49WgE/GdF7772nDRs2KCoqSjfffLPGjx9v783Jy8tTdHS0IiMj7fZOp1OzZ89WYWGhevTooby8PCUmJro9l9Pp1Lx58xo0zoiI0AZt11T9Niem10h9zduk1btUeLS0xnU3xUTp6du71auf8PBW3hxWo/L3OayL6fVJ5tfYlPX5NORs2bJFp0+fVkpKir1s8ODB6tixo9q1a6e9e/cqIyND3377rRYvXixJcrlcbgFHkv24uLi41jalpaU6f/68WrRo4dE4S0pOy7I8Lu+iAgJ+nFRv99ucmF4j9TVvgYEOhYe3UuHRUn1VdKrGNl2i6h9cjh8/o4qKSm8Nr1H4+xzWxfT6JPNr9GV9VX3XxachZ+3atfp//+//6eqrr7aXDR8+3P49JiZGUVFRSk1N1aFDh3TNNdf4cjgXZVnyyR+Yr/ptTkyvkfouH/76Opg+h6bXJ5lfY1PW57Mz8/72t78pJydHw4YNq7VdXFycJOngwYOSftwj43K53NpUPa46j+dibVq3bu3xXhwAAGAmn4WcdevWKSIiQjfddFOt7QoKCiT9M8DEx8dr3759Kikpsdvk5OSodevW6tq1q91m+/btbv3k5OQoPj7eewUAAAC/5pOQU1lZqXXr1unuu+9WUNA/j4gdOnRIS5YsUX5+vo4cOaIPP/xQ06ZN04033qhu3X48QdDpdKpr166aOnWq9uzZo23btmnhwoUaOXKkQkJCJEkjRozQ4cOH9eKLL+rAgQNatWqVNm/erNTUVF+UAwAA/JBPzsnJyclRUVGR7rnnHrflwcHBys3NVXZ2ts6ePasOHTpo4MCBGj9+vN0mMDBQS5cu1ezZszV8+HC1bNlSKSkpbvfV6dy5s5YtW6b09HRlZ2erffv2ev7557lHDgAAsPkk5DidTu3du7fa8g4dOmjlypV1bv+LX/xCmZmZtbbp16+f1q9f39AhAgAAw3FLUAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYyesh5+WXX1ZMTIzbzx133GGv/+GHH5SWlqZ+/fopISFBTz75pFwul1sfRUVFGjt2rOLi4pSYmKgXXnhB5eXlbm0+/fRTpaSkqFevXhowYIDWrVvn7VIAAIAfC/JFp9dff72ysrLsx4GBgfbv8+bN09atW7Vw4UKFhoZq7ty5mjBhglavXi1Jqqio0Lhx4xQZGanVq1fr6NGjmjZtmoKDg/XUU09Jkg4fPqxx48ZpxIgRysjIUG5urmbMmKGoqCglJSX5oiQAAOBnfBJyAgMDFRUVVW356dOntXbtWmVkZCgxMVHSj6HnzjvvVF5enuLj4/XJJ5+osLBQWVlZioyMVPfu3TVp0iRlZGRowoQJCgkJ0erVq9WpUyc988wzkqQuXbpo586deuONNxoUcgICLq3ei/Xn7X6bE9NrpL6m5XAEKKCWwQUGev9Ie3N9LS6muc/hpTK9Psn8Gn1ZX3379EnIOXjwoJxOp6644grFx8drypQp6tixo/Lz81VWVqb+/fvbbbt06aKOHTvaIScvL0/R0dGKjIy02zidTs2ePVuFhYXq0aOH8vLy7JD00zbz5s1r0HgjIkIbVmgT9ducmF4j9TWNikpLgY7G++QPD2/VaM/lbc11Dr3F9Pok82tsyvq8HnJ69+6t9PR0XXfddSouLtaSJUs0cuRIvffee3K5XAoODlabNm3ctomIiFBxcbEkyeVyuQUcSfbjutqUlpbq/PnzatGihUdjLik5LcvyaJNaBQT8OKne7rc5Mb1G6ms6gYEOhYe30qTVu1R4tLTGNjfFROnp27t57TmPHz+jiopKr/XXGJrzHHqD6fVJ5tfoy/qq+q6L10NOcnKy/Xu3bt0UFxenm2++WZs3b/Y4fDQWy5JP/sB81W9zYnqN1Nd0Co+W6quiUzWu6xLl/T0vzfV1qEtznkNvML0+yfwam7I+n19C3qZNG1177bU6dOiQIiMjVVZWplOn3D+4SkpK7HN4IiMjq11tVfW4rjatW7dutkEKAAA0Lp+HnDNnzujw4cOKiopSr169FBwcrNzcXHv9N998o6KiIsXHx0uS4uPjtW/fPpWUlNhtcnJy1Lp1a3Xt2tVus337drfnycnJsfsAAADwesh54YUX9Nlnn+nIkSP64osvNGHCBDkcDg0ePFihoaG65557NH/+fG3fvl35+fmaPn26EhIS7IDidDrVtWtXTZ06VXv27NG2bdu0cOFCjRw5UiEhIZKkESNG6PDhw3rxxRd14MABrVq1Sps3b1Zqaqq3ywEAAH7K6+fkfP/993rqqad04sQJtW3bVjfccIPeeecdtW3bVpI0ffp0ORwOTZw4URcuXJDT6dSsWbPs7QMDA7V06VLNnj1bw4cPV8uWLZWSkqKJEyfabTp37qxly5YpPT1d2dnZat++vZ5//nnukQOgQeq6JL2y0lJlpcEnTQCG8nrI+eMf/1jr+iuuuEKzZs1yCzY/94tf/EKZmZm19tOvXz+tX7++IUMEAElSVOsrVFFpqU2blrW2K6+o1MkTZwk6gJ/xyX1yAMAftGkZpEBHQK2Xq3dt11ovjUiQwxFAyAH8DCEHwGWvtsvVAfgvvoUcAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMFJQUw8AwOXF4QiQwxFw0fWBgfzfC4B3EHIANBqHI0BhV12pIIIMgEZAyAHQaByOAAUFOjRp9S4VHi2tsc1NMVF6+vZujTwyACYi5ABodIVHS/VV0aka13WJatXIowFgKvYZAwAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiUvIAaAe6roTc2WlpcpKq5FGA6A+CDkAUIuo1leootJSmzYta21XXlGpkyfOEnSAZoSQAwC1aNMySIGOgFrv0ty1XWu9NCJBDkcAIQdoRgg5AFAPtd2lGUDzxInHAADASOzJAeA1DkeAHI6Ai66v6+RdAPAmQg4Ar3A4AhR21ZUKIsgAaCa8HnKWLVumv/71r/rmm2/UokULJSQk6He/+51++ctf2m0efPBBffbZZ27bDR8+XHPmzLEfFxUVafbs2fr000915ZVX6u6779aUKVMUFPTPIX/66aeaP3++9u/frw4dOujxxx/X0KFDvV0SgHpwOAIUFOio9QTdm2Ki9PTt3Rp5ZAAuV14POZ999plGjhyp2NhYVVRUaMGCBRozZow2btyoK6+80m533333aeLEifbjli3/eXlmRUWFxo0bp8jISK1evVpHjx7VtGnTFBwcrKeeekqSdPjwYY0bN04jRoxQRkaGcnNzNWPGDEVFRSkpKcnbZQGop9pO0O0S1aqRRwPgcub1kLNixQq3x/Pnz1diYqK++uor3XjjjfbyFi1aKCoqqsY+PvnkExUWFiorK0uRkZHq3r27Jk2apIyMDE2YMEEhISFavXq1OnXqpGeeeUaS1KVLF+3cuVNvvPGGxyEn4OKnEDRIVX/e7rc5Mb1G6kNDNdZravocml6fZH6Nvqyvvn36/Jyc06dPS5LCwsLclr/33nvasGGDoqKidPPNN2v8+PH23py8vDxFR0crMjLSbu90OjV79mwVFhaqR48eysvLU2JiolufTqdT8+bN83iMERGhHm/TlP02J6bXSH3wRHh44++pMn0OTa9PMr/GpqzPpyGnsrJS8+bNU58+fRQdHW0vHzx4sDp27Kh27dpp7969ysjI0LfffqvFixdLklwul1vAkWQ/Li4urrVNaWmpzp8/rxYtWtR7nCUlp2V58f5dAQE/Tqq3+21OTK+R+jwXGOhokn/km5Pjx8+ooqKyUZ6Lv1H/Z3qNvqyvqu+6+DTkpKWlaf/+/Xrrrbfclg8fPtz+PSYmRlFRUUpNTdWhQ4d0zTXX+HJINbIs+eQPzFf9Niem10h98ERTfL+V6XNoen2S+TU2ZX0+Czlz5szRRx99pJUrV6p9+/a1to2Li5MkHTx4UNdcc40iIyO1e/dutzYul0uS7PN4IiMj7WU/bdO6dWuP9uIAwKXi+62A5snrIceyLM2dO1cffPCB3nzzTXXu3LnObQoKCiT9M8DEx8dr6dKlKikpUUREhCQpJydHrVu3VteuXe02H3/8sVs/OTk5io+P92I1AFA3vt8KaJ68HnLS0tL0l7/8Ra+88opatWpln0MTGhqqFi1a6NChQ3rvvfeUnJysq666Snv37lV6erpuvPFGdev24/0znE6nunbtqqlTp+rpp59WcXGxFi5cqJEjRyokJESSNGLECK1atUovvvii7rnnHm3fvl2bN2/WsmXLvF0SANQL328FNC9eDzl//vOfJf14w7+fSk9P19ChQxUcHKzc3FxlZ2fr7Nmz6tChgwYOHKjx48fbbQMDA7V06VLNnj1bw4cPV8uWLZWSkuJ2X53OnTtr2bJlSk9PV3Z2ttq3b6/nn3+ee+QAPsJXNgDwN14POXv37q11fYcOHbRy5co6+/nFL36hzMzMWtv069dP69ev92R4ABqAr2wA4I/47ioAdeIrGwD4I0IOgHrjKxsA+BP2PQMAACMRcgAAgJEIOQAAwEickwMAjagpvvoBuFwRcgCgEfDVD0DjI+QAl7m6bvIncaM/b+CrH4DGR8gBLmMOR4DahHGTv8bEVz8AjYeQA1zGAgLqvsmfxI3+APgnQg6AOvcucKM/AP6IfdQAAMBIhBwAAGAkDlcBBqvryimumgJgMkIOYCiHI0BhV9V+5VRd92xB06jPDQMti0vMgboQcgBDORx1XznFVVPNiyc3DDx18mwjjQrwX4QcwHC1XTnFVVPNiyc3DAwIqP0GjgAIOQDQ7HDDQMA7CDmAn+KkYgCoHSEH8EP1OakYZqsKsRcLs3ybOUDIAfwSJxVfvn5+cnJ4eM3nVfFt5gAhB/BrnFR8+eHbzIH6I+QAzRDn26Au9Tk5uT732yEEwWSEHKCZ4XwbXCpP7rfDIS2YjJADNLL67KXhfBtcCg5pAT8i5ACNyJO9NJxvg0vFIS1c7gg5gBexlwb+gkNauBwQcgAvYS8N/AmHtHA5IOQAuvgeGE+uYmIvDfwRh7RgMkIOjFfXIaSAgAC1Dm1R4x6Yn95oraLSUmAt/VRhLw1MwSEt+DtCDvzapQSYn6vPHhj20uBy4skhreDgQFVUVF7S87FHCN5GyPEhdvFeXF3hpD68HWDqsweGvTS4HNX2d1/fvT312RNaXlGp0tPnZVn//Fz8+efo5fy5Cc8RcnzA4QhQRaV10e+UqVLTG/rnvPWGrk+oqM9z/byfhtx5t77hpL6Hh7wVYAB4rj57e+qzJ/TGa8M1c3BPXXXVlW7Lf/452pifm/B/hBwfCAgIqPNNf7E39M/V5w1dn/HUJ1TU9Vw19VNTkPNmOCHAAP7BG3tCG/NzkyB0eSDk+FBjvaG9ESrq+1x19ePtcEKAAS4vzSkIeYupgcobpx34mt+HnFWrVmnFihUqLi5Wt27dNHPmTPXu3buph1Vvl/qG9maoqO9zEU4ANKXm9B/Ihp5r9FP++IW7npx24HAEqKKiaUKeX4ecTZs2KT09XWlpaYqLi9Of/vQnjRkzRu+//74iIiKaenhe05ihgnACwASN+R/ISw1U4eGtvBaoGrONVPue/aor7wICAiQRcjyWlZWl++67T/fcc48kKS0tTR999JHWrl2rsWPH1rsfh0Py5h7LgH/8XfTs2EYtQwJrbNMlqjVtaOMXbZrjmGhDG2+0uSLIcdE2If/YQ3GpbcKvDFGgI0BLPzqgopPnamwT+4sw3du3s1+2qa32K4J+fH0CAn78d9abAup5lCzAaowDkj5w4cIFxcfHa9GiRbrtttvs5dOmTdOpU6f06quvNuHoAABAU/O/A4H/cPz4cVVUVFQ7LBURESGXy9VEowIAAM2F34YcAACA2vhtyAkPD1dgYKBKSkrclpeUlCgyMrKJRgUAAJoLvw05ISEh6tmzp3Jzc+1llZWVys3NVUJCQhOODAAANAd+fXXV6NGjNW3aNPXq1Uu9e/fWn/70J507d05Dhw5t6qEBAIAm5tch584779SxY8e0aNEiFRcXq3v37lq+fDmHqwAAgP9eQg4AAFAbvz0nBwAAoDaEHAAAYCRCDgAAMBIhBwAAGImQ0wCvvvqqRowYobi4OPXt27de21iWpZdeeklOp1O9e/dWamqqvvvuO7c2J06c0JQpU9SnTx/17dtX06dP15kzZ3xQQd08HcuRI0cUExNT48/mzZvtdjWt37hxY2OU5KYhr/WDDz5YbezPPfecW5uioiKNHTtWcXFxSkxM1AsvvKDy8nJflnJRntZ44sQJzZ07V7fffrt69+6tm266Sc8//7xOnz7t1q6p5nDVqlW65ZZbFBsbq3vvvVe7d++utf3mzZt1xx13KDY2VkOGDNHWrVvd1tfnPdnYPKnxnXfe0f33368bb7xRN954o1JTU6u1f+aZZ6rN1ZgxY3xdxkV5Ut+6deuqjT02NtatTXObQ0/qq+nzJCYmxu3LpZvT/H3++ed67LHH5HQ6FRMToy1bttS5zaeffqqUlBT16tVLAwYM0Lp166q18fR97TELHnvppZesrKwsKz093brhhhvqtc2yZcusG264wfrggw+sgoIC67HHHrNuueUW6/z583abMWPGWHfddZeVl5dnff7559aAAQOsp556yldl1MrTsZSXl1tHjx51+3n55Zet+Ph4q7S01G4XHR1trV271q3dT1+DxtKQ1/qBBx6wZsyY4Tb206dP2+vLy8utwYMHW6mpqdbXX39tffTRR1a/fv2sP/zhD74up0ae1rh3715rwoQJ1ocffmgdPHjQysnJsQYOHGg9+eSTbu2aYg43btxo9ezZ01qzZo21f/9+a8aMGVbfvn0tl8tVY/udO3da3bt3tzIzM63CwkLrj3/8o9WzZ09r7969dpv6vCcbk6c1PvXUU9bKlSutr7/+2iosLLSeeeYZ64YbbrC+//57u820adOsMWPGuM3ViRMnGqskN57Wt3btWqtPnz5uYy8uLnZr05zm0NP6jh8/7lbbvn37rO7du1tr16612zSn+fvoo4+sBQsWWH/961+t6Oho64MPPqi1/aFDh6y4uDgrPT3dKiwstN58802re/fu1scff2y38fQ1awhCziVYu3ZtvUJOZWWl9etf/9pavny5vezUqVNWr169rL/85S+WZVlWYWGhFR0dbe3evdtus3XrVismJsbtQ6sxeGssv/3tb63f//73bsvq8+bwtYbW98ADD1jPP//8Rdd/9NFHVrdu3dw+iN966y2rT58+1g8//OCdwdeTt+Zw06ZNVs+ePa2ysjJ7WVPM4bBhw6y0tDT7cUVFheV0Oq1ly5bV2H7SpEnW2LFj3Zbde++91syZMy3Lqt97srF5WuPPlZeXWwkJCda7775rL5s2bZr1+OOPe3uoDeJpfXV9vja3ObzU+cvKyrISEhKsM2fO2Mua0/z9VH0+A1588UVr0KBBbssmT55sPfzww/bjS33N6oPDVY3gyJEjKi4uVv/+/e1loaGhiouL065duyRJu3btUps2bdx2x/bv318Oh8P7u+/q4I2x5Ofnq6CgQMOGDau2Li0tTf369dOwYcO0Zs0aWY18q6ZLqe+9995Tv379NHjwYP3hD3/QuXPn7HV5eXmKjo52uxml0+lUaWmpCgsLvV9ILbz191RaWqrWrVsrKMj9vqGNOYcXLlzQV1995fb+cTgc6t+/v/3++bm8vDwlJia6LXM6ncrLy5NUv/dkY2pIjT937tw5lZeXKywszG35Z599psTERN1+++2aNWuWjh8/7tWx10dD6zt79qxuvvlmJScn6/HHH9f+/fvtdc1pDr0xf2vXrtWgQYN05ZVXui1vDvPXEHW9B73xmtWHX9/x2F8UFxdLkiIiItyWR0REyOVySZJcLpfatm3rtj4oKEhhYWH29o3FG2NZs2aNunTpoj59+rgtnzhxon71q1+pZcuW+uSTT5SWlqazZ89q1KhRXht/XRpa3+DBg9WxY0e1a9dOe/fuVUZGhr799lstXrzY7vfnd9uueuyPc3js2DG98sorGj58uNvyxp7D48ePq6Kiosb3zzfffFPjNjXNxU/fb/V5TzamhtT4cxkZGWrXrp3bPxpJSUkaMGCAOnXqpMOHD2vBggV69NFH9fbbbyswMNCrNdSmIfVdd911mjdvnmJiYnT69Gm9/vrrGjFihDZu3Kj27ds3qzm81PnbvXu39u3bp//4j/9wW95c5q8hLvZ5WFpaqvPnz+vkyZOX/DdfH4Scf8jIyFBmZmatbTZt2qQuXbo00oi8r741Xqrz58/rL3/5i8aPH19t3RNPPGH/3qNHD507d04rVqzwyj+Qvq7vp//Yx8TEKCoqSqmpqTp06JCuueaaBvfricaaw9LSUo0bN05dunTRhAkT3Nb5cg7RMK+99po2bdqk7OxsXXHFFfbyQYMG2b9Xnbh622232XsHmrOEhAS3L1tOSEjQnXfeqdWrV2vy5MlNNzAfWLNmjaKjo9W7d2+35f48f80FIecfHn74YaWkpNTapnPnzg3qOyoqSpJUUlKidu3a2ctLSkrUrVs3ST8m3GPHjrltV15erpMnT9rbX6r61nipY3n//fd1/vx53X333XW2jYuL0yuvvKILFy4oJCSkzva1aaz6qsTFxUmSDh48qGuuuUaRkZHVDgVV/Y/Sn+awtLRUjzzyiFq1aqUlS5YoODi41vbenMOahIeHKzAwUCUlJW7LS0pKLvo9dZGRkdX+N//T9vV5TzamhtRYZcWKFXrttdeUlZVV59g7d+6s8PBwHTx4sFH/kbyU+qoEBwere/fuOnTokKTmNYeXUt/Zs2e1ceNGTZw4sc7naar5a4ia3oMul0utW7dWixYt5HA4Lvlvoj44J+cf2rZtqy5dutT609AP8E6dOikqKkq5ubn2stLSUn355Zf2/1QSEhJ06tQp5efn2222b9+uysrKaum+oepb46WOZe3atbrllluqHS6pSUFBgcLCwrzyj2Nj1ffTsUv//LCNj4/Xvn373N60OTk5at26tbp27XrJ9TVGjaWlpRozZoyCg4P16quvuu0VuBhvzmFNQkJC1LNnT7f3T2VlpXJzc93+p/9T8fHx2r59u9uynJwcxcfHS6rfe7IxNaRGScrMzNQrr7yi5cuXV7u8uibff/+9Tpw44bXQXV8Nre+nKioqtG/fPnvszWkOL6W+999/XxcuXNBdd91V5/M01fw1RF3vQW/8TdSL105hvoz87W9/s77++mv7Eumvv/7a+vrrr90ulb799tutv/71r/bjZcuWWX379rW2bNli7dmzx3r88cdrvIT87rvvtr788ktrx44d1sCBA5v0EvLaxvL9999bt99+u/Xll1+6bffdd99ZMTEx1tatW6v1+eGHH1rvvPOOtXfvXuu7776zVq1aZcXFxVkvvfSSz+v5OU/rO3jwoLV48WLrf//3f63Dhw9bW7ZssW699VZr5MiR9jZVl5A//PDDVkFBgfXxxx9bv/rVr5r0EnJPajx9+rR17733WoMHD7YOHjzodtlqeXm5ZVlNN4cbN260evXqZa1bt84qLCy0Zs6cafXt29e+ku3pp5+2MjIy7PY7d+60evToYa1YscIqLCy0Fi1aVOMl5HW9JxuTpzUuW7bM6tmzp/X++++7zVXV51Bpaak1f/58a9euXdbhw4etnJwcKyUlxRo4cGCjX+3XkPpefvlla9u2bdahQ4es/Px869///d+t2NhYa//+/Xab5jSHntZX5d/+7d+syZMnV1ve3OavtLTU/rcuOjraysrKsr7++mvrb3/7m2VZlpWRkWE9/fTTdvuqS8hfeOEFq7Cw0Fq5cmWNl5DX9pp5A4erGmDRokV699137cdVh2Wys7PVr18/SdK3337rdhO1Rx99VOfOndNzzz2nU6dO6YYbbtDy5cvd/qeckZGhuXPn6qGHHpLD4dDAgQM1Y8aMxinqZ+oaS1lZmb799lu3q4ukH/fitG/fXk6ns1qfQUFBWrVqlebNmydJuuaaa/TMM8/ovvvu820xNfC0vuDgYOXm5io7O1tnz55Vhw4dNHDgQLfzjgIDA7V06VLNnj1bw4cPV8uWLZWSklKv3dC+4GmNX331lb788ktJ0oABA9z6+vDDD9WpU6cmm8M777xTx44d06JFi1RcXKzu3btr+fLl9m7tv//973I4/rljuk+fPsrIyNDChQu1YMECXXvttVqyZImio6PtNvV5TzYmT2tcvXq1ysrKqv19TZgwQU8++aQCAwO1b98+rV+/XqdPn1a7du3061//WpMmTfLZXrfaeFrfqVOnNHPmTBUXFyssLEw9e/bU6tWr3faKNqc59LQ+Sfrmm2+0c+dOvf7669X6a27zl5+f73beXXp6uiQpJSVF8+fPV3Fxsf7+97/b6zt37qxly5YpPT1d2dnZat++vZ5//nklJSXZbep6zbwhwLIa+fpdAACARsA5OQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAw0v8HH2o/IsWOjt0AAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Axes(0.125,0.11;0.775x0.77)\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAArR0lEQVR4nO3deXRUZZ7G8acqCyCEEJIAIsxRgiyShATjYcgUTbeCOKbxTLBtGGxZhmFTGtpm0UF2aEIzaQcYaIOBjhOkjYwijQP0tNoueFgGGGKaRYaAjLFxIBWWEGTJcucPJjWWQJKCW1WpN9/POTnH3Hrr3vf+bop6fO/7Vjksy7IEAABgGGewOwAAAOAPhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYKTwYHegMSgruyg7v9zC4ZBiY6Ns3y+8UefAodaBQZ0DgzoHhj/rXLvv+hByJFmW/PKH7q/9wht1DhxqHRjUOTCoc2AEs87crgIAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJL6F3I/CwurOkDU1lmpq+ApcAAD8gZDjB06nQ9U1lmJiWtbZrqq6RhfOf0PQAQDADwg5fuBwOBTmdGhqwQEVn6m4aZuu7VppxfBUOZ0OQg4AAH7g85ycvXv3auLEiXK5XOrevbvef/99r8cty9KKFSvkcrmUnJys0aNH6+TJk15tzp8/r2nTpqlPnz5KS0vTrFmzdOnSJa82n3/+uUaMGKGkpCQNGDBAubm5N/Rl+/bteuyxx5SUlKQhQ4bo448/9vV0/Kr4TIUOnSq/6c+twg8AALCHzyHnm2++Uffu3TVv3rybPp6bm6v169dr/vz52rhxo1q0aKGxY8fq6tWrnjbTp09XcXGx8vLylJOTo3379mnu3LmexysqKjR27Fh17NhRmzZt0syZM7Vq1Sq9+eabnjb/+Z//qWnTpulHP/qRNm/erEceeUTPPfec/uu//svXUwIAAAbyOeQMGDBAzz//vAYNGnTDY5ZlKT8/X5MmTdLAgQPVo0cPLVu2TGfOnPGM+Bw/flw7duzQ4sWL1bt3b6WlpWn27NnaunWrTp8+LUnasmWLKisrtWTJEt1///3KyMjQM888o7y8PM+x8vPz1b9/f/393/+9EhIS9LOf/UwPPPCAXn/99dutBQAAMIitc3K++uorlZaWKj093bMtKipKvXv31oEDB5SRkaEDBw6odevWSkpK8rRJT0+X0+lUUVGRBg0apMLCQqWlpSkyMtLTxuVyKTc3VxcuXFB0dLQKCws1evRor+O7XK4bbp81hMPh+7nauT+7j99U1NaN+vkftQ4M6hwY1Dkw/Fnnhu7T1pBTWloqSYqNjfXaHhsbK7fbLUlyu91q27atdyfCwxUdHe15vtvtVqdOnbzaxMXFeR6Ljo6W2+32bLvZcXwRGxvl83PsUt8KLNQvmNevqaHWgUGdA4M6B0Yw68zqKkllZRdl2bjAKTzcqTZtGhZezp27pOrqGvsO3oQ4HNdfPHZfP9yIWgcGdQ4M6hwY/qxz7b7rY2vIiY+PlySVlZWpXbt2nu1lZWXq0aOHpOsjMmfPnvV6XlVVlS5cuOB5flxc3A0jMrW/147e3KxNWVnZDaM7DWFZsvUC+LovXmR3xu7rh1uj1oFBnQODOgdGMOts69c6dOrUSfHx8dq1a5dnW0VFhT777DOlpqZKklJTU1VeXq6DBw962uzevVs1NTVKTk6WJKWkpGjfvn2qrKz0tNm5c6fuu+8+RUdHe9rs3r3b6/g7d+5USkqKnacEAABClM8h59KlSzpy5IiOHDki6fpk4yNHjujUqVNyOBwaOXKkXnnlFX3wwQc6evSoZs6cqXbt2mngwIGSpISEBPXv319z5sxRUVGR9u/fr0WLFikjI0Pt27eXJA0ZMkQRERF66aWXdOzYMW3btk35+fkaM2aMpx8jR47Ujh079Jvf/EbHjx/XP//zP+vgwYP6yU9+YkddAABAiPP5dtXBgwc1cuRIz+9ZWVmSpMzMTC1dulTjxo3T5cuXNXfuXJWXl+vBBx/U2rVr1axZM89zsrOztWjRIo0aNUpOp1OPPvqoZs+e7Xk8KipK69at08KFCzV06FDFxMTo2Wef1bBhwzxt+vTpo+zsbC1fvlwvv/yy7r33Xq1evVrdunW7rUIAAACzOCyLO5Jut/0Tj2NiWipj5Q4dOlV+0za9OrbW1in9de7cJVVVMfH4djgcUlxclO3XDzei1oFBnQODOgeGP+tcu+/62DonBwAAoLEg5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAI9kecqqrq7V8+XI9/PDDSk5O1sCBA7V69WpZluVpY1mWVqxYIZfLpeTkZI0ePVonT5702s/58+c1bdo09enTR2lpaZo1a5YuXbrk1ebzzz/XiBEjlJSUpAEDBig3N9fu0wEAACHK9pCTm5urN954Q3PnztW2bds0ffp0rV27VuvXr/dqs379es2fP18bN25UixYtNHbsWF29etXTZvr06SouLlZeXp5ycnK0b98+zZ071/N4RUWFxo4dq44dO2rTpk2aOXOmVq1apTfffNPuUwIAACHI9pBz4MABPfLII/r+97+vTp066bHHHpPL5VJRUZGk66M4+fn5mjRpkgYOHKgePXpo2bJlOnPmjN5//31J0vHjx7Vjxw4tXrxYvXv3VlpammbPnq2tW7fq9OnTkqQtW7aosrJSS5Ys0f3336+MjAw988wzysvLs/uUAABACLI95KSmpmr37t364osvJF2/pbR//35973vfkyR99dVXKi0tVXp6uuc5UVFR6t27tw4cOCDpelBq3bq1kpKSPG3S09PldDo9YamwsFBpaWmKjIz0tHG5XPriiy904cIFn/rscNj/E+zjN5Uf6ketTfuhztTZpB9/1rkhwn17O67f+PHjVVFRob/+679WWFiYqqur9fzzz+uJJ56QJJWWlkqSYmNjvZ4XGxsrt9stSXK73Wrbtq13R8PDFR0d7Xm+2+1Wp06dvNrExcV5HouOjm5wn2Njo3w4Q3vFxLQM2rFNEczr19RQ68CgzoFBnQMjmHW2PeRs375d7777rn71q1+pa9euOnLkiLKystSuXTtlZmbafThblJVd1LfmRd+x8HCn2rRpWHg5d+6Sqqtr7Dt4E+JwXH/x2H39cCNqHRjUOTCoc2D4s861+66P7SFn2bJlGj9+vDIyMiRJ3bt316lTp7RmzRplZmYqPj5eklRWVqZ27dp5nldWVqYePXpIuj4ic/bsWa/9VlVV6cKFC57nx8XFeUZ+atX+Xjui01CWJVsvgK/74kV2Z+y+frg1ah0Y1DkwqHNgBLPOts/JuXLlihzfuVkWFhbmWULeqVMnxcfHa9euXZ7HKyoq9Nlnnyk1NVXS9Xk95eXlOnjwoKfN7t27VVNTo+TkZElSSkqK9u3bp8rKSk+bnTt36r777vPpVhUAADCT7SHnBz/4gXJycvTRRx/pq6++0nvvvae8vDwNHDhQkuRwODRy5Ei98sor+uCDD3T06FHNnDlT7dq187RJSEhQ//79NWfOHBUVFWn//v1atGiRMjIy1L59e0nSkCFDFBERoZdeeknHjh3Ttm3blJ+frzFjxth9SgAAIATZfrtq9uzZWrFihRYsWOC5JTVs2DA999xznjbjxo3T5cuXNXfuXJWXl+vBBx/U2rVr1axZM0+b7OxsLVq0SKNGjZLT6dSjjz6q2bNnex6PiorSunXrtHDhQg0dOlQxMTF69tlnNWzYMLtPCQAAhCCHZXFH0u22f+JxTExLZazcoUOnym/aplfH1to6pb/OnbukqiomHt8Oh0OKi4uy/frhRtQ6MKhzYFDnwPBnnWv3XR++uwoAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjOSXkHP69GlNnz5dffv2VXJysoYMGaI//elPnscty9KKFSvkcrmUnJys0aNH6+TJk177OH/+vKZNm6Y+ffooLS1Ns2bN0qVLl7zafP755xoxYoSSkpI0YMAA5ebm+uN0AABACLI95Fy4cEF/+7d/q4iICOXm5mrr1q164YUXFB0d7WmTm5ur9evXa/78+dq4caNatGihsWPH6urVq54206dPV3FxsfLy8pSTk6N9+/Zp7ty5nscrKio0duxYdezYUZs2bdLMmTO1atUqvfnmm3afEgAACEHhdu8wNzdXHTp0UFZWlmdb586dPf9tWZby8/M1adIkDRw4UJK0bNkypaen6/3331dGRoaOHz+uHTt26K233lJSUpIkafbs2Ro/frxmzpyp9u3ba8uWLaqsrNSSJUsUGRmp+++/X0eOHFFeXp6GDRtm92kBAIAQY3vI+eMf/yiXy6UpU6Zo7969at++vUaMGKEf//jHkqSvvvpKpaWlSk9P9zwnKipKvXv31oEDB5SRkaEDBw6odevWnoAjSenp6XI6nSoqKtKgQYNUWFiotLQ0RUZGetq4XC7l5ubqwoULXiNH9XE4bDjxO9if3cdvKmrrRv38j1oHBnUODOocGP6sc0P3aXvIKSkp0RtvvKExY8Zo4sSJ+tOf/qTFixcrIiJCmZmZKi0tlSTFxsZ6PS82NlZut1uS5Ha71bZtW++OhocrOjra83y3261OnTp5tYmLi/M85kvIiY2N8u0kbRQT0zJoxzZFMK9fU0OtA4M6BwZ1Doxg1tn2kGNZlhITE/Xzn/9ckvTAAw/o2LFjKigoUGZmpt2Hs0VZ2UVZln37Cw93qk2bhoWXc+cuqbq6xr6DNyEOx/UXj93XDzei1oFBnQODOgeGP+tcu+/62B5y4uPjlZCQ4LWtS5cu+vd//3fP45JUVlamdu3aedqUlZWpR48ekq6PyJw9e9ZrH1VVVbpw4YLn+XFxcZ6Rn1q1v9eO6DSUZcnWC+DrvniR3Rm7rx9ujVoHBnUODOocGMGss+2rq/r06aMvvvjCa9vJkyd1zz33SJI6deqk+Ph47dq1y/N4RUWFPvvsM6WmpkqSUlNTVV5eroMHD3ra7N69WzU1NUpOTpYkpaSkaN++faqsrPS02blzp+677z6fblUBAAAz2R5yRo0apc8++0w5OTn67//+b7377rvauHGjRowYIUlyOBwaOXKkXnnlFX3wwQc6evSoZs6cqXbt2nlWWyUkJKh///6aM2eOioqKtH//fi1atEgZGRlq3769JGnIkCGKiIjQSy+9pGPHjmnbtm3Kz8/XmDFj7D4lAAAQgmy/XZWcnKxVq1bp5Zdf1urVq9WpUyfNmjVLTzzxhKfNuHHjdPnyZc2dO1fl5eV68MEHtXbtWjVr1szTJjs7W4sWLdKoUaPkdDr16KOPavbs2Z7Ho6KitG7dOi1cuFBDhw5VTEyMnn32WZaPAwAASZLDsrgj6XbbP/E4JqalMlbu0KFT5Tdt06tja22d0l/nzl1SVRUTj2+HwyHFxUXZfv1wI2odGNQ5MKhzYPizzrX7rg/fXQUAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADBSeLA7AAAAQo/T6ZDT6Qh2N+pEyAEAAD5xOh2KbnOXwsPqviFUXWPJ6XSoutoKUM+8EXIAAIBPnE6HwsOcmlpwQMVnKm7apmu7VloxPFUOh0MSIQcAAISQ4jMVOnSqPNjduCUmHgMAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjOT3kPPqq6+qe/fu+sUvfuHZdvXqVS1YsEB9+/ZVamqqfvrTn8rtdns979SpUxo/frx69+6tfv366Ze//KWqqqq82uzZs0eZmZlKTEzUoEGDtGnTJn+fDgAACBF+DTlFRUUqKChQ9+7dvbYvWbJEH374oZYvX67169frzJkzmjx5sufx6upqTZgwQZWVlSooKNDSpUv1zjvvaOXKlZ42JSUlmjBhgvr27avf/e53GjVqlGbPnq0dO3b485QAAECI8FvIuXTpkmbMmKHFixcrOjras/3ixYt6++239eKLL6pfv35KTEzUkiVLdODAARUWFkqSPv30UxUXF+sf//Ef1bNnTw0YMEBTp07Vhg0bdO3aNUlSQUGBOnXqpBdffFEJCQn6yU9+osGDB+u1117z1ykBAIAQ4reQs3DhQg0YMEDp6ele2w8ePKjKykqv7QkJCerYsaMn5BQWFqpbt26Ki4vztHG5XKqoqFBxcbGnTb9+/bz27XK5PPsAAABNW7g/drp161YdPnxYb7311g2Pud1uRUREqHXr1l7bY2NjVVpa6mnz7YAjyfN7fW0qKip05coVNW/evMH9dTga3NQv+7P7+E1Fbd2on/9R68CgzoFBnQPL4Qje+6ztIefrr7/WL37xC/3mN79Rs2bN7N69X8TGRgXt2DExLYN2bFME8/o1NdQ6MKhzYFDnwGjTJnjvc7aHnEOHDqmsrExDhw71bKuurtbevXu1YcMGrVu3TpWVlSovL/cazSkrK1N8fLyk6yMyRUVFXvutXX317TbfXZHldrvVqlUrn0Zxrh/7oizLp6fUKTzc2eCLeu7cJVVX19h38CbE4bj+j5Td1w83otaBQZ0DgzrfubAwZ4P/J/38+UuqqrL3fa72GtbH9pDzl3/5l3r33Xe9tv3DP/yDunTponHjxunuu+9WRESEdu3apcGDB0uSTpw4oVOnTiklJUWSlJKSopycHJWVlSk2NlaStHPnTrVq1Updu3b1tPnkk0+8jrNz507PPnxhWbL1D93XffEiuzN2Xz/cGrUODOocGNQ5MIJZZ9tDTqtWrdStWzevbXfddZfatGnj2f7kk09q6dKlio6OVqtWrbR48WKlpqZ6AorL5VLXrl01c+ZMzZgxQ6WlpVq+fLmefvppRUZGSpKGDx+uDRs2aNmyZXryySe1e/dubd++XWvWrLH7lAAAQAjyy8Tj+syaNUtOp1NTpkzRtWvX5HK5NG/ePM/jYWFhysnJ0fz58zVs2DC1aNFCmZmZmjJliqdN586dtWbNGmVlZSk/P18dOnTQ4sWL1b9//2CcEgAAaGQCEnLWr1/v9XuzZs00b948r2DzXffcc49yc3Pr3G/fvn21efNmO7oIAAAMw3dXAQAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASOHB7gDunNPpkNPpqLNNTY2lmhorQD0CACD4CDmNXH0BxuFwqFVUc4WH1T0oV1VdowvnvyHoAACaDEJOI+Z0OhTd5q56A4wkTS04oOIzFTd9rGu7VloxPFVOp4OQAwBoMgg5jZjT6VB4mLPOAPP97vGaMbiHis9U6NCp8gD3EACAxouQEwLqCjAJ8S0D3BsAAEIDq6sAAICRGMlpQsLqmdvDCiwAgEkIOU1AfKtmqq6x1Lp1izrbsQILAGASQk4T0LpFuMKcDlZgAQCaFEJOE8IKLABAU8LEYwAAYCRGcuCFyckAAFMQciCJyckAAPMQciCJyckAAPMQcuCFyckAAFMw8RgAABiJkRz4jMnJAIBQQMhBgzE5GQAQSgg5aDAmJwMAQgkhBz5jcjIAIBQQcuAXzNsBAAQbIQe28mXeTsXFK7KsWwcdghAA4E4QcmCrhszbeejeGM35YS+1aXNXnftiAjMA4E4QcuAXdc3bSYhvyQRmAIDfEXIQNExgBgD4E594DAAAjMRIDhq1+lZpAQBwK4ScIKvrTbwpv8E3dJVWdY0lp9Oh6mrm7QAAvBFygqShb+JNlS+fruxwOCQRcgAA3gg5QdKQN/Hvd4/XjME9AtyzxoXJyQCA20XICbL6llqjfny6MgDgZgg5CFl8KzoAoC62h5w1a9boD3/4g06cOKHmzZsrNTVV06dPV5cuXTxtrl69qqVLl2rbtm26du2aXC6X5s2bp7i4OE+bU6dOaf78+dqzZ4/uuusu/c3f/I2mTZum8PD/7/KePXu0dOlSHTt2THfffbcmTZqkoUOH2n1KaKT4VnQAQF1sX77zH//xH3r66ae1ceNG5eXlqaqqSmPHjtU333zjabNkyRJ9+OGHWr58udavX68zZ85o8uTJnserq6s1YcIEVVZWqqCgQEuXLtU777yjlStXetqUlJRowoQJ6tu3r373u99p1KhRmj17tnbs2GH3KaGRq73ld7OfW4UfAID5bB/JWbdundfvS5cuVb9+/XTo0CE99NBDunjxot5++21lZ2erX79+kq6Hnscff1yFhYVKSUnRp59+quLiYuXl5SkuLk49e/bU1KlTlZ2drcmTJysyMlIFBQXq1KmTXnzxRUlSQkKC9u/fr9dee039+/e3+7QAAECI8fsHsVy8eFGSFB0dLUk6ePCgKisrlZ6e7mmTkJCgjh07qrCwUJJUWFiobt26ed2+crlcqqioUHFxsadNbUj6dpvaffjC4bD/B42PP65zU/qhhtTZpB/qfOf1C/a/vQ3h14nHNTU1WrJkifr06aNu3bpJktxutyIiItS6dWuvtrGxsSotLfW0+XbAkeT5vb42FRUVunLlipo3b97gfsbGRvl2Ygg5MTGsVLMDr5XAoM6BQZ0Do02b4P3769eQs2DBAh07dky//e1v/XmYO1ZWdlGWjXNSw8OdQb2o+H+1K7DCnHXH/qrqGpVfYAXWrTgc198Q7H6twBt1DgzqfOfCwpwN/p/H8+cvqaqqxtbj117D+vgt5CxcuFAfffSRXn/9dXXo0MGzPS4uTpWVlSovL/cazSkrK1N8fLynTVFRkdf+3G63JHm1qd327TatWrXyaRRHkixLtv6h86JpPHz95GSLi1cnu18ruDnqHBjUOTCCWWfbQ45lWVq0aJHee+89rV+/Xp07d/Z6PDExUREREdq1a5cGDx4sSTpx4oROnTqllJQUSVJKSopycnJUVlam2NhYSdLOnTvVqlUrde3a1dPmk08+8dr3zp07PfsAvo1PTgaApsf2iccLFizQli1b9Ktf/UotW7ZUaWmpSktLdeXKFUlSVFSUnnzySS1dulS7d+/WwYMHNWvWLKWmpnoCisvlUteuXTVz5kx9/vnn2rFjh5YvX66nn35akZGRkqThw4erpKREy5Yt0/Hjx7VhwwZt375do0ePtvuUAABACLJ9JOeNN96QJD3zzDNe27Oysjwf1Ddr1iw5nU5NmTLF68MAa4WFhSknJ0fz58/XsGHD1KJFC2VmZmrKlCmeNp07d9aaNWuUlZWl/Px8dejQQYsXL2b5OG4bXw8BAGaxPeQcPXq03jbNmjXTvHnzvILNd91zzz3Kzc2tcz99+/bV5s2bfe0i4IWvhwAAM/HdVWjy+HoIADATIQf4P0xOBgCzEHIAHzBvBwBCByEHaADm7QBA6CHkAA3gy7ydiIgwVVff+tM9Ge0BgMAg5AA+qGveDqM9ANC4EHIAm7BKCwAaF0IOYDNWaQFA42D71zoAAAA0BozkAEHAUnQA8D9CDhBATE4GgMAh5AABZOfkZKfTIafTUefxGBEC0JQRcoAgaMjk5LpuaTkcDrWKaq7wem57MSIEoCkj5ACNTENvaUliuToA1IGQAzQyDbml9f3u8ZoxuAfL1QGgDoQcoJGqK8AkxLcMcG8AIPTwOTkAAMBIhBwAAGAkQg4AADASc3IAw/HpygCaKkIOYCg+XRlAU0fIAQzly6crR0SEqbq6JsA9BAD/IuQAhqtrKbovHzxYXWPJ6XSoupoRHwChgZADNGENGe2R/n/EJzw8TA7HrUd8mN8DoDEh5ACo95OTmd8DIBQRcgDUy85vTweAQCHkAGgwvisLQCgh5ACwVX2fy9MQzO0BYAdCDgBbNHTeTnWNpTCno842zO0BYAdCDgBbNGTezve7x2vG4B7M7QEQEIQcALaqa95OQnzLetvU4usoANwpQg6ARoXl6gDsQsgB0KgE+usoGBECzEXIAdAo2fF1FExyBpo2Qg6AkMMkZwANQcgBELLsmuQMwEyEHABNHiu5ADMRcgA0WazkAsxGyAHQZNm5kovRHqDxIeQAaPLsWMlVVV2jiotXZFkEHaCxIOQAQB0aMtrz0L0xmvPDXmrT5q4699WQJe3VNZacToeqqwlLwJ0i5ABAA9S3ksvOJe3h4WFyOLg1BtwpQg4A2OROl7QzERqwFyEHABoJJkID9iLkAEAjE6iJ0AQhmI6QAwAhxM6J0HatCCMsobEi5ABACLrTidB2rghjjhAaK0IOABgqkCvCmCOExoiQAwBNWCBXhDFHCIFGyAEA3Da75whx26txcDodctZxm7K+L7VtLAg5AIA7dqe3xhp626shGjoixLfP35zT6VB0m7sUHiJBpi6EHABAQNhx26uhE6HrujXmcDhUXWMpJqblHe3HTg0JVPWNrtglLMyp8DBng+ZrNXaEHABA0DXktldDJkI39NaYpIB+H9mdrlBr6OiKXf2RGjZfq7Ej5AAAGo07nQjty6qxQKw+s2uFmi+jK3a1MQEhBwBgHLtGIewIXfW1aeitukD1J1RGaRqCkAMAQBD5cqsOviHkAADQCDSV0ZVACv31YQAAADcR8iFnw4YNevjhh5WUlKSnnnpKRUVFwe4SAABoBEI65Gzbtk1ZWVl67rnn9M4776hHjx4aO3asysrKgt01AAAQZCEdcvLy8vTjH/9YTz75pLp27aoFCxaoefPmevvtt4PdNQAAEGQhO/H42rVrOnTokCZMmODZ5nQ6lZ6ergMHDvi0L6dTsvMDLR3/9xlLvTq2VovIsJu2SYhvRRvahESbxtgn2tCGNo2/TZe465OlHY7r77N2cjTwg58dViA+r9oPTp8+re9973sqKChQamqqZ/uyZcu0d+9e/eu//msQewcAAIItpG9XAQAA3ErIhpyYmBiFhYXdMMm4rKxMcXFxQeoVAABoLEI25ERGRqpXr17atWuXZ1tNTY127drldfsKAAA0TSE78ViSxowZoxdeeEGJiYlKTk7Wv/zLv+jy5csaOnRosLsGAACCLKRDzuOPP66zZ89q5cqVKi0tVc+ePbV27VpuVwEAgNBdXQUAAFCXkJ2TAwAAUBdCDgAAMBIhBwAAGImQAwAAjETIuU0bNmzQww8/rKSkJD311FMqKiqqs/327dv12GOPKSkpSUOGDNHHH38coJ6GNl/qvHHjRo0YMUIPPfSQHnroIY0ePbre64LrfP17rrV161Z1795dzz77rJ97aA5fa11eXq4FCxbI5XIpMTFRgwcP5t+PBvC1zq+99poGDx6s5ORkDRgwQEuWLNHVq1cD1NvQtHfvXk2cOFEul0vdu3fX+++/X+9z9uzZo8zMTCUmJmrQoEHatGmTfztpwWdbt261evXqZb311lvWsWPHrNmzZ1tpaWmW2+2+afv9+/dbPXv2tHJzc63i4mLrn/7pn6xevXpZR48eDXDPQ4uvdf75z39uvf7669bhw4et4uJi68UXX7QefPBB63/+538C3PPQ4muda5WUlFj9+/e3RowYYU2aNClAvQ1tvtb66tWr1tChQ61x48ZZ+/bts0pKSqw9e/ZYR44cCXDPQ4uvdd6yZYuVmJhobdmyxSopKbF27Nhh/dVf/ZW1ZMmSAPc8tHz00UfWyy+/bP3hD3+wunXrZr333nt1tv/yyy+t3r17W1lZWVZxcbG1fv16q2fPntYnn3zitz4Scm7Dj370I2vBggWe36urqy2Xy2WtWbPmpu2nTp1qjR8/3mvbU089Zc2ZM8ev/Qx1vtb5u6qqqqzU1FTrnXfe8VMPzXA7da6qqrKGDRtmbdy40XrhhRcIOQ3ka61/+9vfWo888oh17dq1QHXRCL7WecGCBdbIkSO9tmVlZVnDhw/3az9N0pCQs2zZMisjI8Nr289+9jPr7/7u7/zWL25X+ejatWs6dOiQ0tPTPducTqfS09N14MCBmz6nsLBQ/fr189rmcrlUWFjoz66GtNup83ddvnxZVVVVio6O9lc3Q97t1nn16tWKjY3VU089FYhuGuF2av3HP/5RKSkpWrhwodLT0/XDH/5QOTk5qq6uDlS3Q87t1Dk1NVWHDh3y3NIqKSnRxx9/rAEDBgSkz01FMN4LQ/oTj4Ph3Llzqq6uVmxsrNf22NhYnThx4qbPcbvdN3wKc2xsrNxut9/6Gepup87flZ2drXbt2nn9Ywdvt1Pnffv26a233tLmzZsD0ENz3E6tS0pKtHv3bg0ZMkSvvvqqvvzySy1YsEBVVVWaPHlyILodcm6nzkOGDNG5c+c0YsQIWZalqqoqDR8+XBMnTgxEl5uMm70XxsXFqaKiQleuXFHz5s1tPyYjOTDSq6++qm3btmnVqlVq1qxZsLtjjIqKCs2cOVOLFi1S27Ztg90d41mWpdjYWC1atEiJiYl6/PHHNXHiRBUUFAS7a0bZs2eP1qxZo3nz5mnTpk1atWqVPv74Y61evTrYXcMdYiTHRzExMQoLC1NZWZnX9rKyslt+Z1ZcXNwNozZ1tcft1bnWunXr9OqrryovL089evTwZzdDnq91Likp0Z///GdNmjTJs62mpkaS9MADD+j3v/+9/uIv/sK/nQ5Rt/M3HR8fr/DwcIWFhXm2denSRaWlpbp27ZoiIyP92udQdDt1XrFihZ544gnP7dfu3bvrm2++0dy5czVp0iQ5nYwH2OFm74Vut1utWrXyyyiOxEiOzyIjI9WrVy/t2rXLs62mpka7du1SamrqTZ+TkpKi3bt3e23buXOnUlJS/NnVkHY7dZak3Nxc/frXv9batWuVlJQUiK6GNF/r3KVLF7377rvavHmz5+fhhx9W3759tXnzZnXo0CGQ3Q8pt/M33adPH3355ZeeIClJJ0+eVHx8PAHnFm6nzleuXLkhyNQGS4uvd7RNUN4L/Tal2WBbt261EhMTrU2bNlnFxcXWnDlzrLS0NKu0tNSyLMuaMWOGlZ2d7Wm/f/9+64EHHrDWrVtnFRcXWytXrmQJeQP4Wuc1a9ZYvXr1sn7/+99bZ86c8fxUVFQE6xRCgq91/i5WVzWcr7U+deqUlZqaai1cuNA6ceKE9eGHH1r9+vWzfv3rXwfrFEKCr3VeuXKllZqaav3bv/2b9eWXX1qffvqpNXDgQGvq1KlBOoPQUFFRYR0+fNg6fPiw1a1bNysvL886fPiw9ec//9myLMvKzs62ZsyY4Wlfu4T8l7/8pVVcXGy9/vrrfl9Czu2q2/D444/r7NmzWrlypUpLS9WzZ0+tXbvWMxT69ddfe/1fQZ8+fZSdna3ly5fr5Zdf1r333qvVq1erW7duwTqFkOBrnQsKClRZWakpU6Z47Wfy5Mn66U9/GtC+hxJf64zb52ut7777bq1bt05ZWVl64okn1L59e40cOVLjxo0L1imEBF/rPGnSJDkcDi1fvlynT59W27Zt9YMf/EDPP/98sE4hJBw8eFAjR470/J6VlSVJyszM1NKlS1VaWqqvv/7a83jnzp21Zs0aZWVlKT8/Xx06dNDixYvVv39/v/XRYVmMxQEAAPPwv2cAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGOl/Afll2nSo/BpoAAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "df[df[\"preference\"]][\"cer_diff_preference\"].hist(bins=50)\n", "print(df[df[\"preference\"]][\"cer_diff_preference\"].quantile(0.95))\n", "plt.show()\n", "print(df[df[\"preference\"]][\"cer\"].hist(bins=50))\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:40.799375Z", "start_time": "2024-05-16T13:59:36.394236Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:05:10.728412Z", "iopub.status.busy": "2024-10-05T17:05:10.728262Z", "iopub.status.idle": "2024-10-05T17:05:11.917057Z", "shell.execute_reply": "2024-10-05T17:05:11.916517Z", "shell.execute_reply.started": "2024-10-05T17:05:10.728398Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "11391\n", "good_continue_at\n", "True 139728\n", "False 424\n", "Name: count, dtype: int64\n", "\n", " Check some basics... \n", " preference\n", "False 70076\n", "True 70076\n", "Name: count, dtype: int64 is_30b\n", "True 140152\n", "Name: count, dtype: int64 model_name\n", "chirp-v3p5-engine-t-3 140152\n", "Name: count, dtype: int64 preference model_name \n", "False chirp-v3p5-engine-t-3 70076\n", "True chirp-v3p5-engine-t-3 70076\n", "Name: count, dtype: int64\n", "task\n", " 113536\n", "extend 26490\n", "cover 114\n", "infill 12\n", "Name: count, dtype: int64\n" ] } ], "source": [ "df[\"id\"] = df[\"str_id\"]\n", "# get the original duration of the clips, if they are concacted\n", "df[\"original_duration_s\"] = df[\"total_start_s\"] + df[\"duration\"]\n", "# classify the continue at behavoirs by the duration choice\n", "audio_prompt_id_to_continue_at = {}\n", "for _, row in df[~df[\"continued_parent\"].isna()].iterrows():\n", " audio_prompt_id = row[\"continued_parent\"]\n", " if audio_prompt_id not in audio_prompt_id_to_continue_at:\n", " audio_prompt_id_to_continue_at[audio_prompt_id] = row[\"continue_at\"]\n", " else:\n", " # pick the max\n", " audio_prompt_id = max(\n", " audio_prompt_id_to_continue_at[audio_prompt_id], row[\"continue_at\"]\n", " )\n", "print(len(audio_prompt_id_to_continue_at))\n", "df[\"has_continue_and_start_continue_at\"] = df[\"id\"].apply(\n", " lambda x: audio_prompt_id_to_continue_at.get(x)\n", ")\n", "# we want continue at to be at most of the clip...\n", "df[\"good_continue_at\"] = (\n", " (df[\"has_continue_and_start_continue_at\"] / df[\"duration\"]) > 0.9\n", ") | df[\"has_continue_and_start_continue_at\"].isna()\n", "print(df[\"good_continue_at\"].value_counts())\n", "\n", "\n", "print(\n", " \"\\n Check some basics... \\n\",\n", " df[\"preference\"].value_counts(),\n", " df[\"is_30b\"].value_counts(),\n", " df[\"model_name\"].value_counts(),\n", " df.groupby([\"preference\"])[\"model_name\"].value_counts(),\n", ")\n", "\n", "df = df.sort_values(by=[\"request_id\", \"preference\"])\n", "df[\"duration_rel_diff\"] = df[\"duration\"].diff()\n", "df[\"play_rel_diff\"] = df[\"reaction_play_count\"].diff()\n", "print(df[\"task\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.035167Z", "start_time": "2024-05-16T13:59:40.801098Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:05:11.917934Z", "iopub.status.busy": "2024-10-05T17:05:11.917777Z", "iopub.status.idle": "2024-10-05T17:05:12.148641Z", "shell.execute_reply": "2024-10-05T17:05:12.148093Z", "shell.execute_reply.started": "2024-10-05T17:05:11.917919Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "negative 69367 positive 10604\n", "total pair requests 70076 --> selected pair requests 10252 frac 0.146\n" ] } ], "source": [ "normal_pos_play_count = 3\n", "# this is lower, cause a concat is probably already ensuring that it is good\n", "concat_pos_play_count = 1\n", "# this is a filter on the concated clip\n", "concat_total_play_count = 3\n", "\n", "neg_filter_selection_mask = (\n", " (~df[\"preference\"]) # get basics aligned\n", " & (df[\"reaction_play_count\"] >= 1) # has to be played once\n", " # & (df[\"play_count\"] <= 3) # if it is actually bad, shouldn't be listened often\n", " & (df[\"duration\"] >= 10) # can't be too short, otherwise it is obvious\n", " & (df[\"duration\"] <= 240) # can't be badly long\n", " & (df[\"has_continue_and_start_continue_at\"].isna()) # won't have any continues\n", " & (df[\"upvote_count\"] == 0) # won't have any likes\n", " & ((df[\"norm_play_frac\"] <= 3.1))\n", " # & (df[\"dislike_count\"] >= 1) # this is kinda strict\n", " # & (\n", " # (df_slice[\"is_in_playlist\"] == False)\n", " # & (df_slice[\"concat_in_playlist\"] == False)\n", " # ) # can't be part of a playlist -- otherwise there are some like signal in it?\n", ")\n", "pos_filter_selectin_mask = (\n", " (df[\"preference\"]) # get basics aligned\n", " & (\n", " df[\"good_continue_at\"]\n", " ) # if continue, needs to continue off a certain percentage\n", " & (df[\"reaction_play_count\"] >= 1)\n", " & (df[\"play_rel_diff\"] >= 0) # this is more like quality assurance\n", " & (df[\"duration\"] >= 10) # can't be too short, otherwise it is obvious\n", " & (df[\"duration\"] <= 240) # can't be badly long\n", " & (df[\"dislike_count\"] == 0) # can't have dislikes\n", " & (df[\"flag_count\"] == 0) # can't have issues\n", " & (\n", " (\n", " (df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= concat_pos_play_count)\n", " & (df[\"concat_play_counts\"] >= concat_total_play_count)\n", " )\n", " | (\n", " (~df[\"part_of_concat\"])\n", " & (df[\"reaction_play_count\"] >= normal_pos_play_count)\n", " # & (df[\"norm_play_frac\"] >= 1.5)\n", " )\n", " \n", " )\n", " # & (df[\"norm_play_frac\"] >= 1.9)\n", " & (df[\"user_n_clips\"] >= 40) # user needs to have genereated at least 20\n", " # & (df[\"duration_rel_diff\"] < 10) # positive isn't just longer\n", " & ((df[\"upvote_count\"] >= 1) | (df[\"reaction_play_count\"] >= 3)| (df[\"concat_play_counts\"] >= 3))\n", " & (df[\"pos_diff_preference\"] == 2)\n", " & ((df[\"task\"] != \"infill\") & (df[\"task\"] != \"cover\"))\n", " & ((df[\"cer_diff_preference\"] < 0.5) & (df[\"cer\"] < 0.95)) # cut on hoot cer difference and abs cer\n", ")\n", "print(\n", " \"negative\",\n", " sum(neg_filter_selection_mask),\n", " \"positive\",\n", " sum(pos_filter_selectin_mask),\n", ")\n", "\n", "neg_filter_requests = df[neg_filter_selection_mask][\"request_id\"].unique()\n", "pos_filter_requests = df[pos_filter_selectin_mask][\"request_id\"].unique()\n", "# looking for very strong signal here:\n", "# listen to the positive/negative more than once\n", "# disliked one of the clips\n", "unique_requests = set(pos_filter_requests).intersection(neg_filter_requests)\n", "print(\n", " \"total pair requests\",\n", " df[\"request_id\"].nunique(),\n", " \" --> selected pair requests\",\n", " len(unique_requests),\n", " f\"frac {len(unique_requests) / df['request_id'].nunique():.3f}\",\n", ")" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.250737Z", "start_time": "2024-05-16T13:59:41.036434Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:05:12.149464Z", "iopub.status.busy": "2024-10-05T17:05:12.149315Z", "iopub.status.idle": "2024-10-05T17:05:12.250886Z", "shell.execute_reply": "2024-10-05T17:05:12.250363Z", "shell.execute_reply.started": "2024-10-05T17:05:12.149449Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "30b_t3_v13 requests 10252 clips 20504 total khrs 1.117; N gpus for 1000 iters 1.282; 4 gpus for x iters 320.375; n unique users 7780 n pro users 5358\n" ] } ], "source": [ "df_slice = df[df[\"request_id\"].isin(set(unique_requests))].copy()\n", "print(\n", " f\"{os.path.basename(OUT_DATA_DIR)} requests\",\n", " df_slice[\"request_id\"].nunique(),\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"4 gpus for x iters {df_slice.shape[0] / 8 / 2 / 4:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", " f\"n pro users {df_slice[df_slice['is_pro_user']]['user_id'].nunique()}\",\n", ")\n", "# 76171 152342 total khrs 2.880 n gpus for 1250 iters 3.809\n", "# v10 has 78866\n", "# v14 has 110402" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "execution": { "iopub.execute_input": "2024-10-05T17:05:12.251702Z", "iopub.status.busy": "2024-10-05T17:05:12.251542Z", "iopub.status.idle": "2024-10-05T17:05:12.253754Z", "shell.execute_reply": "2024-10-05T17:05:12.253370Z", "shell.execute_reply.started": "2024-10-05T17:05:12.251687Z" } }, "outputs": [], "source": [ "# v1 requests 17681 clips 35362 total khrs 1.815; N gpus for 1500 iters 1.473; 4 gpus for x iters 552.531; n unique users 15179 n pro users 5983" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.277006Z", "start_time": "2024-05-16T13:59:41.252105Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:05:12.254537Z", "iopub.status.busy": "2024-10-05T17:05:12.254411Z", "iopub.status.idle": "2024-10-05T17:05:12.296009Z", "shell.execute_reply": "2024-10-05T17:05:12.295555Z", "shell.execute_reply.started": "2024-10-05T17:05:12.254525Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "positive in playlist (2108, 112)\n" ] } ], "source": [ "test_mask = (df_slice[\"preference\"]) & (\n", " (df_slice[\"is_in_playlist\"]) | (df_slice[\"concat_in_playlist\"])\n", ")\n", "print(\"positive in playlist\", df_slice[test_mask].shape)" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.323409Z", "start_time": "2024-05-16T13:59:41.278278Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:05:12.296711Z", "iopub.status.busy": "2024-10-05T17:05:12.296574Z", "iopub.status.idle": "2024-10-05T17:05:12.331680Z", "shell.execute_reply": "2024-10-05T17:05:12.331312Z", "shell.execute_reply.started": "2024-10-05T17:05:12.296697Z" } }, "outputs": [], "source": [ "# interesting_clips_must_be_positive_mask = (\n", "# (df_slice[\"upvoted\"] == True)\n", "# | (df_slice[\"has_action\"] == True)\n", "# | (df_slice[\"part_of_concat\"] == True)\n", "# )\n", "# interesting_clips_must_be_not_negative_mask = (df_slice[\"downvoted\"] == False) # & (df_slice[\"dislike_count\"] < 1)\n", "# interesting_clips_mask = interesting_clips_must_be_positive_mask & interesting_clips_must_be_not_negative_mask\n", "# assert interesting_clips_mask.eq(df_slice[\"preference\"]).all()" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.392244Z", "start_time": "2024-05-16T13:59:41.324472Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:05:12.332323Z", "iopub.status.busy": "2024-10-05T17:05:12.332186Z", "iopub.status.idle": "2024-10-05T17:05:12.370629Z", "shell.execute_reply": "2024-10-05T17:05:12.370261Z", "shell.execute_reply.started": "2024-10-05T17:05:12.332310Z" } }, "outputs": [], "source": [ "# save positive ids\n", "# positive_preference_ids = df_slice[df_slice[\"preference\"] == False][\"s3_id\"].to_json(orient='values')\n", "# with open('/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id_negative.json', 'w') as file:\n", "# file.write(positive_preference_ids)" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "execution": { "iopub.execute_input": "2024-10-05T17:05:12.371253Z", "iopub.status.busy": "2024-10-05T17:05:12.371129Z", "iopub.status.idle": "2024-10-05T17:05:12.406244Z", "shell.execute_reply": "2024-10-05T17:05:12.405882Z", "shell.execute_reply.started": "2024-10-05T17:05:12.371241Z" } }, "outputs": [], "source": [ "# df_slice_2 = pd.read_csv(\"/home/tony/Data/Preference/30b_v1/interesting_clips_v4_t_1_20240808_slice.csv\")\n", "# df_total = pd.concat([df_slice, df_slice_2])\n", "# print(df_total.shape)" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "execution": { "iopub.execute_input": "2024-10-05T17:05:12.406904Z", "iopub.status.busy": "2024-10-05T17:05:12.406757Z", "iopub.status.idle": "2024-10-05T17:05:12.446576Z", "shell.execute_reply": "2024-10-05T17:05:12.446207Z", "shell.execute_reply.started": "2024-10-05T17:05:12.406891Z" } }, "outputs": [], "source": [ "# print(df_slices_2.shape, df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "execution": { "iopub.execute_input": "2024-10-05T17:05:12.447200Z", "iopub.status.busy": "2024-10-05T17:05:12.447078Z", "iopub.status.idle": "2024-10-05T17:05:12.486458Z", "shell.execute_reply": "2024-10-05T17:05:12.486083Z", "shell.execute_reply.started": "2024-10-05T17:05:12.447187Z" } }, "outputs": [], "source": [ "# print(df_total.shape)\n", "# df_slice = df_total.copy()" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T14:00:20.866354Z", "start_time": "2024-05-16T14:00:12.443344Z" }, "execution": { "iopub.execute_input": "2024-10-05T17:05:23.536367Z", "iopub.status.busy": "2024-10-05T17:05:23.535906Z", "iopub.status.idle": "2024-10-05T17:05:24.095071Z", "shell.execute_reply": "2024-10-05T17:05:24.094519Z", "shell.execute_reply.started": "2024-10-05T17:05:23.536351Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(20504, 112)\n" ] } ], "source": [ "# df_slice.to_pickle(\"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_v13_20240902_slice.pkl\")\n", "print(df_slice.shape)\n", "# BREAK" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932296Z", "start_time": "2024-05-16T13:59:41.932287Z" }, "execution": { "iopub.execute_input": "2024-09-10T03:36:52.169384Z", "iopub.status.busy": "2024-09-10T03:36:52.169254Z", "iopub.status.idle": "2024-09-10T03:36:52.244010Z", "shell.execute_reply": "2024-09-10T03:36:52.243502Z", "shell.execute_reply.started": "2024-09-10T03:36:52.169371Z" } }, "outputs": [ { "data": { "text/plain": [ "(9003, 10252)" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# don't have continue at\n", "df_slice[\"request_id\"] = df_slice[\"request_id\"].astype(str)\n", "df_slice[df_slice[\"continue_at\"].isna()][\"request_id\"].nunique(), df_slice[\"request_id\"].nunique()" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "execution": { "iopub.execute_input": "2024-09-10T03:36:52.244824Z", "iopub.status.busy": "2024-09-10T03:36:52.244673Z", "iopub.status.idle": "2024-09-10T03:36:52.284551Z", "shell.execute_reply": "2024-09-10T03:36:52.283974Z", "shell.execute_reply.started": "2024-09-10T03:36:52.244810Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(20504, 112)\n", "(20504, 112)\n", "(20504, 112)\n" ] } ], "source": [ "print(df_slice.shape)\n", "df_slice = df_slice[df_slice[\"request_id\"].apply(lambda x: len(x) > 3)]\n", "print(df_slice.shape)\n", "# df_slice = df_slice[df_slice[\"is_pro_user\"]].copy()\n", "print(df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932966Z", "start_time": "2024-05-16T13:59:41.932957Z" }, "execution": { "iopub.execute_input": "2024-09-10T03:36:52.285399Z", "iopub.status.busy": "2024-09-10T03:36:52.285252Z", "iopub.status.idle": "2024-09-10T03:36:52.309585Z", "shell.execute_reply": "2024-09-10T03:36:52.309035Z", "shell.execute_reply.started": "2024-09-10T03:36:52.285386Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "10252\n" ] } ], "source": [ "final_filtered_requests = df_slice[\"request_id\"].astype(str).unique()\n", "# final_filtered_requests = df_slice[df_slice[\"is_pro_user\"]][\"request_id\"].astype(str).unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.933558Z", "start_time": "2024-05-16T13:59:41.933550Z" }, "execution": { "iopub.execute_input": "2024-09-10T03:36:52.310413Z", "iopub.status.busy": "2024-09-10T03:36:52.310265Z", "iopub.status.idle": "2024-09-10T03:36:52.561099Z", "shell.execute_reply": "2024-09-10T03:36:52.560625Z", "shell.execute_reply.started": "2024-09-10T03:36:52.310399Z" } }, "outputs": [], "source": [ "# df_slice.to_csv(\"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_20240902_slice.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934277Z", "start_time": "2024-05-16T13:59:41.934268Z" }, "execution": { "iopub.execute_input": "2024-09-10T03:36:52.561873Z", "iopub.status.busy": "2024-09-10T03:36:52.561726Z", "iopub.status.idle": "2024-09-10T03:36:52.652410Z", "shell.execute_reply": "2024-09-10T03:36:52.651814Z", "shell.execute_reply.started": "2024-09-10T03:36:52.561860Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "10149 103\n", "(20298, 113) (206, 113)\n" ] } ], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"request_id\"].astype(str).isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"request_id\", \"preference\"])\n", "train_df = train_df.reset_index()\n", "val_df = val_df.sort_values(by=[\"request_id\", \"preference\"])\n", "val_df = val_df.reset_index()\n", "\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934954Z", "start_time": "2024-05-16T13:59:41.934946Z" }, "execution": { "iopub.execute_input": "2024-09-10T03:36:52.653275Z", "iopub.status.busy": "2024-09-10T03:36:52.653126Z", "iopub.status.idle": "2024-09-10T03:36:52.850061Z", "shell.execute_reply": "2024-09-10T03:36:52.849575Z", "shell.execute_reply.started": "2024-09-10T03:36:52.653260Z" } }, "outputs": [], "source": [ "# val_df[[\"request_id\", \"metadata\", \"updated_at\", \"user_id\", \"preference\"]].head()" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.935620Z", "start_time": "2024-05-16T13:59:41.935613Z" }, "execution": { "iopub.execute_input": "2024-09-10T03:36:52.850818Z", "iopub.status.busy": "2024-09-10T03:36:52.850671Z", "iopub.status.idle": "2024-09-10T03:36:53.548765Z", "shell.execute_reply": "2024-09-10T03:36:53.548072Z", "shell.execute_reply.started": "2024-09-10T03:36:52.850804Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 20298/20298 [00:00<00:00, 31028.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "1,106 hours of 20298 clips, 1.268625 nodes, 211.4375 iters\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "total_duration = 0\n", "for i, row in tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " try:\n", " assert row[\"preference\"] == (i % 2 == 1)\n", " total_duration += row[\"duration\"]\n", " except Exception as E:\n", " print(i, row)\n", " print(E)\n", " raise ValueError()\n", " \n", "print(\n", " f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1000} nodes, {train_df.shape[0] / 8 / 2 / 6} iters\"\n", ")" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936268Z", "start_time": "2024-05-16T13:59:41.936260Z" }, "execution": { "iopub.execute_input": "2024-09-10T03:36:53.549923Z", "iopub.status.busy": "2024-09-10T03:36:53.549598Z", "iopub.status.idle": "2024-09-10T03:36:59.661118Z", "shell.execute_reply": "2024-09-10T03:36:59.660413Z", "shell.execute_reply.started": "2024-09-10T03:36:53.549904Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 206/206 [00:06<00:00, 33.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 206 clips, 4 different prompts\n", "5 hours of False\n", "5 hours of True\n", "Done\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "make_dataset(val_df, OUT_DATA_DIR, is_val=True, npz_dir=NPZ_DIR)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936964Z", "start_time": "2024-05-16T13:59:41.936957Z" }, "execution": { "iopub.execute_input": "2024-09-10T03:36:59.662152Z", "iopub.status.busy": "2024-09-10T03:36:59.661981Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 88%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 17960/20298 [08:08<00:57, 40.35it/s]" ] } ], "source": [ "make_dataset(train_df, OUT_DATA_DIR, is_val=False, npz_dir=NPZ_DIR)" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.937879Z", "start_time": "2024-05-16T13:59:41.937870Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, 6016, 13)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000\n", "assert mm[:100, :, 1:].min() >= 0\n", "assert mm[:100, :, 1:].max() <= 2048" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.938629Z", "start_time": "2024-05-16T13:59:41.938621Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as codec_encode,\n", "# decode_stream_to_full_audio as codec_decode,\n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# decode as decode\n", "# )\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "# assert len(test_metas) == len(mm)\n", "# idx_list = list(range(len(test_metas)))\n", "# # random.shuffle(idx_list)\n", "# # idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939205Z", "start_time": "2024-05-16T13:59:41.939198Z" } }, "outputs": [], "source": [ "# import random\n", "# idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "# assert \"original_duration_s\" in test_metas[idx]\n", "# # positive index should be shifted by 1\n", "# pos_idx = idx + 1\n", "# print(\n", "# \"tags:\",\n", "# test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", "# test_metas[idx].get(\"tags\"),\n", "# )\n", "# arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[: pad_idx_arr[0], :]\n", "# pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pos_pad_idx_arr) > 0:\n", "# pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "# a = decode(arr)\n", "# print(\"\\n negative example \\n\", test_metas[idx])\n", "# a.play(compress=False)\n", "# pos_a = decode(pos_arr)\n", "# print(\"\\n positive example \\n\", test_metas[pos_idx])\n", "# pos_a.play(compress=False)\n", "# print(\n", "# \"text:\",\n", "# test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", "# test_metas[idx].get(\"text\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939977Z", "start_time": "2024-05-16T13:59:41.939969Z" } }, "outputs": [], "source": [ "# val_df[val_df[\"tags\"] == 'a vibrant blend of experimental jazz fusion, drum-and-bass and swagger fuzzed-out guitars']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.940610Z", "start_time": "2024-05-16T13:59:41.940603Z" } }, "outputs": [], "source": [ "# from collections import Counter\n", "# c = Counter()\n", "# for _, row in df_slice.iterrows():\n", "# # print(row[\"metadata\"])\n", "# for k in ast.literal_eval(row[\"metadata\"]).keys():\n", "# c[k] += 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941167Z", "start_time": "2024-05-16T13:59:41.941159Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941801Z", "start_time": "2024-05-16T13:59:41.941793Z" } }, "outputs": [], "source": [ "def validation_on_metas(input_metas):\n", " total_bad = 0\n", " total_good = 0\n", " for idx in range(len(input_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if input_metas[idx].get(\"tags\") != input_metas[pos_idx].get(\"tags\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", " print(total_good, total_bad)\n", " return\n", "\n", "\n", "validation_on_metas(test_metas)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.942520Z", "start_time": "2024-05-16T13:59:41.942511Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.943072Z", "start_time": "2024-05-16T13:59:41.943065Z" } }, "outputs": [], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.944246Z", "start_time": "2024-05-16T13:59:41.944237Z" } }, "outputs": [], "source": [ "total_iters = len(n_neg_tr) + len(n_pos_tr)\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945249Z", "start_time": "2024-05-16T13:59:41.945241Z" } }, "outputs": [], "source": [ "print(\"1 epoch per batch 2, total\", total_iters / 8 / 2 / 4)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945972Z", "start_time": "2024-05-16T13:59:41.945964Z" } }, "outputs": [], "source": [ "!cd /home/tony/Work/tony/slurm/30b_dpo && sbatch sbatch_ipo_30b" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import shutil\n", "\n", "# Basic file copy\n", "shutil.copy('/home/tony/Work/tony/Preference/make_dataset_13b_v4_t3.ipynb', os.path.join(OUT_DATA_DIR, \"make_dataset.ipynb\"))\n", "print(\"Cache kept!\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# some gymathtics loading prev data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.946562Z", "start_time": "2024-05-16T13:59:41.946555Z" } }, "outputs": [], "source": [ "# prev_v3_data = \"/app/suno/data/dpo/7v_v20_full/\"\n", "\n", "# test_val_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_val.jsonl\"))\n", "# test_tr_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_tr.jsonl\"))\n", "\n", "# all_ids = set()\n", "# for meta in test_val_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# for meta in test_tr_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# print(len(all_ids), len(test_val_metas) + len(test_tr_metas))\n", "\n", "# all_ids = list(all_ids)\n", "# with open(\"/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id.json\", \"w\") as fp:\n", "# json.dump(all_ids, fp)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "val_df.head(n=10)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }