{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.040680Z", "start_time": "2024-05-16T13:58:19.777010Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:28:18.735344Z", "iopub.status.busy": "2024-09-09T02:28:18.735197Z", "iopub.status.idle": "2024-09-09T02:28:20.298977Z", "shell.execute_reply": "2024-09-09T02:28:20.298329Z", "shell.execute_reply.started": "2024-09-09T02:28:18.735328Z" } }, "outputs": [], "source": [ "import ast\n", "import os\n", "import shutil\n", "import sys\n", "from collections import defaultdict\n", "\n", "import numpy as np\n", "import pandas as pd\n", "from preference_data_preparation_4min_concat import *\n", "from preference_helper import *\n", "from sklearn.model_selection import train_test_split\n", "from suno_utils.utils.s3 import download_s3_files\n", "from suno_utils.utils.text import read_json, read_jsonl, write_json, write_jsonl\n", "from tqdm import tqdm\n", "\n", "pd.set_option(\"display.max_rows\", 500)\n", "pd.set_option(\"display.max_columns\", 500)\n", "pd.set_option(\"display.width\", 1000)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:21.082172Z", "start_time": "2024-05-16T13:58:21.041926Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:28:20.299952Z", "iopub.status.busy": "2024-09-09T02:28:20.299742Z", "iopub.status.idle": "2024-09-09T02:28:20.326623Z", "shell.execute_reply": "2024-09-09T02:28:20.326090Z", "shell.execute_reply.started": "2024-09-09T02:28:20.299936Z" } }, "outputs": [], "source": [ "OUT_DATA_DIR = \"/app/suno/data/dpo/concat_v0_cycle\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "shutil.copyfile(\n", " \"/app/suno/data/dpo/7v_v20_full/tokenizer_60k.json\",\n", " os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"),\n", ")\n", "NPZ_DIR = \"/app/suno/data/dpo/concat_cycle_npz\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:53.962528Z", "start_time": "2024-05-16T13:58:21.105919Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:28:20.327465Z", "iopub.status.busy": "2024-09-09T02:28:20.327321Z", "iopub.status.idle": "2024-09-09T02:28:23.375710Z", "shell.execute_reply": "2024-09-09T02:28:23.374956Z", "shell.execute_reply.started": "2024-09-09T02:28:20.327452Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Preference data shape (243001, 43)\n" ] } ], "source": [ "df = pd.read_pickle(\n", " \"/home/tony/Data/Preference/30b_v2/concat_clips_20240908_v0.pkl\"\n", ") # , engine='python')\n", "# df = pd.read_csv(\n", "# \"/home/tony/Data/Preference/30b_v0/interesting_clips_v4_t_1_20240808.csv\"\n", "# ) # , engine='python')\n", "print(\"Preference data shape\", df.shape)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2024-09-09T02:28:23.376761Z", "iopub.status.busy": "2024-09-09T02:28:23.376583Z", "iopub.status.idle": "2024-09-09T02:28:23.561998Z", "shell.execute_reply": "2024-09-09T02:28:23.561282Z", "shell.execute_reply.started": "2024-09-09T02:28:23.376745Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(243001, 43)\n", "(243001, 37)\n" ] } ], "source": [ "print(df.shape)\n", "df = df.dropna(axis=1, how=\"all\")\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.199480Z", "start_time": "2024-05-16T13:58:53.963687Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:28:23.563181Z", "iopub.status.busy": "2024-09-09T02:28:23.562846Z", "iopub.status.idle": "2024-09-09T02:28:27.071820Z", "shell.execute_reply": "2024-09-09T02:28:27.071084Z", "shell.execute_reply.started": "2024-09-09T02:28:23.563162Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "243001\n", "243001\n", "pre-downloaded df (243001, 37)\n", "downloaded df (243001, 37)\n" ] } ], "source": [ "converted_paths = os.listdir(NPZ_DIR)\n", "print(len(converted_paths))\n", "\n", "converted_paths = set([f.replace(\"_gen_cycle.npz\", \"\") for f in converted_paths])\n", "print(len(converted_paths))\n", "\n", "print(\"pre-downloaded df\", df.shape)\n", "df[df[\"s3_id\"].isin(converted_paths)].shape\n", "df = df[df[\"s3_id\"].isin(converted_paths)].copy()\n", "print(\"downloaded df\", df.shape)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:58:56.467253Z", "start_time": "2024-05-16T13:58:56.207647Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:28:27.074228Z", "iopub.status.busy": "2024-09-09T02:28:27.073698Z", "iopub.status.idle": "2024-09-09T02:28:27.130238Z", "shell.execute_reply": "2024-09-09T02:28:27.129681Z", "shell.execute_reply.started": "2024-09-09T02:28:27.074207Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "is_30b\n", "False 229798\n", "True 13203\n", "Name: count, dtype: int64\n" ] }, { "data": { "text/plain": [ "task\n", " 242998\n", "cover 3\n", "Name: count, dtype: int64" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"is_30b\"] = df[\"model_name\"].str.contains(\"-t\")\n", "print(df[\"is_30b\"].value_counts())\n", "df[\"is_30b\"] = True\n", "df[\"task\"].value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# LET's do the data prep" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "execution": { "iopub.execute_input": "2024-09-09T02:28:27.131248Z", "iopub.status.busy": "2024-09-09T02:28:27.131103Z", "iopub.status.idle": "2024-09-09T02:28:27.281745Z", "shell.execute_reply": "2024-09-09T02:28:27.281213Z", "shell.execute_reply.started": "2024-09-09T02:28:27.131233Z" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAt8klEQVR4nO3de3BUdZ7//1d3QgYkF0M6iFlQ+RGSsFxyEYtJ7AxfRxBLcKuAGaEGRKyM4ICKDooUg0KAIkGRRZAdKGSiCC6yCzje2C2dHR2oBFEnGSpsVAIzApNx6G6BXLjl0r8/2JyhJZBu6OQknzwfVV3VOefTJ5/zpjm8+JzPOcfh9/v9AgAAMIzT7g4AAAC0BUIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADBSpN0d6Ah8vhpJUkJCjHy+GvGgi/bjcFB3O1B3e1B3e1B3e7Rl3Zu33RpCjhRQfL9f/CWwAXW3B3W3B3W3B3W3h51153QVAAAwEiEHAAAYiZADAACMRMgBAABGCinkbNiwQRMnTlRmZqays7M1a9YsHTlyJKDNgw8+qNTU1IDX888/H9CmqqpKM2bMUHp6urKzs7VixQo1NDQEtPn00081fvx4DRkyRKNHj9bOnTsv68/WrVv14x//WEOHDtVPf/pTHThwIJTdAQAABgsp5Ozfv19TpkzR9u3bVVRUpIaGBuXl5enMmTMB7R544AHt3bvXes2bN89a19jYqJkzZ6q+vl7btm1TYWGhdu3apTVr1lhtjh07ppkzZ2rEiBH67W9/q4ceekgLFy7Unj17rDYffPCBCgoKNHv2bO3atUtpaWnKy8uTz+e71loAAACDhBRyNm3apAkTJmjgwIFKS0tTYWGhqqqqdPDgwYB23bt3V2JiovWKjo621u3du1eVlZV68cUXNWjQII0cOVJz5szR1q1bdeHCBUnStm3b1LdvX82fP18DBgzQ1KlTNWbMGL322mvWdoqKivTAAw9o4sSJSk5OVn5+vrp3764dO3ZcRzkAAIAprmtOTk3NxZvoxcXFBSx/9913NWLECI0bN04vvfSSzp49a60rKytTSkqKXC6Xtcztdqu2tlaVlZVWm+zs7IBtut1ulZWVSZIuXLiggwcPKicn5x874nQqJydHpaWl17NLAADAENd8M8CmpiYtX75cWVlZSklJsZaPGzdOSUlJ6t27t7766iutXLlSf/7zn/XKK69Ikrxeb0DAkWT97PF4rtqmtrZW586d0+nTp9XY2KiEhISANgkJCZfNEQqGw9Hye7S95npT9/ZF3e1B3e1B3e3RlnUPdpvXHHLy8/N16NAhvfnmmwHLJ02aZL1PTU1VYmKipk+frqNHj+qWW2651l/Xpi69NXQwt4lG+FF3e1B3e1B3e1B3e9hZ92sKOUuWLNHHH3+sLVu2qE+fPldtm56eLkn65ptvdMstt8jlcl12FZTX65UkJSYmSro4atO87NI20dHR6t69u5xOpyIiIi6bZOzz+S4bAQoGz66yD8+UsQd1twd1twd1t0ene3aV3+/X0qVL9eGHH+qNN95Qv379Wv1MRUWFpH8EmIyMDK1fv14+n8863VRcXKzo6GglJydbbf7whz8EbKe4uFgZGRmSpKioKA0ePFglJSUaNWqUpIunz0pKSjR16tRQdun/9ivwPX8J2h91twd1twd1twd1t0eneXZVfn6+3nnnHb300kvq2bOnPB6PPB6Pzp07J0k6evSo1q1bp/Lych0/fly/+93v9Oyzz+qOO+5QWlqapIsTiJOTkzVv3jx9+eWX2rNnj1avXq0pU6YoKipKkjR58mQdO3ZML7zwgg4fPqytW7dq9+7dmj59utWXhx9+WNu3b9euXbt0+PBhLV68WGfPntWECRPCVBoAANCZOfz+4PNVampqi8sLCgo0YcIE/e1vf9MzzzyjQ4cO6cyZM7r55ps1atQozZo1K+Ay8r/+9a9avHix9u/frx49emj8+PGaO3euIiP/MbD06aefqqCgQJWVlerTp49mzZp1WYDZsmWLNm3aJI/Ho0GDBmnhwoXW6bFQeL0XT1e5XDHyehnObE8OB3W3A3W3B3W/OqfTIafz6jNKm5r8amoKrXjU3R5tWffmbbfaLpSQYypCjn04+NiDutuDul+Z0+lQ3I03KDLi6icYGhqbdPrUmZCCDnW3R0cIOdd8dRUAAOHidDoUGeHUnG2lqjxR22Kb5N7RenlyppxOR8ijOeiaCDkAgA6j8kStDlZV290NGIKnkAMAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASCGFnA0bNmjixInKzMxUdna2Zs2apSNHjgS0OX/+vPLz8zVixAhlZmbq8ccfl9frDWhTVVWlGTNmKD09XdnZ2VqxYoUaGhoC2nz66acaP368hgwZotGjR2vnzp2X9Wfr1q368Y9/rKFDh+qnP/2pDhw4EMruAAAAg4UUcvbv368pU6Zo+/btKioqUkNDg/Ly8nTmzBmrzfLly/X73/9eq1ev1htvvKETJ07oscces9Y3NjZq5syZqq+v17Zt21RYWKhdu3ZpzZo1Vptjx45p5syZGjFihH7729/qoYce0sKFC7Vnzx6rzQcffKCCggLNnj1bu3btUlpamvLy8uTz+a6nHgAAwBAhhZxNmzZpwoQJGjhwoNLS0lRYWKiqqiodPHhQklRTU6MdO3Zo/vz5ys7O1pAhQ7R8+XKVlpaqrKxMkrR3715VVlbqxRdf1KBBgzRy5EjNmTNHW7du1YULFyRJ27ZtU9++fTV//nwNGDBAU6dO1ZgxY/Taa69ZfSkqKtIDDzygiRMnKjk5Wfn5+erevbt27NgRnsoAAIBO7brm5NTU1EiS4uLiJEnl5eWqr69XTk6O1WbAgAFKSkqyQk5ZWZlSUlLkcrmsNm63W7W1taqsrLTaZGdnB/wut9ttbePChQs6ePBgwO9xOp3KyclRaWlpyPvhcFx8XfqeV/u9qDt170ov6n7lulzLcZu6d+xXW9Y9GJGhf7Uuampq0vLly5WVlaWUlBRJktfrVbdu3RQbGxvQNiEhQR6Px2pzacCRZP3cWpva2lqdO3dOp0+fVmNjoxISEi77Pd+fIxSMhISYFt+j/VB3e1B3e1D36xMf3/OaPkfd7WFn3a855OTn5+vQoUN68803w9kfW/h8F0ekEhJi5PPVyO+3uUNdiMNB3e1A3e1B3a8sIsIZdHg5ebJOjY1NQW+butujLevevO3WXFPIWbJkiT7++GNt2bJFffr0sZa7XC7V19eruro6YDTH5/MpMTHRavP9q6Car766tM33r8jyer2Kjo5W9+7d5XQ6FRERcdkkY5/Pd9kIUDAuLb7fL/4S2IC624O624O6X79rqR91t4eddQ9pTo7f79eSJUv04Ycf6vXXX1e/fv0C1g8ZMkTdunVTSUmJtezIkSOqqqpSRkaGJCkjI0Nff/11QEApLi5WdHS0kpOTrTb79u0L2HZxcbG1jaioKA0ePDjg9zQ1NamkpESZmZmh7BIAADBUSCEnPz9f77zzjl566SX17NlTHo9HHo9H586dkyTFxMRo4sSJKiws1L59+1ReXq4FCxYoMzPTCihut1vJycmaN2+evvzyS+3Zs0erV6/WlClTFBUVJUmaPHmyjh07phdeeEGHDx/W1q1btXv3bk2fPt3qy8MPP6zt27dr165dOnz4sBYvXqyzZ89qwoQJ4akMAADo1EI6XfXv//7vkqQHH3wwYHlBQYEVLhYsWCCn06knnnhCFy5ckNvt1qJFi6y2ERERWr9+vRYvXqxJkyapR48eGj9+vJ544gmrTb9+/bRhwwYVFBRo8+bN6tOnj5YtW6bc3FyrzX333afvvvtOa9askcfj0aBBg/Tqq69e0+kqAABgHoffzxlKr/fixGOXK0ZeLxPT2pPDQd3tQN3tQd2vLDLy4sTjsWv26GBVdYttBifF6v0ncnXyZJ0aGkKbeEzd219b1r15263h2VUAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjBRyyPnss8/06KOPyu12KzU1VR999FHA+vnz5ys1NTXglZeXF9Dm1KlTmjt3rrKysjR8+HAtWLBAdXV1AW2+/PJL/exnP9PQoUM1cuRIbdy48bK+7N69W/fee6+GDh2q+++/X5988kmouwMAAAwVcsg5c+aMUlNTtWjRoiu2yc3N1d69e63XqlWrAtY//fTTqqysVFFRkdavX6/PP/9czz//vLW+trZWeXl5SkpK0s6dOzVv3jy98soreuutt6w2f/zjHzV37lz95Cc/0dtvv627775bs2fP1tdffx3qLgEAAANFhvqBkSNHauTIkVdtExUVpcTExBbXHT58WHv27NF//ud/aujQoZKkhQsXasaMGZo3b55uuukmvfPOO6qvr9fy5csVFRWlgQMHqqKiQkVFRZo0aZIkafPmzcrNzdXPf/5zSdKTTz6p4uJibdmyRUuWLAl1twAAgGFCDjnB2L9/v7KzsxUbG6sf/vCHevLJJxUfHy9JKi0tVWxsrBVwJCknJ0dOp1MHDhzQ6NGjVVZWpuHDhysqKspq43a7tXHjRp0+fVpxcXEqKyvT9OnTA36v2+2+7PRZMByOlt+j7TXXm7q3L+puD+oePqHUkLrboy3rHuw2wx5ycnNzNXr0aPXt21fHjh3TqlWr9Mgjj+itt95SRESEvF6vevXqFdiJyEjFxcXJ4/FIkrxer/r27RvQxuVyWevi4uLk9XqtZc0SEhLk9XpD7nNCQkyL79F+qLs9qLs9qPv1iY/veU2fo+72sLPuYQ85Y8eOtd43TzweNWqUNbrTEfl8NZIu/kH4fDXy+23uUBficFB3O1B3e1D3K4uIcAYdXk6erFNjY1PQ26bu9mjLujdvuzVtcrrqUv369VN8fLy++eYbZWdny+Vy6bvvvgto09DQoNOnT1vzeFwu12UjMs0/N4/etNTG5/NdNroTjEuL7/eLvwQ2oO72oO72oO7X71rqR93tYWfd2/w+Od9++61OnTplBZjMzExVV1ervLzcarNv3z41NTVp2LBhkqSMjAx9/vnnqq+vt9oUFxerf//+iouLs9rs27cv4HcVFxcrIyOjjfcIAAB0BiGHnLq6OlVUVKiiokKSdPz4cVVUVKiqqkp1dXVasWKFysrKdPz4cZWUlGjWrFm69dZblZubK0kaMGCAcnNz9dxzz+nAgQP64osvtHTpUo0dO1Y33XSTJOn+++9Xt27d9Ktf/UqHDh3SBx98oM2bN+vhhx+2+jFt2jTt2bNHv/nNb3T48GGtXbtW5eXlmjp1ajjqAgAAOrmQT1eVl5dr2rRp1s8FBQWSpPHjx2vx4sX6+uuv9fbbb6umpka9e/fWnXfeqTlz5gRcKbVy5UotXbpUDz30kJxOp+655x4tXLjQWh8TE6NNmzZpyZIlmjBhguLj4zVr1izr8nFJysrK0sqVK7V69WqtWrVKt912m9atW6eUlJRrKgQAADCLw+/nDKXXe3HiscsVI6+XiWntyeGg7nag7vag7lcWGXlx4vHYNXt0sKq6xTaDk2L1/hO5OnmyTg0NoU08pu7try3r3rzt1vDsKgAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMFHLI+eyzz/Too4/K7XYrNTVVH330UcB6v9+vl19+WW63W8OGDdP06dP1l7/8JaDNqVOnNHfuXGVlZWn48OFasGCB6urqAtp8+eWX+tnPfqahQ4dq5MiR2rhx42V92b17t+69914NHTpU999/vz755JNQdwcAABgq5JBz5swZpaamatGiRS2u37hxo9544w0tXrxY27dvV48ePZSXl6fz589bbZ5++mlVVlaqqKhI69ev1+eff67nn3/eWl9bW6u8vDwlJSVp586dmjdvnl555RW99dZbVps//vGPmjt3rn7yk5/o7bff1t13363Zs2fr66+/DnWXAACAgUIOOSNHjtRTTz2l0aNHX7bO7/dr8+bN+sUvfqFRo0YpLS1NL7zwgk6cOGGN+Bw+fFh79uzRsmXLlJ6eruHDh2vhwoV6//339fe//12S9M4776i+vl7Lly/XwIEDNXbsWD344IMqKiqyftfmzZuVm5urn//85xowYICefPJJ/fM//7O2bNlyrbUAAAAGiQznxo4fPy6Px6OcnBxrWUxMjNLT01VaWqqxY8eqtLRUsbGxGjp0qNUmJydHTqdTBw4c0OjRo1VWVqbhw4crKirKauN2u7Vx40adPn1acXFxKisr0/Tp0wN+v9vtvuz0WTAcjpbfo+0115u6ty/qbg/qHj6h1JC626Mt6x7sNsMacjwejyQpISEhYHlCQoK8Xq8kyev1qlevXoGdiIxUXFyc9Xmv16u+ffsGtHG5XNa6uLg4eb1ea1lLvycUCQkxLb5H+6Hu9qDu9qDu1yc+vuc1fY6628POuoc15HRWPl+NpIt/ED5fjfx+mzvUhTgc1N0O1N0e1P3KIiKcQYeXkyfr1NjYFPS2qbs92rLuzdtuTVhDTmJioiTJ5/Opd+/e1nKfz6e0tDRJF0dkvvvuu4DPNTQ06PTp09bnXS7XZSMyzT83j9601Mbn8102uhOMS4vv94u/BDag7vag7vag7tfvWupH3e1hZ93Dep+cvn37KjExUSUlJday2tpa/elPf1JmZqYkKTMzU9XV1SovL7fa7Nu3T01NTRo2bJgkKSMjQ59//rnq6+utNsXFxerfv7/i4uKsNvv27Qv4/cXFxcrIyAjnLgEAgE4q5JBTV1eniooKVVRUSLo42biiokJVVVVyOByaNm2afv3rX+t3v/udvvrqK82bN0+9e/fWqFGjJEkDBgxQbm6unnvuOR04cEBffPGFli5dqrFjx+qmm26SJN1///3q1q2bfvWrX+nQoUP64IMPtHnzZj388MNWP6ZNm6Y9e/boN7/5jQ4fPqy1a9eqvLxcU6dODUddAABAJxfy6ary8nJNmzbN+rmgoECSNH78eBUWFuqRRx7R2bNn9fzzz6u6ulq33367Xn31Vf3gBz+wPrNy5UotXbpUDz30kJxOp+655x4tXLjQWh8TE6NNmzZpyZIlmjBhguLj4zVr1ixNmjTJapOVlaWVK1dq9erVWrVqlW677TatW7dOKSkp11QIAABgFoffzxlKr/fixGOXK0ZeLxPT2pPDQd3tQN3tQd2vLDLy4sTjsWv26GBVdYttBifF6v0ncnXyZJ0aGkKbeEzd219b1r15263h2VUAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGCnkp5ADQEfgdDrkdDqu2qapya+mJp7ICHRVhBwAnY7T6VDcjTcoMuLqg9ENjU06feoMQQfoogg5ADodp9OhyAin5mwrVeWJ2hbbJPeO1suTM+V0Ogg5QBdFyAHQaVWeqNXBqmq7uwGgg2LiMQAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGCkSLs7AABdhdPpkCRFRFz5/5dNTX41Nfnbq0uA0Qg5ALo0p9NhhY8rCUfwcDodio27QZIUH9/ziu0aGpt0+tQZgg4QBoQcAF2W0+lQ3I03KPIqIytScMGjtbAUEeFUZIRTc7aVqvJEbYttkntH6+XJmXI6HYQcIAwIOQC6LKfTEZbgEWxYkqTKE7U6WFV9Xf0GEBxCDoAOJ5hRkXAKJnhc7XcGM0rz/1IT9cyYtOvqJ4DQEHIAdCihjIq0h8ToH6ixya/Y2B6ttr1aWBqQeOV5OADaBiEHQLsKx9yV9hwVie0RqQino8P0B0DwCDkA2k245q7YMSrSkfrTXleEAZ0dIQdAuwlmoi+jIlcXzivCANMRcgC0u440KtLZhOuKMJNdaaTr0snjjHR1DYQcAOiErveKMMnMf+ivNtJ16U0YGenqGgg5AMKmvS/9RsuCvSLMxH/oGenCpQg5AMKio1363ay1+9uYKJgrwkz/h56bLkIi5AAIk442qTiU+9uYin/o0dURcgCEVUeZVNyZ72/TnqNPXXHeDroOQg4Ao3WU0BWM9hx96srzdtB1EHIAoINoz9En5u2gKyDkAEAH056jT8zbgckIOQCAqzJ13o6p+4V/IOQAAFpk6rydUPartuac/P4r7xdBqGMj5ABdXDAPewyGqfec6cpMnbcTzH7dcVu8nhs3WDfeeMNVt9XZAl5XQ8gBujCn06HYuOBu4NfY5FdEGMIQOh9T5+20NvfJxIDX1RBygC7M4Wj9Bn7SP67o6Yz3nEHH0Fkf+WFqwOsqCDkAWj2QN1/R05nuOYOOo6M+8gPmI+QABuus/3tG59PaXZrD+ciPrvg8MlwbQg5gqGD+99yVn+uE8AjlLs3XOxLYUZ9HxqXoHRchBzBUR3tgJszU0e7S3BEfAssVWPYh5ACGYx4N2kNHuUtzR3sILFdg2YuQAwDAdQjmCixOadkj7DO01q5dq9TU1IDXvffea60/f/688vPzNWLECGVmZurxxx+X1+sN2EZVVZVmzJih9PR0ZWdna8WKFWpoaAho8+mnn2r8+PEaMmSIRo8erZ07d4Z7VwAAuC6XntKKj+95xVfcjTeE5aacCNQmIzkDBw5UUVGR9XNERIT1fvny5frkk0+0evVqxcTEaOnSpXrssce0bds2SVJjY6Nmzpwpl8ulbdu26cSJE3r22WfVrVs3/fKXv5QkHTt2TDNnztTkyZO1cuVKlZSUaOHChUpMTFRubm5b7BIAACHjlJa92iTkREREKDEx8bLlNTU12rFjh1auXKns7GxJF0PPfffdp7KyMmVkZGjv3r2qrKxUUVGRXC6XBg0apDlz5mjlypV67LHHFBUVpW3btqlv376aP3++JGnAgAH64osv9NprrxFyAAAdDjcVtEeb3FDgm2++kdvt1t133625c+eqqqpKklReXq76+nrl5ORYbQcMGKCkpCSVlZVJksrKypSSkiKXy2W1cbvdqq2tVWVlpdWmOSRd2qZ5G6FyOC6+Ln3Pq/1e1L3t6gqg84iIcCoy8sqviAiH7ceVazkO2XmMC/tIzrBhw1RQUKD+/fvL4/Fo3bp1mjJlit599115vV5169ZNsbGxAZ9JSEiQx+ORJHm93oCAI8n6ubU2tbW1OnfunLp37x5SnxMSYlp8j/ZD3QF0VcFeit5Znx9n5/E97CFn5MiR1vu0tDSlp6frrrvu0u7du0MOH+3F56uRdPEPwuerkZ9Tou3G4aDu1yKUB2sC6NhCmbdz8mSdGhub2rmH16Ytj+/N225Nm19CHhsbq9tuu01Hjx5VTk6O6uvrVV1dHTCa4/P5rDk8LpdLBw4cCNhG89VXl7b5/hVZXq9X0dHR1xSkLi2+3y/+sbUBdQ8UzOMYuNEfYJZg5+10tmOlncf3Ng85dXV1OnbsmBITEzVkyBB169ZNJSUlGjNmjCTpyJEjqqqqUkZGhiQpIyND69evl8/nU0JCgiSpuLhY0dHRSk5Ottr84Q9/CPg9xcXF1jaAziyUhxl2lJuiAUBHFPaQs2LFCt11111KSkrSiRMntHbtWjmdTo0bN04xMTGaOHGiCgsLFRcXp+joaC1btkyZmZlWQHG73UpOTta8efP0zDPPyOPxaPXq1ZoyZYqioqIkSZMnT9bWrVv1wgsvaOLEidq3b592796tDRs2hHt3gHbH4xgAIDzCHnK+/fZb/fKXv9SpU6fUq1cv3X777dq+fbt69eolSVqwYIGcTqeeeOIJXbhwQW63W4sWLbI+HxERofXr12vx4sWaNGmSevToofHjx+uJJ56w2vTr108bNmxQQUGBNm/erD59+mjZsmVcPg6jMEoDANcn7CHnX//1X6+6/gc/+IEWLVoUEGy+75/+6Z+0cePGq25nxIgRevvtt6+liwAAoAvg0gwAAGAkHtAJAEAnwYM+Q0PIAQCggwv2hoENjU2qrTkn/1Wu2e5KQYiQA4RRa/e3kbrWAQZAeARzw8A7bovXc+MG68Ybb7jqthoam3T61JkucRwi5ABhEuz9bbrSAQZAeLV21SVPPA9EyAHCJJj72zQfYLp1i7jirdlbO+cOAFfDE8//gZADhNnVDjDBnlcHgLbUVSYwE3KAdhTMeXXuZgygrYQygdmE0+qEHMAG3M0YgB1CeeJ5a/N2grnQwm6EHAAAupjrnbcT7IUWjU1+OZ0ONTbaMyJEyAEAACEJ5UILh8MhiZADdGitDc1yVRQAk1ztmNa8rqNfyUXIAYIQ7NAsAHR2Jl0FSsgBghDM0CxXRQEwgUlXgRJygBBwVRSArsKE4x1j7wAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEs+uAnTxAZxOp+OK6yN4+jgAdDqEHHR5TqdDcTfeoEiCDAAYhZCDLs/pdCgywqk520pVeaK2xTb/LzVRz4xJa+eeAQCuByEH+D+VJ2p1sKq6xXUDEnu2c28AANeL8XkAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJG4Tw6MxyMbAKBrIuTAaDyyAQC6LkIOjMYjGwCg6yLkoEvgkQ0A0PUQctCpMd8GAHAlhBx0WK0FGIfDoeiY7sy3AQC0iJCDDimUCcPMtwEAtISQgw4plAnDzLcBALSEkIMOjQADALhWhBzY4vvzbb4/QZgJwwCA60XIQbtrab5NfDyjMgCA8CLkoN1xgz4AQHsg5CDsgr13DfNtAABtiZCDsOJZUQCAjoKQg7DiVBQAoKMg5CAknIoCAHQWhBwEjVNRAIDOhJADSzCjNJyKAgB0FoScLiKcD7vkVBQAoDMg5BggnAGGURoAgCkIOR1cewcYRmkAAKYg5NiIAAMAQNsh5NgklCuVCDAAAISu04ecrVu3atOmTfJ4PEpLS9Nzzz2nYcOG2d2tsF6pRIABACB0nTrkfPDBByooKFB+fr7S09P1+uuvKy8vT//1X/+lhIQE2/oVyigNAQYAgLbRqUNOUVGRHnjgAU2cOFGSlJ+fr48//lg7duzQjBkzbOsXjzYAAMB+nTbkXLhwQQcPHtTMmTOtZU6nUzk5OSotLQ1pW05n4Hu/Pzx9/EGkUz2iIlpcF/V/ozyDk2Kv2GZAYjRtaGN7m47YJ9rQhjYdv83/57p4NsLhCPx3NhwcV54NEtjO7w/XP+nt6+9//7t+9KMfadu2bcrMzLSWv/DCC/rss8/0H//xHzb2DgAA2I2HEAEAACN12pATHx+viIgI+Xy+gOU+n08ul8umXgEAgI6i04acqKgoDR48WCUlJdaypqYmlZSUBJy+AgAAXVOnnXgsSQ8//LCeffZZDRkyRMOGDdPrr7+us2fPasKECXZ3DQAA2KxTh5z77rtP3333ndasWSOPx6NBgwbp1Vdf5XQVAADovFdXAQAAXE2nnZMDAABwNYQcAABgJEIOAAAwEiEHAAAYqcuFnLVr1yo1NTXgde+991rrz58/r/z8fI0YMUKZmZl6/PHH5fV6bexx5/TZZ5/p0UcfldvtVmpqqj766KOA9X6/Xy+//LLcbreGDRum6dOn6y9/+UtAm1OnTmnu3LnKysrS8OHDtWDBAtXV1bXjXnQ+rdV9/vz5l33/8/LyAtpQ99Bs2LBBEydOVGZmprKzszVr1iwdOXIkoE0wx5WqqirNmDFD6enpys7O1ooVK9TQ0NCeu9KpBFP3Bx988LLv+/PPPx/QhrqH5s0339T999+vrKwsZWVladKkSfrkk0+s9R3tu97lQo4kDRw4UHv37rVeb775prVu+fLl+v3vf6/Vq1frjTfe0IkTJ/TYY4/Z2NvO6cyZM0pNTdWiRYtaXL9x40a98cYbWrx4sbZv364ePXooLy9P58+ft9o8/fTTqqysVFFRkdavX6/PP//8sgMUArVWd0nKzc0N+P6vWrUqYD11D83+/fs1ZcoUbd++XUVFRWpoaFBeXp7OnDljtWntuNLY2KiZM2eqvr5e27ZtU2FhoXbt2qU1a9bYsUudQjB1l6QHHngg4Ps+b948ax11D12fPn309NNPa+fOndqxY4d++MMfavbs2Tp06JCkDvhd93cxa9as8f/Lv/xLi+uqq6v9gwcP9u/evdtaVllZ6U9JSfGXlpa2Uw/Nk5KS4v/www+tn5uamvx33nmn/9VXX7WWVVdX+4cMGeJ/7733/H7/P+p+4MABq80nn3ziT01N9X/77bft1/lO7Pt19/v9/meffdb/i1/84oqfoe7Xz+fz+VNSUvz79+/3+/3BHVc+/vhjf1pamt/j8Vht3nzzTX9WVpb//Pnz7dr/zur7dff7/f6pU6f6ly1bdsXPUPfwuOOOO/zbt2/vkN/1LjmS880338jtduvuu+/W3LlzVVVVJUkqLy9XfX29cnJyrLYDBgxQUlKSysrKbOqteY4fPy6PxxNQ55iYGKWnp6u0tFSSVFpaqtjYWA0dOtRqk5OTI6fTqQMHDrR7n02yf/9+ZWdna8yYMVq0aJFOnjxpraPu16+mpkaSFBcXJym440pZWZlSUlICbmTqdrtVW1urysrK9ut8J/b9ujd79913NWLECI0bN04vvfSSzp49a62j7tensbFR77//vs6cOaPMzMwO+V3v1Hc8vhbDhg1TQUGB+vfvL4/Ho3Xr1mnKlCl699135fV61a1bN8XGxgZ8JiEhQR6Px6Yem6e5lgkJCQHLExISrHO3Xq9XvXr1ClgfGRmpuLg4/iyuQ25urkaPHq2+ffvq2LFjWrVqlR555BG99dZbioiIoO7XqampScuXL1dWVpZSUlIkKajjitfrvexO7c0/U/fWtVR3SRo3bpySkpLUu3dvffXVV1q5cqX+/Oc/65VXXpFE3a/VV199pcmTJ+v8+fO64YYbtG7dOiUnJ6uioqLDfde7XMgZOXKk9T4tLU3p6em66667tHv3bnXv3t3GngFtb+zYsdb75omYo0aNskZ3cH3y8/N16NChgHl+aHtXqvukSZOs96mpqUpMTNT06dN19OhR3XLLLe3dTWP0799fb7/9tmpqavTf//3fevbZZ7Vlyxa7u9WiLnm66lKxsbG67bbbdPToUblcLtXX16u6ujqgjc/nU2Jiok09NE9zLX0+X8Byn89nJXqXy6XvvvsuYH1DQ4NOnz7Nn0UY9evXT/Hx8frmm28kUffrsWTJEn388cd6/fXX1adPH2t5MMcVl8t12RUozT9T96u7Ut1bkp6eLkkB33fqHrqoqCjdeuutGjJkiObOnau0tDRt3ry5Q37Xu3zIqaur07Fjx5SYmKghQ4aoW7duKikpsdYfOXJEVVVVysjIsK+Thunbt68SExMD6lxbW6s//elPyszMlCRlZmaqurpa5eXlVpt9+/apqalJw4YNa/c+m+rbb7/VqVOnrIMLdQ+d3+/XkiVL9OGHH+r1119Xv379AtYHc1zJyMjQ119/HRD8i4uLFR0dreTk5HbZj86mtbq3pKKiQtI//jGl7uHR1NSkCxcudMjvepc7XbVixQrdddddSkpK0okTJ7R27Vo5nU6NGzdOMTExmjhxogoLCxUXF6fo6GgtW7ZMmZmZhJwQ1dXV6ejRo9bPx48fV0VFheLi4pSUlKRp06bp17/+tW699Vb17dtXL7/8snr37q1Ro0ZJujhZLTc3V88995zy8/NVX1+vpUuXauzYsbrpppvs2q0O72p1j4uL0yuvvKIxY8bI5XLp2LFjevHFF3XrrbcqNzdXEnW/Fvn5+Xrvvff0b//2b+rZs6c1ryAmJkbdu3cP6rjidruVnJysefPm6ZlnnpHH49Hq1as1ZcoURUVF2bh3HVdrdT969KjeffddjRw5UjfeeKO++uorFRQU6I477lBaWpok6n4tXnrpJf3oRz/SzTffrLq6Or333nvav3+/Nm3a1CG/613uKeRPPfWUPvvsM506dUq9evXS7bffrqeeeso6P3v+/HkVFhbq/fff14ULF+R2u7Vo0SKGLkP06aefatq0aZctHz9+vAoLC+X3+7VmzRpt375d1dXVuv3227Vo0SL179/fanvq1CktXbpU//M//yOn06l77rlHCxcuVM+ePdtzVzqVq9V98eLFmj17tv73f/9XNTU16t27t+68807NmTMnYCIgdQ9Nampqi8sLCgo0YcIEScEdV/76179q8eLF2r9/v3r06KHx48dr7ty5iozscv8XDUprdf/b3/6mZ555RocOHdKZM2d08803a9SoUZo1a5aio6Ot9tQ9NAsWLNC+fft04sQJxcTEKDU1VY888ojuvPNOSR3vu97lQg4AAOgauvycHAAAYCZCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACM9P8DsEtoMlUBKzEAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "df[\"duration\"].hist(bins=50)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "execution": { "iopub.execute_input": "2024-09-09T02:28:27.282806Z", "iopub.status.busy": "2024-09-09T02:28:27.282654Z", "iopub.status.idle": "2024-09-09T02:29:23.421948Z", "shell.execute_reply": "2024-09-09T02:29:23.421198Z", "shell.execute_reply.started": "2024-09-09T02:28:27.282792Z" } }, "outputs": [], "source": [ "# Let's use the old selection for now -- for quality assurance\n", "# expand the metadata columns -- this takes forever...~ 6 mins\n", "test_slice = df[\"metadata\"].apply(lambda x: ast.literal_eval(str(x)))\n", "test_slice_series = test_slice.apply(pd.Series)\n", "df = pd.concat([df, test_slice_series], axis=1, join=\"inner\")" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2024-09-09T02:29:23.423098Z", "iopub.status.busy": "2024-09-09T02:29:23.422924Z", "iopub.status.idle": "2024-09-09T02:29:23.875029Z", "shell.execute_reply": "2024-09-09T02:29:23.874293Z", "shell.execute_reply.started": "2024-09-09T02:29:23.423082Z" } }, "outputs": [], "source": [ "df = df.loc[:, ~df.columns.duplicated()].copy()" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.250737Z", "start_time": "2024-05-16T13:59:41.036434Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:29:23.876124Z", "iopub.status.busy": "2024-09-09T02:29:23.875944Z", "iopub.status.idle": "2024-09-09T02:29:23.960837Z", "shell.execute_reply": "2024-09-09T02:29:23.960135Z", "shell.execute_reply.started": "2024-09-09T02:29:23.876107Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "concat_v0_cycle requests clips 243001 total khrs 13.876; N gpus for 1000 iters 15.188; 2 gpus for x iters 7593.781; n unique users 61062\n" ] } ], "source": [ "df_slice = df.copy()\n", "print(\n", " f\"{os.path.basename(OUT_DATA_DIR)} requests\",\n", " \"clips\",\n", " df_slice.shape[0],\n", " f\"total khrs {sum(df_slice['duration'] / 3600 / 1000):.3f};\",\n", " f\"N gpus for 1000 iters {df_slice.shape[0] / 8 / 2 / 1000:.3f};\",\n", " f\"2 gpus for x iters {df_slice.shape[0] / 8 / 2 / 2:.3f};\",\n", " f\"n unique users {df_slice['user_id'].nunique()}\",\n", ")\n", "# 76171 152342 total khrs 2.880 n gpus for 1250 iters 3.809\n", "# v10 has 78866\n", "# v14 has 110402" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "execution": { "iopub.execute_input": "2024-09-09T02:29:23.961846Z", "iopub.status.busy": "2024-09-09T02:29:23.961683Z", "iopub.status.idle": "2024-09-09T02:29:23.964250Z", "shell.execute_reply": "2024-09-09T02:29:23.963765Z", "shell.execute_reply.started": "2024-09-09T02:29:23.961830Z" } }, "outputs": [], "source": [ "# v1 requests 17681 clips 35362 total khrs 1.815; N gpus for 1500 iters 1.473; 4 gpus for x iters 552.531; n unique users 15179 n pro users 5983" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.323409Z", "start_time": "2024-05-16T13:59:41.278278Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:29:23.965219Z", "iopub.status.busy": "2024-09-09T02:29:23.965070Z", "iopub.status.idle": "2024-09-09T02:29:24.001359Z", "shell.execute_reply": "2024-09-09T02:29:24.000893Z", "shell.execute_reply.started": "2024-09-09T02:29:23.965205Z" } }, "outputs": [], "source": [ "# interesting_clips_must_be_positive_mask = (\n", "# (df_slice[\"upvoted\"] == True)\n", "# | (df_slice[\"has_action\"] == True)\n", "# | (df_slice[\"part_of_concat\"] == True)\n", "# )\n", "# interesting_clips_must_be_not_negative_mask = (df_slice[\"downvoted\"] == False) # & (df_slice[\"dislike_count\"] < 1)\n", "# interesting_clips_mask = interesting_clips_must_be_positive_mask & interesting_clips_must_be_not_negative_mask\n", "# assert interesting_clips_mask.eq(df_slice[\"preference\"]).all()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.392244Z", "start_time": "2024-05-16T13:59:41.324472Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:29:24.002273Z", "iopub.status.busy": "2024-09-09T02:29:24.001971Z", "iopub.status.idle": "2024-09-09T02:29:24.036230Z", "shell.execute_reply": "2024-09-09T02:29:24.035761Z", "shell.execute_reply.started": "2024-09-09T02:29:24.002258Z" } }, "outputs": [], "source": [ "# save positive ids\n", "# positive_preference_ids = df_slice[df_slice[\"preference\"] == False][\"s3_id\"].to_json(orient='values')\n", "# with open('/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id_negative.json', 'w') as file:\n", "# file.write(positive_preference_ids)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "execution": { "iopub.execute_input": "2024-09-09T02:29:24.036971Z", "iopub.status.busy": "2024-09-09T02:29:24.036835Z", "iopub.status.idle": "2024-09-09T02:29:24.073717Z", "shell.execute_reply": "2024-09-09T02:29:24.073255Z", "shell.execute_reply.started": "2024-09-09T02:29:24.036958Z" } }, "outputs": [], "source": [ "# df_slice_2 = pd.read_csv(\"/home/tony/Data/Preference/30b_v1/interesting_clips_v4_t_1_20240808_slice.csv\")\n", "# df_total = pd.concat([df_slice, df_slice_2])\n", "# print(df_total.shape)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "execution": { "iopub.execute_input": "2024-09-09T02:29:24.074442Z", "iopub.status.busy": "2024-09-09T02:29:24.074310Z", "iopub.status.idle": "2024-09-09T02:29:24.110065Z", "shell.execute_reply": "2024-09-09T02:29:24.109605Z", "shell.execute_reply.started": "2024-09-09T02:29:24.074429Z" } }, "outputs": [], "source": [ "# print(df_slices_2.shape, df_slice.shape)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "execution": { "iopub.execute_input": "2024-09-09T02:29:24.110961Z", "iopub.status.busy": "2024-09-09T02:29:24.110665Z", "iopub.status.idle": "2024-09-09T02:29:24.150002Z", "shell.execute_reply": "2024-09-09T02:29:24.149546Z", "shell.execute_reply.started": "2024-09-09T02:29:24.110946Z" } }, "outputs": [], "source": [ "# print(df_total.shape)\n", "# df_slice = df_total.copy()" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T14:00:20.866354Z", "start_time": "2024-05-16T14:00:12.443344Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:29:24.150716Z", "iopub.status.busy": "2024-09-09T02:29:24.150589Z", "iopub.status.idle": "2024-09-09T02:29:24.184862Z", "shell.execute_reply": "2024-09-09T02:29:24.184377Z", "shell.execute_reply.started": "2024-09-09T02:29:24.150704Z" } }, "outputs": [], "source": [ "# df_slice.to_csv(\"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_20240902_slice.csv\", index=False)\n", "# BREAK" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Need to kick out the ones has gpt prompt -- these are pairs with different text inputs" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.932966Z", "start_time": "2024-05-16T13:59:41.932957Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:29:24.185592Z", "iopub.status.busy": "2024-09-09T02:29:24.185458Z", "iopub.status.idle": "2024-09-09T02:29:24.250359Z", "shell.execute_reply": "2024-09-09T02:29:24.249790Z", "shell.execute_reply.started": "2024-09-09T02:29:24.185579Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "243001\n" ] } ], "source": [ "final_filtered_requests = df_slice[\"s3_id\"].astype(str).unique()\n", "print(len(final_filtered_requests))" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.933558Z", "start_time": "2024-05-16T13:59:41.933550Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:29:24.251199Z", "iopub.status.busy": "2024-09-09T02:29:24.251050Z", "iopub.status.idle": "2024-09-09T02:29:24.254937Z", "shell.execute_reply": "2024-09-09T02:29:24.254465Z", "shell.execute_reply.started": "2024-09-09T02:29:24.251185Z" } }, "outputs": [], "source": [ "# df_slice.to_csv(\"/home/tony/Data/Preference/30b_v2/interesting_clips_v4_t_3_20240902_slice.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934277Z", "start_time": "2024-05-16T13:59:41.934268Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:29:24.255721Z", "iopub.status.busy": "2024-09-09T02:29:24.255584Z", "iopub.status.idle": "2024-09-09T02:29:25.198316Z", "shell.execute_reply": "2024-09-09T02:29:25.197600Z", "shell.execute_reply.started": "2024-09-09T02:29:24.255708Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "240570 2431\n", "(240570, 63) (2431, 63)\n" ] } ], "source": [ "train_requests, val_requests = train_test_split(\n", " sorted(list(final_filtered_requests)), test_size=0.01, random_state=42\n", ")\n", "print(len(train_requests), len(val_requests))\n", "\n", "train_df = df_slice[df_slice[\"s3_id\"].astype(str).isin(set(train_requests))].copy()\n", "val_df = df_slice[df_slice[\"s3_id\"].astype(str).isin(set(val_requests))].copy()\n", "train_df = train_df.sort_values(by=[\"s3_id\"])\n", "train_df = train_df.reset_index()\n", "val_df = val_df.sort_values(by=[\"s3_id\"])\n", "val_df = val_df.reset_index()\n", "\n", "print(train_df.shape, val_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Actually make" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.934954Z", "start_time": "2024-05-16T13:59:41.934946Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:29:25.201414Z", "iopub.status.busy": "2024-09-09T02:29:25.200946Z", "iopub.status.idle": "2024-09-09T02:29:25.203694Z", "shell.execute_reply": "2024-09-09T02:29:25.203191Z", "shell.execute_reply.started": "2024-09-09T02:29:25.201394Z" } }, "outputs": [], "source": [ "# val_df[[\"request_id\", \"metadata\", \"updated_at\", \"user_id\", \"preference\"]].head()" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.935620Z", "start_time": "2024-05-16T13:59:41.935613Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:29:25.204482Z", "iopub.status.busy": "2024-09-09T02:29:25.204346Z", "iopub.status.idle": "2024-09-09T02:29:32.572128Z", "shell.execute_reply": "2024-09-09T02:29:32.571402Z", "shell.execute_reply.started": "2024-09-09T02:29:25.204469Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████| 240570/240570 [00:07<00:00, 32846.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "13,738 hours of 240570 clips, 15.035625 nodes, 2505.9375 iters\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "total_duration = 0\n", "for i, row in tqdm(train_df.iterrows(), total=len(train_df)):\n", " # we need to alternate between preference: neg, pos\n", " # print(i, row)\n", " total_duration += row[\"duration\"]\n", " \n", "print(\n", " f\"{round(total_duration / 60 / 60):,} hours of {train_df.shape[0]} clips, {train_df.shape[0] / 8 / 2 / 1000} nodes, {train_df.shape[0] / 8 / 2 / 6} iters\"\n", ")" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936268Z", "start_time": "2024-05-16T13:59:41.936260Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:29:32.573139Z", "iopub.status.busy": "2024-09-09T02:29:32.572974Z", "iopub.status.idle": "2024-09-09T02:30:04.425584Z", "shell.execute_reply": "2024-09-09T02:30:04.424866Z", "shell.execute_reply.started": "2024-09-09T02:29:32.573122Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2431/2431 [00:31<00:00, 76.36it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 2431 clips, 0 different prompts\n", "134 hours of concat\n", "Done\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "make_dataset(val_df, OUT_DATA_DIR, is_val=True, npz_dir=NPZ_DIR)" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.936964Z", "start_time": "2024-05-16T13:59:41.936957Z" }, "execution": { "iopub.execute_input": "2024-09-09T02:30:04.426584Z", "iopub.status.busy": "2024-09-09T02:30:04.426416Z", "iopub.status.idle": "2024-09-09T03:19:39.778005Z", "shell.execute_reply": "2024-09-09T03:19:39.777436Z", "shell.execute_reply.started": "2024-09-09T02:30:04.426568Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 240570/240570 [49:35<00:00, 80.86it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Total 240570 clips, 0 different prompts\n", "13,267 hours of concat\n", "Done\n" ] } ], "source": [ "make_dataset(train_df, OUT_DATA_DIR, is_val=False, npz_dir=NPZ_DIR)" ] }, { "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2024-01-29T19:46:47.549860Z", "start_time": "2024-01-29T19:46:47.548015Z" } }, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.937879Z", "start_time": "2024-05-16T13:59:41.937870Z" }, "execution": { "iopub.execute_input": "2024-09-09T03:19:39.778896Z", "iopub.status.busy": "2024-09-09T03:19:39.778738Z", "iopub.status.idle": "2024-09-09T03:19:39.928185Z", "shell.execute_reply": "2024-09-09T03:19:39.927695Z", "shell.execute_reply.started": "2024-09-09T03:19:39.778880Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, f\"meta_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_val.json\"))\n", "mm = mm.reshape(-1, 6016, 13)\n", "assert len(mm) == len(test_metas)\n", "assert mm[:100, :, 0].min() >= 0\n", "assert mm[:100, :, 0].max() <= 4000\n", "assert mm[:100, :, 1:].min() >= 0\n", "assert mm[:100, :, 1:].max() <= 2048" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.938629Z", "start_time": "2024-05-16T13:59:41.938621Z" }, "execution": { "iopub.execute_input": "2024-09-09T03:19:39.929116Z", "iopub.status.busy": "2024-09-09T03:19:39.928832Z", "iopub.status.idle": "2024-09-09T03:19:39.931192Z", "shell.execute_reply": "2024-09-09T03:19:39.930804Z", "shell.execute_reply.started": "2024-09-09T03:19:39.929099Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c_12cb import (\n", "# encode as codec_encode,\n", "# decode_stream_to_full_audio as codec_decode,\n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# decode as decode\n", "# )\n", "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "# _ = preload_codec_models(\"/app/suno/data/dpo/models/dac_2c_25x12.pt\", device=\"cuda\")\n", "# assert len(test_metas) == len(mm)\n", "# idx_list = list(range(len(test_metas)))\n", "# # random.shuffle(idx_list)\n", "# # idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939205Z", "start_time": "2024-05-16T13:59:41.939198Z" }, "execution": { "iopub.execute_input": "2024-09-09T03:19:39.932012Z", "iopub.status.busy": "2024-09-09T03:19:39.931873Z", "iopub.status.idle": "2024-09-09T03:19:39.968076Z", "shell.execute_reply": "2024-09-09T03:19:39.967693Z", "shell.execute_reply.started": "2024-09-09T03:19:39.931999Z" } }, "outputs": [], "source": [ "# import random\n", "# idx = random.choice(test_info[\"perference_0\"][\"idx_list\"])\n", "# assert \"original_duration_s\" in test_metas[idx]\n", "# # positive index should be shifted by 1\n", "# pos_idx = idx + 1\n", "# print(\n", "# \"tags:\",\n", "# test_metas[idx].get(\"tags\") == test_metas[pos_idx].get(\"tags\"),\n", "# test_metas[idx].get(\"tags\"),\n", "# )\n", "# arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pos_arr = mm[pos_idx, 1:].copy().astype(np.int16)[:, 1:]\n", "# pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[: pad_idx_arr[0], :]\n", "# pos_pad_idx_arr = np.where(pos_arr == COARSE_PAD_TOKEN)[0]\n", "# if len(pos_pad_idx_arr) > 0:\n", "# pos_arr = pos_arr[: pos_pad_idx_arr[0], :]\n", "# a = decode(arr)\n", "# print(\"\\n negative example \\n\", test_metas[idx])\n", "# a.play(compress=False)\n", "# pos_a = decode(pos_arr)\n", "# print(\"\\n positive example \\n\", test_metas[pos_idx])\n", "# pos_a.play(compress=False)\n", "# print(\n", "# \"text:\",\n", "# test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"),\n", "# test_metas[idx].get(\"text\"),\n", "# )" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.939977Z", "start_time": "2024-05-16T13:59:41.939969Z" }, "execution": { "iopub.execute_input": "2024-09-09T03:19:39.968717Z", "iopub.status.busy": "2024-09-09T03:19:39.968583Z", "iopub.status.idle": "2024-09-09T03:19:40.004055Z", "shell.execute_reply": "2024-09-09T03:19:40.003675Z", "shell.execute_reply.started": "2024-09-09T03:19:39.968703Z" } }, "outputs": [], "source": [ "# val_df[val_df[\"tags\"] == 'a vibrant blend of experimental jazz fusion, drum-and-bass and swagger fuzzed-out guitars']" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.940610Z", "start_time": "2024-05-16T13:59:41.940603Z" }, "execution": { "iopub.execute_input": "2024-09-09T03:19:40.004795Z", "iopub.status.busy": "2024-09-09T03:19:40.004670Z", "iopub.status.idle": "2024-09-09T03:19:40.039770Z", "shell.execute_reply": "2024-09-09T03:19:40.039382Z", "shell.execute_reply.started": "2024-09-09T03:19:40.004782Z" } }, "outputs": [], "source": [ "# from collections import Counter\n", "# c = Counter()\n", "# for _, row in df_slice.iterrows():\n", "# # print(row[\"metadata\"])\n", "# for k in ast.literal_eval(row[\"metadata\"]).keys():\n", "# c[k] += 1" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941167Z", "start_time": "2024-05-16T13:59:41.941159Z" }, "execution": { "iopub.execute_input": "2024-09-09T03:19:40.040405Z", "iopub.status.busy": "2024-09-09T03:19:40.040278Z", "iopub.status.idle": "2024-09-09T03:19:40.077067Z", "shell.execute_reply": "2024-09-09T03:19:40.076692Z", "shell.execute_reply.started": "2024-09-09T03:19:40.040392Z" } }, "outputs": [], "source": [ "# original_npz_path = f\"/app/suno/data/dpo/7b_npz/{test_metas[idx]['id']}.npz\"\n", "# original_npz_path = \"/app/suno/data/dpo/7b_npz/729c3011-f672-4ccd-8d82-1cbf2b52ff69.npz\"\n", "# original_arr = np.load(original_npz_path)[\"v2_raw\"]" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.941801Z", "start_time": "2024-05-16T13:59:41.941793Z" }, "execution": { "iopub.execute_input": "2024-09-09T03:19:40.077683Z", "iopub.status.busy": "2024-09-09T03:19:40.077562Z", "iopub.status.idle": "2024-09-09T03:19:40.626154Z", "shell.execute_reply": "2024-09-09T03:19:40.625564Z", "shell.execute_reply.started": "2024-09-09T03:19:40.077671Z" } }, "outputs": [ { "ename": "IndexError", "evalue": "list index out of range", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[31], line 16\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28mprint\u001b[39m(total_good, total_bad)\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[0;32m---> 16\u001b[0m \u001b[43mvalidation_on_metas\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtest_metas\u001b[49m\u001b[43m)\u001b[49m\n", "Cell \u001b[0;32mIn[31], line 7\u001b[0m, in \u001b[0;36mvalidation_on_metas\u001b[0;34m(input_metas)\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m idx \u001b[38;5;241m%\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 6\u001b[0m pos_idx \u001b[38;5;241m=\u001b[39m idx \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[0;32m----> 7\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m input_metas[idx]\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtags\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;241m!=\u001b[39m \u001b[43minput_metas\u001b[49m\u001b[43m[\u001b[49m\u001b[43mpos_idx\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtags\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m 8\u001b[0m \u001b[38;5;66;03m# print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\u001b[39;00m\n\u001b[1;32m 9\u001b[0m total_bad \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", "\u001b[0;31mIndexError\u001b[0m: list index out of range" ] } ], "source": [ "def validation_on_metas(input_metas):\n", " total_bad = 0\n", " total_good = 0\n", " for idx in range(len(input_metas)):\n", " if idx % 2 == 0:\n", " pos_idx = idx + 1\n", " if input_metas[idx].get(\"tags\") != input_metas[pos_idx].get(\"tags\"):\n", " # print(test_metas[idx].get(\"text\") == test_metas[pos_idx].get(\"text\"), test_metas[idx].get(\"tags\"), test_metas[pos_idx].get(\"tags\"))\n", " total_bad += 1\n", " else:\n", " total_good += 1\n", " print(total_good, total_bad)\n", " return\n", "\n", "\n", "validation_on_metas(test_metas)" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.942520Z", "start_time": "2024-05-16T13:59:41.942511Z" }, "execution": { "iopub.execute_input": "2024-09-09T03:33:52.258824Z", "iopub.status.busy": "2024-09-09T03:33:52.258353Z", "iopub.status.idle": "2024-09-09T03:33:52.286021Z", "shell.execute_reply": "2024-09-09T03:33:52.285553Z", "shell.execute_reply.started": "2024-09-09T03:33:52.258805Z" } }, "outputs": [], "source": [ "train_info = read_json(os.path.join(OUT_DATA_DIR, f\"info_tr.json\"))" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.943072Z", "start_time": "2024-05-16T13:59:41.943065Z" }, "execution": { "iopub.execute_input": "2024-09-09T03:33:52.470306Z", "iopub.status.busy": "2024-09-09T03:33:52.470021Z", "iopub.status.idle": "2024-09-09T03:33:52.486063Z", "shell.execute_reply": "2024-09-09T03:33:52.485491Z", "shell.execute_reply.started": "2024-09-09T03:33:52.470290Z" } }, "outputs": [ { "ename": "KeyError", "evalue": "'perference_0'", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[33], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m n_neg_tr \u001b[38;5;241m=\u001b[39m \u001b[43mtrain_info\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mperference_0\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124midx_list\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 2\u001b[0m n_pos_tr \u001b[38;5;241m=\u001b[39m train_info[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mperference_1\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124midx_list\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(n_pos_tr) \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mlen\u001b[39m(n_neg_tr)\n", "\u001b[0;31mKeyError\u001b[0m: 'perference_0'" ] } ], "source": [ "n_neg_tr = train_info[\"perference_0\"][\"idx_list\"]\n", "n_pos_tr = train_info[\"perference_1\"][\"idx_list\"]\n", "assert len(n_pos_tr) == len(n_neg_tr)" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.944246Z", "start_time": "2024-05-16T13:59:41.944237Z" }, "execution": { "iopub.execute_input": "2024-09-09T03:33:52.638268Z", "iopub.status.busy": "2024-09-09T03:33:52.637998Z", "iopub.status.idle": "2024-09-09T03:33:52.649847Z", "shell.execute_reply": "2024-09-09T03:33:52.649302Z", "shell.execute_reply.started": "2024-09-09T03:33:52.638252Z" } }, "outputs": [ { "ename": "NameError", "evalue": "name 'n_neg_tr' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[34], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m total_iters \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(\u001b[43mn_neg_tr\u001b[49m) \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mlen\u001b[39m(n_pos_tr)\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtotal samples\u001b[39m\u001b[38;5;124m\"\u001b[39m, total_iters, train_df\u001b[38;5;241m.\u001b[39mshape)\n", "\u001b[0;31mNameError\u001b[0m: name 'n_neg_tr' is not defined" ] } ], "source": [ "total_iters = len(n_neg_tr) + len(n_pos_tr)\n", "print(\"total samples\", total_iters, train_df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945249Z", "start_time": "2024-05-16T13:59:41.945241Z" }, "execution": { "iopub.status.busy": "2024-09-09T03:19:40.629292Z", "iopub.status.idle": "2024-09-09T03:19:40.629461Z", "shell.execute_reply": "2024-09-09T03:19:40.629378Z", "shell.execute_reply.started": "2024-09-09T03:19:40.629370Z" } }, "outputs": [], "source": [ "print(\"1 epoch per batch 2, total\", total_iters / 8 / 2 / 4)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.945972Z", "start_time": "2024-05-16T13:59:41.945964Z" }, "execution": { "iopub.status.busy": "2024-09-09T03:19:40.629929Z", "iopub.status.idle": "2024-09-09T03:19:40.630084Z", "shell.execute_reply": "2024-09-09T03:19:40.630009Z", "shell.execute_reply.started": "2024-09-09T03:19:40.630002Z" } }, "outputs": [], "source": [ "# !cd /home/tony/Work/tony/slurm/30b_dpo && sbatch sbatch_ipo_30b" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.status.busy": "2024-09-09T03:19:40.630571Z", "iopub.status.idle": "2024-09-09T03:19:40.630727Z", "shell.execute_reply": "2024-09-09T03:19:40.630655Z", "shell.execute_reply.started": "2024-09-09T03:19:40.630647Z" } }, "outputs": [], "source": [ "import shutil\n", "\n", "# Basic file copy\n", "shutil.copy('/home/tony/Work/tony/Preference/make_dataset_13b_v4_concat.ipynb', os.path.join(OUT_DATA_DIR, \"make_dataset.ipynb\"))\n", "print(\"Cache kept!\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# some gymathtics loading prev data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-05-16T13:59:41.946562Z", "start_time": "2024-05-16T13:59:41.946555Z" }, "execution": { "iopub.status.busy": "2024-09-09T03:19:40.631238Z", "iopub.status.idle": "2024-09-09T03:19:40.631396Z", "shell.execute_reply": "2024-09-09T03:19:40.631322Z", "shell.execute_reply.started": "2024-09-09T03:19:40.631314Z" } }, "outputs": [], "source": [ "# prev_v3_data = \"/app/suno/data/dpo/7v_v20_full/\"\n", "\n", "# test_val_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_val.jsonl\"))\n", "# test_tr_metas = read_jsonl(os.path.join(prev_v3_data, f\"meta_tr.jsonl\"))\n", "\n", "# all_ids = set()\n", "# for meta in test_val_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# for meta in test_tr_metas:\n", "# all_ids.add(meta[\"id\"])\n", "# print(len(all_ids), len(test_val_metas) + len(test_tr_metas))\n", "\n", "# all_ids = list(all_ids)\n", "# with open(\"/home/tony/Data/Preference/7b_v2/7v_v20_full_recut_id.json\", \"w\") as fp:\n", "# json.dump(all_ids, fp)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }