{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "100815b2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-04-17T14:03:41.146790Z",
     "start_time": "2024-04-17T14:03:39.701191Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1234263/325804709.py:1: DeprecationWarning: \n",
      "Pyarrow will become a required dependency of pandas in the next major release of pandas (pandas 3.0),\n",
      "(to allow more performant data types, such as the Arrow string type, and better interoperability with other libraries)\n",
      "but was not found to be installed on your system.\n",
      "If this would cause problems for you,\n",
      "please provide us feedback at https://github.com/pandas-dev/pandas/issues/54466\n",
      "        \n",
      "  import pandas as pd\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import train_test_split\n",
    "from suno_utils.utils.s3 import download_s3_files\n",
    "import sys\n",
    "from collections import defaultdict\n",
    "from suno_utils.utils.text import (\n",
    "    write_jsonl,\n",
    "    read_jsonl,\n",
    "    write_json,\n",
    "    read_json,\n",
    ")\n",
    "import shutil\n",
    "import ast\n",
    "from preference_helper import *\n",
    "from preference_data_preparation import make_dataset\n",
    "\n",
    "sys.path.insert(0, \"/home/tony/Work/glockenspiel/sunoGPT/scripts/\")\n",
    "\n",
    "from data_preparation_7b import *\n",
    "import numpy as np\n",
    "\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "98b3542c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-04-17T14:05:09.043810Z",
     "start_time": "2024-04-17T14:05:08.991125Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/app/suno/data/dpo/7v_v2_mix_v0v1/tokenizer_60k.json'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v3_7b_path = \"/app/suno/data/dpo/7v_v20_full_recode/\"\n",
    "v2_3b_path = \"/app/suno/data/dpo/v0_1/\"\n",
    "OUT_DATA_DIR = \"/app/suno/data/dpo/7v_v2_mix_v0v1/\"\n",
    "# OUT_DATA_DIR = \"/home/tony/Data/test/7v_v6_full/\"\n",
    "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n",
    "_ = shutil.copyfile(\"/app/suno/data/dpo/7v_v1_full/tokenizer_60k.json\", os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "98cef2de",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-04-17T14:07:22.324932Z",
     "start_time": "2024-04-17T14:07:22.254108Z"
    }
   },
   "outputs": [],
   "source": [
    "v2_mm = np.memmap(os.path.join(v2_3b_path, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n",
    "v2_test_metas = read_jsonl(os.path.join(v2_3b_path, f\"meta_val.jsonl\"))\n",
    "v2_test_info = read_json(os.path.join(v2_3b_path, f\"info_val.json\"))\n",
    "\n",
    "v3_mm = np.memmap(os.path.join(v3_7b_path, f\"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n",
    "v3_test_metas = read_jsonl(os.path.join(v3_7b_path, f\"meta_val.jsonl\"))\n",
    "v3_test_info = read_json(os.path.join(v3_7b_path, f\"info_val.json\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94d4414c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}