{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c666bc86",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-06T15:08:59.383231Z",
     "start_time": "2024-02-06T15:08:59.261451Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:16:15.434923Z",
     "iopub.status.busy": "2024-09-25T02:16:15.434788Z",
     "iopub.status.idle": "2024-09-25T02:16:15.567125Z",
     "shell.execute_reply": "2024-09-25T02:16:15.566661Z",
     "shell.execute_reply.started": "2024-09-25T02:16:15.434909Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "compute-h100-ord-node-13\n"
     ]
    }
   ],
   "source": [
    "!echo $HOSTNAME\n",
    "\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"\n",
    "\n",
    "# setup autoload\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "76fc0b2c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-06T15:08:59.448966Z",
     "start_time": "2024-02-06T15:08:59.387735Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:16:16.395515Z",
     "iopub.status.busy": "2024-09-25T02:16:16.395363Z",
     "iopub.status.idle": "2024-09-25T02:16:16.401927Z",
     "shell.execute_reply": "2024-09-25T02:16:16.401525Z",
     "shell.execute_reply.started": "2024-09-25T02:16:16.395501Z"
    }
   },
   "outputs": [],
   "source": [
    "# vocab: \n",
    "#   0-60_000 text\n",
    "#   1x0-3999   semantic\n",
    "#   8x0-4095  coarse\n",
    "\n",
    "#   4000 semantic pad token\n",
    "#   4001 semantic infer token\n",
    "#   2048 coarse pad token\n",
    "#   2049 coarse infer token\n",
    "\n",
    "# Memmaps:\n",
    "#   Nx9x3584 for audio tokens\n",
    "# Jsons:\n",
    "#   N*Dict with meta keys \n",
    "#     \"dataset\"\n",
    "#     \"original_id\", \"original_duration_s\",\n",
    "#     \"start_s\", \"end_s\", \n",
    "#     \"text_segments\", \"private_text_segments\",\n",
    "#     \"text\", \"private_text\",\n",
    "#     \"tags\", \"private_tags\",\n",
    "#     \"views\",\n",
    "#   Dict with meta keys {\"dataset\": [\"idx_list\"]}\n",
    "\n",
    "# Bundles (mert_v2_2x1k & dac_2c_25_8):\n",
    "# s3://suno-data/datasets/bundles/\n",
    "#  v1/youtube_music\n",
    "#  v1/genius_hq\n",
    "#  v1/jamendo\n",
    "#  v1/imslp\n",
    "#  v2/pond5\n",
    "#  v2/deezer\n",
    "#  v2/ytm_tagged\n",
    "#  v2/ytm_mb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ce43e8ad",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-06T15:09:00.103537Z",
     "start_time": "2024-02-06T15:08:59.449891Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:16:17.570935Z",
     "iopub.status.busy": "2024-09-25T02:16:17.570785Z",
     "iopub.status.idle": "2024-09-25T02:16:18.442601Z",
     "shell.execute_reply": "2024-09-25T02:16:18.442102Z",
     "shell.execute_reply.started": "2024-09-25T02:16:17.570922Z"
    }
   },
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "import sys\n",
    "\n",
    "sys.path.insert(0, \"/home/tony/Work/neon/sunoGPT/scripts/\")\n",
    "\n",
    "from data_preparation_30b import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d188ba34",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-06T15:09:00.107096Z",
     "start_time": "2024-02-06T15:09:00.105478Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-05T19:53:39.907185Z",
     "iopub.status.busy": "2024-09-05T19:53:39.906820Z",
     "iopub.status.idle": "2024-09-05T19:53:39.915512Z",
     "shell.execute_reply": "2024-09-05T19:53:39.915095Z",
     "shell.execute_reply.started": "2024-09-05T19:53:39.907164Z"
    }
   },
   "outputs": [],
   "source": [
    "# !aws s3 cp s3://suno-data/georg/data/chirp_v2_5/filtered_metas/genius_hq.jsonl /app/suno/data/chirp_v3/metadata/\n",
    "# !aws s3 cp s3://suno-data/georg/data/chirp_v2_5/filtered_metas/youtube_music.jsonl /app/suno/data/chirp_v3/metadata/\n",
    "# !aws s3 cp s3://suno-data/georg/data/chirp_v2_5/filtered_metas/jamendo.jsonl /app/suno/data/chirp_v3/metadata/\n",
    "# !aws s3 cp s3://suno-data/georg/data/chirp_v2_5/filtered_metas/imslp.jsonl /app/suno/data/chirp_v3/metadata/\n",
    "# !aws s3 cp s3://suno-data/georg/data/chirp_v2_5/filtered_metas/pond5_music.jsonl /app/suno/data/chirp_v3/metadata/\n",
    "# !aws s3 cp s3://suno-data/georg/data/chirp_v2_5/filtered_metas/deezer.jsonl /app/suno/data/chirp_v3/metadata/\n",
    "# !aws s3 cp s3://suno-data/georg/data/chirp_v2_5/filtered_metas/ytm_tagged.jsonl /app/suno/data/chirp_v3/metadata/\n",
    "# !aws s3 cp s3://suno-data/georg/data/chirp_v2_5/filtered_metas/musescore.jsonl /app/suno/data/chirp_v3/metadata/\n",
    "# !aws s3 cp s3://suno-data/georg/data/chirp_v2_5/filtered_metas/ytm_mb.jsonl /app/suno/data/chirp_v3/metadata/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "59ad0b39",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-06T15:09:00.170614Z",
     "start_time": "2024-02-06T15:09:00.108075Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:16:29.495977Z",
     "iopub.status.busy": "2024-09-25T02:16:29.495262Z",
     "iopub.status.idle": "2024-09-25T02:16:29.510797Z",
     "shell.execute_reply": "2024-09-25T02:16:29.510330Z",
     "shell.execute_reply.started": "2024-09-25T02:16:29.495960Z"
    }
   },
   "outputs": [],
   "source": [
    "OUT_DATA_DIR = \"/app/suno/data/dpo/chirp_v4_multi_ft_classical_t1\"\n",
    "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n",
    "METAS_DIR = \"metadata/\"\n",
    "NJOBS = 64\n",
    "CHUNKSIZE = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b30bfb77",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-06T15:12:28.656744Z",
     "start_time": "2024-02-06T15:09:00.171671Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:16:46.291452Z",
     "iopub.status.busy": "2024-09-25T02:16:46.290843Z",
     "iopub.status.idle": "2024-09-25T02:16:46.427681Z",
     "shell.execute_reply": "2024-09-25T02:16:46.427205Z",
     "shell.execute_reply.started": "2024-09-25T02:16:46.291435Z"
    }
   },
   "outputs": [],
   "source": [
    "# load manifests of IDs and text and tags etc\n",
    "# loading everything can take a while...4 mins\n",
    "# meta_info_map = {\n",
    "#     \"genius_hq\": {m[\"id\"]: m for m in read_jsonl(os.path.join(\"/home/tony/Work/tony/FineTuning_chirp_v3_4min/metadata/\", \"genius_hq_v5.jsonl\"))},\n",
    "#     \"youtube_music\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"youtube_music.jsonl\"))},\n",
    "#     \"jamendo\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"jamendo.jsonl\"))},\n",
    "#     \"imslp\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"imslp.jsonl\"))},\n",
    "#     \"pond5_music\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"pond5_music.jsonl\"))},\n",
    "#     \"deezer\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"deezer.jsonl\"))},\n",
    "#     \"ytm_tagged\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"ytm_tagged.jsonl\"))},\n",
    "#     \"musescore\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"musescore.jsonl\"))},\n",
    "#     # \"ytm_mb\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"ytm_mb.jsonl\"))}, # this is the large one in case we load full\n",
    "# }\n",
    "\n",
    "meta_info_map = {\n",
    "    # \"genius_hq\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"genius_hq_v8_t3.jsonl\"))},\n",
    "    # \"youtube_music\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"youtube_music_v8_t3.jsonl\"))},\n",
    "    # \"jamendo\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"jamendo_t3.jsonl\"))},\n",
    "    # \"pond5_music\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"pond5_music_t3.jsonl\"))},\n",
    "    \"imslp\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"imslp_filtered_4min.jsonl\"))},\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1f621940",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-06T15:12:30.341906Z",
     "start_time": "2024-02-06T15:12:28.658178Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:16:47.294656Z",
     "iopub.status.busy": "2024-09-25T02:16:47.294194Z",
     "iopub.status.idle": "2024-09-25T02:16:47.303991Z",
     "shell.execute_reply": "2024-09-25T02:16:47.303554Z",
     "shell.execute_reply.started": "2024-09-25T02:16:47.294641Z"
    }
   },
   "outputs": [],
   "source": [
    "# with open(\"/app/suno/data/preprocessing/meta_cutoff_freq.json\", \"r\") as fp:\n",
    "#     meta_cutoff_freq = json.load(fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0103ec4c",
   "metadata": {},
   "source": [
    "## Validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2a01e35c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-06T15:16:03.108826Z",
     "start_time": "2024-02-06T15:12:30.343172Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:17:09.847672Z",
     "iopub.status.busy": "2024-09-25T02:17:09.847053Z",
     "iopub.status.idle": "2024-09-25T02:17:12.188074Z",
     "shell.execute_reply": "2024-09-25T02:17:12.187552Z",
     "shell.execute_reply.started": "2024-09-25T02:17:09.847655Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start prepare data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:02<00:00,  2.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 hours of imslp\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# (start_idx, end_idx), n_archives_semantic, n_archives_coarse\n",
    "datasets = [\n",
    "    # (\"youtube_music\", \"v1\", (0, 3), 1, 1, \"default\"),\n",
    "    # (\"genius_hq\", \"v1\", (0, 3), 1, 1, \"default\"),\n",
    "    (\"imslp\", \"v1\", (0, 1), 1, 1, \"default\"),\n",
    "    # (\"jamendo\", \"v1\", (0, 2), 1, 1, \"default\"),\n",
    "    # (\"pond5_music\", \"v2\", (0, 3), 1, 1, \"default\"),\n",
    "    # (\"deezer\", \"v2\", (0, 1), 1, 1),\n",
    "    # (\"ytm_tagged\", \"v2\", (0, 1), 1, 1),\n",
    "    # (\"musescore\", \"v2\", (0, 1), 1, 1),\n",
    "    # (\"ytm_mb\", \"v2\", (0, 1), 1, 1),\n",
    "]\n",
    "prep_data(\n",
    "    datasets,\n",
    "    out_data_dir=OUT_DATA_DIR,\n",
    "    meta_info_map=meta_info_map,\n",
    "    is_val=True,\n",
    "    njobs=NJOBS,\n",
    "    chunksize=CHUNKSIZE,\n",
    ")\n",
    "# 24 hours of youtube_music\n",
    "#  3 hours of youtube_music_lyrics\n",
    "#  2 hours of youtube_music_lyrics_foreign\n",
    "#  4 hours of genius_hq\n",
    "#  9 hours of genius_hq_lyrics\n",
    "#  5 hours of genius_hq_lyrics_foreign\n",
    "# 17 hours of imslp\n",
    "# 30 hours of jamendo\n",
    "# 14 hours of pond5_music\n",
    "# 12 hours of deezer\n",
    "#  7 hours of deezer_lyrics\n",
    "#  8 hours of deezer_lyrics_foreign\n",
    "# 30 hours of ytm_tagged\n",
    "#  3 hours of musescore_lyrics\n",
    "# 30 hours of ytm_mb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "956bd1f3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-06T15:16:03.111591Z",
     "start_time": "2024-02-06T15:16:03.110031Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:17:14.838748Z",
     "iopub.status.busy": "2024-09-25T02:17:14.838246Z",
     "iopub.status.idle": "2024-09-25T02:17:15.112501Z",
     "shell.execute_reply": "2024-09-25T02:17:15.112052Z",
     "shell.execute_reply.started": "2024-09-25T02:17:14.838732Z"
    }
   },
   "outputs": [],
   "source": [
    "# Stats:\n",
    "# tot expected: 4.5Tb\n",
    "# speed for ytm_mb: 53129/200*(3*60+40)/60/60 == 16 hours"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f177ab5",
   "metadata": {},
   "source": [
    "## Train data ~ 6hrs (1.2hr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a2dfa127",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.190Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:17:32.247016Z",
     "iopub.status.busy": "2024-09-25T02:17:32.246734Z",
     "iopub.status.idle": "2024-09-25T02:19:43.239604Z",
     "shell.execute_reply": "2024-09-25T02:19:43.238948Z",
     "shell.execute_reply.started": "2024-09-25T02:17:32.247000Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start prepare data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [02:10<00:00, 14.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "935 hours of imslp\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# (start_idx, end_idx), n_archives_semantic, n_archives_coarse\n",
    "datasets = [\n",
    "    # (\"youtube_music\", \"v1\", (3, 4204), 1, 1, \"default\"),\n",
    "    # (\"genius_hq\", \"v1\", (3, 4302), 1, 1, \"default\"),\n",
    "    (\"imslp\", \"v1\", (1, 558), 1, 1, \"default\"),\n",
    "    # (\"jamendo\", \"v1\", (2, 112), 1, 1, \"default\"),\n",
    "    # (\"pond5_music\", \"v2\", (3, 4138), 1, 1, \"default\"),\n",
    "    # (\"deezer\", \"v2\", (1, 1538), 1, 1),\n",
    "    # (\"ytm_tagged\", \"v2\", (1, 5545), 1, 1),\n",
    "    # (\"musescore\", \"v2\", (1, 33), 1, 1),\n",
    "#     (\"ytm_mb\", \"v2\", (1, 53129), 1, 1),\n",
    "#     (\"ytm_mb\", \"v2\", (1, 50001), 1, 1),\n",
    "]\n",
    "prep_data(\n",
    "    datasets,\n",
    "    out_data_dir=OUT_DATA_DIR,\n",
    "    meta_info_map=meta_info_map,\n",
    "    is_val=False,\n",
    "    njobs=NJOBS,\n",
    "    chunksize=CHUNKSIZE,\n",
    ")\n",
    "# youtube_music: ~1h runtime\n",
    "#    105,498 hours of youtube_music\n",
    "#     12,168 hours of youtube_music_lyrics\n",
    "#     10,363 hours of youtube_music_lyrics_foreign\n",
    "# genius_hq: ~1h runtime\n",
    "#     17,784 hours of genius_hq\n",
    "#     41,544 hours of genius_hq_lyrics\n",
    "#     17,469 hours of genius_hq_lyrics_foreign\n",
    "# imslp: ~8m runtime\n",
    "#     12,175 hours of imslp\n",
    "# jamendo: ~27m runtime\n",
    "#      3,251 hours of jamendo\n",
    "# pond5_music: ~51m runtime\n",
    "#     58,592 hours of pond5_music\n",
    "# deezer: ~19m runtime\n",
    "#     12,181 hours of deezer\n",
    "#     10,132 hours of deezer_lyrics\n",
    "#      6,770 hours of deezer_lyrics_foreign\n",
    "# ytm_tagged: ~1h runtime\n",
    "#    141,476 hours of ytm_tagged\n",
    "# musescore: ~5m runtime\n",
    "#        103 hours of ytm_tagged\n",
    "# ytm_mb: ~12h runtime\n",
    "#  1,642,164 hours of ytm_mb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "446fc413-18bd-4a85-8dbe-e67a09fc51c8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-25T02:20:31.367184Z",
     "iopub.status.busy": "2024-09-25T02:20:31.366719Z",
     "iopub.status.idle": "2024-09-25T02:20:31.825388Z",
     "shell.execute_reply": "2024-09-25T02:20:31.824831Z",
     "shell.execute_reply.started": "2024-09-25T02:20:31.367163Z"
    }
   },
   "outputs": [],
   "source": [
    "# !cd /home/tony/Work/tony/slurm/30b_ft && sbatch sbatch_run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0e44f3e9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-07T01:31:05.958033Z",
     "start_time": "2024-02-07T01:31:05.955694Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:20:31.826536Z",
     "iopub.status.busy": "2024-09-25T02:20:31.826375Z",
     "iopub.status.idle": "2024-09-25T02:20:31.869801Z",
     "shell.execute_reply": "2024-09-25T02:20:31.869317Z",
     "shell.execute_reply.started": "2024-09-25T02:20:31.826522Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/app/suno/data/dpo/chirp_v4_multi_ft_classical_t1\n"
     ]
    }
   ],
   "source": [
    "print(OUT_DATA_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2ab57df8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-07T01:32:40.626130Z",
     "start_time": "2024-02-07T01:32:40.428418Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:20:32.922324Z",
     "iopub.status.busy": "2024-09-25T02:20:32.922165Z",
     "iopub.status.idle": "2024-09-25T02:20:32.950554Z",
     "shell.execute_reply": "2024-09-25T02:20:32.950115Z",
     "shell.execute_reply.started": "2024-09-25T02:20:32.922310Z"
    }
   },
   "outputs": [],
   "source": [
    "# verify\n",
    "mm = np.memmap(os.path.join(OUT_DATA_DIR, \"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n",
    "metas = read_jsonl(os.path.join(OUT_DATA_DIR, \"metas_val.jsonl\"))\n",
    "mm = mm.reshape(-1, N_TOKENS_MEMMAP, 13)\n",
    "assert(len(mm) == len(metas))\n",
    "assert(mm[:100,:,0].min() >= 0)\n",
    "assert(mm[:100,:,0].max() <= 4000)\n",
    "assert(mm[:100,:,1:].min() >= 0)\n",
    "assert(mm[:100,:,1:].max() <= 2048)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e92d9857",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-07T01:35:45.914782Z",
     "start_time": "2024-02-07T01:32:44.592042Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:20:34.054200Z",
     "iopub.status.busy": "2024-09-25T02:20:34.054046Z",
     "iopub.status.idle": "2024-09-25T02:20:34.214534Z",
     "shell.execute_reply": "2024-09-25T02:20:34.214092Z",
     "shell.execute_reply.started": "2024-09-25T02:20:34.054187Z"
    }
   },
   "outputs": [],
   "source": [
    "train_metas = read_jsonl(os.path.join(OUT_DATA_DIR, \"metas_tr.jsonl\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1ccdc2db-791a-498f-b788-872b42f22c80",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-25T02:20:34.498466Z",
     "iopub.status.busy": "2024-09-25T02:20:34.498114Z",
     "iopub.status.idle": "2024-09-25T02:20:34.507594Z",
     "shell.execute_reply": "2024-09-25T02:20:34.507160Z",
     "shell.execute_reply.started": "2024-09-25T02:20:34.498450Z"
    }
   },
   "outputs": [],
   "source": [
    "from suno_utils.utils.text import read_jsonl, read_json\n",
    "test_info = read_json(os.path.join(OUT_DATA_DIR, \"info_val.json\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "15a2265d-c3bf-4b91-91d5-ddee396252b9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-25T02:20:35.026096Z",
     "iopub.status.busy": "2024-09-25T02:20:35.025939Z",
     "iopub.status.idle": "2024-09-25T02:20:35.036927Z",
     "shell.execute_reply": "2024-09-25T02:20:35.036510Z",
     "shell.execute_reply.started": "2024-09-25T02:20:35.026083Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['imslp'])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_info.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2bddbfd2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-07T01:35:45.920764Z",
     "start_time": "2024-02-07T01:35:45.916711Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-25T02:29:46.423698Z",
     "iopub.status.busy": "2024-09-25T02:29:46.423309Z",
     "iopub.status.idle": "2024-09-25T02:29:46.819750Z",
     "shell.execute_reply": "2024-09-25T02:29:46.819324Z",
     "shell.execute_reply.started": "2024-09-25T02:29:46.423678Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(29076, 605.75)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_metas), len(train_metas) / 8 / 3 / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c531753b-cb94-43b1-bed3-fcb1f5d9ddbd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-25T02:20:43.763181Z",
     "iopub.status.busy": "2024-09-25T02:20:43.762863Z",
     "iopub.status.idle": "2024-09-25T02:20:43.774916Z",
     "shell.execute_reply": "2024-09-25T02:20:43.774502Z",
     "shell.execute_reply.started": "2024-09-25T02:20:43.763164Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/app/suno/data/dpo/chirp_v4_multi_ft_classical_t1'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "OUT_DATA_DIR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "239421b5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-07T01:35:46.003673Z",
     "start_time": "2024-02-07T01:35:45.921856Z"
    },
    "execution": {
     "iopub.execute_input": "2024-09-05T21:30:54.169900Z",
     "iopub.status.busy": "2024-09-05T21:30:54.169523Z",
     "iopub.status.idle": "2024-09-05T21:30:54.372299Z",
     "shell.execute_reply": "2024-09-05T21:30:54.371619Z",
     "shell.execute_reply.started": "2024-09-05T21:30:54.169885Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'BREAK' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[21], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDone\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m \u001b[43mBREAK\u001b[49m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined"
     ]
    }
   ],
   "source": [
    "print(\"Done\")\n",
    "BREAK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5bdf8ac",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.192Z"
    }
   },
   "outputs": [],
   "source": [
    "# !du -hs /mnt/data/georg/data/chirp_v2/data_tr.bin\n",
    "# # 1003G"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbcbca32",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.192Z"
    }
   },
   "outputs": [],
   "source": [
    "# # randomly listen to some stuff\n",
    "# from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n",
    "# from suno_utils.tasks.dac_2c_12cb import (\n",
    "#     encode as codec_encode,\n",
    "#     decode as codec_decode, \n",
    "#     EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n",
    "# )\n",
    "# _ = preload_codec_models(\"/app/suno/models/dac_2c_25x12.pt\")\n",
    "# mm = np.memmap(\"/mnt/localdisk/test_data/data_val.bin\", dtype=np.uint16, mode=\"r\")\n",
    "# mm = mm.reshape(-1, 3008, 13)\n",
    "# test_metas = read_jsonl(\"/mnt/localdisk/data/metas_val.jsonl\")\n",
    "# test_info = read_json(\"/mnt/localdisk/data/info_val.json\")\n",
    "# assert(len(test_metas) == len(mm))\n",
    "# idx_list = list(range(len(test_metas)))\n",
    "# #random.shuffle(idx_list)\n",
    "# #idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n",
    "# print(len(mm))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7008ddc7",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.193Z"
    }
   },
   "outputs": [],
   "source": [
    "# # show text and audio\n",
    "# # idx = random.choice(idx_list)\n",
    "# # idx = random.choice(test_info[\"pond5_music\"][\"idx_list\"])\n",
    "# idx = random.choice(test_info[\"genius_hq_lyrics\"][\"idx_list\"])\n",
    "# # idx = random.choice(test_info[\"musescore_lyrics\"][\"idx_list\"])\n",
    "# # idx = random.choice(test_info[\"deezer_lyrics\"][\"idx_list\"])\n",
    "# # idx = random.choice(test_info[\"youtube_music_lyrics_foreign\"][\"idx_list\"])\n",
    "# # idx_key = random.choice(list(test_info.keys()))\n",
    "# # print(idx_key)\n",
    "# # idx = random.choice(test_info[idx_key][\"idx_list\"])\n",
    "# assert(\"original_duration_s\" in test_metas[idx])\n",
    "# print(\"tags:\", test_metas[idx].get(\"tags\"))\n",
    "# arr = mm[idx,1:].copy().astype(np.int16)[:,1:]\n",
    "# pad_idx_arr = np.where(arr == 4096)[0]\n",
    "# if len(pad_idx_arr) > 0:\n",
    "#     arr = arr[:pad_idx_arr[0],:]\n",
    "# a = codec_decode(arr)\n",
    "# a.play(compress=False)\n",
    "# print(\"text:\", test_metas[idx].get(\"text\"))\n",
    "# plt.plot(a.array_float[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f818ab33",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87b856f6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "62fb16fc",
   "metadata": {},
   "source": [
    "### try same with train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f987040e",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.194Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4471a7fb",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.195Z"
    }
   },
   "outputs": [],
   "source": [
    "# randomly listen to some stuff\n",
    "from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n",
    "from suno_utils.tasks.dac_2c_12cb import (\n",
    "    encode as codec_encode,\n",
    "    decode as codec_decode, \n",
    "    EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n",
    ")\n",
    "from suno_utils.utils.text import read_jsonl, read_json\n",
    "_ = preload_codec_models(\"/app/suno/models/dac_2c_25x12.pt\")\n",
    "mm = np.memmap(\"/mnt/localdisk/data/data_tr.bin\", dtype=np.uint16, mode=\"r\")\n",
    "mm = mm.reshape(-1, 3008, 13)\n",
    "test_metas = read_jsonl(\"/mnt/localdisk/data/metas_tr.jsonl\")\n",
    "test_info = read_json(\"/mnt/localdisk/data/info_tr.json\")\n",
    "assert(len(test_metas) == len(mm))\n",
    "idx_list = list(range(len(test_metas)))\n",
    "#random.shuffle(idx_list)\n",
    "#idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n",
    "print(len(mm))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecb73f42",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.195Z"
    }
   },
   "outputs": [],
   "source": [
    "import random\n",
    "# show text and audio\n",
    "# idx = random.choice(idx_list)\n",
    "# idx = random.choice(test_info[\"pond5_music\"][\"idx_list\"])\n",
    "# idx = random.choice(test_info[\"genius_hq_lyrics\"][\"idx_list\"])\n",
    "idx = random.choice(test_info[\"ytm_mb\"][\"idx_list\"])\n",
    "# idx = random.choice(test_info[\"musescore_lyrics\"][\"idx_list\"])\n",
    "# idx = random.choice(test_info[\"deezer_lyrics\"][\"idx_list\"])\n",
    "# idx = random.choice(test_info[\"youtube_music_lyrics_foreign\"][\"idx_list\"])\n",
    "# idx_key = random.choice(list(test_info.keys()))\n",
    "# print(idx_key)\n",
    "# idx = random.choice(test_info[idx_key][\"idx_list\"])\n",
    "assert(\"original_duration_s\" in test_metas[idx])\n",
    "print(\"tags:\", test_metas[idx].get(\"tags\"))\n",
    "arr = mm[idx,1:].copy().astype(np.int16)[:,1:]\n",
    "pad_idx_arr = np.where(arr == 4096)[0]\n",
    "if len(pad_idx_arr) > 0:\n",
    "    arr = arr[:pad_idx_arr[0],:]\n",
    "a = codec_decode(arr)\n",
    "a.play(compress=False)\n",
    "print(\"text:\", test_metas[idx].get(\"text\"))\n",
    "plt.plot(a.array_float[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee88a2eb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d977073",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2653b41",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "067a561b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73c92bda",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "08665ec0",
   "metadata": {},
   "source": [
    "## Playground"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "747f0363",
   "metadata": {},
   "source": [
    "#### get all tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73286512",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.196Z"
    }
   },
   "outputs": [],
   "source": [
    "test_metas = read_jsonl(\"/mnt/data/georg/data/chirp_v2/metas_val.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3ed7922",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.197Z"
    }
   },
   "outputs": [],
   "source": [
    "tags = []\n",
    "for e in train_metas:\n",
    "    tags.extend(e.get(\"tags\", []))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2eed69a8",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.197Z"
    }
   },
   "outputs": [],
   "source": [
    "vc_tags = pd.Series(tags).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf1c83e8",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.198Z"
    }
   },
   "outputs": [],
   "source": [
    "vc_tags.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f76f1e83",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.198Z"
    }
   },
   "outputs": [],
   "source": [
    "vc_tags = vc_tags.to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6766efb1",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.198Z"
    }
   },
   "outputs": [],
   "source": [
    "vc_tags.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fece8109",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.199Z"
    }
   },
   "outputs": [],
   "source": [
    "a = vc_tags[vc_tags[\"count\"]>=3][\"count\"].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52e1a982",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.199Z"
    }
   },
   "outputs": [],
   "source": [
    "test_metas = read_jsonl(\"/mnt/data/georg/data/chirp_v2/metas_val.jsonl\")\n",
    "test_info = read_json(\"/mnt/data/georg/data/chirp_v2/info_val.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6b9f19e",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-06T15:08:59.200Z"
    }
   },
   "outputs": [],
   "source": [
    "idx = random.choice(test_info[\"musescore_lyrics\"][\"idx_list\"])\n",
    "print(idx)\n",
    "m = test_metas[idx]\n",
    "re.sub(r\"\\[.*?\\]\", \"\", m[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dffab2d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4531aac2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0e1aafb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "143ac0a7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}