{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "561dc275", "metadata": { "execution": { "iopub.execute_input": "2024-08-03T04:23:42.040756Z", "iopub.status.busy": "2024-08-03T04:23:42.040253Z", "iopub.status.idle": "2024-08-03T04:23:42.160433Z", "shell.execute_reply": "2024-08-03T04:23:42.159878Z", "shell.execute_reply.started": "2024-08-03T04:23:42.040735Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "compute-h100-ord-node-635\n" ] } ], "source": [ "!echo $HOSTNAME" ] }, { "cell_type": "code", "execution_count": 2, "id": "250e8fae", "metadata": { "execution": { "iopub.execute_input": "2024-08-03T04:23:43.088776Z", "iopub.status.busy": "2024-08-03T04:23:43.088376Z", "iopub.status.idle": "2024-08-03T04:23:43.090944Z", "shell.execute_reply": "2024-08-03T04:23:43.090541Z", "shell.execute_reply.started": "2024-08-03T04:23:43.088756Z" } }, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": 3, "id": "ed2662f0", "metadata": { "execution": { "iopub.execute_input": "2024-08-03T04:23:45.709527Z", "iopub.status.busy": "2024-08-03T04:23:45.709154Z", "iopub.status.idle": "2024-08-03T04:23:45.711790Z", "shell.execute_reply": "2024-08-03T04:23:45.711402Z", "shell.execute_reply.started": "2024-08-03T04:23:45.709509Z" } }, "outputs": [], "source": [ "# vocab: \n", "# 0-60_000 text\n", "# 1x0-3999 semantic\n", "# 12x0-2047 coarse\n", "\n", "# 4000 semantic pad token\n", "# 4001 semantic infer token\n", "# 2048 coarse pad token\n", "# 2049 coarse infer token\n", "\n", "# Memmaps:\n", "# Nx9x3584 for audio tokens\n", "# Jsons:\n", "# N*Dict with meta keys \n", "# \"dataset\"\n", "# \"original_id\", \"original_duration_s\",\n", "# \"start_s\", \"end_s\", \n", "# \"text_segments\", \"private_text_segments\",\n", "# \"text\", \"private_text\",\n", "# \"tags\", \"private_tags\",\n", "# \"views\",\n", "# Dict with meta keys {\"dataset\": [\"idx_list\"]}\n", "\n", "# Bundles (mert_25_2x4k & dac_2c_25_12):\n", "# s3://suno-data/datasets/bundles/\n", "# v1/youtube_music\n", "# v1/genius_hq\n", "# v1/jamendo\n", "# v1/imslp\n", "# v2/pond5_music\n", "# v2/deezer\n", "# v2/ytm_tagged\n", "# v3/discogs\n", "# v3/discogs_covers" ] }, { "cell_type": "code", "execution_count": 4, "id": "9c4df11a", "metadata": { "execution": { "iopub.execute_input": "2024-08-03T04:23:46.291989Z", "iopub.status.busy": "2024-08-03T04:23:46.291831Z", "iopub.status.idle": "2024-08-03T04:23:46.666115Z", "shell.execute_reply": "2024-08-03T04:23:46.665638Z", "shell.execute_reply.started": "2024-08-03T04:23:46.291973Z" } }, "outputs": [], "source": [ "%matplotlib inline\n", "from matplotlib import pyplot as plt" ] }, { "cell_type": "code", "execution_count": 5, "id": "c7c6906e-4b9b-4f4c-ac73-69067a16f4d5", "metadata": { "execution": { "iopub.execute_input": "2024-08-03T04:23:46.760159Z", "iopub.status.busy": "2024-08-03T04:23:46.759819Z", "iopub.status.idle": "2024-08-03T04:23:46.872715Z", "shell.execute_reply": "2024-08-03T04:23:46.872152Z", "shell.execute_reply.started": "2024-08-03T04:23:46.760141Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/tony/Work/tony/FineTuning_chirp_v4\n" ] } ], "source": [ "!pwd" ] }, { "cell_type": "code", "execution_count": 9, "id": "6b35b54b", "metadata": { "execution": { "iopub.execute_input": "2024-08-03T04:24:11.761403Z", "iopub.status.busy": "2024-08-03T04:24:11.761026Z", "iopub.status.idle": "2024-08-03T04:24:11.768247Z", "shell.execute_reply": "2024-08-03T04:24:11.767825Z", "shell.execute_reply.started": "2024-08-03T04:24:11.761384Z" } }, "outputs": [], "source": [ "import sys\n", "import shutil\n", "\n", "sys.path.insert(0, \"/home/tony/Work/neon/sunoGPT/scripts/\")\n", "\n", "from data_preparation_30b import *\n", "\n", "METAS_DIR = \"metadata\"\n", "OUT_DATA_DIR = \"/app/suno/data/dpo/chirp_v4_multi_ft_t1\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 7, "id": "8e842574", "metadata": { "execution": { "iopub.execute_input": "2024-06-29T13:41:46.778069Z", "iopub.status.busy": "2024-06-29T13:41:46.777870Z", "iopub.status.idle": "2024-06-29T13:41:46.780169Z", "shell.execute_reply": "2024-06-29T13:41:46.779783Z", "shell.execute_reply.started": "2024-06-29T13:41:46.778053Z" } }, "outputs": [], "source": [ "# !aws s3 cp s3://suno-data/datasets/metadata/chirp_v4/genius_hq_v5_plus.jsonl /app/suno/data/chirp_v4/metadata/ \n", "# !aws s3 cp s3://suno-data/datasets/metadata/chirp_v4/youtube_music.jsonl /app/suno/data/chirp_v4/metadata/\n", "# !aws s3 cp s3://suno-data/datasets/metadata/chirp_v4/jamendo.jsonl /app/suno/data/chirp_v4/metadata/\n", "# !aws s3 cp s3://suno-data/datasets/metadata/chirp_v4/imslp.jsonl /app/suno/data/chirp_v4/metadata/\n", "# !aws s3 cp s3://suno-data/datasets/metadata/chirp_v4/pond5_music.jsonl /app/suno/data/chirp_v4/metadata/\n", "# !aws s3 cp s3://suno-data/datasets/metadata/chirp_v4/deezer.jsonl /app/suno/data/chirp_v4/metadata/\n", "# !aws s3 cp s3://suno-data/datasets/metadata/chirp_v4/ytm_tagged.jsonl /app/suno/data/chirp_v4/metadata/\n", "# !aws s3 cp s3://suno-data/datasets/metadata/chirp_v4/discogs.jsonl /app/suno/data/chirp_v4/metadata/\n", "# !aws s3 cp s3://suno-data/datasets/metadata/chirp_v4/bundle_cover_metas.jsonl /app/suno/data/chirp_v4/metadata/\n", "# !aws s3 cp s3://suno-data/georg/trained_models/chirp_v2/tokenizer_60k.json /app/suno/data/chirp_v4/multi/\n", "# !aws s3 cp s3://suno-data/datasets/harvest/ytm2/ytm_tagged_songs_rym.jsonl /app/suno/data/chirp_v4/metadata/" ] }, { "cell_type": "code", "execution_count": 8, "id": "15a1934f", "metadata": { "execution": { "iopub.execute_input": "2024-06-29T13:41:46.780861Z", "iopub.status.busy": "2024-06-29T13:41:46.780727Z", "iopub.status.idle": "2024-06-29T13:41:46.814001Z", "shell.execute_reply": "2024-06-29T13:41:46.813635Z", "shell.execute_reply.started": "2024-06-29T13:41:46.780846Z" } }, "outputs": [], "source": [ "# _ = shutil.copyfile(\"/app/suno/data/chirp_v4/multi/tokenizer_60k.json\", os.path.join(OUT_DATA_DIR, \"tokenizer_60k.json\"))" ] }, { "cell_type": "code", "execution_count": 9, "id": "8a4257e2", "metadata": { "execution": { "iopub.execute_input": "2024-06-29T13:41:46.814762Z", "iopub.status.busy": "2024-06-29T13:41:46.814624Z", "iopub.status.idle": "2024-06-29T13:43:42.650855Z", "shell.execute_reply": "2024-06-29T13:43:42.650291Z", "shell.execute_reply.started": "2024-06-29T13:41:46.814748Z" } }, "outputs": [], "source": [ "# load manifests of IDs and text and tags etc\n", "meta_info_map = {\n", " \"genius_hq\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"genius_hq_v5_plus.jsonl\"))},\n", " # \"youtube_music\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"youtube_music.jsonl\"))},\n", " # \"jamendo\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"jamendo.jsonl\"))},\n", " # \"imslp\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"imslp.jsonl\"))},\n", " # \"pond5_music\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"pond5_music.jsonl\"))},\n", " # \"deezer\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"deezer.jsonl\"))},\n", " # \"ytm_tagged\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"ytm_tagged_songs_rym.jsonl\"))},\n", " # \"discogs\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"discogs.jsonl\"))},\n", " # \"discogs_covers\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"bundle_cover_metas.jsonl\"))},\n", "}" ] }, { "cell_type": "code", "execution_count": 10, "id": "2da8574b", "metadata": { "execution": { "iopub.execute_input": "2024-06-29T13:43:42.651747Z", "iopub.status.busy": "2024-06-29T13:43:42.651593Z", "iopub.status.idle": "2024-06-29T13:43:42.654097Z", "shell.execute_reply": "2024-06-29T13:43:42.653705Z", "shell.execute_reply.started": "2024-06-29T13:43:42.651730Z" } }, "outputs": [], "source": [ "NJOBS = 64\n", "CHUNKSIZE = 64" ] }, { "cell_type": "code", "execution_count": 11, "id": "9af61149", "metadata": { "execution": { "iopub.execute_input": "2024-06-29T13:43:42.654819Z", "iopub.status.busy": "2024-06-29T13:43:42.654680Z", "iopub.status.idle": "2024-06-29T13:43:50.853396Z", "shell.execute_reply": "2024-06-29T13:43:50.852848Z", "shell.execute_reply.started": "2024-06-29T13:43:42.654804Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "start prepare data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:07<00:00, 7.72s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "15 hours of genius_hq_lyrics_foreign\n", "20 hours of genius_hq_lyrics\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "# (start_idx, end_idx), n_archives_semantic, n_archives_coarse\n", "datasets = [\n", " # (\"youtube_music\", \"v1\", (0, 1), 1, 1, \"default\"),\n", " (\"genius_hq\", \"v1\", (0, 1), 1, 1, \"default\"),\n", " # (\"jamendo\", \"v1\", (0, 1), 1, 1, \"default\"),\n", " # (\"imslp\", \"v1\", (0, 1), 1, 1, \"default\"),\n", " # (\"pond5_music\", \"v2\", (0, 1), 1, 1, \"default\"),\n", " # (\"deezer\", \"v2\", (0, 1), 1, 1, \"default\"),\n", " # (\"ytm_tagged\", \"v2\", (0, 1), 1, 1, \"default\"),\n", " # (\"discogs\", \"v3\", (0, 1), 1, 1, \"default\"),\n", " # (\"discogs_covers\", \"v3\", (0, 1), 1, 1, \"covers\"),\n", "]\n", "prep_data(\n", " datasets,\n", " out_data_dir=OUT_DATA_DIR,\n", " meta_info_map=meta_info_map,\n", " is_val=True,\n", " njobs=NJOBS,\n", " chunksize=CHUNKSIZE,\n", ")\n", "# 19 hours of youtube_music\n", "# 5 hours of youtube_music_lyrics\n", "# 4 hours of youtube_music_lyrics_foreign\n", "# 20 hours of genius_hq_lyrics\n", "# 15 hours of genius_hq_lyrics_foreign\n", "# 30 hours of jamendo\n", "# 17 hours of imslp\n", "# 14 hours of pond5_music\n", "# 2 hours of deezer\n", "# 11 hours of deezer_lyrics\n", "# 14 hours of deezer_lyrics_foreign\n", "# 30 hours of ytm_tagged\n", "# 28 hours of ytm_mb\n", "# 25 hours of discogs\n", "# 2 hours of discogs_lyrics\n", "# 1 hours of discogs_lyrics_foreign\n", "# 25 hours of discogs_covers" ] }, { "cell_type": "code", "execution_count": 12, "id": "f64a38dd", "metadata": { "execution": { "iopub.execute_input": "2024-06-29T13:43:50.854238Z", "iopub.status.busy": "2024-06-29T13:43:50.854088Z", "iopub.status.idle": "2024-06-29T14:59:44.517064Z", "shell.execute_reply": "2024-06-29T14:59:44.516384Z", "shell.execute_reply.started": "2024-06-29T13:43:50.854222Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "start prepare data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 68/68 [1:15:02<00:00, 66.21s/it]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "90,948 hours of genius_hq_lyrics\n", "49,259 hours of genius_hq_lyrics_foreign\n" ] } ], "source": [ "# (start_idx, end_idx), n_archives_semantic, n_archives_coarse\n", "datasets = [\n", " # (\"youtube_music\", \"v1\", (1, 4204), 1, 1, \"default\"),\n", " (\"genius_hq\", \"v1\", (1, 4302), 1, 1, \"default\"),\n", " # (\"jamendo\", \"v1\", (1, 112), 1, 1, \"default\"),\n", " # (\"imslp\", \"v1\", (1, 558), 1, 1, \"default\"),\n", " # (\"pond5_music\", \"v2\", (1, 4138), 1, 1, \"default\"),\n", " # (\"deezer\", \"v2\", (1, 1538), 1, 1, \"default\"),\n", " # (\"ytm_tagged\", \"v2\", (1, 5545), 1, 1, \"default\"),\n", " # (\"discogs\", \"v3\", (1, 106_715), 1, 1, \"default\"),\n", " # (\"discogs_covers\", \"v3\", (1, 3_689), 1, 1, \"covers\"),\n", "]\n", "prep_data(\n", " datasets,\n", " out_data_dir=OUT_DATA_DIR,\n", " meta_info_map=meta_info_map,\n", " is_val=False,\n", " njobs=NJOBS,\n", " chunksize=CHUNKSIZE,\n", ")\n", "# youtube_music: ~45m prep\n", "# 88,538 hours of youtube_music\n", "# 21,141 hours of youtube_music_lyrics\n", "# 18,351 hours of youtube_music_lyrics_foreign\n", "# genius_hq_lyrics: ~50m prep\n", "# 90,948 hours of genius_hq_lyrics\n", "# 49,259 hours of genius_hq_lyrics_foreign\n", "# jamendo: ~3m prep\n", "# 3,251 hours of jamendo\n", "# imslp: ~7m prep\n", "# 12,176 hours of imslp\n", "# pond5_music: ~36m prep\n", "# 58,593 hours of pond5_music\n", "# deezer: ~15m prep\n", "# 1,389 hours of deezer\n", "# 16,560 hours of deezer_lyrics\n", "# 11,133 hours of deezer_lyrics_foreign\n", "# ytm_tagged: ~1h prep\n", "# 141,478 hours of ytm_tagged\n", "# discogs: ~19h prep\n", "# 2,538,773 hours of discogs\n", "# 184,039 hours of discogs_lyrics\n", "# 168,766 hours of discogs_lyrics_foreign\n", "# discogs_covers: ~XXh prep\n", "# XX,XXX hours of discogs_covers" ] }, { "cell_type": "code", "execution_count": 13, "id": "0eab56db-f1b9-431a-bab7-0e087b8c0533", "metadata": { "execution": { "iopub.execute_input": "2024-06-29T14:59:44.518239Z", "iopub.status.busy": "2024-06-29T14:59:44.518048Z", "iopub.status.idle": "2024-06-29T14:59:44.875558Z", "shell.execute_reply": "2024-06-29T14:59:44.874184Z", "shell.execute_reply.started": "2024-06-29T14:59:44.518219Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "DONE\n" ] }, { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[13], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDONE\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "print(\"DONE\")\n", "BREAK" ] }, { "cell_type": "markdown", "id": "1b0f3596", "metadata": {}, "source": [ "### rewrite covers info" ] }, { "cell_type": "code", "execution_count": null, "id": "759137f1", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.876122Z", "iopub.status.idle": "2024-06-29T14:59:44.876326Z", "shell.execute_reply": "2024-06-29T14:59:44.876232Z", "shell.execute_reply.started": "2024-06-29T14:59:44.876221Z" } }, "outputs": [], "source": [ "# TODO: they all fit here, so alignment not obvious if we need to chop. train as 8min??" ] }, { "cell_type": "code", "execution_count": null, "id": "6c1f93d7", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.876969Z", "iopub.status.idle": "2024-06-29T14:59:44.877143Z", "shell.execute_reply": "2024-06-29T14:59:44.877061Z", "shell.execute_reply.started": "2024-06-29T14:59:44.877051Z" } }, "outputs": [], "source": [ "metas_fn = \"metas_val.jsonl\"\n", "info_fn = \"info_val.json\"\n", "tmp_metas = read_jsonl(os.path.join(OUT_DATA_DIR, metas_fn))\n", "tmp_info = read_json(os.path.join(OUT_DATA_DIR, info_fn))\n", "for dset in [\"discogs_covers\"]:\n", " print(f\"{dset}:\")\n", " tmp_metas_sub = [m for m in tmp_metas if m[\"dataset\"] == \"discogs_covers\"]\n", " parent_ids = set([m[\"id\"] for m in tmp_metas_sub if \"parent_id\" not in m])\n", " id_to_idx_map = {m[\"id\"]: idx for idx, m in enumerate(tmp_metas) if int(round(m[\"start_s\"])) == 0}\n", " info_dict = defaultdict(list)\n", " for m in tmp_metas_sub:\n", " if int(round(m[\"start_s\"])) != 0:\n", " continue\n", " if \"parent_id\" in m and m[\"parent_id\"] in parent_ids:\n", " info_dict[id_to_idx_map[m[\"parent_id\"]]].append(id_to_idx_map[m[\"id\"]])\n", " info_dict = dict(info_dict)\n", " print(f\" {len(info_dict):,}/{len(tmp_metas_sub):,} main tracks retrained\")\n", " print(f\" {sum(len(v) for _, v in info_dict.items()):,}/{len(tmp_metas_sub):,} cover tracks retrained\")\n", " tmp_info[dset] = {\n", " \"idx_map\": info_dict, \n", " \"task\": \"covers\",\n", " }\n", "write_json(tmp_info, os.path.join(OUT_DATA_DIR, info_fn))" ] }, { "cell_type": "code", "execution_count": null, "id": "3272adef", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.877756Z", "iopub.status.idle": "2024-06-29T14:59:44.877918Z", "shell.execute_reply": "2024-06-29T14:59:44.877841Z", "shell.execute_reply.started": "2024-06-29T14:59:44.877832Z" } }, "outputs": [], "source": [ "metas_fn = \"metas_tr.jsonl\"\n", "info_fn = \"info_tr.json\"\n", "tmp_metas = read_jsonl(os.path.join(OUT_DATA_DIR, metas_fn), progress=True)\n", "tmp_info = read_json(os.path.join(OUT_DATA_DIR, info_fn))\n", "for dset in [\"discogs_covers\"]:\n", " print(f\"{dset}:\")\n", " tmp_metas_sub = [m for m in tmp_metas if m[\"dataset\"] == \"discogs_covers\"]\n", " parent_ids = set([m[\"id\"] for m in tmp_metas_sub if \"parent_id\" not in m])\n", " id_to_idx_map = {m[\"id\"]: idx for idx, m in enumerate(tmp_metas) if int(round(m[\"start_s\"])) == 0}\n", " info_dict = defaultdict(list)\n", " for m in tmp_metas_sub:\n", " if int(round(m[\"start_s\"])) != 0:\n", " continue\n", " if \"parent_id\" in m and m[\"parent_id\"] in parent_ids:\n", " info_dict[id_to_idx_map[m[\"parent_id\"]]].append(id_to_idx_map[m[\"id\"]])\n", " info_dict = dict(info_dict)\n", " print(f\" {len(info_dict):,}/{len(tmp_metas_sub):,} main tracks retrained\")\n", " print(f\" {sum(len(v) for _, v in info_dict.items()):,}/{len(tmp_metas_sub):,} cover tracks retrained\")\n", " tmp_info[dset] = {\n", " \"idx_map\": info_dict, \n", " \"task\": \"covers\",\n", " }\n", "write_json(tmp_info, os.path.join(OUT_DATA_DIR, info_fn))" ] }, { "cell_type": "code", "execution_count": null, "id": "8efe003d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a82b6de5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3ffce572", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "31159a07-0c94-43c1-9a7c-77dc8a0f8a7a", "metadata": {}, "source": [ "### " ] }, { "cell_type": "markdown", "id": "2109985b", "metadata": {}, "source": [ "### Test output" ] }, { "cell_type": "code", "execution_count": null, "id": "eddbf8a8", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.878439Z", "iopub.status.idle": "2024-06-29T14:59:44.878609Z", "shell.execute_reply": "2024-06-29T14:59:44.878531Z", "shell.execute_reply.started": "2024-06-29T14:59:44.878522Z" } }, "outputs": [], "source": [ "# !ls -lah /app/suno/data/chirp_v4/multi\n", "# # total 13T\n", "# # drwxrwxr-x 2 georg georg 7 Apr 10 17:09 .\n", "# # drwxrwxr-x 3 georg georg 1 Apr 9 16:38 ..\n", "# # -rw-rw-r-- 1 georg georg 13T Apr 10 16:49 data_tr.bin\n", "# # -rw-rw-r-- 1 georg georg 806M Apr 9 18:23 data_val.bin\n", "# # -rw-rw-r-- 1 georg georg 840M Apr 10 16:57 info_tr.json\n", "# # -rw-rw-r-- 1 georg georg 32K Apr 9 18:23 info_val.json\n", "# # -rw-rw-r-- 1 georg georg 32G Apr 10 16:49 metas_tr.jsonl\n", "# # -rw-rw-r-- 1 georg georg 4.4M Apr 9 18:23 metas_val.jsonl\n", "# # -rw-rw-r-- 1 georg georg 1.3M Apr 10 17:09 tokenizer_60k.json" ] }, { "cell_type": "code", "execution_count": null, "id": "e5355c5d", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.879191Z", "iopub.status.idle": "2024-06-29T14:59:44.879349Z", "shell.execute_reply": "2024-06-29T14:59:44.879276Z", "shell.execute_reply.started": "2024-06-29T14:59:44.879267Z" } }, "outputs": [], "source": [ "# verify\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, \"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "metas = read_jsonl(os.path.join(OUT_DATA_DIR,\"metas_val.jsonl\"))\n", "mm = mm.reshape(-1, N_TOKENS_MEMMAP, 13)\n", "assert(len(mm) == len(metas))\n", "assert(mm[:100,:,0].min() >= 0)\n", "assert(mm[:100,:,0].max() <= SEMANTIC_CODEBOOK_SIZE)\n", "assert(mm[:100,:,1:].min() >= 0)\n", "assert(mm[:100,:,1:].max() <= COARSE_CODEBOOK_SIZE)" ] }, { "cell_type": "code", "execution_count": null, "id": "88922ef1", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.880073Z", "iopub.status.idle": "2024-06-29T14:59:44.880238Z", "shell.execute_reply": "2024-06-29T14:59:44.880158Z", "shell.execute_reply.started": "2024-06-29T14:59:44.880149Z" } }, "outputs": [], "source": [ "# !aws s3 cp s3://suno-data/georg/models/codec/dac_2c_25x12.pt /app/suno/data/chirp_v4/models/\n", "# !aws s3 cp s3://suno-data/georg/trained_models/chirp_v2/tokenizer_60k.json /app/suno/data/chirp_v4/models/\n", "# !aws s3 cp s3://suno-data/georg/models/semantic/mert_25.pt /app/suno/data/chirp_v4/models/\n", "# !aws s3 cp s3://suno-data/georg/models/semantic/mert_25_2x4k.npy /app/suno/data/chirp_v4/models/\n", "# !aws s3 cp s3://suno-data/checkpoints/hoot_v3/hoot_ckpt.pt /app/suno/data/chirp_v4/models/\n", "# !aws s3 cp s3://suno-data/checkpoints/hoot_v3/tokenizer.model /app/suno/data/chirp_v4/models/hoot_tokenizer.model\n", "# !aws s3 cp s3://suno-data/georg/models/codec/dac_2c_25x12.pt /app/suno/data/chirp_v4/models/\n", "# !aws s3 cp s3://suno-data/georg/trained_models/chirp_v1/lid.176.bin /app/suno/data/chirp_v4/models/" ] }, { "cell_type": "code", "execution_count": null, "id": "846797cd", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.880608Z", "iopub.status.idle": "2024-06-29T14:59:44.880763Z", "shell.execute_reply": "2024-06-29T14:59:44.880691Z", "shell.execute_reply.started": "2024-06-29T14:59:44.880682Z" } }, "outputs": [], "source": [ "# randomly listen to some stuff\n", "from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "from suno_utils.tasks.dac_2c_12cb import (\n", " encode as codec_encode,\n", " decode as codec_decode,\n", " EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", ")\n", "\n", "_ = preload_codec_models(\"/app/suno/data/chirp_v4/models/dac_2c_25x12.pt\")\n", "mm = np.memmap(os.path.join(OUT_DATA_DIR, \"data_val.bin\"), dtype=np.uint16, mode=\"r\")\n", "mm = mm.reshape(-1, N_TOKENS_MEMMAP, SEMANTIC_N_CODEBOOKS+COARSE_N_CODEBOOKS)\n", "test_metas = read_jsonl(os.path.join(OUT_DATA_DIR, \"metas_val.jsonl\"))\n", "test_info = read_json(os.path.join(OUT_DATA_DIR, \"info_val.json\"))\n", "assert len(test_metas) == len(mm)\n", "idx_list = list(range(len(test_metas)))\n", "# random.shuffle(idx_list)\n", "# idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "print(len(mm))" ] }, { "cell_type": "code", "execution_count": null, "id": "da0c7e34", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.881334Z", "iopub.status.idle": "2024-06-29T14:59:44.881505Z", "shell.execute_reply": "2024-06-29T14:59:44.881429Z", "shell.execute_reply.started": "2024-06-29T14:59:44.881419Z" } }, "outputs": [], "source": [ "# show text and audio\n", "# idx = 171\n", "# idx = random.choice(idx_list)\n", "idx = random.choice(test_info[\"discogs_lyrics\"][\"idx_list\"])\n", "# idx_key = random.choice(list(test_info.keys()))\n", "# print(idx_key)\n", "# idx = random.choice(test_info[idx_key][\"idx_list\"])\n", "assert \"original_duration_s\" in test_metas[idx]\n", "print(\"dataset:\", test_metas[idx].get(\"dataset\"))\n", "print(\"tags:\", test_metas[idx].get(\"tags\"))\n", "arr = mm[idx, 1:].copy().astype(np.int16)[:, 1:]\n", "pad_idx_arr = np.where(arr == COARSE_PAD_TOKEN)[0]\n", "if len(pad_idx_arr) > 0:\n", " arr = arr[: pad_idx_arr[0], :]\n", "a = codec_decode(arr)\n", "a.play(compress=False)\n", "print(\"text:\", test_metas[idx].get(\"text\"))\n", "# plt.plot(a.array_float[0])" ] }, { "cell_type": "code", "execution_count": null, "id": "a2e60f8c", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.882129Z", "iopub.status.idle": "2024-06-29T14:59:44.882290Z", "shell.execute_reply": "2024-06-29T14:59:44.882214Z", "shell.execute_reply.started": "2024-06-29T14:59:44.882205Z" } }, "outputs": [], "source": [ "test_metas[idx]" ] }, { "cell_type": "code", "execution_count": null, "id": "19f5baa2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e507506e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "7cd26f13", "metadata": {}, "source": [ "## Test dataload" ] }, { "cell_type": "code", "execution_count": null, "id": "f90714bd", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.882766Z", "iopub.status.idle": "2024-06-29T14:59:44.882924Z", "shell.execute_reply": "2024-06-29T14:59:44.882850Z", "shell.execute_reply.started": "2024-06-29T14:59:44.882841Z" } }, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "6e8b43cc", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.883280Z", "iopub.status.idle": "2024-06-29T14:59:44.883448Z", "shell.execute_reply": "2024-06-29T14:59:44.883369Z", "shell.execute_reply.started": "2024-06-29T14:59:44.883359Z" } }, "outputs": [], "source": [ "import sys\n", "sys.path.insert(0, \"/home/georg/code/neon/sunoGPT/\")\n", "from data_utils import get_batch, tokenize_batch, _load_tokenizer, get_sample" ] }, { "cell_type": "code", "execution_count": null, "id": "3bc93e35", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.883891Z", "iopub.status.idle": "2024-06-29T14:59:44.884048Z", "shell.execute_reply": "2024-06-29T14:59:44.883975Z", "shell.execute_reply.started": "2024-06-29T14:59:44.883966Z" } }, "outputs": [], "source": [ "from modules.gpt import GPT, GPTConfig, GPTTrainConfig\n", "model_cfg = GPTConfig(\n", " n_layer=24,\n", " n_head=24,\n", " d_head=128,\n", " n_kv_head=4,\n", " block_size=8704,\n", " t_memmap=6016,\n", " t_audio=6144,\n", " t_text=2560,\n", ")\n", "train_cfg = GPTTrainConfig()\n", "model = GPT(model_cfg, train_cfg)\n", "cfg = model.config\n", "device = \"cpu\"\n", "tokenizer = _load_tokenizer(tokenizer_fp=\"/app/suno/data/chirp_v4/base/tokenizer_60k.json\")" ] }, { "cell_type": "code", "execution_count": null, "id": "7560d3f4", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.884563Z", "iopub.status.idle": "2024-06-29T14:59:44.884717Z", "shell.execute_reply": "2024-06-29T14:59:44.884645Z", "shell.execute_reply.started": "2024-06-29T14:59:44.884637Z" } }, "outputs": [], "source": [ "from suno_utils.tasks.dac_2c_12cb import preload_models as preload_codec_models\n", "from suno_utils.tasks.dac_2c_12cb import (\n", " encode as codec_encode,\n", " decode as codec_decode,\n", " EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", ")\n", "_ = preload_codec_models(\"/app/suno/models/chirp_v2/dac_2c_25x12.pt\")" ] }, { "cell_type": "code", "execution_count": null, "id": "742f9172", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.885244Z", "iopub.status.idle": "2024-06-29T14:59:44.885399Z", "shell.execute_reply": "2024-06-29T14:59:44.885325Z", "shell.execute_reply.started": "2024-06-29T14:59:44.885317Z" } }, "outputs": [], "source": [ "import random\n", "import numpy as np\n", "from collections import defaultdict\n", "import json\n", "from data_utils import read_jsonl\n", "from suno_utils.utils.text import read_json\n", "\n", "artist_condition = True\n", "cover_condition = True\n", "pack = True\n", "batch_size = 2\n", "batch_size_tokens = cfg.block_size * batch_size\n", "local_data_shard_dir = None\n", "\n", "t_memmap = cfg.t_memmap\n", "t_text = cfg.t_text\n", "semantic_n_codebooks = cfg.semantic_n_codebooks\n", "coarse_n_codebooks = cfg.coarse_n_codebooks\n", "semantic_vocab_size = cfg.semantic_vocab_size\n", "coarse_vocab_size = cfg.coarse_vocab_size\n", "\n", "print_with_time_master = print\n", "\n", "def load_dataset(\n", " data_dir: str,\n", " filename: str,\n", " info_filename: str,\n", " metas_filename: str,\n", " weights_multiplier_map: dict,\n", " is_finetune: bool,\n", ") -> list:\n", " dataset_names = []\n", " data_idx_lists = [] # used to randomly sample from the dataset\n", " data_weights = []\n", " data = np.memmap(os.path.join(data_dir, filename), dtype=np.uint16, mode=\"r\")\n", " data = data.reshape(-1, t_memmap, semantic_n_codebooks + coarse_n_codebooks)\n", " assert data[:100, :, :semantic_n_codebooks].max() <= semantic_vocab_size\n", " assert data[:100, :, semantic_n_codebooks:].max() <= coarse_vocab_size\n", " with open(os.path.join(data_dir, info_filename)) as f:\n", " infos = json.load(f)\n", " metas = read_jsonl(os.path.join(data_dir, metas_filename))\n", " assert len(data) == len(metas), (len(data), len(metas))\n", "\n", " artist_to_songs = defaultdict(list)\n", " for i, m in enumerate(metas):\n", " if \"artist\" in m:\n", " artist_to_songs[f\"{m['dataset']}__{m['artist']}\"].append(i)\n", " artist_to_songs = {k: v for k, v in artist_to_songs.items() if len(v) > 1}\n", " if artist_condition:\n", " assert len(artist_to_songs) > 0, \"no artist data found\"\n", " print_with_time_master(f\"found {len(artist_to_songs):,} samples with artists on main process\")\n", "\n", " idx_set = set() # for checking that we don't have any duplicates\n", " has_cover = False\n", " # make sure we turn into int since keys in json get auto turned into strings\n", " for dset_name in infos.keys():\n", " if \"idx_map\" in infos[dset_name]:\n", " infos[dset_name][\"idx_map\"] = {int(k): v for k, v in infos[dset_name][\"idx_map\"].items()}\n", " for dset_name, info in infos.items():\n", " dataset_names.append(dset_name)\n", "\n", " if info.get(\"task\", \"default\") == \"default\":\n", " assert \"idx_list\" in info\n", " idx_list = info[\"idx_list\"][:]\n", " idx_set |= set(idx_list)\n", " elif info[\"task\"] == \"covers\":\n", " assert pack, \"for now pack needs to be active to do covers\"\n", " assert batch_size_tokens >= t_memmap * 2 + t_text, \"for covers we need double the blocksize\"\n", " has_cover = True\n", " # dict from original idx to list of covers idx\n", " # use original idxs as the idx_list\n", " idx_list = []\n", " n_covers = 0\n", " for idx, child_idx_l in info[\"idx_map\"].items():\n", " idx_list.append(int(idx))\n", " idx_set.add(int(idx))\n", " idx_set |= set(child_idx_l)\n", " n_covers += len(child_idx_l)\n", " print_with_time_master(\n", " f\"found {len(idx_list):,} samples with {n_covers:,} total covers on main process\"\n", " )\n", " else:\n", " raise ValueError(f\"unknown task for {dset_name} in info file\")\n", " random.shuffle(idx_list)\n", "\n", " data_idx_lists.append(idx_list)\n", " data_weights.append(len(idx_list) * weights_multiplier_map.get(dset_name, 1.0))\n", " if cover_condition:\n", " assert has_cover, \"no cover data found\"\n", " weights_norm = np.sum(data_weights)\n", " data_weights = [v / weights_norm for v in data_weights]\n", "\n", " if not is_finetune:\n", " print_with_time_master(f\"indexed {len(idx_set)/len(data)*100:.1f}% of data\")\n", " for k in weights_multiplier_map.keys():\n", " assert k in dataset_names\n", "\n", " del idx_set\n", " shard_info = \"\" if local_data_shard_dir is None else \" (sharded)\"\n", " print_with_time_master(f\"{len(data):,} lines of {filename} loaded.{shard_info}\")\n", " assert len(data) == len(metas)\n", " assert len(infos) == len(dataset_names) == len(data_weights) == len(data_idx_lists)\n", " return (\n", " dataset_names,\n", " data_idx_lists,\n", " data_weights,\n", " data,\n", " metas,\n", " infos,\n", " artist_to_songs,\n", " )" ] }, { "cell_type": "code", "execution_count": null, "id": "f9cd0f7e", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.885935Z", "iopub.status.idle": "2024-06-29T14:59:44.886096Z", "shell.execute_reply": "2024-06-29T14:59:44.886017Z", "shell.execute_reply.started": "2024-06-29T14:59:44.886008Z" } }, "outputs": [], "source": [ "data_dir = \"/app/suno/data/chirp_v4/multi\"\n", "\n", "val_filename = \"data_val.bin\"\n", "val_info_filename = \"info_val.json\"\n", "val_metas_filename = \"metas_val.jsonl\"\n", "\n", "weights_multiplier_map = {}\n", "is_finetune = False\n", "\n", "(\n", " val_dataset_names,\n", " val_data_idx_lists,\n", " val_data_weights,\n", " val_data,\n", " val_metas,\n", " val_info,\n", " val_artist_to_songs,\n", ") = load_dataset(\n", " data_dir,\n", " val_filename,\n", " val_info_filename,\n", " val_metas_filename,\n", " weights_multiplier_map,\n", " is_finetune,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "6bab0566", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.886582Z", "iopub.status.idle": "2024-06-29T14:59:44.886738Z", "shell.execute_reply": "2024-06-29T14:59:44.886666Z", "shell.execute_reply.started": "2024-06-29T14:59:44.886657Z" } }, "outputs": [], "source": [ "data_sampling_info = {\n", " \"cfg\": cfg,\n", " \"train_cfg\": train_cfg,\n", " \"batch_size\": batch_size,\n", " \"batch_size_tokens\": batch_size_tokens,\n", " \"tokenizer_fp\": \"/app/suno/data/chirp_v4/base/tokenizer_60k.json\",\n", " \"device\": device,\n", " \"device_type\": \"cuda\" if \"cuda\" in str(device) else \"cpu\",\n", " \"val\": {\n", " \"data\": val_data,\n", " \"metas\": val_metas,\n", " \"infos\": val_info,\n", " \"artist_to_songs\": val_artist_to_songs,\n", " \"names\": val_dataset_names,\n", " \"weights\": val_data_weights,\n", " \"idx_lists\": val_data_idx_lists,\n", " },\n", "}" ] }, { "cell_type": "markdown", "id": "6e6221a1", "metadata": {}, "source": [ "### get_sample" ] }, { "cell_type": "code", "execution_count": null, "id": "e7e7f262", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.887312Z", "iopub.status.idle": "2024-06-29T14:59:44.887466Z", "shell.execute_reply": "2024-06-29T14:59:44.887396Z", "shell.execute_reply.started": "2024-06-29T14:59:44.887387Z" } }, "outputs": [], "source": [ "suffix_first = True\n", "row_idx, x_arr = get_sample(\n", " data_sampling_info,\n", " \"val\",\n", " dataset_idx=None,\n", " row_idx=None, # absolute, overrides dataset_idx\n", " use_private=False,\n", " inference=False,\n", " suppress_text=False,\n", " dummy_data=False,\n", " min_text_offs=None,\n", " suffix_first=suffix_first,\n", " dropout_semantic=False,\n", " allow_artist_condition=artist_condition,\n", " allow_cover=cover_condition,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "29848739", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bc21d875", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "4e3f4425", "metadata": {}, "source": [ "# TODO: finish updating exploration code below" ] }, { "cell_type": "code", "execution_count": null, "id": "e7cbfc9d", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.888158Z", "iopub.status.idle": "2024-06-29T14:59:44.888315Z", "shell.execute_reply": "2024-06-29T14:59:44.888241Z", "shell.execute_reply.started": "2024-06-29T14:59:44.888233Z" } }, "outputs": [], "source": [ "m = []\n", "max_len = 0\n", "for n in range(1, 13):\n", " v = x_audio[n][51+5*(n-1):]\n", " max_idx = np.where(v != 2048)[0][-1]\n", " m.append(v[:max_idx+1])\n", "audio_arr = np.stack(m)" ] }, { "cell_type": "code", "execution_count": null, "id": "0f23b57f", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.888822Z", "iopub.status.idle": "2024-06-29T14:59:44.888992Z", "shell.execute_reply": "2024-06-29T14:59:44.888916Z", "shell.execute_reply.started": "2024-06-29T14:59:44.888907Z" } }, "outputs": [], "source": [ "codec_decode(audio_arr.T[:25*50]).play(compress=False)" ] }, { "cell_type": "markdown", "id": "05596ed1", "metadata": {}, "source": [ "### get_bach" ] }, { "cell_type": "code", "execution_count": null, "id": "5382892f", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.889522Z", "iopub.status.idle": "2024-06-29T14:59:44.889676Z", "shell.execute_reply": "2024-06-29T14:59:44.889605Z", "shell.execute_reply.started": "2024-06-29T14:59:44.889596Z" } }, "outputs": [], "source": [ "text_offset, x, y = get_batch(\n", " data_sampling_info,\n", " \"val\",\n", " dataset_idx=None,\n", " row_idx=None,\n", " use_private=False,\n", " inference=False,\n", " min_text_offs=None,\n", " suppress_text=False,\n", " dummy_data=False,\n", " return_idx=False,\n", " n_offs=None,\n", ")\n", "text_offset" ] }, { "cell_type": "code", "execution_count": null, "id": "726a5201", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.890040Z", "iopub.status.idle": "2024-06-29T14:59:44.890188Z", "shell.execute_reply": "2024-06-29T14:59:44.890119Z", "shell.execute_reply.started": "2024-06-29T14:59:44.890110Z" } }, "outputs": [], "source": [ "n_row = 0\n", "tokenizer.decode(x[n_row,0,:411]).replace(\" ##\", \"\")" ] }, { "cell_type": "code", "execution_count": null, "id": "9d77ac7a", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.890594Z", "iopub.status.idle": "2024-06-29T14:59:44.890748Z", "shell.execute_reply": "2024-06-29T14:59:44.890677Z", "shell.execute_reply.started": "2024-06-29T14:59:44.890669Z" } }, "outputs": [], "source": [ "m = []\n", "max_len = 0\n", "for n in range(2, 14):\n", " v = x[n_row,n,text_offset+51+5*(n-1):]\n", " max_idx = np.where(v != 2048)[0][-1]\n", " m.append(v[:max_idx+1])\n", "audio_arr = np.stack(m)" ] }, { "cell_type": "code", "execution_count": null, "id": "3bb54a6b", "metadata": { "execution": { "iopub.status.busy": "2024-06-29T14:59:44.891189Z", "iopub.status.idle": "2024-06-29T14:59:44.891355Z", "shell.execute_reply": "2024-06-29T14:59:44.891278Z", "shell.execute_reply.started": "2024-06-29T14:59:44.891269Z" } }, "outputs": [], "source": [ "codec_decode(audio_arr.T[:25*50]).play(compress=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "847885fd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "795e6dd8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a45a9ae6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a6f5e88a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d21501f7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d7a1a3b9", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" } }, "nbformat": 4, "nbformat_minor": 5 }