{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "ffdbda4b", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T13:27:22.576933Z", "start_time": "2023-10-28T13:27:22.575445Z" } }, "outputs": [], "source": [ "import os\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "dd41c10c", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T13:27:31.662980Z", "start_time": "2023-10-28T13:27:25.080606Z" } }, "outputs": [], "source": [ "import random\n", "import json\n", "import numpy as np\n", "import tqdm\n", "import torch\n", "import funcy\n", "import time\n", "import gc\n", "from scipy.io import wavfile\n", "import tempfile\n", "import collections\n", "from collections import defaultdict\n", "from joblib import Parallel, delayed\n", "\n", "from suno_utils.utils.s3 import _apply_mp\n", "from suno_utils.audio import Audio\n", "from suno_utils.tasks.data_loader import load_audio_mp\n", "from suno_utils.utils.text import write_jsonl, read_jsonl, write_json, read_json\n", "from suno_utils.utils.s3 import read_from_s3, check_s3_file_exists, open_from_s3\n", "from suno_utils.audio.conversion import convert_audio_files\n", "\n", "SAMPLE_RATE = 24_000\n", "EMBEDDING_RATE = 25\n", "N_CODEBOOKS = 8\n", "\n", "IN_DATA_DIR = \"/app/suno/data/mert_25hz\"\n", "IN_AUDIO_DIR = os.path.join(IN_DATA_DIR, \"audio\")\n", "IN_TSV_DIR = os.path.join(IN_DATA_DIR, \"audio_tsv\")\n", "IN_LABEL_DIR = os.path.join(IN_DATA_DIR, \"label\")\n", "\n", "OUT_DATA_DIR = \"/app/suno/data/mert_25hz_long/\"\n", "OUT_AUDIO_DIR = os.path.join(OUT_DATA_DIR, \"audio\")\n", "OUT_TSV_DIR = os.path.join(OUT_DATA_DIR, \"audio_tsv\")\n", "OUT_LABEL_DIR = os.path.join(OUT_DATA_DIR, \"label\")\n", "OUT_TEMP_DIR = os.path.join(OUT_DATA_DIR, \"temp\")\n", "\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "os.makedirs(OUT_AUDIO_DIR, exist_ok=True)\n", "os.makedirs(OUT_TSV_DIR, exist_ok=True)\n", "os.makedirs(OUT_LABEL_DIR, exist_ok=True)\n", "os.makedirs(OUT_TEMP_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 3, "id": "195c03f2", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T13:27:31.666592Z", "start_time": "2023-10-28T13:27:31.664957Z" } }, "outputs": [], "source": [ "MIN_SIZE = SAMPLE_RATE * 5\n", "MAX_SIZE = SAMPLE_RATE * 60\n", "MAX_ARRAY_SIZE = MAX_SIZE // SAMPLE_RATE * 25" ] }, { "cell_type": "markdown", "id": "c729d86d", "metadata": {}, "source": [ "# File loading" ] }, { "cell_type": "code", "execution_count": 16, "id": "bd6a01a1", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T13:32:07.377721Z", "start_time": "2023-10-28T13:32:07.375949Z" } }, "outputs": [], "source": [ "data_split = \"train\"" ] }, { "cell_type": "code", "execution_count": 17, "id": "66631638", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T13:32:08.262919Z", "start_time": "2023-10-28T13:32:07.591725Z" } }, "outputs": [], "source": [ "# load current data\n", "tsv_info = []\n", "with open(os.path.join(IN_TSV_DIR, f\"{data_split}.tsv\"), \"r\") as f:\n", " for line in f.read().strip().split(\"\\n\"):\n", " if len(line.strip()) == 0:\n", " continue\n", " tsv_info.append(line.strip().split(\"\\t\"))" ] }, { "cell_type": "code", "execution_count": 18, "id": "cad52faa", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T13:32:08.266362Z", "start_time": "2023-10-28T13:32:08.264492Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "436804\n" ] } ], "source": [ "print(len(tsv_info))" ] }, { "cell_type": "code", "execution_count": 19, "id": "741d5c23", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T13:32:08.317360Z", "start_time": "2023-10-28T13:32:08.267380Z" } }, "outputs": [], "source": [ "tsv_data_path = tsv_info[0]\n", "tsv_data = tsv_info[1:]" ] }, { "cell_type": "markdown", "id": "d17fd56f", "metadata": {}, "source": [ "## Slice the TSV files" ] }, { "cell_type": "code", "execution_count": 20, "id": "6a6a9b0e", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T13:32:08.363886Z", "start_time": "2023-10-28T13:32:08.323736Z" } }, "outputs": [], "source": [ "# tsv_data[0:10]" ] }, { "cell_type": "code", "execution_count": 21, "id": "c4906e01", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T13:32:12.303402Z", "start_time": "2023-10-28T13:32:08.513124Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████| 436803/436803 [00:03<00:00, 115653.52it/s]\n" ] } ], "source": [ "new_tsv_data = []\n", "# list of list of tuples,\n", "# each index is the audio's codec spllit list (start, end)\n", "split_times = []\n", "for n_row, (audio_fn, audio_size) in tqdm.tqdm(\n", " enumerate(tsv_data), total=len(tsv_data)\n", "):\n", " new_splits = []\n", " for n in range(int(np.ceil(int(audio_size) / MAX_SIZE))):\n", " start_idx = n * MAX_SIZE\n", " end_idx = min(int(audio_size), (n + 1) * MAX_SIZE)\n", " if end_idx - start_idx < MIN_SIZE:\n", " continue\n", " input_file_id = audio_fn.replace(\".wav\", \"\")\n", " out_audio_fn = f\"{input_file_id}_{n}.wav\"\n", " new_tsv_data.append((audio_fn, out_audio_fn, start_idx, end_idx))\n", " start_idx_embed = int(round(start_idx / SAMPLE_RATE * EMBEDDING_RATE))\n", " end_idx_embed = int(round(end_idx / SAMPLE_RATE * EMBEDDING_RATE))\n", " new_splits.append((start_idx_embed, end_idx_embed))\n", " split_times.append(new_splits)" ] }, { "cell_type": "markdown", "id": "4defb74e", "metadata": {}, "source": [ "## Slice the codebook labels" ] }, { "cell_type": "code", "execution_count": 22, "id": "a0b384f7", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T13:32:12.306945Z", "start_time": "2023-10-28T13:32:12.304840Z" } }, "outputs": [ { "data": { "text/plain": [ "436803" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(split_times)" ] }, { "cell_type": "code", "execution_count": 23, "id": "c7a427db", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T13:32:12.922876Z", "start_time": "2023-10-28T13:32:12.921181Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[(0, 1500), (1500, 3000), (3000, 3437)]\n" ] } ], "source": [ "print(split_times[0])" ] }, { "cell_type": "code", "execution_count": 24, "id": "990c025e", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T13:32:13.116825Z", "start_time": "2023-10-28T13:32:13.115417Z" } }, "outputs": [], "source": [ "total_rows = len(new_tsv_data)" ] }, { "cell_type": "code", "execution_count": 25, "id": "9065a223", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T14:23:55.464638Z", "start_time": "2023-10-28T13:32:13.825423Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "codebook: 0\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "436803it [05:10, 1408.99it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "codebook: 1\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "436803it [05:11, 1402.05it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "codebook: 2\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "436803it [05:10, 1405.88it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "codebook: 3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "436803it [05:11, 1403.75it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "codebook: 4\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "436803it [05:11, 1402.97it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "codebook: 5\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "436803it [05:10, 1407.63it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "codebook: 6\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "436803it [05:10, 1406.28it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "codebook: 7\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "436803it [05:10, 1406.22it/s]\n" ] } ], "source": [ "output_arrays = [\n", " np.ones((total_rows, MAX_ARRAY_SIZE), dtype=np.int16) * -1\n", " for _ in range(N_CODEBOOKS)\n", "]\n", "# need to pad the arrays\n", "for codebook in range(N_CODEBOOKS):\n", " print(\"codebook:\", codebook)\n", " n_output_row = 0\n", " with open(os.path.join(IN_LABEL_DIR, f\"{data_split}.codec_{codebook}\"), \"r\") as f:\n", " for n_row, line in tqdm.tqdm(enumerate(f.readlines())):\n", " line = line.split()\n", " split_time_row = split_times[n_row]\n", " for start_idx_embed, end_idx_embed in split_time_row:\n", " codec_values = np.array(\n", " [int(i) for i in line[start_idx_embed:end_idx_embed]]\n", " ).astype(np.int16)\n", " output_arrays[codebook][n_output_row, :codec_values.shape[0]] = codec_values\n", " n_output_row += 1" ] }, { "cell_type": "code", "execution_count": 26, "id": "145dcfe9", "metadata": { "ExecuteTime": { "end_time": "2023-10-28T14:24:56.858209Z", "start_time": "2023-10-28T14:23:55.466436Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "codebook: 0\n", "codebook: 1\n", "codebook: 2\n", "codebook: 3\n", "codebook: 4\n", "codebook: 5\n", "codebook: 6\n", "codebook: 7\n", "Finished saving all codec labels\n" ] } ], "source": [ "for codebook in range(N_CODEBOOKS):\n", " print(\"codebook:\", codebook)\n", " with open(os.path.join(OUT_LABEL_DIR, f\"{data_split}.codec_{codebook}.npy\"), \"wb\") as f_out:\n", " np.save(f_out, output_arrays[codebook].astype(np.int16))\n", "print(\"Finished saving all codec labels\")" ] }, { "cell_type": "markdown", "id": "135eec50", "metadata": {}, "source": [ "# Convert the labels to np arrays" ] }, { "cell_type": "code", "execution_count": 49, "id": "cb93f04e", "metadata": { "ExecuteTime": { "end_time": "2023-10-26T20:06:08.517664Z", "start_time": "2023-10-26T20:06:08.467362Z" } }, "outputs": [], "source": [ "# total_rows = len(new_tsv_data)" ] }, { "cell_type": "code", "execution_count": 48, "id": "82d42119", "metadata": { "ExecuteTime": { "end_time": "2023-10-26T20:06:08.465975Z", "start_time": "2023-10-26T20:06:08.463979Z" } }, "outputs": [], "source": [ "# i = 0\n", "# max_shape = 300\n", "# data_split = \"valid\"\n", "# output_arrays = [\n", "# np.ones((total_rows, max_shape), dtype=np.int16) * -1 for _ in range(N_CODEBOOKS)\n", "# ]\n", "# for origin_id, temp_tsv_data in tqdm.tqdm(enumerate(tsv_data)):\n", "# test_file_name = temp_tsv_data[0]\n", "# test_id = test_file_name.replace(\".wav\", \"\").replace(\"speech_\", \"\")\n", "# file_to_fetch = f\"{IN_DATA_DIR}/audio_label/{data_split}_{test_id}.npy\"\n", "# if not os.path.exists(file_to_fetch):\n", "# print(\"WTF\", file_to_fetch)\n", "# break\n", "# test_outputs = np.load(file_to_fetch)\n", "# # print(test_outputs.shape)\n", "# split_time = split_times[origin_id]\n", "# # if split_time[-1][1] + 1 != test_outputs.shape[0]:\n", "# # print(\"WTF, shape doesn't match\", split_time[-1][1], test_outputs.shape[0])\n", "# for split_start, split_end in split_time:\n", "# for codebook in range(N_CODEBOOKS):\n", "# # print(output_arrays[codebook][i:i + n_batch_size, :].shape, test_outputs[:, :, codebook].shape)\n", "# test_slice = test_outputs[split_start:split_end, codebook]\n", "# # this is very fucked up\n", "# # np.random.shuffle(test_slice)\n", "# # test_slice = np.random.randint(0, 4096, test_slice.shape[0])\n", "# output_arrays[codebook][i, : test_slice.shape[0]] = test_slice\n", "# i += 1\n", "# print(i)" ] }, { "cell_type": "markdown", "id": "20c16724", "metadata": {}, "source": [ "## some validation" ] }, { "cell_type": "code", "execution_count": 90, "id": "171b0063", "metadata": { "ExecuteTime": { "end_time": "2023-10-26T20:33:02.523222Z", "start_time": "2023-10-26T20:32:59.630399Z" } }, "outputs": [], "source": [ "from suno_utils.tasks import dac\n", "dac.preload_models(checkpoint_filepath=\"/home/tony/Data/MERT/codec.pt\")" ] }, { "cell_type": "code", "execution_count": 97, "id": "ffd23c72", "metadata": { "ExecuteTime": { "end_time": "2023-10-26T20:35:03.814738Z", "start_time": "2023-10-26T20:35:03.812318Z" } }, "outputs": [], "source": [ "test_row_id = 2563\n", "test_file_name = new_tsv_data[test_row_id][1]\n", "test_resembed = [output_arrays[codebook][test_row_id, :] for codebook in range(N_CODEBOOKS)]\n", "test_resembed_array = np.array(test_resembed).swapaxes(0, 1)" ] }, { "cell_type": "code", "execution_count": 100, "id": "8fdda711", "metadata": { "ExecuteTime": { "end_time": "2023-10-26T20:37:38.256128Z", "start_time": "2023-10-26T20:37:38.210297Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "24000\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "test_audio = Audio.from_file(os.path.join(OUT_AUDIO_DIR, test_file_name))\n", "print(test_audio.sample_rate)\n", "test_audio.play(compress=False)" ] }, { "cell_type": "code", "execution_count": 99, "id": "6d3ceb62", "metadata": { "ExecuteTime": { "end_time": "2023-10-26T20:35:15.167336Z", "start_time": "2023-10-26T20:35:05.579369Z" } }, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "decoded_audio = dac.decode(test_resembed_array)\n", "print(test_audio.decoded_audio)\n", "decoded_audio.play(compress=False)" ] }, { "cell_type": "markdown", "id": "5c75504e", "metadata": {}, "source": [ "# Convert the labels to text" ] }, { "cell_type": "code", "execution_count": 77, "id": "02a83804", "metadata": { "ExecuteTime": { "end_time": "2023-10-26T20:13:27.535252Z", "start_time": "2023-10-26T20:13:27.446462Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/app/suno/data/mert_25hz/label/dict.codec_0.txt\n", "/app/suno/data/mert_25hz/label/dict.codec_1.txt\n", "/app/suno/data/mert_25hz/label/dict.codec_2.txt\n", "/app/suno/data/mert_25hz/label/dict.codec_3.txt\n", "/app/suno/data/mert_25hz/label/dict.codec_4.txt\n", "/app/suno/data/mert_25hz/label/dict.codec_5.txt\n", "/app/suno/data/mert_25hz/label/dict.codec_6.txt\n", "/app/suno/data/mert_25hz/label/dict.codec_7.txt\n" ] } ], "source": [ "# FIX?!?\n", "for codebook in range(N_CODEBOOKS):\n", " with open(os.path.join(OUT_LABEL_DIR, f\"dict.codec_{codebook}.txt\"), \"w\") as f:\n", " print(os.path.join(OUT_LABEL_DIR, f\"dict.codec_{codebook}.txt\"))\n", " for n in range(4097):\n", " f.write(f\"{n} {1}\\n\")" ] }, { "cell_type": "markdown", "id": "37937441", "metadata": {}, "source": [ "## Write the train TSV" ] }, { "cell_type": "code", "execution_count": 15, "id": "b0d051d7", "metadata": { "ExecuteTime": { "end_time": "2023-10-26T20:41:45.849884Z", "start_time": "2023-10-26T20:41:45.238161Z" } }, "outputs": [], "source": [ "with open(os.path.join(OUT_TSV_DIR, f\"{data_split}.tsv\"), \"w\") as f:\n", " f.write(OUT_AUDIO_DIR + \"\\n\")\n", " f.write(\n", " \"\\n\".join(\n", " [\n", " fn + \"\\t\" + str(int(idx_end - idx_start))\n", " for _, fn, idx_start, idx_end in new_tsv_data\n", " ]\n", " )\n", " )" ] }, { "cell_type": "markdown", "id": "6bd58f17", "metadata": {}, "source": [ "## Generate the split audios Job file" ] }, { "cell_type": "code", "execution_count": 16, "id": "28864363", "metadata": { "ExecuteTime": { "end_time": "2023-10-26T20:41:51.881506Z", "start_time": "2023-10-26T20:41:49.551526Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "436803 work_items\n" ] } ], "source": [ "# write new audios to disk\n", "def _write_new_wav_files(work_item):\n", " from_fn, to_work_items = work_item\n", " _, audio_arr = wavfile.read(os.path.join(IN_AUDIO_DIR, from_fn))\n", " for to_fn, (start_idx, end_idx) in to_work_items:\n", " wavfile.write(\n", " os.path.join(OUT_AUDIO_DIR, to_fn),\n", " SAMPLE_RATE,\n", " audio_arr[start_idx:end_idx],\n", " )\n", "work_items = defaultdict(list)\n", "for from_fn, to_fn, start_idx, end_idx in new_tsv_data:\n", " work_items[from_fn].append((to_fn, (start_idx, end_idx)))\n", "work_items = [(k, v) for k, v in work_items.items()]\n", "print(len(work_items), \"work_items\")" ] }, { "cell_type": "code", "execution_count": 17, "id": "4e77c558", "metadata": { "ExecuteTime": { "end_time": "2023-10-26T20:41:58.375373Z", "start_time": "2023-10-26T20:41:51.884155Z" } }, "outputs": [], "source": [ "with open(f\"{data_split}_origin_speech.json\", \"w\") as fp:\n", " json.dump(work_items, fp)" ] }, { "cell_type": "code", "execution_count": 33, "id": "12018181", "metadata": { "ExecuteTime": { "end_time": "2023-10-13T13:59:39.895788Z", "start_time": "2023-10-13T13:59:39.893990Z" } }, "outputs": [], "source": [ "# _ = _apply_mp(\n", "# _write_new_wav_files,\n", "# work_items,\n", "# chunksize=1,\n", "# n_cores=32,\n", "# joblib_backend=\"threads\",\n", "# )" ] }, { "cell_type": "code", "execution_count": 12, "id": "738138b8", "metadata": { "ExecuteTime": { "end_time": "2023-09-06T17:42:30.677232Z", "start_time": "2023-09-06T17:42:30.625842Z" } }, "outputs": [], "source": [ "# codec_data = defaultdict(list)\n", "# for codebook in range(N_CODEBOOKS):\n", "# # print(i)\n", "# with open(os.path.join(IN_LABEL_DIR, f\"train.codec_{codebook}\"), \"r\") as f:\n", "# for line in f.read().strip().split(\"\\n\"):\n", "# if len(line.strip()) == 0:\n", "# continue\n", "# codec_data[codebook].append(line.strip().split())\n", "# # break\n", "# for codebook in range(N_CODEBOOKS):\n", "# with open(os.path.join(OUT_LABEL_DIR, f\"train.codec_{codebook}\"), \"w\") as f:\n", "# f.write(\"\\n\".join(new_codec_data[codebook]))" ] }, { "cell_type": "code", "execution_count": null, "id": "7428b9fc", "metadata": { "ExecuteTime": { "start_time": "2023-09-07T06:24:59.615Z" } }, "outputs": [], "source": [ "files = os.listdir(\"/app/suno/data/mert_25hz_short/audio\")" ] }, { "cell_type": "code", "execution_count": null, "id": "be9669ca", "metadata": { "ExecuteTime": { "start_time": "2023-09-07T06:25:00.056Z" } }, "outputs": [], "source": [ "len(files)" ] }, { "cell_type": "code", "execution_count": 202, "id": "1f86eb36", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T05:49:51.825989Z", "start_time": "2023-09-07T05:49:51.590995Z" } }, "outputs": [], "source": [ "selected = [f for f in files if \"_\" in f]" ] }, { "cell_type": "code", "execution_count": 203, "id": "3d49a73b", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T05:49:51.829834Z", "start_time": "2023-09-07T05:49:51.827571Z" } }, "outputs": [ { "data": { "text/plain": [ "(6896994, 7085918)" ] }, "execution_count": 203, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(selected), len(new_tsv_data)" ] }, { "cell_type": "code", "execution_count": 213, "id": "c8eceb78", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T06:02:08.734815Z", "start_time": "2023-09-07T06:02:08.732246Z" } }, "outputs": [ { "data": { "text/plain": [ "('3.wav', '2_4.wav', 1152000, 1440000)" ] }, "execution_count": 213, "metadata": {}, "output_type": "execute_result" } ], "source": [ "new_tsv_data[30]" ] }, { "cell_type": "code", "execution_count": 230, "id": "97b70c7d", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T06:23:08.640223Z", "start_time": "2023-09-07T06:23:04.856146Z" } }, "outputs": [], "source": [ "bad_map = {}\n", "for i in new_tsv_data:\n", " # bad_map[int(i[0].split(\".wav\")[0])] = int(i[1].split(\"_\")[0])\n", " bad_map[int(i[1].split(\"_\")[0])] = int(i[0].split(\".wav\")[0])\n", "# if int(i[0].split(\".wav\")[0]) == 436817:\n", "# print(i)\n", "# if int(i[1].split(\"_\")[0]) == 436817:\n", "# print(i)" ] }, { "cell_type": "code", "execution_count": 205, "id": "c27f546d", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T05:49:53.725164Z", "start_time": "2023-09-07T05:49:52.247231Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "436803\n" ] } ], "source": [ "must_unique_f = set(int(i[0].split(\".wav\")[0]) for i in new_tsv_data)\n", "print(len(must_unique_f))" ] }, { "cell_type": "code", "execution_count": 206, "id": "daec6223", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T05:49:53.739580Z", "start_time": "2023-09-07T05:49:53.726550Z" } }, "outputs": [ { "data": { "text/plain": [ "(1, 463758)" ] }, "execution_count": 206, "metadata": {}, "output_type": "execute_result" } ], "source": [ "min(must_unique_f), max(must_unique_f)" ] }, { "cell_type": "code", "execution_count": 207, "id": "e6891f7f", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T05:49:55.345744Z", "start_time": "2023-09-07T05:49:53.819901Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "436803\n" ] } ], "source": [ "must_unique_f_1 = set(int(i[1].split(\"_\")[0]) for i in new_tsv_data)\n", "print(len(must_unique_f_1))" ] }, { "cell_type": "code", "execution_count": 221, "id": "868c9acd", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T06:18:46.028766Z", "start_time": "2023-09-07T06:18:46.018969Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "25604\n" ] } ], "source": [ "print(len(must_unique_f.difference(must_unique_f_1)))" ] }, { "cell_type": "code", "execution_count": 37, "id": "410c5abb", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:26:31.449837Z", "start_time": "2023-09-07T15:26:31.447994Z" } }, "outputs": [], "source": [ "# for i in must_unique_f:\n", "# if i not in must_unique_f_1:\n", "# print(i)" ] }, { "cell_type": "code", "execution_count": 38, "id": "19a77adf", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:26:33.541989Z", "start_time": "2023-09-07T15:26:33.540195Z" } }, "outputs": [], "source": [ "# for i in must_unique_f_1:\n", "# if i not in must_unique_f:\n", "# print(i)" ] }, { "cell_type": "code", "execution_count": 208, "id": "95dffc70", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T05:49:55.349478Z", "start_time": "2023-09-07T05:49:55.347112Z" } }, "outputs": [ { "data": { "text/plain": [ "(6896994, 7085918)" ] }, "execution_count": 208, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(selected), len(new_tsv_data)" ] }, { "cell_type": "code", "execution_count": 209, "id": "ef64eb3a", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T05:49:57.736167Z", "start_time": "2023-09-07T05:49:56.307871Z" } }, "outputs": [], "source": [ "unique_f = set(int(f.split(\"_\")[0]) for f in selected)" ] }, { "cell_type": "code", "execution_count": 210, "id": "c564b71e", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T05:50:06.069558Z", "start_time": "2023-09-07T05:50:06.067137Z" } }, "outputs": [ { "data": { "text/plain": [ "427307" ] }, "execution_count": 210, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(unique_f)" ] }, { "cell_type": "code", "execution_count": 187, "id": "4101cd9a", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T05:44:15.653998Z", "start_time": "2023-09-07T05:44:15.651596Z" } }, "outputs": [ { "data": { "text/plain": [ "427307" ] }, "execution_count": 187, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(unique_f)" ] }, { "cell_type": "code", "execution_count": 50, "id": "5a97a281", "metadata": { "ExecuteTime": { "end_time": "2023-09-06T21:59:37.236355Z", "start_time": "2023-09-06T21:59:37.232184Z" } }, "outputs": [ { "data": { "text/plain": [ "[112736,\n", " 112737,\n", " 112738,\n", " 112739,\n", " 112740,\n", " 112741,\n", " 112742,\n", " 112743,\n", " 112744,\n", " 112745,\n", " 112746,\n", " 112747,\n", " 112748,\n", " 112749,\n", " 112750,\n", " 112751,\n", " 112752,\n", " 112753,\n", " 112754,\n", " 112755]" ] }, "execution_count": 50, "metadata": {}, "output_type": "execute_result" } ], "source": [ "list(sorted(unique_f))[-20:]" ] }, { "cell_type": "code", "execution_count": 19, "id": "d824c90b", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:19:39.034143Z", "start_time": "2023-09-07T15:19:39.031323Z" } }, "outputs": [ { "data": { "text/plain": [ "('5887.wav', '5579_5.wav', 1440000, 1728000)" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "new_tsv_data[102030]" ] }, { "cell_type": "markdown", "id": "066bb0b2", "metadata": {}, "source": [ "# Some validations" ] }, { "cell_type": "code", "execution_count": 108, "id": "cc4169ce", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T18:25:53.048806Z", "start_time": "2023-09-07T18:25:53.046931Z" } }, "outputs": [], "source": [ "selected_file = new_tsv_data[0][1]" ] }, { "cell_type": "code", "execution_count": 109, "id": "db72b8b7", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T18:25:53.711891Z", "start_time": "2023-09-07T18:25:53.702216Z" } }, "outputs": [], "source": [ "new_wav = wavfile.read(os.path.join(OUT_AUDIO_DIR, selected_file))" ] }, { "cell_type": "code", "execution_count": 110, "id": "6b67883e", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T18:25:53.827291Z", "start_time": "2023-09-07T18:25:53.825112Z" } }, "outputs": [ { "data": { "text/plain": [ "(288000, 1)" ] }, "execution_count": 110, "metadata": {}, "output_type": "execute_result" } ], "source": [ "new_wav[1].reshape(-1, 1).shape" ] }, { "cell_type": "code", "execution_count": 111, "id": "94c6df8e", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T18:25:54.052800Z", "start_time": "2023-09-07T18:25:54.050693Z" } }, "outputs": [], "source": [ "audio = Audio.from_file(os.path.join(OUT_AUDIO_DIR, selected_file))" ] }, { "cell_type": "code", "execution_count": 112, "id": "2d720608", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T18:25:54.443771Z", "start_time": "2023-09-07T18:25:54.381841Z" } }, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "audio.play()" ] }, { "cell_type": "code", "execution_count": 30, "id": "d45020c4", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:20:27.594159Z", "start_time": "2023-09-07T15:20:27.591239Z" } }, "outputs": [ { "data": { "text/plain": [ "(24000,\n", " array([ 598, -553, 1800, ..., -5042, -7360, -11655], dtype=int16))" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "new_wav" ] }, { "cell_type": "code", "execution_count": 55, "id": "f2f87e70", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T00:52:29.427411Z", "start_time": "2023-09-07T00:52:27.396720Z" } }, "outputs": [], "source": [ "from nnAudio import features as nnAudioFeatures" ] }, { "cell_type": "code", "execution_count": 60, "id": "91d01f60", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T00:53:28.148373Z", "start_time": "2023-09-07T00:53:27.640337Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CQT kernels created, time used = 0.5050 seconds\n" ] } ], "source": [ "spec_layer = nnAudioFeatures.cqt.CQT(\n", " sr=new_wav[0],\n", " hop_length=new_wav[0] // EMBEDDING_RATE,\n", " fmin=32.7,\n", " fmax=None,\n", " n_bins=336,\n", " bins_per_octave=336 // 7,\n", " filter_scale=1,\n", " norm=1,\n", " window=\"hann\",\n", " center=True,\n", " pad_mode=\"constant\",\n", " trainable=False,\n", " output_format=\"Magnitude\",\n", " verbose=True,\n", ")" ] }, { "cell_type": "code", "execution_count": 75, "id": "58f004c5", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T01:00:35.908643Z", "start_time": "2023-09-07T01:00:35.904899Z" } }, "outputs": [ { "data": { "text/plain": [ "torch.Size([288000])" ] }, "execution_count": 75, "metadata": {}, "output_type": "execute_result" } ], "source": [ "input_tensor = torch.from_numpy(new_wav[1]).float()\n", "input_tensor.shape" ] }, { "cell_type": "code", "execution_count": 31, "id": "39ae6ed8", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:20:59.919522Z", "start_time": "2023-09-07T15:20:59.916810Z" } }, "outputs": [ { "data": { "text/plain": [ "300" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "288000 // new_wav[0] * EMBEDDING_RATE" ] }, { "cell_type": "code", "execution_count": 77, "id": "144f4bcc", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T01:00:38.572993Z", "start_time": "2023-09-07T01:00:38.459900Z" } }, "outputs": [], "source": [ "output_tensor = spec_layer(input_tensor)" ] }, { "cell_type": "code", "execution_count": 78, "id": "574e5193", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T01:00:39.115600Z", "start_time": "2023-09-07T01:00:39.112815Z" } }, "outputs": [ { "data": { "text/plain": [ "torch.Size([1, 336, 301])" ] }, "execution_count": 78, "metadata": {}, "output_type": "execute_result" } ], "source": [ "output_tensor.shape" ] }, { "cell_type": "code", "execution_count": 79, "id": "7a151f4d", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T01:00:43.598526Z", "start_time": "2023-09-07T01:00:43.540744Z" } }, "outputs": [ { "data": { "text/plain": [ "tensor([[[ 49.1279, 52.6179, 55.2570, ..., 2552.8076, 1880.0348,\n", " 1353.1643],\n", " [ 52.0929, 58.9065, 66.4197, ..., 1992.2830, 1287.6638,\n", " 714.6923],\n", " [ 51.8104, 59.5420, 68.0058, ..., 2252.4263, 1565.3777,\n", " 1002.5173],\n", " ...,\n", " [ 0.0000, 0.0000, 0.0000, ..., 330.8419, 308.4335,\n", " 925.4127],\n", " [ 0.0000, 0.0000, 0.0000, ..., 837.1494, 576.7127,\n", " 765.4628],\n", " [ 0.0000, 0.0000, 0.0000, ..., 868.1017, 926.3876,\n", " 564.1667]]])" ] }, "execution_count": 79, "metadata": {}, "output_type": "execute_result" } ], "source": [ "output_tensor" ] }, { "cell_type": "code", "execution_count": 80, "id": "f50e9751", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T01:01:44.413557Z", "start_time": "2023-09-07T01:01:44.349371Z" } }, "outputs": [], "source": [ "output_tensor_half = spec_layer(input_tensor[:144000])" ] }, { "cell_type": "code", "execution_count": 81, "id": "cadada08", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T01:01:46.820207Z", "start_time": "2023-09-07T01:01:46.816609Z" } }, "outputs": [ { "data": { "text/plain": [ "tensor([[[ 49.1279, 52.6179, 55.2570, ..., 149.2399, 147.4749, 144.6790],\n", " [ 52.0929, 58.9065, 66.4197, ..., 125.9033, 125.5422, 124.2411],\n", " [ 51.8104, 59.5420, 68.0058, ..., 95.9982, 95.1028, 93.6845],\n", " ...,\n", " [ 0.0000, 0.0000, 0.0000, ..., 45.0109, 18.7750, 76.7034],\n", " [ 0.0000, 0.0000, 0.0000, ..., 26.5843, 46.0399, 61.8331],\n", " [ 0.0000, 0.0000, 0.0000, ..., 6.8963, 67.4006, 36.9114]]])" ] }, "execution_count": 81, "metadata": {}, "output_type": "execute_result" } ], "source": [ "output_tensor_half" ] }, { "cell_type": "code", "execution_count": 57, "id": "618fa4e4", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:54:49.096938Z", "start_time": "2023-09-07T15:54:49.094965Z" } }, "outputs": [], "source": [ "# for i in range(151):\n", "# print(i, torch.equal(output_tensor_half[:, :, i], output_tensor[:, :, i]))" ] }, { "cell_type": "code", "execution_count": null, "id": "218a4b9c", "metadata": { "ExecuteTime": { "start_time": "2023-09-07T06:58:49.727Z" } }, "outputs": [], "source": [ "len(new_tsv_data)" ] }, { "cell_type": "markdown", "id": "f4962921", "metadata": {}, "source": [ "# Some validation on codebooks" ] }, { "cell_type": "code", "execution_count": 32, "id": "b3302b81", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:22:32.292137Z", "start_time": "2023-09-07T15:22:32.289446Z" } }, "outputs": [], "source": [ "codebook_1 = \"\"\"\n", "2717 1007 1023 1007 1023 1023 686 3971 1934 1116 408 2840 2969 603 2372 956 2711 3109 1354 2566 3435 952 81 1686 411 3993 2622 1019 1951 3153 1037 4\n", "002 146 2737 1464 3805 1029 3462 130 3343 2816 3204 2155 686 686 400 1991 3442 3965 2392 2341 2654 948 1740 850 7 1888 1826 2156 961 3679 1607 44 29\n", " 2578 1317 2817 1607 44 555 3351 2857 913 2796 2867 22 1825 2243 3327 1683 2625 1316 3008 3533 3389 518 492 3581 2359 1072 1220 2740 2063 3591 1393\n", "330 1915 3073 2302 3445 1904 1885 115 1281 3730 2811 1727 873 2048 1773 2947 2997 696 3269 1787 260 2381 3941 2345 2105 998 2234 2523 2105 1904 1188\n", " 3088 4037 1698 2739 483 2269 3068 4023 559 2037 3151 3761 3574 2167 2432 1054 114 787 2966 3969 1215 709 3534 2858 3364 3834 30 3055 1274 605 3091\n", "2470 2415 2392 836 298 617 1686 105 1738 1878 3714 2392 1776 1537 3610 2565 1251 3317 1561 3200 1072 1869 929 1206 889 3302 2044 782 1546 1822 1712\n", "1471 2342 1712 3144 215 3620 1779 2344 2857 924 175 2063 1763 1569 2142 330 2238 3378 1810 2300 2002 1560 3077 1263 2867 2048 1796 3136 3218 3855 30\n", "61 1629 3218 710 182 1658 3218 3855 2578 3267 2857 2777 993 1372 603 1128 3948 3596 960 3728 3247 1363 603 581 393 651 3752 3144 1870 511 3941 3163\n", "2714 998 2630 3157 1419 232 2623 2036 531 3999 2402 1104 1539 270 2963 951 3811 88 2800 2678 3466 3289 306 2880 734 1735 625 1793 2086 3283 920 1759\n", " 679 2481 1741 3667 3767 242 1572 317 3460 3042 3207 1687 1538 3868 1626 2326 3832 1965\"\"\"" ] }, { "cell_type": "code", "execution_count": 40, "id": "a4065474", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:26:58.519293Z", "start_time": "2023-09-07T15:26:58.517066Z" } }, "outputs": [], "source": [ "import sys\n", "sys.path.insert(0, \"/home/tony/Work/MERT/\")\n", "\n", "from mert_fairseq.tasks.mert_pretraining import LabelEncoder, Dictionary" ] }, { "cell_type": "code", "execution_count": 41, "id": "151f16d9", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:27:02.777782Z", "start_time": "2023-09-07T15:27:02.772316Z" } }, "outputs": [], "source": [ "test_dict = Dictionary.load(os.path.join(IN_LABEL_DIR, f\"dict.codec_{1}.txt\"))" ] }, { "cell_type": "code", "execution_count": 49, "id": "22be0497", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:30:28.498496Z", "start_time": "2023-09-07T15:30:28.496009Z" } }, "outputs": [ { "data": { "text/plain": [ "4101" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(test_dict.symbols)" ] }, { "cell_type": "code", "execution_count": 43, "id": "722915e6", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:27:21.067507Z", "start_time": "2023-09-07T15:27:21.065406Z" } }, "outputs": [], "source": [ "label_encoder = LabelEncoder(test_dict)" ] }, { "cell_type": "code", "execution_count": 44, "id": "c2554b68", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:27:38.459487Z", "start_time": "2023-09-07T15:27:38.434383Z" } }, "outputs": [ { "data": { "text/plain": [ "tensor([2721, 1011, 1027, 1011, 1027, 1027, 690, 3975, 1938, 1120, 412, 2844,\n", " 2973, 607, 2376, 960, 2715, 3113, 1358, 2570, 3439, 956, 85, 1690,\n", " 415, 3997, 2626, 1023, 1955, 3157, 1041, 8, 3, 150, 2741, 1468,\n", " 3809, 1033, 3466, 134, 3347, 2820, 3208, 2159, 690, 690, 404, 1995,\n", " 3446, 3969, 2396, 2345, 2658, 952, 1744, 854, 11, 1892, 1830, 2160,\n", " 965, 3683, 1611, 48, 33, 2582, 1321, 2821, 1611, 48, 559, 3355,\n", " 2861, 917, 2800, 2871, 26, 1829, 2247, 3331, 1687, 2629, 1320, 3012,\n", " 3537, 3393, 522, 496, 3585, 2363, 1076, 1224, 2744, 2067, 3595, 1397,\n", " 334, 1919, 3077, 2306, 3449, 1908, 1889, 119, 1285, 3734, 2815, 1731,\n", " 877, 2052, 1777, 2951, 3001, 700, 3273, 1791, 264, 2385, 3945, 2349,\n", " 2109, 1002, 2238, 2527, 2109, 1908, 1192, 3092, 4041, 1702, 2743, 487,\n", " 2273, 3072, 4027, 563, 2041, 3155, 3765, 3578, 2171, 2436, 1058, 118,\n", " 791, 2970, 3973, 1219, 713, 3538, 2862, 3368, 3838, 34, 3059, 1278,\n", " 609, 3095, 2474, 2419, 2396, 840, 302, 621, 1690, 109, 1742, 1882,\n", " 3718, 2396, 1780, 1541, 3614, 2569, 1255, 3321, 1565, 3204, 1076, 1873,\n", " 933, 1210, 893, 3306, 2048, 786, 1550, 1826, 1716, 1475, 2346, 1716,\n", " 3148, 219, 3624, 1783, 2348, 2861, 928, 179, 2067, 1767, 1573, 2146,\n", " 334, 2242, 3382, 1814, 2304, 2006, 1564, 3081, 1267, 2871, 2052, 1800,\n", " 3140, 3222, 3859, 34, 65, 1633, 3222, 714, 186, 1662, 3222, 3859,\n", " 2582, 3271, 2861, 2781, 997, 1376, 607, 1132, 3952, 3600, 964, 3732,\n", " 3251, 1367, 607, 585, 397, 655, 3756, 3148, 1874, 515, 3945, 3167,\n", " 2718, 1002, 2634, 3161, 1423, 236, 2627, 2040, 535, 4003, 2406, 1108,\n", " 1543, 274, 2967, 955, 3815, 92, 2804, 2682, 3470, 3293, 310, 2884,\n", " 738, 1739, 629, 1797, 2090, 3287, 924, 1763, 683, 2485, 1745, 3671,\n", " 3771, 246, 1576, 321, 3464, 3046, 3211, 1691, 1542, 3872, 1630, 2330,\n", " 3836, 1969], dtype=torch.int32)" ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "label_encoder(codebook_1)" ] }, { "cell_type": "code", "execution_count": 50, "id": "cf8d991c", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:31:10.891011Z", "start_time": "2023-09-07T15:31:10.881330Z" } }, "outputs": [], "source": [ "test_dict_2 = Dictionary.load(os.path.join(OUT_LABEL_DIR, f\"dict.codec_{1}.txt\"))" ] }, { "cell_type": "code", "execution_count": 52, "id": "7ac7ca22", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:31:45.816488Z", "start_time": "2023-09-07T15:31:45.813987Z" } }, "outputs": [ { "data": { "text/plain": [ "12" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(test_dict_2.symbols)" ] }, { "cell_type": "code", "execution_count": 53, "id": "44edfa8a", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:31:56.496676Z", "start_time": "2023-09-07T15:31:56.494800Z" } }, "outputs": [], "source": [ "label_encoder_2 = LabelEncoder(test_dict_2)" ] }, { "cell_type": "code", "execution_count": 54, "id": "5505a37d", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T15:32:01.026153Z", "start_time": "2023-09-07T15:32:01.021221Z" } }, "outputs": [ { "data": { "text/plain": [ "tensor([ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 11, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", " 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3],\n", " dtype=torch.int32)" ] }, "execution_count": 54, "metadata": {}, "output_type": "execute_result" } ], "source": [ "label_encoder_2(codebook_1)" ] }, { "cell_type": "code", "execution_count": null, "id": "3b4e1865", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 82, "id": "93fb3bab", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T18:21:47.044946Z", "start_time": "2023-09-07T18:21:38.914139Z" } }, "outputs": [], "source": [ "from collections import deque\n", "from suno_utils.tasks.gpt_v2.chirp_v1 import codec_decode, preload_codec_models\n", "\n", "_ = preload_codec_models(\"s3://suno-data/georg/trained_models/chirp_v1/codec.pt\")" ] }, { "cell_type": "code", "execution_count": 106, "id": "004a848a", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T18:25:17.817977Z", "start_time": "2023-09-07T18:25:17.815622Z" } }, "outputs": [], "source": [ "def head(filename, n=0):\n", " 'Return the first x line of a file'\n", " buf = bytearray(10240)\n", " with open(filename, \"rb\") as f:\n", " s = f.readinto(buf)\n", " # print(buf.decode().split(\"\\n\"))\n", " output = buf.decode().split(\"\\n\")[n]\n", " # print(output)\n", " return output" ] }, { "cell_type": "code", "execution_count": 107, "id": "6ed5c10a", "metadata": { "ExecuteTime": { "end_time": "2023-09-07T18:25:20.180858Z", "start_time": "2023-09-07T18:25:18.105300Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0\n", "1\n", "2\n", "3\n", "4\n", "5\n", "6\n", "7\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "data = []\n", "for n in range(8):\n", " print(n)\n", " line = head(f\"/app/suno/data/mert_25hz_short/label/train.codec_{n}\").strip()\n", " data.append(np.array([int(e) for e in line.split()]))\n", "codec_decode(np.stack(data).T).play()" ] }, { "cell_type": "markdown", "id": "f827fb11", "metadata": {}, "source": [ "# train" ] }, { "cell_type": "code", "execution_count": null, "id": "c4653e39", "metadata": {}, "outputs": [], "source": [ "OMP_NUM_THREADS=6 python -u /home/georg/code/fairseq/fairseq_cli/hydra_train.py \\\n", " --config-dir '/home/tony/Work/MERT/mert_fairseq/config/pretrain' \\\n", " --config-name 'custom_95M' \\\n", " common.user_dir='/home/georg/code/MERT/mert_fairseq' \\\n", " common.wandb_project='mert_test' \\\n", " checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_tony_testcpu' \\\n", " distributed_training.distributed_rank=0 \\\n", " distributed_training.distributed_world_size=4 \\\n", " distributed_training.nprocs_per_node=4 \\\n", " optimization.update_freq='[2]' \\\n", " distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n", " task.data='/app/suno/data/mert_25hz_short/audio_tsv' \\\n", " task.label_dir='/app/suno/data/mert_25hz_short/label' \\\n", " task.labels='[\"codec_0\",\"codec_1\",\"codec_2\",\"codec_3\",\"codec_4\",\"codec_5\",\"codec_6\",\"codec_7\"]' \\\n", " dataset.num_workers=6 \\\n", " dataset.max_tokens=1800000 \\\n", " dataset.disable_validation=true \\\n", " model.label_rate=25" ] }, { "cell_type": "code", "execution_count": null, "id": "e8eff264", "metadata": {}, "outputs": [], "source": [ "OMP_NUM_THREADS=6 python -u /home/georg/code/fairseq/fairseq_cli/hydra_train.py \\\n", " --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n", " --config-name 'custom_95M' \\\n", " common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n", " common.wandb_project='mert_test' \\\n", " checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_tony_test_cqtgpu' \\\n", " distributed_training.distributed_rank=0 \\\n", " distributed_training.distributed_world_size=4 \\\n", " distributed_training.nprocs_per_node=4 \\\n", " optimization.update_freq='[2]' \\\n", " distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n", " task.data='/app/suno/data/mert_25hz_short/audio_tsv' \\\n", " task.label_dir='/app/suno/data/mert_25hz_short/label' \\\n", " task.labels='[\"codec_0\",\"codec_1\",\"codec_2\",\"codec_3\",\"codec_4\",\"codec_5\",\"codec_6\",\"codec_7\"]' \\\n", " dataset.num_workers=6 \\\n", " dataset.max_tokens=1800000 \\\n", " dataset.disable_validation=true \\\n", " model.label_rate=25" ] }, { "cell_type": "code", "execution_count": null, "id": "37a8c716", "metadata": {}, "outputs": [], "source": [ "OMP_NUM_THREADS=6 python -u /home/georg/code/fairseq/fairseq_cli/hydra_train.py \\\n", " --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n", " --config-name 'custom_95M_long' \\\n", " common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n", " common.wandb_project='mert_test' \\\n", " checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_tony_test_12sec' \\\n", " distributed_training.distributed_rank=0 \\\n", " distributed_training.distributed_world_size=4 \\\n", " distributed_training.nprocs_per_node=4 \\\n", " optimization.update_freq='[2]' \\\n", " distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n", " task.data='/app/suno/data/mert_25hz_short/audio_tsv' \\\n", " task.label_dir='/app/suno/data/mert_25hz_short/label' \\\n", " task.labels='[\"codec_0\",\"codec_1\",\"codec_2\",\"codec_3\",\"codec_4\",\"codec_5\",\"codec_6\",\"codec_7\"]' \\\n", " dataset.num_workers=6 \\\n", " dataset.max_tokens=1800000 \\\n", " dataset.disable_validation=true \\\n", " model.label_rate=25" ] }, { "cell_type": "code", "execution_count": null, "id": "a563019a", "metadata": {}, "outputs": [], "source": [ "OMP_NUM_THREADS=6 python -u /home/tony/Work/glockenspiel/mert_training/src/fairseq/fairseq_cli/hydra_train.py \\\n", " --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n", " --config-name 'custom_95M_2' \\\n", " common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n", " common.wandb_project='mert_test' \\\n", " checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_tony_test_12sec' \\\n", " distributed_training.distributed_rank=0 \\\n", " distributed_training.distributed_world_size=4 \\\n", " distributed_training.nprocs_per_node=4 \\\n", " optimization.update_freq='[2]' \\\n", " distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n", " task.data='/app/suno/data/mert_25hz_short/audio_tsv' \\\n", " task.label_dir='/app/suno/data/mert_25hz_short/label' \\\n", " task.labels='[\"codec_0\",\"codec_1\",\"codec_2\",\"codec_3\",\"codec_4\",\"codec_5\",\"codec_6\",\"codec_7\"]' \\\n", " dataset.num_workers=6 \\\n", " dataset.max_tokens=1800000 \\\n", " dataset.disable_validation=true \\\n", " model.label_rate=25" ] }, { "cell_type": "code", "execution_count": null, "id": "1c2adfa1", "metadata": {}, "outputs": [], "source": [ "OMP_NUM_THREADS=6 python -u /home/tony/Work/glockenspiel/mert_training/src/fairseq/fairseq_cli/hydra_train.py \\\n", " --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n", " --config-name 'custom_95M_2' \\\n", " common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n", " common.wandb_project='mert_test' \\\n", " checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_tony_test_update4_3sec_maxtoken20_att_relax' \\\n", " distributed_training.distributed_rank=0 \\\n", " distributed_training.distributed_world_size=4 \\\n", " distributed_training.nprocs_per_node=4 \\\n", " optimization.update_freq='[4]' \\\n", " distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n", " task.data='/app/suno/data/mert_25hz_short/audio_tsv' \\\n", " task.label_dir='/app/suno/data/mert_25hz_short/label' \\\n", " task.labels='[\"codec_0\",\"codec_1\",\"codec_2\",\"codec_3\",\"codec_4\",\"codec_5\",\"codec_6\",\"codec_7\"]' \\\n", " dataset.num_workers=6 \\\n", " dataset.max_tokens=1000000 \\\n", " dataset.disable_validation=true \\\n", " model.label_rate=25" ] }, { "cell_type": "code", "execution_count": null, "id": "4a6960f2", "metadata": {}, "outputs": [], "source": [ "OMP_NUM_THREADS=6 python -u /home/tony/Work/glockenspiel/mert_training/src/fairseq/fairseq_cli/hydra_train.py \\\n", " --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n", " --config-name 'custom_95M_2' \\\n", " common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n", " common.wandb_project='mert_test' \\\n", " checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_tony_test_update4_5sec_maxtoken18_att_relax_reweight_highres3' \\\n", " distributed_training.distributed_rank=0 \\\n", " distributed_training.distributed_world_size=4 \\\n", " distributed_training.nprocs_per_node=4 \\\n", " optimization.update_freq='[4]' \\\n", " distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n", " task.data='/app/suno/data/mert_25hz_short/audio_tsv' \\\n", " task.label_dir='/app/suno/data/mert_25hz_short/label' \\\n", " task.labels='[\"codec_0\",\"codec_1\",\"codec_2\",\"codec_3\",\"codec_4\",\"codec_5\",\"codec_6\",\"codec_7\"]' \\\n", " dataset.num_workers=6 \\\n", " dataset.max_tokens=1800000 \\\n", " dataset.disable_validation=true \\\n", " model.label_rate=25" ] }, { "cell_type": "code", "execution_count": null, "id": "95059ba0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "481b2af8", "metadata": {}, "outputs": [], "source": [ " OMP_NUM_THREADS=6 python -u /home/tony/Work/glockenspiel/mert_training/src/fairseq/fairseq_cli/hydra_train.py \\\n", " --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n", " --config-name 'custom_95M' \\\n", " common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n", " common.wandb_project='mert_test' \\\n", " checkpoint.save_dir='/app/suno/checkpoints/mert_speech_test_valid' \\\n", " distributed_training.distributed_rank=0 \\\n", " distributed_training.distributed_world_size=4 \\\n", " distributed_training.nprocs_per_node=4 \\\n", " optimization.update_freq='[4]' \\\n", " distributed_training.distributed_init_method=\"tcp://127.0.0.2:39683\" \\\n", " task.data='/app/suno/data/mert_25hz_speech_short/audio_tsv' \\\n", " task.label_dir='/app/suno/data/mert_25hz_speech_short/label' \\\n", " task.labels='[\"codec_0\", \"codec_1\"]' \\\n", " task.do_valid_set_test=true \\\n", " dataset.num_workers=6 \\\n", " dataset.max_tokens=1800000 \\\n", " dataset.disable_validation=true \\\n", " model.label_rate=25" ] }, { "cell_type": "code", "execution_count": null, "id": "0194f166", "metadata": {}, "outputs": [], "source": [ "# on A100\n", "OMP_NUM_THREADS=6 python -u /home/tony/Work/glockenspiel/mert_training/src/fairseq/fairseq_cli/hydra_train.py \\\n", " --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n", " --config-name 'custom_95M_2' \\\n", " common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n", " common.wandb_project='mert_test' \\\n", " checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_speech_only' \\\n", " distributed_training.distributed_rank=0 \\\n", " distributed_training.distributed_world_size=8 \\\n", " distributed_training.nprocs_per_node=8 \\\n", " optimization.update_freq='[1]' \\\n", " distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n", " task.data='/app/suno/data/mert_25hz_speech_short/audio_tsv' \\\n", " task.label_dir='/app/suno/data/mert_25hz_speech_short/label' \\\n", " task.labels='[\"codec_0\", \"codec_1\"]' \\\n", " dataset.num_workers=6 \\\n", " dataset.max_tokens=4000000 \\\n", " dataset.disable_validation=false \\\n", " model.label_rate=25" ] }, { "cell_type": "code", "execution_count": null, "id": "0c67c7bd", "metadata": {}, "outputs": [], "source": [ " OMP_NUM_THREADS=6 python -u /home/tony/Work/glockenspiel/mert_training/src/fairseq/fairseq_cli/hydra_train.py \\\n", " --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n", " --config-name 'custom_95M' \\\n", " common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n", " common.wandb_project='mert_test' \\\n", " checkpoint.save_dir='/app/suno/checkpoints/mert_speech_test_valid_mixed' \\\n", " distributed_training.distributed_rank=0 \\\n", " distributed_training.distributed_world_size=4 \\\n", " distributed_training.nprocs_per_node=4 \\\n", " optimization.update_freq='[4]' \\\n", " distributed_training.distributed_init_method=\"tcp://127.0.0.2:39683\" \\\n", " task.data='/app/suno/data/mert_25hz_speech_short/audio_tsv' \\\n", " task.label_dir='/app/suno/data/mert_25hz_speech_short/label_mixed' \\\n", " task.labels='[\"codec_0\", \"codec_1\"]' \\\n", " task.do_valid_set_test=true \\\n", " dataset.num_workers=6 \\\n", " dataset.max_tokens=1800000 \\\n", " dataset.disable_validation=true \\\n", " model.label_rate=25" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }