{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "ffdbda4b",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-15T18:22:54.537900Z",
"start_time": "2023-10-15T18:22:54.536429Z"
}
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\""
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "dd41c10c",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-15T18:23:02.580061Z",
"start_time": "2023-10-15T18:23:00.744864Z"
}
},
"outputs": [],
"source": [
"import random\n",
"import json\n",
"import numpy as np\n",
"import tqdm\n",
"import torch\n",
"import funcy\n",
"import time\n",
"import gc\n",
"from scipy.io import wavfile\n",
"import tempfile\n",
"import collections\n",
"from collections import defaultdict\n",
"from joblib import Parallel, delayed\n",
"\n",
"from suno_utils.utils.s3 import _apply_mp\n",
"from suno_utils.audio import Audio\n",
"from suno_utils.tasks.data_loader import load_audio_mp\n",
"from suno_utils.utils.text import write_jsonl, read_jsonl, write_json, read_json\n",
"from suno_utils.utils.s3 import read_from_s3, check_s3_file_exists, open_from_s3\n",
"from suno_utils.audio.conversion import convert_audio_files\n",
"\n",
"SAMPLE_RATE = 24_000\n",
"EMBEDDING_RATE = 25\n",
"N_CODEBOOKS = 8\n",
"\n",
"IN_DATA_DIR = \"/app/suno/data/mert_25hz_speech_short\"\n",
"IN_AUDIO_DIR = os.path.join(IN_DATA_DIR, \"audio\")\n",
"IN_TSV_DIR = os.path.join(IN_DATA_DIR, \"audio_tsv\")\n",
"IN_LABEL_DIR = os.path.join(IN_DATA_DIR, \"label\")\n",
"\n",
"OUT_DATA_DIR = \"/app/suno/data/mert_25hz_short\"\n",
"OUT_AUDIO_DIR = os.path.join(OUT_DATA_DIR, \"audio\")\n",
"OUT_TSV_DIR = os.path.join(OUT_DATA_DIR, \"audio_tsv\")\n",
"OUT_LABEL_DIR = os.path.join(OUT_DATA_DIR, \"label\")\n",
"OUT_TEMP_DIR = os.path.join(OUT_DATA_DIR, \"temp\")\n",
"\n",
"os.makedirs(OUT_DATA_DIR, exist_ok=True)\n",
"os.makedirs(OUT_AUDIO_DIR, exist_ok=True)\n",
"os.makedirs(OUT_TSV_DIR, exist_ok=True)\n",
"os.makedirs(OUT_LABEL_DIR, exist_ok=True)\n",
"os.makedirs(OUT_TEMP_DIR, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "195c03f2",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-15T18:23:02.583149Z",
"start_time": "2023-10-15T18:23:02.581547Z"
}
},
"outputs": [],
"source": [
"MIN_SIZE = SAMPLE_RATE * 5\n",
"MAX_SIZE = SAMPLE_RATE * 12"
]
},
{
"cell_type": "markdown",
"id": "c729d86d",
"metadata": {},
"source": [
"# File loading"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "66631638",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-15T18:23:05.249745Z",
"start_time": "2023-10-15T18:23:05.128822Z"
}
},
"outputs": [],
"source": [
"# load current data\n",
"tsv_info = []\n",
"with open(os.path.join(IN_TSV_DIR, \"valid.tsv\"), \"r\") as f:\n",
" for line in f.read().strip().split(\"\\n\"):\n",
" if len(line.strip()) == 0:\n",
" continue\n",
" tsv_info.append(line.strip().split(\"\\t\"))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "cad52faa",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-15T18:23:06.026833Z",
"start_time": "2023-10-15T18:23:06.024962Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"77365\n"
]
}
],
"source": [
"print(len(tsv_info))"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "741d5c23",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-15T18:23:09.802793Z",
"start_time": "2023-10-15T18:23:09.800823Z"
}
},
"outputs": [],
"source": [
"tsv_data_path = tsv_info[0]\n",
"tsv_data = tsv_info[1:]"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "cb97b66b",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-15T18:23:38.877369Z",
"start_time": "2023-10-15T18:23:38.875845Z"
}
},
"outputs": [],
"source": [
"import shutil\n",
"# shutil.copyfile(os.path.join(log_dir, f), os.path.join(log_dir, f.replace(\".log\", \"_40k.log\")))"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "5f7ea21d",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-15T19:27:25.262537Z",
"start_time": "2023-10-15T18:25:26.808723Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 77364/77364 [1:01:58<00:00, 20.81it/s]\n"
]
}
],
"source": [
"for data in tqdm.tqdm(tsv_data):\n",
" # print(data)\n",
" # print(os.path.exists(os.path.join(OUT_AUDIO_DIR, data[0])))\n",
" shutil.copyfile(\n",
" os.path.join(IN_AUDIO_DIR, data[0]), os.path.join(OUT_AUDIO_DIR, data[0])\n",
" )\n",
" # print(os.path.exists(os.path.join(OUT_AUDIO_DIR, data[0])))\n",
" # break"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "38e506cc",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "d17fd56f",
"metadata": {},
"source": [
"## Slice the TSV files"
]
},
{
"cell_type": "code",
"execution_count": 89,
"id": "6a6a9b0e",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:14:28.152219Z",
"start_time": "2023-10-13T15:14:28.149885Z"
}
},
"outputs": [],
"source": [
"# tsv_data[0:10]"
]
},
{
"cell_type": "code",
"execution_count": 90,
"id": "c4906e01",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:14:31.459109Z",
"start_time": "2023-10-13T15:14:31.336948Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████| 950/950 [00:00<00:00, 8679.66it/s]\n"
]
}
],
"source": [
"new_tsv_data = []\n",
"# list of list of tuples,\n",
"# each index is the audio's codec spllit list (start, end)\n",
"split_times = []\n",
"for n_row, (audio_fn, audio_size) in tqdm.tqdm(\n",
" enumerate(tsv_data), total=len(tsv_data)\n",
"):\n",
" new_splits = []\n",
" for n in range(int(np.ceil(int(audio_size) / MAX_SIZE))):\n",
" start_idx = n * MAX_SIZE\n",
" end_idx = min(int(audio_size), (n + 1) * MAX_SIZE)\n",
" if end_idx - start_idx < MIN_SIZE:\n",
" continue\n",
" input_file_id = audio_fn.replace(\".wav\", \"\")\n",
" out_audio_fn = f\"speech_{input_file_id}_{n}.wav\"\n",
" new_tsv_data.append((audio_fn, out_audio_fn, start_idx, end_idx))\n",
" start_idx_embed = int(round(start_idx / SAMPLE_RATE * EMBEDDING_RATE))\n",
" end_idx_embed = int(round(end_idx / SAMPLE_RATE * EMBEDDING_RATE))\n",
" new_splits.append((start_idx_embed, end_idx_embed))\n",
" split_times.append(new_splits)"
]
},
{
"cell_type": "markdown",
"id": "4defb74e",
"metadata": {},
"source": [
"## Slice the codebook labels"
]
},
{
"cell_type": "code",
"execution_count": 91,
"id": "a0b384f7",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:14:35.029117Z",
"start_time": "2023-10-13T15:14:35.026298Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"950"
]
},
"execution_count": 91,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(split_times)"
]
},
{
"cell_type": "code",
"execution_count": 92,
"id": "c7a427db",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:14:35.615010Z",
"start_time": "2023-10-13T15:14:35.612772Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[(0, 300), (300, 600), (600, 900), (900, 1200), (1200, 1500), (1500, 1800), (1800, 2100), (2100, 2400), (2400, 2700), (2700, 3000), (3000, 3300), (3300, 3600), (3600, 3900), (3900, 4200), (4200, 4500), (4500, 4800), (4800, 5100), (5100, 5400), (5400, 5700), (5700, 6000), (6000, 6300), (6300, 6496)]\n"
]
}
],
"source": [
"print(split_times[0])"
]
},
{
"cell_type": "code",
"execution_count": 93,
"id": "9065a223",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:14:36.659228Z",
"start_time": "2023-10-13T15:14:36.656355Z"
}
},
"outputs": [],
"source": [
"# for codebook in range(N_CODEBOOKS):\n",
"# print(\"codebook:\", codebook)\n",
"# bunchsize = 100\n",
"# bunch = []\n",
"# with open(os.path.join(OUT_LABEL_DIR, f\"train.codec_{codebook}\"), \"w\") as f_out:\n",
"# with open(os.path.join(IN_LABEL_DIR, f\"train.codec_{codebook}\"), \"r\") as f:\n",
"# n_curr = 0\n",
"# for n_row, line in tqdm.tqdm(enumerate(f.readlines())):\n",
"# line = line.split()\n",
"# split_time_row = split_times[n_row]\n",
"# for (start_idx_embed, end_idx_embed) in split_time_row:\n",
"# bunch.append(\" \".join(line[start_idx_embed:end_idx_embed]) + \"\\n\")\n",
"# n_curr += 1\n",
"# if n_curr == bunchsize:\n",
"# n_curr = 0\n",
"# f_out.writelines(bunch)\n",
"# bunch = []\n",
"# # J also do the last one\n",
"# f_out.writelines(bunch)"
]
},
{
"cell_type": "code",
"execution_count": 94,
"id": "d236c653",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:14:37.336389Z",
"start_time": "2023-10-13T15:14:37.333730Z"
}
},
"outputs": [],
"source": [
"# out_mm = np.memmap(\"test.npy\", dtype=np.uint16, mode=\"w+\", shape=(1,))\n",
"# n_offs = 0\n",
"# for arr, arr_meta in encoded_arrays:\n",
"# out_mm[n_offs : n_offs + arr.size] = arr.reshape(\n",
"# -1,\n",
"# )\n",
"# n_offs += arr.size\n",
"# out_mm.flush()\n",
"# del out_mm"
]
},
{
"cell_type": "markdown",
"id": "135eec50",
"metadata": {},
"source": [
"# Convert the labels to np arrays"
]
},
{
"cell_type": "code",
"execution_count": 96,
"id": "cb93f04e",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:14:44.730346Z",
"start_time": "2023-10-13T15:14:44.727902Z"
}
},
"outputs": [],
"source": [
"total_rows = len(new_tsv_data)"
]
},
{
"cell_type": "code",
"execution_count": 97,
"id": "82d42119",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:15:03.500880Z",
"start_time": "2023-10-13T15:15:01.462929Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"950it [00:01, 499.96it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"77364\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"i = 0\n",
"max_shape = 300\n",
"data_split = \"valid\"\n",
"output_arrays = [np.ones((total_rows, max_shape), dtype=np.int16) * -1 for _ in range(N_CODEBOOKS)]\n",
"for origin_id, temp_tsv_data in tqdm.tqdm(enumerate(tsv_data)):\n",
" test_file_name = temp_tsv_data[0]\n",
" test_id = test_file_name.replace(\".wav\", \"\").replace(\"speech_\", \"\")\n",
" file_to_fetch = f\"{IN_DATA_DIR}/audio_label/{data_split}_{test_id}.npy\"\n",
" if not os.path.exists(file_to_fetch):\n",
" print(\"WTF\", file_to_fetch)\n",
" break\n",
" test_outputs = np.load(file_to_fetch)\n",
" # print(test_outputs.shape)\n",
" split_time = split_times[origin_id]\n",
" # if split_time[-1][1] + 1 != test_outputs.shape[0]:\n",
" # print(\"WTF, shape doesn't match\", split_time[-1][1], test_outputs.shape[0])\n",
" for split_start, split_end in (split_time):\n",
" for codebook in range(N_CODEBOOKS):\n",
" # print(output_arrays[codebook][i:i + n_batch_size, :].shape, test_outputs[:, :, codebook].shape)\n",
" test_slice = test_outputs[split_start:split_end, codebook]\n",
" output_arrays[codebook][i, :test_slice.shape[0]] = test_slice\n",
" i += 1\n",
"print(i)"
]
},
{
"cell_type": "code",
"execution_count": 98,
"id": "b14bc55f",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:15:13.170879Z",
"start_time": "2023-10-13T15:15:12.026708Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"codebook: 0\n",
"codebook: 1\n",
"codebook: 2\n",
"codebook: 3\n",
"codebook: 4\n",
"codebook: 5\n",
"codebook: 6\n",
"codebook: 7\n"
]
}
],
"source": [
"for codebook in range(N_CODEBOOKS):\n",
" print(\"codebook:\", codebook)\n",
" with open(os.path.join(OUT_LABEL_DIR, f\"{data_split}.codec_{codebook}.npy\"), \"wb\") as f_out:\n",
" np.save(f_out, output_arrays[codebook].astype(np.int16))"
]
},
{
"cell_type": "markdown",
"id": "20c16724",
"metadata": {},
"source": [
"## some validation"
]
},
{
"cell_type": "code",
"execution_count": 102,
"id": "ffd23c72",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:33:31.080768Z",
"start_time": "2023-10-13T15:33:31.078329Z"
}
},
"outputs": [],
"source": [
"test_row_id = 1234\n",
"test_file_name = new_tsv_data[test_row_id][1]\n",
"test_resembed = [output_arrays[codebook][test_row_id, :] for codebook in range(N_CODEBOOKS)]"
]
},
{
"cell_type": "code",
"execution_count": 103,
"id": "8fdda711",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:33:32.042469Z",
"start_time": "2023-10-13T15:33:31.978653Z"
}
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"test_audio = Audio.from_file(os.path.join(OUT_AUDIO_DIR, test_file_name))\n",
"test_resembed_array = np.array(test_resembed).swapaxes(0, 1)\n",
"test_audio.play()"
]
},
{
"cell_type": "code",
"execution_count": 104,
"id": "6d3ceb62",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:33:38.360349Z",
"start_time": "2023-10-13T15:33:34.073690Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/tony/anaconda3/envs/suno_env/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:30: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.\n",
" warnings.warn(\"torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.\")\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from suno_utils.tasks import dac\n",
"dac.preload_models(checkpoint_filepath=\"/home/tony/Data/MERT/codec.pt\")\n",
"dac.decode(test_resembed_array).play()"
]
},
{
"cell_type": "markdown",
"id": "5c75504e",
"metadata": {},
"source": [
"# Convert the labels to text"
]
},
{
"cell_type": "code",
"execution_count": 60,
"id": "177c9530",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T14:59:11.998380Z",
"start_time": "2023-10-13T14:59:11.996017Z"
}
},
"outputs": [],
"source": [
"test_resembed = [output_arrays[i][12345, :] for i in range(N_CODEBOOKS)]"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "02a83804",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T14:39:59.014292Z",
"start_time": "2023-09-07T14:39:58.911359Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/app/suno/data/mert_25hz/label/dict.codec_0.txt\n",
"/app/suno/data/mert_25hz/label/dict.codec_1.txt\n",
"/app/suno/data/mert_25hz/label/dict.codec_2.txt\n",
"/app/suno/data/mert_25hz/label/dict.codec_3.txt\n",
"/app/suno/data/mert_25hz/label/dict.codec_4.txt\n",
"/app/suno/data/mert_25hz/label/dict.codec_5.txt\n",
"/app/suno/data/mert_25hz/label/dict.codec_6.txt\n",
"/app/suno/data/mert_25hz/label/dict.codec_7.txt\n"
]
}
],
"source": [
"# FIX?!?\n",
"for codebook in range(N_CODEBOOKS):\n",
" with open(os.path.join(IN_LABEL_DIR, f\"dict.codec_{codebook}.txt\"), \"w\") as f:\n",
" print(os.path.join(IN_LABEL_DIR, f\"dict.codec_{codebook}.txt\"))\n",
" for n in range(4097):dd\n",
" f.write(f\"{n} {1}\\n\")"
]
},
{
"cell_type": "code",
"execution_count": 58,
"id": "678cc9e0",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:55:32.721283Z",
"start_time": "2023-09-07T15:55:32.576006Z"
}
},
"outputs": [],
"source": [
"for codebook in range(N_CODEBOOKS):\n",
" with open(os.path.join(OUT_LABEL_DIR, f\"dict.codec_{codebook}.txt\"), \"w\") as f:\n",
" for n in range(4097):\n",
" f.write(f\"{n} {1}\\n\")"
]
},
{
"cell_type": "markdown",
"id": "37937441",
"metadata": {},
"source": [
"## Write the train TSV"
]
},
{
"cell_type": "code",
"execution_count": 99,
"id": "b0d051d7",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:16:07.491220Z",
"start_time": "2023-10-13T15:16:07.420505Z"
}
},
"outputs": [],
"source": [
"with open(os.path.join(OUT_TSV_DIR, f\"{data_split}.tsv\"), \"w\") as f:\n",
" f.write(OUT_AUDIO_DIR + \"\\n\")\n",
" f.write(\n",
" \"\\n\".join(\n",
" [\n",
" fn + \"\\t\" + str(int(idx_end - idx_start))\n",
" for _, fn, idx_start, idx_end in new_tsv_data\n",
" ]\n",
" )\n",
" )"
]
},
{
"cell_type": "markdown",
"id": "6bd58f17",
"metadata": {},
"source": [
"## Generate the split audios Job file"
]
},
{
"cell_type": "code",
"execution_count": 100,
"id": "28864363",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:16:25.813609Z",
"start_time": "2023-10-13T15:16:25.580399Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"950 work_items\n"
]
}
],
"source": [
"# write new audios to disk\n",
"def _write_new_wav_files(work_item):\n",
" from_fn, to_work_items = work_item\n",
" _, audio_arr = wavfile.read(os.path.join(IN_AUDIO_DIR, from_fn))\n",
" for to_fn, (start_idx, end_idx) in to_work_items:\n",
" wavfile.write(\n",
" os.path.join(OUT_AUDIO_DIR, to_fn),\n",
" SAMPLE_RATE,\n",
" audio_arr[start_idx:end_idx],\n",
" )\n",
"work_items = defaultdict(list)\n",
"for from_fn, to_fn, start_idx, end_idx in new_tsv_data:\n",
" work_items[from_fn].append((to_fn, (start_idx, end_idx)))\n",
"work_items = [(k, v) for k, v in work_items.items()]\n",
"print(len(work_items), \"work_items\")"
]
},
{
"cell_type": "code",
"execution_count": 101,
"id": "4e77c558",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T15:16:40.509846Z",
"start_time": "2023-10-13T15:16:40.237307Z"
}
},
"outputs": [],
"source": [
"with open(f\"{data_split}_origin_speech.json\", \"w\") as fp:\n",
" json.dump(work_items, fp)"
]
},
{
"cell_type": "code",
"execution_count": 33,
"id": "12018181",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T13:59:39.895788Z",
"start_time": "2023-10-13T13:59:39.893990Z"
}
},
"outputs": [],
"source": [
"# _ = _apply_mp(\n",
"# _write_new_wav_files,\n",
"# work_items,\n",
"# chunksize=1,\n",
"# n_cores=32,\n",
"# joblib_backend=\"threads\",\n",
"# )"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "738138b8",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-06T17:42:30.677232Z",
"start_time": "2023-09-06T17:42:30.625842Z"
}
},
"outputs": [],
"source": [
"# codec_data = defaultdict(list)\n",
"# for codebook in range(N_CODEBOOKS):\n",
"# # print(i)\n",
"# with open(os.path.join(IN_LABEL_DIR, f\"train.codec_{codebook}\"), \"r\") as f:\n",
"# for line in f.read().strip().split(\"\\n\"):\n",
"# if len(line.strip()) == 0:\n",
"# continue\n",
"# codec_data[codebook].append(line.strip().split())\n",
"# # break\n",
"# for codebook in range(N_CODEBOOKS):\n",
"# with open(os.path.join(OUT_LABEL_DIR, f\"train.codec_{codebook}\"), \"w\") as f:\n",
"# f.write(\"\\n\".join(new_codec_data[codebook]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7428b9fc",
"metadata": {
"ExecuteTime": {
"start_time": "2023-09-07T06:24:59.615Z"
}
},
"outputs": [],
"source": [
"files = os.listdir(\"/app/suno/data/mert_25hz_short/audio\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "be9669ca",
"metadata": {
"ExecuteTime": {
"start_time": "2023-09-07T06:25:00.056Z"
}
},
"outputs": [],
"source": [
"len(files)"
]
},
{
"cell_type": "code",
"execution_count": 202,
"id": "1f86eb36",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T05:49:51.825989Z",
"start_time": "2023-09-07T05:49:51.590995Z"
}
},
"outputs": [],
"source": [
"selected = [f for f in files if \"_\" in f]"
]
},
{
"cell_type": "code",
"execution_count": 203,
"id": "3d49a73b",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T05:49:51.829834Z",
"start_time": "2023-09-07T05:49:51.827571Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"(6896994, 7085918)"
]
},
"execution_count": 203,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(selected), len(new_tsv_data)"
]
},
{
"cell_type": "code",
"execution_count": 213,
"id": "c8eceb78",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T06:02:08.734815Z",
"start_time": "2023-09-07T06:02:08.732246Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"('3.wav', '2_4.wav', 1152000, 1440000)"
]
},
"execution_count": 213,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"new_tsv_data[30]"
]
},
{
"cell_type": "code",
"execution_count": 230,
"id": "97b70c7d",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T06:23:08.640223Z",
"start_time": "2023-09-07T06:23:04.856146Z"
}
},
"outputs": [],
"source": [
"bad_map = {}\n",
"for i in new_tsv_data:\n",
" # bad_map[int(i[0].split(\".wav\")[0])] = int(i[1].split(\"_\")[0])\n",
" bad_map[int(i[1].split(\"_\")[0])] = int(i[0].split(\".wav\")[0])\n",
"# if int(i[0].split(\".wav\")[0]) == 436817:\n",
"# print(i)\n",
"# if int(i[1].split(\"_\")[0]) == 436817:\n",
"# print(i)"
]
},
{
"cell_type": "code",
"execution_count": 205,
"id": "c27f546d",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T05:49:53.725164Z",
"start_time": "2023-09-07T05:49:52.247231Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"436803\n"
]
}
],
"source": [
"must_unique_f = set(int(i[0].split(\".wav\")[0]) for i in new_tsv_data)\n",
"print(len(must_unique_f))"
]
},
{
"cell_type": "code",
"execution_count": 206,
"id": "daec6223",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T05:49:53.739580Z",
"start_time": "2023-09-07T05:49:53.726550Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"(1, 463758)"
]
},
"execution_count": 206,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"min(must_unique_f), max(must_unique_f)"
]
},
{
"cell_type": "code",
"execution_count": 207,
"id": "e6891f7f",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T05:49:55.345744Z",
"start_time": "2023-09-07T05:49:53.819901Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"436803\n"
]
}
],
"source": [
"must_unique_f_1 = set(int(i[1].split(\"_\")[0]) for i in new_tsv_data)\n",
"print(len(must_unique_f_1))"
]
},
{
"cell_type": "code",
"execution_count": 221,
"id": "868c9acd",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T06:18:46.028766Z",
"start_time": "2023-09-07T06:18:46.018969Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"25604\n"
]
}
],
"source": [
"print(len(must_unique_f.difference(must_unique_f_1)))"
]
},
{
"cell_type": "code",
"execution_count": 37,
"id": "410c5abb",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:26:31.449837Z",
"start_time": "2023-09-07T15:26:31.447994Z"
}
},
"outputs": [],
"source": [
"# for i in must_unique_f:\n",
"# if i not in must_unique_f_1:\n",
"# print(i)"
]
},
{
"cell_type": "code",
"execution_count": 38,
"id": "19a77adf",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:26:33.541989Z",
"start_time": "2023-09-07T15:26:33.540195Z"
}
},
"outputs": [],
"source": [
"# for i in must_unique_f_1:\n",
"# if i not in must_unique_f:\n",
"# print(i)"
]
},
{
"cell_type": "code",
"execution_count": 208,
"id": "95dffc70",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T05:49:55.349478Z",
"start_time": "2023-09-07T05:49:55.347112Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"(6896994, 7085918)"
]
},
"execution_count": 208,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(selected), len(new_tsv_data)"
]
},
{
"cell_type": "code",
"execution_count": 209,
"id": "ef64eb3a",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T05:49:57.736167Z",
"start_time": "2023-09-07T05:49:56.307871Z"
}
},
"outputs": [],
"source": [
"unique_f = set(int(f.split(\"_\")[0]) for f in selected)"
]
},
{
"cell_type": "code",
"execution_count": 210,
"id": "c564b71e",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T05:50:06.069558Z",
"start_time": "2023-09-07T05:50:06.067137Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"427307"
]
},
"execution_count": 210,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(unique_f)"
]
},
{
"cell_type": "code",
"execution_count": 187,
"id": "4101cd9a",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T05:44:15.653998Z",
"start_time": "2023-09-07T05:44:15.651596Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"427307"
]
},
"execution_count": 187,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(unique_f)"
]
},
{
"cell_type": "code",
"execution_count": 50,
"id": "5a97a281",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-06T21:59:37.236355Z",
"start_time": "2023-09-06T21:59:37.232184Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"[112736,\n",
" 112737,\n",
" 112738,\n",
" 112739,\n",
" 112740,\n",
" 112741,\n",
" 112742,\n",
" 112743,\n",
" 112744,\n",
" 112745,\n",
" 112746,\n",
" 112747,\n",
" 112748,\n",
" 112749,\n",
" 112750,\n",
" 112751,\n",
" 112752,\n",
" 112753,\n",
" 112754,\n",
" 112755]"
]
},
"execution_count": 50,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"list(sorted(unique_f))[-20:]"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "d824c90b",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:19:39.034143Z",
"start_time": "2023-09-07T15:19:39.031323Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"('5887.wav', '5579_5.wav', 1440000, 1728000)"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"new_tsv_data[102030]"
]
},
{
"cell_type": "markdown",
"id": "066bb0b2",
"metadata": {},
"source": [
"# Some validations"
]
},
{
"cell_type": "code",
"execution_count": 108,
"id": "cc4169ce",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T18:25:53.048806Z",
"start_time": "2023-09-07T18:25:53.046931Z"
}
},
"outputs": [],
"source": [
"selected_file = new_tsv_data[0][1]"
]
},
{
"cell_type": "code",
"execution_count": 109,
"id": "db72b8b7",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T18:25:53.711891Z",
"start_time": "2023-09-07T18:25:53.702216Z"
}
},
"outputs": [],
"source": [
"new_wav = wavfile.read(os.path.join(OUT_AUDIO_DIR, selected_file))"
]
},
{
"cell_type": "code",
"execution_count": 110,
"id": "6b67883e",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T18:25:53.827291Z",
"start_time": "2023-09-07T18:25:53.825112Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"(288000, 1)"
]
},
"execution_count": 110,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"new_wav[1].reshape(-1, 1).shape"
]
},
{
"cell_type": "code",
"execution_count": 111,
"id": "94c6df8e",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T18:25:54.052800Z",
"start_time": "2023-09-07T18:25:54.050693Z"
}
},
"outputs": [],
"source": [
"audio = Audio.from_file(os.path.join(OUT_AUDIO_DIR, selected_file))"
]
},
{
"cell_type": "code",
"execution_count": 112,
"id": "2d720608",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T18:25:54.443771Z",
"start_time": "2023-09-07T18:25:54.381841Z"
}
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"audio.play()"
]
},
{
"cell_type": "code",
"execution_count": 30,
"id": "d45020c4",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:20:27.594159Z",
"start_time": "2023-09-07T15:20:27.591239Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"(24000,\n",
" array([ 598, -553, 1800, ..., -5042, -7360, -11655], dtype=int16))"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"new_wav"
]
},
{
"cell_type": "code",
"execution_count": 55,
"id": "f2f87e70",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T00:52:29.427411Z",
"start_time": "2023-09-07T00:52:27.396720Z"
}
},
"outputs": [],
"source": [
"from nnAudio import features as nnAudioFeatures"
]
},
{
"cell_type": "code",
"execution_count": 60,
"id": "91d01f60",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T00:53:28.148373Z",
"start_time": "2023-09-07T00:53:27.640337Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CQT kernels created, time used = 0.5050 seconds\n"
]
}
],
"source": [
"spec_layer = nnAudioFeatures.cqt.CQT(\n",
" sr=new_wav[0],\n",
" hop_length=new_wav[0] // EMBEDDING_RATE,\n",
" fmin=32.7,\n",
" fmax=None,\n",
" n_bins=336,\n",
" bins_per_octave=336 // 7,\n",
" filter_scale=1,\n",
" norm=1,\n",
" window=\"hann\",\n",
" center=True,\n",
" pad_mode=\"constant\",\n",
" trainable=False,\n",
" output_format=\"Magnitude\",\n",
" verbose=True,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 75,
"id": "58f004c5",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T01:00:35.908643Z",
"start_time": "2023-09-07T01:00:35.904899Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([288000])"
]
},
"execution_count": 75,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"input_tensor = torch.from_numpy(new_wav[1]).float()\n",
"input_tensor.shape"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "39ae6ed8",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:20:59.919522Z",
"start_time": "2023-09-07T15:20:59.916810Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"300"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"288000 // new_wav[0] * EMBEDDING_RATE"
]
},
{
"cell_type": "code",
"execution_count": 77,
"id": "144f4bcc",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T01:00:38.572993Z",
"start_time": "2023-09-07T01:00:38.459900Z"
}
},
"outputs": [],
"source": [
"output_tensor = spec_layer(input_tensor)"
]
},
{
"cell_type": "code",
"execution_count": 78,
"id": "574e5193",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T01:00:39.115600Z",
"start_time": "2023-09-07T01:00:39.112815Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([1, 336, 301])"
]
},
"execution_count": 78,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"output_tensor.shape"
]
},
{
"cell_type": "code",
"execution_count": 79,
"id": "7a151f4d",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T01:00:43.598526Z",
"start_time": "2023-09-07T01:00:43.540744Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[[ 49.1279, 52.6179, 55.2570, ..., 2552.8076, 1880.0348,\n",
" 1353.1643],\n",
" [ 52.0929, 58.9065, 66.4197, ..., 1992.2830, 1287.6638,\n",
" 714.6923],\n",
" [ 51.8104, 59.5420, 68.0058, ..., 2252.4263, 1565.3777,\n",
" 1002.5173],\n",
" ...,\n",
" [ 0.0000, 0.0000, 0.0000, ..., 330.8419, 308.4335,\n",
" 925.4127],\n",
" [ 0.0000, 0.0000, 0.0000, ..., 837.1494, 576.7127,\n",
" 765.4628],\n",
" [ 0.0000, 0.0000, 0.0000, ..., 868.1017, 926.3876,\n",
" 564.1667]]])"
]
},
"execution_count": 79,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"output_tensor"
]
},
{
"cell_type": "code",
"execution_count": 80,
"id": "f50e9751",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T01:01:44.413557Z",
"start_time": "2023-09-07T01:01:44.349371Z"
}
},
"outputs": [],
"source": [
"output_tensor_half = spec_layer(input_tensor[:144000])"
]
},
{
"cell_type": "code",
"execution_count": 81,
"id": "cadada08",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T01:01:46.820207Z",
"start_time": "2023-09-07T01:01:46.816609Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[[ 49.1279, 52.6179, 55.2570, ..., 149.2399, 147.4749, 144.6790],\n",
" [ 52.0929, 58.9065, 66.4197, ..., 125.9033, 125.5422, 124.2411],\n",
" [ 51.8104, 59.5420, 68.0058, ..., 95.9982, 95.1028, 93.6845],\n",
" ...,\n",
" [ 0.0000, 0.0000, 0.0000, ..., 45.0109, 18.7750, 76.7034],\n",
" [ 0.0000, 0.0000, 0.0000, ..., 26.5843, 46.0399, 61.8331],\n",
" [ 0.0000, 0.0000, 0.0000, ..., 6.8963, 67.4006, 36.9114]]])"
]
},
"execution_count": 81,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"output_tensor_half"
]
},
{
"cell_type": "code",
"execution_count": 57,
"id": "618fa4e4",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:54:49.096938Z",
"start_time": "2023-09-07T15:54:49.094965Z"
}
},
"outputs": [],
"source": [
"# for i in range(151):\n",
"# print(i, torch.equal(output_tensor_half[:, :, i], output_tensor[:, :, i]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "218a4b9c",
"metadata": {
"ExecuteTime": {
"start_time": "2023-09-07T06:58:49.727Z"
}
},
"outputs": [],
"source": [
"len(new_tsv_data)"
]
},
{
"cell_type": "markdown",
"id": "f4962921",
"metadata": {},
"source": [
"# Some validation on codebooks"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "b3302b81",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:22:32.292137Z",
"start_time": "2023-09-07T15:22:32.289446Z"
}
},
"outputs": [],
"source": [
"codebook_1 = \"\"\"\n",
"2717 1007 1023 1007 1023 1023 686 3971 1934 1116 408 2840 2969 603 2372 956 2711 3109 1354 2566 3435 952 81 1686 411 3993 2622 1019 1951 3153 1037 4\n",
"002 146 2737 1464 3805 1029 3462 130 3343 2816 3204 2155 686 686 400 1991 3442 3965 2392 2341 2654 948 1740 850 7 1888 1826 2156 961 3679 1607 44 29\n",
" 2578 1317 2817 1607 44 555 3351 2857 913 2796 2867 22 1825 2243 3327 1683 2625 1316 3008 3533 3389 518 492 3581 2359 1072 1220 2740 2063 3591 1393\n",
"330 1915 3073 2302 3445 1904 1885 115 1281 3730 2811 1727 873 2048 1773 2947 2997 696 3269 1787 260 2381 3941 2345 2105 998 2234 2523 2105 1904 1188\n",
" 3088 4037 1698 2739 483 2269 3068 4023 559 2037 3151 3761 3574 2167 2432 1054 114 787 2966 3969 1215 709 3534 2858 3364 3834 30 3055 1274 605 3091\n",
"2470 2415 2392 836 298 617 1686 105 1738 1878 3714 2392 1776 1537 3610 2565 1251 3317 1561 3200 1072 1869 929 1206 889 3302 2044 782 1546 1822 1712\n",
"1471 2342 1712 3144 215 3620 1779 2344 2857 924 175 2063 1763 1569 2142 330 2238 3378 1810 2300 2002 1560 3077 1263 2867 2048 1796 3136 3218 3855 30\n",
"61 1629 3218 710 182 1658 3218 3855 2578 3267 2857 2777 993 1372 603 1128 3948 3596 960 3728 3247 1363 603 581 393 651 3752 3144 1870 511 3941 3163\n",
"2714 998 2630 3157 1419 232 2623 2036 531 3999 2402 1104 1539 270 2963 951 3811 88 2800 2678 3466 3289 306 2880 734 1735 625 1793 2086 3283 920 1759\n",
" 679 2481 1741 3667 3767 242 1572 317 3460 3042 3207 1687 1538 3868 1626 2326 3832 1965\"\"\""
]
},
{
"cell_type": "code",
"execution_count": 40,
"id": "a4065474",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:26:58.519293Z",
"start_time": "2023-09-07T15:26:58.517066Z"
}
},
"outputs": [],
"source": [
"import sys\n",
"sys.path.insert(0, \"/home/tony/Work/MERT/\")\n",
"\n",
"from mert_fairseq.tasks.mert_pretraining import LabelEncoder, Dictionary"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "151f16d9",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:27:02.777782Z",
"start_time": "2023-09-07T15:27:02.772316Z"
}
},
"outputs": [],
"source": [
"test_dict = Dictionary.load(os.path.join(IN_LABEL_DIR, f\"dict.codec_{1}.txt\"))"
]
},
{
"cell_type": "code",
"execution_count": 49,
"id": "22be0497",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:30:28.498496Z",
"start_time": "2023-09-07T15:30:28.496009Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"4101"
]
},
"execution_count": 49,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(test_dict.symbols)"
]
},
{
"cell_type": "code",
"execution_count": 43,
"id": "722915e6",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:27:21.067507Z",
"start_time": "2023-09-07T15:27:21.065406Z"
}
},
"outputs": [],
"source": [
"label_encoder = LabelEncoder(test_dict)"
]
},
{
"cell_type": "code",
"execution_count": 44,
"id": "c2554b68",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:27:38.459487Z",
"start_time": "2023-09-07T15:27:38.434383Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([2721, 1011, 1027, 1011, 1027, 1027, 690, 3975, 1938, 1120, 412, 2844,\n",
" 2973, 607, 2376, 960, 2715, 3113, 1358, 2570, 3439, 956, 85, 1690,\n",
" 415, 3997, 2626, 1023, 1955, 3157, 1041, 8, 3, 150, 2741, 1468,\n",
" 3809, 1033, 3466, 134, 3347, 2820, 3208, 2159, 690, 690, 404, 1995,\n",
" 3446, 3969, 2396, 2345, 2658, 952, 1744, 854, 11, 1892, 1830, 2160,\n",
" 965, 3683, 1611, 48, 33, 2582, 1321, 2821, 1611, 48, 559, 3355,\n",
" 2861, 917, 2800, 2871, 26, 1829, 2247, 3331, 1687, 2629, 1320, 3012,\n",
" 3537, 3393, 522, 496, 3585, 2363, 1076, 1224, 2744, 2067, 3595, 1397,\n",
" 334, 1919, 3077, 2306, 3449, 1908, 1889, 119, 1285, 3734, 2815, 1731,\n",
" 877, 2052, 1777, 2951, 3001, 700, 3273, 1791, 264, 2385, 3945, 2349,\n",
" 2109, 1002, 2238, 2527, 2109, 1908, 1192, 3092, 4041, 1702, 2743, 487,\n",
" 2273, 3072, 4027, 563, 2041, 3155, 3765, 3578, 2171, 2436, 1058, 118,\n",
" 791, 2970, 3973, 1219, 713, 3538, 2862, 3368, 3838, 34, 3059, 1278,\n",
" 609, 3095, 2474, 2419, 2396, 840, 302, 621, 1690, 109, 1742, 1882,\n",
" 3718, 2396, 1780, 1541, 3614, 2569, 1255, 3321, 1565, 3204, 1076, 1873,\n",
" 933, 1210, 893, 3306, 2048, 786, 1550, 1826, 1716, 1475, 2346, 1716,\n",
" 3148, 219, 3624, 1783, 2348, 2861, 928, 179, 2067, 1767, 1573, 2146,\n",
" 334, 2242, 3382, 1814, 2304, 2006, 1564, 3081, 1267, 2871, 2052, 1800,\n",
" 3140, 3222, 3859, 34, 65, 1633, 3222, 714, 186, 1662, 3222, 3859,\n",
" 2582, 3271, 2861, 2781, 997, 1376, 607, 1132, 3952, 3600, 964, 3732,\n",
" 3251, 1367, 607, 585, 397, 655, 3756, 3148, 1874, 515, 3945, 3167,\n",
" 2718, 1002, 2634, 3161, 1423, 236, 2627, 2040, 535, 4003, 2406, 1108,\n",
" 1543, 274, 2967, 955, 3815, 92, 2804, 2682, 3470, 3293, 310, 2884,\n",
" 738, 1739, 629, 1797, 2090, 3287, 924, 1763, 683, 2485, 1745, 3671,\n",
" 3771, 246, 1576, 321, 3464, 3046, 3211, 1691, 1542, 3872, 1630, 2330,\n",
" 3836, 1969], dtype=torch.int32)"
]
},
"execution_count": 44,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"label_encoder(codebook_1)"
]
},
{
"cell_type": "code",
"execution_count": 50,
"id": "cf8d991c",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:31:10.891011Z",
"start_time": "2023-09-07T15:31:10.881330Z"
}
},
"outputs": [],
"source": [
"test_dict_2 = Dictionary.load(os.path.join(OUT_LABEL_DIR, f\"dict.codec_{1}.txt\"))"
]
},
{
"cell_type": "code",
"execution_count": 52,
"id": "7ac7ca22",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:31:45.816488Z",
"start_time": "2023-09-07T15:31:45.813987Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"12"
]
},
"execution_count": 52,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(test_dict_2.symbols)"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "44edfa8a",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:31:56.496676Z",
"start_time": "2023-09-07T15:31:56.494800Z"
}
},
"outputs": [],
"source": [
"label_encoder_2 = LabelEncoder(test_dict_2)"
]
},
{
"cell_type": "code",
"execution_count": 54,
"id": "5505a37d",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T15:32:01.026153Z",
"start_time": "2023-09-07T15:32:01.021221Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 11, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
" 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3],\n",
" dtype=torch.int32)"
]
},
"execution_count": 54,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"label_encoder_2(codebook_1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3b4e1865",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 82,
"id": "93fb3bab",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T18:21:47.044946Z",
"start_time": "2023-09-07T18:21:38.914139Z"
}
},
"outputs": [],
"source": [
"from collections import deque\n",
"from suno_utils.tasks.gpt_v2.chirp_v1 import codec_decode, preload_codec_models\n",
"\n",
"_ = preload_codec_models(\"s3://suno-data/georg/trained_models/chirp_v1/codec.pt\")"
]
},
{
"cell_type": "code",
"execution_count": 106,
"id": "004a848a",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T18:25:17.817977Z",
"start_time": "2023-09-07T18:25:17.815622Z"
}
},
"outputs": [],
"source": [
"def head(filename, n=0):\n",
" 'Return the first x line of a file'\n",
" buf = bytearray(10240)\n",
" with open(filename, \"rb\") as f:\n",
" s = f.readinto(buf)\n",
" # print(buf.decode().split(\"\\n\"))\n",
" output = buf.decode().split(\"\\n\")[n]\n",
" # print(output)\n",
" return output"
]
},
{
"cell_type": "code",
"execution_count": 107,
"id": "6ed5c10a",
"metadata": {
"ExecuteTime": {
"end_time": "2023-09-07T18:25:20.180858Z",
"start_time": "2023-09-07T18:25:18.105300Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0\n",
"1\n",
"2\n",
"3\n",
"4\n",
"5\n",
"6\n",
"7\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"data = []\n",
"for n in range(8):\n",
" print(n)\n",
" line = head(f\"/app/suno/data/mert_25hz_short/label/train.codec_{n}\").strip()\n",
" data.append(np.array([int(e) for e in line.split()]))\n",
"codec_decode(np.stack(data).T).play()"
]
},
{
"cell_type": "markdown",
"id": "f827fb11",
"metadata": {},
"source": [
"# train"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c4653e39",
"metadata": {},
"outputs": [],
"source": [
"OMP_NUM_THREADS=6 python -u /home/georg/code/fairseq/fairseq_cli/hydra_train.py \\\n",
" --config-dir '/home/tony/Work/MERT/mert_fairseq/config/pretrain' \\\n",
" --config-name 'custom_95M' \\\n",
" common.user_dir='/home/georg/code/MERT/mert_fairseq' \\\n",
" common.wandb_project='mert_test' \\\n",
" checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_tony_testcpu' \\\n",
" distributed_training.distributed_rank=0 \\\n",
" distributed_training.distributed_world_size=4 \\\n",
" distributed_training.nprocs_per_node=4 \\\n",
" optimization.update_freq='[2]' \\\n",
" distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n",
" task.data='/app/suno/data/mert_25hz_short/audio_tsv' \\\n",
" task.label_dir='/app/suno/data/mert_25hz_short/label' \\\n",
" task.labels='[\"codec_0\",\"codec_1\",\"codec_2\",\"codec_3\",\"codec_4\",\"codec_5\",\"codec_6\",\"codec_7\"]' \\\n",
" dataset.num_workers=6 \\\n",
" dataset.max_tokens=1800000 \\\n",
" dataset.disable_validation=true \\\n",
" model.label_rate=25"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e8eff264",
"metadata": {},
"outputs": [],
"source": [
"OMP_NUM_THREADS=6 python -u /home/georg/code/fairseq/fairseq_cli/hydra_train.py \\\n",
" --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n",
" --config-name 'custom_95M' \\\n",
" common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n",
" common.wandb_project='mert_test' \\\n",
" checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_tony_test_cqtgpu' \\\n",
" distributed_training.distributed_rank=0 \\\n",
" distributed_training.distributed_world_size=4 \\\n",
" distributed_training.nprocs_per_node=4 \\\n",
" optimization.update_freq='[2]' \\\n",
" distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n",
" task.data='/app/suno/data/mert_25hz_short/audio_tsv' \\\n",
" task.label_dir='/app/suno/data/mert_25hz_short/label' \\\n",
" task.labels='[\"codec_0\",\"codec_1\",\"codec_2\",\"codec_3\",\"codec_4\",\"codec_5\",\"codec_6\",\"codec_7\"]' \\\n",
" dataset.num_workers=6 \\\n",
" dataset.max_tokens=1800000 \\\n",
" dataset.disable_validation=true \\\n",
" model.label_rate=25"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "37a8c716",
"metadata": {},
"outputs": [],
"source": [
"OMP_NUM_THREADS=6 python -u /home/georg/code/fairseq/fairseq_cli/hydra_train.py \\\n",
" --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n",
" --config-name 'custom_95M_long' \\\n",
" common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n",
" common.wandb_project='mert_test' \\\n",
" checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_tony_test_12sec' \\\n",
" distributed_training.distributed_rank=0 \\\n",
" distributed_training.distributed_world_size=4 \\\n",
" distributed_training.nprocs_per_node=4 \\\n",
" optimization.update_freq='[2]' \\\n",
" distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n",
" task.data='/app/suno/data/mert_25hz_short/audio_tsv' \\\n",
" task.label_dir='/app/suno/data/mert_25hz_short/label' \\\n",
" task.labels='[\"codec_0\",\"codec_1\",\"codec_2\",\"codec_3\",\"codec_4\",\"codec_5\",\"codec_6\",\"codec_7\"]' \\\n",
" dataset.num_workers=6 \\\n",
" dataset.max_tokens=1800000 \\\n",
" dataset.disable_validation=true \\\n",
" model.label_rate=25"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a563019a",
"metadata": {},
"outputs": [],
"source": [
"OMP_NUM_THREADS=6 python -u /home/tony/Work/glockenspiel/mert_training/src/fairseq/fairseq_cli/hydra_train.py \\\n",
" --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n",
" --config-name 'custom_95M_2' \\\n",
" common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n",
" common.wandb_project='mert_test' \\\n",
" checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_tony_test_12sec' \\\n",
" distributed_training.distributed_rank=0 \\\n",
" distributed_training.distributed_world_size=4 \\\n",
" distributed_training.nprocs_per_node=4 \\\n",
" optimization.update_freq='[2]' \\\n",
" distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n",
" task.data='/app/suno/data/mert_25hz_short/audio_tsv' \\\n",
" task.label_dir='/app/suno/data/mert_25hz_short/label' \\\n",
" task.labels='[\"codec_0\",\"codec_1\",\"codec_2\",\"codec_3\",\"codec_4\",\"codec_5\",\"codec_6\",\"codec_7\"]' \\\n",
" dataset.num_workers=6 \\\n",
" dataset.max_tokens=1800000 \\\n",
" dataset.disable_validation=true \\\n",
" model.label_rate=25"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1c2adfa1",
"metadata": {},
"outputs": [],
"source": [
"OMP_NUM_THREADS=6 python -u /home/tony/Work/glockenspiel/mert_training/src/fairseq/fairseq_cli/hydra_train.py \\\n",
" --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n",
" --config-name 'custom_95M_2' \\\n",
" common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n",
" common.wandb_project='mert_test' \\\n",
" checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_tony_test_update4_3sec_maxtoken20_att_relax' \\\n",
" distributed_training.distributed_rank=0 \\\n",
" distributed_training.distributed_world_size=4 \\\n",
" distributed_training.nprocs_per_node=4 \\\n",
" optimization.update_freq='[4]' \\\n",
" distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n",
" task.data='/app/suno/data/mert_25hz_short/audio_tsv' \\\n",
" task.label_dir='/app/suno/data/mert_25hz_short/label' \\\n",
" task.labels='[\"codec_0\",\"codec_1\",\"codec_2\",\"codec_3\",\"codec_4\",\"codec_5\",\"codec_6\",\"codec_7\"]' \\\n",
" dataset.num_workers=6 \\\n",
" dataset.max_tokens=1000000 \\\n",
" dataset.disable_validation=true \\\n",
" model.label_rate=25"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4a6960f2",
"metadata": {},
"outputs": [],
"source": [
"OMP_NUM_THREADS=6 python -u /home/tony/Work/glockenspiel/mert_training/src/fairseq/fairseq_cli/hydra_train.py \\\n",
" --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n",
" --config-name 'custom_95M_2' \\\n",
" common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n",
" common.wandb_project='mert_test' \\\n",
" checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_tony_test_update4_5sec_maxtoken18_att_relax_reweight_highres3' \\\n",
" distributed_training.distributed_rank=0 \\\n",
" distributed_training.distributed_world_size=4 \\\n",
" distributed_training.nprocs_per_node=4 \\\n",
" optimization.update_freq='[4]' \\\n",
" distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n",
" task.data='/app/suno/data/mert_25hz_short/audio_tsv' \\\n",
" task.label_dir='/app/suno/data/mert_25hz_short/label' \\\n",
" task.labels='[\"codec_0\",\"codec_1\",\"codec_2\",\"codec_3\",\"codec_4\",\"codec_5\",\"codec_6\",\"codec_7\"]' \\\n",
" dataset.num_workers=6 \\\n",
" dataset.max_tokens=1800000 \\\n",
" dataset.disable_validation=true \\\n",
" model.label_rate=25"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "95059ba0",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "481b2af8",
"metadata": {},
"outputs": [],
"source": [
" OMP_NUM_THREADS=6 python -u /home/tony/Work/glockenspiel/mert_training/src/fairseq/fairseq_cli/hydra_train.py \\\n",
" --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n",
" --config-name 'custom_95M' \\\n",
" common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n",
" common.wandb_project='mert_test' \\\n",
" checkpoint.save_dir='/app/suno/checkpoints/mert_speech_test_valid' \\\n",
" distributed_training.distributed_rank=0 \\\n",
" distributed_training.distributed_world_size=4 \\\n",
" distributed_training.nprocs_per_node=4 \\\n",
" optimization.update_freq='[4]' \\\n",
" distributed_training.distributed_init_method=\"tcp://127.0.0.2:39683\" \\\n",
" task.data='/app/suno/data/mert_25hz_speech_short/audio_tsv' \\\n",
" task.label_dir='/app/suno/data/mert_25hz_speech_short/label' \\\n",
" task.labels='[\"codec_0\", \"codec_1\"]' \\\n",
" task.do_valid_set_test=true \\\n",
" dataset.num_workers=6 \\\n",
" dataset.max_tokens=1800000 \\\n",
" dataset.disable_validation=true \\\n",
" model.label_rate=25"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0194f166",
"metadata": {},
"outputs": [],
"source": [
"# on A100\n",
"OMP_NUM_THREADS=6 python -u /home/tony/Work/glockenspiel/mert_training/src/fairseq/fairseq_cli/hydra_train.py \\\n",
" --config-dir '/home/tony/Work/glockenspiel/mert_training/mert_fairseq/config/pretrain' \\\n",
" --config-name 'custom_95M' \\\n",
" common.user_dir='/home/tony/Work/glockenspiel/mert_training/mert_fairseq' \\\n",
" common.wandb_project='mert_test' \\\n",
" checkpoint.save_dir='/app/suno/checkpoints/mert_25hz_speech_only' \\\n",
" distributed_training.distributed_rank=0 \\\n",
" distributed_training.distributed_world_size=8 \\\n",
" distributed_training.nprocs_per_node=8 \\\n",
" optimization.update_freq='[1]' \\\n",
" distributed_training.distributed_init_method=\"tcp://127.0.0.1:39683\" \\\n",
" task.data='/app/suno/data/mert_25hz_speech_short/audio_tsv' \\\n",
" task.label_dir='/app/suno/data/mert_25hz_speech_short/label' \\\n",
" task.labels='[\"codec_0\", \"codec_1\"]' \\\n",
" dataset.num_workers=6 \\\n",
" dataset.max_tokens=4000000 \\\n",
" dataset.disable_validation=false \\\n",
" model.label_rate=25"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 5
}