{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f4b969c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Georg multi\n",
    "OMP_NUM_THREADS=1 torchrun \\\n",
    "    --nnodes=8 --nproc_per_node=8 \\\n",
    "    --rdzv_backend=c10d --rdzv_id=123 --rdzv_endpoint=\"compute-hpc-node-294:1234\" \\\n",
    "    train.py \\\n",
    "    --out_dir=\"/app/suno/checkpoints\" \\\n",
    "    --data_dir=\"/mnt/localdisk/data\" \\\n",
    "    --train_filename=\"data_tr.bin\" \\\n",
    "    --train_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --train_info_filename=\"ft_info_tr.json\" \\\n",
    "    --val_filename=\"data_tr.bin\" \\\n",
    "    --val_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --val_info_filename=\"ft_info_val.json\" \\\n",
    "    --preload_checkpoint=\"/app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt\" \\\n",
    "    --weights_multiplier=\"genius_hq_lyrics:1;pond5_music:1\" \\\n",
    "    --is_finetune=True \\\n",
    "    --learning_rate=2e-5 \\\n",
    "    --max_iters=5_000 \\\n",
    "    --eval_interval=100 \\\n",
    "    --eval_iters=500 \\\n",
    "    --log_interval=10 \\\n",
    "    --block_size=4288 \\\n",
    "    --t_text=1152 \\\n",
    "    --t_memmap=3008 \\\n",
    "    --t_audio=3136 \\\n",
    "    --n_layer=48 \\\n",
    "    --n_head=32 \\\n",
    "    --d_head=64 \\\n",
    "    --coarse_n_codebooks=8 \\\n",
    "    --gradient_accumulation_steps=2 \\\n",
    "    --batch_size=2 \\\n",
    "    --fsdp=True \\\n",
    "    --sharding_strategy=\"full_shard\" \\\n",
    "    --wandb_log=True \\\n",
    "    --wandb_project=\"chirp-v2-ft\" \\\n",
    "    --wandb_run_name=\"xxl_ft\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e262122",
   "metadata": {},
   "outputs": [],
   "source": [
    "OMP_NUM_THREADS=4 torchrun \\\n",
    "    --nnodes=1 --nproc_per_node=8 \\\n",
    "    train.py \\\n",
    "    --out_dir=\"/app/suno/checkpoints\" \\\n",
    "    --data_dir=\"/mnt/localdisk/data\" \\\n",
    "    --train_filename=\"data_tr.bin\" \\\n",
    "    --train_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --train_info_filename=\"ft_info_tr.json\" \\\n",
    "    --val_filename=\"data_tr.bin\" \\\n",
    "    --val_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --val_info_filename=\"ft_info_val.json\" \\\n",
    "    --preload_checkpoint=\"/app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt\" \\\n",
    "    --weights_multiplier=\"genius_hq_lyrics:1;pond5_music:1\" \\\n",
    "    --is_finetune=True \\\n",
    "    --learning_rate=1e-6 \\\n",
    "    --max_iters=5_000 \\\n",
    "    --eval_interval=100 \\\n",
    "    --eval_iters=500 \\\n",
    "    --log_interval=10 \\\n",
    "    --block_size=4288 \\\n",
    "    --t_text=1152 \\\n",
    "    --t_memmap=3008 \\\n",
    "    --t_audio=3136 \\\n",
    "    --n_layer=48 \\\n",
    "    --n_head=32 \\\n",
    "    --d_head=64 \\\n",
    "    --coarse_n_codebooks=8 \\\n",
    "    --gradient_accumulation_steps=2 \\\n",
    "    --batch_size=2 \\\n",
    "    --fsdp=True \\\n",
    "    --sharding_strategy=\"full_shard\" \\\n",
    "    --wandb_log=True \\\n",
    "    --wandb_project=\"chirp-v2-ft-tony\" \\\n",
    "    --wandb_run_name=\"xxl_ft_test\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee108855",
   "metadata": {},
   "outputs": [],
   "source": [
    "OMP_NUM_THREADS=4 torchrun \\\n",
    "    --nnodes=1 --nproc_per_node=8 \\\n",
    "    train.py \\\n",
    "    --out_dir=\"/app/suno/checkpoints\" \\\n",
    "    --data_dir=\"/mnt/localdisk/data\" \\\n",
    "    --train_filename=\"data_tr.bin\" \\\n",
    "    --train_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --train_info_filename=\"ft_info_tr.json\" \\\n",
    "    --val_filename=\"data_tr.bin\" \\\n",
    "    --val_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --val_info_filename=\"ft_info_val.json\" \\\n",
    "    --preload_checkpoint=\"/app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt\" \\\n",
    "    --weights_multiplier=\"genius_hq_lyrics:1;pond5_music:1\" \\\n",
    "    --is_finetune=True \\\n",
    "    --learning_rate=3e-5 \\\n",
    "    --max_iters=20_000 \\\n",
    "    --eval_interval=100 \\\n",
    "    --eval_iters=500 \\\n",
    "    --log_interval=10 \\\n",
    "    --block_size=4288 \\\n",
    "    --t_text=1152 \\\n",
    "    --t_memmap=3008 \\\n",
    "    --t_audio=3136 \\\n",
    "    --n_layer=48 \\\n",
    "    --n_head=32 \\\n",
    "    --d_head=64 \\\n",
    "    --coarse_n_codebooks=8 \\\n",
    "    --gradient_accumulation_steps=1 \\\n",
    "    --batch_size=2 \\\n",
    "    --fsdp=True \\\n",
    "    --sharding_strategy=\"full_shard\" \\\n",
    "    --wandb_log=True \\\n",
    "    --wandb_project=\"chirp-v2-ft-tony\" \\\n",
    "    --wandb_run_name=\"xxl_ft_test_3e5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c2f193c",
   "metadata": {},
   "outputs": [],
   "source": [
    "OMP_NUM_THREADS=4 torchrun \\\n",
    "    --nnodes=1 --nproc_per_node=8 \\\n",
    "    train.py \\\n",
    "    --out_dir=\"/app/suno/checkpoints\" \\\n",
    "    --data_dir=\"/mnt/localdisk/data\" \\\n",
    "    --train_filename=\"data_tr.bin\" \\\n",
    "    --train_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --train_info_filename=\"ft_info_tr.json\" \\\n",
    "    --val_filename=\"data_tr.bin\" \\\n",
    "    --val_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --val_info_filename=\"ft_info_val.json\" \\\n",
    "    --preload_checkpoint=\"/app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt\" \\\n",
    "    --weights_multiplier=\"genius_hq_lyrics:1;pond5_music:1\" \\\n",
    "    --is_finetune=True \\\n",
    "    --learning_rate=3e-5 \\\n",
    "    --max_iters=100_000 \\\n",
    "    --eval_interval=100 \\\n",
    "    --eval_iters=500 \\\n",
    "    --log_interval=10 \\\n",
    "    --block_size=4288 \\\n",
    "    --t_text=1152 \\\n",
    "    --t_memmap=3008 \\\n",
    "    --t_audio=3136 \\\n",
    "    --n_layer=48 \\\n",
    "    --n_head=32 \\\n",
    "    --d_head=64 \\\n",
    "    --coarse_n_codebooks=8 \\\n",
    "    --gradient_accumulation_steps=1 \\\n",
    "    --batch_size=2 \\\n",
    "    --fsdp=True \\\n",
    "    --sharding_strategy=\"full_shard\" \\\n",
    "    --wandb_log=True \\\n",
    "    --wandb_project=\"chirp-v2-ft-tony\" \\\n",
    "    --wandb_run_name=\"xxl_ft_test_longer\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce33a7a1",
   "metadata": {},
   "source": [
    "# Genius fine tune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cba8679",
   "metadata": {},
   "outputs": [],
   "source": [
    "OMP_NUM_THREADS=4 torchrun \\\n",
    "    --nnodes=1 --nproc_per_node=8 \\\n",
    "    train.py \\\n",
    "    --out_dir=\"/app/suno/checkpoints\" \\\n",
    "    --data_dir=\"/app/suno/data/chirp_v2_finetune\" \\\n",
    "    --train_filename=\"data_tr.bin\" \\\n",
    "    --train_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --train_info_filename=\"info_tr.json\" \\\n",
    "    --val_filename=\"data_val.bin\" \\\n",
    "    --val_metas_filename=\"metas_val.jsonl\" \\\n",
    "    --val_info_filename=\"info_val.json\" \\\n",
    "    --preload_checkpoint=\"/app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt\" \\\n",
    "    --weights_multiplier=\"genius_hq_lyrics:1;genius_hq_lyrics_foreign:2\" \\\n",
    "    --is_finetune=True \\\n",
    "    --learning_rate=3e-5 \\\n",
    "    --min_lr=1e-6 \\\n",
    "    --warmup_iters=2_000 \\\n",
    "    --max_iters=10_000 \\\n",
    "    --eval_interval=1_000 \\\n",
    "    --eval_iters=50 \\\n",
    "    --log_interval=10 \\\n",
    "    --block_size=4288 \\\n",
    "    --t_text=1152 \\\n",
    "    --t_memmap=3008 \\\n",
    "    --t_audio=3136 \\\n",
    "    --n_layer=48 \\\n",
    "    --n_head=32 \\\n",
    "    --d_head=64 \\\n",
    "    --coarse_n_codebooks=8 \\\n",
    "    --gradient_accumulation_steps=1 \\\n",
    "    --batch_size=2 \\\n",
    "    --fsdp=True \\\n",
    "    --sharding_strategy=\"full_shard\" \\\n",
    "    --wandb_log=True \\\n",
    "    --wandb_project=\"chirp-v2-ft-tony\" \\\n",
    "    --wandb_run_name=\"xxl_ft_genius\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1933dbca",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af278847",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tony multi: 54, 541, 78, 409\n",
    "OMP_NUM_THREADS=4 torchrun \\\n",
    "    --nnodes=4 --nproc_per_node=8 \\\n",
    "    --rdzv_backend=c10d --rdzv_id=123 --rdzv_endpoint=\"compute-hpc-node-54:1234\" \\\n",
    "    train.py \\\n",
    "    --out_dir=\"/app/suno/checkpoints\" \\\n",
    "    --data_dir=\"/app/suno/data/chirp_v2_finetune\" \\\n",
    "    --train_filename=\"data_tr.bin\" \\\n",
    "    --train_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --train_info_filename=\"info_tr.json\" \\\n",
    "    --val_filename=\"data_val.bin\" \\\n",
    "    --val_metas_filename=\"metas_val.jsonl\" \\\n",
    "    --val_info_filename=\"info_val.json\" \\\n",
    "    --preload_checkpoint=\"/app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt\" \\\n",
    "    --weights_multiplier=\"genius_hq_lyrics:1;genius_hq_lyrics_foreign:2\" \\\n",
    "    --is_finetune=True \\\n",
    "    --learning_rate=3e-5 \\\n",
    "    --min_lr=1e-6 \\\n",
    "    --warmup_iters=5_000 \\\n",
    "    --max_iters=50_000 \\\n",
    "    --eval_interval=2_000 \\\n",
    "    --eval_iters=50 \\\n",
    "    --log_interval=10 \\\n",
    "    --block_size=4288 \\\n",
    "    --t_text=1152 \\\n",
    "    --t_memmap=3008 \\\n",
    "    --t_audio=3136 \\\n",
    "    --n_layer=48 \\\n",
    "    --n_head=32 \\\n",
    "    --d_head=64 \\\n",
    "    --coarse_n_codebooks=8 \\\n",
    "    --gradient_accumulation_steps=1 \\\n",
    "    --batch_size=2 \\\n",
    "    --fsdp=True \\\n",
    "    --sharding_strategy=\"full_shard\" \\\n",
    "    --wandb_log=True \\\n",
    "    --wandb_project=\"chirp-v2-ft-tony\" \\\n",
    "    --wandb_run_name=\"xxl_ft_genius_4x\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c8c40713",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-30T05:41:32.661143Z",
     "start_time": "2023-11-30T05:41:32.657261Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10609.296875"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 678995 / 2 / 32 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fa5d6ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "OMP_NUM_THREADS=4 torchrun \\\n",
    "    --nnodes=3 --nproc_per_node=8 \\\n",
    "    --rdzv_backend=c10d --rdzv_id=123 --rdzv_endpoint=\"compute-hpc-node-54:1234\" \\\n",
    "    train.py \\\n",
    "    --out_dir=\"/app/suno/checkpoints\" \\\n",
    "    --data_dir=\"/app/suno/data/chirp_v2_finetune\" \\\n",
    "    --train_filename=\"data_tr.bin\" \\\n",
    "    --train_metas_filename=\"metas_tr.jsonl\" \\\n",
    "    --train_info_filename=\"info_tr.json\" \\\n",
    "    --val_filename=\"data_val.bin\" \\\n",
    "    --val_metas_filename=\"metas_val.jsonl\" \\\n",
    "    --val_info_filename=\"info_val.json\" \\\n",
    "    --preload_checkpoint=\"/app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt\" \\\n",
    "    --weights_multiplier=\"genius_hq_lyrics:1;genius_hq_lyrics_foreign:2\" \\\n",
    "    --is_finetune=True \\\n",
    "    --learning_rate=3e-5 \\\n",
    "    --min_lr=1e-6 \\\n",
    "    --warmup_iters=2_000 \\\n",
    "    --max_iters=10_000 \\\n",
    "    --eval_interval=1_000 \\\n",
    "    --eval_iters=50 \\\n",
    "    --log_interval=10 \\\n",
    "    --block_size=4288 \\\n",
    "    --t_text=1152 \\\n",
    "    --t_memmap=3008 \\\n",
    "    --t_audio=3136 \\\n",
    "    --n_layer=48 \\\n",
    "    --n_head=32 \\\n",
    "    --d_head=64 \\\n",
    "    --coarse_n_codebooks=8 \\\n",
    "    --gradient_accumulation_steps=1 \\\n",
    "    --batch_size=2 \\\n",
    "    --fsdp=True \\\n",
    "    --sharding_strategy=\"full_shard\" \\\n",
    "    --wandb_log=True \\\n",
    "    --wandb_project=\"chirp-v2-ft-tony\" \\\n",
    "    --wandb_run_name=\"xxl_ft_genius_3x\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "suno_env",
   "language": "python",
   "name": "suno_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}