# fine tune 3b scripts, for book keepings # pre-trained 3b at /app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt OMP_NUM_THREADS=8 torchrun \ --nnodes=6 --nproc_per_node=8 \ --rdzv_backend=c10d --rdzv_id=123 --rdzv_endpoint="compute-hpc-node-294:1234" \ train.py \ --out_dir="/app/suno/checkpoints" \ --data_dir="/app/suno/data/chirp_v2_finetune_v13" \ --train_filename="data_tr.bin" \ --train_metas_filename="metas_tr.jsonl" \ --train_info_filename="info_tr.json" \ --val_filename="data_val.bin" \ --val_metas_filename="metas_val.jsonl" \ --val_info_filename="info_val.json" \ --preload_checkpoint="/app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt" \ --weights_multiplier="genius_hq_lyrics:1;genius_hq_lyrics_foreign:1.2;pond5_music:1;" \ --is_finetune=True \ --learning_rate=1e-5 \ --min_lr=1e-6 \ --warmup_iters=8_000 \ --max_iters=40_000 \ --eval_interval=2_500 \ --eval_iters=50 \ --log_interval=20 \ --block_size=4288 \ --t_text=1152 \ --t_memmap=3008 \ --t_audio=3136 \ --n_layer=48 \ --n_head=32 \ --d_head=64 \ --coarse_n_codebooks=8 \ --coarse_vocab_size=4160 \ --coarse_codebook_size=4096 \ --coarse_pad_token=4096 \ --coarse_infer_token=4097 \ --use_sin_pos_emb=False \ --grad_clip=0.1 \ --step_save_iters=5_000 \ --step_save_infer=True \ --gradient_accumulation_steps=2 \ --batch_size=2 \ --fsdp=True \ --sharding_strategy="full_shard" \ --wandb_log=True \ --wandb_project="chirp-v2-ft-tony" \ --wandb_run_name="xxl_ft_genius_v13_6x_v1" # fine tune 3b original corpus size...let's try it OMP_NUM_THREADS=8 torchrun \ --nnodes=6 --nproc_per_node=8 \ --rdzv_backend=c10d --rdzv_id=123 --rdzv_endpoint="compute-hpc-node-294:1234" \ train.py \ --out_dir="/app/suno/checkpoints" \ --data_dir="/app/suno/data/chirp_v2_finetune_v13_full" \ --train_filename="data_tr.bin" \ --train_metas_filename="metas_tr.jsonl" \ --train_info_filename="ft_v1_info_tr.json" \ --val_filename="data_val.bin" \ --val_metas_filename="metas_val.jsonl" \ --val_info_filename="info_val.json" \ --preload_checkpoint="/app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt" \ --weights_multiplier="youtube_music:0.75;youtube_music_lyrics:2;youtube_music_lyrics_foreign:3;genius_hq_lyrics:2;genius_hq_lyrics_foreign:3;freesound:5;imslp:0.5;jamendo:1;pond5_music:0.75;deezer:1;deezer_lyrics:2;deezer_lyrics_foreign:3;ytm_tagged:0.25;musescore_lyrics:5;" \ --is_finetune=True \ --learning_rate=5e-5 \ --min_lr=1e-6 \ --warmup_iters=5_000 \ --max_iters=100_000 \ --eval_interval=2_500 \ --eval_iters=50 \ --log_interval=20 \ --block_size=4288 \ --t_text=1152 \ --t_memmap=3008 \ --t_audio=3136 \ --n_layer=48 \ --n_head=32 \ --d_head=64 \ --coarse_n_codebooks=8 \ --coarse_vocab_size=4160 \ --coarse_codebook_size=4096 \ --coarse_pad_token=4096 \ --coarse_infer_token=4097 \ --use_sin_pos_emb=False \ --grad_clip=0.1 \ --step_save_iters=10_000 \ --step_save_infer=True \ --gradient_accumulation_steps=4 \ --batch_size=2 \ --fsdp=True \ --sharding_strategy="full_shard" \ --wandb_log=True \ --wandb_project="chirp-v2-ft-tony" \ --wandb_run_name="xxl_ft_genius_v13_full_6x_v2" # 4 node for FT OMP_NUM_THREADS=8 torchrun \ --nnodes=4 --nproc_per_node=8 \ --rdzv_backend=c10d --rdzv_id=123 --rdzv_endpoint="compute-hpc-node-54:1234" \ train.py \ --out_dir="/app/suno/checkpoints" \ --data_dir="/app/suno/data/chirp_v2_finetune_v13" \ --train_filename="data_tr.bin" \ --train_metas_filename="metas_tr.jsonl" \ --train_info_filename="ft_v1_info_tr.json" \ --val_filename="data_val.bin" \ --val_metas_filename="metas_val.jsonl" \ --val_info_filename="ft_v1_info_val.json" \ --preload_checkpoint="/app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt" \ --weights_multiplier="genius_hq_lyrics:1;genius_hq_lyrics_foreign:1.1;pond5_music:1;genius_hq_lyrics_top:3.0;genius_hq_lyrics_foreign_top:2.5;genius_hq_lyrics_mid:1.8;genius_hq_lyrics_foreign_mid:1.6;" \ --is_finetune=True \ --learning_rate=1e-5 \ --min_lr=1e-6 \ --warmup_iters=8_000 \ --max_iters=40_000 \ --eval_interval=2_500 \ --eval_iters=50 \ --log_interval=20 \ --block_size=4288 \ --t_text=1152 \ --t_memmap=3008 \ --t_audio=3136 \ --n_layer=48 \ --n_head=32 \ --d_head=64 \ --coarse_n_codebooks=8 \ --coarse_vocab_size=4160 \ --coarse_codebook_size=4096 \ --coarse_pad_token=4096 \ --coarse_infer_token=4097 \ --use_sin_pos_emb=False \ --grad_clip=0.1 \ --step_save_iters=5_000 \ --step_save_infer=True \ --gradient_accumulation_steps=3 \ --batch_size=2 \ --fsdp=True \ --sharding_strategy="full_shard" \ --wandb_log=True \ --wandb_project="chirp-v2-ft-tony" \ --wandb_run_name="xxl_ft_genius_v13_4x_v2" # one node for FT OMP_NUM_THREADS=8 torchrun \ --nnodes=1 --nproc_per_node=8 \ train.py \ --out_dir="/app/suno/checkpoints" \ --data_dir="/app/suno/data/chirp_v2_finetune_v11" \ --train_filename="data_tr.bin" \ --train_metas_filename="metas_tr.jsonl" \ --train_info_filename="info_tr.json" \ --val_filename="data_val.bin" \ --val_metas_filename="metas_val.jsonl" \ --val_info_filename="info_val.json" \ --preload_checkpoint="/app/suno/checkpoints/2023-10-17_16-01-32/last_ckpt.pt" \ --weights_multiplier="genius_hq_lyrics:1;genius_hq_lyrics_foreign:1.25;pond5_music:1;" \ --is_finetune=True \ --learning_rate=3e-5 \ --min_lr=1e-6 \ --warmup_iters=8_000 \ --max_iters=40_000 \ --eval_interval=2_500 \ --eval_iters=50 \ --log_interval=20 \ --block_size=4288 \ --t_text=1152 \ --t_memmap=3008 \ --t_audio=3136 \ --n_layer=48 \ --n_head=32 \ --d_head=64 \ --coarse_n_codebooks=8 \ --coarse_vocab_size=4160 \ --coarse_codebook_size=4096 \ --coarse_pad_token=4096 \ --coarse_infer_token=4097 \ --use_sin_pos_emb=False \ --gradient_accumulation_steps=1 \ --batch_size=2 \ --fsdp=True \ --sharding_strategy="full_shard" \ --wandb_log=True \ --wandb_project="chirp-v2-ft-tony" \ --wandb_run_name="xxl_ft_genius_v11_1x_v1" # ft on one node for just classical OMP_NUM_THREADS=8 torchrun \ --nnodes=1 --nproc_per_node=8 \ train.py \ --out_dir="/app/suno/checkpoints" \ --data_dir="/app/suno/data/chirp_v2_finetune_v7_classical_3" \ --train_filename="data_tr.bin" \ --train_metas_filename="metas_tr.jsonl" \ --train_info_filename="info_tr.json" \ --val_filename="data_val.bin" \ --val_metas_filename="metas_val.jsonl" \ --val_info_filename="info_val.json" \ --preload_checkpoint="/app/suno/checkpoints/2023-12-10_05-20-55/last_ckpt.pt" \ --weights_multiplier="imslp_lyrics_foreign:1;" \ --is_finetune=True \ --learning_rate=3e-5 \ --min_lr=1e-6 \ --warmup_iters=5_000 \ --max_iters=50_000 \ --eval_interval=2_500 \ --eval_iters=50 \ --log_interval=10 \ --t_memmap=3008 \ --t_text=1152 \ --t_audio=3136 \ --block_size=4288 \ --n_layer=48 \ --n_head=32 \ --d_head=64 \ --coarse_n_codebooks=8 \ --coarse_vocab_size=4160 \ --coarse_codebook_size=4096 \ --coarse_pad_token=4096 \ --coarse_infer_token=4097 \ --last_codebook_weight=0.75 \ --use_sin_pos_emb=False \ --gradient_accumulation_steps=2 \ --batch_size=2 \ --fsdp=True \ --sharding_strategy="full_shard" \ --wandb_log=True \ --wandb_project="chirp-v2-ft-tony" \ --wandb_run_name="xxl_ft_genius_v8_classical_v3" # dpo on a small set OMP_NUM_THREADS=4 torchrun \ --nnodes=1 --nproc_per_node=8 \ dpo.py \ --out_dir="/app/suno/checkpoints" \ --data_dir="/app/suno/data/chirp_v2_finetune_v5_extra_filter" \ --train_filename="data_val.bin" \ --train_metas_filename="metas_val.jsonl" \ --train_info_filename="info_val.json" \ --val_filename="data_val.bin" \ --val_metas_filename="metas_val.jsonl" \ --val_info_filename="info_val.json" \ --preload_checkpoint="/app/suno/checkpoints/2023-12-02_23-30-13/last_ckpt.pt" \ --weights_multiplier="genius_hq_lyrics:1;genius_hq_lyrics_foreign:1.2" \ --is_finetune=True \ --learning_rate=3e-5 \ --min_lr=1e-6 \ --warmup_iters=8_000 \ --max_iters=40_000 \ --eval_interval=2_000 \ --eval_iters=50 \ --log_interval=10 \ --block_size=4288 \ --t_text=1152 \ --t_memmap=3008 \ --t_audio=3136 \ --n_layer=48 \ --n_head=32 \ --d_head=64 \ --coarse_n_codebooks=8 \ --coarse_vocab_size=4160 \ --coarse_codebook_size=4096 \ --use_sin_pos_emb=False \ --custom_seed_offset=1 \ --gradient_accumulation_steps=1 \ --batch_size=1 \ --fsdp=True \ --sharding_strategy="full_shard" \ --wandb_log=False \ --wandb_project="chirp-v2-ft-tony" \ --wandb_run_name="xxl_ft_genius_v4_1x_filter" # eval loss # for val OMP_NUM_THREADS=4 torchrun \ --nnodes=1 --nproc_per_node=4 \ eval_loss.py \ --out_dir="/app/suno/data/chirp_v2_finetune_v4_extra_filter" \ --data_dir="/app/suno/data/chirp_v2_finetune_v4_extra_filter" \ --preload_checkpoint="/app/suno/checkpoints/2023-12-02_23-30-13/last_ckpt.pt" \ --block_size=4288 \ --t_text=1152 \ --t_memmap=3008 \ --t_audio=3136 \ --n_layer=48 \ --n_head=32 \ --d_head=64 \ --coarse_n_codebooks=8 \ --coarse_vocab_size=4160 \ --coarse_codebook_size=4096 \ --coarse_pad_token=4096 \ --coarse_infer_token=4097 \ --use_sin_pos_emb=False # for train, we want to use a v7 fine tuned on everything, so it is in distribution # batch size 24 on A100 using 39 G now... don't increase any more OMP_NUM_THREADS=8 torchrun \ --nnodes=6 --nproc_per_node=8 \ --rdzv_backend=c10d --rdzv_id=123 --rdzv_endpoint="compute-hpc-node-294:1234" \ eval_loss.py \ --out_dir="/app/suno/data/chirp_v2_finetune_v13_full" \ --data_dir="/app/suno/data/chirp_v2_finetune_v13_full" \ --preload_checkpoint="/app/suno/checkpoints/2023-12-17_19-09-54/last_ckpt.pt" \ --val_filename="data_tr.bin" \ --val_metas_filename="metas_tr.jsonl" \ --val_info_filename="info_tr.json" \ --batch_size=24 \ --block_size=4288 \ --t_text=1152 \ --t_memmap=3008 \ --t_audio=3136 \ --n_layer=48 \ --n_head=32 \ --d_head=64 \ --coarse_n_codebooks=8 \ --coarse_vocab_size=4160 \ --coarse_codebook_size=4096 \ --coarse_pad_token=4096 \ --coarse_infer_token=4097 \ --use_sin_pos_emb=False