#!/bin/bash
# Test script for layer-specific targets
# Tests that continuous_semantic can use intermediate layer outputs

CUDA_VISIBLE_DEVICES=0 /home/victor/anaconda3/envs/suno_env_fa2_fa3_pt29/bin/python -u train.py \
    --debug_val_only=True \
    --master_port=12897 \
    --out_dir="/app2/suno/checkpoints/test_layer_targets" \
    --data_dir="/app2/suno/data/auk_v0" \
    --train_metas_filename="metas_v8_tr_mini.jsonl" \
    --val_metas_filename="metas_v8_val.jsonl" \
    --step_save_iters=2_000 \
    --eval_interval=1_000 \
    --eval_iters=5 \
    --batch_store_size=1 \
    \
    --n_layer=2 \
    --n_head=32 \
    --stem_active_sections_weight=400000 \
    --d_head=128 \
    \
    --learning_rate=5e-5 \
    --max_iters=4 \
    --warmup_iters=1_000 \
    --block_size=32000 \
    --batch_size=1 \
    \
    --fsdp=True \
    --grad_checkpointing=True \
    --compile=False \
    \
    --wandb_log=False \
    --preload_optimizer=False \
    --allow_skip=True \
    \
    --repa_semantic=True \
    --repa_layer=0