{
    "training": {
        "wandb_name": "v2_infill_v1_syn_corr_5E6_beta100_n16_bt2_3k_acc4",
        "wandb_project": "diff_dpo",
        "model_type": "prefix",
        "tot_num_steps": 3000,
        "ckpt_every": 6000,
        "batch_size": 2,
        "learning_rate": 5e-06,
        "lr_warmup": 250,
        "lr_scheduler": "constant",
        "weight_decay": 0.001,
        "clip_grad_norm": 0.5,
        "ipo_beta": 100.0,
        "betas": [
            0.9,
            0.999
        ],
        "seed_offset": 42,
        "compute_metrics": false,
        "use_ema": false,
        "activation_checkpointing": false,
        "compile": true,
        "preload_checkpoint": "/app/suno/data/dpo/models/diff_v2_2b_2mil_ft_infill_20250421_v2.pt",
        "preload_optimizer": false,
        "revert_label": false,
        "semantic_chunk_frac": 0.0,
        "t_discretize": 0,
        "ave_latent_frac": 0.0,
        "ctx_vae_noise": 1.0,
        "ctx_vae_noise_frac": 1.0,
        "ctx_vae_mask_frac": 0.0,
        "grad_accum_steps" : 4,
        "sft_loss_weight": 0.0
    },
    "model": {
        "io_hz": 25,
        "io_channels": 128,
        "embed_dim": 2048,
        "depth": 32,
        "n_heads": 32,
        "qk_norm": true,
        "block_size": 750,
        "cond_semantic_n_vocab": 4001,
        "cond_semantic_len": 750,
        "cond_text_n_vocab": 60001,
        "cond_text_len": 1536,
        "ctx_len": 750,
        "infill_ctx_len": 0,
        "shared_ctx": true
    },
    "data": {
        "mode": "dpo",
        "shard_data": false,
        "shard_data_dir": "/mnt/localdisk/tmp",
        "allow_shard_reuse": false,
        "dataset_type": "general_memmap",
        "dataset_dir": "/app/suno/data/diff_syn_dpo/genius_t6_sampled_10k_corrupted",
        "train_vae_memmap_filename": "data_vae_tr.bin",
        "train_semantic_memmap_filename": "data_semantic_tr.bin",
        "train_metas_filename": "metas_tr.jsonl",
        "val_vae_memmap_filename": "data_vae_val.bin",
        "val_semantic_memmap_filename": "data_semantic_val.bin",
        "val_metas_filename": "metas_val.jsonl",
        "vae_dim": 128,
        "semantic_rate_hz": 25,
        "vae_scale_factor": 0.4,
        "codec_dir": "/app/suno/models/codecs",
        "scale_vae_ctx" : true,
        "noise_ctx" : 0.0,
        "prev_vae_ctx": true,
        "infill_vae_ctx": false
    }
}