{
    "model": {
        "vae_dim": 128,
        "enc_embed_dim": 768,
        "enc_layers": 12,
        "enc_heads": 12,
        "enc_ff": 3072,
        "dec_embed_dim": 512,
        "dec_layers": 8,
        "dec_heads": 16,
        "dec_ff": 2048,
        "dropout": 0.1,
        "max_len": 750,
        "pos_type": "learned",
        "proj_to_dec": true
    },
    "training": {
        "wandb_project": "ear",
        "wandb_name": "ear-pretrain-t8",
        "pretrain": true,
        "batch_size": 32,
        "num_workers": 16,
        "val_every": 100,
        "log_every": 10,
        "ckpt_every": 1000,
        "max_steps": 100000,
        "grad_clip": 1.0,
        "lr": 1e-5,
        "compile": false,
        "seed_offset": 0,
        "preload_checkpoint": null
    },
    "data": {
        "dataset_type": "jsonl",
        "dataset_dir": "/home/christian/code/christian/metadata/reward_model/pretrain",
        "train_metas_filename": "metas_tr_t0.jsonl",
        "val_metas_filename": "metas_val_t0.jsonl",
        "mask_prob": 0.5,
        "vae_scale_factor": 0.4
    }
}