| { | |
| "num_heads": 16, | |
| "encoder_dim": 256, | |
| "ffn_ratio": 4, | |
| "prefix_layers": 4, | |
| "n_value_classes": 3, | |
| "dropout": 0.05, | |
| "gradient_checkpointing": false, | |
| "use_qk_norm": true, | |
| "use_swiglu": true, | |
| "recurrent_layers": 7, | |
| "hidden_dim": 1216, | |
| "recurrent_unrolls": 3, | |
| "zero_init_out_proj": true, | |
| "policy_head_dim": 384, | |
| "suffix_layers": 4, | |
| "value_hidden": 384 | |
| } | |