MLA_config.json

{
    "d_model": 2048,
    "rms_norm_eps": 1e-05,
    "vocab_size": null,
    "d_inner": 2048,
    "d_xb": 512,
    "intermediate_size": 8192,
    "hidden_act": "silu",
    "n_layer": 16,
    "attn_layers": []
}