pub struct ModelConfig {Show 149 fields
pub hidden_size: usize,
pub num_hidden_layers: usize,
pub intermediate_size: usize,
pub vocab_size: usize,
pub num_attention_heads: usize,
pub num_attention_heads_per_layer: Vec<usize>,
pub num_key_value_heads: usize,
pub head_dim: usize,
pub partial_rotary_factor: f64,
pub linear_num_key_heads: usize,
pub linear_key_head_dim: usize,
pub linear_num_value_heads: usize,
pub linear_value_head_dim: usize,
pub linear_conv_kernel_dim: usize,
pub num_experts: usize,
pub zero_expert_num: usize,
pub num_experts_per_tok: usize,
pub moe_intermediate_size: usize,
pub shared_expert_intermediate_size: usize,
pub norm_topk_prob: bool,
pub decoder_sparse_step: usize,
pub layer_types: Vec<LayerType>,
pub mtp_layer_types: Vec<LayerType>,
pub full_attention_interval: usize,
pub sliding_window: u32,
pub max_position_embeddings: usize,
pub rope_theta: f64,
pub rms_norm_eps: f64,
pub bos_token_id: u32,
pub glm5next_router_mode: Glm5NextRouterMode,
pub eos_token_id: u32,
pub eos_token_ids: Vec<u32>,
pub tie_word_embeddings: bool,
pub lm_head_bf16_override: Option<bool>,
pub lm_head_fp8: bool,
pub model_type: String,
pub mtp_num_hidden_layers: usize,
pub dspark_block_size: usize,
pub dspark_noise_token_id: u32,
pub dspark_target_layer_ids: Vec<usize>,
pub dspark_markov_rank: usize,
pub hybrid_override_pattern: String,
pub mamba_num_heads: usize,
pub mamba_head_dim: usize,
pub ssm_state_size: usize,
pub n_groups: usize,
pub expand: usize,
pub n_routed_experts: usize,
pub norm_eps: f64,
pub conv_kernel: usize,
pub moe_shared_expert_intermediate_size: usize,
pub routed_scaling_factor: f64,
pub linear_gate_lower_bound: f32,
pub swiglu_limit: f32,
pub mlp_only_layers: Vec<usize>,
pub moe_latent_size: usize,
pub moe_intermediate_sizes: Vec<usize>,
pub num_experts_per_toks: Vec<usize>,
pub kv_lora_rank: usize,
pub kv_layer_dims: Vec<(usize, usize)>,
pub q_lora_rank: usize,
pub qk_nope_head_dim: usize,
pub qk_rope_head_dim: usize,
pub v_head_dim: usize,
pub ngram_vocab_size_ratio: usize,
pub emb_neighbor_num: usize,
pub emb_split_num: usize,
pub ngram_vocab_size_base: usize,
pub ngram_split_parts: usize,
pub ple_layer_ids: Vec<usize>,
pub ple_conv_kernel_size: usize,
pub o_lora_rank: usize,
pub o_groups: usize,
pub yarn_mscale: f32,
pub yarn_mscale_all_dim: f32,
pub hc_mult: usize,
pub hc_sinkhorn_iters: usize,
pub hc_eps: f32,
pub hc_lowrank: usize,
pub final_norm_identity: bool,
pub compress_ratios: Vec<usize>,
pub index_n_heads: usize,
pub index_head_dim: usize,
pub index_topk: usize,
pub index_compress_ratio: usize,
pub index_kpool: usize,
pub index_kpool_always_select_tail: bool,
pub num_hash_layers: usize,
pub compress_rope_theta: f32,
pub kv_source_layer_ids: Vec<usize>,
pub index_source_layer_ids: Vec<usize>,
pub candidate_source_layer_id: Option<usize>,
pub candidate_topk_blocks: usize,
pub candidate_block_size: usize,
pub engram_layer_ids: Vec<usize>,
pub engram_num_embeddings: Vec<u64>,
pub engram_max_ngram_size: usize,
pub engram_vocab_size: usize,
pub engram_n_heads: usize,
pub engram_head_dim: usize,
pub engram_pad_token_id: u32,
pub engram_compressed_vocab_size: usize,
pub engram_multipliers: Vec<u64>,
pub engram_primes: Vec<u64>,
pub engram_offsets: Vec<u64>,
pub yarn_factor: f32,
pub yarn_beta_slow: f32,
pub yarn_beta_fast: f32,
pub yarn_original_max_position_embeddings: usize,
pub yarn_attention_factor: f32,
pub llama_4_scaling_beta: f32,
pub llama_4_scaling_original_max_position_embeddings: usize,
pub vision: Option<VisionConfig>,
pub quantization_config: Option<QuantizationConfig>,
pub attn_gated: bool,
pub gdn_norm_sigmoid: bool,
pub nested_config: bool,
pub mrope_section: [usize; 3],
pub mrope_interleaved: bool,
pub weight_prefix: String,
pub profile: bool,
pub ep_rank: usize,
pub ep_world_size: usize,
pub tp_rank: usize,
pub tp_world_size: usize,
pub serve_max_seq_len: usize,
pub fp8_kv_calibration_tokens: usize,
pub fp8_kv_headroom: f32,
pub final_logit_softcapping: f32,
pub embed_scale: f32,
pub scoring_func: String,
pub use_routing_bias: bool,
pub qk_norm_type: String,
pub num_mtp_modules: usize,
pub mtp_transformer_layers: usize,
pub rotary_dim: usize,
pub attn_position_encoding: Option<AttnPositionEncoding>,
pub dflash_capture_layers: Vec<usize>,
pub dflash_gamma: Option<usize>,
pub adapter_max_rank: usize,
pub attn_res_block_size: usize,
pub use_full_rank_gate: bool,
pub mla_use_nope: bool,
pub mla_use_output_gate: bool,
pub latent_moe_use_norm: bool,
pub hidden_act: String,
pub activation_situ_beta: f32,
pub activation_situ_linear_beta: f32,
pub n_shared_experts: usize,
}Fields§
§intermediate_size: usize§vocab_size: usize§num_attention_heads: usize§num_attention_heads_per_layer: Vec<usize>2026-09-26: Per-layer Q-head counts (the laguna parser fills it). When non-empty,
validate_config requires one entry per layer.
num_key_value_heads: usize§head_dim: usize§partial_rotary_factor: f642026-09-26: Fraction of head_dim that rotary_dim() rotates when rotary_dim is 0;
1.0 when absent.
linear_num_key_heads: usize§linear_key_head_dim: usize§linear_num_value_heads: usize§linear_value_head_dim: usize§linear_conv_kernel_dim: usize§num_experts: usize§zero_expert_num: usize§num_experts_per_tok: usize§moe_intermediate_size: usize§norm_topk_prob: bool§decoder_sparse_step: usize§layer_types: Vec<LayerType>2026-09-26: Per-layer kind; when empty, layer_type() falls back to
full_attention_interval.
mtp_layer_types: Vec<LayerType>2026-09-26: Per-layer kind of the MTP / NextN layers past num_hidden_layers; empty
when there are none. They stay out of layer_types, whose length validate_config
checks against num_hidden_layers; ModelConfig::layer_type_at reads both.
full_attention_interval: usize2026-09-26: With layer_types empty, every full_attention_interval-th layer is
FullAttention and the rest LinearAttention.
sliding_window: u322026-09-26: 0 when absent or JSON null.
max_position_embeddings: usize§rope_theta: f64§rms_norm_eps: f64§bos_token_id: u322026-09-26: JSON null gives 0.
glm5next_router_mode: Glm5NextRouterMode2026-09-26: See Glm5NextRouterMode; HfFp32 when absent.
eos_token_id: u322026-09-26: The primary stop token; ModelConfig::eos_ids gives the full set.
eos_token_ids: Vec<u32>2026-09-26: Every declared stop id, primary first. parse_config fills it for every
family. Empty means not populated (a hand-built ModelConfig), not “no stop tokens”,
so read it through ModelConfig::eos_ids.
tie_word_embeddings: bool§lm_head_bf16_override: Option<bool>2026-09-26: Set at serve startup from --lm-head-dtype.
Some(true) forces a BF16 LM head, Some(false) a quantized one, and None leaves
the choice to skip_lm_head_quantization()’s model rules.
lm_head_fp8: bool2026-09-26: Set by --lm-head-dtype fp8. When skip_lm_head_quantization() is false
the LM head is FP8 (the checkpoint’s own FP8 head, else a quantized copy) instead of
NVFP4; a pre-packed NVFP4 head is used whatever this says.
model_type: String§dspark_block_size: usize§dspark_noise_token_id: u32§dspark_target_layer_ids: Vec<usize>§dspark_markov_rank: usize§hybrid_override_pattern: String§mamba_num_heads: usize§mamba_head_dim: usize§ssm_state_size: usize§n_groups: usize§expand: usize§n_routed_experts: usize2026-09-26: Nemotron-H name; the nemotron_h arm of parse_config copies it into
num_experts when that is 0.
norm_eps: f642026-09-26: Nemotron-H name; copied into rms_norm_eps when that is still the default.
conv_kernel: usize2026-09-26: Nemotron-H name; copied into linear_conv_kernel_dim when that is still the
default.
2026-09-26: Nemotron-H name; copied into shared_expert_intermediate_size when that is 0.
routed_scaling_factor: f64§linear_gate_lower_bound: f322026-09-26: KDA forget-gate lower bound (linear_attn_config.gate_lower_bound). The
glm5_next parser refuses a config without it; for Kimi K3, 0.0 (absent) makes the host
KDA path run unbounded.
swiglu_limit: f322026-09-26: SwiGLU clamp bound (swiglu_limit); 0.0 means no clamp. The glm5_next
parser refuses a config without a positive one.
mlp_only_layers: Vec<usize>§moe_latent_size: usize§moe_intermediate_sizes: Vec<usize>2026-09-26: Per-layer routed-expert intermediate sizes, set by the Puzzle parser (one
entry per layer, 0 for non-MoE layers) and never read from JSON. Empty means the scalar
moe_intermediate_size for every layer.
num_experts_per_toks: Vec<usize>2026-09-26: Per-layer top-k expert counts, same layout as moe_intermediate_sizes.
Empty means the scalar num_experts_per_tok.
kv_lora_rank: usize2026-09-26: 0 means standard attention; non-zero makes capabilities() report MLA.
kv_layer_dims: Vec<(usize, usize)>2026-09-26: Per-layer KV cache (num_kv_heads, head_dim), set by the model factory from
the weight loader; never read from JSON.
q_lora_rank: usize§qk_nope_head_dim: usize§qk_rope_head_dim: usize§v_head_dim: usize§ngram_vocab_size_ratio: usize§emb_neighbor_num: usize§emb_split_num: usize§ngram_vocab_size_base: usize§ngram_split_parts: usize§ple_layer_ids: Vec<usize>§ple_conv_kernel_size: usize§o_lora_rank: usize§o_groups: usize§yarn_mscale: f32§yarn_mscale_all_dim: f32§hc_mult: usize§hc_sinkhorn_iters: usize§hc_eps: f32§hc_lowrank: usize§final_norm_identity: bool§compress_ratios: Vec<usize>2026-09-26: Per-layer attention compression ratios; 0 means uncompressed. The GGUF
builder accepts more entries than num_hidden_layers, never fewer.
index_n_heads: usize§index_head_dim: usize§index_topk: usize§index_compress_ratio: usize2026-09-26: The QSA indexer’s compression ratio. The qwen4_exp parser sets it and
leaves compress_ratios empty; 0 means no indexer.
index_kpool: usize§index_kpool_always_select_tail: bool§num_hash_layers: usize§compress_rope_theta: f322026-09-26: The compressor’s RoPE base, separate from rope_theta; the GGUF builder
reads attention.compress_rope_freq_base.
kv_source_layer_ids: Vec<usize>§index_source_layer_ids: Vec<usize>§candidate_source_layer_id: Option<usize>§candidate_topk_blocks: usize§candidate_block_size: usize§engram_layer_ids: Vec<usize>§engram_num_embeddings: Vec<u64>§engram_max_ngram_size: usize§engram_vocab_size: usize§engram_n_heads: usize§engram_head_dim: usize§engram_pad_token_id: u32§engram_compressed_vocab_size: usize§engram_multipliers: Vec<u64>§engram_primes: Vec<u64>§engram_offsets: Vec<u64>§yarn_factor: f32§yarn_beta_slow: f32§yarn_beta_fast: f32§yarn_original_max_position_embeddings: usize§yarn_attention_factor: f32§llama_4_scaling_beta: f32§llama_4_scaling_original_max_position_embeddings: usize§vision: Option<VisionConfig>2026-09-26: None for text-only models, and for GLM-5.3 unless glm_vision_enabled().
quantization_config: Option<QuantizationConfig>2026-09-26: Set by finalize_config from quantization_config. When that leaves it
None, serve fills it from a sibling hf_quant_config.json
(merge_sidecar_quant_config).
attn_gated: bool2026-09-26: Whether the Q projection carries an output gate. The parsers set it: false
for qwen3_vl_moe, Nemotron-H, and the GGUF llama/qwen2 mapping.
gdn_norm_sigmoid: bool2026-09-26: The GDN gated norm uses a sigmoid gate instead of SiLU; the qwen4_exp
parser sets it for output_gate_type: "sigmoid".
nested_config: bool2026-09-26: The text model’s fields sit under a nested key such as text_config.
Weight-prefix detection reads it when weight_prefix is empty.
mrope_section: [usize; 3]2026-09-26: MRoPE section sizes; [0, 0, 0] when the config declares none.
mrope_interleaved: bool§weight_prefix: String§profile: bool2026-09-26: Set from serve’s --profile.
ep_rank: usize§ep_world_size: usize§tp_rank: usize§tp_world_size: usize§serve_max_seq_len: usize2026-09-26: The serve’s --max-seq-len, set at startup. It is 0 when nothing set it (a
unit test, an offline tool), which readers must treat as unknown, not as zero context.
fp8_kv_calibration_tokens: usize§fp8_kv_headroom: f32§final_logit_softcapping: f32§embed_scale: f32§scoring_func: String§use_routing_bias: bool§qk_norm_type: String§num_mtp_modules: usize§mtp_transformer_layers: usize§rotary_dim: usize2026-09-26: When non-zero, rotary_dim() returns it instead of
partial_rotary_factor * head_dim.
attn_position_encoding: Option<AttnPositionEncoding>2026-09-29: Full attention’s positional encoding. Never read from JSON: a family parser
whose attention has none declares it, and resolve_attn_position_encoding settles
every other config to RoPE. Read it through ModelConfig::attn_position_encoding.
dflash_capture_layers: Vec<usize>§dflash_gamma: Option<usize>2026-09-26: The DFlash drafter γ, set by the model factory with dflash_capture_layers;
None when DFlash is inactive.
adapter_max_rank: usize2026-09-26: --max-lora-rank, set by the model factory. BufferSizes sizes the LoRA
scratch from it and allocates none when it is 0.
attn_res_block_size: usize2026-09-26: attn_res_block_size; the kimi_k3 parser requires it.
use_full_rank_gate: bool2026-09-26: linear_attn_config.use_full_rank_gate.
mla_use_nope: bool§mla_use_output_gate: bool§latent_moe_use_norm: bool2026-09-26: HF hidden_act; empty when absent.
activation_situ_beta: f32§activation_situ_linear_beta: f322026-09-26: num_shared_experts, else n_shared_experts (kimi_k3 parser).
Implementations§
Source§impl ModelConfig
impl ModelConfig
pub fn qwen3_next_80b_nvfp4() -> Self
Source§impl ModelConfig
impl ModelConfig
Sourcepub fn kv_only_prefix_cache_is_safe(&self) -> bool
pub fn kv_only_prefix_cache_is_safe(&self) -> bool
2026-09-26: Whether the radix prefix cache captures every state needed to resume this
model exactly. preflight_reserve and build_prefix_cache both read it.
METRALE_GLM53_PREFIX_CACHE_UNPROVEN (any value, 0 included) also opens it for
glm5_next. kv_only_swap_out_is_safe ignores that switch: the swap image cannot
carry the DSA rows (see there).
Sourcepub fn kv_only_swap_out_is_safe(&self) -> bool
pub fn kv_only_swap_out_is_safe(&self) -> bool
2026-09-26: Whether a sequence may be swapped out to the --swap-space-gb pool and
restored from it; resolve_swap_space_gb reads it. save_sequence_state_dispatch
writes the KV blocks and the SsmLayerState of each LayerType::LinearAttention
layer and nothing else, so any other per-sequence state would not survive the swap.
Source§impl ModelConfig
impl ModelConfig
Sourcepub fn eos_ids(&self) -> Vec<u32>
pub fn eos_ids(&self) -> Vec<u32>
2026-09-26: Every configured stop-token id, primary first; [eos_token_id] when
eos_token_ids is empty.
pub fn is_eos(&self, id: u32) -> bool
Sourcepub fn gqa_ratio(&self) -> usize
pub fn gqa_ratio(&self) -> usize
2026-09-26: Q heads per KV head; 1 when num_key_value_heads is 0.
Sourcepub fn layer_type_at(&self, layer_idx: usize) -> Option<LayerType>
pub fn layer_type_at(&self, layer_idx: usize) -> Option<LayerType>
2026-09-26: Layer kind for any index in the checkpoint. Text-stack indices go through
layer_type; indices >= num_hidden_layers are MTP/NextN layers and resolve through
mtp_layer_types, and past both the answer is None.
Sourcepub fn sparse_attention_layers(&self) -> Vec<usize>
pub fn sparse_attention_layers(&self) -> Vec<usize>
2026-09-26: Text-stack layers whose mixer is LayerType::SparseAttention.
Sourcepub fn has_sparse_attention(&self) -> bool
pub fn has_sparse_attention(&self) -> bool
2026-09-26: True when any layer, in the text stack or the MTP layers, is sparse attention.
Sourcepub fn layer_type(&self, layer_idx: usize) -> LayerType
pub fn layer_type(&self, layer_idx: usize) -> LayerType
2026-09-26: Layer kind of a text-stack index: layer_types[layer_idx] (FullAttention
past its end), or, when layer_types is empty, FullAttention on every
full_attention_interval-th layer and LinearAttention elsewhere.
Sourcepub fn num_attention_layers(&self) -> usize
pub fn num_attention_layers(&self) -> usize
2026-09-26: Number of layers that read the paged KV cache: every layer for which
LayerType::is_attention holds (full, sliding and sparse). Without layer_types,
num_hidden_layers / full_attention_interval, or every layer when the interval is 0.
pub fn num_ssm_layers(&self) -> usize
Sourcepub fn has_recurrent_state(&self) -> bool
pub fn has_recurrent_state(&self) -> bool
2026-09-26: Whether the model carries recurrent (SSM / linear-attention) state:
num_ssm_layers() > 0. An SSM snapshot tier requested for a model without it is a
startup error (ensure_ssm_tier_capability_from).
pub fn has_experts(&self) -> bool
Sourcepub fn rotary_dim(&self) -> usize
pub fn rotary_dim(&self) -> usize
2026-09-29: Dims of Q/K that RoPE rotates; 0 when attention has no positional
encoding, so a RoPE launch that skipped the attn_position_encoding check trips the
rotary_dim > 0 assert in ops::rope instead of rotating silently.
pub fn ssm_qkvz_size(&self) -> usize
pub fn ssm_qkv_size(&self) -> usize
pub fn ssm_z_size(&self) -> usize
pub fn ssm_ba_size(&self) -> usize
Sourcepub fn local_expert_range(&self) -> (usize, usize)
pub fn local_expert_range(&self) -> (usize, usize)
2026-09-26: (start, end), end exclusive, of the experts this EP rank owns; the last
rank also takes the remainder of num_experts / ep_world_size.
pub fn is_local_expert(&self, expert_id: usize) -> bool
Sourcepub fn tp_shard_range(&self, total: usize) -> (usize, usize)
pub fn tp_shard_range(&self, total: usize) -> (usize, usize)
2026-09-26: Range [start, end) of a total-sized dimension owned by this TP rank.
total must be divisible by tp_world_size (debug-asserted). (0, total) when
tp_world_size <= 1.
pub fn tp_shard_dim(&self, total: usize) -> usize
Sourcepub fn layer_prefix(&self, layer_idx: usize) -> String
pub fn layer_prefix(&self, layer_idx: usize) -> String
2026-09-26: {weight_prefix}.layers.{layer_idx}, with model when weight_prefix
is empty.
pub fn capabilities(&self) -> ModelCapabilities
pub fn is_qwen35(&self) -> bool
pub fn is_qwen35_dense(&self) -> bool
pub fn is_qwen3_vl(&self) -> bool
Sourcepub fn skip_lm_head_quantization(&self) -> bool
pub fn skip_lm_head_quantization(&self) -> bool
2026-09-26: Whether to keep the LM head in BF16 instead of quantizing it to NVFP4 at
load. lm_head_bf16_override (set by serve’s --lm-head-dtype) wins when set;
otherwise yes for MLA models (kv_lora_rank > 0), for laguna, and for dense
gemma4 unless METRALE_GEMMA4_LMHEAD_NVFP4=1, and no for the rest.
pub fn mamba2_d_inner(&self) -> usize
pub fn mamba2_d_xbc(&self) -> usize
pub fn mamba2_in_proj_size(&self) -> usize
Sourcepub fn has_mamba2_layers(&self) -> bool
pub fn has_mamba2_layers(&self) -> bool
2026-09-29: Whether the model has Mamba-2 SSM layers (Nemotron-H).
pub fn ssm_h_state_bytes(&self) -> usize
pub fn ssm_conv_state_bytes(&self) -> usize
pub fn ssm_state_norm_dims(&self) -> (usize, usize, usize)
pub fn moe_input_size(&self) -> usize
Sourcepub fn moe_intermediate_size_for(&self, layer: usize) -> usize
pub fn moe_intermediate_size_for(&self, layer: usize) -> usize
2026-09-26: Routed-expert intermediate size for layer: its entry in
moe_intermediate_sizes when present and non-zero (Puzzle schedules), else the scalar.
Sourcepub fn num_experts_per_tok_for(&self, layer: usize) -> usize
pub fn num_experts_per_tok_for(&self, layer: usize) -> usize
2026-09-26: Top-k experts per token for layer, by the same rule over
num_experts_per_toks.
pub fn max_moe_intermediate_size(&self) -> usize
pub fn num_moe_layers(&self) -> usize
Source§impl ModelConfig
impl ModelConfig
Sourcepub fn attn_position_encoding(&self) -> Result<AttnPositionEncoding>
pub fn attn_position_encoding(&self) -> Result<AttnPositionEncoding>
2026-09-29: The resolved positional encoding of full attention. An error for a config
that never went through parse_config / config_from_gguf and was not built by a
factory that sets it.
Trait Implementations§
Source§impl Clone for ModelConfig
impl Clone for ModelConfig
Source§fn clone(&self) -> ModelConfig
fn clone(&self) -> ModelConfig
1.0.0 · Source§fn clone_from(&mut self, source: &Self)
fn clone_from(&mut self, source: &Self)
source. Read more