| { |
| "hidden_size": 896, |
| "num_hidden_layers": 16, |
| "intermediate_size": 256, |
| "vocab_size": 32000, |
| "num_attention_heads": 14, |
| "num_key_value_heads": 2, |
| "attention_type": "gqa", |
| "attention_dropout": 0.0, |
| "use_qk_norm": true, |
| "sliding_window": null, |
| "causal_conv_kernel_size": 4, |
| "causal_conv_dilation_cycle": 8, |
| "causal_state_rank": 16, |
| "causal_context_gate_init": 1.0, |
| "causal_contextual_mix_init": 0.0, |
| "causal_context_fusion_size": 0, |
| "causal_stable_delta": false, |
| "causal_delta_chunk_size": 512, |
| "causal_delta_timescales": 1, |
| "causal_delta_collision_normalized": false, |
| "causal_delta_lexical_values": false, |
| "causal_delta_lexical_forge": false, |
| "causal_delta_occurrence_address": false, |
| "tr_mha_num_experts": 4, |
| "tr_mha_adapter_rank": 8, |
| "tr_mha_top_k": 2, |
| "tr_mha_adapter_gate_init": 0.1, |
| "tr_mha_id_primary_logit": 2.0, |
| "tr_mha_id_secondary_logit": 1.0, |
| "tr_mha_id_other_logit": -2.0, |
| "tr_mha_verifier_gate_init": 0.1, |
| "tr_mha_verifier_temperature": 1.0, |
| "tr_mha_targets": "qv", |
| "max_position_embeddings": 2048, |
| "rope_theta": 10000.0, |
| "rope_type": "standard", |
| "rope_fraction": 1.0, |
| "mlp_type": "tr_hash_engine", |
| "hidden_act": "silu", |
| "num_experts": 4, |
| "expert_initialization": "gpt_normal", |
| "token_frequencies": null, |
| "routing_strategy": "token_id_multi_hash", |
| "route_hash_count": 2, |
| "lsh_routing": false, |
| "lsh_bits": 0, |
| "lsh_from_layer": 0, |
| "lsh_threshold_mode": "zero", |
| "shared_expert": true, |
| "shared_intermediate_size": 3072, |
| "shared_expert_chunk_tokens": 0, |
| "use_shared_routed_gates": false, |
| "shared_gate_init": 1.0, |
| "routed_gate_init": 1.0, |
| "shared_output_scale": 1.0, |
| "routed_output_scale": 2.0, |
| "shared_output_scale_first_layer": null, |
| "shared_output_scale_last_layer": null, |
| "routed_output_scale_first_layer": null, |
| "routed_output_scale_last_layer": null, |
| "top_k_primary_weight": 0.5, |
| "learn_hash_pair_gates": false, |
| "hash_pair_gate_init": 0.5, |
| "learn_hash_channel_modulation": false, |
| "hash_channel_scale_init": 0.0, |
| "static_expert_capacity": false, |
| "use_custom_kernels": "auto", |
| "collect_moe_telemetry": false, |
| "use_cggr": "auto", |
| "active_num_experts": null, |
| "active_expert_width": null, |
| "lexical_object_rank": 16, |
| "lexical_object_gate_init": 0.1, |
| "tie_lexical_object_embeddings": false, |
| "micro_num_experts": 4, |
| "micro_expert_width": 16, |
| "micro_expert_gate_init": 0.1, |
| "lexical_gqa_rank": 16, |
| "lexical_gqa_gate_init": 0.0, |
| "lexical_gqa_use_token_code": true, |
| "lexical_key_gate_init": 0.05, |
| "lexical_zipf_path": null, |
| "lexical_zipf_mode": "uniform", |
| "lexical_zipf_alpha": 0.25, |
| "lexical_zipf_floor": 0.1, |
| "lexical_zipf_permutation_seed": 1729, |
| "norm_type": "rmsnorm", |
| "norm_eps": 1e-06, |
| "tie_word_embeddings": true, |
| "use_sdpa": true, |
| "use_cache": true, |
| "is_causal": true, |
| "initializer_range": 0.02, |
| "use_mup_init": false, |
| "use_mup_attn_scale": false, |
| "use_mup_output_mult": false, |
| "mup_base_width": 256, |
| "extra_config": {}, |
| "model_type": "tr_hash_moe", |
| "architectures": [ |
| "TRHashForCausalLM" |
| ], |
| "auto_map": { |
| "AutoConfig": "configuration_tr_hash_moe.TRHashConfig", |
| "AutoModelForCausalLM": "modeling_tr_hash_moe.TRHashForCausalLM" |
| }, |
| "num_experts_per_tok": 2 |
| } |
|
|
