{ "alt_layer_every": -1, "alt_rope_partial_dim": 64, "alt_rope_theta": 10000.0, "alt_share_v0": 0, "alt_token_mixer": "swa", "alt_token_mixer_class_path": null, "architectures": [ "RWKV7BackboneForCausalLM" ], "auto_map": { "AutoConfig": "rwkv7_backbone.MixerConfig", "AutoModelForCausalLM": "rwkv7_backbone.RWKV7BackboneForCausalLM" }, "bos_token_id": null, "chunk_len": 256, "hidden_size": 1024, "d_head": null, "d_qk_head": null, "d_v_head": null, "dtype": "bfloat16", "eos_token_id": null, "ffn_expansion": 4.0, "gated_deltanet_conv_bias": 0, "gated_deltanet_conv_size": 4, "gated_deltanet_expand_k": 0.75, "gated_deltanet_expand_v": 1.5, "gated_deltanet_num_kv_heads": null, "gated_deltanet_token_shift_mode": "none", "gated_deltanet_use_mamba_gate": 1, "gated_deltanet_value_residual_mode": "none", "gptalpha_token_shift_mode": "rwkv", "gptalpha_value_residual_mode": "rwkv", "kda_allow_neg_eigval": 0, "kda_conv_bias": 0, "kda_conv_size": 4, "kda_expand_k": 1.0, "kda_expand_v": 1.0, "kda_token_shift_mode": "none", "kda_value_residual_mode": "none", "kvm_apply_merge_gate_to_appends": 1, "kvm_apply_merge_gate_to_initial_state": 0, "kvm_token_shift_mode": "rwkv", "kvm_use_head_temps": 1, "kvm_use_merge_gate_keys": 1, "kvm_use_merge_gate_values": 1, "kvm_use_vlens": 1, "kvm_value_residual_mode": "rwkv", "lact_actual_rope_partial_dim": -1, "lact_attn_qk_norm": 0, "lact_chunk_size": -1, "lact_factor": 1.0, "lact_fw_init_gain": 0.5, "lact_inter_multi": 1.0, "lact_lact_window_size": -1, "lact_learnable_muon_lr": 0, "lact_muon_lr_reduce_exp": 2, "lact_no_v_silu": 0, "lact_norm_eps": 1e-06, "lact_num_fw_heads": -1, "lact_token_shift_mode": "rwkv", "lact_ttt_lag": 0, "lact_ttt_nope": 0, "lact_ttt_prenorm": 0, "lact_use_bswa": 0, "lact_use_momentum": 1, "lact_use_muon": 0, "lact_value_residual_mode": "rwkv", "lact_w0_w2_low_rank": 32, "logit_softcap": 0.0, "max_position_embeddings": 1073741824, "model_class_path": "model.rwkv7_backbone.RWKV7BackboneForCausalLM", "model_type": "rwkv7backbone", "mup_base_dim": 768, "n_bswa_chunks": 2, "num_attention_heads": 8, "n_layer": null, "n_max_d_chunks": 1, "num_hidden_layers": 24, "ovq_chunk_size": 128, "ovq_max_centroids": 2048, "ovq_normalize_centroids": 0, "ovq_token_shift_mode": "rwkv", "ovq_use_counts_for_lr_only": 0, "ovq_use_delta_rule": 0, "ovq_use_min_state_size": 0, "ovq_value_residual_mode": "rwkv", "ovq_window_size": 128, "pad_token_id": null, "param_dtype": "bfloat16", "residual_scale": 0.5, "rope_partial_dim": 64, "rope_theta": 10000.0, "sink_len": 1, "state_budget_mode": "fixed", "state_growth_exponent": 0.5, "state_growth_factor": 1.0, "state_min_len": -1, "state_round_down": 1, "state_saturation_n": null, "swa_token_shift_mode": "rwkv", "swa_value_residual_mode": "rwkv", "swa_window_size": -1, "tie_word_embeddings": false, "token_mixer": "gptalpha", "token_mixer_class_path": null, "transformers_version": "5.3.0", "use_block_lambdas": 0, "use_l2wrap": 0, "use_overflow_v_key_weighting": 1, "use_skip_connections": 0, "use_state_decay": 0, "use_state_deltarule": 0, "use_tokenshift_att": 1, "use_tokenshift_ffn": 1, "use_value_residual": 1, "vocab_size": 50304 }