kvmpaper_gptalpha_350M / config.json
SmerkyG's picture
Upload folder using huggingface_hub
cfa2b34 verified
Raw History Blame Contribute Delete
3.39 kB
{
"alt_layer_every": -1,
"alt_rope_partial_dim": 64,
"alt_rope_theta": 10000.0,
"alt_share_v0": 0,
"alt_token_mixer": "swa",
"alt_token_mixer_class_path": null,
"architectures": [
"RWKV7BackboneForCausalLM"
],
"auto_map": {
"AutoConfig": "rwkv7_backbone.MixerConfig",
"AutoModelForCausalLM": "rwkv7_backbone.RWKV7BackboneForCausalLM"
},
"bos_token_id": null,
"chunk_len": 256,
"hidden_size": 1024,
"d_head": null,
"d_qk_head": null,
"d_v_head": null,
"dtype": "bfloat16",
"eos_token_id": null,
"ffn_expansion": 4.0,
"gated_deltanet_conv_bias": 0,
"gated_deltanet_conv_size": 4,
"gated_deltanet_expand_k": 0.75,
"gated_deltanet_expand_v": 1.5,
"gated_deltanet_num_kv_heads": null,
"gated_deltanet_token_shift_mode": "none",
"gated_deltanet_use_mamba_gate": 1,
"gated_deltanet_value_residual_mode": "none",
"gptalpha_token_shift_mode": "rwkv",
"gptalpha_value_residual_mode": "rwkv",
"kda_allow_neg_eigval": 0,
"kda_conv_bias": 0,
"kda_conv_size": 4,
"kda_expand_k": 1.0,
"kda_expand_v": 1.0,
"kda_token_shift_mode": "none",
"kda_value_residual_mode": "none",
"kvm_apply_merge_gate_to_appends": 1,
"kvm_apply_merge_gate_to_initial_state": 0,
"kvm_token_shift_mode": "rwkv",
"kvm_use_head_temps": 1,
"kvm_use_merge_gate_keys": 1,
"kvm_use_merge_gate_values": 1,
"kvm_use_vlens": 1,
"kvm_value_residual_mode": "rwkv",
"lact_actual_rope_partial_dim": -1,
"lact_attn_qk_norm": 0,
"lact_chunk_size": -1,
"lact_factor": 1.0,
"lact_fw_init_gain": 0.5,
"lact_inter_multi": 1.0,
"lact_lact_window_size": -1,
"lact_learnable_muon_lr": 0,
"lact_muon_lr_reduce_exp": 2,
"lact_no_v_silu": 0,
"lact_norm_eps": 1e-06,
"lact_num_fw_heads": -1,
"lact_token_shift_mode": "rwkv",
"lact_ttt_lag": 0,
"lact_ttt_nope": 0,
"lact_ttt_prenorm": 0,
"lact_use_bswa": 0,
"lact_use_momentum": 1,
"lact_use_muon": 0,
"lact_value_residual_mode": "rwkv",
"lact_w0_w2_low_rank": 32,
"logit_softcap": 0.0,
"max_position_embeddings": 1073741824,
"model_class_path": "model.rwkv7_backbone.RWKV7BackboneForCausalLM",
"model_type": "rwkv7backbone",
"mup_base_dim": 768,
"n_bswa_chunks": 2,
"num_attention_heads": 8,
"n_layer": null,
"n_max_d_chunks": 1,
"num_hidden_layers": 24,
"ovq_chunk_size": 128,
"ovq_max_centroids": 2048,
"ovq_normalize_centroids": 0,
"ovq_token_shift_mode": "rwkv",
"ovq_use_counts_for_lr_only": 0,
"ovq_use_delta_rule": 0,
"ovq_use_min_state_size": 0,
"ovq_value_residual_mode": "rwkv",
"ovq_window_size": 128,
"pad_token_id": null,
"param_dtype": "bfloat16",
"residual_scale": 0.5,
"rope_partial_dim": 64,
"rope_theta": 10000.0,
"sink_len": 1,
"state_budget_mode": "fixed",
"state_growth_exponent": 0.5,
"state_growth_factor": 1.0,
"state_min_len": -1,
"state_round_down": 1,
"state_saturation_n": null,
"swa_token_shift_mode": "rwkv",
"swa_value_residual_mode": "rwkv",
"swa_window_size": -1,
"tie_word_embeddings": false,
"token_mixer": "gptalpha",
"token_mixer_class_path": null,
"transformers_version": "5.3.0",
"use_block_lambdas": 0,
"use_l2wrap": 0,
"use_overflow_v_key_weighting": 1,
"use_skip_connections": 0,
"use_state_decay": 0,
"use_state_deltarule": 0,
"use_tokenshift_att": 1,
"use_tokenshift_ffn": 1,
"use_value_residual": 1,
"vocab_size": 50304
}