Download config.json from featherless-ai/kvmpaper_gptalpha_350M: direct link, hf CLI and curl.
- Browser
- Download file 3.39 kB
-
https://huggingface.co/featherless-ai/kvmpaper_gptalpha_350M/resolve/main/config.json
- Command line
-
hf download hf://featherless-ai/kvmpaper_gptalpha_350M/config.json
-
curl -L -o config.json https://huggingface.co/featherless-ai/kvmpaper_gptalpha_350M/resolve/main/config.json
3.39 kB
| { | |
| "alt_layer_every": -1, | |
| "alt_rope_partial_dim": 64, | |
| "alt_rope_theta": 10000.0, | |
| "alt_share_v0": 0, | |
| "alt_token_mixer": "swa", | |
| "alt_token_mixer_class_path": null, | |
| "architectures": [ | |
| "RWKV7BackboneForCausalLM" | |
| ], | |
| "auto_map": { | |
| "AutoConfig": "rwkv7_backbone.MixerConfig", | |
| "AutoModelForCausalLM": "rwkv7_backbone.RWKV7BackboneForCausalLM" | |
| }, | |
| "bos_token_id": null, | |
| "chunk_len": 256, | |
| "hidden_size": 1024, | |
| "d_head": null, | |
| "d_qk_head": null, | |
| "d_v_head": null, | |
| "dtype": "bfloat16", | |
| "eos_token_id": null, | |
| "ffn_expansion": 4.0, | |
| "gated_deltanet_conv_bias": 0, | |
| "gated_deltanet_conv_size": 4, | |
| "gated_deltanet_expand_k": 0.75, | |
| "gated_deltanet_expand_v": 1.5, | |
| "gated_deltanet_num_kv_heads": null, | |
| "gated_deltanet_token_shift_mode": "none", | |
| "gated_deltanet_use_mamba_gate": 1, | |
| "gated_deltanet_value_residual_mode": "none", | |
| "gptalpha_token_shift_mode": "rwkv", | |
| "gptalpha_value_residual_mode": "rwkv", | |
| "kda_allow_neg_eigval": 0, | |
| "kda_conv_bias": 0, | |
| "kda_conv_size": 4, | |
| "kda_expand_k": 1.0, | |
| "kda_expand_v": 1.0, | |
| "kda_token_shift_mode": "none", | |
| "kda_value_residual_mode": "none", | |
| "kvm_apply_merge_gate_to_appends": 1, | |
| "kvm_apply_merge_gate_to_initial_state": 0, | |
| "kvm_token_shift_mode": "rwkv", | |
| "kvm_use_head_temps": 1, | |
| "kvm_use_merge_gate_keys": 1, | |
| "kvm_use_merge_gate_values": 1, | |
| "kvm_use_vlens": 1, | |
| "kvm_value_residual_mode": "rwkv", | |
| "lact_actual_rope_partial_dim": -1, | |
| "lact_attn_qk_norm": 0, | |
| "lact_chunk_size": -1, | |
| "lact_factor": 1.0, | |
| "lact_fw_init_gain": 0.5, | |
| "lact_inter_multi": 1.0, | |
| "lact_lact_window_size": -1, | |
| "lact_learnable_muon_lr": 0, | |
| "lact_muon_lr_reduce_exp": 2, | |
| "lact_no_v_silu": 0, | |
| "lact_norm_eps": 1e-06, | |
| "lact_num_fw_heads": -1, | |
| "lact_token_shift_mode": "rwkv", | |
| "lact_ttt_lag": 0, | |
| "lact_ttt_nope": 0, | |
| "lact_ttt_prenorm": 0, | |
| "lact_use_bswa": 0, | |
| "lact_use_momentum": 1, | |
| "lact_use_muon": 0, | |
| "lact_value_residual_mode": "rwkv", | |
| "lact_w0_w2_low_rank": 32, | |
| "logit_softcap": 0.0, | |
| "max_position_embeddings": 1073741824, | |
| "model_class_path": "model.rwkv7_backbone.RWKV7BackboneForCausalLM", | |
| "model_type": "rwkv7backbone", | |
| "mup_base_dim": 768, | |
| "n_bswa_chunks": 2, | |
| "num_attention_heads": 8, | |
| "n_layer": null, | |
| "n_max_d_chunks": 1, | |
| "num_hidden_layers": 24, | |
| "ovq_chunk_size": 128, | |
| "ovq_max_centroids": 2048, | |
| "ovq_normalize_centroids": 0, | |
| "ovq_token_shift_mode": "rwkv", | |
| "ovq_use_counts_for_lr_only": 0, | |
| "ovq_use_delta_rule": 0, | |
| "ovq_use_min_state_size": 0, | |
| "ovq_value_residual_mode": "rwkv", | |
| "ovq_window_size": 128, | |
| "pad_token_id": null, | |
| "param_dtype": "bfloat16", | |
| "residual_scale": 0.5, | |
| "rope_partial_dim": 64, | |
| "rope_theta": 10000.0, | |
| "sink_len": 1, | |
| "state_budget_mode": "fixed", | |
| "state_growth_exponent": 0.5, | |
| "state_growth_factor": 1.0, | |
| "state_min_len": -1, | |
| "state_round_down": 1, | |
| "state_saturation_n": null, | |
| "swa_token_shift_mode": "rwkv", | |
| "swa_value_residual_mode": "rwkv", | |
| "swa_window_size": -1, | |
| "tie_word_embeddings": false, | |
| "token_mixer": "gptalpha", | |
| "token_mixer_class_path": null, | |
| "transformers_version": "5.3.0", | |
| "use_block_lambdas": 0, | |
| "use_l2wrap": 0, | |
| "use_overflow_v_key_weighting": 1, | |
| "use_skip_connections": 0, | |
| "use_state_decay": 0, | |
| "use_state_deltarule": 0, | |
| "use_tokenshift_att": 1, | |
| "use_tokenshift_ffn": 1, | |
| "use_value_residual": 1, | |
| "vocab_size": 50304 | |
| } | |