{
  "architectures": [
    "Fuse3ForCausalLM"
  ],
  "block_auto_adjust_ff_dim": false,
  "block_dim": 2048,
  "block_ffn_dim_multiplier": 1.0,
  "block_mlp_init_scale": 1.0,
  "block_multiple_of": 256,
  "block_norm_eps": 1e-05,
  "block_out_init_scale": 1.0,
  "block_use_swiglu": true,
  "block_use_xavier_init": true,
  "bos_token_id": 124894,
  "coding_enabled": true,
  "conv_L_cache": 3,
  "conv_bias": false,
  "conv_dim": 2048,
  "conv_use_xavier_init": true,
  "dtype": "bfloat16",
  "eos_token_id": 124900,
  "expert_intermediate_size": 512,
  "expert_scale_init": -5.0,
  "experts_per_layer": {
    "0": [
      62,
      75,
      174,
      231,
      211,
      36,
      134,
      159,
      92,
      70,
      81,
      188,
      144,
      138,
      232,
      241,
      123,
      254,
      191,
      128,
      78,
      37,
      223,
      187,
      11,
      124,
      142,
      48,
      117,
      226,
      77,
      116
    ],
    "1": [
      193,
      254,
      51,
      33,
      80,
      235,
      107,
      66,
      169,
      141,
      13,
      120,
      209,
      75,
      133,
      25,
      40,
      96,
      48,
      153,
      222,
      167,
      162,
      217,
      214,
      55,
      23,
      101,
      74,
      17,
      121,
      185
    ],
    "2": [
      217,
      39,
      115,
      92,
      20,
      216,
      169,
      164,
      223,
      111,
      252,
      182,
      207,
      191,
      193,
      90,
      153,
      157,
      36,
      10,
      150,
      149,
      34,
      42,
      167,
      103,
      145,
      22,
      128,
      187,
      176,
      41
    ],
    "3": [
      196,
      6,
      69,
      224,
      201,
      199,
      149,
      163,
      251,
      17,
      36,
      43,
      212,
      154,
      92,
      166,
      119,
      56,
      197,
      75,
      126,
      221,
      109,
      81,
      113,
      93,
      183,
      22,
      31,
      173,
      255,
      100
    ],
    "4": [
      175,
      253,
      152,
      90,
      28,
      38,
      143,
      36,
      8,
      221,
      189,
      228,
      148,
      160,
      134,
      121,
      157,
      6,
      195,
      235,
      44,
      187,
      140,
      206,
      65,
      46,
      132,
      70,
      156,
      161,
      216,
      94
    ],
    "5": [
      153,
      58,
      251,
      50,
      70,
      22,
      177,
      60,
      218,
      0,
      77,
      222,
      159,
      210,
      163,
      19,
      146,
      8,
      53,
      42,
      74,
      111,
      172,
      137,
      61,
      64,
      7,
      108,
      136,
      164,
      185,
      122
    ],
    "6": [
      26,
      12,
      102,
      159,
      210,
      125,
      30,
      212,
      238,
      64,
      168,
      34,
      195,
      249,
      196,
      79,
      90,
      92,
      169,
      186,
      209,
      237,
      43,
      72,
      120,
      60,
      126,
      218,
      105,
      69,
      113,
      202
    ],
    "7": [
      185,
      78,
      245,
      175,
      227,
      80,
      92,
      49,
      233,
      120,
      174,
      244,
      3,
      142,
      5,
      12,
      104,
      226,
      246,
      132,
      184,
      183,
      240,
      81,
      39,
      160,
      131,
      108,
      75,
      26,
      140,
      187
    ],
    "8": [
      133,
      72,
      148,
      23,
      53,
      16,
      145,
      224,
      188,
      233,
      85,
      76,
      181,
      195,
      180,
      95,
      121,
      71,
      164,
      249,
      98,
      41,
      22,
      193,
      162,
      1,
      171,
      73,
      99,
      192,
      240,
      151
    ],
    "9": [
      99,
      207,
      1,
      186,
      161,
      255,
      130,
      81,
      229,
      188,
      68,
      172,
      254,
      32,
      28,
      114,
      137,
      54,
      179,
      199,
      225,
      120,
      95,
      61,
      251,
      178,
      176,
      233,
      140,
      164,
      20,
      168
    ],
    "10": [
      116,
      132,
      217,
      156,
      12,
      207,
      233,
      146,
      165,
      134,
      234,
      242,
      190,
      148,
      147,
      250,
      149,
      126,
      205,
      176,
      172,
      184,
      23,
      124,
      248,
      137,
      61,
      38,
      108,
      54,
      135,
      14
    ],
    "11": [
      60,
      44,
      141,
      97,
      32,
      204,
      223,
      130,
      107,
      222,
      37,
      63,
      142,
      15,
      218,
      220,
      230,
      134,
      153,
      170,
      10,
      213,
      98,
      180,
      139,
      33,
      20,
      71,
      215,
      214,
      162,
      143
    ],
    "12": [
      116,
      250,
      176,
      161,
      163,
      143,
      157,
      10,
      21,
      38,
      244,
      196,
      1,
      15,
      50,
      137,
      189,
      2,
      39,
      186,
      128,
      14,
      166,
      67,
      63,
      211,
      240,
      17,
      13,
      145,
      55,
      115
    ],
    "13": [
      93,
      212,
      45,
      145,
      219,
      165,
      14,
      83,
      208,
      180,
      183,
      252,
      42,
      120,
      0,
      248,
      151,
      232,
      170,
      166,
      58,
      202,
      66,
      100,
      194,
      34,
      92,
      98,
      158,
      19,
      174,
      144
    ],
    "14": [
      204,
      44,
      132,
      253,
      73,
      194,
      51,
      164,
      202,
      90,
      248,
      214,
      96,
      166,
      189,
      54,
      21,
      64,
      165,
      17,
      225,
      117,
      187,
      113,
      198,
      67,
      179,
      115,
      63,
      163,
      88,
      167
    ],
    "15": [
      147,
      12,
      139,
      14,
      169,
      235,
      206,
      238,
      18,
      1,
      98,
      170,
      114,
      145,
      228,
      195,
      50,
      165,
      242,
      227,
      148,
      74,
      134,
      248,
      3,
      215,
      28,
      90,
      21,
      240,
      81,
      0
    ],
    "16": [
      72,
      172,
      117,
      191,
      183,
      252,
      56,
      112,
      130,
      28,
      59,
      104,
      189,
      155,
      230,
      247,
      160,
      234,
      171,
      195,
      53,
      228,
      25,
      35,
      74,
      116,
      47,
      147,
      209,
      37,
      190,
      5
    ],
    "17": [
      89,
      69,
      38,
      1,
      185,
      152,
      233,
      20,
      19,
      98,
      209,
      100,
      8,
      58,
      44,
      21,
      62,
      222,
      140,
      219,
      125,
      204,
      117,
      217,
      135,
      223,
      189,
      9,
      54,
      4,
      126,
      169
    ],
    "18": [
      77,
      91,
      176,
      138,
      43,
      139,
      175,
      126,
      116,
      225,
      69,
      212,
      144,
      245,
      93,
      152,
      236,
      204,
      66,
      171,
      209,
      146,
      92,
      64,
      108,
      13,
      224,
      241,
      54,
      123,
      163,
      223
    ],
    "19": [
      212,
      59,
      122,
      75,
      115,
      35,
      101,
      88,
      164,
      143,
      203,
      226,
      61,
      95,
      60,
      137,
      99,
      33,
      219,
      183,
      22,
      114,
      116,
      47,
      85,
      248,
      67,
      133,
      231,
      221,
      11,
      25
    ],
    "20": [
      133,
      72,
      23,
      148,
      53,
      195,
      181,
      224,
      71,
      171,
      145,
      95,
      233,
      76,
      85,
      41,
      98,
      188,
      249,
      99,
      1,
      172,
      192,
      197,
      240,
      116,
      173,
      206,
      193,
      151,
      234,
      121
    ],
    "21": [
      207,
      99,
      186,
      1,
      188,
      81,
      251,
      130,
      229,
      95,
      172,
      255,
      68,
      254,
      137,
      32,
      199,
      114,
      179,
      225,
      140,
      20,
      28,
      233,
      54,
      72,
      243,
      193,
      178,
      164,
      183,
      211
    ],
    "22": [
      116,
      156,
      124,
      132,
      12,
      40,
      148,
      217,
      147,
      233,
      134,
      149,
      207,
      165,
      109,
      172,
      205,
      126,
      137,
      250,
      190,
      242,
      234,
      65,
      77,
      182,
      184,
      163,
      135,
      146,
      160,
      108
    ],
    "23": [
      60,
      204,
      141,
      32,
      44,
      223,
      222,
      107,
      37,
      63,
      213,
      134,
      193,
      142,
      220,
      218,
      15,
      10,
      130,
      61,
      153,
      230,
      139,
      143,
      121,
      59,
      20,
      64,
      162,
      80,
      117,
      170
    ],
    "24": [
      1,
      116,
      163,
      176,
      250,
      157,
      55,
      143,
      244,
      39,
      50,
      166,
      68,
      10,
      137,
      196,
      161,
      2,
      211,
      38,
      199,
      167,
      14,
      67,
      21,
      158,
      186,
      93,
      238,
      217,
      240,
      99
    ],
    "25": [
      93,
      145,
      165,
      45,
      208,
      42,
      219,
      212,
      14,
      0,
      47,
      101,
      183,
      202,
      83,
      252,
      166,
      120,
      100,
      232,
      151,
      180,
      155,
      203,
      248,
      58,
      251,
      34,
      170,
      66,
      243,
      233
    ],
    "26": [
      166,
      204,
      214,
      132,
      194,
      90,
      73,
      202,
      44,
      248,
      54,
      96,
      164,
      67,
      137,
      17,
      64,
      165,
      21,
      117,
      253,
      189,
      78,
      88,
      163,
      113,
      225,
      198,
      38,
      115,
      179,
      94
    ],
    "27": [
      147,
      139,
      98,
      206,
      12,
      18,
      114,
      169,
      14,
      85,
      1,
      28,
      50,
      242,
      170,
      182,
      240,
      228,
      235,
      195,
      145,
      215,
      134,
      3,
      99,
      70,
      238,
      74,
      183,
      150,
      190,
      103
    ],
    "28": [
      72,
      152,
      247,
      183,
      191,
      56,
      130,
      59,
      117,
      234,
      172,
      28,
      155,
      112,
      228,
      104,
      74,
      165,
      35,
      252,
      116,
      171,
      230,
      53,
      166,
      37,
      179,
      54,
      210,
      80,
      212,
      238
    ],
    "29": [
      89,
      1,
      38,
      69,
      185,
      19,
      125,
      248,
      98,
      152,
      100,
      222,
      117,
      9,
      4,
      54,
      204,
      91,
      209,
      140,
      44,
      233,
      167,
      155,
      238,
      219,
      79,
      192,
      21,
      58,
      150,
      236
    ]
  },
  "full_attn_idxs": null,
  "hidden_size": 2048,
  "initializer_range": 0.02,
  "intermediate_size": 10752,
  "layer_types": [
    "conv",
    "conv",
    "full_attention",
    "conv",
    "conv",
    "full_attention",
    "conv",
    "conv",
    "conv",
    "full_attention",
    "conv",
    "conv",
    "conv",
    "full_attention",
    "conv",
    "conv",
    "conv",
    "full_attention",
    "conv",
    "conv",
    "conv",
    "full_attention",
    "conv",
    "conv",
    "full_attention",
    "conv",
    "conv",
    "full_attention",
    "conv",
    "conv"
  ],
  "load_balance_coef": 0.001,
  "max_position_embeddings": 131072,
  "model_type": "fuse3",
  "norm_eps": 1e-05,
  "num_attention_heads": 32,
  "num_augmented_layers": 30,
  "num_heads": 32,
  "num_hidden_layers": 30,
  "num_key_value_heads": 8,
  "pad_token_id": 124893,
  "rope_parameters": {
    "rope_theta": 10000000.0,
    "rope_type": "default"
  },
  "router_init_scale": -2.0,
  "swiglu_limit": 10.0,
  "tie_word_embeddings": true,
  "top_k_experts": 8,
  "transformers_version": "5.14.1",
  "use_cache": false,
  "use_pos_enc": true,
  "vocab_size": 128000,
  "auto_map": {
    "AutoConfig": "fuse3_model.Fuse3Config",
    "AutoModelForCausalLM": "fuse3_model.Fuse3ForCausalLM"
  }
}