Upload 7 files
Browse files- checkpoint_metadata.json +5 -0
- config.json +27 -0
- model.pt +3 -0
- tokenizer.json +0 -0
- tokenizer_config.json +0 -0
- training_state.json +21 -0
- training_state.pt +3 -0
checkpoint_metadata.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"global_step": 15000,
|
| 3 |
+
"kind": "regular",
|
| 4 |
+
"saved_at": "2026-08-05T21:19:13.263280+00:00"
|
| 5 |
+
}
|
config.json
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architecture": "FastDeepHybridLM",
|
| 3 |
+
"attention_every": 4,
|
| 4 |
+
"attention_residual_group_size": 4,
|
| 5 |
+
"bos_token_id": 1,
|
| 6 |
+
"conv_kernel_size": 4,
|
| 7 |
+
"embedding_size": 256,
|
| 8 |
+
"eos_token_id": 2,
|
| 9 |
+
"ffn_latent_size": 256,
|
| 10 |
+
"hidden_size": 512,
|
| 11 |
+
"initializer_range": 0.02,
|
| 12 |
+
"intermediate_size": 1728,
|
| 13 |
+
"max_position_embeddings": 4096,
|
| 14 |
+
"memory_heads": 4,
|
| 15 |
+
"memory_size": 128,
|
| 16 |
+
"model_type": "byte-deep-hybrid",
|
| 17 |
+
"mtp_loss_weight": 0.2,
|
| 18 |
+
"num_attention_heads": 8,
|
| 19 |
+
"num_hidden_layers": 24,
|
| 20 |
+
"num_key_value_heads": 2,
|
| 21 |
+
"pad_token_id": 0,
|
| 22 |
+
"padded_vocab_size": 6464,
|
| 23 |
+
"rms_norm_eps": 1e-05,
|
| 24 |
+
"rope_theta": 10000.0,
|
| 25 |
+
"vocab_size": 6422,
|
| 26 |
+
"window_size": 512
|
| 27 |
+
}
|
model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a7b9484613e9b99bbda7fd0884cdcffdd7b9386ade1de77c51dc7a81c994c766
|
| 3 |
+
size 238179151
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
training_state.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"batch_in_epoch": 12608,
|
| 3 |
+
"best_checkpoint": "/home/ventu/bytefalcon_h1_60m/runs/bytefast-60m/best",
|
| 4 |
+
"best_validation_loss": 0.9608736783266068,
|
| 5 |
+
"epoch": 4,
|
| 6 |
+
"global_step": 15000,
|
| 7 |
+
"last_finite_step": 15000,
|
| 8 |
+
"lr_decay_steps": 40000,
|
| 9 |
+
"nonfinite_events": 0,
|
| 10 |
+
"packed_fingerprint": {
|
| 11 |
+
"context_length": 4096,
|
| 12 |
+
"data_sha256": "901a662f4f079b2221bc5498f38cc422cdc404072a16ca094b3d226bbd4d83a0",
|
| 13 |
+
"format": "instruction\\n\\n\"text\"\\n\\n\"output\"<eos>",
|
| 14 |
+
"packing_version": 2,
|
| 15 |
+
"tokenizer_sha256": "4dcec7707ad71bb41520a167f37ad7446e0241a91363931c7840cec1a6482575",
|
| 16 |
+
"validation_ratio": 0.005
|
| 17 |
+
},
|
| 18 |
+
"started_at": "2026-08-05T18:30:50.533268+00:00",
|
| 19 |
+
"tokens_seen": 491520000,
|
| 20 |
+
"warmup_steps": 1000
|
| 21 |
+
}
|
training_state.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:70b95c0fe68bc841abe15def8785a649e47162c1a319cebeb090b3c423898194
|
| 3 |
+
size 476434986
|