Qwen90M step 0; complete resumable checkpoint

README.md

@@ -0,0 +1,15 @@
+---
+library_name: transformers
+pipeline_tag: text-generation
+tags: [qwen3, byte-level, pretraining]
+---
+# Qwen90M Byte
+24 vanilla Qwen3 layers, width 640, MLP 1440, 10 query / 2 KV heads, head dimension 64.
+90,148,352 tied-embedding parameters. 259 tokens: bytes 0–255, PAD 256, BOS 257, EOS 258.
+Trained from scratch; this is a base model, not a Qwen instruction checkpoint.
+
+Load the model with `AutoModelForCausalLM.from_pretrained(repo)`.
+Load its custom byte tokenizer with `AutoTokenizer.from_pretrained(repo, trust_remote_code=True)`.
+Training records begin with BOS; prepend `tokenizer.bos_token_id` to encoded generation prompts.
+See training_config.json and coverage.json for the mixture and consumed data statistics.
+Model weights are FP32 masters trained under BF16 autocast. Use dtype=torch.bfloat16 for GPU inference.

byte_vocab.json

@@ -0,0 +1,261 @@
+{
+  "<0x00>": 0,
+  "<0x01>": 1,
+  "<0x02>": 2,
+  "<0x03>": 3,
+  "<0x04>": 4,
+  "<0x05>": 5,
+  "<0x06>": 6,
+  "<0x07>": 7,
+  "<0x08>": 8,
+  "<0x09>": 9,
+  "<0x0A>": 10,
+  "<0x0B>": 11,
+  "<0x0C>": 12,
+  "<0x0D>": 13,
+  "<0x0E>": 14,
+  "<0x0F>": 15,
+  "<0x10>": 16,
+  "<0x11>": 17,
+  "<0x12>": 18,
+  "<0x13>": 19,
+  "<0x14>": 20,
+  "<0x15>": 21,
+  "<0x16>": 22,
+  "<0x17>": 23,
+  "<0x18>": 24,
+  "<0x19>": 25,
+  "<0x1A>": 26,
+  "<0x1B>": 27,
+  "<0x1C>": 28,
+  "<0x1D>": 29,
+  "<0x1E>": 30,
+  "<0x1F>": 31,
+  "<0x20>": 32,
+  "<0x21>": 33,
+  "<0x22>": 34,
+  "<0x23>": 35,
+  "<0x24>": 36,
+  "<0x25>": 37,
+  "<0x26>": 38,
+  "<0x27>": 39,
+  "<0x28>": 40,
+  "<0x29>": 41,
+  "<0x2A>": 42,
+  "<0x2B>": 43,
+  "<0x2C>": 44,
+  "<0x2D>": 45,
+  "<0x2E>": 46,
+  "<0x2F>": 47,
+  "<0x30>": 48,
+  "<0x31>": 49,
+  "<0x32>": 50,
+  "<0x33>": 51,
+  "<0x34>": 52,
+  "<0x35>": 53,
+  "<0x36>": 54,
+  "<0x37>": 55,
+  "<0x38>": 56,
+  "<0x39>": 57,
+  "<0x3A>": 58,
+  "<0x3B>": 59,
+  "<0x3C>": 60,
+  "<0x3D>": 61,
+  "<0x3E>": 62,
+  "<0x3F>": 63,
+  "<0x40>": 64,
+  "<0x41>": 65,
+  "<0x42>": 66,
+  "<0x43>": 67,
+  "<0x44>": 68,
+  "<0x45>": 69,
+  "<0x46>": 70,
+  "<0x47>": 71,
+  "<0x48>": 72,
+  "<0x49>": 73,
+  "<0x4A>": 74,
+  "<0x4B>": 75,
+  "<0x4C>": 76,
+  "<0x4D>": 77,
+  "<0x4E>": 78,
+  "<0x4F>": 79,
+  "<0x50>": 80,
+  "<0x51>": 81,
+  "<0x52>": 82,
+  "<0x53>": 83,
+  "<0x54>": 84,
+  "<0x55>": 85,
+  "<0x56>": 86,
+  "<0x57>": 87,
+  "<0x58>": 88,
+  "<0x59>": 89,
+  "<0x5A>": 90,
+  "<0x5B>": 91,
+  "<0x5C>": 92,
+  "<0x5D>": 93,
+  "<0x5E>": 94,
+  "<0x5F>": 95,
+  "<0x60>": 96,
+  "<0x61>": 97,
+  "<0x62>": 98,
+  "<0x63>": 99,
+  "<0x64>": 100,
+  "<0x65>": 101,
+  "<0x66>": 102,
+  "<0x67>": 103,
+  "<0x68>": 104,
+  "<0x69>": 105,
+  "<0x6A>": 106,
+  "<0x6B>": 107,
+  "<0x6C>": 108,
+  "<0x6D>": 109,
+  "<0x6E>": 110,
+  "<0x6F>": 111,
+  "<0x70>": 112,
+  "<0x71>": 113,
+  "<0x72>": 114,
+  "<0x73>": 115,
+  "<0x74>": 116,
+  "<0x75>": 117,
+  "<0x76>": 118,
+  "<0x77>": 119,
+  "<0x78>": 120,
+  "<0x79>": 121,
+  "<0x7A>": 122,
+  "<0x7B>": 123,
+  "<0x7C>": 124,
+  "<0x7D>": 125,
+  "<0x7E>": 126,
+  "<0x7F>": 127,
+  "<0x80>": 128,
+  "<0x81>": 129,
+  "<0x82>": 130,
+  "<0x83>": 131,
+  "<0x84>": 132,
+  "<0x85>": 133,
+  "<0x86>": 134,
+  "<0x87>": 135,
+  "<0x88>": 136,
+  "<0x89>": 137,
+  "<0x8A>": 138,
+  "<0x8B>": 139,
+  "<0x8C>": 140,
+  "<0x8D>": 141,
+  "<0x8E>": 142,
+  "<0x8F>": 143,
+  "<0x90>": 144,
+  "<0x91>": 145,
+  "<0x92>": 146,
+  "<0x93>": 147,
+  "<0x94>": 148,
+  "<0x95>": 149,
+  "<0x96>": 150,
+  "<0x97>": 151,
+  "<0x98>": 152,
+  "<0x99>": 153,
+  "<0x9A>": 154,
+  "<0x9B>": 155,
+  "<0x9C>": 156,
+  "<0x9D>": 157,
+  "<0x9E>": 158,
+  "<0x9F>": 159,
+  "<0xA0>": 160,
+  "<0xA1>": 161,
+  "<0xA2>": 162,
+  "<0xA3>": 163,
+  "<0xA4>": 164,
+  "<0xA5>": 165,
+  "<0xA6>": 166,
+  "<0xA7>": 167,
+  "<0xA8>": 168,
+  "<0xA9>": 169,
+  "<0xAA>": 170,
+  "<0xAB>": 171,
+  "<0xAC>": 172,
+  "<0xAD>": 173,
+  "<0xAE>": 174,
+  "<0xAF>": 175,
+  "<0xB0>": 176,
+  "<0xB1>": 177,
+  "<0xB2>": 178,
+  "<0xB3>": 179,
+  "<0xB4>": 180,
+  "<0xB5>": 181,
+  "<0xB6>": 182,
+  "<0xB7>": 183,
+  "<0xB8>": 184,
+  "<0xB9>": 185,
+  "<0xBA>": 186,
+  "<0xBB>": 187,
+  "<0xBC>": 188,
+  "<0xBD>": 189,
+  "<0xBE>": 190,
+  "<0xBF>": 191,
+  "<0xC0>": 192,
+  "<0xC1>": 193,
+  "<0xC2>": 194,
+  "<0xC3>": 195,
+  "<0xC4>": 196,
+  "<0xC5>": 197,
+  "<0xC6>": 198,
+  "<0xC7>": 199,
+  "<0xC8>": 200,
+  "<0xC9>": 201,
+  "<0xCA>": 202,
+  "<0xCB>": 203,
+  "<0xCC>": 204,
+  "<0xCD>": 205,
+  "<0xCE>": 206,
+  "<0xCF>": 207,
+  "<0xD0>": 208,
+  "<0xD1>": 209,
+  "<0xD2>": 210,
+  "<0xD3>": 211,
+  "<0xD4>": 212,
+  "<0xD5>": 213,
+  "<0xD6>": 214,
+  "<0xD7>": 215,
+  "<0xD8>": 216,
+  "<0xD9>": 217,
+  "<0xDA>": 218,
+  "<0xDB>": 219,
+  "<0xDC>": 220,
+  "<0xDD>": 221,
+  "<0xDE>": 222,
+  "<0xDF>": 223,
+  "<0xE0>": 224,
+  "<0xE1>": 225,
+  "<0xE2>": 226,
+  "<0xE3>": 227,
+  "<0xE4>": 228,
+  "<0xE5>": 229,
+  "<0xE6>": 230,
+  "<0xE7>": 231,
+  "<0xE8>": 232,
+  "<0xE9>": 233,
+  "<0xEA>": 234,
+  "<0xEB>": 235,
+  "<0xEC>": 236,
+  "<0xED>": 237,
+  "<0xEE>": 238,
+  "<0xEF>": 239,
+  "<0xF0>": 240,
+  "<0xF1>": 241,
+  "<0xF2>": 242,
+  "<0xF3>": 243,
+  "<0xF4>": 244,
+  "<0xF5>": 245,
+  "<0xF6>": 246,
+  "<0xF7>": 247,
+  "<0xF8>": 248,
+  "<0xF9>": 249,
+  "<0xFA>": 250,
+  "<0xFB>": 251,
+  "<0xFC>": 252,
+  "<0xFD>": 253,
+  "<0xFE>": 254,
+  "<0xFF>": 255,
+  "<bos>": 257,
+  "<eos>": 258,
+  "<pad>": 256
+}

complete.json

@@ -0,0 +1,19 @@
+{
+  "step": 0,
+  "files": {
+    "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
+    "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
+    "model.safetensors": "1c688d2ef10bd45dcb2ed3ca6945c4cc383ba4cc356d90bbb93b725e427338c1",
+    "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
+    "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
+    "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
+    "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
+    "trainer.pt": "21aa2cccd888f3918a44b02c1ad179a74c9924c4304886988affaf97c4132422",
+    "training_config.json": "528161bcd9687da465595131ceb9bfb223ac3e494a1c6d5fcfdbd97b88324bd8",
+    "performance.json": "b95a1befd575f8d9a02b43fd0cc41c54243f5297127b93d2656e0292ba15e77e",
+    "shard_scheduler.json": "c16912023ae706fc358e0de73ec412ec549d6d440abc128e7900e41c5c63a6f3",
+    "coverage.json": "1db676f88d72dde495669f8f42a2a0104a6dc4ec5bfa810967263316d2335d79",
+    "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
+  },
+  "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"
+}

config.json

@@ -0,0 +1,57 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 257,
+  "dtype": "float32",
+  "eos_token_id": 258,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 640,
+  "initializer_range": 0.02,
+  "intermediate_size": 1440,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 2048,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 10,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "pad_token_id": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 100000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.1",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 259
+}

coverage.json

@@ -0,0 +1,54 @@
+{
+  "step": 0,
+  "stats": {},
+  "sources": {
+    "climbmix": {
+      "page": 0,
+      "row": 0,
+      "start": null,
+      "epoch": 0,
+      "offset": 0,
+      "documents": 0
+    },
+    "rewrite6": {
+      "page": 0,
+      "row": 0,
+      "start": null,
+      "epoch": 0,
+      "offset": 0,
+      "documents": 0
+    },
+    "openmath": {
+      "page": 0,
+      "row": 0,
+      "start": null,
+      "epoch": 0,
+      "offset": 0,
+      "documents": 0
+    },
+    "ultra_qa": {
+      "page": 0,
+      "row": 0,
+      "start": null,
+      "epoch": 0,
+      "offset": 0,
+      "documents": 0
+    },
+    "ultra_style": {
+      "page": 0,
+      "row": 0,
+      "start": null,
+      "epoch": 0,
+      "offset": 0,
+      "documents": 0
+    },
+    "cortex": {
+      "page": 0,
+      "row": 0,
+      "start": null,
+      "epoch": 0,
+      "offset": 0,
+      "documents": 0
+    }
+  }
+}

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 257,
+  "eos_token_id": 258,
+  "pad_token_id": 256,
+  "transformers_version": "4.57.1",
+  "use_cache": false
+}

latest.json

@@ -0,0 +1 @@
+{"step": 0, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "1c688d2ef10bd45dcb2ed3ca6945c4cc383ba4cc356d90bbb93b725e427338c1", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "21aa2cccd888f3918a44b02c1ad179a74c9924c4304886988affaf97c4132422", "training_config.json": "528161bcd9687da465595131ceb9bfb223ac3e494a1c6d5fcfdbd97b88324bd8", "performance.json": "b95a1befd575f8d9a02b43fd0cc41c54243f5297127b93d2656e0292ba15e77e", "shard_scheduler.json": "c16912023ae706fc358e0de73ec412ec549d6d440abc128e7900e41c5c63a6f3", "coverage.json": "1db676f88d72dde495669f8f42a2a0104a6dc4ec5bfa810967263316d2335d79", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000000000"}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c688d2ef10bd45dcb2ed3ca6945c4cc383ba4cc356d90bbb93b725e427338c1
+size 360622744

performance.json

@@ -0,0 +1,222 @@
+{
+  "cache_key": {
+    "version": 1,
+    "kind": "gpu",
+    "gpu": {
+      "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
+      "total_memory": 101975851008,
+      "capability": [
+        12,
+        0
+      ]
+    },
+    "torch": "2.8.0+cu128",
+    "cuda": "12.8",
+    "python": "3.13.11",
+    "packages": {
+      "transformers": "4.57.1",
+      "numpy": "2.2.6",
+      "pyarrow": "21.0.0",
+      "huggingface_hub": "0.36.0"
+    },
+    "settings": {
+      "context": 2048,
+      "global_batch": 90,
+      "microbatch_candidates": [
+        8,
+        16,
+        32,
+        48,
+        64,
+        72,
+        76,
+        80,
+        86,
+        90
+      ],
+      "batch_selection": "max_batch",
+      "gradient_checkpointing": false,
+      "compile_model": true,
+      "autotune_microbatch": true,
+      "attention_backend": "flash",
+      "vram_fraction": 0.95
+    },
+    "source": "e6d135a4487a9fd1907e12903cda79e75fd61305fad149fda7ec5281279dc18b",
+    "requested_microbatch_ceiling": 90
+  },
+  "cache_reused": false,
+  "attention": "PyTorch FlashAttention (forced; forward/backward verified)",
+  "selected": {
+    "microbatch": 86,
+    "mode": "compiled",
+    "microbatch_seconds": 0.8818629400000191,
+    "peak_gb": 84.78368663787842,
+    "fits_budget": true,
+    "estimated_update_seconds": 0.9349941874999956,
+    "input_tokens_per_second": 199722.64624250587
+  },
+  "measurements": [
+    {
+      "microbatch": 8,
+      "mode": "eager",
+      "microbatch_seconds": 0.15782159200000478,
+      "peak_gb": 14.079077243804932,
+      "fits_budget": true,
+      "estimated_update_seconds": 1.793421506500053,
+      "input_tokens_per_second": 103813.42497165725
+    },
+    {
+      "microbatch": 16,
+      "mode": "eager",
+      "microbatch_seconds": 0.3450626039999918,
+      "peak_gb": 27.397329807281494,
+      "fits_budget": true,
+      "estimated_update_seconds": 1.9234990809999601,
+      "input_tokens_per_second": 94962.47817106481
+    },
+    {
+      "microbatch": 32,
+      "mode": "eager",
+      "microbatch_seconds": 0.7818640030000097,
+      "peak_gb": 54.14767265319824,
+      "fits_budget": true,
+      "estimated_update_seconds": 2.185234111500023,
+      "input_tokens_per_second": 83820.20370363462
+    },
+    {
+      "microbatch": 48,
+      "mode": "eager",
+      "microbatch_seconds": 1.1685763079999845,
+      "peak_gb": 80.94732475280762,
+      "fits_budget": true,
+      "estimated_update_seconds": 2.190914332499986,
+      "input_tokens_per_second": 84122.87612457851
+    },
+    {
+      "microbatch": 8,
+      "mode": "compiled",
+      "microbatch_seconds": 0.08803204299999834,
+      "peak_gb": 8.428553104400635,
+      "fits_budget": true,
+      "estimated_update_seconds": 1.0060347614999898,
+      "input_tokens_per_second": 186114.0494035826
+    },
+    {
+      "microbatch": 16,
+      "mode": "compiled",
+      "microbatch_seconds": 0.1704058820000114,
+      "peak_gb": 16.191233158111572,
+      "fits_budget": true,
+      "estimated_update_seconds": 0.9576335650000658,
+      "input_tokens_per_second": 192293.8317352086
+    },
+    {
+      "microbatch": 32,
+      "mode": "compiled",
+      "microbatch_seconds": 0.34082671699997036,
+      "peak_gb": 31.810625553131104,
+      "fits_budget": true,
+      "estimated_update_seconds": 0.9493586249999737,
+      "input_tokens_per_second": 192285.39527904938
+    },
+    {
+      "microbatch": 48,
+      "mode": "compiled",
+      "microbatch_seconds": 0.5061537779999981,
+      "peak_gb": 47.43971490859985,
+      "fits_budget": true,
+      "estimated_update_seconds": 0.9468006980000041,
+      "input_tokens_per_second": 194217.6553308279
+    },
+    {
+      "microbatch": 64,
+      "mode": "compiled",
+      "microbatch_seconds": 0.6658512800000267,
+      "peak_gb": 63.073320388793945,
+      "fits_budget": true,
+      "estimated_update_seconds": 0.9295456565000109,
+      "input_tokens_per_second": 196848.76178355434
+    },
+    {
+      "microbatch": 72,
+      "mode": "compiled",
+      "microbatch_seconds": 0.7430151979999664,
+      "peak_gb": 70.97307014465332,
+      "fits_budget": true,
+      "estimated_update_seconds": 0.9250642054999503,
+      "input_tokens_per_second": 198456.23669195347
+    },
+    {
+      "microbatch": 76,
+      "mode": "compiled",
+      "microbatch_seconds": 0.78068104700003,
+      "peak_gb": 74.84185981750488,
+      "fits_budget": true,
+      "estimated_update_seconds": 0.9241739715000392,
+      "input_tokens_per_second": 199374.63654089966
+    },
+    {
+      "microbatch": 80,
+      "mode": "compiled",
+      "microbatch_seconds": 0.8210239410000213,
+      "peak_gb": 78.72572612762451,
+      "fits_budget": true,
+      "estimated_update_seconds": 0.9229792625000073,
+      "input_tokens_per_second": 199555.69115370748
+    },
+    {
+      "microbatch": 86,
+      "mode": "compiled",
+      "microbatch_seconds": 0.8818629400000191,
+      "peak_gb": 84.78368663787842,
+      "fits_budget": true,
+      "estimated_update_seconds": 0.9349941874999956,
+      "input_tokens_per_second": 199722.64624250587
+    },
+    {
+      "microbatch": 90,
+      "mode": "compiled",
+      "microbatch_seconds": 0.9259720020000373,
+      "peak_gb": 88.6924238204956,
+      "fits_budget": false,
+      "estimated_update_seconds": 0.9259720020000373,
+      "input_tokens_per_second": 199055.69455867045
+    }
+  ],
+  "failures": [
+    {
+      "microbatch": 64,
+      "mode": "eager",
+      "reason": "oom"
+    },
+    {
+      "microbatch": 72,
+      "mode": "eager",
+      "reason": "oom"
+    },
+    {
+      "microbatch": 76,
+      "mode": "eager",
+      "reason": "oom"
+    },
+    {
+      "microbatch": 80,
+      "mode": "eager",
+      "reason": "oom"
+    },
+    {
+      "microbatch": 86,
+      "mode": "eager",
+      "reason": "oom"
+    },
+    {
+      "microbatch": 90,
+      "mode": "eager",
+      "reason": "oom"
+    }
+  ],
+  "selection_policy": "max_batch",
+  "budget_gb": 89.6964394569397,
+  "reserve_gb": 2.1791439056396484,
+  "note": "Forward/backward throughput; excludes optimizer, data, logging and checkpoint time."
+}

shard_scheduler.json

@@ -0,0 +1,230 @@
+{
+  "version": 1,
+  "startup_batch_seconds": [
+    17.862166522000052,
+    0.01599195200003578,
+    0.012736249999989013,
+    0.10922379700002693,
+    0.015912060000005113,
+    0.014246264000007614,
+    0.01397435499995936,
+    0.10559937200002878
+  ],
+  "source_benchmarks": [
+    {
+      "source": "climbmix",
+      "rows": 256,
+      "tokens": 721064,
+      "bytes": 302776,
+      "generation_seconds": 0.6739276440000594,
+      "write_seconds": 0.00442227799999273,
+      "read_seconds": 0.04565144100001817,
+      "tokens_per_second": 1062967.6168812835,
+      "note": "Bounded fresh sample; full-page latency is an extrapolation.",
+      "estimated_pages_for_buffer": 1,
+      "required_token_positions_per_second": 78853.96613762408,
+      "required_pages_per_second_upper_estimate": 0.00683486193125923
+    },
+    {
+      "source": "rewrite6",
+      "rows": 256,
+      "tokens": 156675,
+      "bytes": 68694,
+      "generation_seconds": 0.12810849499999222,
+      "write_seconds": 0.001732062999963091,
+      "read_seconds": 0.002349199000036606,
+      "tokens_per_second": 1206672.263377472,
+      "note": "Bounded fresh sample; full-page latency is an extrapolation.",
+      "estimated_pages_for_buffer": 1,
+      "required_token_positions_per_second": 9856.74576720301,
+      "required_pages_per_second_upper_estimate": 0.003932003258019391
+    },
+    {
+      "source": "openmath",
+      "rows": 256,
+      "tokens": 270174,
+      "bytes": 90063,
+      "generation_seconds": 0.6564187429999038,
+      "write_seconds": 0.002473517999987962,
+      "read_seconds": 0.0032515120000198294,
+      "tokens_per_second": 410042.7581134458,
+      "note": "Bounded fresh sample; full-page latency is an extrapolation.",
+      "estimated_pages_for_buffer": 1,
+      "required_token_positions_per_second": 19713.49153440602,
+      "required_pages_per_second_upper_estimate": 0.004560369320883491
+    },
+    {
+      "source": "ultra_qa",
+      "rows": 256,
+      "tokens": 1026361,
+      "bytes": 365167,
+      "generation_seconds": 0.46967248600003586,
+      "write_seconds": 0.004728889000034542,
+      "read_seconds": 0.0050225659999796335,
+      "tokens_per_second": 2163486.5624068812,
+      "note": "Bounded fresh sample; full-page latency is an extrapolation.",
+      "estimated_pages_for_buffer": 1,
+      "required_token_positions_per_second": 29570.23730160903,
+      "required_pages_per_second_upper_estimate": 0.0018006723086229548
+    },
+    {
+      "source": "ultra_style",
+      "rows": 256,
+      "tokens": 811870,
+      "bytes": 338806,
+      "generation_seconds": 0.5208254859999215,
+      "write_seconds": 0.004081093000081637,
+      "read_seconds": 0.004787273000033565,
+      "tokens_per_second": 1546694.2737633225,
+      "note": "Bounded fresh sample; full-page latency is an extrapolation.",
+      "estimated_pages_for_buffer": 1,
+      "required_token_positions_per_second": 29570.23730160903,
+      "required_pages_per_second_upper_estimate": 0.0022763987231337094
+    },
+    {
+      "source": "cortex",
+      "rows": 128,
+      "tokens": 80537,
+      "bytes": 22745,
+      "generation_seconds": 0.07537080299994159,
+      "write_seconds": 0.003962095000019872,
+      "read_seconds": 0.042780299000014566,
+      "tokens_per_second": 1015177.8395898147,
+      "note": "Bounded fresh sample; full-page latency is an extrapolation.",
+      "estimated_pages_for_buffer": 23,
+      "required_token_positions_per_second": 29570.23730160903,
+      "required_pages_per_second_upper_estimate": 0.36716338206798155
+    }
+  ],
+  "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
+  "layout": "Logical pages unchanged; multiple pages per immutable TAR shard.",
+  "candidate_shard_mib": [
+    16,
+    32,
+    64,
+    128,
+    256
+  ],
+  "local_shard_size_benchmark": [
+    {
+      "target_mib": 16,
+      "actual_bytes": 16107520,
+      "pages": 6,
+      "packing_seconds": 0.03508346299997811,
+      "read_seconds": 0.0015898530000413302,
+      "io_bytes_per_second": 439216350.1111124,
+      "target_reached": true
+    },
+    {
+      "target_mib": 32,
+      "actual_bytes": 18657280,
+      "pages": 8,
+      "packing_seconds": 0.04401178199998412,
+      "read_seconds": 0.0016850460000341627,
+      "io_bytes_per_second": 408283918.5247723,
+      "target_reached": false
+    },
+    {
+      "target_mib": 64,
+      "actual_bytes": 18657280,
+      "pages": 8,
+      "packing_seconds": 0.04403130000002875,
+      "read_seconds": 0.0015284270000393008,
+      "io_bytes_per_second": 409512550.4148024,
+      "target_reached": false
+    },
+    {
+      "target_mib": 128,
+      "actual_bytes": 18657280,
+      "pages": 8,
+      "packing_seconds": 0.04190044699998907,
+      "read_seconds": 0.001583749000019452,
+      "io_bytes_per_second": 429058870.0316856,
+      "target_reached": false
+    },
+    {
+      "target_mib": 256,
+      "actual_bytes": 18657280,
+      "pages": 8,
+      "packing_seconds": 0.042332877000035296,
+      "read_seconds": 0.0015621619999137693,
+      "io_bytes_per_second": 425043021.37700915,
+      "target_reached": false
+    }
+  ],
+  "shard_benchmark_scope": "Local packing/read measurements; upload capacity measured only on real data commits. Targets larger than available pages are sample-limited.",
+  "estimated_fresh_generation_batch_seconds": 0.1434574083060457,
+  "selected": {
+    "prefetch_batches": 65,
+    "prefetch_capacity_limit": 512,
+    "batch_memory_estimate_bytes": 688946,
+    "producer_batch_seconds": 2.3582960475177965,
+    "producer_p95_seconds": 17.862166522000052,
+    "gpu_update_seconds": 0.9349941874999956,
+    "producer_batches_per_second": 0.4240349726458394,
+    "gpu_batches_per_second": 1.06952536536491,
+    "producer_headroom": 0.39647023472057946,
+    "sustainable": false,
+    "buffer_seconds": 60.774622187499716,
+    "buffer_drain_seconds": 100.69863274989008,
+    "shard_target_mib": 32,
+    "upload_pages_per_commit": 15,
+    "max_pending_pages": 1842,
+    "generation_bytes_per_second": 1035119.2626246791,
+    "upload_bytes_per_second": 6146691.738378885,
+    "upload_sustainable": true,
+    "logical_page_rows": 4096,
+    "logical_cortex_page_rows": 128,
+    "estimated_full_shards_per_hour": 111.0562487080349,
+    "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
+  },
+  "upload_measurements": [
+    {
+      "seconds": 0.8211703489999991,
+      "bytes": 5047481,
+      "pages": 1,
+      "shards": 1
+    }
+  ],
+  "cache_key": {
+    "version": 1,
+    "kind": "data",
+    "gpu": {
+      "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
+      "total_memory": 101975851008,
+      "capability": [
+        12,
+        0
+      ]
+    },
+    "torch": "2.8.0+cu128",
+    "cuda": "12.8",
+    "python": "3.13.11",
+    "packages": {
+      "transformers": "4.57.1",
+      "numpy": "2.2.6",
+      "pyarrow": "21.0.0",
+      "huggingface_hub": "0.36.0"
+    },
+    "settings": {
+      "context": 2048,
+      "global_batch": 90,
+      "page_rows": 4096,
+      "cortex_page_rows": 128,
+      "mix": {
+        "climbmix": 40,
+        "ultra_qa": 15,
+        "ultra_style": 15,
+        "cortex": 15,
+        "openmath": 10,
+        "rewrite6": 5
+      },
+      "cpu_threads": 4
+    },
+    "source": "f8722b66ff2f92bca174f36564336149e85c65c48b964d05ff3a4d5d85da308f",
+    "cpu": "unknown",
+    "cpu_count": 20,
+    "microbatch": 86
+  },
+  "cache_reused": false
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<bos>",
+  "eos_token": "<eos>",
+  "pad_token": "<pad>"
+}

tokenization_byte.py

@@ -0,0 +1,138 @@
+import json
+import os
+from transformers import PreTrainedTokenizer
+
+
+BYTE_PREFIX = "<0x"
+PAD_TOKEN = "<pad>"
+BOS_TOKEN = "<bos>"
+EOS_TOKEN = "<eos>"
+
+
+class ByteTokenizer(PreTrainedTokenizer):
+    """Lossless UTF-8 byte tokenizer used by Qwen90M.
+
+    IDs:
+      0..255 -> raw byte values
+      256    -> PAD
+      257    -> BOS
+      258    -> EOS
+
+    No UNK token is required because every UTF-8 string is representable as bytes.
+    """
+
+    vocab_files_names = {"vocab_file": "byte_vocab.json"}
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file=None,
+        pad_token=PAD_TOKEN,
+        bos_token=BOS_TOKEN,
+        eos_token=EOS_TOKEN,
+        unk_token=None,
+        model_max_length=1024,
+        padding_side="left",
+        clean_up_tokenization_spaces=False,
+        **kwargs,
+    ):
+        # Transformers v5 loads values from tokenizer_config.json into this
+        # constructor. Make every value that we also forward to PythonBackend
+        # an explicit argument so it is consumed exactly once instead of being
+        # duplicated inside **kwargs.
+        self.vocab_file = vocab_file
+        kwargs.setdefault("split_special_tokens",True)
+        super().__init__(
+            pad_token=pad_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            model_max_length=model_max_length,
+            padding_side=padding_side,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs,
+        )
+
+    @property
+    def vocab_size(self):
+        return 259
+
+    def get_vocab(self):
+        vocab = {f"<0x{i:02X}>": i for i in range(256)}
+        vocab[PAD_TOKEN] = 256
+        vocab[BOS_TOKEN] = 257
+        vocab[EOS_TOKEN] = 258
+        return vocab
+
+    def _tokenize(self, text, **kwargs):
+        return [f"<0x{b:02X}>" for b in text.encode("utf-8", errors="replace")]
+
+    def _convert_token_to_id(self, token):
+        if token == PAD_TOKEN:
+            return 256
+        if token == BOS_TOKEN:
+            return 257
+        if token == EOS_TOKEN:
+            return 258
+        if isinstance(token, str) and token.startswith(BYTE_PREFIX) and token.endswith(">"):
+            try:
+                value = int(token[3:-1], 16)
+                if 0 <= value <= 255:
+                    return value
+            except ValueError:
+                pass
+        # This branch should be unreachable for text encoded by this tokenizer.
+        return 0
+
+    def _convert_id_to_token(self, index):
+        index = int(index)
+        if 0 <= index <= 255:
+            return f"<0x{index:02X}>"
+        if index == 256:
+            return PAD_TOKEN
+        if index == 257:
+            return BOS_TOKEN
+        if index == 258:
+            return EOS_TOKEN
+        return "<0x00>"
+
+    def convert_tokens_to_string(self, tokens):
+        out = []
+        buf = bytearray()
+
+        def flush():
+            nonlocal buf
+            if buf:
+                out.append(bytes(buf).decode("utf-8", errors="replace"))
+                buf = bytearray()
+
+        for token in tokens:
+            idx = self._convert_token_to_id(token)
+            if isinstance(token, str) and 0 <= idx <= 255 and token.startswith(BYTE_PREFIX):
+                buf.append(idx)
+            else:
+                flush()
+                out.append(str(token))
+        flush()
+        return "".join(out)
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        # BET pretraining did not automatically insert BOS/EOS around ordinary text.
+        if token_ids_1 is None:
+            return list(token_ids_0)
+        return list(token_ids_0) + list(token_ids_1)
+
+    def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
+        n = len(token_ids_0) + (len(token_ids_1) if token_ids_1 is not None else 0)
+        return [0] * n
+
+    def save_vocabulary(self, save_directory, filename_prefix=None):
+        os.makedirs(save_directory, exist_ok=True)
+        name = "byte_vocab.json" if filename_prefix is None else f"{filename_prefix}-byte_vocab.json"
+        path = os.path.join(save_directory, name)
+        vocab = {f"<0x{i:02X}>": i for i in range(256)}
+        vocab.update({PAD_TOKEN: 256, BOS_TOKEN: 257, EOS_TOKEN: 258})
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(vocab, f, indent=2, sort_keys=True)
+        return (path,)
+

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "added_tokens_decoder": {
+    "256": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "257": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "258": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_byte.ByteTokenizer",
+      null
+    ]
+  },
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 2048,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "split_special_tokens": true,
+  "tokenizer_class": "ByteTokenizer",
+  "unk_token": null
+}

trainer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21aa2cccd888f3918a44b02c1ad179a74c9924c4304886988affaf97c4132422
+size 340197

training_config.json

@@ -0,0 +1,75 @@
+{
+  "model_repo": "appvoid/void-byte",
+  "data_repo": "appvoid/void-byte-data",
+  "run_name": "void-byte-v1",
+  "context": 2048,
+  "global_batch": 90,
+  "micro_batch": 86,
+  "microbatch_candidates": [
+    8,
+    16,
+    32,
+    48,
+    64,
+    72,
+    76,
+    80,
+    86,
+    90
+  ],
+  "batch_selection": "max_batch",
+  "allow_global_batch_change": true,
+  "max_update_retries": 16,
+  "fp32_micro_batch": 2,
+  "min_recovery_lr_factor": 0.015625,
+  "eval_generation_families": 12,
+  "eval_max_new_tokens": 192,
+  "mix": {
+    "climbmix": 40,
+    "ultra_qa": 15,
+    "ultra_style": 15,
+    "cortex": 15,
+    "openmath": 10,
+    "rewrite6": 5
+  },
+  "lr": 0.0003,
+  "warmup_steps": 1000,
+  "max_steps": null,
+  "min_lr_ratio": 0.1,
+  "weight_decay": 0.1,
+  "grad_clip": 1.0,
+  "seed": 20260925,
+  "save_every": 5000,
+  "eval_every": 5000,
+  "log_every": 100,
+  "gradient_checkpointing": false,
+  "cpu_threads": 4,
+  "page_rows": 4096,
+  "cortex_page_rows": 128,
+  "prefetch_batches": 65,
+  "prefetch_low_water": 32,
+  "memory_pages": 12,
+  "cache_gb": 20,
+  "force_rebenchmark": false,
+  "scheduler_benchmark_batches": 8,
+  "scheduler_buffer_seconds": 60,
+  "scheduler_max_prefetch_gb": 1.0,
+  "scheduler_max_batches": 512,
+  "scheduler_safety": 1.5,
+  "scheduler_update_every": 100,
+  "shard_target_mib": 64,
+  "data_upload_pages": 32,
+  "data_upload_seconds": 120,
+  "data_upload_max_pending": 256,
+  "metrics_upload_every": 100,
+  "compile_model": true,
+  "benchmark_compile": true,
+  "autotune_microbatch": true,
+  "attention_backend": "flash",
+  "vram_fraction": 0.95,
+  "upload": true,
+  "hub_resume": true,
+  "work_dir": "/marimo/qwen90m_workspace/runs/void-byte-v1",
+  "data_recipe": "c0b0679d54277afcf4c85726fd0998265697a759188400782be4a859641ba658",
+  "trainer_recipe": "bc45094ca6fe6129cd1201f38cc5978b80014fa21be8d01ea684cb56d8d1644b"
+}