Publish OpenSML-150M release

CLEAN_DOWNLOAD_VERIFICATION.json

@@ -0,0 +1,26 @@
+{
+  "repository": "wzebrowski/OpenSML-150M",
+  "revision": "27ba9cd101492c0dbfeb6cea0147e429a86d8648",
+  "download": "Fresh force-download of 17 release files with no authentication",
+  "hashes": "All SHA256SUMS entries verified",
+  "working_directory": "/private/tmp (outside original project)",
+  "environment": "Separate temporary virtual environment; published requirements installed from PyPI",
+  "mlx": "0.32.2",
+  "mlx-metal": "0.32.2",
+  "tokenizers": "0.22.2",
+  "generation": {
+    "text": " I am a student.",
+    "token_ids": [
+      345,
+      768,
+      261,
+      2560,
+      17,
+      1
+    ],
+    "stop_reason": "eos",
+    "prompt_tokens": 13,
+    "generated_tokens": 6
+  },
+  "scope": "Strict loading and short inference test; no benchmark rerun or broad quality claim."
+}

INFERENCE_VERIFICATION.json

@@ -0,0 +1,29 @@
+{
+  "model_name": "OpenSML-150M",
+  "weights_sha256": "cbd3e3fb4ada74d7264371b79cee7598f513b7b950f1f141ef9f1a43bd2e1b4e",
+  "strict_loading": "passed",
+  "prompt": "Say hello in one sentence.",
+  "generation": {
+    "text": " I am a student.",
+    "token_ids": [
+      345,
+      768,
+      261,
+      2560,
+      17,
+      1
+    ],
+    "stop_reason": "eos",
+    "prompt_tokens": 13,
+    "generated_tokens": 6
+  },
+  "precision": "FP32",
+  "attention": "vanilla",
+  "ffn": "reference",
+  "scope": "Short local loading/generation check; no benchmark rerun or cross-framework parity claim.",
+  "parameter_count": 150439168,
+  "tested_mlx_runtime": "0.32.3.dev20260920+7241f12",
+  "cached_full_prefix_argmax": "matched on checked prompt",
+  "context_overflow": "rejected",
+  "dependency_scope": "PyPI requirements are a baseline; smoke check used the recorded local development runtime."
+}

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,282 @@
+---
+license: apache-2.0
+datasets:
+- HuggingFaceTB/smollm-corpus
+- HuggingFaceTB/dclm-edu
+- HuggingFaceFW/finewiki
+- HuggingFaceTB/smol-smoltalk
+- HuggingFaceH4/ultrachat_200k
+- rajpurkar/squad_v2
+- allenai/ai2_arc
+- databricks/databricks-dolly-15k
+- allenai/tulu-3-sft-personas-instruction-following
+- HuggingFaceTB/smoltalk
+- allenai/sciq
+language:
+- en
+pipeline_tag: text-generation
+tags:
+- text-generation
+- mlx
+- pretraining
+- from-scratch
+- small-language-model
+- post-training
+- silicon
+
+---
+
+
+# OpenSML-150M
+
+
+**Author:** William Zebrowski · **Checkpoint:** OpenSML-150M Instruct · **Status:** research preview
+
+
+[Technical report](https://github.com/williamzebrowskI/opensml-150m/blob/main/sml-mlx-v1/docs/TECHNICAL_REPORT.md) · [Results and provenance](TRIAL28_RESULTS.json) · [Tokenizer](tokenizer/tokenizer.json)
+
+
+An English-first language model trained from scratch with Apple's MLX framework.
+Pretraining processed **7.800B tokens** across Stage A and Stage B on a four-Mac
+cluster, later expanded to five Macs using Thunderbolt RDMA. Three supervised
+fine-tuning stages produced the selected OpenSML-150M checkpoint.
+
+
+## Availability and Inference
+
+
+The selected **OpenSML-150M** weights, frozen tokenizer, and standalone native MLX inference code are included. The **OpenSML-150M Instruct** checkpoint contains the selected fine-tuned FP32 weights; optimizer state is excluded. Its training-step identity is retained in the checkpoint provenance.
+
+
+Use Python 3.11+ on Apple Silicon. Install the Hugging Face CLI first and authenticate if this repository is private:
+
+
+```bash
+hf download wzebrowski/OpenSML-150M --local-dir ./OpenSML-150M
+cd OpenSML-150M
+python -m pip install -r requirements.txt
+python inference.py --prompt "Say hello in one sentence." --max-new-tokens 32
+```
+
+
+The CLI verifies the weight hash and tokenizer manifest, then uses greedy FP32 reference MLX inference. Prompts use `User: {prompt}\nAssistant:`; `--raw-completion` skips this wrapper. The 2,048-token context includes the requested generation budget; overflow is rejected. Output includes text, generated token IDs, and the stop reason. Native loading and a short generation were verified locally; this is **not a Transformers AutoModel or mlx-lm loader package**. Other formats and cross-framework parity remain unverified.
+
+
+See [checkpoint provenance](checkpoint_provenance.json), [loading verification](INFERENCE_VERIFICATION.json), and [bundle hashes](SHA256SUMS).
+
+
+## Model Specification
+
+
+| Field | Value |
+| --- | --- |
+| Parameters (model-card label) | 150,439,188 (150.44M) |
+| Layers / hidden width / FFN width | 20 / 768 / 2,048 |
+| Attention | GQA: 12 query heads / 4 KV heads; head dimension 64 |
+| Position encoding | RoPE, base 10,000 |
+| Normalization / MLP | RMSNorm, Q/K normalization, SwiGLU |
+| Vocabulary / context | 32,000 / 2,048 |
+| Embeddings | Tied input and output |
+| Linear biases / dropout | Neither used |
+| Pretraining precision | BF16 compute; FP32 master weights and optimizer state |
+| Benchmark precision | FP32 scoring, explicit vanilla MLX attention |
+
+
+The independently fitted tokenizer is a 32,000-token byte-level BPE. Its recorded
+held-out audit had zero round-trip failures. Full configuration, parameter
+accounting, and tokenizer checks are in the [technical report](https://github.com/williamzebrowskI/opensml-150m/blob/main/sml-mlx-v1/docs/TECHNICAL_REPORT.md).
+
+
+## Training and Model Lineage
+
+
+The selected pretrained base is **Stage B step 73,243**, after **7,800,086,528
+lifetime pretraining tokens**. Stage B continued Stage A with the adjusted mixture below.
+Percentages are configured token shares.
+
+
+| Source | Initial token share | Stage B token share | Selection |
+| --- | ---: | ---: | --- |
+| [FineWeb-Edu](https://huggingface.co/datasets/HuggingFaceTB/smollm-corpus) | 55% | 55% | `fineweb-edu-dedup` |
+| [DCLM-Edu](https://huggingface.co/datasets/HuggingFaceTB/dclm-edu) | 25% | 20% | `edu_int_score >= 3` |
+| [FineWiki](https://huggingface.co/datasets/HuggingFaceFW/finewiki) | 10% | 15% | English |
+| [Cosmopedia v2](https://huggingface.co/datasets/HuggingFaceTB/smollm-corpus) | 10% | 10% | Textbook/tutorial/blog/educational format filter |
+
+
+FineWeb-Edu and Cosmopedia v2 come from two subsets of the same SmolLM Corpus
+repository. Pretraining used no dedicated code or math dataset.
+
+
+```text
+V1 pretrained base → Unified384 → Repair512 → OpenSML-150M (SFT step 768)
+                      +384         +128       +256 updates
+```
+
+
+Unified384 used Smol-SmolTalk, UltraChat, SQuAD v2, ARC training questions, and
+locally authored follow-ups. Repair512 added Dolly and Tulu Persona instruction
+examples. The final stage used SmolTalk constraints, SQuAD v2, SciQ, and Repair512
+replay. All model parameters were updated; the selected checkpoint is a direct
+continuation, without LoRA or parameter interpolation. Dataset exposure, source
+pins, optimizer schedules, and checkpoint hashes are in the technical report.
+
+
+### Pretraining Loss Curves
+
+
+![Recorded pretraining validation loss for Stage A and Stage B](pretraining-overview.svg)
+
+
+Recorded held-out loss without smoothing. The star marks the selected base;
+later points are outside its training exposure. The late-training panel uses a
+magnified scale. This chart uses the same original validation set throughout.
+
+
+<details>
+<summary>Stage B detail and per-source curves</summary>
+
+
+![Stage B validation loss on the expanded held-out set](pretraining-phase-b.svg)
+
+
+This panel uses a different, expanded validation set; its absolute losses should
+not be merged with the overview. The selected base has original-set loss
+2.81855024 and expanded-set loss 2.76700753.
+
+
+![Original-set validation loss by data source](pretraining-sources.svg)
+
+
+Per-source panels use separate vertical scales. Full validation accounting and
+historical measurements are in the technical report.
+
+
+</details>
+
+
+## Evaluation
+
+
+### Zero-shot Likelihood Benchmarks
+
+
+Existing full-split evaluations cover **15,428 multiple-choice examples**.
+The pretrained base and selected SFT model are reported separately.
+
+
+| Benchmark | Split | Examples | Pretrained base acc / acc_norm | OpenSML-150M acc / acc_norm |
+| --- | --- | ---: | ---: | ---: |
+| ARC-Easy | Test | 2,376 | 54.67% / 48.53% | 56.65% / 55.43% |
+| ARC-Challenge | Test | 1,172 | 23.38% / 26.88% | 26.02% / 29.52% |
+| PIQA | Validation | 1,838 | 65.23% / 64.36% | 64.53% / 64.09% |
+| HellaSwag | Validation | 10,042 | 30.21% / 34.44% | 30.55% / 33.94% |
+
+
+SFT improves both ARC metrics. PIQA declines slightly; HellaSwag raw accuracy
+rises slightly while normalized accuracy declines.
+
+
+<details>
+<summary>Likelihood prompts and scoring</summary>
+
+
+The native evaluator uses zero-shot raw completion prompts, FP32 scoring,
+reference MLX attention, batch size 1, and no chat template or added BOS/EOS.
+`acc` ranks summed candidate log-likelihood; `acc_norm` divides by candidate
+Unicode character count, excluding the leading delimiter. No generation or LLM
+judge is involved. The evaluator follows pinned harness conventions but is not
+an installed lm-evaluation-harness run. Dataset pins and integrity receipts are
+in the technical report and results record.
+
+
+</details>
+
+
+### Comparison with Small Language Models
+
+
+External scores below are **previously published measurements**. Each cell shows
+`acc / acc_norm`; **—** means not reported in the selected source. Bold marks the
+highest available reported value.
+
+
+| Model | ARC-Easy acc / acc_norm | ARC-Challenge acc / acc_norm | PIQA acc / acc_norm | HellaSwag acc / acc_norm |
+| --- | ---: | ---: | ---: | ---: |
+| **OpenSML-150M (SFT)** | **56.65% / 55.43%** | **26.02% / 29.52%** | **64.53% / 64.09%** | **30.55% / 33.94%** |
+| [GPT-2 (124M, base)](https://huggingface.co/openai-community/gpt2) | — / 39.48% | — / — | — / 62.51% | 28.92% / 31.14% |
+| [OPT-125M (base)](https://huggingface.co/facebook/opt-125m) | 43.52% / 39.98% | 18.94% / 22.78% | 63.00% / 62.02% | — / — |
+| [Pythia-160M (base)](https://huggingface.co/EleutherAI/pythia-160m) | 43.52% / 39.65% | 18.77% / 23.29% | 62.73% / 61.64% | — / — |
+
+
+OpenSML's largest reported advantage is normalized ARC-Easy: +15.45 percentage
+points over OPT-125M and +15.78 over Pythia-160M. These are cross-source
+comparisons, not a controlled ranking: prompts, normalization, numerical settings,
+and dataset revisions are not verified identical. OpenSML is SFT-trained; the
+references are base models. Source records are in the technical report.
+
+
+### Instruction Following: IFEval
+
+
+| Checkpoint | Prompt strict | Instruction strict | Prompt loose | Instruction loose | 1,280-token cap hits |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| OpenSML-150M | 15.16% (82/541) | 25.30% (211/834) | 15.71% (85/541) | 25.78% (215/834) | 33/541 |
+
+
+These results apply to OpenSML-150M, not the pretrained base. All 541 prompts and
+834 instructions were scored using programmatic strict/loose verifiers, without
+an LLM judge.
+
+
+<details>
+<summary>IFEval formatting, generation, and scoring</summary>
+
+
+Generation is zero-shot and greedy, with plain `User: {prompt}\nAssistant:`
+formatting, no added system message, and a 1,280-new-token cap. The context is
+2,048 tokens; document-end EOS is ID 1. No additional stop strings or repetition
+penalty are used. All responses are scored as produced: 508 stop at EOS and 33
+reach the generation cap; none hit the context limit. The dataset's only split
+is named `train`, but these prompts are evaluation inputs, not supervised records.
+
+
+</details>
+
+
+### Complete-answer Diagnostics
+
+
+Saved development checks report a follow-up joint proxy of 22/32 and named-constraint
+passes on 5/32 public cases. Repetition was detected on 4/63 legacy turns and
+18/73 public-development turns. These are mechanical development proxies;
+a systematic independent review of complete-answer correctness is still pending.
+Likelihood and IFEval do not establish reliable factual answers. No MT-Bench or
+coding-success score is reported for the selected model.
+
+
+## Intended Use and Limitations
+
+
+Intended for small-language-model research and controlled experimentation.
+Responses may be incorrect, repetitive, biased, or unsafe. Production use,
+safety behavior, tool use, multilingual ability, and extended dialogue remain
+unvalidated. Benchmark reuse during selection limits evaluation independence;
+a comprehensive contamination audit has not been established.
+
+
+## Checkpoint Verification
+
+
+Saved hashes and integrity receipts identify the selected weights and completed
+evaluations. They do not establish cross-framework inference parity. An unchanged native weight export and standalone inference CLI are included.
+The export has a local loading/generation check; a full benchmark rerun of this download bundle has not been performed. See the
+[results and provenance record](TRIAL28_RESULTS.json) and technical report.
+
+
+## Licensing and Data Provenance
+
+
+William Zebrowski releases the OpenSML-150M model weights, tokenizer and author-controlled project contributions under [Apache-2.0](LICENSE). Anyone may use, modify, fine-tune and redistribute these contributions for personal, research or commercial purposes, subject to the license terms.
+
+This grant covers rights held by the author and does not relicense third-party training materials. Upstream materials retain their own terms, including SQuAD v2 (CC-BY-SA-4.0) and SciQ (CC-BY-NC-3.0). The model license does not grant blanket commercial rights to those datasets.
+

SHA256SUMS

@@ -0,0 +1,16 @@
+6050506f4e9411f7720754591436ec5477ee04d8c136f69dedb5365280d73a2e  INFERENCE_VERIFICATION.json
+cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30  LICENSE
+7edb9a16eaf1ef9232bf5eb3b82dd6a6e5df36fe63108d1525bce00777025bdc  README.md
+b5483cebea706f876ff0deddf843e046e0e963ff76e6d97b18d521da03ebeae5  checkpoint_provenance.json
+6d0fc4cb42c9fafb832f57153842bdb71cee5d72678fd2e4cb43b6f3c881604b  config.json
+e4c47b7b6aba08f948de7c2c7cb94c58dbd09dde2ded6ac185c60856ee686d27  corpus.json
+f53e9934693d5513a3500c4ba31462b406b4537cf23d37bfc803a962c3b94827  inference.py
+1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb  manifest.json
+cbd3e3fb4ada74d7264371b79cee7598f513b7b950f1f141ef9f1a43bd2e1b4e  model.safetensors
+1eee6f8afe78767ff6b9c604ccf2328c98b24bc94dbad6291a0ad699dd7345fe  native_model.py
+a4b98c7860e40509313e719e3cd235dbf7afa7b81965d217af57416e5ff7c9bb  native_tokenizer.py
+2be27ef4bac5f2490b1faaf12e9487eda46356225be220139939459f5105e7af  native_utils.py
+e871f690eabebf82da173f0e406f38b79a10cd7d2647bb3592c3fc35ae9a2f16  report.json
+2d5c8eddf30346ebeffdc25fc07552d0e8a9789e967f89e7ff53dc525c7ce569  requirements.txt
+e095f0413a11fa91fbb58f37d00e4c6bcda59beb89b28f6f680db1436a7e8891  tokenizer.json
+619f500553367eb76f5be00a98625130ec857b9c367561334984096c029ba2a9  tokenizer_config.json

TECHNICAL_REPORT.md

@@ -0,0 +1,1356 @@
+# OpenSML-150M — Technical Report
+
+William Zebrowski · V2 training and selected post-training checkpoint · Updated October 2, 2026
+
+This report is the detailed companion to the OpenSML-150M model card. The current
+record below covers the V2 base at Stage B step 73,243 (7,800,086,528 pretraining
+tokens) and the selected SFT checkpoint at total step 768. The September 18
+report is preserved in a clearly dated historical appendix; its training status,
+hardware, and unfinished-evaluation statements describe that earlier boundary.
+Current selection and release statements below take precedence.
+
+## Parameter-count accounting note
+
+The owner-requested model-card wording reports 150,439,188 parameters (150.44M).
+The existing architecture accounting and configuration records total 150,439,168.
+This 20-parameter discrepancy is unresolved; the presentation edit was not a
+model-weight change. The original component accounting is preserved in the
+historical appendix.
+
+## Current V2 Training and Evaluation Record
+
+## Availability and Inference
+
+This repository currently documents the selected fine-tuned model and its pretrained
+base, and provides their shared tokenizer
+and historical configuration files. It does not provide model weights or a tested
+standalone inference bundle. A download-and-generate quick start is therefore not
+yet available. The local project uses native MLX inference on Apple hardware;
+compatibility with Transformers AutoModel, GGUF, ONNX, vLLM, and llama.cpp has not
+been established for this repository. The selected SFT checkpoint uses the native plain User/Assistant format described
+below; the pretrained base remains a completion model.
+
+## At a Glance
+
+| Item | Configuration |
+| --- | --- |
+| Model type | Decoder-only language model; selected instruction-tuned checkpoint with base results separately reported |
+| Initialization | Random weights; no pretrained model initialization |
+| Parameters | 150,439,168 |
+| Backbone | 20 layers, width 768 |
+| Context | 2,048 tokens |
+| Tokenizer | Independently trained 32,000-token byte-level BPE |
+| Training framework | MLX, with custom training and cluster orchestration |
+| Five-Mac cluster | One M5 Ultra, one M3 Ultra, and three M4 Max machines |
+| Communication | Native JACCL over Thunderbolt RDMA; four-node ring initially, five-node continuation |
+| Selected pretrained base | Step 73,243; 7,800,086,528 lifetime pretraining tokens |
+| Original exposure target | 15B total tokens; a plan, not the completed exposure of the selected base |
+| Selected fine-tuned model | OpenSML-150M; total SFT step 768 |
+| Current status | Base and selected SFT checkpoint documented; weight release pending |
+
+## Model Specification
+
+| Field | Value |
+| --- | --- |
+| Unique parameters | 150,439,168 |
+| Layers / hidden width / FFN width | 20 / 768 / 2,048 |
+| Attention | GQA: 12 query heads / 4 KV heads; head dimension 64 |
+| Position encoding | RoPE, base 10,000 |
+| Normalization / MLP | RMSNorm, Q/K normalization, SwiGLU |
+| Vocabulary / context | 32,000 / 2,048 |
+| Embeddings | Tied input and output |
+| Linear biases / dropout | Neither used |
+| Pretraining precision | BF16 compute; FP32 master weights and optimizer state |
+| Benchmark precision | FP32 scoring, explicit vanilla MLX attention |
+
+### Architecture
+
+The model uses grouped-query attention with 12 query heads and 4 key/value heads
+(head dimension 64), rotary position embeddings with base 10,000, RMSNorm, Q/K
+normalization, and SwiGLU feed-forward layers with intermediate width 2,048.
+Input and output embeddings are tied; linear layers are bias-free.
+
+The tied vocabulary matrix contains 24,576,000 parameters. Initialization uses
+normal standard deviation 0.02 with residual output projections scaled by
+1/sqrt(2L). These are established Transformer components in our configuration,
+not a claim of a new attention architecture.
+
+## Pretraining Data
+
+The model uses a prose-first mixture streamed from Hugging Face. This replaces
+the earlier OpenWebText description on this page; that description does not apply
+to OpenSML-150M.
+
+| Source | Initial token share | Stage B token share | Selection |
+| --- | ---: | ---: | --- |
+| [FineWeb-Edu](https://huggingface.co/datasets/HuggingFaceTB/smollm-corpus) | 55% | 55% | `fineweb-edu-dedup` |
+| [DCLM-Edu](https://huggingface.co/datasets/HuggingFaceTB/dclm-edu) | 25% | 20% | `edu_int_score >= 3` |
+| [FineWiki](https://huggingface.co/datasets/HuggingFaceFW/finewiki) | 10% | 15% | English |
+| [Cosmopedia v2](https://huggingface.co/datasets/HuggingFaceTB/smollm-corpus) | 10% | 10% | Textbook/tutorial/blog/educational format filter |
+
+The [original corpus recipe](https://huggingface.co/wzebrowski/OpenSML-150M/blob/main/configs/corpus.json) records the initial mixture.
+Stage B began at step 73,008, after 7,775,059,968 tokens, and changed the mixture
+as shown above. No dedicated code or math source was included in this prose
+pretraining mixture; prose can still contain technical material, equations,
+and code.
+
+Pretraining uses three dataset repositories: FineWeb-Edu and Cosmopedia
+v2 are two distinct subsets of SmolLM Corpus. All four pretraining sources are
+listed above. Dataset tags also include the public sources used across the
+selected checkpoint's SFT lineage, described below.
+
+Normalized document hashes assign 97% to training and 1% each to tokenizer
+development, validation, and a reserved final split. Original text is preserved
+for model input. A bounded window of 10,000 recent accepted hashes suppresses
+nearby exact duplicates across sources; it is not complete corpus deduplication
+or a benchmark contamination audit.
+
+Early training read each source in dataset order without a document shuffle
+buffer. From step 70,710 (7,530,332,160 tokens), the data-review continuation
+introduced a resumable 256-document shuffle buffer per source. This is bounded
+shuffling, not a global random permutation. Documents receive one explicit EOS
+and are packed into causal windows that can cross document boundaries;
+attention is not reset at every document.
+
+Mac-1 handles filtering, tokenization, and RAM prefetch, then distributes disjoint
+rank slices. The training path does not save corpus/token shards to disk.
+Checkpoint cursors, tokenizer-fitting samples, logs, staged files, and small HF
+metadata caches are separate storage uses. Exact-resume state tracks consumed
+batches, not the prefetch worker's advanced position.
+
+## Tokenizer
+
+**ByteBPE32K v1 is completed and frozen.** It was fitted on our source mixture,
+not borrowed from another model. Tokenizer-fitting shares are measured in UTF-8
+bytes; pretraining shares are measured in tokens.
+
+| Audit item | Recorded result |
+| --- | --- |
+| Fitting sample | 500,048,186 UTF-8 bytes; 113,898 documents |
+| Held-out audit | 4,010,605 bytes; 970 documents |
+| Vocabulary | Exactly 32,000 IDs including structural tokens |
+| Round-trip failures | Zero across the held-out audit |
+| Regression fixtures | 9 passed |
+| Special IDs | Padding 0; document-end 1; turn-start 2; turn-end 3 |
+| Encoding | Full byte alphabet; no text normalization or implicit BOS/EOS |
+
+Literal control-token spellings encode as ordinary text. Reserved turn tokens
+do not mean the base model has been chat-trained. The selected SFT model uses
+plain role labels and document-end EOS rather than relying on those reserved turn tokens. Held-out bytes/token were
+4.6255 for web, 4.4532 for DCLM, 4.0593 for wiki, and 5.0610 for Cosmopedia.
+These are encoding checks, not model capability results.
+
+Files: [tokenizer](https://huggingface.co/wzebrowski/OpenSML-150M/blob/main/tokenizer/tokenizer.json),
+[configuration](https://huggingface.co/wzebrowski/OpenSML-150M/blob/main/tokenizer/tokenizer_config.json),
+[manifest](https://huggingface.co/wzebrowski/OpenSML-150M/blob/main/tokenizer/manifest.json), and [audit](https://huggingface.co/wzebrowski/OpenSML-150M/blob/main/tokenizer/report.json).
+The artifact is provided with its structural-token contract; standard text
+encoding alone does not reproduce every custom trainer boundary rule.
+
+```text
+Tokenizer artifact fingerprint:
+1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb
+```
+
+## Training Method
+
+| Setting | Value |
+| --- | --- |
+| Optimizer | AdamW; betas 0.9/0.95; epsilon 1e-8 |
+| Precision | BF16 compute; FP32 master weights, moments, and accumulated/reduced gradients |
+| Regularization | Matrix weight decay 0.1; gradient norm clip 1.0 |
+| Local microbatches | Initially 4 / 3 / 3 / 3; five-Mac continuation 4 / 3 / 2 / 2 / 2 sequences |
+| Gradient accumulation | 4 |
+| Global update | 52 sequences; 106,496 tokens |
+| Learning rate | Initial peak 3e-4; later validation-plateau schedule, then 5e-6 floor; selected Stage B base used 5e-6 |
+| Validation / samples | Every 25M / 50M tokens |
+| Routine checkpoint saves | Every 10M tokens, plus best/final/clean-stop saves |
+
+### Four-to-five-Mac training history
+
+The original cluster used one M3 Ultra and three M4 Max Mac Studios. At step
+**50,096**, after **5,335,023,616 tokens**, training continued with an M5 Ultra
+coordinator added to form a five-Mac ring. This transition is recorded in the
+selected pretrained checkpoint's provenance; the model was not trained on
+five Macs from initialization.
+
+| Machine | Unified memory | Local microbatch | Tokens per global update |
+| --- | ---: | ---: | ---: |
+| M5 Ultra | 96 GB | 4 | 32,768 |
+| M3 Ultra | 96 GB | 3 | 24,576 |
+| M4 Max | 36 GB | 2 | 16,384 |
+| M4 Max | 36 GB | 2 | 16,384 |
+| M4 Max | 36 GB | 2 | 16,384 |
+
+The physical ring is **Mac-1 → Mac-2 → Mac-3 → Mac-5 → Mac-4 → Mac-1**.
+This is replicated data parallelism: each Mac holds the complete model and
+optimizer, processes disjoint examples, combines gradients with batch-size
+weighting, and applies the same update. It is not a model split across five
+machines. Mac-1 streams/tokenizes the corpus, distributes batches, evaluates,
+and saves checkpoints.
+
+Native JACCL over Thunderbolt RDMA carries tensors. Ethernet/SSH and
+job-scoped coordinator tunnels handle control traffic. With context 2,048 and
+four accumulation steps, the five-Mac allocation preserves the earlier effective
+batch: `(4 + 3 + 2 + 2 + 2) × 4 × 2048 = 106,496` tokens per update.
+The transition preserves architecture, tokenizer and optimizer state. No
+controlled four-versus-five-Mac throughput comparison is claimed.
+
+The implementation uses compiled backward computation, fast attention, custom
+Metal cross-entropy, and packed SwiGLU. Their presence is not, by itself, evidence
+of a measured speedup for each individual optimization.
+
+### Pilot and Continuation
+
+The pilot finished at step 4,696 after 500,105,216 tokens. It used token-based
+warmup, a stable phase, and a final cosine decay to 3e-5. The initial continuation restored
+its weights, optimizer and consumed data cursor, reheated to 3e-4 over 50M
+additional tokens, and originally targeted 15B total tokens. Later training
+changed schedules through explicit checkpoint transitions: a validation-plateau
+controller at step 38,056; a lower-LR continuation at step 60,263; and the
+shuffle/validation transition at step 70,710. Stage B began from step 73,008
+at 5e-6. The selected Stage B base is step **73,243**, with **7,800,086,528**
+lifetime pretraining tokens.
+
+This is **not a single unchanged 15B schedule from initialization**, and the
+15B target must not be read as the training exposure of these checkpoints.
+The linked [pilot](https://huggingface.co/wzebrowski/OpenSML-150M/blob/main/configs/pilot.json) and [early continuation configuration](https://huggingface.co/wzebrowski/OpenSML-150M/blob/main/configs/training_recipe.json)
+are historical recipes; the later transitions are summarized here.
+
+## Training and Model Lineage
+
+The selected fine-tuned model is **OpenSML-150M**, retained by the
+project owner as the current baseline. Its internal experiment identifier is
+Trial 28 — Repair512 +256; that identifier remains in provenance records only. Its ancestry is:
+
+```text
+Pretrained V2: Stage B step 73,243 (7.800B pretraining tokens)
+  → Unified384: 384 supervised updates from the base
+  → Repair512: 128 additional public-repair updates
+  → OpenSML-150M: 256 additional updates; total SFT step 768
+```
+
+These SFT labels are update counts, not pretraining steps or token counts.
+This step 768 is distinct from the earlier Text Follow-up 768 branch. The model
+is a direct fine-tuned checkpoint, without parameter interpolation. Other search
+candidates are separate experiments and are not ancestors of these weights.
+
+### Fine-tuning Data and Settings
+
+| Stage | Data and method | Retained exposure |
+| --- | --- | --- |
+| Unified384 | Smol-SmolTalk and UltraChat conversations, writing/summarization, authored recovery/format follow-ups, SQuAD v2 passage reading, and ARC training questions; assistant-only supervised learning | First 384 updates of the 768-update recipe |
+| Repair512 | Short Smol-SmolTalk/UltraChat chats, Dolly ordinary and grounded answers, and verified Tulu 3 Persona instruction examples; assistant-only supervised learning | 128 additional updates from Unified384 |
+| OpenSML-150M | SmolTalk constraints, SQuAD v2 training passages, SciQ training labels, and Repair512 replay; supervised learning with repetition and retention objectives | 256 additional updates; first 4,096 ordered training records |
+
+Trial 28 prepared an 8,192-record, 512-update experiment. Its retained checkpoint
+is the **+256 midpoint**, not the final +512 model. The full prepared mixture was
+3,072 SmolTalk constraint records, 2,048 SQuAD records, 1,024 SciQ records, and
+2,048 replay records, including repeated passes through fresh examples. Those
+full-plan counts must not be interpreted as the retained midpoint's exposure.
+
+Trial 28 updates all model parameters, including tied embeddings; no LoRA or
+frozen-layer restriction is used. It retains the architecture, context length,
+and tokenizer of the base. Batches contain 16 whole conversations. Supervised
+cross-entropy weights conversations equally and trains assistant targets plus
+EOS while masking prompt/history positions. Targets are retained whole; records
+that exceed length/context limits are excluded.
+
+The Trial 28 optimizer uses AdamW, betas 0.9/0.95, zero weight decay, clipping at
+norm 1, and a 16-update warmup to 2e-6 followed by cosine decay toward 2e-7 over
+the planned 512 updates. The selected midpoint therefore stops partway through
+that schedule. The objective adds a selective repetition unlikelihood term
+(weight 0.3) and a frozen-Repair512 forward-KL retention term (weight 2), restricted
+to replay examples. This is supervised post-training with auxiliary objectives;
+it does not use DPO or a reward-model policy-optimization stage in this lineage.
+
+Trial 28 source revisions are SmolTalk `5feaf2fd3ffca7c237fc38d1861bc30365d48ffa`,
+SQuAD v2 `3ffb306f725f7d2ce8394bc1873b24868140c412`, and SciQ
+`2c94ad3e1aafab77146f384e23536f97a4849815`.
+Training preparation excludes known benchmark overlaps and separates development
+and reserved records, but this is not a comprehensive contamination audit.
+Public targets have not all received independent factual review.
+
+Selection followed repeated development and public-benchmark comparisons.
+Trial 28 passed the recorded benchmark gate but failed the chat-repetition
+component of the development gate; the owner explicitly retained it. It is the
+**selected current baseline**, not a statistically established best model across
+all capabilities. Reused benchmarks are exploratory development evidence.
+
+## Pretraining Loss Curves
+
+These charts show recorded held-out next-token validation loss from the selected
+pretraining lineage, without smoothing. **Blue is Phase A** (the initial mixture
+and its continuations); **teal is Phase B** (the Stage B prose refinement).
+Phase B starts at step **73,008**, after **7.775B tokens**. The star marks the
+selected pretrained base at step **73,243**, after **7.800B tokens**. Later points
+show continued pretraining beyond the selected checkpoint, not additional
+training included in that checkpoint. Lower loss is better.
+
+### Full History and Late-training Zoom
+
+![OpenSML pretraining validation loss: full Phase A and Phase B history with a late-training zoom](https://huggingface.co/wzebrowski/OpenSML-150M/resolve/main/pretraining-overview.svg)
+
+The overview and zoom use the **same original held-out set** throughout
+(262,144 target positions). The zoom makes the shorter Phase B visible; its
+vertical scale is magnified. Aggregate validation weights remain fixed at
+55% web, 25% DCLM, 10% Wikipedia, and 10% Cosmopedia across both phases,
+even though the Phase B training mixture changes. The selected checkpoint's
+loss on this original set is **2.81855024**.
+
+### Phase B Detail
+
+![OpenSML Phase B validation loss on the expanded held-out set, with the selected checkpoint marked](https://huggingface.co/wzebrowski/OpenSML-150M/resolve/main/pretraining-phase-b.svg)
+
+This close-up uses the **expanded held-out set** (4,194,304 target positions),
+introduced late in Phase A. Its absolute loss values are separate from the
+original-set curves above and below. The selected checkpoint's expanded loss
+is **2.76700753**. The tightly zoomed vertical scale shows small changes; the
+later increase is less than 0.001 loss. Validation weights remain fixed to the
+original reference mixture.
+
+### Validation Loss by Data Source
+
+![OpenSML original-set validation loss for web, DCLM, Wikipedia, and Cosmopedia across both pretraining phases](https://huggingface.co/wzebrowski/OpenSML-150M/resolve/main/pretraining-sources.svg)
+
+Each source is evaluated on the original held-out set throughout. Panels use
+their own vertical scales; they show each source's trajectory rather than a
+common-scale ranking of source difficulty. The dashed line marks the Phase B
+transition and the star marks the selected base.
+
+## Interim Results
+
+**Dated snapshot: September 18, 2026, through step 8,217. Not a live counter.**
+
+| Selected milestone | Total tokens | Validation loss |
+| --- | ---: | ---: |
+| Pilot endpoint, step 4,696 | 500,105,216 | 3.3536 |
+| Near end of continuation rewarm, step 5,165 | 550,051,840 | 3.4327 |
+| Best at this snapshot, step 8,217 | 875,077,632 | 3.3078 |
+
+These are selected milestones, not a complete learning curve. The loss rise and
+recovery do not isolate the causal effect of reheating from additional training.
+
+At step 8,217, per-source losses were web **3.4116**, DCLM **3.6108**, wiki
+**2.7977**, and Cosmopedia **2.4896**. Evaluation uses 16 batches of two sequences
+per source: 262,144 target positions in total. The aggregate applies the training
+mixture weights. Pilot and continuation preserve the same held-out fingerprint.
+
+One earlier production readout at step 7,980 reported 13,794 interval tokens/sec
+and 13,578 run tokens/sec across the cluster. This is an observed counter, not a
+controlled performance benchmark, utilization measurement, or total runtime.
+
+## Selected Pretrained Base
+
+The selected pretrained checkpoint is the preserved Stage B base at
+step **73,243**, after **7,800,086,528 pretraining tokens**. Its original legacy
+validation loss is **2.81855024**. Its expanded validation loss is **2.76700753**.
+These are different evaluation sets and must not be merged into one loss curve.
+The expanded set has 256 batches of two sequences per source, or 4,194,304
+positions in total; the legacy set has 262,144 positions.
+
+The historical September 18 milestones above remain a dated record, not the
+current training counter. Lower next-token loss does not by itself establish
+correct answers, reasoning, or reliable instruction following.
+
+## Evaluation
+
+Results identify either the **pretrained Stage B base at step 73,243** or the
+**selected OpenSML-150M SFT checkpoint at total step 768**. Their identities and
+results are kept separate. The pretrained weight SHA-256 is:
+
+```text
+ca9bd5d82005f28e77f319d3a5a29f29fb4e4f86e7ceea049f01162fd8aae80f
+```
+
+### Zero-shot Likelihood Benchmarks
+
+The complete evaluation scored **15,428 multiple-choice examples**. Higher is
+better; raw and normalized accuracy are separate metrics.
+
+| Benchmark | Split | Examples | Pretrained base acc / acc_norm | OpenSML-150M acc / acc_norm |
+| --- | --- | ---: | ---: | ---: |
+| ARC-Easy | Test | 2,376 | 54.67% / 48.53% | 56.65% / 55.43% |
+| ARC-Challenge | Test | 1,172 | 23.38% / 26.88% | 26.02% / 29.52% |
+| PIQA | Validation | 1,838 | 65.23% / 64.36% | 64.53% / 64.09% |
+| HellaSwag | Validation | 10,042 | 30.21% / 34.44% | 30.55% / 33.94% |
+
+Under the recorded protocol, the selected SFT checkpoint improves both ARC
+metrics over the base. PIQA is slightly lower; HellaSwag raw accuracy increases
+slightly while normalized accuracy decreases. Fine-tuning therefore shows a
+mixed result rather than uniform gains.
+
+Scoring ranks supplied answer continuations by their summed log-likelihood
+(`acc`). `acc_norm` divides that score by the candidate text's Unicode character
+count, rather than token count. The evaluator uses plain task completion prompts,
+no few-shot demonstrations, no chat template, and no added BOS/EOS tokens. It
+loads the selected checkpoint in FP32, uses reference feed-forward operations
+and explicit vanilla MLX attention, and scores one example at a time. No response
+sampling or LLM judge is involved.
+
+The custom evaluator follows the pinned lm-evaluation-harness task conventions
+at revision `b954108c9baaaa934b4ad842033b31a97ee30816`; this is not an installed
+harness run or an official leaderboard submission. All expected rows completed,
+and the saved integrity receipt records unchanged inputs. Different prompt,
+normalization, tokenizer, or numerical settings can change scores. These numbers
+are not automatically comparable with external model cards. A comprehensive
+pretraining contamination audit has not been established, and these are not
+claimed to be untouched final-test results.
+
+### Comparison with Small Language Models
+
+**OpenSML-150M** is compared below with recognizable models in the same approximate
+parameter range: GPT-2 (124M), OPT-125M, and Pythia-160M. All external values are
+**previously published results**. OpenSML values are
+our existing measurements of the selected fine-tuned checkpoint.
+
+Each cell shows **acc / acc_norm**; **—** means the cited source does not report
+that metric. Bold marks the highest available reported value in each column.
+These are cross-source comparisons, not a controlled common-harness ranking.
+
+| Model | ARC-Easy acc / acc_norm | ARC-Challenge acc / acc_norm | PIQA acc / acc_norm | HellaSwag acc / acc_norm |
+| --- | ---: | ---: | ---: | ---: |
+| **OpenSML-150M (SFT)** | **56.65% / 55.43%** | **26.02% / 29.52%** | **64.53% / 64.09%** | **30.55% / 33.94%** |
+| [GPT-2 (124M, base)](https://huggingface.co/openai-community/gpt2) | — / 39.48% | — / — | — / 62.51% | 28.92% / 31.14% |
+| [OPT-125M (base)](https://huggingface.co/facebook/opt-125m) | 43.52% / 39.98% | 18.94% / 22.78% | 63.00% / 62.02% | — / — |
+| [Pythia-160M (base)](https://huggingface.co/EleutherAI/pythia-160m) | 43.52% / 39.65% | 18.77% / 23.29% | 62.73% / 61.64% | — / — |
+
+**The strongest reported margin is ARC-Easy:** OpenSML-150M's normalized score
+is +15.45 percentage points above OPT-125M and +15.78 points above Pythia-160M.
+On normalized ARC-Challenge, the respective differences are +6.74 and +6.23
+points. OpenSML also exceeds the available published PIQA and GPT-2 HellaSwag
+values shown here. These differences describe the reported numbers, not
+statistically established superiority under identical evaluation settings.
+
+OpenSML uses the zero-shot native-MLX protocol described above, with
+character-normalized `acc_norm`. External metrics retain their source definitions;
+we have not verified identical prompts, normalization, precision, or dataset
+revisions across all sources. OpenSML is instruction fine-tuned, whereas the
+three references are pretrained base models. The comparison therefore does not
+isolate architecture, pretraining quality, or instruction-following ability.
+Missing external metrics have not been filled by running new evaluations.
+
+### Instruction Following: IFEval
+
+These results apply only to **OpenSML-150M**, not the pretrained base.
+
+| Checkpoint | Prompt strict | Instruction strict | Prompt loose | Instruction loose | 1,280-token cap hits |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| OpenSML-150M | 15.16% (82/541) | 25.30% (211/834) | 15.71% (85/541) | 25.78% (215/834) | 33/541 |
+
+The evaluation completes all 541 official prompts and 834 instructions using
+zero-shot greedy generation and the pinned programmatic strict/loose verifiers,
+without an LLM judge. Each unchanged prompt is formatted as
+`User: {prompt}\nAssistant:` with no added system message. Generation uses cached
+native MLX decoding, context 2,048, a 1,280-new-token cap, document-end EOS ID 1,
+no additional stop strings, and no repetition penalty. Outputs are scored as
+produced, including length-capped answers. Of the responses, 508 stop at EOS,
+33 reach the generation cap, and none hit the context limit.
+
+The dataset's official split is named `train`; these evaluation examples are
+not used as supervised training records in the documented experiment. Benchmark
+inspection across multiple candidates nevertheless limits independence of model
+selection. No pretrained-base IFEval score is reported here.
+
+### Complete-answer Diagnostics
+
+For OpenSML-150M, saved development generations cover 63 legacy-retention turns
+and 73 public-development turns. The recorded follow-up joint proxy is 22/32;
+5/32 public named-constraint checks pass. Repetition is detected on 4/63 legacy
+turns and 18/73 public turns; in the chat subset it rises from Repair512's 3/18
+to 4/18. These are development diagnostics and mechanical proxies, not
+independent human-reviewed complete-answer correctness rates.
+
+A systematic complete-answer review comparable to PetitGPT's published suite
+has not been established for the selected checkpoint. Likelihood and IFEval
+scores do not establish factual correctness or reliable conversational answers.
+
+### Reading and Fact-sensitivity Diagnostics (Pretrained Base)
+
+The following diagnostics apply to the pretrained base, not Trial 28.
+A separate September 24 audit used **400-example subsets per task**, distinct
+from the full benchmark results above. On the same selected rows, OpenSML base
+and a pinned SmolLM2-135M **base** reference scored:
+
+| Diagnostic | OpenSML base | SmolLM2-135M base |
+| --- | ---: | ---: |
+| PIQA | 65.00% | 66.00% |
+| HellaSwag | 34.25% | 42.25% |
+| BoolQ | 62.75% | 62.75% |
+
+The reference revision was `93efa2f097d58c2a74874c7e644dbc9b0cee75a2`.
+These are local matched-subset measurements, not the reference model's published
+full-benchmark scores. Different data, architecture, tokenizers, and training
+budgets prevent attributing the differences to pretraining exposure alone.
+
+BoolQ accuracy did not establish reliable reading: OpenSML selected "yes" on
+95.25% of items; without the passage it scored 58.75%. In a separate authored
+fact-swap diagnostic, the model answered both variants correctly on **18/120
+pairs** in question/answer format. Swapping the passage changes the correct
+choice, so this diagnostic checks whether recognition follows supplied facts.
+The pairs share templates and are not 240 independent task families.
+
+### What These Results Establish
+
+Likelihood benchmarks measure recognition among supplied choices. They do not
+measure whether the model can independently write a factually correct, complete
+answer. The reading diagnostics also score choices rather than complete assistant
+responses. A systematic complete-answer evaluation comparable to a reviewed
+free-form answer suite has not been established for this pretrained checkpoint.
+IFEval for the selected SFT checkpoint is reported separately above. No
+MT-Bench or coding-success score is claimed for that checkpoint.
+
+## Checkpoint Identity and Verification
+
+The pretrained checkpoint is `step_0073243_7f3070237eea`, selected from the Stage B run.
+The selected SFT checkpoint is `step_0000768_4980617cb301`, with weight SHA-256:
+
+```text
+cbd3e3fb4ada74d7264371b79cee7598f513b7b950f1f141ef9f1a43bd2e1b4e
+```
+
+The SFT weight hash was verified against the retained bundle. Its full multiple-choice
+and IFEval integrity receipts record 15,428 and 541 completed examples respectively,
+with unchanged inputs. Checkpoint metadata pins Repair512 as its direct parent.
+A portable [Trial 28 results and provenance record](https://huggingface.co/wzebrowski/OpenSML-150M/resolve/main/TRIAL28_RESULTS.json) preserves
+exact scores, protocol settings, dataset pins, and source-record hashes.
+Local evaluation records pin its weight hash, tokenizer fingerprint, dataset
+revisions, evaluator source hashes, and runtime settings. The full-benchmark
+integrity receipt records 15,428 completed examples with unchanged inputs.
+The tokenizer round-trip audit is described above. These checks establish
+artifact identity and the scope of the recorded evaluation; they do not establish
+cross-framework inference parity or deployment readiness. A released-weight
+export verification and reproducible inference guide remain pending.
+
+## Limitations and Intended Use
+
+The eventual model is intended for small-language-model research and controlled
+experimentation. The selected checkpoint is instruction-tuned, but is not a validated production assistant. It may generate
+incorrect, repetitive, biased, offensive, or unsafe text. Do not rely on it for
+high-stakes decisions. Broader capability and safety assessments remain pending.
+
+Architecture, tokenizer, data, and context differ from earlier experiments;
+this run alone cannot attribute any improvement to a single change.
+
+## Repository Contents and Release Plan
+
+- This model card documents the selected SFT checkpoint, pretrained ancestry, and separately labeled evaluations.
+- `configs/` contains the original pinned corpus and pilot/early-continuation recipes; these are historical, not the complete later training history.
+- `tokenizer/` contains the completed tokenizer, its corpus contract, and audit.
+- Model weights, optimizer states, raw datasets, credentials, and private network configuration are not included.
+
+Before a weight release, freeze the selected checkpoint and evaluation evidence,
+add tested inference instructions and necessary implementation code, and review
+third-party attribution. No release date is promised.
+
+## License
+
+OpenSML-150M project contributions in this repository are licensed under the
+[Apache License 2.0](https://huggingface.co/wzebrowski/OpenSML-150M/resolve/main/LICENSE), for rights controlled by the author.
+Upstream datasets and third-party materials retain their own licenses and
+attribution requirements; the repository license does not relicense them.
+
+Post-training sources also retain their own terms. Trial 28 source records list
+SmolTalk as Apache-2.0, SQuAD v2 as CC-BY-SA-4.0, and SciQ as CC-BY-NC-3.0.
+Apache-2.0 metadata for author-controlled project contributions is not a blanket
+commercial-use license for all upstream materials or a completed weight-release
+licensing review.
+
+## External Benchmark Source Records
+
+External comparison values are published measurements, not results from local
+baseline evaluations. Incomplete local baseline runs are not used in the model card.
+
+- GPT-2 normalized ARC-Easy and PIQA: [nano-chatgpt published table](https://github.com/Seif-Yasser-Ahmed/nano-chatgpt/blob/2c6b48b4e2eb612447b0858460c3d0472f00a98b/README.md#presentation-benchmark-table); shot counts are unspecified in that table. GPT-2 HellaSwag acc/acc_norm: [saved baseline JSON](https://github.com/Seif-Yasser-Ahmed/nano-chatgpt/blob/2c6b48b4e2eb612447b0858460c3d0472f00a98b/out_gpt2/10B/checkpoint_19072/evaluation_results/hellaswag_metrics.json). The project's comparison script invokes default HellaSwag few-shot settings. These are project measurements, not official OpenAI claims.
+- OPT-125M: [EleutherAI baseline JSON](https://github.com/EleutherAI/pythia/blob/a19eecb807ec2c79a39ebf18108816e6ffffc1d5/evals/opt/opt-125m.json), `num_fewshot: 0`, `limit: null`; HellaSwag absent.
+- Pythia-160M: [EleutherAI official non-deduplicated suite, step 143,000](https://github.com/EleutherAI/pythia/blob/a19eecb807ec2c79a39ebf18108816e6ffffc1d5/evals/pythia-v1/pythia-160m/zero-shot/160m_step143000.json), `num_fewshot: 0`, `limit: null`; HellaSwag absent.
+
+## Retained SFT Dataset Exposure
+
+These are record exposures, not token shares or unique-example counts. The
+trainers consume ordered batches of 16 records. Tables count the first 6,144,
+2,048, and 4,096 records at the three retained training boundaries respectively.
+
+### Unified384
+
+| Source | Record exposures | Share | Role |
+| --- | ---: | ---: | --- |
+| HuggingFaceTB/smol-smoltalk | 3,071 | 49.98% | Stage mixture |
+| arc_easy | 198 | 3.22% | Stage mixture |
+| HuggingFaceH4/ultrachat_200k | 1,761 | 28.66% | Stage mixture |
+| authored-given-facts-v1 | 384 | 6.25% | Stage mixture |
+| authored-fictional-text-v1 | 384 | 6.25% | Stage mixture |
+| squad_v2 | 160 | 2.60% | Stage mixture |
+| arc_challenge | 186 | 3.03% | Stage mixture |
+| **Total** | **6,144** | **100%** | |
+
+### Repair512
+
+| Source | Record exposures | Share | Role |
+| --- | ---: | ---: | --- |
+| HuggingFaceTB/smol-smoltalk | 259 | 12.65% | Stage mixture |
+| databricks/databricks-dolly-15k | 1,024 | 50.00% | Stage mixture |
+| HuggingFaceH4/ultrachat_200k | 253 | 12.35% | Stage mixture |
+| allenai/tulu-3-sft-personas-instruction-following | 512 | 25.00% | Stage mixture |
+| **Total** | **2,048** | **100%** | |
+
+### OpenSML-150M final continuation
+
+| Source | Record exposures | Share | Role |
+| --- | ---: | ---: | --- |
+| allenai/tulu-3-sft-personas-instruction-following | 256 | 6.25% | Repair512 replay |
+| HuggingFaceTB/smoltalk | 1,536 | 37.50% | Stage mixture |
+| rajpurkar/squad_v2 | 1,024 | 25.00% | Stage mixture |
+| databricks/databricks-dolly-15k | 512 | 12.50% | Repair512 replay |
+| allenai/sciq | 512 | 12.50% | Stage mixture |
+| HuggingFaceTB/smol-smoltalk | 132 | 3.22% | Repair512 replay |
+| HuggingFaceH4/ultrachat_200k | 124 | 3.03% | Repair512 replay |
+| **Total** | **4,096** | **100%** | |
+
+
+## Historical Appendix — September 18, 2026
+
+The following is the preserved original report at its 1B-token reporting boundary.
+It is historical evidence, not the current model-card release status. Relative
+links in this appendix refer to the original V2 documentation tree.
+
+### Historical OpenSML-152M: Pretraining a Small Language Model on Apple Silicon
+
+**William Zebrowski | Working technical report | September 18, 2026**
+
+**Status: pretraining in progress.** Quantitative results below are frozen at
+step 9,391, or 1,000,103,936 consumed training tokens. The planned endpoint is
+15 billion tokens; it is not a completed result.
+
+Project overview (`../README.md` in the original V2 documentation tree) | Data and tokenizer (`DATA_AND_TOKENIZER.md` in the original V2 documentation tree) |
+Results notes (`RESULTS.md` in the original V2 documentation tree) | Evidence index (`EVIDENCE.md` in the original V2 documentation tree)
+
+## Abstract
+
+OpenSML-152M is an English-first, decoder-only language model trained from random
+initialization on a four-Mac Studio cluster. Its implemented architecture contains
+**150,439,168 trainable parameters**, with 20 Transformer blocks, a hidden width
+of 768, grouped-query attention, tied embeddings, and a 2,048-token context.
+The project name is a rounded model label, not the exact parameter count.
+
+A custom 32,000-token byte-level BPE tokenizer was fitted to a representative
+sample of the selected educational-prose corpus. Pretraining streams four
+source components from three Hugging Face repositories, using a target token
+mixture of 55% FineWeb-Edu, 25% DCLM-Edu, 10% English FineWiki, and 10% filtered
+Cosmopedia. The training system combines BF16 computation with FP32 optimizer
+state and gradients, unequal per-device batches, and Thunderbolt RDMA ring
+communication through MLX/JACCL.
+
+At the reporting boundary, aggregate development cross-entropy is **3.2812**
+and perplexity is **26.61** on the fixed reference batches. These are interim
+next-token prediction measurements, not downstream capability scores. Repetitive
+and factually incorrect completions remain visible. This report documents the
+implemented method and its evidence while leaving final evaluation, post-training,
+and release claims explicitly open.
+
+## Contents
+
+1. [Scope and Reporting Conventions](#1-scope-and-reporting-conventions)
+2. [Model Architecture](#2-model-architecture)
+3. [Tokenizer](#3-tokenizer)
+4. [Pretraining Data](#4-pretraining-data)
+5. [Optimization and Training Schedule](#5-optimization-and-training-schedule)
+6. [Distributed Execution](#6-distributed-execution)
+7. [Checkpointing and Reproducibility](#7-checkpointing-and-reproducibility)
+8. [Evaluation Methodology](#8-evaluation-methodology)
+9. [Interim Results](#9-interim-results)
+10. [Post-Training and Release Status](#10-post-training-and-release-status)
+11. [Limitations and Intended Use](#11-limitations-and-intended-use)
+12. [Evidence and Completion Checklist](#12-evidence-and-completion-checklist)
+
+## 1. Scope and Reporting Conventions
+
+### 1.1 Research objective
+
+The objective is a useful small text-generation model with a traceable training
+process on locally available Apple Silicon hardware. Throughput matters as a
+resource constraint, but model selection is not based on throughput alone.
+The architecture and mixture are this project's chosen configuration of
+established components, not a claim of architectural novelty.
+
+These weights were initialized from scratch. Earlier models and their
+fine-tuning experiments are not ancestors of these weights and are not evidence
+of this model's capabilities. Reused implementation components are separately
+recorded in code provenance (`../provenance.json` in the original V2 documentation tree).
+
+### 1.2 What the report distinguishes
+
+| Term | Meaning here |
+| --- | --- |
+| Configured | Present in the recorded recipe or implemented code |
+| Observed | Supported by an identified log, audit, or artifact |
+| Planned | Not yet a completed experiment or measured outcome |
+| Consumed training tokens | Token positions included in completed optimizer updates along this run's lineage |
+| Validation loss | Cross-entropy on fixed held-out reference batches, not generated-answer accuracy |
+| Best checkpoint | Lowest observed weighted validation loss under this evaluation contract |
+| Latest checkpoint | Most recent successfully committed resumable state |
+
+Validation records, qualitative probes, and future independent benchmarks are
+different evidence types. Repeated use of reference batches makes them
+development evidence, not an untouched final test. Results from another
+tokenizer, corpus, or evaluation protocol are not directly comparable by raw loss.
+
+### 1.3 Reporting boundary
+
+This document is a versioned snapshot, not a live status dashboard. Training may
+have advanced beyond the values shown. A machine-readable
+validation table (`tables/validation_through_0009391.csv` in the original V2 documentation tree) and its
+provenance record (`tables/validation_through_0009391.json` in the original V2 documentation tree) accompany this revision.
+Earlier supporting notes may use earlier, explicitly dated cutoffs.
+
+## 2. Model Architecture
+
+### 2.1 Transformer configuration
+
+| Component | Implemented value |
+| --- | --- |
+| Model family | Autoregressive, decoder-only Transformer |
+| Trainable parameters | 150,439,168 |
+| Transformer blocks | 20 |
+| Hidden width | 768 |
+| Context length | 2,048 tokens |
+| Vocabulary | 32,000 |
+| Query / key-value heads | 12 / 4 |
+| Head dimension | 64 |
+| Position representation | Rotary position embeddings; base 10,000 |
+| Normalization | Pre-norm RMSNorm; per-head query/key RMSNorm; final RMSNorm |
+| Feed-forward | SwiGLU; intermediate width 2,048 |
+| Vocabulary interface | Shared input embedding and output projection |
+| Linear biases / dropout | No linear biases; no dropout layers |
+
+Each attention block uses a fused QKV projection. Its output width is
+768 + 256 + 256 = 1,280. Grouped-query attention shares four key/value heads
+across twelve query heads. The configured fast-attention path passes grouped
+heads to MLX's scaled dot-product attention rather than explicitly expanding the
+key/value tensors in Python.
+
+### 2.2 Block computation
+
+For residual input x, each block computes:
+
+~~~text
+h = x + Attention(RMSNorm(x))
+y = h + SwiGLU(RMSNorm(h))
+
+SwiGLU(z) = down(silu(gate(z)) * up(z))
+logits = RMSNorm(final_hidden) @ embedding_weight.T
+~~~
+
+Query and key normalization occur before rotary position encoding. Attention
+uses a causal mask and scale 1/sqrt(64). The configured MLP ratio of 4 is a
+construction parameter: applying the SwiGLU width adjustment and alignment
+produces an actual intermediate width of 2,048, not 3,072.
+
+### 2.3 Parameter accounting
+
+The input embedding and output projection are tied and counted once.
+
+| Component | Parameters across the model |
+| --- | ---: |
+| Shared token embedding / output matrix | 24,576,000 |
+| QKV projections, 20 blocks | 19,660,800 |
+| Attention output projections, 20 blocks | 11,796,480 |
+| SwiGLU projections, 20 blocks | 94,371,840 |
+| Block and Q/K normalization weights | 33,280 |
+| Final normalization | 768 |
+| **Total** | **150,439,168** |
+
+The shared vocabulary matrix accounts for approximately 16.3% of parameters.
+The model has no separate learned positional embedding table.
+
+### 2.4 Initialization and numerical representation
+
+Matrix weights use normal initialization with standard deviation 0.02.
+Attention output and feed-forward down projections are scaled by
+1/sqrt(2L), where L = 20. Normalization weights start at one.
+
+BF16 is used for model computation. FP32 master weights, optimizer moments,
+gradient accumulation, and reduced gradients preserve precision during updates.
+This is mixed-precision training, not an all-BF16 optimizer.
+
+Implementation evidence: model (`../sml_v2/model.py` in the original V2 documentation tree),
+precision and optimizer (`../sml_v2/precision.py` in the original V2 documentation tree).
+
+## 3. Tokenizer
+
+### 3.1 Construction
+
+The tokenizer is **ByteBPE32K v1**, trained for this corpus rather than inherited
+from the earlier model. Its fitting sample follows the same source proportions
+as pretraining, but sampling quotas are measured in UTF-8 bytes rather than
+model tokens.
+
+| Property | Recorded value |
+| --- | --- |
+| Algorithm | Byte-level BPE |
+| Vocabulary size | 32,000 IDs including four structural tokens |
+| Initial alphabet | Full 256-byte alphabet |
+| Minimum merge frequency | 2 |
+| Fitting sample | 500,048,186 UTF-8 bytes; 113,898 documents |
+| Development audit | 4,010,605 UTF-8 bytes; 970 documents |
+| Tokenizers library | 0.22.2 |
+| Normalization / prefix space | No text normalizer; no automatic prefix space |
+| Implicit BOS/EOS insertion | None |
+| Sampling | Seeded source shuffling; buffer size 10,000 |
+
+A 500 MB tokenizer fitting sample is not the pretraining dataset size.
+The fitted tokenizer is frozen during model training.
+
+### 3.2 Encoding contract
+
+| ID | Token | Role |
+| ---: | --- | --- |
+| 0 | `<|pad|>` | Reserved padding |
+| 1 | `<|doc_end|>` | Explicit document boundary |
+| 2 | `<|turn_start|>` | Reserved for potential conversational formatting |
+| 3 | `<|turn_end|>` | Reserved for potential conversational formatting |
+
+Content encoding does not insert structural IDs automatically. Literal spellings
+of structural tokens in input text are encoded as ordinary content; the wrapper
+rejects an unexpected reserved ID in encoded content. Pretraining appends the
+document-end token explicitly. Reserved turn tokens do not imply that chat
+formatting or instruction following has been trained.
+
+### 3.3 Audit results
+
+| Held-out source | Documents | UTF-8 bytes | Encoded tokens | Bytes/token |
+| --- | ---: | ---: | ---: | ---: |
+| FineWeb-Edu | 559 | 2,205,062 | 476,722 | 4.6255 |
+| DCLM-Edu | 165 | 1,001,509 | 224,896 | 4.4532 |
+| FineWiki | 143 | 401,775 | 98,977 | 4.0593 |
+| Cosmopedia | 103 | 402,259 | 79,482 | 5.0610 |
+
+The held-out audit recorded zero round-trip failures; nine regression fixtures
+also passed. These checks establish properties of the audited encoding, not
+downstream quality or an exhaustive audit of future streamed documents.
+
+The manifest (`../tokenizer/bytebpe32k_v1/manifest.json` in the original V2 documentation tree) and
+audit report (`../tokenizer/bytebpe32k_v1/report.json` in the original V2 documentation tree) retain the detailed records.
+The tokenizer artifact fingerprint is:
+
+~~~text
+1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb
+~~~
+
+## 4. Pretraining Data
+
+### 4.1 Source mixture
+
+Pretraining uses four components from three upstream repositories. All are read
+from their upstream training splits, with project-specific content-hash splits
+applied afterward.
+
+| Component | Upstream repository / configuration | Target token share | Additional selection |
+| --- | --- | ---: | --- |
+| Web | [HuggingFaceTB/smollm-corpus](https://huggingface.co/datasets/HuggingFaceTB/smollm-corpus), `fineweb-edu-dedup` | 55% | Shared text-quality filters |
+| DCLM | [HuggingFaceTB/dclm-edu](https://huggingface.co/datasets/HuggingFaceTB/dclm-edu) | 25% | `edu_int_score >= 3` |
+| Wiki | [HuggingFaceFW/finewiki](https://huggingface.co/datasets/HuggingFaceFW/finewiki), `en` | 10% | English configuration |
+| Cosmopedia | SmolLM corpus, `cosmopedia-v2` | 10% | Format contains textbook, tutorial, blog, or educational |
+
+Pinned revisions:
+
+~~~text
+SmolLM corpus: 3ba9d605774198c5868892d7a8deda78031a781f
+DCLM-Edu:      dbad8ad71224482740cd9c9d353591adbf62fe04
+FineWiki:      8bd13e72e6a002407649b3e898535f42ceb1aeb9
+~~~
+
+The mixture is prose-focused; no dedicated coding or mathematics source is
+selected. This does not mean every document is free of code, equations, or
+incidental non-English text. Shares describe target consumed token proportions,
+not document counts or a completed inventory of selected documents.
+
+The corpus configuration records source license labels and provenance.
+Those labels are not blanket permission to redistribute all underlying content.
+See the complete corpus contract (`../configs/corpus.json` in the original V2 documentation tree).
+
+### 4.2 Filtering and split assignment
+
+Documents must contain text, meet the 200-character minimum, and remain within
+200,000 UTF-8 bytes. Filters reject NUL characters and more than two Unicode
+replacement characters. Documents with at least ten non-empty lines are rejected
+when fewer than half those lines are distinct. Source-specific score and format
+requirements are checked in addition to these shared rules.
+
+For identity and split assignment, text is NFC-normalized and whitespace is
+collapsed before SHA-256 hashing. The first sixteen hexadecimal hash digits,
+modulo 10,000, assign documents as follows:
+
+| Bucket range | Allocation |
+| --- | --- |
+| 0-99 | Reserved final split |
+| 100-199 | Validation |
+| 200-299 | Tokenizer development |
+| 300-9,999 | Training |
+
+These are deterministic hash allocations, not guarantees of exact byte or token
+percentages. Model input retains the original text; identity normalization does
+not rewrite the training text.
+
+A cross-source window of 10,000 recent accepted document hashes suppresses
+normalized exact duplicates. This is bounded deduplication, not full-corpus
+near-duplicate removal. Hash splitting separates identical normalized documents
+across roles, but does not establish benchmark decontamination.
+
+### 4.3 Streaming, scheduling, and packing
+
+Mac-1 reads the pinned Hugging Face sources, filters documents, tokenizes them,
+and constructs a global batch. The live training stream follows source order;
+unlike tokenizer fitting, it does not use a document shuffle buffer.
+
+Source scheduling chooses the most underrepresented component using consumed
+token offsets divided by target weights. Each source maintains its own pending
+token buffer. Documents receive one explicit document-end token, with no
+automatic beginning-of-document token.
+
+A row requires 2,049 buffered tokens: 2,048 input positions and their one-token
+shifted targets. Advancing by 2,048 preserves the final lookahead token for the
+next row. Packed windows can cross document boundaries within a source.
+Attention is not reset at a document-end token.
+
+The reader is configured for four prefetched global updates and five read
+attempts. Source exhaustion or the configured five-million-row-per-source limit
+is an error, not a trigger to silently recycle data. Reaching 15B tokens therefore
+depends on sufficient eligible source material remaining available.
+
+### 4.4 Storage and token accounting
+
+No full training corpus or token-shard collection is materialized on disk.
+Streaming still requires RAM for reader state, token buffers, fixed validation
+batches, and bounded prefetch. Checkpoints persist the consumed cursor and pending
+buffer state; tokenizer samples, logs, staged inputs, and metadata are separate
+disk uses.
+
+| Quantity | Value / definition |
+| --- | --- |
+| Planned endpoint | 15,000,000,000 cumulative training positions |
+| Positions per optimizer update | 106,496 |
+| Consumed at this report's boundary | 1,000,103,936 |
+| Expected full-update endpoint if geometry is unchanged | 15,000,068,096 |
+| Expected endpoint step under that assumption | 140,851 |
+
+Counts include boundary-token positions processed by the objective. They exclude
+validation and generation, and do not count prefetch as consumed training.
+They are not counts of unique words, documents, or facts. Restarting from an older
+checkpoint can repeat computation; lineage token counts are not an accounting
+of every token processed across failed or discarded work.
+
+Implementation evidence: corpus (`../sml_v2/corpus.py` in the original V2 documentation tree),
+streaming (`../sml_v2/stream.py` in the original V2 documentation tree).
+
+## 5. Optimization and Training Schedule
+
+### 5.1 Objective and optimizer
+
+The objective is causal next-token cross-entropy over packed sequences:
+
+~~~text
+L = -(1 / N) * sum(log p(target_token | preceding_tokens))
+~~~
+
+All packed target positions contribute, including document boundaries. This is
+base-model pretraining, not assistant-response-only supervision.
+
+| Hyperparameter | Value |
+| --- | --- |
+| Optimizer | AdamW |
+| Beta1 / beta2 | 0.9 / 0.95 |
+| Epsilon | 1e-8 |
+| Weight decay | 0.1 on matrix parameters; normalization weights excluded |
+| Global gradient clipping | L2 norm 1.0, after distributed reduction |
+| Peak / minimum learning rate | 3e-4 / 3e-5 |
+| Accumulation | Four microsteps per update |
+| Random seed | 7,337 |
+| Schedule coordinate | Consumed training tokens |
+
+Local gradients are averaged across accumulation microsteps in FP32. The
+distributed weighted mean is then clipped before each replica applies its
+optimizer update.
+
+### 5.2 Effective batch
+
+~~~text
+(4 + 3 + 3 + 3) sequences/microstep
+    * 4 accumulation microsteps
+    = 52 sequences/update
+
+52 * 2,048 = 106,496 tokens/update
+~~~
+
+The update contains **52 sequences**, not 208. Unequal device batches are a
+compute-balancing choice and are reflected in the gradient weights.
+
+### 5.3 Recorded learning-rate trajectory
+
+The training budget was extended at 500,105,216 consumed tokens. Model,
+optimizer, and data state were preserved; the schedule was explicitly changed.
+This is one weight lineage, but not an uninterrupted 15B-token schedule defined
+at initialization.
+
+| Token interval / boundary | Learning-rate behavior |
+| --- | --- |
+| 0 to 5M | Linear warmup toward 3e-4 |
+| 5M to 400M | Hold at 3e-4 |
+| 400M to approximately 500M | Cosine decay toward 3e-5 under the original budget |
+| 500,105,216 | Resume anchor at step 4,696; LR approximately 3e-5 |
+| Next 50M tokens | Linear rewarm toward 3e-4 |
+| Approximately 550.1M to 12B | Hold at 3e-4 |
+| 12B to 15B | Cosine decay toward 3e-5 |
+
+The rate is evaluated at the upcoming update's ending token count, so boundaries
+are realized at full updates. Resuming this continuation preserves its original
+rewarm anchor rather than restarting warmup.
+
+The scheduler is token-based, not a validation-plateau controller. No controlled
+experiment has established that this schedule is optimal. The early decay and
+rewarm must remain visible when interpreting learning curves.
+
+## 6. Distributed Execution
+
+### 6.1 Hardware and rank assignment
+
+Training uses one M3 Ultra Mac Studio and three M4 Max Mac Studios.
+The logical JACCL ring is Mac-1 -> Mac-2 -> Mac-4 -> Mac-3 -> Mac-1.
+
+| Logical rank | Physical host | Chip family | Microbatch | Sequences/update | Tokens/update |
+| ---: | --- | --- | ---: | ---: | ---: |
+| 0 | Mac-1 | M3 Ultra | 4 | 16 | 32,768 |
+| 1 | Mac-2 | M4 Max | 3 | 12 | 24,576 |
+| 2 | Mac-4 | M4 Max | 3 | 12 | 24,576 |
+| 3 | Mac-3 | M4 Max | 3 | 12 | 24,576 |
+| **Total** | | | **13** | **52** | **106,496** |
+
+This is replicated data-parallel training, not model sharding. Each Mac stores
+the complete model and optimizer state and computes on a disjoint slice of the
+global batch. Four physical Thunderbolt links form the ring.
+
+### 6.2 One optimizer update
+
+1. Mac-1 constructs and distributes the global input/target arrays.
+2. Each rank selects its assigned sequences and performs four local microsteps.
+3. Accumulated FP32 gradients are weighted by local batch contribution.
+4. Distributed gradient reduction produces the global mean.
+5. Each replica clips that mean and applies the same AdamW update.
+6. When due, ranks coordinate evaluation, generation, or checkpoint commitment.
+
+For local mean gradients g_r and microbatch sizes b_r, the intended gradient is:
+
+~~~text
+g_global = sum(b_r * g_r) / sum(b_r)
+~~~
+
+Before MLX's equal-rank averaging operation, the implementation multiplies each
+local mean by world_size * b_r / sum(b_r). Equal accumulation counts and full
+sequence lengths make sequence weighting equivalent to token weighting here.
+A plain unweighted mean of the four ranks would overrepresent the smaller batches.
+
+### 6.3 Data plane and control plane
+
+Tensor transfers use JACCL over Thunderbolt RDMA. Wired SSH and loopback-only
+coordinator tunnels carry control traffic; Mac-4's wired control path is relayed
+through Mac-2. Hugging Face streaming uses the host's internet connection and is
+separate from inter-Mac tensor communication.
+
+The launcher verifies physical identities, ring links, RDMA, wired control
+connectivity, and synchronized inputs. This is custom orchestration around MLX,
+not an unmodified upstream example. Existing cluster/network helpers outside the
+dedicated model directory remain runtime dependencies.
+
+### 6.4 Kernels and performance boundaries
+
+The configured compute path uses compiled backward computation, MLX fast
+attention, a custom Metal cross-entropy implementation, and packed Metal SwiGLU.
+Cross-entropy reductions and optimizer-related state retain FP32 precision.
+
+Gradient groups use a 32 MiB reduction setting. The trainer explicitly
+materializes gradients before communication; this report does not claim
+communication/backward overlap. The implementation's collective stream setting
+does not mean that model forward/backward computation runs on the CPU.
+
+No isolated speedup is assigned to any kernel without a matched measurement.
+An earlier architecture's throughput is not a controlled baseline for this
+20-layer, 2,048-context model.
+
+Implementation evidence: trainer (`../sml_v2/pretrain.py` in the original V2 documentation tree),
+launcher (`../sml_v2/launch.py` in the original V2 documentation tree).
+
+## 7. Checkpointing and Reproducibility
+
+### 7.1 What a resumable checkpoint preserves
+
+Checkpoint bundles preserve model weights, FP32 master weights and optimizer
+state, update/token counters, recipe and artifact contracts, validation state,
+and consumed stream state. Replica hashes and data-cursor checks accompany
+commitment. Writes are staged before a completed bundle becomes the new pointer.
+
+The stream cursor corresponds to consumed work, not the producer's most advanced
+prefetched position. Unconsumed queued batches are reconstructed after resume.
+Underlying readers may reread remote content to recover a position; that alone
+does not establish repeated training exposure.
+
+### 7.2 Selection and retention
+
+The latest pointer identifies the newest committed resume point. The best pointer
+identifies the lowest aggregate validation loss observed under the fixed contract.
+These may refer to the same bundle.
+
+Routine saves are requested every 10M tokens, with additional saves for new
+validation bests, clean stops, and completion. The configured retention policy
+keeps two recent checkpoints and protects the best, so a distinct older best can
+require an additional bundle. Logs and staged job inputs have separate storage
+lifetimes.
+
+An 8 GiB reserve, plus estimated checkpoint requirements, guards saving. Failed
+or interrupted work after the last committed bundle is not guaranteed recoverable.
+The system does not automatically restart failed training.
+
+### 7.3 Reproduction contract
+
+The job snapshots recipe, corpus definition, tokenizer, and code inputs and
+verifies copies and package versions across hosts. Resume validation uses
+artifact contracts rather than assuming that files with the same name are
+equivalent.
+
+The dependency specification pins MLX and mlx-metal 0.32.2, NumPy 2.4.2,
+datasets 4.5.0, huggingface-hub 1.4.0, and tokenizers 0.22.2. The project requires
+Python 3.11 or later. Exact future release reproduction should additionally
+retain observed interpreter, macOS, hardware, and networking inventories.
+
+These controls improve traceability; they do not constitute a claim of
+bitwise-identical execution across different machines or software releases.
+Operating instructions remain in the project README (`../README.md` in the original V2 documentation tree).
+
+## 8. Evaluation Methodology
+
+### 8.1 Fixed reference loss
+
+The validation stream materializes sixteen batches per source, each containing
+two sequences of 2,048 targets. This yields:
+
+| Quantity | Count |
+| --- | ---: |
+| Source components | 4 |
+| Batches per component | 16 |
+| Total batches | 64 |
+| Target positions per component | 65,536 |
+| Total target positions | 262,144 |
+
+Per-source cross-entropies are combined using the training mixture's 55/25/10/10
+weights. Because the source sample sizes are equal but weights differ, this is
+not a simple equal average across all batches.
+
+~~~text
+L_reference = 0.55 * L_web + 0.25 * L_dclm
+            + 0.10 * L_wiki + 0.10 * L_cosmopedia
+
+reference_perplexity = exp(L_reference)
+~~~
+
+Validation runs every 25M consumed tokens, approximately every 235 updates at
+the current batch geometry. The reference fingerprint remains unchanged across
+the documented continuation:
+
+~~~text
+99e7d4417061db5a70138701694ad5577d0f9ff0d0fc729ddc6beff06b3a7081
+~~~
+
+These fixed batches support longitudinal comparisons within this model's run.
+They are small relative to the corpus and do not quantify uncertainty across
+all possible held-out documents.
+
+### 8.2 Generation probes
+
+Five fixed raw-text prefixes are sampled every 50M tokens:
+
+| Probe | Full prefix |
+| --- | --- |
+| Narrative continuation | The library was closed for repairs, so Maya decided to |
+| Basic explanation | Plants need sunlight because |
+| Learning advice | A useful way to remember a new word is to |
+| Contextual continuation | The train arrived late because of heavy snow. When the passengers stepped outside, |
+| Basic explanation | Clean drinking water is important because |
+
+The recorded comparison path uses greedy decoding, temperature 0, at most 64
+new tokens, and the model's own frozen tokenizer. It uses cached autoregressive
+generation and stops at document end. It does not apply a repetition penalty
+or top-k sampling.
+
+These settings deliberately make comparisons stable. Greedy failure cases are
+not the only possible outputs, but changing temperature does not by itself
+establish that a factual or contextual weakness is solved. Probes are qualitative
+development diagnostics with no reported automatic capability score.
+
+### 8.3 Independent evaluation still to perform
+
+Before any final capability claim, freeze candidate weights and publish the
+evaluation settings, prompts or task versions, decoding parameters, and scoring
+rules. The intended evaluation record should separate held-out language modeling,
+factual recall, use of supplied context, narrative coherence, and repetition.
+
+No downstream benchmark score, human-review pass rate, or untouched final-split
+result has been established in this report. Future comparisons must use documented
+protocols and identify possible training-data overlap.
+
+## 9. Interim Results
+
+### 9.1 Reference-loss milestones
+
+| Step | Consumed training tokens | Reference loss | Perplexity | Context |
+| ---: | ---: | ---: | ---: | --- |
+| 235 | 25,026,560 | 5.7345 | 309.37 | Early training |
+| 4,696 | 500,105,216 | 3.3536 | 28.61 | Budget-extension boundary |
+| 5,165 | 550,051,840 | 3.4327 | 30.96 | Near the end of rewarm |
+| 7,982 | 850,051,072 | 3.3160 | 27.55 | Continued pretraining |
+| 9,156 | 975,077,376 | 3.2832 | 26.66 | Continued pretraining |
+| **9,391** | **1,000,103,936** | **3.2812** | **26.61** | **Reporting cutoff** |
+
+All logged evaluations through the cutoff are retained in the
+validation CSV (`tables/validation_through_0009391.csv` in the original V2 documentation tree), including repeated
+observations if present. The table above selects milestones, not every event.
+Loss and perplexity retain their logged rounding.
+
+Loss increased during rewarm and subsequently fell below the earlier boundary's
+value. This is a temporal association with the schedule change, not a controlled
+causal result. More consumed data and more optimizer updates also changed.
+
+### 9.2 Per-source reference loss
+
+| Source | Step 4,696 | Step 9,391 |
+| --- | ---: | ---: |
+| Web | 3.4531 | 3.3884 |
+| DCLM | 3.6497 | 3.5756 |
+| Wiki | 2.8654 | 2.7904 |
+| Cosmopedia | 2.5543 | 2.4468 |
+
+All four fixed source references improved between these boundaries. Differences
+between sources reflect different reference distributions as well as learned
+prediction; a lower source loss is not a ranking of factual competence.
+
+### 9.3 Throughput
+
+As one identifiable production observation, step 9,280 logged **12,889 tokens/s**
+for its reporting interval and **13,570 run tokens/s**. These are aggregate
+cluster training-position rates, not inference generation speed and not a
+fourfold sum of independently reported rank rates.
+
+The run counter describes that worker invocation's timed training loop, not all
+project wall time. Startup, interrupted sessions, and external preparation must
+be accounted for separately before reporting total training duration or cost.
+A final throughput summary, distribution of step times, peak-memory audit, and
+energy measurement remain pending. No model FLOP utilization is claimed.
+
+### 9.4 Qualitative limitations observed so far
+
+The logged prefixes elicit grammatical fragments but also repetitive clauses,
+topic drift, and factual errors. At step 5,165, for example, the word-learning
+prefix began with useful advice about using a word in a sentence and then
+repeated that idea. The plant and water explanations were not dependable.
+
+Improving reference loss is therefore evidence of better prediction on those
+reference batches, not proof of reliable open-ended generation. A final report
+should retain complete successes and failures rather than presenting only
+favorable excerpts. The full probe prefixes and generation settings above make
+that review reproducible.
+
+## 10. Post-Training and Release Status
+
+| Item | Status at this report boundary |
+| --- | --- |
+| Tokenizer fitting and development audit | Completed |
+| From-scratch pretraining | In progress |
+| 15B-token endpoint | Planned, not reached |
+| Final pretrained checkpoint selection | Pending |
+| Instruction fine-tuning for these weights | Not reported |
+| Preference optimization / reinforcement learning | Not reported |
+| Independent downstream evaluation | Pending |
+| Export parity and fresh-download validation of final weights | Pending |
+| Public final-weight release | Pending |
+
+The prepared [OpenSML repository](https://huggingface.co/wzebrowski/OpenSML) is
+private at this revision. Its model card uses Apache-2.0 metadata for the planned
+release. That does not relicense upstream datasets or remove obligations on
+third-party code and content.
+
+A final export should include weights, matching tokenizer and configuration,
+inference instructions, provenance, evaluation settings, and a reproducible
+loading/generation check. Compatibility with an inference library should be
+verified rather than inferred from the model's general Transformer architecture.
+
+No existing fine-tuned checkpoint from an earlier model is presented as a
+post-trained version of OpenSML-152M.
+
+## 11. Limitations and Intended Use
+
+- **Incomplete training:** the reported snapshot covers approximately 1B tokens, not the full 15B target.
+- **Generation reliability:** repetitive and incorrect text remains an observed failure mode.
+- **Development reuse:** reference batches and fixed prompts have been inspected repeatedly.
+- **Data coverage:** educational prose does not guarantee balanced coverage of every domain or language.
+- **Deduplication limits:** a bounded exact-hash window does not remove all duplicates, paraphrases, or benchmark overlap.
+- **Sequence boundaries:** packing allows attention across document boundaries without an attention reset.
+- **Source order:** live streaming does not perform the seeded document shuffle used for tokenizer fitting.
+- **Schedule confounding:** the recorded early decay and rewarm prevent describing this as a single unchanged training schedule.
+- **Uncontrolled design choices:** architecture, tokenizer, and data choices have not been individually validated by matched ablations.
+- **Infrastructure dependence:** wired routing, tunnels, and project-specific collective safeguards add operational complexity.
+- **Unmeasured efficiency:** no end-to-end energy result, MFU measurement, or final total training cost is reported.
+
+Current intended use is research into small-model training and text completion.
+This base model should not be treated as a reliable assistant or a source of
+high-stakes factual advice.
+
+## 12. Evidence and Completion Checklist
+
+### 12.1 Local evidence map
+
+| Claim group | Primary records |
+| --- | --- |
+| Architecture and parameter construction | Model implementation (`../sml_v2/model.py` in the original V2 documentation tree) |
+| Optimizer and precision | Precision implementation (`../sml_v2/precision.py` in the original V2 documentation tree) |
+| Source weights, fields, revisions | Corpus configuration (`../configs/corpus.json` in the original V2 documentation tree) |
+| Filtering, identity, split rules | Corpus implementation (`../sml_v2/corpus.py` in the original V2 documentation tree) |
+| Stream cursors and packing | Streaming implementation (`../sml_v2/stream.py` in the original V2 documentation tree) |
+| Tokenizer construction and audits | Manifest (`../tokenizer/bytebpe32k_v1/manifest.json` in the original V2 documentation tree), audit (`../tokenizer/bytebpe32k_v1/report.json` in the original V2 documentation tree) |
+| Live continuation settings | Recorded job recipe (`../runs/full_15b_v1/jobs/20260918_085811_b4e5c849/input/recipe.json` in the original V2 documentation tree) |
+| Distributed updates and evaluation | Trainer (`../sml_v2/pretrain.py` in the original V2 documentation tree), launcher (`../sml_v2/launch.py` in the original V2 documentation tree) |
+| Reported validation observations | CSV (`tables/validation_through_0009391.csv` in the original V2 documentation tree), provenance (`tables/validation_through_0009391.json` in the original V2 documentation tree) |
+| Implementation ancestry | Code provenance (`../provenance.json` in the original V2 documentation tree) |
+
+The evidence table references the local project layout. A future published report
+must package durable copies or revision-pinned links; local run directories alone
+are not a public reproduction artifact.
+
+The companion evidence exporter (`../scripts/export_report_evidence.py` in the original V2 documentation tree) reads
+logs without loading a model, contacting workers, or changing training. It
+records source-log hashes, line numbers, and the exported table's hash. Active
+logs grow, so their hashes identify the snapshot read rather than an immutable
+future file.
+
+### 12.2 Reporting approach and attribution
+
+The organization was informed by the transparent reporting approach in
+[PetitGPT's technical report](https://github.com/yangqi0/petitgpt/blob/53a5bb33052fa8092987eacc1047e040a7c5ce30/docs/petitgpt-v1/TECHNICAL_REPORT.md).
+The prose, model configuration, implementation description, and measurements
+here describe this project. PetitGPT's results are not imported as our results
+or used as matched baselines.
+
+The system builds on established Transformer methods, MLX, Hugging Face tooling,
+and the cited data sources. Component-specific attribution and inherited
+implementation history should remain attached to the released code.
+
+### 12.3 Items required for a completed report
+
+- [ ] Freeze the final training history, schedule, source consumption, and selected checkpoint.
+- [ ] Publish the complete loss curve with restart and schedule boundaries identified.
+- [ ] Summarize sustained throughput, memory, elapsed time, and measurement scope.
+- [ ] Evaluate the reserved final split without using it for checkpoint selection.
+- [ ] Run independently specified capability and generation evaluations.
+- [ ] Publish complete qualitative examples with prompt and decoding settings.
+- [ ] Document any post-training experiments separately from base-model results.
+- [ ] Verify exported weights, tokenizer, loading, and generation from a fresh download.
+- [ ] Package durable evidence and review dataset/code attribution before release.
+
+Until those items are completed, this document is a detailed methods and interim
+results report, not a final model-quality claim.

TRAINING_RECORDS_MANIFEST.json

@@ -0,0 +1,847 @@
+{
+  "scope": "Original selected SFT lineage inputs, recipe code, logs and evaluation receipts; no weights, optimizer tensors or complete pretraining cache. Historical V2 source names are preserved.",
+  "files": [
+    {
+      "path": "exports/opensml-150m/SOURCE_NOTICE.md",
+      "bytes": 1717,
+      "sha256": "f23e5289cac871e325fd06edc4b538167e8c4cb737491adabc591d1535810c1d"
+    },
+    {
+      "path": "exports/opensml-150m/THIRD_PARTY_NOTICES.md",
+      "bytes": 5918,
+      "sha256": "7d5e0edccfe69bed72de19aa6a004256262da3c271e8397e7ba72ec67767948f"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/README.txt",
+      "bytes": 5653,
+      "sha256": "3deb2a3f65b5c42a936a5cb9fd2e99914a1d09cdb9729d799750216e29edc6e1"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/checkpoint512_v1/analyze.py",
+      "bytes": 11931,
+      "sha256": "0c086b2082ea257c7026b9f90f72a2b7d699279f53eda0f1474b97cb2e180b31"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/checkpoint512_v1/arc_challenge_changed_rows.json",
+      "bytes": 319279,
+      "sha256": "e02a6b6bd21e31d023f8537adb710b146dc52dfa5e8bcd6b2f3ace21c8cc6be3"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/checkpoint512_v1/arc_easy_changed_rows.json",
+      "bytes": 664901,
+      "sha256": "eda40bc6ca822381dc78c993e9726c0ce3f9b8b610a8fe19e8cee1a365a8d4e9"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/checkpoint512_v1/build_report.py",
+      "bytes": 7558,
+      "sha256": "1a157b7890ef1d84009f5d2e29131da3f574cae42a9640cf88cee8423321b70e"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/checkpoint512_v1/hellaswag_changed_rows.json",
+      "bytes": 564949,
+      "sha256": "cc6ca7f8ffa4c34955ad522fbb8c19aa205bcaadafe7c2e001dcdafaafb18c0d"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/checkpoint512_v1/ifeval_all_annotated.json",
+      "bytes": 1467781,
+      "sha256": "f1c35030244ccfad6c33817729e1403978803dc4a0e076be7dba6fa6f74d0f4c"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/checkpoint512_v1/piqa_changed_rows.json",
+      "bytes": 81333,
+      "sha256": "e41ed6e8793acb2eb38235368b5fc11935286d289b451aee97dcf05ac0de3e39"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/checkpoint512_v1/public_dev_changes.json",
+      "bytes": 220329,
+      "sha256": "8064c6650b7957f2bf0a429fbe5aa05488225ffc961d9f4cf1754199239bb739"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/checkpoint512_v1/receipt.json",
+      "bytes": 5295,
+      "sha256": "31d3b6e7f99da6d92cdba7a5e41dfe976e4722311c4628db44bc8c8b661e9e01"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/checkpoint512_v1/summary.json",
+      "bytes": 37976,
+      "sha256": "c13a62d05d415828981bccc6cf23112e918e3a12e4b0016c62fcfc86a173cc52"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/next_route_20261001/analyze.py",
+      "bytes": 5199,
+      "sha256": "45784c21764aaefd9f5f8d21fb67445cc73c073210358006b1d097cded7e17b0"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/next_route_20261001/findings.json",
+      "bytes": 10493,
+      "sha256": "3c6b0842660c1e66dc9a3f2ebf1712bf4c824302ee560e6a4300fbd24b1392e1"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/next_route_20261001/probe.py",
+      "bytes": 9260,
+      "sha256": "a1494c32dea72810cc86702b16b838e4c78bd9d3f01fa45d088f15b6fd63ac78"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/audits/next_route_20261001/sampling.json",
+      "bytes": 236002,
+      "sha256": "daeae2df4c849dd0d38e15b88b576edc2a90f5f51c6f3768e6ed5d7c9a653764"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/benchmarks/public-repair-384-plus-128/ifeval/integrity.json",
+      "bytes": 161,
+      "sha256": "e76134f1a82f5f456056baf463c16e6a81a15f170c9b3d847327d11c3087085e"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/benchmarks/public-repair-384-plus-128/ifeval/manifest.json",
+      "bytes": 172290,
+      "sha256": "7c9e9627a22d4862193817a987d5060704b5a125dc285b892a4c4f0dd48c8b2a"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/benchmarks/public-repair-384-plus-128/ifeval/records.jsonl",
+      "bytes": 2278523,
+      "sha256": "4b1a7c5ce69cde554b2b07dfefe26aa8fc003fe2eece3cb3f7cac6fe9cbaeff2"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/benchmarks/public-repair-384-plus-128/ifeval/summary.json",
+      "bytes": 615,
+      "sha256": "6d5f3a88e0afd009b9f4fd527e7edb961f230678e0bffad3888601ea570d67b6"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/benchmarks/public-repair-384-plus-128/multiple-choice/integrity.json",
+      "bytes": 163,
+      "sha256": "c5059de1980e9abaa6f033e307eb57d98047cf002ae0d1c1d0318a247df3d0b8"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/benchmarks/public-repair-384-plus-128/multiple-choice/manifest.json",
+      "bytes": 22453,
+      "sha256": "e9e35c43bb110f11c1190f9110aee455ffa61cc3a1abfdfc5985d72efb6cc8db"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/benchmarks/public-repair-384-plus-128/multiple-choice/records.jsonl",
+      "bytes": 8905350,
+      "sha256": "58c4fc6c5cf830f345b06d4a75f0a3078171e9a28a595c2bf4e64455c0ec4184"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/benchmarks/public-repair-384-plus-128/multiple-choice/summary.json",
+      "bytes": 690,
+      "sha256": "f2a63e07b2c2db4aa2742fcb88d03ac8af4f4abb5d40092db84f2a3101243d22"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/benchmarks/readiness/readiness_public-repair-384-plus-128_ifeval.json",
+      "bytes": 174893,
+      "sha256": "7260060b8b59a727fc4fce844bd9eac716f28b37acf0d3bc9f125b9ab8f09ad4"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/benchmarks/readiness/readiness_public-repair-384-plus-128_multiple-choice.json",
+      "bytes": 23559,
+      "sha256": "d232fdd220cc578d75b22f8030419da9e7ac62e56ef4c70af701a8040d6d85e7"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/config.json",
+      "bytes": 1194,
+      "sha256": "2fd560ee19aa6da5df60aab88d8fafab0114c598e46ea20a3760b31d83114922"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/data/prepared.json",
+      "bytes": 8450457,
+      "sha256": "71090fe7bd30f9454090ec48fc92ff7af4a27a7443ec4d2b4539aef9c4c0909a"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/readiness.json",
+      "bytes": 19316,
+      "sha256": "397585461f323b229df9e9f23366154ecf0d87c5649ea1f54623b18ced9a17c5"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/contract.json",
+      "bytes": 18350,
+      "sha256": "c2333789e610b71b7b8c046ff53be78732f1829d64153f28289435632a0dbbcf"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/evaluations/update_00000.json",
+      "bytes": 543743,
+      "sha256": "4bbb244cef4bfa53753877295c71a62c7a11e72ed4f1a84425a677fc9598c0e7"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/evaluations/update_00064.json",
+      "bytes": 453277,
+      "sha256": "2d7b9c962208eb1bb175e111c105a05687d77b40905df9404a01d947a3a16259"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/evaluations/update_00128.json",
+      "bytes": 455898,
+      "sha256": "35e20b36b4f111ac55c72211c3bfb55a2726cec51211a6544115dc6b4b0ca851"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/evaluations/update_00192.json",
+      "bytes": 474855,
+      "sha256": "18793a84af62788168f3f65d9d6da0623b0faa5cc21163461025e20ba5a5b813"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/evaluations/update_00256.json",
+      "bytes": 475209,
+      "sha256": "4abb56c59cd3c3363ac7fcc953043f9d52a22b4bedd4b58827c5bec4d129dd41"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/finished.json",
+      "bytes": 99,
+      "sha256": "cb6ffecb12bb4015a64c3d0e23ead5ef8e46b2fd53436e211f90334a1b41f56a"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/latest.json",
+      "bytes": 82,
+      "sha256": "03f0a1ae27ddeea55be00de287ac8f9838b3d89d443eb4d2345b1ad714a90107"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/step_0000448_e6272c82afad/manifest.json",
+      "bytes": 561,
+      "sha256": "c84d5a09b2f21fb80ff00b709ce7bc3a0d17157248fc93509b11af7101b375bb"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/step_0000448_e6272c82afad/model.safetensors.json",
+      "bytes": 560,
+      "sha256": "00e3b5bc1cf131ad80d3788ba546ea155e6a3dc7fc89dbddf4885782938d4796"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/step_0000512_6972bbf9be08/manifest.json",
+      "bytes": 561,
+      "sha256": "eb254f3d11b3e40af726a7ba07a8d123f77c4271f9709bb720560203d131244e"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/step_0000512_6972bbf9be08/model.safetensors.json",
+      "bytes": 561,
+      "sha256": "418784fc8da5cea97100a704f00e646d0feac8613d49b62c0d1e6baea8574c0f"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/step_0000576_69e1ad1810fd/manifest.json",
+      "bytes": 561,
+      "sha256": "fdb1cbf6178a54ad671472ce95a003ce53c58872d9c33a48e8de783653b82f19"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/step_0000576_69e1ad1810fd/model.safetensors.json",
+      "bytes": 561,
+      "sha256": "c1826e8f9f9dde192515bde18c80b1b4a15d0e13007f11751f9beb0573d22e99"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/step_0000640_c0d26296ef71/manifest.json",
+      "bytes": 561,
+      "sha256": "c8295d027fbe49a0db971f4473b4f21bf313009db92cd54c0ff40ee019f1cfbc"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/step_0000640_c0d26296ef71/model.safetensors.json",
+      "bytes": 561,
+      "sha256": "266b4501faad19f123268121f392469564d322f044c3e6b8066c48a7800400e2"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/training.jsonl",
+      "bytes": 44724,
+      "sha256": "6fb3e2999a57d51e3bef729be494de4b30364a2b14d2e0e886eb31821d64e9d8"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/runs/sft/training.log",
+      "bytes": 10065,
+      "sha256": "5303ab85bbeac866fdb68141eb542158f8f24ee2df1bdf247a4575142eb9f12a"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/scripts/benchmark.py",
+      "bytes": 2851,
+      "sha256": "1d5bbae8c2d6262e4e8326b75af941c32a7d863b7e858a7bf17df749ccd77076"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/scripts/data.py",
+      "bytes": 13082,
+      "sha256": "36793fec90c5cd7cb504edc38c57a698115fa8ad8c51a5d97d6826403e87df38"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/scripts/launch.py",
+      "bytes": 15205,
+      "sha256": "348215af57ffeaf85ed68b1ce60413c9b43df6c8f643515bb4ee036920ae930d"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/scripts/test_data.py",
+      "bytes": 2566,
+      "sha256": "8d8bb35a656a989fc007df5602750b2f2a8baf6a86508aa7d82c76f75bdad275"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/scripts/test_evaluation.py",
+      "bytes": 2517,
+      "sha256": "eacf876f9ba03412aba3cc28d0f8bdcac1097ec1a1a6e9768b1d099c40f0144d"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/public_repair_384_v1/source_pins.json",
+      "bytes": 884,
+      "sha256": "a18cd8ba1ec4a68fb22725db75f3ea8138db5f18e9b1bcf2311ef26b9efc99c9"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/KEEP.json",
+      "bytes": 596,
+      "sha256": "718c641093a0acaba2ade0ce3e666221b0e89a49d17d44aefa90f43a4244f65e"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/benchmark-u256.log",
+      "bytes": 27100,
+      "sha256": "319c7b342cfccd03e57c60e06ca525d548e3191c5db2afb46039194469230a20"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/benchmarks/readiness/readiness_repair512-search-028-broad-public-long-stable-u256_ifeval.json",
+      "bytes": 174579,
+      "sha256": "b02861fcd54e8c6bef2ee61e5ff48c3817630908f1d3542452a582bd0f0d632b"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/benchmarks/readiness/readiness_repair512-search-028-broad-public-long-stable-u256_multiple-choice.json",
+      "bytes": 23244,
+      "sha256": "da11352661e8f5e932debde2eb8234061077ff877e81a1358d4f2b84dd422de7"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/benchmarks/repair512-search-028-broad-public-long-stable-u256/ifeval/integrity.json",
+      "bytes": 161,
+      "sha256": "f923ec455d9aaae428370bd7d573462d174d2719eaafce22178b8b4f7fffdd82"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/benchmarks/repair512-search-028-broad-public-long-stable-u256/ifeval/manifest.json",
+      "bytes": 171982,
+      "sha256": "af4b836f50fe1ae10a19eb4634d191c5b8b1ba9e1c3d6ae1d31bec95cdb8010c"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/benchmarks/repair512-search-028-broad-public-long-stable-u256/ifeval/records.jsonl",
+      "bytes": 1563553,
+      "sha256": "19155f279bb543d787f664f3c3f58790ebe5e976421057683ebbfd5dbb9fc638"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/benchmarks/repair512-search-028-broad-public-long-stable-u256/ifeval/summary.json",
+      "bytes": 613,
+      "sha256": "a6716e18e9bccbcc0d8c6cfbc84231f644f0b4baf5438ed4288483324850a58b"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/benchmarks/repair512-search-028-broad-public-long-stable-u256/multiple-choice/integrity.json",
+      "bytes": 163,
+      "sha256": "729c34c8d48b369ac8a615a50db76e8b7d1e9eef7134a1bf34849c573b79a234"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/benchmarks/repair512-search-028-broad-public-long-stable-u256/multiple-choice/manifest.json",
+      "bytes": 22145,
+      "sha256": "ef7197ac9430d226269d45fc0b7d3422c3f0a20ac2817870b13e31249d2e7bca"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/benchmarks/repair512-search-028-broad-public-long-stable-u256/multiple-choice/records.jsonl",
+      "bytes": 8905340,
+      "sha256": "aab6309b9000e5c341cd6e0ba48ab31ffa6f06cc79f25edd488006acfc212c27"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/benchmarks/repair512-search-028-broad-public-long-stable-u256/multiple-choice/summary.json",
+      "bytes": 690,
+      "sha256": "65e9d9819640a3f4f7667a984d3c3055f283bc7d47e8fb94091bca54e7dc97ec"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/checkpoint_cleanup.json",
+      "bytes": 3955,
+      "sha256": "795944665293786f9b7436a8d7df7d9a5c8586674971d311520bada82c84a39b"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/comparison.json",
+      "bytes": 4976,
+      "sha256": "ab774c75d42db577966a4231178fccab9b350af2948c8a206039627f7eed9a12"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/config.json",
+      "bytes": 2203,
+      "sha256": "d06d3552b264aab676d966a74598078955d2ea43683fc038a5401a7e42ffbe0c"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/data.json",
+      "bytes": 15034926,
+      "sha256": "a2d4d9401de47eb44030168be8bb04e865de467081019a7624110f08637f4dfa"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/negatives.json",
+      "bytes": 4229,
+      "sha256": "048cd6f1f30d998e40fdb4d76bf806e18ea66286c40d5a5defcc0843aaa0acc4"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/retained_result.json",
+      "bytes": 6583,
+      "sha256": "20a585841d228c28310a4f26024debb6ef2f2d25ae044b95530d55353c795297"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/run.log",
+      "bytes": 22776,
+      "sha256": "c8e9c9a947bfb006729b73133fdc1cc2d95b41c3ae94a6178f2ae87ac0823dc4"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/contract.json",
+      "bytes": 18725,
+      "sha256": "6268d303653c6751a1e10f1e3c38e21ad64ebd83621e228b87db087b61ef8483"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/evaluations/targeted_00000.json",
+      "bytes": 84837,
+      "sha256": "2760e5d9cd08da92ac80bc0179b74c7947976c30c6778a885d6ef0690e450653"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/evaluations/targeted_00128.json",
+      "bytes": 75746,
+      "sha256": "b6ed077ce1abc47bb460c2aadff12745a1d5218495d1429f61ccbc6f30eae2cd"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/evaluations/targeted_00256.json",
+      "bytes": 78536,
+      "sha256": "91bb4f2b016acae6d5933868987db95f661f687e88228f4c021572727960a49b"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/evaluations/targeted_00384.json",
+      "bytes": 85653,
+      "sha256": "269355ccc3f5c6af97af3a0fb82a6773cfc40c6a32e1d5ce6cc4531ee1b43d00"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/evaluations/targeted_00512.json",
+      "bytes": 80172,
+      "sha256": "c11c626ac249c6de8fa00dddfad0b87139e85153ea7bde9a589d775d7c1292de"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/evaluations/update_00000.json",
+      "bytes": 455938,
+      "sha256": "36aee7f595230203b72490104d775714920de59d513d4a67de5e38532f1a52cf"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/evaluations/update_00128.json",
+      "bytes": 452121,
+      "sha256": "1f7b09e2a166a8392b6c51e00fca4d9075110d7be0c7d6910c47595e344bad61"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/evaluations/update_00256.json",
+      "bytes": 444583,
+      "sha256": "886bc64ccb4e69ebb2d88c20279518e2cb15a8c126dd5435487cb7bf136e880a"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/evaluations/update_00384.json",
+      "bytes": 443219,
+      "sha256": "60e25041a1f05af6f18a94eb5d21c6a995fb9e35c37b7b7b5c938fa598fe2c54"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/evaluations/update_00512.json",
+      "bytes": 456062,
+      "sha256": "40b98ff3a9c54fbd611fb2c7efc48f9aae0c1e7ed9b6b48cc9a600d014fd9b82"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/finished.json",
+      "bytes": 268,
+      "sha256": "0f144440bfc330e533de811655b5c2fcb2188ae737a06d509440f2b33f386aca"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/latest.json",
+      "bytes": 82,
+      "sha256": "b15a5860caf7b126ca2e8ef02718bdccce16faa32e48f8e699b5df9567561e37"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/step_0000768_4980617cb301/manifest.json",
+      "bytes": 561,
+      "sha256": "1bc8d8350f20ac9c5cd69ad79c55b1120c0830dea30000c7576e140b5d2d22a3"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/step_0000768_4980617cb301/model.safetensors.json",
+      "bytes": 525,
+      "sha256": "7c0e307d7bc1973a1d5b8c683b2911c4dd2fead65d49226cdce53d6dfe098908"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/runs/sft/training.jsonl",
+      "bytes": 148139,
+      "sha256": "90bf86c280f79327762c752059cb65d9cbf134f9bbd696b9f59698bdc8e82758"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/selection.json",
+      "bytes": 1832,
+      "sha256": "50fca08ffe0fc3d0cd759ef4ec09509bb836ff92ea09932111db43edf78e7d9d"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/repair512_search_v1/candidates/028-broad-public-long-stable/source_pins.json",
+      "bytes": 1020,
+      "sha256": "11baf5a81fd28f9e2e114cc7e1c050172175745e168154d2004590925c4577e4"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/README.md",
+      "bytes": 6144,
+      "sha256": "fa54b6faca34e079e2d98a5fa61c46bd4b365d346224d1ea65db76d56fe36e02"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/audits/384_review_v1/additional_findings.json",
+      "bytes": 5310,
+      "sha256": "86955ddde9aa5813f0adc3b1aefae6f15789179c8e93a3ebcc78128136e0a83f"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/audits/384_review_v1/analyze.py",
+      "bytes": 8908,
+      "sha256": "4d5c47d2b9f84bc87ccf33d3b71365e9befcd36706627871ad0efb9b18a927ab"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/audits/384_review_v1/build_report.py",
+      "bytes": 20320,
+      "sha256": "9f0058c6a23f6dcc0e41970e2f236b4775151a74060df1e01b2bd4d03fb7cc14"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/audits/384_review_v1/ifeval_all_annotated.json",
+      "bytes": 1322865,
+      "sha256": "2a11935866624a92879df0f79643fcb69e61b3f5d70f11fec35338d2d6857f51"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/audits/384_review_v1/legacy_chat_all.json",
+      "bytes": 29754,
+      "sha256": "09d26878651c6a4b1e82e0c4aee8094d578765d64c1f6463089d2fcd933b5380"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/audits/384_review_v1/receipt.json",
+      "bytes": 1238,
+      "sha256": "066f55cca00422c993b5950ed59003347d1e2cc9b3a9a54c27a0d5ab308636d4"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/audits/384_review_v1/recovery_all.json",
+      "bytes": 50089,
+      "sha256": "47fe12c3f7374db96704419eec237cf41966b6c6e8ed5266d5cf9442427f587d"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/audits/384_review_v1/summary.json",
+      "bytes": 12875,
+      "sha256": "8a59b73fca8edba04e1a09d1a9cd753350d094adb98c54983ceb0cacd4453917"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/audits/384_review_v1/training_refusal_markers.json",
+      "bytes": 635188,
+      "sha256": "64c1acdecc5d18d6c2d1854ea6ed309c6922aa1b893fea94af59f039e85d6761"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/baselines/retained_768.json",
+      "bytes": 480232,
+      "sha256": "37cd576a70e3981dc95062cb1639e8c5dcf0220325dd9bcf513be0dc4d5e4f43"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/baselines/retained_768_previous_list_definition.json",
+      "bytes": 480163,
+      "sha256": "170b55dd225cb6aae7fef8c02eb15d855b64b56bf014a8636a4011ba3e602572"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/benchmarks/readiness/readiness_unified-text-v2-384_ifeval.json",
+      "bytes": 174984,
+      "sha256": "73072666f43a3e684b2eaac4f6c01fbdcc664c4ddce6faa15e97fb21cc0f9cae"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/benchmarks/readiness/readiness_unified-text-v2-384_multiple-choice.json",
+      "bytes": 23570,
+      "sha256": "dcfb1c954c61718cd2cdb11d179a8a1dc6c5d5c4de759045b2f5dd4e2ba27e3f"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/benchmarks/unified-text-v2-384/ifeval/integrity.json",
+      "bytes": 161,
+      "sha256": "c2ad62d77a74faf44598b992beaf18f4b1603f8d28a4bc0c5d50646cced3ff3d"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/benchmarks/unified-text-v2-384/ifeval/manifest.json",
+      "bytes": 172301,
+      "sha256": "dd0d989aadb4bd6d2c046cb3203f8167aee28d1205de6e52d896c54252296409"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/benchmarks/unified-text-v2-384/ifeval/records.jsonl",
+      "bytes": 2616383,
+      "sha256": "b1c9b1d3bc18878e1b3c33b57edb534ff39959507e86163cc4901e3a7400effa"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/benchmarks/unified-text-v2-384/ifeval/summary.json",
+      "bytes": 614,
+      "sha256": "913db00ad90df8294541d2ff8d0f98b2b26b08d4b9c0c56230a3c0d4a60cd6d8"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/benchmarks/unified-text-v2-384/multiple-choice/integrity.json",
+      "bytes": 163,
+      "sha256": "4b3da697314fe4ac2573820dfcb69396d0890aa7b1f1287535f24489f1552131"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/benchmarks/unified-text-v2-384/multiple-choice/manifest.json",
+      "bytes": 22464,
+      "sha256": "bfc12c9da0a40c9f04b6d46f08898b7be6108113faabe40d3aee1fa928acbfd9"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/benchmarks/unified-text-v2-384/multiple-choice/records.jsonl",
+      "bytes": 8905737,
+      "sha256": "8fc77af506dd4814d249e0f68f15a1f13b1154cdec1fd0d376a72f2521d8db6e"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/benchmarks/unified-text-v2-384/multiple-choice/summary.json",
+      "bytes": 690,
+      "sha256": "260e6ce6208113ba6596d8eacf15a8c24f4484dc3dcab72b091c9d921e866b76"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/comparisons/step_00384.json",
+      "bytes": 2029,
+      "sha256": "77029aa3d4a5c4bb4a1e8afecc7a948ad8c5d800c9dd97c1d535dc44660802a2"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/comparisons/step_00384.md",
+      "bytes": 943,
+      "sha256": "2b59fca5c8548e351799f9810e52a8dd64f31a258ef7072fd8e3a41fa7ce6fc2"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/config.json",
+      "bytes": 914,
+      "sha256": "a7086f6cf57bad7171c011ff2949580ffefc0e5cd23fe7f254f5328dbd642ede"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/data/prepared.json",
+      "bytes": 47629159,
+      "sha256": "14e3e79cce495d880feddd2b905622157b5b796ee43ae06965a86dfb5fff3b1f"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/readiness.json",
+      "bytes": 19772,
+      "sha256": "41fa86eae50cfaef7679d0a40c0814175cb42b170109b279064b8f8b3ea115f5"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/contract.json",
+      "bytes": 18745,
+      "sha256": "da074b0d3ca56df597587d2390613e7824f48c774f41d59650fff46bc9b0e1f5"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00000.json",
+      "bytes": 2063118,
+      "sha256": "4171276fec78087bdeae37747e85617e4f9853573259e1a530f65b3cba739401"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00000.md",
+      "bytes": 148982,
+      "sha256": "71b4b468a12cc9f3cca21eb287ceb347f608544413a62576dd4831d0b4d99c5d"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00064.json",
+      "bytes": 473373,
+      "sha256": "845820f20f781b671ea1505539c610fe6f6488dc4796a592ec8086d13d83483d"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00064.md",
+      "bytes": 50304,
+      "sha256": "506d45f704d5810a598ae9b7a801cba66fb941c50ad13188891e51d4398d26c8"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00128.json",
+      "bytes": 480191,
+      "sha256": "130ae60a75f29b724ea6de590124760602b90be0591c82559414a6db252c0e96"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00128.md",
+      "bytes": 47755,
+      "sha256": "1788f909a719cf0af0591a36ae2a9fd88c2304d1ac25d50d5701e9ca395fa7ff"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00256.json",
+      "bytes": 486927,
+      "sha256": "24a7697fbece4b32f6996be399101e44a8fd48e1df2474bdf22d216fa861bca1"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00256.md",
+      "bytes": 46403,
+      "sha256": "a7db00bfdffa9d22dd211ee5794e40cc9d09a109aeb5de1b5c29639fee22d29d"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00384.json",
+      "bytes": 476887,
+      "sha256": "7e897668a592b7eaa0d94090ee48eca8aaaacf3e53334b4c9181180be9170b42"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00384.md",
+      "bytes": 47974,
+      "sha256": "729e9ff66707bead9503df53b9a29a96385f3d43e0210cf78e0d800be35fc95d"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00512.json",
+      "bytes": 484202,
+      "sha256": "c51ebd7375b3e051730d9be57ce0e8896254a93000f121b9a27311a2db1998ca"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00512.md",
+      "bytes": 46758,
+      "sha256": "44a2a12128769799999f4181dd1d9a6bafb0335c6b5418f4841b4ad2b1b4aa6c"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00640.json",
+      "bytes": 491169,
+      "sha256": "a8633be13429a7fea13714366b8df57d08d0ab397e926d14c25a9c6c0825f42f"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00640.md",
+      "bytes": 47990,
+      "sha256": "a3f2ecebff5c145c8a3a449e3208af65f608e280ffb91b1af0ac2514ef5cdc1c"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00768.json",
+      "bytes": 493756,
+      "sha256": "cf06f3df28190830a589ad0f061514bd32eaf5552fdfa335cf7b0e0279d452a9"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/evaluations/update_00768.md",
+      "bytes": 49557,
+      "sha256": "b920ecef4c3b2a41b7ba7f86868319c47ba80187a9c30ab5c29d6fbba000ec6f"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/inputs/benchmark.py",
+      "bytes": 2936,
+      "sha256": "df13c3567c3723a99d2685347b0ebe9bb42c9fececaf0b7f212b84ecb6dc7e78"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/inputs/compare.py",
+      "bytes": 2146,
+      "sha256": "a58478ebd74e944450f8c9fbe567bc85f361c6b1528a103efb7326ae0bce0f6f"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/inputs/config.json",
+      "bytes": 914,
+      "sha256": "a7086f6cf57bad7171c011ff2949580ffefc0e5cd23fe7f254f5328dbd642ede"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/inputs/data.py",
+      "bytes": 14082,
+      "sha256": "f09cfdf9a1fc0fea8384f7ab6890026b0f653fac9f9506b455c65d6de09224a6"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/inputs/evaluate.py",
+      "bytes": 2912,
+      "sha256": "41e1ecf0ed1da7d5e6b9513c298e82636f3991bc5c4a00a3f6f172f68f161cdc"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/inputs/launch.py",
+      "bytes": 18026,
+      "sha256": "ecad7943c55810d9da4c4043e7eb0c2e1f6939d2b038f24250f85af89a1924e5"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/latest.json",
+      "bytes": 82,
+      "sha256": "29baea72db44893e925638234536df3a651db7a55082baa200789895e327048b"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/report.json",
+      "bytes": 190,
+      "sha256": "a34abd46eb396062a8f44bea7f4481264cc90f2317aa34d71635e2608e2a925b"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000064_26e5f8d01db6/manifest.json",
+      "bytes": 560,
+      "sha256": "9722ab5768a9792a39f302af69abcaf113e3eb36b5d01fb2500b75257b57b08e"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000064_26e5f8d01db6/model.safetensors.json",
+      "bytes": 572,
+      "sha256": "134dfc9bd5dcd077a48b31863f6aa7bebc10c4b861db3a678611683eaf82d7e0"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000128_158ec2e1d74f/manifest.json",
+      "bytes": 561,
+      "sha256": "51d5fa5286917a59a054b14aca69e223bb7f8a9e07461d85332394fa762603ee"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000128_158ec2e1d74f/model.safetensors.json",
+      "bytes": 574,
+      "sha256": "caed696ea736885cac4b4cff1b422efece12c8b20b2101f325cec37bf6f46e1a"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000256_b0e8d0cf7cb2/manifest.json",
+      "bytes": 561,
+      "sha256": "8520bed611f0e04470461e48e31647dc24ddada0e8268f89c1740a1581599ad4"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000256_b0e8d0cf7cb2/model.safetensors.json",
+      "bytes": 574,
+      "sha256": "0a1ef903461c68f8e31ff7848758534380b88e236dd7ef48cd0ab1cd669f5b0f"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000384_807628c9dbdc/manifest.json",
+      "bytes": 561,
+      "sha256": "1010a05a709e3da8fffd73deb8ce649026e4ed212b01a762ad569c15edaee9a1"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000384_807628c9dbdc/model.safetensors.json",
+      "bytes": 574,
+      "sha256": "ca07ddc3db511d5458397e4d0651dc791f0e3ac6b0279a2f887dd4ef7df5bc6b"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000512_0d93b325c545/manifest.json",
+      "bytes": 561,
+      "sha256": "8c9ed76764064f2a3a1836a07075ce2cd18de95b22240f55e43f1805cfeacf62"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000512_0d93b325c545/model.safetensors.json",
+      "bytes": 574,
+      "sha256": "0413ed5c82b1bf3e5fb58d87aafc4e8472cf0a6ef99a259f6a0db890aedd250d"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000640_000f4e4da94e/manifest.json",
+      "bytes": 561,
+      "sha256": "b4fe64901c96a87bf1510efda315ebfa3005e6d16dcc0c780534cba015af0f36"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000640_000f4e4da94e/model.safetensors.json",
+      "bytes": 575,
+      "sha256": "3d30d9b7ec060c3f5f8414930544ebe9206cb2a855025406b3bb26566fd7a5f2"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000768_bbb87b1b9e42/manifest.json",
+      "bytes": 561,
+      "sha256": "da58cd27f19dfd5e2842552bb3d6fe2f80808d141c8774fa2f4dc72dbeaa0540"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/step_0000768_bbb87b1b9e42/model.safetensors.json",
+      "bytes": 575,
+      "sha256": "8d3b044abedcd35b9583fc365aee8d081e783a32ed18f45323dd1cdd59262c7a"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/training.jsonl",
+      "bytes": 221436,
+      "sha256": "1f0f46d77253cbbdf594702f707de45150fadad89f1a2a4ce752a057c06463ce"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/runs/sft/training.log",
+      "bytes": 24809,
+      "sha256": "13b9180dff22b0c1dc4be75b6203ce405e1ef43ab131155d2a3f9b3215f0dc6a"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/scripts/benchmark.py",
+      "bytes": 2936,
+      "sha256": "df13c3567c3723a99d2685347b0ebe9bb42c9fececaf0b7f212b84ecb6dc7e78"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/scripts/compare.py",
+      "bytes": 2146,
+      "sha256": "a58478ebd74e944450f8c9fbe567bc85f361c6b1528a103efb7326ae0bce0f6f"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/scripts/data.py",
+      "bytes": 14082,
+      "sha256": "f09cfdf9a1fc0fea8384f7ab6890026b0f653fac9f9506b455c65d6de09224a6"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/scripts/evaluate.py",
+      "bytes": 2912,
+      "sha256": "41e1ecf0ed1da7d5e6b9513c298e82636f3991bc5c4a00a3f6f172f68f161cdc"
+    },
+    {
+      "path": "sml-mlx-v2/experiments/unified_text_v2_sft_v1/scripts/launch.py",
+      "bytes": 18026,
+      "sha256": "ecad7943c55810d9da4c4043e7eb0c2e1f6939d2b038f24250f85af89a1924e5"
+    },
+    {
+      "path": "sml-mlx-v2/sft/conversation_foundation_v1/prepared.json",
+      "bytes": 179779711,
+      "sha256": "ae8a93a306bda7b996e33292ad9e25c9efda11379f772ccadad054430975dd29"
+    },
+    {
+      "path": "sml-mlx-v2/sft/grounded_rank_384/prepared.json",
+      "bytes": 5298180,
+      "sha256": "cf0a379a6bd78440152a57ffc1d98853de6ac25932052926e1d451f44d5a9454"
+    },
+    {
+      "path": "sml-mlx-v2/sft/preference_long_640/prepared.json",
+      "bytes": 29979623,
+      "sha256": "56d24fa9b368a17870c82e10320c2286297f3b6d389c50afb099822dce21af13"
+    },
+    {
+      "path": "sml-mlx-v2/sft/text_followup_512_v1/prepared.json",
+      "bytes": 14235573,
+      "sha256": "f7676dd515a9755bd26d460b486eb940e484081ae8910c044008f03bada61c51"
+    }
+  ],
+  "archive": "OpenSML-150M-training-records.tar.gz",
+  "archive_sha256": "45d442f2a9713b73f14d4301d96a0c4b1d963985c706b3c7a7ec9631029503a3"
+}

TRIAL28_RESULTS.json

@@ -0,0 +1,262 @@
+{
+  "checkpoint": "OpenSML-150M",
+  "bundle": "step_0000768_4980617cb301",
+  "weight_sha256": "cbd3e3fb4ada74d7264371b79cee7598f513b7b950f1f141ef9f1a43bd2e1b4e",
+  "parent_weight_sha256": "465ce42aad2e7c765f169aa75aa4124ec03fec17b46e6f5eabf75c2c4a8d4772",
+  "base_weight_sha256": "ca9bd5d82005f28e77f319d3a5a29f29fb4e4f86e7ceea049f01162fd8aae80f",
+  "lineage": [
+    "Pretrained Stage B step 73243",
+    "Unified384",
+    "Repair512",
+    "Trial28 +256"
+  ],
+  "total_sft_step": 768,
+  "trial_additional_updates": 256,
+  "trial_planned_updates": 512,
+  "trial_consumed_records": 4096,
+  "selection": "Owner-selected current baseline; exploratory selection after repeated benchmark inspection; chat-repetition development gate failed",
+  "trial_config": {
+    "anchor_replay_only": true,
+    "automatic_selection": false,
+    "balanced_instructions": false,
+    "batch_conversations": 16,
+    "batch_counts": {
+      "chat": 4,
+      "grounded": 4,
+      "human": 4,
+      "instructions": 4
+    },
+    "betas": [
+      0.9,
+      0.95
+    ],
+    "ce_weight": 1.0,
+    "checkpoint_keep": 4,
+    "clip_norm": 1.0,
+    "context": 2048,
+    "data_focused": false,
+    "dataset_plan": "breadth-stable",
+    "dpo_beta": 0.1,
+    "dpo_weight": 0.0,
+    "epochs": 1,
+    "evaluation_updates": [
+      0,
+      128,
+      256,
+      384,
+      512
+    ],
+    "final_lr": 2e-07,
+    "format_preference_beta": 5.0,
+    "format_preference_weight": 0.0,
+    "freeze_embeddings": false,
+    "fresh_per_batch": 4,
+    "generated_history": false,
+    "generation_conversations": 56,
+    "generation_per_source": {
+      "chat": 8,
+      "grounded": 8,
+      "human": 8,
+      "instructions": 32
+    },
+    "holdout_per_source": 32,
+    "instruction_edge_weight": 1.0,
+    "instruction_weight": 1.0,
+    "kl_weight": 2.0,
+    "max_assistant_tokens": 192,
+    "max_new_tokens": 384,
+    "maximum_instruction_word_count": 120,
+    "method": "Broader verified public instruction/annotated QA mixture; two fresh passes plus repair512 replay; assistant-only CE + EOS; repetition penalty .3 and replay KL anchor 2",
+    "minimum_free_gib": 12,
+    "name": "public-repair-384-v1",
+    "negative_replay_only": true,
+    "on_policy": true,
+    "peak_lr": 2e-06,
+    "policy_kl_weight": 0.0,
+    "ranking_weight": 0.0,
+    "reference_aware": true,
+    "repetition_allowance": 2,
+    "repetition_ngram": 4,
+    "replay_only": false,
+    "seed": 202610011,
+    "source_caps": {
+      "chat": 192,
+      "grounded": 160,
+      "human": 160,
+      "instructions": 192
+    },
+    "source_model_sha256": "465ce42aad2e7c765f169aa75aa4124ec03fec17b46e6f5eabf75c2c4a8d4772",
+    "source_step": 512,
+    "train_all_norms": false,
+    "train_final_norm": false,
+    "train_last_blocks": 0,
+    "training_counts": {
+      "chat": 512,
+      "grounded": 2560,
+      "human": 1536,
+      "instructions": 3584
+    },
+    "ul_weight": 0.3,
+    "updates": 512,
+    "validation_conversations": 128,
+    "warmup_updates": 16,
+    "weight_decay": 0.0,
+    "weighting": "conversation"
+  },
+  "source_pins": {
+    "constraints": {
+      "file": "data/smol-constraints/train-00000-of-00001.parquet",
+      "license": "Apache-2.0",
+      "local_file": "constraints.parquet",
+      "repo": "HuggingFaceTB/smoltalk",
+      "revision": "5feaf2fd3ffca7c237fc38d1861bc30365d48ffa",
+      "sha256": "3369eaff911d3511ee21561a3b8607e1acfd773a6ade80f4759490ffdcce7ba4",
+      "split": "train"
+    },
+    "squad": {
+      "file": "squad_v2/train-00000-of-00001.parquet",
+      "license": "CC-BY-SA-4.0",
+      "local_file": "squad.parquet",
+      "repo": "rajpurkar/squad_v2",
+      "revision": "3ffb306f725f7d2ce8394bc1873b24868140c412",
+      "sha256": "f6da32ffb482ff463ad056477740d1bb284b96a45db3a08bee6a225ca6abf291",
+      "split": "train"
+    },
+    "sciq": {
+      "repo": "allenai/sciq",
+      "revision": "2c94ad3e1aafab77146f384e23536f97a4849815",
+      "file": "data/train-00000-of-00001.parquet",
+      "local_file": "sciq.parquet",
+      "license": "CC-BY-NC-3.0",
+      "split": "train",
+      "sha256": "19644360954006d06e9ad3df07bddb34f8535c081b831d48f604603c713ac167"
+    }
+  },
+  "benchmarks": {
+    "multiple-choice": {
+      "summary": {
+        "completed": 15428,
+        "expected": 15428,
+        "status": "complete",
+        "tasks": {
+          "arc_challenge": {
+            "acc": 0.26023890784982934,
+            "acc_norm": 0.295221843003413,
+            "completed": 1172,
+            "expected": 1172
+          },
+          "arc_easy": {
+            "acc": 0.5664983164983165,
+            "acc_norm": 0.5542929292929293,
+            "completed": 2376,
+            "expected": 2376
+          },
+          "hellaswag": {
+            "acc": 0.30551682931686913,
+            "acc_norm": 0.33937462656841266,
+            "completed": 10042,
+            "expected": 10042
+          },
+          "piqa": {
+            "acc": 0.6452665941240479,
+            "acc_norm": 0.6409140369967355,
+            "completed": 1838,
+            "expected": 1838
+          }
+        },
+        "training": false
+      },
+      "integrity": {
+        "completed": 15428,
+        "inputs_unchanged": true,
+        "manifest_sha256": "ef7197ac9430d226269d45fc0b7d3422c3f0a20ac2817870b13e31249d2e7bca",
+        "training": false
+      },
+      "protocol": {
+        "decode_mode": "cached",
+        "eos": 1,
+        "extra_stop_strings": [],
+        "greedy": true,
+        "harness": "b954108c9baaaa934b4ad842033b31a97ee30816",
+        "ifeval_template": "User: {unchanged prompt}\nAssistant:",
+        "max_new_tokens": 1280,
+        "mc_metrics": [
+          "acc",
+          "acc_norm"
+        ],
+        "mc_template": "plain official question/continuation; no chat/BOS/EOS",
+        "repetition_penalty": 1,
+        "seed": 24092026
+      },
+      "fp32": true,
+      "attention": "MLX explicit vanilla attention; PyTorch equivalence not claimed",
+      "batch_size": 1,
+      "context": 2048,
+      "prepared_sha256": "88b30e3a273f6817e9af1e360dbeb44714c58f2b345ca875a0ad83a0e8c05da5"
+    },
+    "ifeval": {
+      "summary": {
+        "completed": 541,
+        "expected": 541,
+        "loose": {
+          "instruction_accuracy": 0.2577937649880096,
+          "instruction_correct": 215,
+          "instruction_total": 834,
+          "prompt_accuracy": 0.15711645101663585,
+          "prompt_correct": 85,
+          "prompt_total": 541
+        },
+        "status": "complete",
+        "stop_counts": {
+          "context_limit": 0,
+          "eos": 508,
+          "length": 33
+        },
+        "strict": {
+          "instruction_accuracy": 0.2529976019184652,
+          "instruction_correct": 211,
+          "instruction_total": 834,
+          "prompt_accuracy": 0.15157116451016636,
+          "prompt_correct": 82,
+          "prompt_total": 541
+        },
+        "training": false
+      },
+      "integrity": {
+        "completed": 541,
+        "inputs_unchanged": true,
+        "manifest_sha256": "af4b836f50fe1ae10a19eb4634d191c5b8b1ba9e1c3d6ae1d31bec95cdb8010c",
+        "training": false
+      },
+      "protocol": {
+        "decode_mode": "cached",
+        "eos": 1,
+        "extra_stop_strings": [],
+        "greedy": true,
+        "harness": "b954108c9baaaa934b4ad842033b31a97ee30816",
+        "ifeval_template": "User: {unchanged prompt}\nAssistant:",
+        "max_new_tokens": 1280,
+        "mc_metrics": [
+          "acc",
+          "acc_norm"
+        ],
+        "mc_template": "plain official question/continuation; no chat/BOS/EOS",
+        "repetition_penalty": 1,
+        "seed": 24092026
+      },
+      "fp32": true,
+      "attention": "MLX explicit vanilla attention; PyTorch equivalence not claimed",
+      "batch_size": 1,
+      "context": 2048,
+      "prepared_sha256": "88b30e3a273f6817e9af1e360dbeb44714c58f2b345ca875a0ad83a0e8c05da5"
+    }
+  },
+  "source_record_sha256": {
+    "config.json": "d06d3552b264aab676d966a74598078955d2ea43683fc038a5401a7e42ffbe0c",
+    "selection.json": "50fca08ffe0fc3d0cd759ef4ec09509bb836ff92ea09932111db43edf78e7d9d",
+    "source_pins.json": "11baf5a81fd28f9e2e114cc7e1c050172175745e168154d2004590925c4577e4",
+    "retained_result.json": "20a585841d228c28310a4f26024debb6ef2f2d25ae044b95530d55353c795297"
+  },
+  "internal_experiment": "Trial 28 \u2014 Repair512 +256",
+  "public_model_name": "OpenSML-150M"
+}

checkpoint_provenance.json

@@ -0,0 +1,38 @@
+{
+  "model_name": "OpenSML-150M",
+  "step": 768,
+  "additional_updates": 256,
+  "weights_sha256": "cbd3e3fb4ada74d7264371b79cee7598f513b7b950f1f141ef9f1a43bd2e1b4e",
+  "source_metadata": {
+    "additional_updates": 256,
+    "automatic_promotion": false,
+    "contract": "e7ad0c5c033aa86c115458bb4be6efe1d20e9d0c73aa8231480e16ff6f1f4fdd",
+    "experimental": true,
+    "source_bundle": "experiments/public_repair_384_v1/runs/sft/step_0000512_6972bbf9be08",
+    "source_model_sha256": "465ce42aad2e7c765f169aa75aa4124ec03fec17b46e6f5eabf75c2c4a8d4772",
+    "source_step": 512,
+    "step": 768,
+    "tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb",
+    "training_format": "plain-user-assistant-eos-v1"
+  },
+  "source_manifest": {
+    "created_ns": 1790893470379935000,
+    "files": {
+      "model.safetensors": {
+        "bytes": 601772434,
+        "sha256": "cbd3e3fb4ada74d7264371b79cee7598f513b7b950f1f141ef9f1a43bd2e1b4e"
+      },
+      "model.safetensors.json": {
+        "bytes": 525,
+        "sha256": "7c0e307d7bc1973a1d5b8c683b2911c4dd2fead65d49226cdce53d6dfe098908"
+      },
+      "model.safetensors.optimizer.safetensors": {
+        "bytes": 1805319748,
+        "sha256": "a23dd9b5ca2af99ac3a7648a7b9a70acb04c183aacbea71fdaabd2a1fdc2c720"
+      }
+    },
+    "format": "sml-pretrain-bundle-v1",
+    "step": 768
+  },
+  "export": "Exact unchanged FP32 weights; optimizer tensors excluded. Native reference MLX inference."
+}

config.json

@@ -0,0 +1,24 @@
+{
+  "model_name": "OpenSML-150M",
+  "format": "opensml-native-mlx-v1",
+  "model": {
+    "attention_impl": "vanilla",
+    "bias": false,
+    "ce_impl": "reference",
+    "d_model": 768,
+    "ffn_impl": "reference",
+    "loss_dtype": "float32",
+    "max_seq_len": 2048,
+    "mlp_multiple_of": 256,
+    "mlp_ratio": 4.0,
+    "n_heads": 12,
+    "n_kv_heads": 4,
+    "n_layers": 20,
+    "qk_norm": true,
+    "vocab_size": 32000
+  },
+  "training_format": "plain-user-assistant-eos-v1",
+  "dtype": "float32",
+  "context_length": 2048,
+  "weights_sha256": "cbd3e3fb4ada74d7264371b79cee7598f513b7b950f1f141ef9f1a43bd2e1b4e"
+}

corpus.json

@@ -0,0 +1,60 @@
+{
+  "max_bytes": 200000,
+  "max_rows_per_source": 5000000,
+  "min_chars": 200,
+  "name": "sml-v2-educational-prose-v1",
+  "seed": 7337,
+  "shuffle_buffer": 10000,
+  "sources": [
+    {
+      "config": "fineweb-edu-dedup",
+      "label": "web",
+      "license": "ODC-BY; preserve underlying source notices",
+      "repo": "HuggingFaceTB/smollm-corpus",
+      "revision": "3ba9d605774198c5868892d7a8deda78031a781f",
+      "split": "train",
+      "text_field": "text",
+      "weight": 55
+    },
+    {
+      "label": "dclm",
+      "license": "CC-BY-4.0",
+      "min_score": {
+        "field": "edu_int_score",
+        "value": 3
+      },
+      "repo": "HuggingFaceTB/dclm-edu",
+      "revision": "dbad8ad71224482740cd9c9d353591adbf62fe04",
+      "split": "train",
+      "text_field": "text",
+      "weight": 25
+    },
+    {
+      "config": "en",
+      "label": "wiki",
+      "license": "CC-BY-SA-4.0 / GFDL",
+      "repo": "HuggingFaceFW/finewiki",
+      "revision": "8bd13e72e6a002407649b3e898535f42ceb1aeb9",
+      "split": "train",
+      "text_field": "text",
+      "weight": 10
+    },
+    {
+      "config": "cosmopedia-v2",
+      "format_contains": [
+        "textbook",
+        "tutorial",
+        "blog",
+        "educational"
+      ],
+      "label": "cosmopedia",
+      "license": "ODC-BY; synthetic content; preserve provenance",
+      "repo": "HuggingFaceTB/smollm-corpus",
+      "revision": "3ba9d605774198c5868892d7a8deda78031a781f",
+      "split": "train",
+      "text_field": "text",
+      "weight": 10
+    }
+  ],
+  "split_buckets": 10000
+}

inference.py

@@ -0,0 +1,36 @@
+"""Standalone OpenSML-150M inference on Apple Silicon using native MLX."""
+import argparse,json
+from pathlib import Path
+import mlx.core as mx
+from native_model import TransformerConfig,TransformerLM,count_parameters
+from native_tokenizer import Tokenizer
+from native_utils import file_sha256
+
+def load_model(directory):
+ root=Path(directory).resolve();cfg=json.loads((root/'config.json').read_text())
+ if cfg['format']!='opensml-native-mlx-v1':raise ValueError('Unsupported model format')
+ if file_sha256(root/'model.safetensors')!=cfg['weights_sha256']:raise ValueError('Weights checksum mismatch')
+ tok=Tokenizer(root)
+ if tok.vocab_size!=cfg['model']['vocab_size']:raise ValueError('Tokenizer/model vocabulary mismatch')
+ model=TransformerLM(TransformerConfig(**cfg['model']));model.load_weights(str(root/'model.safetensors'),strict=True);model.eval();mx.eval(model.parameters())
+ return model,tok
+
+def generate(model,tokenizer,prompt,max_new_tokens=128,raw_completion=False):
+ if max_new_tokens<1:raise ValueError('max_new_tokens must be positive')
+ text=prompt if raw_completion else f'User: {prompt}\nAssistant:'
+ ids=tokenizer.encode(text)
+ if not ids or len(ids)+max_new_tokens>model.cfg.max_seq_len:raise ValueError('Prompt plus output budget must fit the 2048-token context')
+ generated=[];caches=None;inputs=mx.array([ids]);stop='max_new_tokens'
+ for _ in range(max_new_tokens):
+  logits,caches=model.step(inputs,caches=caches)
+  token=int(mx.argmax(logits,axis=-1).item());generated.append(token)
+  if token==tokenizer.eos:stop='eos';break
+  inputs=mx.array([[token]])
+ answer=generated[:-1] if generated and generated[-1]==tokenizer.eos else generated
+ return {'text':tokenizer.decode(answer),'token_ids':generated,'stop_reason':stop,'prompt_tokens':len(ids),'generated_tokens':len(generated)}
+
+def main():
+ p=argparse.ArgumentParser(description=__doc__);p.add_argument('--model-directory',type=Path,default=Path(__file__).resolve().parent);p.add_argument('--prompt',required=True);p.add_argument('--max-new-tokens',type=int,default=128);p.add_argument('--raw-completion',action='store_true',help='Skip the User/Assistant wrapper for base-style completion prompts');a=p.parse_args()
+ model,tok=load_model(a.model_directory)
+ print(json.dumps(generate(model,tok,a.prompt,a.max_new_tokens,a.raw_completion),ensure_ascii=False,indent=2))
+if __name__=='__main__':main()

manifest.json

@@ -0,0 +1,31 @@
+{
+  "boundary_policy": "raw-text-then-one-document-EOS-v1",
+  "corpus_fingerprint": "a89f69d6f63cfa4385da233693a445404234a18194170fe1d2b4a74bf501f5d6",
+  "format": "sml-v2-tokenizer-v1",
+  "recipe": {
+    "corpus": "a89f69d6f63cfa4385da233693a445404234a18194170fe1d2b4a74bf501f5d6",
+    "min_frequency": 2,
+    "sample_manifest": "30ead6f502f11e99b479e3711a81ce7198787408d0700153615fc3255b244f46",
+    "specials": [
+      "<|pad|>",
+      "<|doc_end|>",
+      "<|turn_start|>",
+      "<|turn_end|>"
+    ],
+    "tokenizers": "0.22.2",
+    "version": 1,
+    "vocab_size": 32000
+  },
+  "sha256": {
+    "corpus.json": "e4c47b7b6aba08f948de7c2c7cb94c58dbd09dde2ded6ac185c60856ee686d27",
+    "tokenizer.json": "e095f0413a11fa91fbb58f37d00e4c6bcda59beb89b28f6f680db1436a7e8891",
+    "tokenizer_config.json": "619f500553367eb76f5be00a98625130ec857b9c367561334984096c029ba2a9"
+  },
+  "special_ids": {
+    "eos": 1,
+    "pad": 0,
+    "turn_end": 3,
+    "turn_start": 2
+  },
+  "vocab_size": 32000
+}

native_model.py

@@ -0,0 +1,278 @@
+#!/usr/bin/env python3
+"""MLX-only transformer model for small causal language modeling."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+import mlx.core as mx
+import mlx.nn as nn
+import mlx.nn.losses as losses
+
+
+@dataclass
+class TransformerConfig:
+    vocab_size: int
+    max_seq_len: int = 1024
+    d_model: int = 768
+    n_heads: int = 12
+    n_kv_heads: Optional[int] = None
+    n_layers: int = 12
+    mlp_ratio: float = 4.0
+    mlp_multiple_of: int = 256
+    rope_base: float = 10000.0
+    bias: bool = False
+    qk_norm: bool = True
+    attention_impl: str = "fast"
+    loss_dtype: str = "float32"
+    ce_impl: str = "reference"
+    ffn_impl: str = "reference"
+
+    def __post_init__(self) -> None:
+        if self.vocab_size <= 0:
+            raise ValueError("vocab_size must be > 0")
+        if self.max_seq_len <= 0:
+            raise ValueError("max_seq_len must be > 0")
+        if self.d_model <= 0:
+            raise ValueError("d_model must be > 0")
+        if self.n_heads <= 0 or self.d_model % self.n_heads != 0:
+            raise ValueError("n_heads must divide d_model")
+        if self.n_kv_heads is None:
+            self.n_kv_heads = self.n_heads
+        if self.n_kv_heads <= 0 or self.n_heads % self.n_kv_heads != 0:
+            raise ValueError("n_kv_heads must be positive and divide n_heads")
+        if self.n_layers <= 0:
+            raise ValueError("n_layers must be > 0")
+        if self.mlp_ratio <= 1.0:
+            raise ValueError("mlp_ratio must be > 1.0")
+        if self.mlp_multiple_of <= 0:
+            raise ValueError("mlp_multiple_of must be > 0")
+        if self.attention_impl not in {"fast", "vanilla"}:
+            raise ValueError("attention_impl must be one of: fast, vanilla")
+        if self.loss_dtype not in {"float32", "model"}:
+            raise ValueError("loss_dtype must be one of: float32, model")
+        if self.ce_impl not in {"reference", "metal"}:
+            raise ValueError("ce_impl must be reference or metal")
+        if self.ce_impl == "metal" and self.loss_dtype != "float32":
+            raise ValueError("Metal CE uses FP32 reductions")
+        if self.ffn_impl not in {"reference", "packed-metal"}:
+            raise ValueError("ffn_impl must be reference or packed-metal")
+
+
+def _tree_leaves(tree):
+    if isinstance(tree, dict):
+        for v in tree.values():
+            yield from _tree_leaves(v)
+    elif isinstance(tree, (list, tuple)):
+        for v in tree:
+            yield from _tree_leaves(v)
+    else:
+        yield tree
+
+
+def count_parameters(model: nn.Module) -> int:
+    total = 0
+    for leaf in _tree_leaves(model.parameters()):
+        if isinstance(leaf, mx.array):
+            n = 1
+            for d in leaf.shape:
+                n *= int(d)
+            total += n
+    return total
+
+
+class CausalSelfAttention(nn.Module):
+    def __init__(self, cfg: TransformerConfig):
+        super().__init__()
+        self.n_heads = cfg.n_heads
+        self.n_kv_heads = cfg.n_kv_heads
+        self.head_dim = cfg.d_model // cfg.n_heads
+        self.q_dim = cfg.n_heads * self.head_dim
+        self.kv_dim = cfg.n_kv_heads * self.head_dim
+        self.max_seq_len = cfg.max_seq_len
+
+        self.qkv_proj = nn.Linear(cfg.d_model, self.q_dim + 2 * self.kv_dim, bias=cfg.bias)
+        self.proj = nn.Linear(cfg.d_model, cfg.d_model, bias=cfg.bias)
+        self.q_norm = nn.RMSNorm(self.head_dim) if cfg.qk_norm else None
+        self.k_norm = nn.RMSNorm(self.head_dim) if cfg.qk_norm else None
+        self.rope = nn.RoPE(self.head_dim, base=cfg.rope_base)
+        self.attention_impl = cfg.attention_impl
+
+    def __call__(
+        self,
+        x: mx.array,
+        cache: Optional[Tuple[mx.array, mx.array]] = None,
+    ) -> Tuple[mx.array, Tuple[mx.array, mx.array]]:
+        bsz, seqlen, d_model = x.shape
+        qkv = self.qkv_proj(x)
+        q, k, v = mx.split(qkv, [self.q_dim, self.q_dim + self.kv_dim], axis=-1)
+
+        def split_heads(t: mx.array, n_heads: int) -> mx.array:
+            t = t.reshape(bsz, seqlen, n_heads, self.head_dim)
+            return t.transpose(0, 2, 1, 3)
+
+        q = split_heads(q, self.n_heads)
+        k = split_heads(k, self.n_kv_heads)
+        v = split_heads(v, self.n_kv_heads)
+        if self.q_norm is not None:
+            q = self.q_norm(q)
+            k = self.k_norm(k)
+
+        cache_offset = 0
+        if cache is not None and cache[0] is not None:
+            cache_offset = int(cache[0].shape[2])
+        q = self.rope(q, offset=cache_offset)
+        k = self.rope(k, offset=cache_offset)
+
+        if cache is not None:
+            k_cache, v_cache = cache
+            if k_cache is not None and v_cache is not None:
+                k = mx.concatenate([k_cache, k], axis=2)
+                v = mx.concatenate([v_cache, v], axis=2)
+
+        q_len = q.shape[2]
+        k_len = k.shape[2]
+        if q_len > self.max_seq_len or k_len > self.max_seq_len:
+            raise ValueError(
+                f"Sequence length {q_len}/{k_len} exceeds max_seq_len={self.max_seq_len}"
+            )
+        scale = 1.0 / math.sqrt(self.head_dim)
+        # Cache compact KV heads, not the expanded heads used by vanilla GQA.
+        next_cache = (k, v)
+        if self.attention_impl == "fast":
+            attn = mx.fast.scaled_dot_product_attention(
+                q,
+                k,
+                v,
+                scale=scale,
+                mask="causal",
+            )
+        else:
+            repeats = self.n_heads // self.n_kv_heads
+            if repeats > 1:
+                k = mx.repeat(k, repeats, axis=1)
+                v = mx.repeat(v, repeats, axis=1)
+            full_mask = nn.MultiHeadAttention.create_additive_causal_mask(k_len)
+            mask = full_mask[k_len - q_len : k_len, :k_len]
+            mask = mx.expand_dims(mx.expand_dims(mask, axis=0), axis=0).astype(q.dtype)
+            scores = mx.matmul(q, k.transpose(0, 1, 3, 2)) * scale
+            scores = scores + mask
+            probs = mx.softmax(scores.astype(mx.float32), axis=-1).astype(v.dtype)
+            attn = mx.matmul(probs, v)
+        out = attn.transpose(0, 2, 1, 3).reshape(bsz, q_len, d_model)
+        out = self.proj(out)
+        return out, next_cache
+
+
+class FeedForward(nn.Module):
+    def __init__(self, cfg: TransformerConfig):
+        super().__init__()
+        # SwiGLU uses three projections, so 2/3 keeps parameter count comparable
+        # to a conventional 4x two-projection MLP.
+        hidden_dim = int((2.0 / 3.0) * cfg.d_model * cfg.mlp_ratio)
+        hidden_dim = cfg.mlp_multiple_of * math.ceil(hidden_dim / cfg.mlp_multiple_of)
+        self.hidden_dim = hidden_dim
+        self.ffn_impl = cfg.ffn_impl
+        self.gate_up = nn.Linear(cfg.d_model, 2 * hidden_dim, bias=cfg.bias)
+        self.down = nn.Linear(hidden_dim, cfg.d_model, bias=cfg.bias)
+
+    def __call__(self, x: mx.array) -> mx.array:
+        gate_up = self.gate_up(x)
+        if self.ffn_impl == "packed-metal":
+            try:
+                from .fast_swiglu import packed_swiglu
+            except ImportError:
+                from fast_swiglu import packed_swiglu
+            return self.down(packed_swiglu(gate_up))
+        gate, up = mx.split(gate_up, [self.hidden_dim], axis=-1)
+        return self.down(nn.silu(gate) * up)
+
+
+class TransformerBlock(nn.Module):
+    def __init__(self, cfg: TransformerConfig):
+        super().__init__()
+        self.norm1 = nn.RMSNorm(cfg.d_model)
+        self.attn = CausalSelfAttention(cfg)
+        self.norm2 = nn.RMSNorm(cfg.d_model)
+        self.ffn = FeedForward(cfg)
+
+    def __call__(
+        self,
+        x: mx.array,
+        cache: Optional[Tuple[mx.array, mx.array]] = None,
+    ) -> Tuple[mx.array, Tuple[mx.array, mx.array]]:
+        attn_out, new_cache = self.attn(self.norm1(x), cache=cache)
+        x = x + attn_out
+        x = x + self.ffn(self.norm2(x))
+        return x, new_cache
+
+
+class TransformerLM(nn.Module):
+    def __init__(self, cfg: TransformerConfig):
+        super().__init__()
+        self.cfg = cfg
+        self.embed = nn.Embedding(cfg.vocab_size, cfg.d_model)
+        self.blocks = [TransformerBlock(cfg) for _ in range(cfg.n_layers)]
+        # MLX >= 0.32.2 supplies the memory-reduced Metal RMSNorm backward
+        # for every nn.RMSNorm here, including block and Q/K norms. Keep the
+        # native primitive so compilation/autodiff select that implementation.
+        self.norm = nn.RMSNorm(cfg.d_model)
+
+    def logits(
+        self,
+        input_ids: mx.array,
+        caches: Optional[List[Tuple[mx.array, mx.array]]] = None,
+    ) -> Tuple[mx.array, List[Tuple[mx.array, mx.array]]]:
+        x = self.embed(input_ids)
+        if caches is None:
+            caches = [None] * len(self.blocks)
+
+        new_caches = []
+        for block, block_cache in zip(self.blocks, caches):
+            x, next_cache = block(x, cache=block_cache)
+            new_caches.append(next_cache)
+
+        x = self.norm(x)
+        weight = self.embed.weight
+        logits = mx.matmul(x, weight.transpose(1, 0))
+        return logits, new_caches
+
+    def __call__(
+        self,
+        input_ids: mx.array,
+        targets: Optional[mx.array] = None,
+        ignore_index: int = -100,
+    ) -> dict:
+        logits, _ = self.logits(input_ids, caches=None)
+        out = {"logits": logits}
+        if targets is None:
+            return out
+
+        loss_logits = logits.astype(mx.float32) if self.cfg.loss_dtype == "float32" else logits
+        mask = targets != ignore_index
+        safe_targets = mx.where(mask, targets, mx.zeros_like(targets))
+        if self.cfg.ce_impl == "metal":
+            try:
+                from .fast_loss import cross_entropy
+            except ImportError:
+                from fast_loss import cross_entropy
+            per_token = cross_entropy(logits, safe_targets)
+        else:
+            per_token = losses.cross_entropy(loss_logits, safe_targets, reduction="none")
+        mask = mask.astype(per_token.dtype)
+        denom = mx.maximum(mask.sum(), mx.array(1.0, dtype=per_token.dtype))
+        loss = (per_token * mask).sum() / denom
+        out["loss"] = loss
+        out["token_loss"] = per_token
+        return out
+
+    def step(
+        self,
+        token_ids: mx.array,
+        caches: Optional[List[Tuple[mx.array, mx.array]]] = None,
+    ) -> Tuple[mx.array, List[Tuple[mx.array, mx.array]]]:
+        # token_ids shape: [B, 1]
+        logits, new_caches = self.logits(token_ids, caches=caches)
+        return logits[:, -1, :], new_caches

native_tokenizer.py

@@ -0,0 +1,55 @@
+"""Own vocabulary, explicit structural IDs, verified frozen artifact."""
+
+from pathlib import Path
+from native_utils import file_sha256, read_json
+
+SPECIALS = ["<|pad|>", "<|doc_end|>", "<|turn_start|>", "<|turn_end|>"]
+FIXTURES = ["A plant uses sunlight to make sugars.", "  leading and trailing spaces  \n\n",
+            "\tif x:\n\t\treturn x + 1\n", "caf\u00e9, \u03c0, \u4e2d\u6587, \U0001f680; e\u0301",
+            "17 * 6 = 102; x**2 + y**2 = z**2", '{"items": [1, 2]}', "a\r\nb\r\n", "",
+            "Print " + " and ".join(SPECIALS) + " literally."]
+
+
+class Tokenizer:
+    def __init__(self, directory):
+        from tokenizers import Tokenizer as Backend
+        directory = Path(directory)
+        self.manifest = read_json(directory / "manifest.json")
+        if self.manifest["format"] != "sml-v2-tokenizer-v1":
+            raise ValueError("Not a v2 tokenizer")
+        if not {"tokenizer.json", "tokenizer_config.json", "corpus.json"} <= self.manifest["sha256"].keys():
+            raise ValueError("Incomplete tokenizer integrity manifest")
+        if self.manifest["special_ids"] != dict(pad=0, eos=1, turn_start=2, turn_end=3):
+            raise ValueError("Structural token metadata changed")
+        for name, digest in self.manifest["sha256"].items():
+            if Path(name).name != name or file_sha256(directory / name) != digest:
+                raise ValueError(f"Tokenizer integrity failure: {name}")
+        self.backend = Backend.from_file(str(directory / "tokenizer.json"))
+        self.backend.no_padding()
+        self.backend.no_truncation()
+        self.backend.encode_special_tokens = True
+        self.fingerprint = file_sha256(directory / "manifest.json")
+        self.eos = self.manifest["special_ids"]["eos"]
+        self.pad = self.manifest["special_ids"]["pad"]
+        self.vocab_size = self.backend.get_vocab_size(with_added_tokens=True)
+        ids = set(self.backend.get_vocab().values())
+        if ids != set(range(self.manifest["vocab_size"])):
+            raise ValueError("Tokenizer IDs are not the declared dense vocabulary")
+        for i, token in enumerate(SPECIALS):
+            if self.backend.token_to_id(token) != i:
+                raise ValueError("Special token mapping changed")
+
+    def encode(self, text):
+        ids = self.backend.encode(text, add_special_tokens=False).ids
+        if any(i < len(SPECIALS) for i in ids):
+            raise ValueError("Ordinary text emitted structural token IDs")
+        return ids
+
+    def decode(self, ids):
+        return self.backend.decode([int(i) for i in ids], skip_special_tokens=False)
+
+    def assert_roundtrip(self, text):
+        ids = self.encode(text)
+        if self.decode(ids) != text:
+            raise ValueError(f"Tokenizer round-trip failed: {text[:120]!r}")
+        return ids

native_utils.py

@@ -0,0 +1,8 @@
+import hashlib,json
+from pathlib import Path
+def read_json(p):return json.loads(Path(p).read_text())
+def file_sha256(p):
+ h=hashlib.sha256()
+ with Path(p).open("rb") as f:
+  for b in iter(lambda:f.read(4*1024**2),b""):h.update(b)
+ return h.hexdigest()

report.json

@@ -0,0 +1,38 @@
+{
+  "fixtures_passed": 9,
+  "sources": {
+    "cosmopedia": {
+      "bytes": 402259,
+      "bytes_per_token": 5.061007523716062,
+      "documents": 103,
+      "roundtrip_failures": 0,
+      "seconds": 0.09366637503262609,
+      "tokens": 79482
+    },
+    "dclm": {
+      "bytes": 1001509,
+      "bytes_per_token": 4.453209483494593,
+      "documents": 165,
+      "roundtrip_failures": 0,
+      "seconds": 0.26016941701527685,
+      "tokens": 224896
+    },
+    "web": {
+      "bytes": 2205062,
+      "bytes_per_token": 4.625467253451697,
+      "documents": 559,
+      "roundtrip_failures": 0,
+      "seconds": 0.5487007499905303,
+      "tokens": 476722
+    },
+    "wiki": {
+      "bytes": 401775,
+      "bytes_per_token": 4.059276397546905,
+      "documents": 143,
+      "roundtrip_failures": 0,
+      "seconds": 0.10723229206632823,
+      "tokens": 98977
+    }
+  },
+  "vocab_size": 32000
+}

requirements.txt

@@ -0,0 +1,3 @@
+mlx==0.32.2
+mlx-metal==0.32.2
+tokenizers==0.22.2

tokenizer_config.json

@@ -0,0 +1,8 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|doc_end|>",
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}