Upload Neo50M HF safetensors export

README.md

@@ -1,3 +1,72 @@
 ---
-license: mit
+license: apache-2.0
+language:
+- en
+library_name: transformers
+pipeline_tag: text-generation
+tags:
+- llama
+- causal-lm
+- chat
+- tiny-language-model
+- neo50m
 ---
+
+# Neo50M
+
+Neo50M is a tiny decoder-only chat language model trained from scratch. It is designed for toy/local assistant use, educational experiments, lightweight generation, and testing training pipelines.
+
+## Model Details
+
+- **Type:** decoder-only causal language model, Llama-compatible architecture
+- **Parameters:** approximately 52.6M
+- **Context length target:** 16k tokens
+- **Training target:** about 15B pretraining tokens plus chat/instruction tuning
+- **Hardware:** 8x NVIDIA RTX 5090 cloud GPUs
+- **Tokenizer:** TinyLlama/Llama-style 32k tokenizer with a Neo50M chat template
+
+## Intended Uses
+
+- toy/local assistant experiments
+- educational training and inference demos
+- lightweight generation
+- testing HF, GGUF, ONNX, and distributed training pipelines
+
+## Limitations
+
+Neo50M is very small. It is not reliable for factual accuracy, has limited reasoning ability, may hallucinate, and should not be used for safety-critical decisions or high-stakes advice.
+
+## Transformers Usage
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "KookiesXy/Neo50M"
+tokenizer = AutoTokenizer.from_pretrained(repo_id)
+model = AutoModelForCausalLM.from_pretrained(repo_id, device_map="auto")
+
+messages = [{"role": "user", "content": "Write a short thank-you note."}]
+inputs = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to(model.device)
+out = model.generate(inputs, max_new_tokens=120, temperature=0.7, top_p=0.9)
+print(tokenizer.decode(out[0][inputs.shape[-1]:], skip_special_tokens=True))
+```
+
+## GGUF Usage
+
+After downloading a GGUF file:
+
+```bash
+llama-cli -m neo50m-q4_k_m.gguf -p "User: Write a haiku about GPUs.\nAssistant:"
+```
+
+## ONNX Usage
+
+The ONNX export is intended for forward-pass validation and integration experiments. Use ONNX Runtime to load `onnx/model.onnx` and feed integer `input_ids` plus `attention_mask`.
+
+## Dataset Summary
+
+The training pipeline streams a configurable mixture of FineWeb-Edu, Cosmopedia, Wikipedia-like text, TinyStories, and a small permissive code component. SFT uses OpenHermes-style, UltraChat-style, Alpaca-style, and small refusal/helpfulness examples when available. Dataset availability can change; the exact configs are included with the upload.
+
+## Eval Results
+
+Eval artifacts, when present, are uploaded under `evals/`.

chat_template.jinja

@@ -0,0 +1 @@
+{% for message in messages %}{% if loop.first and message['role'] != 'system' %}{{ bos_token + 'System: You are Neo50M, a concise and helpful assistant.\n' }}{% endif %}{% if message['role'] == 'system' %}{{ bos_token + 'System: ' + message['content'].strip() + '\n' }}{% elif message['role'] == 'user' %}{{ 'User: ' + message['content'].strip() + '\n' }}{% elif message['role'] == 'assistant' %}{{ 'Assistant: ' + message['content'].strip() + eos_token }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant: ' }}{% endif %}

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "max_position_embeddings": 16384,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 2,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 500000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.12.1",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "max_new_tokens": 256,
+  "pad_token_id": 2,
+  "temperature": 0.7,
+  "top_p": 0.9,
+  "transformers_version": "5.12.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc82614943d98a47f0a93bf0ed3748325ce394162fb06a12a235983c79df1db9
+size 210302864

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 16384,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

training_metadata.json

@@ -0,0 +1,96 @@
+{
+  "best_val_loss": 2.788245379924774,
+  "config": {
+    "_config_path": "configs/sft_chat.yaml",
+    "run": {
+      "data_config": "configs/data_mix.yaml",
+      "model_config": "configs/model_neo50m.yaml",
+      "name": "neo50m_sft_chat",
+      "output_dir": "outputs",
+      "seed": 3337
+    },
+    "training": {
+      "beta1": 0.9,
+      "beta2": 0.95,
+      "curriculum": [
+        {
+          "seq_len": 4096,
+          "tokens": 40000000
+        },
+        {
+          "seq_len": 8192,
+          "tokens": 25000000
+        },
+        {
+          "seq_len": 16384,
+          "tokens": 15000000
+        }
+      ],
+      "data_mode": "sft",
+      "eval_batches": 4,
+      "eval_interval_steps": 250,
+      "grad_accum": 8,
+      "grad_clip": 1.0,
+      "keep_last_checkpoints": 2,
+      "learning_rate": 8e-05,
+      "log_interval_steps": 5,
+      "micro_batch_size": 1,
+      "min_lr_ratio": 0.1,
+      "num_workers": 0,
+      "save_interval_steps": 250,
+      "seq_len": 4096,
+      "stage": "sft_chat",
+      "target_tokens": 80000000,
+      "warmup_steps": 200,
+      "weight_decay": 0.0
+    }
+  },
+  "env": {
+    "cuda": "13.0",
+    "git_revision": null,
+    "gpu_count": 8,
+    "gpu_name": "NVIDIA GeForce RTX 5090",
+    "python": "3.12.13",
+    "torch": "2.12.0+cu130"
+  },
+  "global_step": 57750,
+  "global_tokens": 15628763136,
+  "model_config": {
+    "architecture": "llama",
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_implementation": "sdpa",
+    "gradient_checkpointing": false,
+    "hidden_size": 512,
+    "initializer_range": 0.02,
+    "intermediate_size": 1536,
+    "max_position_embeddings": 16384,
+    "mlp_bias": false,
+    "name": "Neo50M",
+    "num_attention_heads": 8,
+    "num_hidden_layers": 12,
+    "num_key_value_heads": 2,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 500000.0,
+    "tie_word_embeddings": true,
+    "tokenizer_id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "torch_dtype": "bfloat16",
+    "use_cache": false
+  },
+  "precision": {
+    "fp8_backend": null,
+    "reason": "FP8 unavailable; torchao is not installed",
+    "requested": "auto",
+    "selected": "bf16"
+  },
+  "stage_id": "sft_chat_0_seq4096",
+  "stage_tokens": 25690112,
+  "tokenizer": {
+    "bos_token_id": 1,
+    "chat_template": "{% for message in messages %}{% if loop.first and message['role'] != 'system' %}{{ bos_token + 'System: You are Neo50M, a concise and helpful assistant.\\n' }}{% endif %}{% if message['role'] == 'system' %}{{ bos_token + 'System: ' + message['content'].strip() + '\\n' }}{% elif message['role'] == 'user' %}{{ 'User: ' + message['content'].strip() + '\\n' }}{% elif message['role'] == 'assistant' %}{{ 'Assistant: ' + message['content'].strip() + eos_token }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant: ' }}{% endif %}",
+    "eos_token_id": 2,
+    "pad_token_id": 2,
+    "tokenizer_id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "vocab_size": 32000
+  }
+}