Upload folder using huggingface_hub

config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "DFlashDraftModel"
+  ],
+  "auto_map": {
+    "": "config.DFlashSpeculatorConfig"
+  },
+  "aux_hidden_state_layer_ids": [
+    1,
+    12,
+    23,
+    34,
+    45
+  ],
+  "block_size": 8,
+  "draft_vocab_size": 32000,
+  "dtype": "bfloat16",
+  "mask_token_id": 151669,
+  "max_anchors": 1024,
+  "sliding_window_non_causal": false,
+  "speculators_config": {
+    "algorithm": "dflash",
+    "default_proposal_method": "greedy",
+    "proposal_methods": [
+      {
+        "accept_tolerance": 0.0,
+        "proposal_type": "greedy",
+        "speculative_tokens": 7,
+        "verifier_accept_k": 1
+      }
+    ],
+    "verifier": {
+      "architectures": [],
+      "name_or_path": "Qwen/Qwen3-30B-A3B-Instruct-2507"
+    }
+  },
+  "speculators_model_type": "dflash",
+  "speculators_version": "0.5.0.dev160",
+  "target_hidden_size": null,
+  "tie_word_embeddings": false,
+  "transformer_layer_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 2048,
+    "initializer_range": 0.02,
+    "intermediate_size": 6144,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention"
+    ],
+    "max_position_embeddings": 262144,
+    "mlp_bias": false,
+    "model_type": "llama",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 5,
+    "num_key_value_heads": 4,
+    "pad_token_id": null,
+    "pretraining_tp": 1,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "sliding_window": 2048,
+    "tie_word_embeddings": false,
+    "use_cache": true,
+    "vocab_size": 151936
+  },
+  "transformers_version": "5.10.2"
+}

config.py

@@ -0,0 +1,102 @@
+from typing import Any, Literal
+
+from pydantic import Field, field_serializer, field_validator
+from transformers import AutoConfig, PretrainedConfig
+from transformers.models.qwen3.modeling_qwen3 import (
+    Qwen3Config,
+)
+
+from speculators import SpeculatorModelConfig
+
+__all__ = [
+    "DFlashSpeculatorConfig",
+]
+
+
+@SpeculatorModelConfig.register("dflash")
+class DFlashSpeculatorConfig(SpeculatorModelConfig):
+    """
+    Configuration for DFlash speculator with vocabulary mapping.
+
+    DFlash features vocabulary mapping between draft (64K) and target (128K)
+    vocabularies, enabling cross-tokenizer speculation.
+
+    :param transformer_layer_config: Configuration for the transformer decoder layer
+    :param draft_vocab_size: Size of draft model vocabulary for speculation
+    """
+
+    speculators_model_type: Literal["dflash"] = "dflash"
+    architectures: list[str] = Field(
+        default_factory=lambda: ["DFlashSpeculator"],
+        description="Model architectures that can load these weights",
+    )
+
+    transformer_layer_config: PretrainedConfig = Field(
+        default_factory=Qwen3Config,
+        description="Configuration for the transformer decoder layer",
+    )
+
+    draft_vocab_size: int = Field(
+        default=32000,
+        description="Size of draft model vocabulary for speculation",
+    )
+
+    block_size: int = Field(
+        default=8,
+        description=(
+            "Default size of the draft block predicted with a forward pass of the model"
+        ),
+    )
+
+    max_anchors: int = Field(
+        default=256,
+        description=(
+            "Maximum number of anchor positions to sample during training "
+            "(controls memory usage and training efficiency)"
+        ),
+    )
+
+    target_hidden_size: int | None = Field(
+        default=None,
+        description="Hidden size of the target model (if different from draft model)",
+    )
+
+    aux_hidden_state_layer_ids: list[int] | None = Field(
+        default=None,
+        description="Layer IDs of the DFlash auxiliary hidden state layers",
+    )
+
+    mask_token_id: int | None = Field(
+        default=None,
+        description="Token ID used for masking",
+    )
+
+    sliding_window_non_causal: bool = Field(
+        default=False,
+        description="Use non-causal (bidirectional) masking within draft blocks for "
+        "sliding window attention layers. Full attention layers are always "
+        "bidirectional.",
+    )
+
+    @field_serializer("transformer_layer_config")
+    def serialize_transformer_config(self, value: PretrainedConfig) -> dict:
+        """Serialize transformer config to dict."""
+        return value.to_diff_dict()
+
+    @field_validator("transformer_layer_config", mode="before")
+    @classmethod
+    def validate_transformer_config(cls, value: Any) -> PretrainedConfig:
+        """Validate and convert transformer config."""
+        if isinstance(value, dict):
+            config_class: type[PretrainedConfig] = Qwen3Config
+            if "model_type" in value:
+                config_class = AutoConfig.for_model(
+                    model_type=value["model_type"]
+                ).__class__
+            return config_class(**value)
+        return value
+
+    @property
+    def target_vocab_size(self) -> int:
+        """Get target vocabulary size from transformer config."""
+        return self.transformer_layer_config.vocab_size

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22ef21bf72671069f80c4564d871f441e6b9535d8acbc82a6ef4a5ac4264a5b4
+size 1362042120

val_metrics.json

@@ -0,0 +1 @@
+{"loss_epoch": 1.4144842092175067, "full_acc_epoch": 0.34676152761132395, "position_1_acc_epoch": 0.6682026932842137, "position_2_acc_epoch": 0.4860215776577793, "position_3_acc_epoch": 0.3740923833465139, "position_4_acc_epoch": 0.2959340704922699, "position_5_acc_epoch": 0.23920370384832654, "position_6_acc_epoch": 0.19724874215872754, "position_7_acc_epoch": 0.16442117170899365}