Upload 14 files

.gitattributes

@@ -0,0 +1,2 @@
+compact-int8/model_int8.safetensors filter=lfs diff=lfs merge=lfs -text
+model.safetensors filter=lfs diff=lfs merge=lfs -text

compact-int8/README.md

@@ -4,12 +4,12 @@ language: [en]
 library_name: transformers
 tags: [decision-model, system-one, calibrated-decisions, multiple-choice, zero-shot-classification, falcondec]
 ---
-# FalconDec v1.0.0 — Falcon Decision
+# LightDec v1.0.1 — Falcon Decision
 
 Single-pass, typed (`choice` / `noul` / `score`), calibrated closed-set decision model. Successor to
 [`Falconsai/proof_v2`](https://huggingface.co/Falconsai/proof_v2).
 
-* Built with FalconDec notebook V2 · backbone `jhu-clsp/ettin-encoder-150m` · mode `scratch` · preset `standard` · lineage: jhu-clsp/ettin-encoder-150m
+* Built with FalconDec notebook V2 · backbone `jhu-clsp/ettin-encoder-150m` · mode `finetune` · preset `full` · lineage: jhu-clsp/ettin-encoder-150m → FalconDec-v1.0.0
 * Parameters: 159.7M · weights: fp16 305 MB, int8 153 MB
 * Layout: `[CLS] question [SEP] [MASK] opt1 … [MASK] optk [SEP] state [SEP]`; set-transformer option head;
   temperature per (question type × option-count bucket).
@@ -19,7 +19,7 @@ Single-pass, typed (`choice` / `noul` / `score`), calibrated closed-set decision
 ```python
 import importlib.util
 from huggingface_hub import snapshot_download
-path = snapshot_download("<repo>")            # or a local FalconDec-v1.0.0 directory
+path = snapshot_download("<repo>")            # or a local FalconDec-v1.0.1 directory
 spec = importlib.util.spec_from_file_location("falcondec_modeling", f"{path}/falcondec_modeling.py")
 fdm = importlib.util.module_from_spec(spec); spec.loader.exec_module(fdm)
 model, tok = fdm.load_falcondec(path)          # add "/compact-int8" for the int8 artefact
@@ -31,69 +31,69 @@ out = fdm.decide(model, tok, state={"body": "I was charged twice, refund me or I
 
 ## Test results
 
-Overall: micro acc **0.725**, task-macro acc **0.725**, ECE **0.025**,
-NLL 0.652, Brier 0.358, AURC 0.097.
+Overall: micro acc **0.781**, task-macro acc **0.769**, ECE **0.020**,
+NLL 0.529, Brier 0.290, AURC 0.064.
 
 | task | domain | heldout | n | chance | acc | proof_v2 (card) | ece |
 |---|---|---|---|---|---|---|---|
-| counsel/critique_quality | agentic | False | 201 | 0.333 | 0.557 |  | 0.094 |
-| counsel/step_has_error | agentic | False | 201 | 0.500 | 0.791 |  | 0.093 |
-| hotpotqa/comparison_yes_no | agentic | False | 17 | 0.500 | 0.824 |  | 0.185 |
-| hotpotqa/retrieve | agentic | False | 298 | 0.168 | 0.842 |  | 0.078 |
-| sst5/score | classification | True | 300 | 0.200 | 0.370 |  | 0.086 |
-| emotion/6way | classification | True | 300 | 0.167 | 0.480 |  | 0.049 |
-| yelp/score | classification | False | 300 | 0.200 | 0.640 |  | 0.069 |
-| ag_news/topic | classification | False | 300 | 0.250 | 0.847 |  | 0.051 |
-| devign/vulnerability | code | False | 300 | 0.500 | 0.553 | 0.542 | 0.026 |
-| mbpp/bugspot | code | False | 156 | 0.413 | 0.731 | 0.475 | 0.086 |
-| humaneval/completion | code | True | 119 | 0.394 | 0.756 | 0.575 | 0.152 |
-| bigclonebench/clone | code | False | 300 | 0.500 | 0.863 | 0.383 | 0.098 |
-| codexglue/func_name | code | False | 275 | 0.263 | 0.938 | 0.901 | 0.021 |
-| codexglue/doc_to_code | code | False | 300 | 0.274 | 0.973 | 0.961 | 0.022 |
-| codexglue/code_to_doc | code | False | 300 | 0.256 | 0.977 | 0.969 | 0.016 |
-| mbpp/solution | code | False | 300 | 0.250 | 0.987 | 0.992 | 0.014 |
-| codexglue/lang_id | code | False | 300 | 0.235 | 1.000 | 0.997 | 0.004 |
-| prompt_injections/detect | guardrails | True | 116 | 0.500 | 0.647 |  | 0.272 |
-| agentharm/refuse | guardrails | True | 416 | 0.500 | 0.654 |  | 0.178 |
-| civil_comments/toxic | guardrails | False | 300 | 0.500 | 0.807 |  | 0.051 |
-| jailbreak/detect | guardrails | False | 262 | 0.500 | 0.966 |  | 0.018 |
-| banking77/intent_77 | intents | True | 300 | 0.013 | 0.470 |  | 0.200 |
-| clinc150/intent | intents | False | 300 | 0.122 | 0.793 | 0.850 | 0.081 |
-| massive_en/intent | intents | False | 300 | 0.122 | 0.907 |  | 0.046 |
-| banking77/intent | intents | True | 300 | 0.317 | 0.923 | 0.883 | 0.037 |
-| policy/table_extreme_transfer | policy | False | 300 | 0.240 | 0.290 |  | 0.036 |
-| policy/invoice_overdue_transfer | policy | False | 300 | 0.500 | 0.407 |  | 0.364 |
-| policy/count_threshold_transfer | policy | False | 300 | 0.179 | 0.523 |  | 0.085 |
-| policy/invoice_total_transfer | policy | False | 300 | 0.500 | 0.523 |  | 0.020 |
-| policy/refund_approval_transfer | policy | False | 300 | 0.333 | 0.710 |  | 0.219 |
-| policy/sla_urgency_transfer | policy | False | 300 | 0.250 | 0.713 |  | 0.214 |
-| policy/free_shipping_transfer | policy | False | 300 | 0.500 | 0.850 |  | 0.067 |
-| policy/access_control_transfer | policy | False | 300 | 0.333 | 1.000 |  | 0.001 |
-| policy/return_window_transfer | policy | False | 300 | 0.333 | 1.000 |  | 0.019 |
-| aqua_rat/math | reasoning | False | 247 | 0.200 | 0.259 |  | 0.071 |
-| hellaswag/continuation | reasoning | False | 300 | 0.250 | 0.383 |  | 0.171 |
-| anli/nli | reasoning | False | 300 | 0.333 | 0.393 |  | 0.185 |
-| mmlu/mcq | reasoning | True | 300 | 0.250 | 0.393 |  | 0.094 |
-| arc_challenge/mcq | reasoning | True | 300 | 0.250 | 0.423 | 0.308 | 0.085 |
-| winogrande/blank | reasoning | False | 300 | 0.500 | 0.540 |  | 0.089 |
-| openbookqa/mcq | reasoning | False | 300 | 0.250 | 0.550 | 0.292 | 0.068 |
-| arc_easy/mcq | reasoning | True | 300 | 0.250 | 0.553 | 0.425 | 0.057 |
-| commonsense_qa/mcq | reasoning | False | 296 | 0.200 | 0.611 | 0.442 | 0.058 |
-| gsm8k/math | reasoning | False | 300 | 0.250 | 0.637 | 0.275 | 0.050 |
-| boolq/yes_no | reasoning | False | 300 | 0.500 | 0.783 | 0.717 | 0.071 |
-| mnli/claim | reasoning | False | 300 | 0.333 | 0.807 | 0.492 | 0.072 |
-| snli/nli | reasoning | False | 594 | 0.333 | 0.837 |  | 0.035 |
-| sciq/mcq | reasoning | False | 300 | 0.250 | 0.950 | 0.692 | 0.023 |
-| scitail/support | reasoning | False | 300 | 0.500 | 0.957 |  | 0.030 |
-| snli/contradicts | reasoning | False | 300 | 0.333 | 0.967 | 0.892 | 0.035 |
-| snli/must_be_true | reasoning | False | 300 | 0.333 | 0.970 | 0.908 | 0.029 |
-| qasc/mcq | reasoning | False | 300 | 0.125 | 0.983 |  | 0.007 |
-| bitext/category | support | False | 300 | 0.172 | 0.993 |  | 0.008 |
-| bitext/route | support | False | 300 | 0.200 | 1.000 | 0.958 | 0.002 |
-| typed_decisions/invoice_processing | workflows | False | 500 | 0.350 | 0.628 |  | 0.101 |
-| typed_decisions/agent_trace_observability | workflows | False | 500 | 0.300 | 0.696 |  | 0.106 |
-| typed_decisions/security_incidents | workflows | False | 500 | 0.340 | 0.712 |  | 0.140 |
-| typed_decisions/customer_service | workflows | False | 500 | 0.280 | 0.720 |  | 0.099 |
+| counsel/critique_quality | agentic | False | 201 | 0.333 | 0.592 |  | 0.227 |
+| counsel/step_has_error | agentic | False | 201 | 0.500 | 0.801 |  | 0.169 |
+| hotpotqa/comparison_yes_no | agentic | False | 26 | 0.500 | 0.846 |  | 0.151 |
+| hotpotqa/retrieve | agentic | False | 497 | 0.167 | 0.873 |  | 0.083 |
+| sst5/score | classification | True | 500 | 0.200 | 0.416 |  | 0.046 |
+| emotion/6way | classification | True | 500 | 0.167 | 0.502 |  | 0.082 |
+| yelp/score | classification | False | 500 | 0.200 | 0.654 |  | 0.069 |
+| ag_news/topic | classification | False | 500 | 0.250 | 0.888 |  | 0.040 |
+| devign/vulnerability | code | False | 500 | 0.500 | 0.596 | 0.542 | 0.050 |
+| humaneval/completion | code | True | 119 | 0.394 | 0.672 | 0.575 | 0.107 |
+| mbpp/bugspot | code | False | 256 | 0.406 | 0.730 | 0.475 | 0.102 |
+| bigclonebench/clone | code | False | 500 | 0.500 | 0.952 | 0.383 | 0.016 |
+| codexglue/func_name | code | False | 467 | 0.255 | 0.953 | 0.901 | 0.023 |
+| codexglue/doc_to_code | code | False | 504 | 0.279 | 0.976 | 0.961 | 0.010 |
+| mbpp/solution | code | False | 500 | 0.250 | 0.982 | 0.992 | 0.015 |
+| codexglue/code_to_doc | code | False | 504 | 0.262 | 0.984 | 0.969 | 0.012 |
+| codexglue/lang_id | code | False | 504 | 0.235 | 1.000 | 0.997 | 0.001 |
+| prompt_injections/detect | guardrails | True | 116 | 0.500 | 0.578 |  | 0.286 |
+| agentharm/refuse | guardrails | True | 416 | 0.500 | 0.743 |  | 0.067 |
+| civil_comments/toxic | guardrails | False | 500 | 0.500 | 0.906 |  | 0.035 |
+| jailbreak/detect | guardrails | False | 262 | 0.500 | 0.954 |  | 0.023 |
+| banking77/intent_77 | intents | True | 300 | 0.013 | 0.533 |  | 0.147 |
+| massive_en/intent | intents | False | 500 | 0.130 | 0.906 |  | 0.022 |
+| banking77/intent | intents | True | 500 | 0.321 | 0.930 | 0.883 | 0.040 |
+| clinc150/intent | intents | False | 500 | 0.124 | 0.942 | 0.850 | 0.036 |
+| policy/invoice_total_transfer | policy | False | 500 | 0.500 | 0.486 |  | 0.032 |
+| policy/invoice_overdue_transfer | policy | False | 500 | 0.500 | 0.600 |  | 0.201 |
+| policy/sla_urgency_transfer | policy | False | 500 | 0.250 | 0.698 |  | 0.212 |
+| policy/free_shipping_transfer | policy | False | 500 | 0.500 | 0.766 |  | 0.101 |
+| policy/refund_approval_transfer | policy | False | 500 | 0.333 | 0.780 |  | 0.186 |
+| policy/count_threshold_transfer | policy | False | 500 | 0.178 | 0.846 |  | 0.109 |
+| policy/table_extreme_transfer | policy | False | 500 | 0.237 | 0.912 |  | 0.014 |
+| policy/return_window_transfer | policy | False | 500 | 0.333 | 0.998 |  | 0.020 |
+| policy/access_control_transfer | policy | False | 500 | 0.333 | 1.000 |  | 0.000 |
+| aqua_rat/math | reasoning | False | 247 | 0.200 | 0.275 |  | 0.042 |
+| mmlu/mcq | reasoning | True | 500 | 0.250 | 0.356 |  | 0.109 |
+| anli/nli | reasoning | False | 498 | 0.333 | 0.416 |  | 0.184 |
+| arc_challenge/mcq | reasoning | True | 500 | 0.250 | 0.460 | 0.308 | 0.088 |
+| hellaswag/continuation | reasoning | False | 500 | 0.250 | 0.580 |  | 0.073 |
+| arc_easy/mcq | reasoning | True | 500 | 0.250 | 0.594 | 0.425 | 0.038 |
+| openbookqa/mcq | reasoning | False | 500 | 0.250 | 0.610 | 0.292 | 0.092 |
+| winogrande/blank | reasoning | False | 500 | 0.500 | 0.612 |  | 0.111 |
+| commonsense_qa/mcq | reasoning | False | 493 | 0.200 | 0.639 | 0.442 | 0.044 |
+| gsm8k/math | reasoning | False | 500 | 0.250 | 0.662 | 0.275 | 0.061 |
+| boolq/yes_no | reasoning | False | 500 | 0.500 | 0.816 | 0.717 | 0.027 |
+| mnli/claim | reasoning | False | 500 | 0.333 | 0.838 | 0.492 | 0.071 |
+| snli/nli | reasoning | False | 988 | 0.333 | 0.880 |  | 0.050 |
+| sciq/mcq | reasoning | False | 498 | 0.250 | 0.954 | 0.692 | 0.031 |
+| scitail/support | reasoning | False | 500 | 0.500 | 0.964 |  | 0.020 |
+| snli/must_be_true | reasoning | False | 500 | 0.333 | 0.980 | 0.908 | 0.029 |
+| snli/contradicts | reasoning | False | 500 | 0.333 | 0.986 | 0.892 | 0.014 |
+| qasc/mcq | reasoning | False | 500 | 0.125 | 0.990 |  | 0.006 |
+| bitext/category | support | False | 500 | 0.161 | 1.000 |  | 0.002 |
+| bitext/route | support | False | 500 | 0.190 | 1.000 | 0.958 | 0.001 |
+| typed_decisions/agent_trace_observability | workflows | False | 500 | 0.300 | 0.712 |  | 0.120 |
+| typed_decisions/security_incidents | workflows | False | 500 | 0.340 | 0.730 |  | 0.144 |
+| typed_decisions/customer_service | workflows | False | 500 | 0.280 | 0.740 |  | 0.085 |
+| typed_decisions/invoice_processing | workflows | False | 500 | 0.350 | 0.838 |  | 0.122 |
 
 
 ## Training
@@ -101,7 +101,7 @@ NLL 0.652, Brier 0.358, AURC 0.097.
 Strictly proper objective (log + 0.5·spherical + 1.0·RPS for ordinal), soft teacher targets where available,
 task-balanced sampling (α = 0.5), NOTA augmentation (8%), per-epoch option reshuffling,
 AdamW + LLRD 0.9, EMA 0.999, 2 epoch(s), RLCD stage off/reverted.
-Training time 46.1 min on NVIDIA GeForce RTX 5090 Laptop GPU.
+Training time 172.2 min on NVIDIA GeForce RTX 5090 Laptop GPU.
 
 ## Limitations
 

compact-int8/falcondec_config.json

@@ -7,13 +7,14 @@
   "max_opts_single_pass": 96,
   "interact_layers": 2,
   "interact_heads": 8,
-  "name": "FalconDec",
+  "name": "LightDec",
   "backbone": "jhu-clsp/ettin-encoder-150m",
   "backbone_init": "pretrained",
   "lineage": [
-    "jhu-clsp/ettin-encoder-150m"
+    "jhu-clsp/ettin-encoder-150m",
+    "FalconDec-v1.0.0"
   ],
-  "parent_version": null,
+  "parent_version": "1.0.0",
   "special": {
     "cls": 50281,
     "sep": 50282,
@@ -27,27 +28,27 @@
     "score": 2
   },
   "n_buckets": 4,
-  "version": "1.0.0",
+  "version": "1.0.1",
   "notebook_version": "V2",
-  "created": "2026-09-26T04:35:05.339042Z",
+  "created": "2026-09-26T11:28:22.623723Z",
   "temperature": [
     [
-      1.7067416906356812,
-      1.3524072170257568,
-      1.4660307168960571,
-      1.3491928577423096
+      1.3723914623260498,
+      1.3805755376815796,
+      1.5033340454101562,
+      1.320592999458313
     ],
     [
-      1.4017767906188965,
-      1.4017767906188965,
-      1.4017767906188965,
-      1.4017767906188965
+      1.3903899192810059,
+      1.3903899192810059,
+      1.3903899192810059,
+      1.3903899192810059
     ],
     [
-      1.4534460306167603,
-      1.4534460306167603,
-      1.4534460306167603,
-      1.4534460306167603
+      1.4738539457321167,
+      1.4738539457321167,
+      1.4738539457321167,
+      1.4738539457321167
     ]
   ],
   "weights": "model_int8.safetensors"

compact-int8/model_int8.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:80796383c3b131a6540b01642a8cb929f53e8d84b4c0f3f159248e56c11a7624
+oid sha256:174e111a329070aaef5597db916a4045aef6de8ce4398c2f73a5975fbe9d700b
 size 160530522

compact-int8/tokenizer/tokenizer_config.json

@@ -2,9 +2,10 @@
   "backend": "tokenizers",
   "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
-  "is_local": false,
+  "is_local": true,
   "local_files_only": false,
   "mask_token": "[MASK]",
+  "max_length": 256,
   "model_input_names": [
     "input_ids",
     "attention_mask"
@@ -12,6 +13,9 @@
   "model_max_length": 8192,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
+  "stride": 0,
   "tokenizer_class": "TokenizersBackend",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "[UNK]"
 }

encoder/config.json

@@ -0,0 +1,79 @@
+{
+  "architectures": [
+    "ModernBertForMaskedLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 50281,
+  "causal_mask": false,
+  "classifier_activation": "gelu",
+  "classifier_bias": false,
+  "classifier_dropout": 0.0,
+  "classifier_pooling": "mean",
+  "cls_token_id": 50281,
+  "decoder_bias": true,
+  "deterministic_flash_attn": false,
+  "dtype": "float32",
+  "embedding_dropout": 0.0,
+  "eos_token_id": 50282,
+  "global_attn_every_n_layers": 3,
+  "gradient_checkpointing": false,
+  "hidden_activation": "gelu",
+  "hidden_size": 768,
+  "initializer_cutoff_factor": 2.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 1152,
+  "is_causal": false,
+  "layer_norm_eps": 1e-05,
+  "layer_types": [
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "local_attention": 128,
+  "max_position_embeddings": 7999,
+  "mlp_bias": false,
+  "mlp_dropout": 0.0,
+  "model_type": "modernbert",
+  "norm_bias": false,
+  "norm_eps": 1e-05,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 22,
+  "pad_token_id": 50283,
+  "position_embedding_type": "sans_pos",
+  "rope_parameters": {
+    "full_attention": {
+      "rope_theta": 160000.0,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "rope_theta": 160000.0,
+      "rope_type": "default"
+    }
+  },
+  "sep_token_id": 50282,
+  "sparse_pred_ignore_index": -100,
+  "sparse_prediction": false,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.17.0",
+  "vocab_size": 50368
+}

falcondec_config.json

@@ -0,0 +1,55 @@
+{
+  "max_len": 512,
+  "long_max_len": 2048,
+  "long_opts_threshold": 24,
+  "head_max_len": 192,
+  "max_tok_per_opt": 24,
+  "max_opts_single_pass": 96,
+  "interact_layers": 2,
+  "interact_heads": 8,
+  "name": "LightDec",
+  "backbone": "jhu-clsp/ettin-encoder-150m",
+  "backbone_init": "pretrained",
+  "lineage": [
+    "jhu-clsp/ettin-encoder-150m",
+    "FalconDec-v1.0.0"
+  ],
+  "parent_version": "1.0.0",
+  "special": {
+    "cls": 50281,
+    "sep": 50282,
+    "mask": 50284,
+    "pad": 50283
+  },
+  "defer_threshold": 0.7,
+  "qtypes": {
+    "choice": 0,
+    "noul": 1,
+    "score": 2
+  },
+  "n_buckets": 4,
+  "version": "1.0.1",
+  "notebook_version": "V2",
+  "created": "2026-09-26T11:28:22.623723Z",
+  "temperature": [
+    [
+      1.3723914623260498,
+      1.3805755376815796,
+      1.5033340454101562,
+      1.320592999458313
+    ],
+    [
+      1.3903899192810059,
+      1.3903899192810059,
+      1.3903899192810059,
+      1.3903899192810059
+    ],
+    [
+      1.4738539457321167,
+      1.4738539457321167,
+      1.4738539457321167,
+      1.4738539457321167
+    ]
+  ],
+  "weights": "model.safetensors"
+}

falcondec_modeling.py

@@ -0,0 +1,358 @@
+# -*- coding: utf-8 -*-
+"""FalconDec — Falcon Decision model: single-pass, typed, calibrated closed-set decisions.
+
+Layout : [CLS] question [SEP] [MASK] opt_1 ... [MASK] opt_k [SEP] state [SEP]
+Head   : marker vectors + CLS context + question-type embedding
+         -> permutation-equivariant set transformer (options attend to each other; no positions)
+         -> MLP -> one logit per option -> softmax over this question's options
+Calib. : temperature per (question type, option-count bucket), stored in the checkpoint
+"""
+from __future__ import annotations
+
+import contextlib
+import json
+import shutil
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+QTYPES = {"choice": 0, "noul": 1, "score": 2}
+N_BUCKETS = 4
+CONFIG_FILE = "falcondec_config.json"
+FP16_FILE = "model.safetensors"
+INT8_FILE = "model_int8.safetensors"
+MODEL_KEYS = ("input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype")
+
+
+def n_bucket(n: int) -> int:
+    return 0 if n <= 2 else 1 if n <= 5 else 2 if n <= 12 else 3
+
+
+def special_ids(tok) -> dict:
+    sp = {"cls": tok.cls_token_id, "sep": tok.sep_token_id, "mask": tok.mask_token_id, "pad": tok.pad_token_id}
+    if sp["cls"] is None:
+        sp["cls"] = tok.bos_token_id
+    if sp["sep"] is None:
+        sp["sep"] = tok.eos_token_id
+    if sp["pad"] is None:
+        sp["pad"] = sp["sep"]
+    if sp["mask"] is None:
+        raise ValueError("FalconDec needs a tokenizer with a mask token (used as the option marker).")
+    return {k: int(v) for k, v in sp.items()}
+
+
+def _as_list(x):
+    return x.tolist() if hasattr(x, "tolist") else list(x)
+
+
+def assemble(q_ids, opt_ids, s_ids, sp, max_len=512, head_max_len=192, max_tok_per_opt=24,
+             long_max_len=2048, long_opts_threshold=24):
+    """Build one input sequence. Returns (ids, marker_positions)."""
+    n = len(opt_ids)
+    eff = max_len if n <= long_opts_threshold else max(max_len, long_max_len)
+    q = _as_list(q_ids)[:96]
+    need = len(q) + 2 + n * (max_tok_per_opt + 1)
+    head = min(eff - 64, max(head_max_len, need))
+    per = max(2, min(max_tok_per_opt, (head - len(q) - 2) // max(n, 1) - 1))
+    ids = [sp["cls"]] + q + [sp["sep"]]
+    markers = []
+    for o in opt_ids:
+        markers.append(len(ids))
+        ids.append(sp["mask"])
+        ids.extend(_as_list(o[:per]))
+    ids.append(sp["sep"])
+    room = eff - len(ids) - 1
+    if room > 0 and s_ids is not None and len(s_ids) > 0:
+        ids.extend(_as_list(s_ids[:room]))
+        ids.append(sp["sep"])
+    if len(ids) > eff:
+        ids = ids[:eff]
+    if markers and markers[-1] >= len(ids):
+        raise ValueError("Too many options for one pass; reduce options or raise long_max_len.")
+    return ids, markers
+
+
+def collate_features(feats, pad_id, device=None):
+    """feats: list of (ids, markers, qtype_index) -> dict of padded tensors matching FalconDec.forward."""
+    B = len(feats)
+    T = max(len(f[0]) for f in feats)
+    K = max(len(f[1]) for f in feats)
+    ids = torch.full((B, T), pad_id, dtype=torch.long)
+    att = torch.zeros((B, T), dtype=torch.long)
+    mpos = torch.zeros((B, K), dtype=torch.long)
+    mmask = torch.zeros((B, K), dtype=torch.bool)
+    qt = torch.zeros(B, dtype=torch.long)
+    for i, (x, mk, q) in enumerate(feats):
+        ids[i, : len(x)] = torch.as_tensor(x, dtype=torch.long)
+        att[i, : len(x)] = 1
+        mpos[i, : len(mk)] = torch.as_tensor(mk, dtype=torch.long)
+        mmask[i, : len(mk)] = True
+        qt[i] = int(q)
+    out = dict(input_ids=ids, attention_mask=att, marker_pos=mpos, marker_mask=mmask, qtype=qt)
+    if device is not None:
+        out = {k: v.to(device, non_blocking=True) for k, v in out.items()}
+    return out
+
+
+class OptionInteraction(nn.Module):
+    """Set transformer over the options of one question (no positional encoding => order-equivariant)."""
+
+    def __init__(self, d, n_layers=2, n_heads=8, dropout=0.1):
+        super().__init__()
+        layer = nn.TransformerEncoderLayer(d, n_heads, dim_feedforward=2 * d, dropout=dropout,
+                                           activation="gelu", batch_first=True, norm_first=True)
+        self.enc = nn.TransformerEncoder(layer, n_layers, enable_nested_tensor=False)
+
+    def forward(self, x, mask):
+        return self.enc(x, src_key_padding_mask=~mask)
+
+
+class FalconDec(nn.Module):
+    def __init__(self, encoder, fcfg: dict):
+        super().__init__()
+        self.encoder = encoder
+        self.fcfg = dict(fcfg)
+        d = encoder.config.hidden_size
+        self.qtype_emb = nn.Embedding(len(QTYPES), d)
+        self.ctx_proj = nn.Linear(d, d)
+        self.opt_norm = nn.LayerNorm(d)
+        self.interact = OptionInteraction(d, int(self.fcfg.get("interact_layers", 2)),
+                                          int(self.fcfg.get("interact_heads", 8)))
+        self.scorer = nn.Sequential(nn.Linear(d, d), nn.GELU(), nn.Dropout(0.1), nn.Linear(d, 1))
+        self.register_buffer("temperature", torch.ones(len(QTYPES), N_BUCKETS))
+
+    @property
+    def device(self):
+        return next(self.parameters()).device
+
+    def num_parameters(self):
+        return sum(p.numel() for p in self.parameters())
+
+    def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype):
+        h = self.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
+        B, K = marker_pos.shape
+        idx = marker_pos.unsqueeze(-1).expand(B, K, h.size(-1))
+        opt = h.gather(1, idx)
+        ctx = self.ctx_proj(h[:, 0]).unsqueeze(1)
+        x = self.opt_norm(opt + ctx + self.qtype_emb(qtype).unsqueeze(1))
+        x = self.interact(x, marker_mask)
+        logits = self.scorer(x).squeeze(-1).float()
+        return logits.masked_fill(~marker_mask, -1e4)
+
+
+# ------------------------------------------------------------------ storage
+def clean_state_dict(sd):
+    return {k.replace("_orig_mod.", ""): v.detach().cpu().contiguous() for k, v in sd.items()}
+
+
+def quantize_int8(sd, min_numel=4096):
+    """Per-output-channel symmetric int8 for every matrix; fp16 for everything else."""
+    out = {}
+    for k, v in sd.items():
+        if v.is_floating_point() and v.ndim == 2 and v.numel() >= min_numel:
+            w = v.float()
+            s = (w.abs().amax(dim=1, keepdim=True) / 127.0).clamp_min(1e-12)
+            out[k] = torch.round(w / s).clamp_(-127, 127).to(torch.int8).contiguous()
+            out[k + "::scale"] = s.contiguous()
+        elif v.is_floating_point():
+            out[k] = v.half().contiguous()
+        else:
+            out[k] = v.contiguous()
+    return out
+
+
+def dequantize_int8(sd):
+    out = {}
+    for k, v in sd.items():
+        if k.endswith("::scale"):
+            continue
+        s = sd.get(k + "::scale")
+        out[k] = (v.float() * s).half() if s is not None else v
+    return out
+
+
+def save_falcondec(model, tok, out_dir, int8=False, extra_files=None):
+    from safetensors.torch import save_file
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    enc_cfg = model.encoder.config
+    if hasattr(enc_cfg, "reference_compile"):
+        enc_cfg.reference_compile = False
+    enc_cfg.save_pretrained(str(out / "encoder"))
+    tok.save_pretrained(str(out / "tokenizer"))
+    sd = clean_state_dict(model.state_dict())
+    fc = dict(model.fcfg)
+    fc["temperature"] = model.temperature.detach().float().cpu().tolist()
+    fc["weights"] = INT8_FILE if int8 else FP16_FILE
+    if int8:
+        save_file(quantize_int8(sd), str(out / INT8_FILE), metadata={"format": "falcondec-int8"})
+    else:
+        save_file({k: (v.half() if v.is_floating_point() else v) for k, v in sd.items()},
+                  str(out / FP16_FILE), metadata={"format": "falcondec-fp16"})
+    (out / CONFIG_FILE).write_text(json.dumps(fc, indent=2, default=str), encoding="utf-8")
+    try:
+        shutil.copy(__file__, out / "falcondec_modeling.py")
+    except Exception:
+        pass
+    for name, content in (extra_files or {}).items():
+        (out / name).write_text(content, encoding="utf-8")
+    return out
+
+
+def load_falcondec(path, device=None, dtype=None, attn_implementation="sdpa"):
+    """Load a FalconDec directory (fp16 or int8) or Hub repo. Returns (model, tokenizer)."""
+    from safetensors.torch import load_file
+    from transformers import AutoConfig, AutoModel, AutoTokenizer
+    p = Path(path)
+    if not p.exists():
+        from huggingface_hub import snapshot_download
+        p = Path(snapshot_download(str(path)))
+    fc = json.loads((p / CONFIG_FILE).read_text(encoding="utf-8"))
+    ecfg = AutoConfig.from_pretrained(str(p / "encoder"))
+    if hasattr(ecfg, "reference_compile"):
+        ecfg.reference_compile = False
+    try:
+        enc = AutoModel.from_config(ecfg, attn_implementation=attn_implementation)
+    except Exception:
+        enc = AutoModel.from_config(ecfg)
+    model = FalconDec(enc, fc)
+    wf = p / fc.get("weights", FP16_FILE)
+    if not wf.exists():
+        wf = p / (INT8_FILE if (p / INT8_FILE).exists() else FP16_FILE)
+    sd = load_file(str(wf))
+    if wf.name == INT8_FILE:
+        sd = dequantize_int8(sd)
+    missing, unexpected = model.load_state_dict(sd, strict=False)
+    if missing or unexpected:
+        print(f"[FalconDec] load warning: missing={list(missing)[:5]} unexpected={list(unexpected)[:5]}")
+    tok = AutoTokenizer.from_pretrained(str(p / "tokenizer"))
+    dev = torch.device(device) if device is not None else torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model.to(dev)
+    if dtype is not None:
+        model.to(dtype)
+    model.eval()
+    return model, tok
+
+
+# ------------------------------------------------------------------ inference
+def _amp(model):
+    if model.device.type == "cuda" and next(model.parameters()).dtype == torch.float32:
+        dt = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
+        return torch.autocast("cuda", dtype=dt)
+    return contextlib.nullcontext()
+
+
+def _state_text(state):
+    if state is None:
+        return ""
+    return state if isinstance(state, str) else json.dumps(state, ensure_ascii=False)
+
+
+@torch.no_grad()
+def score_items(model, tok, items, batch_size=32):
+    """items: [{"state", "question", "options", "type"?, "option_tokens"?, "seq_len"?}] -> list of prob arrays."""
+    fc = model.fcfg
+    M = int(fc.get("max_opts_single_pass", 96))
+    out = [None] * len(items)
+    small = [i for i, it in enumerate(items) if len(it["options"]) <= M]
+    for i, it in enumerate(items):
+        if len(it["options"]) > M:
+            out[i] = _score_large(model, tok, it, batch_size)
+    if not small:
+        return out
+    sp = fc["special"]
+    prepared = {}
+    for i in small:
+        it = items[i]
+        if len(it["options"]) < 2:
+            raise ValueError("Every question needs at least two options.")
+        budget = int(it.get("option_tokens", fc["max_tok_per_opt"]))
+        q_ids = tok(it.get("question", ""), add_special_tokens=False, truncation=True, max_length=96)["input_ids"]
+        o_ids = tok([str(o) for o in it["options"]], add_special_tokens=False, truncation=True,
+                    max_length=budget)["input_ids"]
+        s = _state_text(it.get("state"))
+        s_ids = tok(s, add_special_tokens=False, truncation=True,
+                    max_length=int(fc["long_max_len"]))["input_ids"] if s else []
+        ids, mk = assemble(q_ids, o_ids, s_ids, sp, int(it.get("seq_len", fc["max_len"])), fc["head_max_len"],
+                           budget, fc["long_max_len"], fc["long_opts_threshold"])
+        prepared[i] = (ids, mk, QTYPES[it.get("type", "choice")])
+    order = sorted(small, key=lambda i: len(prepared[i][0]))
+    model.eval()
+    for b0 in range(0, len(order), batch_size):
+        idxs = order[b0: b0 + batch_size]
+        batch = collate_features([prepared[i] for i in idxs], sp["pad"], model.device)
+        with _amp(model):
+            logits = model(**batch)
+        for j, i in enumerate(idxs):
+            n = len(prepared[i][1])
+            T = model.temperature[prepared[i][2], n_bucket(n)].float().clamp_min(1e-3)
+            out[i] = torch.softmax(logits[j, :n].float() / T, -1).cpu().numpy()
+    return out
+
+
+def _score_large(model, tok, it, batch_size):
+    """Tournament for > max_opts_single_pass options: keep the best of each chunk, then one final pass."""
+    M = int(model.fcfg.get("max_opts_single_pass", 96))
+    opts = list(it["options"])
+    cand = list(range(len(opts)))
+    while len(cand) > M:
+        groups = [cand[i: i + M] for i in range(0, len(cand), M)]
+        keep = max(1, M // len(groups))
+        ps = score_items(model, tok, [dict(it, options=[opts[c] for c in g]) for g in groups], batch_size)
+        cand = [g[t] for g, p in zip(groups, ps) for t in np.argsort(-p)[:keep]]
+    p = score_items(model, tok, [dict(it, options=[opts[c] for c in cand])], batch_size)[0]
+    full = np.zeros(len(opts), dtype=np.float32)
+    full[cand] = p
+    return full
+
+
+def _normalize_question(q):
+    qtype = q.get("type", "choice")
+    text = q.get("question") or q.get("instructions") or ""
+    if qtype == "noul":
+        lab = q.get("labels") or {}
+        return qtype, text, [True, False], [str(lab.get("true", "Yes")), str(lab.get("false", "No"))]
+    crit = q.get("criteria", q.get("options"))
+    if qtype == "score":
+        opts = [str(c) for c in crit]
+        return qtype, text, list(range(len(opts))), opts
+    if isinstance(crit, dict):
+        return qtype, text, list(crit), [f"{k}: {v}" if v else str(k) for k, v in crit.items()]
+    return qtype, text, list(crit), [str(c) for c in crit]
+
+
+@torch.no_grad()
+def decide(model, tok, state, questions, defer_threshold=None, batch_size=32):
+    """Answer typed questions about one state.
+
+    questions: list of dicts, or a Jev/Laya-style dict {key: question}. Each question:
+      {"type": "choice"|"noul"|"score", "question"/"instructions": str,
+       "options": [...] or "criteria": {key: description} / [levels], "option_tokens"?: int}
+    """
+    if isinstance(questions, dict):
+        questions = [dict(q, key=k) for k, q in questions.items()]
+    items, metas = [], []
+    for q in questions:
+        qtype, text, keys, opts = _normalize_question(q)
+        item = {"state": state, "question": text, "options": opts, "type": qtype}
+        for extra in ("option_tokens", "seq_len"):
+            if extra in q:
+                item[extra] = q[extra]
+        items.append(item)
+        metas.append((q, qtype, keys, opts))
+    probs = score_items(model, tok, items, batch_size)
+    thr = model.fcfg.get("defer_threshold", 0.0) if defer_threshold is None else defer_threshold
+    results = []
+    for (q, qtype, keys, opts), p in zip(metas, probs):
+        i = int(np.argmax(p))
+        r = {"key": q.get("key"), "type": qtype, "question": items[len(results)]["question"],
+             "choice": keys[i], "choice_text": opts[i], "confidence": float(p[i]),
+             "probs": {str(k): float(v) for k, v in zip(keys, p)}, "defer": float(p[i]) < thr}
+        if qtype == "noul":
+            r["p_true"] = float(p[0])
+        if qtype == "score":
+            r["expected_level"] = float(np.dot(p, np.arange(len(p))))
+        results.append(r)
+    return {"results": results, "answers": {r["key"]: r for r in results if r["key"] is not None}}

falcondec_report.json

@@ -0,0 +1,901 @@
+{
+  "version": "1.0.1",
+  "notebook_version": "V2",
+  "mode": "finetune",
+  "parent_version": "1.0.0",
+  "lineage": [
+    "jhu-clsp/ettin-encoder-150m",
+    "FalconDec-v1.0.0"
+  ],
+  "config": {
+    "MODE": "finetune",
+    "FINETUNE_FROM": "falcondec_runs/FalconDec-v1.0.0",
+    "MODEL_NAME": "LightDec",
+    "BACKBONE": "jhu-clsp/ettin-encoder-150m",
+    "BACKBONE_INIT": "pretrained",
+    "PRESET": "full",
+    "CUSTOM_DATA_JSONL": "",
+    "CUSTOM_WEIGHT": 3.0,
+    "INCLUDE_BUILDERS": [],
+    "EXCLUDE_BUILDERS": [
+      "mind2web"
+    ],
+    "MAX_LEN": 512,
+    "LONG_MAX_LEN": 2048,
+    "LONG_OPTS_THRESHOLD": 24,
+    "HEAD_MAX_LEN": 192,
+    "MAX_TOK_PER_OPT": 24,
+    "MAX_OPTS_SINGLE_PASS": 96,
+    "INTERACT_LAYERS": 2,
+    "INTERACT_HEADS": 8,
+    "BATCH_SIZE": 32,
+    "TOKENS_PER_BATCH": 16384,
+    "GRAD_ACCUM": 1,
+    "LR_ENCODER": 4e-05,
+    "LR_HEAD": 0.0003,
+    "LLRD": 0.9,
+    "WEIGHT_DECAY": 0.01,
+    "WARMUP_FRAC": 0.06,
+    "EPOCHS": 0,
+    "EMA_DECAY": 0.999,
+    "GRAD_CLIP": 1.0,
+    "FINETUNE_LR_SCALE": 0.5,
+    "SPHERICAL_W": 0.5,
+    "BRIER_W": 0.0,
+    "RPS_W": 1.0,
+    "TASK_SAMPLING_ALPHA": 0.5,
+    "NOTA_PROB": 0.08,
+    "RLCD_EPOCHS": 0.0,
+    "RLCD_SAMPLES": 4,
+    "RLCD_SIGMA": 0.3,
+    "RLCD_CE_ANCHOR": 0.5,
+    "DEVICE": "cuda",
+    "SEED": 42,
+    "NUM_WORKERS": 0,
+    "USE_COMPILE": false,
+    "ATTN_IMPL": "auto",
+    "OUTPUT_ROOT": "./falcondec_runs",
+    "EXPORT_INT8": true,
+    "SIZE_TARGET_MB": 250,
+    "DEFER_THRESHOLD": 0.7,
+    "EVAL_BASELINE_PROOF_V2": true,
+    "BASELINE_MAX_PER_TASK": 120,
+    "RUN_CPU_BENCH": true,
+    "PUSH_TO_HUB": false,
+    "HUB_REPO": ""
+  },
+  "epochs": 2,
+  "best_epoch": 1,
+  "rlcd_kept": false,
+  "history": [
+    {
+      "epoch": 1,
+      "train_loss": 0.47463753718974483,
+      "train_acc": 0.8307763076996187,
+      "val_macro": 0.8253118682436162,
+      "val_micro": 0.834065625876414,
+      "val_nll": 0.4055366800702441
+    },
+    {
+      "epoch": 2,
+      "train_loss": 0.36265113689414885,
+      "train_acc": 0.8738680252304237,
+      "val_macro": 0.8252539304231531,
+      "val_micro": 0.8348602411891184,
+      "val_nll": 0.4680047751477799
+    }
+  ],
+  "train_minutes": 172.17712485790253,
+  "temperatures": [
+    [
+      1.3723914623260498,
+      1.3805755376815796,
+      1.5033340454101562,
+      1.320592999458313
+    ],
+    [
+      1.3903899192810059,
+      1.3903899192810059,
+      1.3903899192810059,
+      1.3903899192810059
+    ],
+    [
+      1.4738539457321167,
+      1.4738539457321167,
+      1.4738539457321167,
+      1.4738539457321167
+    ]
+  ],
+  "data_counts": {
+    "train": 946028,
+    "validation": 21394,
+    "test": 26597
+  },
+  "skipped_builders": [],
+  "test_overall": {
+    "n": 26597.0,
+    "micro_acc": 0.7810655337068091,
+    "macro_acc": 0.7694407230754166,
+    "nll": 0.5288964978022598,
+    "brier": 0.2897912847111225,
+    "ece": 0.01969622960708085,
+    "aurc": 0.06403486126686972,
+    "score_mae": 0.4928175845338412,
+    "coverage@0.7": 0.6595104711057638,
+    "acc_on_covered@0.7": 0.9167094236360527
+  },
+  "test_per_domain": {
+    "agentic": 0.7781070271610528,
+    "classification": 0.615,
+    "code": 0.8717717677968562,
+    "guardrails": 0.7951432854293641,
+    "intents": 0.8278333333333333,
+    "policy": 0.7873333333333333,
+    "reasoning": 0.7006267469170804,
+    "support": 1.0,
+    "workflows": 0.755
+  },
+  "test_per_task": [
+    {
+      "task": "ag_news/topic",
+      "n": 500,
+      "acc": 0.888,
+      "chance": 0.25,
+      "nll": 0.28043824258210953,
+      "brier": 0.154232526432949,
+      "heldout": false,
+      "domain": "classification",
+      "ece": 0.03968544936180117,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "agentharm/refuse",
+      "n": 416,
+      "acc": 0.7427884615384616,
+      "chance": 0.5,
+      "nll": 0.5331332034928402,
+      "brier": 0.35988097897984717,
+      "heldout": true,
+      "domain": "guardrails",
+      "ece": 0.06706968007179408,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "anli/nli",
+      "n": 498,
+      "acc": 0.41566265060240964,
+      "chance": 0.33333333333333326,
+      "nll": 1.1498472186171984,
+      "brier": 0.7016487136039561,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.18407419437624845,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "aqua_rat/math",
+      "n": 247,
+      "acc": 0.27530364372469635,
+      "chance": 0.2,
+      "nll": 1.6108178135746645,
+      "brier": 0.7991190068538744,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.04157990880823329,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "arc_challenge/mcq",
+      "n": 500,
+      "acc": 0.46,
+      "chance": 0.25006666666666666,
+      "nll": 1.2087727874126286,
+      "brier": 0.653150732152712,
+      "heldout": true,
+      "domain": "reasoning",
+      "ece": 0.08824656188488007,
+      "proof_v2 (card)": 0.308,
+      "\u0394 vs proof_v2": 0.15200000000000002
+    },
+    {
+      "task": "arc_easy/mcq",
+      "n": 500,
+      "acc": 0.594,
+      "chance": 0.24980000000000002,
+      "nll": 0.9366558539052494,
+      "brier": 0.5142523023860749,
+      "heldout": true,
+      "domain": "reasoning",
+      "ece": 0.0377397992014885,
+      "proof_v2 (card)": 0.425,
+      "\u0394 vs proof_v2": 0.16899999999999998
+    },
+    {
+      "task": "banking77/intent",
+      "n": 500,
+      "acc": 0.93,
+      "chance": 0.3214666666666666,
+      "nll": 0.21486007268911725,
+      "brier": 0.11047492184269238,
+      "heldout": true,
+      "domain": "intents",
+      "ece": 0.04031616294384006,
+      "proof_v2 (card)": 0.883,
+      "\u0394 vs proof_v2": 0.04700000000000004
+    },
+    {
+      "task": "banking77/intent_77",
+      "n": 300,
+      "acc": 0.5333333333333333,
+      "chance": 0.012987012987012986,
+      "nll": 2.063840282613334,
+      "brier": 0.6459762259581862,
+      "heldout": true,
+      "domain": "intents",
+      "ece": 0.14717036068439482,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "bigclonebench/clone",
+      "n": 500,
+      "acc": 0.952,
+      "chance": 0.5,
+      "nll": 0.14842138955372502,
+      "brier": 0.07646761075289837,
+      "heldout": false,
+      "domain": "code",
+      "ece": 0.015586107730865495,
+      "proof_v2 (card)": 0.383,
+      "\u0394 vs proof_v2": 0.569
+    },
+    {
+      "task": "bitext/category",
+      "n": 500,
+      "acc": 1.0,
+      "chance": 0.16128124098124094,
+      "nll": 0.0018036427325441764,
+      "brier": 0.000553129204988908,
+      "heldout": false,
+      "domain": "support",
+      "ece": 0.0015759592056274496,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "bitext/route",
+      "n": 500,
+      "acc": 1.0,
+      "chance": 0.19016313131313134,
+      "nll": 0.0011070070102664432,
+      "brier": 7.369536871875006e-05,
+      "heldout": false,
+      "domain": "support",
+      "ece": 0.0010853933095931502,
+      "proof_v2 (card)": 0.958,
+      "\u0394 vs proof_v2": 0.04200000000000004
+    },
+    {
+      "task": "boolq/yes_no",
+      "n": 500,
+      "acc": 0.816,
+      "chance": 0.5,
+      "nll": 0.4159143784940243,
+      "brier": 0.26228454396614875,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.02717309403419496,
+      "proof_v2 (card)": 0.717,
+      "\u0394 vs proof_v2": 0.09899999999999998
+    },
+    {
+      "task": "civil_comments/toxic",
+      "n": 500,
+      "acc": 0.906,
+      "chance": 0.5,
+      "nll": 0.2400463573904126,
+      "brier": 0.13785060332637392,
+      "heldout": false,
+      "domain": "guardrails",
+      "ece": 0.035369446873664855,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "clinc150/intent",
+      "n": 500,
+      "acc": 0.942,
+      "chance": 0.12404474984829318,
+      "nll": 0.16339450041492637,
+      "brier": 0.08445263661020003,
+      "heldout": false,
+      "domain": "intents",
+      "ece": 0.036179060459136984,
+      "proof_v2 (card)": 0.85,
+      "\u0394 vs proof_v2": 0.09199999999999997
+    },
+    {
+      "task": "codexglue/code_to_doc",
+      "n": 504,
+      "acc": 0.9841269841269841,
+      "chance": 0.26233465608465617,
+      "nll": 0.04367021227094591,
+      "brier": 0.02423435580505018,
+      "heldout": false,
+      "domain": "code",
+      "ece": 0.011909238107147682,
+      "proof_v2 (card)": 0.969,
+      "\u0394 vs proof_v2": 0.0151269841269841
+    },
+    {
+      "task": "codexglue/doc_to_code",
+      "n": 504,
+      "acc": 0.9761904761904762,
+      "chance": 0.2786044973544974,
+      "nll": 0.05729491385996503,
+      "brier": 0.030591094765289834,
+      "heldout": false,
+      "domain": "code",
+      "ece": 0.01021853999959103,
+      "proof_v2 (card)": 0.961,
+      "\u0394 vs proof_v2": 0.015190476190476199
+    },
+    {
+      "task": "codexglue/func_name",
+      "n": 467,
+      "acc": 0.9528907922912205,
+      "chance": 0.25499643112062814,
+      "nll": 0.17043053883545994,
+      "brier": 0.08725536960997705,
+      "heldout": false,
+      "domain": "code",
+      "ece": 0.023257157468183163,
+      "proof_v2 (card)": 0.901,
+      "\u0394 vs proof_v2": 0.05189079229122051
+    },
+    {
+      "task": "codexglue/lang_id",
+      "n": 504,
+      "acc": 1.0,
+      "chance": 0.2354166666666667,
+      "nll": 0.0005584346231148719,
+      "brier": 3.0963766792822474e-06,
+      "heldout": false,
+      "domain": "code",
+      "ece": 0.0005574606004214999,
+      "proof_v2 (card)": 0.997,
+      "\u0394 vs proof_v2": 0.0030000000000000027
+    },
+    {
+      "task": "commonsense_qa/mcq",
+      "n": 493,
+      "acc": 0.6389452332657201,
+      "chance": 0.19999999999999998,
+      "nll": 0.904400206106629,
+      "brier": 0.47107552904904443,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.04405802657589711,
+      "proof_v2 (card)": 0.442,
+      "\u0394 vs proof_v2": 0.19694523326572005
+    },
+    {
+      "task": "counsel/critique_quality",
+      "n": 201,
+      "acc": 0.5920398009950248,
+      "chance": 0.3333333333333333,
+      "nll": 1.3054839486733487,
+      "brier": 0.6194979652829024,
+      "heldout": false,
+      "domain": "agentic",
+      "ece": 0.22704330650135063,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "counsel/step_has_error",
+      "n": 201,
+      "acc": 0.8009950248756219,
+      "chance": 0.5,
+      "nll": 0.8227317674037697,
+      "brier": 0.3473574496250388,
+      "heldout": false,
+      "domain": "agentic",
+      "ece": 0.16924872979595879,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "devign/vulnerability",
+      "n": 500,
+      "acc": 0.596,
+      "chance": 0.5,
+      "nll": 0.6676419580578804,
+      "brier": 0.4753553111644763,
+      "heldout": false,
+      "domain": "code",
+      "ece": 0.05009425449371341,
+      "proof_v2 (card)": 0.542,
+      "\u0394 vs proof_v2": 0.05399999999999994
+    },
+    {
+      "task": "emotion/6way",
+      "n": 500,
+      "acc": 0.502,
+      "chance": 0.16666666666666663,
+      "nll": 1.3070907976925372,
+      "brier": 0.6468766489597455,
+      "heldout": true,
+      "domain": "classification",
+      "ece": 0.08215402710437775,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "gsm8k/math",
+      "n": 500,
+      "acc": 0.662,
+      "chance": 0.25,
+      "nll": 0.7718028125888668,
+      "brier": 0.4427847443219045,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.06086966800689696,
+      "proof_v2 (card)": 0.275,
+      "\u0394 vs proof_v2": 0.387
+    },
+    {
+      "task": "hellaswag/continuation",
+      "n": 500,
+      "acc": 0.58,
+      "chance": 0.25,
+      "nll": 0.9953804229423404,
+      "brier": 0.5371978334623486,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.07333891946077346,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "hotpotqa/comparison_yes_no",
+      "n": 26,
+      "acc": 0.8461538461538461,
+      "chance": 0.5,
+      "nll": 0.43042818936877525,
+      "brier": 0.2727966735853732,
+      "heldout": false,
+      "domain": "agentic",
+      "ece": 0.15112700829139125,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "hotpotqa/retrieve",
+      "n": 497,
+      "acc": 0.8732394366197183,
+      "chance": 0.16741720162243304,
+      "nll": 0.4343729186731075,
+      "brier": 0.20390003436957638,
+      "heldout": false,
+      "domain": "agentic",
+      "ece": 0.08316291382974779,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "humaneval/completion",
+      "n": 119,
+      "acc": 0.6722689075630253,
+      "chance": 0.39355742296918783,
+      "nll": 0.7120104984726616,
+      "brier": 0.44585727429903876,
+      "heldout": true,
+      "domain": "code",
+      "ece": 0.10659772783768277,
+      "proof_v2 (card)": 0.575,
+      "\u0394 vs proof_v2": 0.09726890756302531
+    },
+    {
+      "task": "jailbreak/detect",
+      "n": 262,
+      "acc": 0.9541984732824428,
+      "chance": 0.5,
+      "nll": 0.10692274935157883,
+      "brier": 0.06256811361157136,
+      "heldout": false,
+      "domain": "guardrails",
+      "ece": 0.0232666851455019,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "massive_en/intent",
+      "n": 500,
+      "acc": 0.906,
+      "chance": 0.12958727789085603,
+      "nll": 0.26250843075065133,
+      "brier": 0.12679194471370903,
+      "heldout": false,
+      "domain": "intents",
+      "ece": 0.022001883208751682,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "mbpp/bugspot",
+      "n": 256,
+      "acc": 0.73046875,
+      "chance": 0.40559895833333326,
+      "nll": 0.5201096873639273,
+      "brier": 0.3294220878204956,
+      "heldout": false,
+      "domain": "code",
+      "ece": 0.1015084749087691,
+      "proof_v2 (card)": 0.475,
+      "\u0394 vs proof_v2": 0.25546875
+    },
+    {
+      "task": "mbpp/solution",
+      "n": 500,
+      "acc": 0.982,
+      "chance": 0.25,
+      "nll": 0.06299449819558049,
+      "brier": 0.03283427225700136,
+      "heldout": false,
+      "domain": "code",
+      "ece": 0.014863664209842696,
+      "proof_v2 (card)": 0.992,
+      "\u0394 vs proof_v2": -0.010000000000000009
+    },
+    {
+      "task": "mmlu/mcq",
+      "n": 500,
+      "acc": 0.356,
+      "chance": 0.25,
+      "nll": 1.370702707901597,
+      "brier": 0.738678903692766,
+      "heldout": true,
+      "domain": "reasoning",
+      "ece": 0.10921649205684662,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "mnli/claim",
+      "n": 500,
+      "acc": 0.838,
+      "chance": 0.33333333333333326,
+      "nll": 0.41442326370440424,
+      "brier": 0.23035331540149503,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.07134601444005968,
+      "proof_v2 (card)": 0.492,
+      "\u0394 vs proof_v2": 0.346
+    },
+    {
+      "task": "openbookqa/mcq",
+      "n": 500,
+      "acc": 0.61,
+      "chance": 0.25,
+      "nll": 1.024268753977958,
+      "brier": 0.5355113333378367,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.09224175179004669,
+      "proof_v2 (card)": 0.292,
+      "\u0394 vs proof_v2": 0.318
+    },
+    {
+      "task": "policy/access_control_transfer",
+      "n": 500,
+      "acc": 1.0,
+      "chance": 0.33333333333333326,
+      "nll": 0.0002645651292309594,
+      "brier": 1.1635768837216181e-06,
+      "heldout": false,
+      "domain": "policy",
+      "ece": 0.00026422095298772597,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "policy/count_threshold_transfer",
+      "n": 500,
+      "acc": 0.846,
+      "chance": 0.17848809523809525,
+      "nll": 0.4453474920627887,
+      "brier": 0.239936219050859,
+      "heldout": false,
+      "domain": "policy",
+      "ece": 0.10926523756980895,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "policy/free_shipping_transfer",
+      "n": 500,
+      "acc": 0.766,
+      "chance": 0.5,
+      "nll": 0.39174148760084065,
+      "brier": 0.2747179012451008,
+      "heldout": false,
+      "domain": "policy",
+      "ece": 0.10137953925132755,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "policy/invoice_overdue_transfer",
+      "n": 500,
+      "acc": 0.6,
+      "chance": 0.5,
+      "nll": 0.8301094107693061,
+      "brier": 0.5458620193712137,
+      "heldout": false,
+      "domain": "policy",
+      "ece": 0.20100946557521818,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "policy/invoice_total_transfer",
+      "n": 500,
+      "acc": 0.486,
+      "chance": 0.5,
+      "nll": 0.69399339812994,
+      "brier": 0.5008197939882997,
+      "heldout": false,
+      "domain": "policy",
+      "ece": 0.03210310280323031,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "policy/refund_approval_transfer",
+      "n": 500,
+      "acc": 0.78,
+      "chance": 0.33333333333333326,
+      "nll": 0.8478962361293161,
+      "brier": 0.38071118861218445,
+      "heldout": false,
+      "domain": "policy",
+      "ece": 0.18587953794002532,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "policy/return_window_transfer",
+      "n": 500,
+      "acc": 0.998,
+      "chance": 0.33333333333333326,
+      "nll": 0.02459742584443666,
+      "brier": 0.005749234483560448,
+      "heldout": false,
+      "domain": "policy",
+      "ece": 0.019777906179428068,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "policy/sla_urgency_transfer",
+      "n": 500,
+      "acc": 0.698,
+      "chance": 0.25,
+      "nll": 0.9843494065143168,
+      "brier": 0.49182705060356263,
+      "heldout": false,
+      "domain": "policy",
+      "ece": 0.21234694278240196,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "policy/table_extreme_transfer",
+      "n": 500,
+      "acc": 0.912,
+      "chance": 0.23716666666666666,
+      "nll": 0.22187845180515342,
+      "brier": 0.12271458379265106,
+      "heldout": false,
+      "domain": "policy",
+      "ece": 0.013898970842361484,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "prompt_injections/detect",
+      "n": 116,
+      "acc": 0.5775862068965517,
+      "chance": 0.5,
+      "nll": 1.0671347150446622,
+      "brier": 0.628204618498277,
+      "heldout": true,
+      "domain": "guardrails",
+      "ece": 0.2859799378904803,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "qasc/mcq",
+      "n": 500,
+      "acc": 0.99,
+      "chance": 0.125,
+      "nll": 0.0315895584258742,
+      "brier": 0.01700795004425532,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.006339998722076429,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "sciq/mcq",
+      "n": 498,
+      "acc": 0.9538152610441767,
+      "chance": 0.25,
+      "nll": 0.11897318925779031,
+      "brier": 0.06399753046801272,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.03059945360244999,
+      "proof_v2 (card)": 0.692,
+      "\u0394 vs proof_v2": 0.26181526104417674
+    },
+    {
+      "task": "scitail/support",
+      "n": 500,
+      "acc": 0.964,
+      "chance": 0.5,
+      "nll": 0.10391226119734347,
+      "brier": 0.05616228693429717,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.020327572941780014,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "snli/contradicts",
+      "n": 500,
+      "acc": 0.986,
+      "chance": 0.33333333333333326,
+      "nll": 0.048629490838502536,
+      "brier": 0.024212671137030826,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.01413622707128527,
+      "proof_v2 (card)": 0.892,
+      "\u0394 vs proof_v2": 0.09399999999999997
+    },
+    {
+      "task": "snli/must_be_true",
+      "n": 500,
+      "acc": 0.98,
+      "chance": 0.33333333333333326,
+      "nll": 0.060363937412155795,
+      "brier": 0.02947848703942969,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.02905823123455048,
+      "proof_v2 (card)": 0.908,
+      "\u0394 vs proof_v2": 0.07199999999999995
+    },
+    {
+      "task": "snli/nli",
+      "n": 988,
+      "acc": 0.8795546558704453,
+      "chance": 0.33333333333333326,
+      "nll": 0.3359528171680472,
+      "brier": 0.18385021007121183,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.04961462647688053,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "sst5/score",
+      "n": 500,
+      "acc": 0.416,
+      "chance": 0.2,
+      "nll": 1.246531030535698,
+      "brier": 0.6734807272262489,
+      "heldout": true,
+      "domain": "classification",
+      "ece": 0.04567561271786689,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "typed_decisions/agent_trace_observability",
+      "n": 500,
+      "acc": 0.712,
+      "chance": 0.3,
+      "nll": 0.7305186161920428,
+      "brier": 0.42510022971515116,
+      "heldout": false,
+      "domain": "workflows",
+      "ece": 0.11957757359743115,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "typed_decisions/customer_service",
+      "n": 500,
+      "acc": 0.74,
+      "chance": 0.28,
+      "nll": 0.6714302012249828,
+      "brier": 0.3723593006177565,
+      "heldout": false,
+      "domain": "workflows",
+      "ece": 0.08548467075824738,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "typed_decisions/invoice_processing",
+      "n": 500,
+      "acc": 0.838,
+      "chance": 0.35,
+      "nll": 0.5010932807754725,
+      "brier": 0.26905854083557984,
+      "heldout": false,
+      "domain": "workflows",
+      "ece": 0.12220623338222505,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "typed_decisions/security_incidents",
+      "n": 500,
+      "acc": 0.73,
+      "chance": 0.34,
+      "nll": 0.6859722163826227,
+      "brier": 0.3929527039537006,
+      "heldout": false,
+      "domain": "workflows",
+      "ece": 0.143579844892025,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "winogrande/blank",
+      "n": 500,
+      "acc": 0.612,
+      "chance": 0.5,
+      "nll": 0.7296091562472283,
+      "brier": 0.508893263835874,
+      "heldout": false,
+      "domain": "reasoning",
+      "ece": 0.11118709647655486,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    },
+    {
+      "task": "yelp/score",
+      "n": 500,
+      "acc": 0.654,
+      "chance": 0.2,
+      "nll": 0.7885563414096832,
+      "brier": 0.4562026964016218,
+      "heldout": false,
+      "domain": "classification",
+      "ece": 0.06915212672948838,
+      "proof_v2 (card)": NaN,
+      "\u0394 vs proof_v2": NaN
+    }
+  ],
+  "head_to_head": null,
+  "env": {
+    "torch": "2.11.0+cu128",
+    "transformers": "5.17.0",
+    "python": "3.14.4",
+    "gpu": "NVIDIA GeForce RTX 5090 Laptop GPU"
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80486c432d95a5b8b69f79a728bc8a3e42f12a86cb16abf2016b2a3076faa7fd
+size 319325890

tokenizer/tokenizer_config.json

@@ -2,9 +2,10 @@
   "backend": "tokenizers",
   "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
-  "is_local": false,
+  "is_local": true,
   "local_files_only": false,
   "mask_token": "[MASK]",
+  "max_length": 256,
   "model_input_names": [
     "input_ids",
     "attention_mask"
@@ -12,6 +13,9 @@
   "model_max_length": 8192,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
+  "stride": 0,
   "tokenizer_class": "TokenizersBackend",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "[UNK]"
 }