Invalid JSON:Unexpected token 'N', ..." (card)": NaN,
"... is not valid JSON
| { | |
| "version": "1.0.1", | |
| "notebook_version": "V2", | |
| "mode": "finetune", | |
| "parent_version": "1.0.0", | |
| "lineage": [ | |
| "jhu-clsp/ettin-encoder-150m", | |
| "FalconDec-v1.0.0" | |
| ], | |
| "config": { | |
| "MODE": "finetune", | |
| "FINETUNE_FROM": "falcondec_runs/FalconDec-v1.0.0", | |
| "MODEL_NAME": "LightDec", | |
| "BACKBONE": "jhu-clsp/ettin-encoder-150m", | |
| "BACKBONE_INIT": "pretrained", | |
| "PRESET": "full", | |
| "CUSTOM_DATA_JSONL": "", | |
| "CUSTOM_WEIGHT": 3.0, | |
| "INCLUDE_BUILDERS": [], | |
| "EXCLUDE_BUILDERS": [ | |
| "mind2web" | |
| ], | |
| "MAX_LEN": 512, | |
| "LONG_MAX_LEN": 2048, | |
| "LONG_OPTS_THRESHOLD": 24, | |
| "HEAD_MAX_LEN": 192, | |
| "MAX_TOK_PER_OPT": 24, | |
| "MAX_OPTS_SINGLE_PASS": 96, | |
| "INTERACT_LAYERS": 2, | |
| "INTERACT_HEADS": 8, | |
| "BATCH_SIZE": 32, | |
| "TOKENS_PER_BATCH": 16384, | |
| "GRAD_ACCUM": 1, | |
| "LR_ENCODER": 4e-05, | |
| "LR_HEAD": 0.0003, | |
| "LLRD": 0.9, | |
| "WEIGHT_DECAY": 0.01, | |
| "WARMUP_FRAC": 0.06, | |
| "EPOCHS": 0, | |
| "EMA_DECAY": 0.999, | |
| "GRAD_CLIP": 1.0, | |
| "FINETUNE_LR_SCALE": 0.5, | |
| "SPHERICAL_W": 0.5, | |
| "BRIER_W": 0.0, | |
| "RPS_W": 1.0, | |
| "TASK_SAMPLING_ALPHA": 0.5, | |
| "NOTA_PROB": 0.08, | |
| "RLCD_EPOCHS": 0.0, | |
| "RLCD_SAMPLES": 4, | |
| "RLCD_SIGMA": 0.3, | |
| "RLCD_CE_ANCHOR": 0.5, | |
| "DEVICE": "cuda", | |
| "SEED": 42, | |
| "NUM_WORKERS": 0, | |
| "USE_COMPILE": false, | |
| "ATTN_IMPL": "auto", | |
| "OUTPUT_ROOT": "./falcondec_runs", | |
| "EXPORT_INT8": true, | |
| "SIZE_TARGET_MB": 250, | |
| "DEFER_THRESHOLD": 0.7, | |
| "EVAL_BASELINE_PROOF_V2": true, | |
| "BASELINE_MAX_PER_TASK": 120, | |
| "RUN_CPU_BENCH": true, | |
| "PUSH_TO_HUB": false, | |
| "HUB_REPO": "" | |
| }, | |
| "epochs": 2, | |
| "best_epoch": 1, | |
| "rlcd_kept": false, | |
| "history": [ | |
| { | |
| "epoch": 1, | |
| "train_loss": 0.47463753718974483, | |
| "train_acc": 0.8307763076996187, | |
| "val_macro": 0.8253118682436162, | |
| "val_micro": 0.834065625876414, | |
| "val_nll": 0.4055366800702441 | |
| }, | |
| { | |
| "epoch": 2, | |
| "train_loss": 0.36265113689414885, | |
| "train_acc": 0.8738680252304237, | |
| "val_macro": 0.8252539304231531, | |
| "val_micro": 0.8348602411891184, | |
| "val_nll": 0.4680047751477799 | |
| } | |
| ], | |
| "train_minutes": 172.17712485790253, | |
| "temperatures": [ | |
| [ | |
| 1.3723914623260498, | |
| 1.3805755376815796, | |
| 1.5033340454101562, | |
| 1.320592999458313 | |
| ], | |
| [ | |
| 1.3903899192810059, | |
| 1.3903899192810059, | |
| 1.3903899192810059, | |
| 1.3903899192810059 | |
| ], | |
| [ | |
| 1.4738539457321167, | |
| 1.4738539457321167, | |
| 1.4738539457321167, | |
| 1.4738539457321167 | |
| ] | |
| ], | |
| "data_counts": { | |
| "train": 946028, | |
| "validation": 21394, | |
| "test": 26597 | |
| }, | |
| "skipped_builders": [], | |
| "test_overall": { | |
| "n": 26597.0, | |
| "micro_acc": 0.7810655337068091, | |
| "macro_acc": 0.7694407230754166, | |
| "nll": 0.5288964978022598, | |
| "brier": 0.2897912847111225, | |
| "ece": 0.01969622960708085, | |
| "aurc": 0.06403486126686972, | |
| "score_mae": 0.4928175845338412, | |
| "coverage@0.7": 0.6595104711057638, | |
| "acc_on_covered@0.7": 0.9167094236360527 | |
| }, | |
| "test_per_domain": { | |
| "agentic": 0.7781070271610528, | |
| "classification": 0.615, | |
| "code": 0.8717717677968562, | |
| "guardrails": 0.7951432854293641, | |
| "intents": 0.8278333333333333, | |
| "policy": 0.7873333333333333, | |
| "reasoning": 0.7006267469170804, | |
| "support": 1.0, | |
| "workflows": 0.755 | |
| }, | |
| "test_per_task": [ | |
| { | |
| "task": "ag_news/topic", | |
| "n": 500, | |
| "acc": 0.888, | |
| "chance": 0.25, | |
| "nll": 0.28043824258210953, | |
| "brier": 0.154232526432949, | |
| "heldout": false, | |
| "domain": "classification", | |
| "ece": 0.03968544936180117, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "agentharm/refuse", | |
| "n": 416, | |
| "acc": 0.7427884615384616, | |
| "chance": 0.5, | |
| "nll": 0.5331332034928402, | |
| "brier": 0.35988097897984717, | |
| "heldout": true, | |
| "domain": "guardrails", | |
| "ece": 0.06706968007179408, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "anli/nli", | |
| "n": 498, | |
| "acc": 0.41566265060240964, | |
| "chance": 0.33333333333333326, | |
| "nll": 1.1498472186171984, | |
| "brier": 0.7016487136039561, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.18407419437624845, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "aqua_rat/math", | |
| "n": 247, | |
| "acc": 0.27530364372469635, | |
| "chance": 0.2, | |
| "nll": 1.6108178135746645, | |
| "brier": 0.7991190068538744, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.04157990880823329, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "arc_challenge/mcq", | |
| "n": 500, | |
| "acc": 0.46, | |
| "chance": 0.25006666666666666, | |
| "nll": 1.2087727874126286, | |
| "brier": 0.653150732152712, | |
| "heldout": true, | |
| "domain": "reasoning", | |
| "ece": 0.08824656188488007, | |
| "proof_v2 (card)": 0.308, | |
| "\u0394 vs proof_v2": 0.15200000000000002 | |
| }, | |
| { | |
| "task": "arc_easy/mcq", | |
| "n": 500, | |
| "acc": 0.594, | |
| "chance": 0.24980000000000002, | |
| "nll": 0.9366558539052494, | |
| "brier": 0.5142523023860749, | |
| "heldout": true, | |
| "domain": "reasoning", | |
| "ece": 0.0377397992014885, | |
| "proof_v2 (card)": 0.425, | |
| "\u0394 vs proof_v2": 0.16899999999999998 | |
| }, | |
| { | |
| "task": "banking77/intent", | |
| "n": 500, | |
| "acc": 0.93, | |
| "chance": 0.3214666666666666, | |
| "nll": 0.21486007268911725, | |
| "brier": 0.11047492184269238, | |
| "heldout": true, | |
| "domain": "intents", | |
| "ece": 0.04031616294384006, | |
| "proof_v2 (card)": 0.883, | |
| "\u0394 vs proof_v2": 0.04700000000000004 | |
| }, | |
| { | |
| "task": "banking77/intent_77", | |
| "n": 300, | |
| "acc": 0.5333333333333333, | |
| "chance": 0.012987012987012986, | |
| "nll": 2.063840282613334, | |
| "brier": 0.6459762259581862, | |
| "heldout": true, | |
| "domain": "intents", | |
| "ece": 0.14717036068439482, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "bigclonebench/clone", | |
| "n": 500, | |
| "acc": 0.952, | |
| "chance": 0.5, | |
| "nll": 0.14842138955372502, | |
| "brier": 0.07646761075289837, | |
| "heldout": false, | |
| "domain": "code", | |
| "ece": 0.015586107730865495, | |
| "proof_v2 (card)": 0.383, | |
| "\u0394 vs proof_v2": 0.569 | |
| }, | |
| { | |
| "task": "bitext/category", | |
| "n": 500, | |
| "acc": 1.0, | |
| "chance": 0.16128124098124094, | |
| "nll": 0.0018036427325441764, | |
| "brier": 0.000553129204988908, | |
| "heldout": false, | |
| "domain": "support", | |
| "ece": 0.0015759592056274496, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "bitext/route", | |
| "n": 500, | |
| "acc": 1.0, | |
| "chance": 0.19016313131313134, | |
| "nll": 0.0011070070102664432, | |
| "brier": 7.369536871875006e-05, | |
| "heldout": false, | |
| "domain": "support", | |
| "ece": 0.0010853933095931502, | |
| "proof_v2 (card)": 0.958, | |
| "\u0394 vs proof_v2": 0.04200000000000004 | |
| }, | |
| { | |
| "task": "boolq/yes_no", | |
| "n": 500, | |
| "acc": 0.816, | |
| "chance": 0.5, | |
| "nll": 0.4159143784940243, | |
| "brier": 0.26228454396614875, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.02717309403419496, | |
| "proof_v2 (card)": 0.717, | |
| "\u0394 vs proof_v2": 0.09899999999999998 | |
| }, | |
| { | |
| "task": "civil_comments/toxic", | |
| "n": 500, | |
| "acc": 0.906, | |
| "chance": 0.5, | |
| "nll": 0.2400463573904126, | |
| "brier": 0.13785060332637392, | |
| "heldout": false, | |
| "domain": "guardrails", | |
| "ece": 0.035369446873664855, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "clinc150/intent", | |
| "n": 500, | |
| "acc": 0.942, | |
| "chance": 0.12404474984829318, | |
| "nll": 0.16339450041492637, | |
| "brier": 0.08445263661020003, | |
| "heldout": false, | |
| "domain": "intents", | |
| "ece": 0.036179060459136984, | |
| "proof_v2 (card)": 0.85, | |
| "\u0394 vs proof_v2": 0.09199999999999997 | |
| }, | |
| { | |
| "task": "codexglue/code_to_doc", | |
| "n": 504, | |
| "acc": 0.9841269841269841, | |
| "chance": 0.26233465608465617, | |
| "nll": 0.04367021227094591, | |
| "brier": 0.02423435580505018, | |
| "heldout": false, | |
| "domain": "code", | |
| "ece": 0.011909238107147682, | |
| "proof_v2 (card)": 0.969, | |
| "\u0394 vs proof_v2": 0.0151269841269841 | |
| }, | |
| { | |
| "task": "codexglue/doc_to_code", | |
| "n": 504, | |
| "acc": 0.9761904761904762, | |
| "chance": 0.2786044973544974, | |
| "nll": 0.05729491385996503, | |
| "brier": 0.030591094765289834, | |
| "heldout": false, | |
| "domain": "code", | |
| "ece": 0.01021853999959103, | |
| "proof_v2 (card)": 0.961, | |
| "\u0394 vs proof_v2": 0.015190476190476199 | |
| }, | |
| { | |
| "task": "codexglue/func_name", | |
| "n": 467, | |
| "acc": 0.9528907922912205, | |
| "chance": 0.25499643112062814, | |
| "nll": 0.17043053883545994, | |
| "brier": 0.08725536960997705, | |
| "heldout": false, | |
| "domain": "code", | |
| "ece": 0.023257157468183163, | |
| "proof_v2 (card)": 0.901, | |
| "\u0394 vs proof_v2": 0.05189079229122051 | |
| }, | |
| { | |
| "task": "codexglue/lang_id", | |
| "n": 504, | |
| "acc": 1.0, | |
| "chance": 0.2354166666666667, | |
| "nll": 0.0005584346231148719, | |
| "brier": 3.0963766792822474e-06, | |
| "heldout": false, | |
| "domain": "code", | |
| "ece": 0.0005574606004214999, | |
| "proof_v2 (card)": 0.997, | |
| "\u0394 vs proof_v2": 0.0030000000000000027 | |
| }, | |
| { | |
| "task": "commonsense_qa/mcq", | |
| "n": 493, | |
| "acc": 0.6389452332657201, | |
| "chance": 0.19999999999999998, | |
| "nll": 0.904400206106629, | |
| "brier": 0.47107552904904443, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.04405802657589711, | |
| "proof_v2 (card)": 0.442, | |
| "\u0394 vs proof_v2": 0.19694523326572005 | |
| }, | |
| { | |
| "task": "counsel/critique_quality", | |
| "n": 201, | |
| "acc": 0.5920398009950248, | |
| "chance": 0.3333333333333333, | |
| "nll": 1.3054839486733487, | |
| "brier": 0.6194979652829024, | |
| "heldout": false, | |
| "domain": "agentic", | |
| "ece": 0.22704330650135063, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "counsel/step_has_error", | |
| "n": 201, | |
| "acc": 0.8009950248756219, | |
| "chance": 0.5, | |
| "nll": 0.8227317674037697, | |
| "brier": 0.3473574496250388, | |
| "heldout": false, | |
| "domain": "agentic", | |
| "ece": 0.16924872979595879, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "devign/vulnerability", | |
| "n": 500, | |
| "acc": 0.596, | |
| "chance": 0.5, | |
| "nll": 0.6676419580578804, | |
| "brier": 0.4753553111644763, | |
| "heldout": false, | |
| "domain": "code", | |
| "ece": 0.05009425449371341, | |
| "proof_v2 (card)": 0.542, | |
| "\u0394 vs proof_v2": 0.05399999999999994 | |
| }, | |
| { | |
| "task": "emotion/6way", | |
| "n": 500, | |
| "acc": 0.502, | |
| "chance": 0.16666666666666663, | |
| "nll": 1.3070907976925372, | |
| "brier": 0.6468766489597455, | |
| "heldout": true, | |
| "domain": "classification", | |
| "ece": 0.08215402710437775, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "gsm8k/math", | |
| "n": 500, | |
| "acc": 0.662, | |
| "chance": 0.25, | |
| "nll": 0.7718028125888668, | |
| "brier": 0.4427847443219045, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.06086966800689696, | |
| "proof_v2 (card)": 0.275, | |
| "\u0394 vs proof_v2": 0.387 | |
| }, | |
| { | |
| "task": "hellaswag/continuation", | |
| "n": 500, | |
| "acc": 0.58, | |
| "chance": 0.25, | |
| "nll": 0.9953804229423404, | |
| "brier": 0.5371978334623486, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.07333891946077346, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "hotpotqa/comparison_yes_no", | |
| "n": 26, | |
| "acc": 0.8461538461538461, | |
| "chance": 0.5, | |
| "nll": 0.43042818936877525, | |
| "brier": 0.2727966735853732, | |
| "heldout": false, | |
| "domain": "agentic", | |
| "ece": 0.15112700829139125, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "hotpotqa/retrieve", | |
| "n": 497, | |
| "acc": 0.8732394366197183, | |
| "chance": 0.16741720162243304, | |
| "nll": 0.4343729186731075, | |
| "brier": 0.20390003436957638, | |
| "heldout": false, | |
| "domain": "agentic", | |
| "ece": 0.08316291382974779, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "humaneval/completion", | |
| "n": 119, | |
| "acc": 0.6722689075630253, | |
| "chance": 0.39355742296918783, | |
| "nll": 0.7120104984726616, | |
| "brier": 0.44585727429903876, | |
| "heldout": true, | |
| "domain": "code", | |
| "ece": 0.10659772783768277, | |
| "proof_v2 (card)": 0.575, | |
| "\u0394 vs proof_v2": 0.09726890756302531 | |
| }, | |
| { | |
| "task": "jailbreak/detect", | |
| "n": 262, | |
| "acc": 0.9541984732824428, | |
| "chance": 0.5, | |
| "nll": 0.10692274935157883, | |
| "brier": 0.06256811361157136, | |
| "heldout": false, | |
| "domain": "guardrails", | |
| "ece": 0.0232666851455019, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "massive_en/intent", | |
| "n": 500, | |
| "acc": 0.906, | |
| "chance": 0.12958727789085603, | |
| "nll": 0.26250843075065133, | |
| "brier": 0.12679194471370903, | |
| "heldout": false, | |
| "domain": "intents", | |
| "ece": 0.022001883208751682, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "mbpp/bugspot", | |
| "n": 256, | |
| "acc": 0.73046875, | |
| "chance": 0.40559895833333326, | |
| "nll": 0.5201096873639273, | |
| "brier": 0.3294220878204956, | |
| "heldout": false, | |
| "domain": "code", | |
| "ece": 0.1015084749087691, | |
| "proof_v2 (card)": 0.475, | |
| "\u0394 vs proof_v2": 0.25546875 | |
| }, | |
| { | |
| "task": "mbpp/solution", | |
| "n": 500, | |
| "acc": 0.982, | |
| "chance": 0.25, | |
| "nll": 0.06299449819558049, | |
| "brier": 0.03283427225700136, | |
| "heldout": false, | |
| "domain": "code", | |
| "ece": 0.014863664209842696, | |
| "proof_v2 (card)": 0.992, | |
| "\u0394 vs proof_v2": -0.010000000000000009 | |
| }, | |
| { | |
| "task": "mmlu/mcq", | |
| "n": 500, | |
| "acc": 0.356, | |
| "chance": 0.25, | |
| "nll": 1.370702707901597, | |
| "brier": 0.738678903692766, | |
| "heldout": true, | |
| "domain": "reasoning", | |
| "ece": 0.10921649205684662, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "mnli/claim", | |
| "n": 500, | |
| "acc": 0.838, | |
| "chance": 0.33333333333333326, | |
| "nll": 0.41442326370440424, | |
| "brier": 0.23035331540149503, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.07134601444005968, | |
| "proof_v2 (card)": 0.492, | |
| "\u0394 vs proof_v2": 0.346 | |
| }, | |
| { | |
| "task": "openbookqa/mcq", | |
| "n": 500, | |
| "acc": 0.61, | |
| "chance": 0.25, | |
| "nll": 1.024268753977958, | |
| "brier": 0.5355113333378367, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.09224175179004669, | |
| "proof_v2 (card)": 0.292, | |
| "\u0394 vs proof_v2": 0.318 | |
| }, | |
| { | |
| "task": "policy/access_control_transfer", | |
| "n": 500, | |
| "acc": 1.0, | |
| "chance": 0.33333333333333326, | |
| "nll": 0.0002645651292309594, | |
| "brier": 1.1635768837216181e-06, | |
| "heldout": false, | |
| "domain": "policy", | |
| "ece": 0.00026422095298772597, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "policy/count_threshold_transfer", | |
| "n": 500, | |
| "acc": 0.846, | |
| "chance": 0.17848809523809525, | |
| "nll": 0.4453474920627887, | |
| "brier": 0.239936219050859, | |
| "heldout": false, | |
| "domain": "policy", | |
| "ece": 0.10926523756980895, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "policy/free_shipping_transfer", | |
| "n": 500, | |
| "acc": 0.766, | |
| "chance": 0.5, | |
| "nll": 0.39174148760084065, | |
| "brier": 0.2747179012451008, | |
| "heldout": false, | |
| "domain": "policy", | |
| "ece": 0.10137953925132755, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "policy/invoice_overdue_transfer", | |
| "n": 500, | |
| "acc": 0.6, | |
| "chance": 0.5, | |
| "nll": 0.8301094107693061, | |
| "brier": 0.5458620193712137, | |
| "heldout": false, | |
| "domain": "policy", | |
| "ece": 0.20100946557521818, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "policy/invoice_total_transfer", | |
| "n": 500, | |
| "acc": 0.486, | |
| "chance": 0.5, | |
| "nll": 0.69399339812994, | |
| "brier": 0.5008197939882997, | |
| "heldout": false, | |
| "domain": "policy", | |
| "ece": 0.03210310280323031, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "policy/refund_approval_transfer", | |
| "n": 500, | |
| "acc": 0.78, | |
| "chance": 0.33333333333333326, | |
| "nll": 0.8478962361293161, | |
| "brier": 0.38071118861218445, | |
| "heldout": false, | |
| "domain": "policy", | |
| "ece": 0.18587953794002532, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "policy/return_window_transfer", | |
| "n": 500, | |
| "acc": 0.998, | |
| "chance": 0.33333333333333326, | |
| "nll": 0.02459742584443666, | |
| "brier": 0.005749234483560448, | |
| "heldout": false, | |
| "domain": "policy", | |
| "ece": 0.019777906179428068, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "policy/sla_urgency_transfer", | |
| "n": 500, | |
| "acc": 0.698, | |
| "chance": 0.25, | |
| "nll": 0.9843494065143168, | |
| "brier": 0.49182705060356263, | |
| "heldout": false, | |
| "domain": "policy", | |
| "ece": 0.21234694278240196, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "policy/table_extreme_transfer", | |
| "n": 500, | |
| "acc": 0.912, | |
| "chance": 0.23716666666666666, | |
| "nll": 0.22187845180515342, | |
| "brier": 0.12271458379265106, | |
| "heldout": false, | |
| "domain": "policy", | |
| "ece": 0.013898970842361484, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "prompt_injections/detect", | |
| "n": 116, | |
| "acc": 0.5775862068965517, | |
| "chance": 0.5, | |
| "nll": 1.0671347150446622, | |
| "brier": 0.628204618498277, | |
| "heldout": true, | |
| "domain": "guardrails", | |
| "ece": 0.2859799378904803, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "qasc/mcq", | |
| "n": 500, | |
| "acc": 0.99, | |
| "chance": 0.125, | |
| "nll": 0.0315895584258742, | |
| "brier": 0.01700795004425532, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.006339998722076429, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "sciq/mcq", | |
| "n": 498, | |
| "acc": 0.9538152610441767, | |
| "chance": 0.25, | |
| "nll": 0.11897318925779031, | |
| "brier": 0.06399753046801272, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.03059945360244999, | |
| "proof_v2 (card)": 0.692, | |
| "\u0394 vs proof_v2": 0.26181526104417674 | |
| }, | |
| { | |
| "task": "scitail/support", | |
| "n": 500, | |
| "acc": 0.964, | |
| "chance": 0.5, | |
| "nll": 0.10391226119734347, | |
| "brier": 0.05616228693429717, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.020327572941780014, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "snli/contradicts", | |
| "n": 500, | |
| "acc": 0.986, | |
| "chance": 0.33333333333333326, | |
| "nll": 0.048629490838502536, | |
| "brier": 0.024212671137030826, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.01413622707128527, | |
| "proof_v2 (card)": 0.892, | |
| "\u0394 vs proof_v2": 0.09399999999999997 | |
| }, | |
| { | |
| "task": "snli/must_be_true", | |
| "n": 500, | |
| "acc": 0.98, | |
| "chance": 0.33333333333333326, | |
| "nll": 0.060363937412155795, | |
| "brier": 0.02947848703942969, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.02905823123455048, | |
| "proof_v2 (card)": 0.908, | |
| "\u0394 vs proof_v2": 0.07199999999999995 | |
| }, | |
| { | |
| "task": "snli/nli", | |
| "n": 988, | |
| "acc": 0.8795546558704453, | |
| "chance": 0.33333333333333326, | |
| "nll": 0.3359528171680472, | |
| "brier": 0.18385021007121183, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.04961462647688053, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "sst5/score", | |
| "n": 500, | |
| "acc": 0.416, | |
| "chance": 0.2, | |
| "nll": 1.246531030535698, | |
| "brier": 0.6734807272262489, | |
| "heldout": true, | |
| "domain": "classification", | |
| "ece": 0.04567561271786689, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "typed_decisions/agent_trace_observability", | |
| "n": 500, | |
| "acc": 0.712, | |
| "chance": 0.3, | |
| "nll": 0.7305186161920428, | |
| "brier": 0.42510022971515116, | |
| "heldout": false, | |
| "domain": "workflows", | |
| "ece": 0.11957757359743115, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "typed_decisions/customer_service", | |
| "n": 500, | |
| "acc": 0.74, | |
| "chance": 0.28, | |
| "nll": 0.6714302012249828, | |
| "brier": 0.3723593006177565, | |
| "heldout": false, | |
| "domain": "workflows", | |
| "ece": 0.08548467075824738, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "typed_decisions/invoice_processing", | |
| "n": 500, | |
| "acc": 0.838, | |
| "chance": 0.35, | |
| "nll": 0.5010932807754725, | |
| "brier": 0.26905854083557984, | |
| "heldout": false, | |
| "domain": "workflows", | |
| "ece": 0.12220623338222505, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "typed_decisions/security_incidents", | |
| "n": 500, | |
| "acc": 0.73, | |
| "chance": 0.34, | |
| "nll": 0.6859722163826227, | |
| "brier": 0.3929527039537006, | |
| "heldout": false, | |
| "domain": "workflows", | |
| "ece": 0.143579844892025, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "winogrande/blank", | |
| "n": 500, | |
| "acc": 0.612, | |
| "chance": 0.5, | |
| "nll": 0.7296091562472283, | |
| "brier": 0.508893263835874, | |
| "heldout": false, | |
| "domain": "reasoning", | |
| "ece": 0.11118709647655486, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| }, | |
| { | |
| "task": "yelp/score", | |
| "n": 500, | |
| "acc": 0.654, | |
| "chance": 0.2, | |
| "nll": 0.7885563414096832, | |
| "brier": 0.4562026964016218, | |
| "heldout": false, | |
| "domain": "classification", | |
| "ece": 0.06915212672948838, | |
| "proof_v2 (card)": NaN, | |
| "\u0394 vs proof_v2": NaN | |
| } | |
| ], | |
| "head_to_head": null, | |
| "env": { | |
| "torch": "2.11.0+cu128", | |
| "transformers": "5.17.0", | |
| "python": "3.14.4", | |
| "gpu": "NVIDIA GeForce RTX 5090 Laptop GPU" | |
| } | |
| } |