{
  "version": "1.0.0",
  "notebook_version": "V3.7",
  "mode": "scratch",
  "parent_version": null,
  "lineage": [
    "jhu-clsp/ettin-encoder-150m"
  ],
  "config": {
    "MODE": "scratch",
    "FINETUNE_FROM": "latest",
    "MODEL_NAME": "LightDec_V2_Long",
    "BACKBONE": "jhu-clsp/ettin-encoder-150m",
    "BACKBONE_INIT": "pretrained",
    "PRESET": "long",
    "CUSTOM_DATA_JSONL": "",
    "CUSTOM_WEIGHT": 3.0,
    "INCLUDE_BUILDERS": [],
    "EXCLUDE_BUILDERS": [
      "mind2web"
    ],
    "TEV1_DIR": "./tev1",
    "TEV1_AUTOBUILD": true,
    "TEV1_POLICY_CAP": 13500,
    "TEV1_ROUTING_CAP": 6000,
    "TEV1_RESEARCH_CAP": 1000,
    "TRIAGE_EMAILS": 200,
    "TEV1_RETRIES": 3,
    "TEV1_STEP_TIMEOUT_MIN": 180,
    "TEV1_STALL_MIN": 15,
    "RETRY_FAILED_BUILDERS": true,
    "LONG_TASK_TRAIN": 1500,
    "MAX_LEN": 2048,
    "LONG_MAX_LEN": 2048,
    "LONG_OPTS_THRESHOLD": 24,
    "HEAD_MAX_LEN": 192,
    "MAX_TOK_PER_OPT": 24,
    "MAX_OPTS_SINGLE_PASS": 96,
    "INTERACT_LAYERS": 2,
    "INTERACT_HEADS": 8,
    "BATCH_SIZE": 128,
    "TOKENS_PER_BATCH": 65536,
    "GRAD_ACCUM": 1,
    "LR_ENCODER": 8e-05,
    "LR_HEAD": 0.0006,
    "LLRD": 0.9,
    "WEIGHT_DECAY": 0.01,
    "WARMUP_FRAC": 0.06,
    "EPOCHS": 0,
    "EMA_DECAY": 0.999,
    "GRAD_CLIP": 1.0,
    "FINETUNE_LR_SCALE": 0.5,
    "SPHERICAL_W": 0.5,
    "BRIER_W": 0.0,
    "RPS_W": 1.0,
    "TASK_SAMPLING_ALPHA": 0.5,
    "NOTA_PROB": 0.08,
    "RLCD_EPOCHS": 0.0,
    "RLCD_SAMPLES": 4,
    "RLCD_SIGMA": 0.3,
    "RLCD_CE_ANCHOR": 0.5,
    "DEVICE": "cuda",
    "SEED": 42,
    "NUM_WORKERS": 8,
    "USE_COMPILE": true,
    "ATTN_IMPL": "auto",
    "OUTPUT_ROOT": "./falcondec_runs",
    "EXPORT_INT8": true,
    "SIZE_TARGET_MB": 250,
    "DEFER_THRESHOLD": 0.7,
    "EVAL_BASELINE_PROOF_V2": true,
    "BASELINE_MAX_PER_TASK": 120,
    "RUN_CPU_BENCH": true,
    "PUSH_TO_HUB": false,
    "HUB_REPO": ""
  },
  "epochs": 10,
  "best_epoch": 4,
  "rlcd_kept": false,
  "history": [
    {
      "epoch": 1,
      "train_loss": 0.7058714814313827,
      "train_acc": 0.7342094450703411,
      "val_macro": 0.8188568656508199,
      "val_micro": 0.820702479338843,
      "val_nll": 0.4076740937270036
    },
    {
      "epoch": 2,
      "train_loss": 0.431469622218984,
      "train_acc": 0.8473141073247583,
      "val_macro": 0.8450872188411825,
      "val_micro": 0.8444628099173553,
      "val_nll": 0.3623910443100994
    },
    {
      "epoch": 3,
      "train_loss": 0.34276796075090776,
      "train_acc": 0.8791940521680108,
      "val_macro": 0.8501904675940947,
      "val_micro": 0.8499173553719008,
      "val_nll": 0.39227130460368825
    },
    {
      "epoch": 4,
      "train_loss": 0.2773282952523419,
      "train_acc": 0.9039863196679513,
      "val_macro": 0.8519172452047367,
      "val_micro": 0.8519834710743802,
      "val_nll": 0.4529550437184426
    },
    {
      "epoch": 5,
      "train_loss": 0.22030453415052323,
      "train_acc": 0.925851485588573,
      "val_macro": 0.8488092001804082,
      "val_micro": 0.85,
      "val_nll": 0.5369876145373608
    },
    {
      "epoch": 6,
      "train_loss": 0.16953414903032543,
      "train_acc": 0.9449156076226879,
      "val_macro": 0.8474279008050457,
      "val_micro": 0.8488842975206612,
      "val_nll": 0.6705887193589142
    },
    {
      "epoch": 7,
      "train_loss": 0.126976966611751,
      "train_acc": 0.9602833527848939,
      "val_macro": 0.8453322694829587,
      "val_micro": 0.8476446280991735,
      "val_nll": 0.8430423012207805
    },
    {
      "epoch": 8,
      "train_loss": 0.09470793687766035,
      "train_acc": 0.9714013975720073,
      "val_macro": 0.8451769009269958,
      "val_micro": 0.8467355371900827,
      "val_nll": 1.0336680755654943
    },
    {
      "epoch": 9,
      "train_loss": 0.0710737590306488,
      "train_acc": 0.9794225926173407,
      "val_macro": 0.8435159403610613,
      "val_micro": 0.8458677685950413,
      "val_nll": 1.210751206553627
    },
    {
      "epoch": 10,
      "train_loss": 0.05665995772408678,
      "train_acc": 0.9843832093140154,
      "val_macro": 0.844272704258198,
      "val_micro": 0.8474380165289256,
      "val_nll": 1.41872131482475
    }
  ],
  "train_minutes": 291.67019440333047,
  "temperatures": [
    [
      2.37591814994812,
      2.0426506996154785,
      1.6455482244491577,
      1.8417285680770874
    ],
    [
      2.4885058403015137,
      2.4885058403015137,
      2.4885058403015137,
      2.4885058403015137
    ],
    [
      2.2925639152526855,
      2.2925639152526855,
      2.2925639152526855,
      2.2925639152526855
    ]
  ],
  "data_counts": {
    "train": 1023814,
    "validation": 24200,
    "test": 31990
  },
  "skipped_builders": [],
  "test_overall": {
    "n": 31990.0,
    "micro_acc": 0.784057517974367,
    "macro_acc": 0.7887006006875437,
    "nll": 0.5630830966790886,
    "brier": 0.2954194093899411,
    "ece": 0.02856345933167103,
    "aurc": 0.06778106632931055,
    "score_mae": 0.5060575584353655,
    "coverage@0.7": 0.6797436698968428,
    "acc_on_covered@0.7": 0.9059094044607956
  },
  "test_per_domain": {
    "agentic": 0.821117397177404,
    "classification": 0.6115,
    "code": 0.9108344674425007,
    "guardrails": 0.7984073149310548,
    "intents": 0.8505,
    "long_context": 1.0,
    "policy": 0.8061818181818182,
    "reasoning": 0.7180877154615182,
    "support": 0.9815181518151815,
    "tev1_benchmark": 0.6541666666666667,
    "workflows": 0.773
  },
  "test_per_task": [
    {
      "task": "ag_news/topic",
      "n": 500,
      "acc": 0.9,
      "chance": 0.25,
      "nll": 0.2674782864307199,
      "brier": 0.14296993909824685,
      "heldout": false,
      "domain": "classification",
      "ece": 0.05165432274341582,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "agentharm/refuse",
      "n": 416,
      "acc": 0.6995192307692307,
      "chance": 0.5,
      "nll": 0.6008703433729422,
      "brier": 0.4076762512706606,
      "heldout": true,
      "domain": "guardrails",
      "ece": 0.08916855761064935,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "agenttrek/finish_now",
      "n": 151,
      "acc": 0.7880794701986755,
      "chance": 0.5,
      "nll": 0.445978463758558,
      "brier": 0.30066953632150406,
      "heldout": false,
      "domain": "agentic",
      "ece": 0.07512128787324918,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "agenttrek/next_action_type",
      "n": 487,
      "acc": 0.864476386036961,
      "chance": 0.19054952576513148,
      "nll": 0.3929848723152591,
      "brier": 0.2184262054212701,
      "heldout": false,
      "domain": "agentic",
      "ece": 0.09202881025827395,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "anli/nli",
      "n": 498,
      "acc": 0.4859437751004016,
      "chance": 0.33333333333333326,
      "nll": 1.069221544292677,
      "brier": 0.6544603492386665,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.14589204983299517,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "aqua_rat/math",
      "n": 247,
      "acc": 0.340080971659919,
      "chance": 0.2,
      "nll": 1.54525283087603,
      "brier": 0.7739737508966391,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.080350092668765,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "arc_challenge/mcq",
      "n": 500,
      "acc": 0.49,
      "chance": 0.25006666666666666,
      "nll": 1.3021090898208785,
      "brier": 0.6752390805252713,
      "heldout": true,
      "domain": "reasoning",
      "ece": 0.1680490539073944,
      "proof_v2 (card)": 0.308,
      "\u0394 vs proof_v2": 0.182
    },
    {
      "task": "arc_easy/mcq",
      "n": 500,
      "acc": 0.612,
      "chance": 0.24980000000000002,
      "nll": 0.9830388274090365,
      "brier": 0.5264415342806388,
      "heldout": true,
      "domain": "reasoning",
      "ece": 0.1215095258355141,
      "proof_v2 (card)": 0.425,
      "\u0394 vs proof_v2": 0.187
    },
    {
      "task": "banking77/intent",
      "n": 500,
      "acc": 0.922,
      "chance": 0.3214666666666666,
      "nll": 0.23128961177584006,
      "brier": 0.11756106243365176,
      "heldout": true,
      "domain": "intents",
      "ece": 0.037492330908775316,
      "proof_v2 (card)": 0.883,
      "\u0394 vs proof_v2": 0.039000000000000035
    },
    {
      "task": "banking77/intent_77",
      "n": 300,
      "acc": 0.58,
      "chance": 0.012987012987012986,
      "nll": 2.159930980304489,
      "brier": 0.6431857202493357,
      "heldout": true,
      "domain": "intents",
      "ece": 0.24079640249411272,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "bigclonebench/clone",
      "n": 500,
      "acc": 0.962,
      "chance": 0.5,
      "nll": 0.1312969167594565,
      "brier": 0.06444151702850223,
      "heldout": false,
      "domain": "code",
      "ece": 0.03135248208045959,
      "proof_v2 (card)": 0.383,
      "\u0394 vs proof_v2": 0.579
    },
    {
      "task": "bitext/category",
      "n": 500,
      "acc": 1.0,
      "chance": 0.16128124098124094,
      "nll": 0.0013673773490934594,
      "brier": 0.0003828941821233322,
      "heldout": false,
      "domain": "support",
      "ece": 0.001247308015823329,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "bitext/route",
      "n": 500,
      "acc": 1.0,
      "chance": 0.19016313131313134,
      "nll": 0.00031508885690954,
      "brier": 4.903548307607296e-06,
      "heldout": false,
      "domain": "support",
      "ece": 0.0003137041330337764,
      "proof_v2 (card)": 0.958,
      "\u0394 vs proof_v2": 0.04200000000000004
    },
    {
      "task": "boolq/yes_no",
      "n": 500,
      "acc": 0.822,
      "chance": 0.5,
      "nll": 0.4842173567947466,
      "brier": 0.2840138305161832,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.0795972980260849,
      "proof_v2 (card)": 0.717,
      "\u0394 vs proof_v2": 0.10499999999999998
    },
    {
      "task": "civil_comments/toxic",
      "n": 500,
      "acc": 0.926,
      "chance": 0.5,
      "nll": 0.2234244500696659,
      "brier": 0.1209681824839501,
      "heldout": false,
      "domain": "guardrails",
      "ece": 0.05123518884181974,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "clinc150/intent",
      "n": 500,
      "acc": 0.97,
      "chance": 0.12404474984829318,
      "nll": 0.09906445816022005,
      "brier": 0.04996487012600131,
      "heldout": false,
      "domain": "intents",
      "ece": 0.014944501101970714,
      "proof_v2 (card)": 0.85,
      "\u0394 vs proof_v2": 0.12
    },
    {
      "task": "codexglue/code_to_doc",
      "n": 504,
      "acc": 0.9900793650793651,
      "chance": 0.26233465608465617,
      "nll": 0.032474144255576765,
      "brier": 0.016158895201785088,
      "heldout": false,
      "domain": "code",
      "ece": 0.009654374349685009,
      "proof_v2 (card)": 0.969,
      "\u0394 vs proof_v2": 0.021079365079365142
    },
    {
      "task": "codexglue/doc_to_code",
      "n": 504,
      "acc": 0.9761904761904762,
      "chance": 0.2786044973544973,
      "nll": 0.05970113779317181,
      "brier": 0.03072688989396561,
      "heldout": false,
      "domain": "code",
      "ece": 0.012547755761752008,
      "proof_v2 (card)": 0.961,
      "\u0394 vs proof_v2": 0.015190476190476199
    },
    {
      "task": "codexglue/func_name",
      "n": 467,
      "acc": 0.9550321199143469,
      "chance": 0.25499643112062814,
      "nll": 0.14961532151979784,
      "brier": 0.07053876912275295,
      "heldout": false,
      "domain": "code",
      "ece": 0.03948740715388316,
      "proof_v2 (card)": 0.901,
      "\u0394 vs proof_v2": 0.054032119914346866
    },
    {
      "task": "codexglue/lang_id",
      "n": 504,
      "acc": 1.0,
      "chance": 0.2354166666666667,
      "nll": 0.0014352430686462487,
      "brier": 1.2864369183890018e-05,
      "heldout": false,
      "domain": "code",
      "ece": 0.0014309174721203188,
      "proof_v2 (card)": 0.997,
      "\u0394 vs proof_v2": 0.0030000000000000027
    },
    {
      "task": "commonsense_qa/mcq",
      "n": 493,
      "acc": 0.6430020283975659,
      "chance": 0.19999999999999998,
      "nll": 0.9523430099929484,
      "brier": 0.49435087368161623,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.14308575696442233,
      "proof_v2 (card)": 0.442,
      "\u0394 vs proof_v2": 0.20100202839756592
    },
    {
      "task": "counsel/critique_quality",
      "n": 201,
      "acc": 0.6268656716417911,
      "chance": 0.3333333333333333,
      "nll": 1.5466346353110312,
      "brier": 0.6427543071457474,
      "heldout": false,
      "domain": "agentic",
      "ece": 0.27062572442477023,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "counsel/step_has_error",
      "n": 201,
      "acc": 0.8308457711442786,
      "chance": 0.5,
      "nll": 0.8236887436017318,
      "brier": 0.3201374357369142,
      "heldout": false,
      "domain": "agentic",
      "ece": 0.14827988011326954,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "devign/vulnerability",
      "n": 500,
      "acc": 0.632,
      "chance": 0.5,
      "nll": 0.6074751503933221,
      "brier": 0.42706214521965163,
      "heldout": false,
      "domain": "code",
      "ece": 0.07018342161178594,
      "proof_v2 (card)": 0.542,
      "\u0394 vs proof_v2": 0.08999999999999997
    },
    {
      "task": "emotion/6way",
      "n": 500,
      "acc": 0.472,
      "chance": 0.16666666666666663,
      "nll": 1.561316128242761,
      "brier": 0.7356062387487318,
      "heldout": true,
      "domain": "classification",
      "ece": 0.21054179659485817,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "gsm8k/math",
      "n": 500,
      "acc": 0.7,
      "chance": 0.25,
      "nll": 0.6923719964642078,
      "brier": 0.4014884319624292,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.08541783547401426,
      "proof_v2 (card)": 0.275,
      "\u0394 vs proof_v2": 0.42499999999999993
    },
    {
      "task": "hellaswag/continuation",
      "n": 500,
      "acc": 0.576,
      "chance": 0.25,
      "nll": 0.968790470642969,
      "brier": 0.5288133586177948,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.07543198531866073,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "hotpotqa/comparison_yes_no",
      "n": 26,
      "acc": 0.9230769230769231,
      "chance": 0.5,
      "nll": 0.3400710394176153,
      "brier": 0.15484679762361436,
      "heldout": false,
      "domain": "agentic",
      "ece": 0.08232885369887719,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "hotpotqa/retrieve",
      "n": 497,
      "acc": 0.8933601609657947,
      "chance": 0.16741720162243304,
      "nll": 0.3457089332123268,
      "brier": 0.16216258689991922,
      "heldout": false,
      "domain": "agentic",
      "ece": 0.04268557528854612,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "humaneval/completion",
      "n": 119,
      "acc": 0.8487394957983193,
      "chance": 0.39355742296918783,
      "nll": 0.46361927195851294,
      "brier": 0.2730045795813753,
      "heldout": true,
      "domain": "code",
      "ece": 0.13989568708323633,
      "proof_v2 (card)": 0.575,
      "\u0394 vs proof_v2": 0.27373949579831935
    },
    {
      "task": "jailbreak/detect",
      "n": 262,
      "acc": 0.9732824427480916,
      "chance": 0.5,
      "nll": 0.08469655378148465,
      "brier": 0.044003931208438284,
      "heldout": false,
      "domain": "guardrails",
      "ece": 0.030764950368240628,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "long/contract_clause",
      "n": 150,
      "acc": 1.0,
      "chance": 0.19999999999999996,
      "nll": 0.0002789537957808837,
      "brier": 4.274148862877054e-07,
      "heldout": false,
      "domain": "long_context",
      "ece": 0.00027882695198055973,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "long/email_thread",
      "n": 150,
      "acc": 1.0,
      "chance": 0.25,
      "nll": 0.0003266528345905802,
      "brier": 2.717841913247229e-06,
      "heldout": false,
      "domain": "long_context",
      "ece": 0.0003259424368540209,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "long/service_log",
      "n": 150,
      "acc": 1.0,
      "chance": 0.19999999999999996,
      "nll": 0.0010035930563268873,
      "brier": 2.0987077842670875e-06,
      "heldout": false,
      "domain": "long_context",
      "ece": 0.0010028688112894146,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "massive_en/intent",
      "n": 500,
      "acc": 0.93,
      "chance": 0.12958727789085603,
      "nll": 0.20636169074449573,
      "brier": 0.10375277943487987,
      "heldout": false,
      "domain": "intents",
      "ece": 0.0284088468849659,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "mbpp/bugspot",
      "n": 256,
      "acc": 0.85546875,
      "chance": 0.40559895833333326,
      "nll": 0.3667814989712497,
      "brier": 0.21843909652329055,
      "heldout": false,
      "domain": "code",
      "ece": 0.037241218378767364,
      "proof_v2 (card)": 0.475,
      "\u0394 vs proof_v2": 0.38046875
    },
    {
      "task": "mbpp/solution",
      "n": 500,
      "acc": 0.978,
      "chance": 0.25,
      "nll": 0.05863042902501184,
      "brier": 0.02981698831494921,
      "heldout": false,
      "domain": "code",
      "ece": 0.01960155099630357,
      "proof_v2 (card)": 0.992,
      "\u0394 vs proof_v2": -0.014000000000000012
    },
    {
      "task": "mmlu/mcq",
      "n": 500,
      "acc": 0.39,
      "chance": 0.25,
      "nll": 1.4543649319559335,
      "brier": 0.7702575193705765,
      "heldout": true,
      "domain": "reasoning",
      "ece": 0.15507307499647138,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "mnli/claim",
      "n": 500,
      "acc": 0.86,
      "chance": 0.33333333333333326,
      "nll": 0.4006310784481466,
      "brier": 0.2131232579488494,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.081218329668045,
      "proof_v2 (card)": 0.492,
      "\u0394 vs proof_v2": 0.368
    },
    {
      "task": "openbookqa/mcq",
      "n": 500,
      "acc": 0.572,
      "chance": 0.25,
      "nll": 1.1920220527790952,
      "brier": 0.6155284183368397,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.21775564682483672,
      "proof_v2 (card)": 0.292,
      "\u0394 vs proof_v2": 0.27999999999999997
    },
    {
      "task": "policy/access_control_transfer",
      "n": 500,
      "acc": 1.0,
      "chance": 0.33333333333333326,
      "nll": 0.002006467206090747,
      "brier": 2.4728843418299015e-05,
      "heldout": false,
      "domain": "policy",
      "ece": 0.001999327063560541,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/count_threshold_transfer",
      "n": 500,
      "acc": 0.88,
      "chance": 0.10537052392052391,
      "nll": 0.319376940273738,
      "brier": 0.1809248790341789,
      "heldout": false,
      "domain": "policy",
      "ece": 0.04758520478010175,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/free_shipping_transfer",
      "n": 500,
      "acc": 0.938,
      "chance": 0.5,
      "nll": 0.1287872482436942,
      "brier": 0.08322479293591872,
      "heldout": false,
      "domain": "policy",
      "ece": 0.041308605194091734,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/invoice_overdue_transfer",
      "n": 500,
      "acc": 0.824,
      "chance": 0.5,
      "nll": 0.3965394942490384,
      "brier": 0.2504247577332596,
      "heldout": false,
      "domain": "policy",
      "ece": 0.01985593855381013,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/invoice_total_transfer",
      "n": 500,
      "acc": 0.474,
      "chance": 0.5,
      "nll": 0.710681935429573,
      "brier": 0.5164436840846652,
      "heldout": false,
      "domain": "policy",
      "ece": 0.08196303224563599,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/refund_approval_transfer",
      "n": 500,
      "acc": 0.962,
      "chance": 0.33333333333333326,
      "nll": 0.10006511305032836,
      "brier": 0.057969574647252664,
      "heldout": false,
      "domain": "policy",
      "ece": 0.018834196925163266,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/return_window_transfer",
      "n": 500,
      "acc": 1.0,
      "chance": 0.33333333333333326,
      "nll": 0.02710788446944207,
      "brier": 0.004185976644178241,
      "heldout": false,
      "domain": "policy",
      "ece": 0.02567652928829193,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/sla_urgency_transfer",
      "n": 500,
      "acc": 0.606,
      "chance": 0.25,
      "nll": 0.8170870101451874,
      "brier": 0.5123369210022903,
      "heldout": false,
      "domain": "policy",
      "ece": 0.13068743020296097,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/table_compare_transfer",
      "n": 500,
      "acc": 0.928,
      "chance": 0.5,
      "nll": 0.2105378701629961,
      "brier": 0.11898007306418568,
      "heldout": false,
      "domain": "policy",
      "ece": 0.04929606854915622,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/table_count_transfer",
      "n": 500,
      "acc": 0.324,
      "chance": 0.12114228549228548,
      "nll": 3.24442003638536,
      "brier": 1.1488026091975905,
      "heldout": false,
      "domain": "policy",
      "ece": 0.5407210965156555,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/table_extreme_transfer",
      "n": 500,
      "acc": 0.932,
      "chance": 0.12210815295815294,
      "nll": 0.268062693382595,
      "brier": 0.11322196827103552,
      "heldout": false,
      "domain": "policy",
      "ece": 0.05259446609020234,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "prompt_injections/detect",
      "n": 116,
      "acc": 0.5948275862068966,
      "chance": 0.5,
      "nll": 1.1838965486195179,
      "brier": 0.6826645118148511,
      "heldout": true,
      "domain": "guardrails",
      "ece": 0.33272201953263114,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "qasc/mcq",
      "n": 500,
      "acc": 0.986,
      "chance": 0.125,
      "nll": 0.028506858559036514,
      "brier": 0.017274368003708154,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.007774531126022318,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "sciq/mcq",
      "n": 498,
      "acc": 0.9538152610441767,
      "chance": 0.25,
      "nll": 0.1506263591345834,
      "brier": 0.06985463059035983,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.020590302994452258,
      "proof_v2 (card)": 0.692,
      "\u0394 vs proof_v2": 0.26181526104417674
    },
    {
      "task": "scitail/support",
      "n": 500,
      "acc": 0.962,
      "chance": 0.5,
      "nll": 0.11612921302905306,
      "brier": 0.05990850379879065,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.03015338957309728,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "snli/contradicts",
      "n": 500,
      "acc": 0.99,
      "chance": 0.33333333333333326,
      "nll": 0.042843449617153966,
      "brier": 0.017027529190616314,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.018511961221695038,
      "proof_v2 (card)": 0.892,
      "\u0394 vs proof_v2": 0.09799999999999998
    },
    {
      "task": "snli/must_be_true",
      "n": 500,
      "acc": 0.986,
      "chance": 0.33333333333333326,
      "nll": 0.06753946821717545,
      "brier": 0.02560326154020915,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.043202445626258815,
      "proof_v2 (card)": 0.908,
      "\u0394 vs proof_v2": 0.07799999999999996
    },
    {
      "task": "snli/nli",
      "n": 988,
      "acc": 0.8947368421052632,
      "chance": 0.33333333333333326,
      "nll": 0.3554420507991845,
      "brier": 0.17963376213248355,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.10791712172842222,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "sst5/score",
      "n": 500,
      "acc": 0.406,
      "chance": 0.2,
      "nll": 1.3002369542717933,
      "brier": 0.6888741553408958,
      "heldout": true,
      "domain": "classification",
      "ece": 0.068851743131876,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "tev1_test/ag_news",
      "n": 150,
      "acc": 0.92,
      "chance": 0.25,
      "nll": 0.2815313008365532,
      "brier": 0.13456139206846243,
      "heldout": true,
      "domain": "tev1_benchmark",
      "ece": 0.09770219445228576,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "tev1_test/banking77",
      "n": 200,
      "acc": 0.72,
      "chance": 0.17791666666666664,
      "nll": 0.777236735031438,
      "brier": 0.3902663054271286,
      "heldout": true,
      "domain": "tev1_benchmark",
      "ece": 0.13721376925706863,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "tev1_test/boolq",
      "n": 200,
      "acc": 0.835,
      "chance": 0.5,
      "nll": 0.40902234488283284,
      "brier": 0.24753055362669119,
      "heldout": true,
      "domain": "tev1_benchmark",
      "ece": 0.04955918818712238,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "tev1_test/mnli",
      "n": 300,
      "acc": 0.7233333333333334,
      "chance": 0.3333333333333333,
      "nll": 0.6736717029288412,
      "brier": 0.39782655881245704,
      "heldout": true,
      "domain": "tev1_benchmark",
      "ece": 0.07533363938331603,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "tev1_test/policy",
      "n": 1200,
      "acc": 0.5291666666666667,
      "chance": 0.3333333333333333,
      "nll": 0.8761954319352905,
      "brier": 0.5462664224307302,
      "heldout": true,
      "domain": "tev1_benchmark",
      "ece": 0.05868758827447889,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "tev1_test/routing",
      "n": 600,
      "acc": 0.4583333333333333,
      "chance": 0.2,
      "nll": 1.120521725914441,
      "brier": 0.5789269511485964,
      "heldout": true,
      "domain": "tev1_benchmark",
      "ece": 0.037736288358767814,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "tev1_test/sst5",
      "n": 150,
      "acc": 0.3933333333333333,
      "chance": 0.19999999999999996,
      "nll": 1.294863009850184,
      "brier": 0.6810328679494898,
      "heldout": true,
      "domain": "tev1_benchmark",
      "ece": 0.1081918982664744,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "triage/support_email",
      "n": 505,
      "acc": 0.9445544554455445,
      "chance": 0.32333333333333325,
      "nll": 0.15705669974757994,
      "brier": 0.07848840114983771,
      "heldout": false,
      "domain": "support",
      "ece": 0.045869263741049465,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "typed_decisions/agent_trace_observability",
      "n": 500,
      "acc": 0.734,
      "chance": 0.3,
      "nll": 0.8089503426551818,
      "brier": 0.45125534421065666,
      "heldout": false,
      "domain": "workflows",
      "ece": 0.22166652178764346,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "typed_decisions/customer_service",
      "n": 500,
      "acc": 0.764,
      "chance": 0.28,
      "nll": 0.7428050636351109,
      "brier": 0.40177602517263195,
      "heldout": false,
      "domain": "workflows",
      "ece": 0.20972245779633528,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "typed_decisions/invoice_processing",
      "n": 500,
      "acc": 0.826,
      "chance": 0.35,
      "nll": 0.5732684296518564,
      "brier": 0.2997603036136731,
      "heldout": false,
      "domain": "workflows",
      "ece": 0.19463082921504973,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "typed_decisions/security_incidents",
      "n": 500,
      "acc": 0.768,
      "chance": 0.34,
      "nll": 0.7502468670606613,
      "brier": 0.4238190137148829,
      "heldout": false,
      "domain": "workflows",
      "ece": 0.23508695220947262,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "winogrande/blank",
      "n": 500,
      "acc": 0.662,
      "chance": 0.5,
      "nll": 0.6651458515003323,
      "brier": 0.45323706188140617,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.1159557296037674,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "yelp/score",
      "n": 500,
      "acc": 0.668,
      "chance": 0.2,
      "nll": 0.7834262755662202,
      "brier": 0.4423740240858843,
      "heldout": false,
      "domain": "classification",
      "ece": 0.09068749487400055,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    }
  ],
  "head_to_head": [
    {
      "task": "ag_news/topic",
      "n": 120,
      "LightDec": 0.85,
      "proof_v2": 0.45,
      "\u0394": 0.39999999999999997
    },
    {
      "task": "agentharm/refuse",
      "n": 120,
      "LightDec": 0.6666666666666666,
      "proof_v2": 0.45,
      "\u0394": 0.21666666666666662
    },
    {
      "task": "agenttrek/finish_now",
      "n": 120,
      "LightDec": 0.7833333333333333,
      "proof_v2": 0.25833333333333336,
      "\u0394": 0.5249999999999999
    },
    {
      "task": "agenttrek/next_action_type",
      "n": 120,
      "LightDec": 0.875,
      "proof_v2": 0.075,
      "\u0394": 0.8
    },
    {
      "task": "anli/nli",
      "n": 120,
      "LightDec": 0.5916666666666667,
      "proof_v2": 0.325,
      "\u0394": 0.26666666666666666
    },
    {
      "task": "aqua_rat/math",
      "n": 120,
      "LightDec": 0.35,
      "proof_v2": 0.24166666666666667,
      "\u0394": 0.10833333333333331
    },
    {
      "task": "arc_challenge/mcq",
      "n": 120,
      "LightDec": 0.5333333333333333,
      "proof_v2": 0.30833333333333335,
      "\u0394": 0.22499999999999998
    },
    {
      "task": "arc_easy/mcq",
      "n": 120,
      "LightDec": 0.65,
      "proof_v2": 0.4583333333333333,
      "\u0394": 0.1916666666666667
    },
    {
      "task": "banking77/intent",
      "n": 120,
      "LightDec": 0.925,
      "proof_v2": 0.8666666666666667,
      "\u0394": 0.05833333333333335
    },
    {
      "task": "bigclonebench/clone",
      "n": 120,
      "LightDec": 0.9666666666666667,
      "proof_v2": 0.24166666666666667,
      "\u0394": 0.725
    },
    {
      "task": "bitext/category",
      "n": 120,
      "LightDec": 1.0,
      "proof_v2": 0.8583333333333333,
      "\u0394": 0.14166666666666672
    },
    {
      "task": "bitext/route",
      "n": 120,
      "LightDec": 1.0,
      "proof_v2": 0.925,
      "\u0394": 0.07499999999999996
    },
    {
      "task": "boolq/yes_no",
      "n": 120,
      "LightDec": 0.825,
      "proof_v2": 0.6666666666666666,
      "\u0394": 0.15833333333333333
    },
    {
      "task": "civil_comments/toxic",
      "n": 120,
      "LightDec": 0.9333333333333333,
      "proof_v2": 0.35833333333333334,
      "\u0394": 0.575
    },
    {
      "task": "clinc150/intent",
      "n": 120,
      "LightDec": 0.975,
      "proof_v2": 0.6416666666666667,
      "\u0394": 0.33333333333333326
    },
    {
      "task": "codexglue/code_to_doc",
      "n": 120,
      "LightDec": 1.0,
      "proof_v2": 0.9333333333333333,
      "\u0394": 0.06666666666666665
    },
    {
      "task": "codexglue/doc_to_code",
      "n": 120,
      "LightDec": 0.9833333333333333,
      "proof_v2": 0.975,
      "\u0394": 0.008333333333333304
    },
    {
      "task": "codexglue/func_name",
      "n": 120,
      "LightDec": 0.9916666666666667,
      "proof_v2": 0.9166666666666666,
      "\u0394": 0.07500000000000007
    },
    {
      "task": "codexglue/lang_id",
      "n": 120,
      "LightDec": 1.0,
      "proof_v2": 1.0,
      "\u0394": 0.0
    },
    {
      "task": "commonsense_qa/mcq",
      "n": 120,
      "LightDec": 0.6833333333333333,
      "proof_v2": 0.4083333333333333,
      "\u0394": 0.275
    },
    {
      "task": "counsel/critique_quality",
      "n": 120,
      "LightDec": 0.6083333333333333,
      "proof_v2": 0.25833333333333336,
      "\u0394": 0.3499999999999999
    },
    {
      "task": "counsel/step_has_error",
      "n": 120,
      "LightDec": 0.825,
      "proof_v2": 0.75,
      "\u0394": 0.07499999999999996
    },
    {
      "task": "devign/vulnerability",
      "n": 120,
      "LightDec": 0.6666666666666666,
      "proof_v2": 0.5666666666666667,
      "\u0394": 0.09999999999999998
    },
    {
      "task": "emotion/6way",
      "n": 120,
      "LightDec": 0.49166666666666664,
      "proof_v2": 0.36666666666666664,
      "\u0394": 0.125
    },
    {
      "task": "gsm8k/math",
      "n": 120,
      "LightDec": 0.7333333333333333,
      "proof_v2": 0.125,
      "\u0394": 0.6083333333333333
    },
    {
      "task": "hellaswag/continuation",
      "n": 120,
      "LightDec": 0.6083333333333333,
      "proof_v2": 0.2916666666666667,
      "\u0394": 0.3166666666666666
    },
    {
      "task": "hotpotqa/comparison_yes_no",
      "n": 26,
      "LightDec": 0.9230769230769231,
      "proof_v2": 0.46153846153846156,
      "\u0394": 0.46153846153846156
    },
    {
      "task": "hotpotqa/retrieve",
      "n": 120,
      "LightDec": 0.875,
      "proof_v2": 0.2916666666666667,
      "\u0394": 0.5833333333333333
    },
    {
      "task": "humaneval/completion",
      "n": 119,
      "LightDec": 0.8487394957983193,
      "proof_v2": 0.5126050420168067,
      "\u0394": 0.33613445378151263
    },
    {
      "task": "jailbreak/detect",
      "n": 120,
      "LightDec": 0.9833333333333333,
      "proof_v2": 0.5666666666666667,
      "\u0394": 0.41666666666666663
    },
    {
      "task": "long/contract_clause",
      "n": 120,
      "LightDec": 1.0,
      "proof_v2": 0.23333333333333334,
      "\u0394": 0.7666666666666666
    },
    {
      "task": "long/email_thread",
      "n": 120,
      "LightDec": 1.0,
      "proof_v2": 0.05,
      "\u0394": 0.95
    },
    {
      "task": "long/service_log",
      "n": 120,
      "LightDec": 1.0,
      "proof_v2": 0.24166666666666667,
      "\u0394": 0.7583333333333333
    },
    {
      "task": "massive_en/intent",
      "n": 120,
      "LightDec": 0.95,
      "proof_v2": 0.7416666666666667,
      "\u0394": 0.20833333333333326
    },
    {
      "task": "mbpp/bugspot",
      "n": 120,
      "LightDec": 0.8583333333333333,
      "proof_v2": 0.575,
      "\u0394": 0.2833333333333333
    },
    {
      "task": "mbpp/solution",
      "n": 120,
      "LightDec": 1.0,
      "proof_v2": 0.85,
      "\u0394": 0.15000000000000002
    },
    {
      "task": "mmlu/mcq",
      "n": 120,
      "LightDec": 0.4083333333333333,
      "proof_v2": 0.30833333333333335,
      "\u0394": 0.09999999999999998
    },
    {
      "task": "mnli/claim",
      "n": 120,
      "LightDec": 0.875,
      "proof_v2": 0.31666666666666665,
      "\u0394": 0.5583333333333333
    },
    {
      "task": "openbookqa/mcq",
      "n": 120,
      "LightDec": 0.65,
      "proof_v2": 0.2833333333333333,
      "\u0394": 0.3666666666666667
    },
    {
      "task": "policy/access_control_transfer",
      "n": 120,
      "LightDec": 1.0,
      "proof_v2": 0.23333333333333334,
      "\u0394": 0.7666666666666666
    },
    {
      "task": "policy/count_threshold_transfer",
      "n": 120,
      "LightDec": 0.8666666666666667,
      "proof_v2": 0.10833333333333334,
      "\u0394": 0.7583333333333333
    },
    {
      "task": "policy/free_shipping_transfer",
      "n": 120,
      "LightDec": 0.925,
      "proof_v2": 0.7083333333333334,
      "\u0394": 0.21666666666666667
    },
    {
      "task": "policy/invoice_overdue_transfer",
      "n": 120,
      "LightDec": 0.8083333333333333,
      "proof_v2": 0.6583333333333333,
      "\u0394": 0.15000000000000002
    },
    {
      "task": "policy/invoice_total_transfer",
      "n": 120,
      "LightDec": 0.45,
      "proof_v2": 0.525,
      "\u0394": -0.07500000000000001
    },
    {
      "task": "policy/refund_approval_transfer",
      "n": 120,
      "LightDec": 0.9583333333333334,
      "proof_v2": 0.325,
      "\u0394": 0.6333333333333333
    },
    {
      "task": "policy/return_window_transfer",
      "n": 120,
      "LightDec": 1.0,
      "proof_v2": 0.3416666666666667,
      "\u0394": 0.6583333333333333
    },
    {
      "task": "policy/sla_urgency_transfer",
      "n": 120,
      "LightDec": 0.5166666666666667,
      "proof_v2": 0.44166666666666665,
      "\u0394": 0.07500000000000007
    },
    {
      "task": "policy/table_compare_transfer",
      "n": 120,
      "LightDec": 0.9166666666666666,
      "proof_v2": 0.49166666666666664,
      "\u0394": 0.425
    },
    {
      "task": "policy/table_count_transfer",
      "n": 120,
      "LightDec": 0.31666666666666665,
      "proof_v2": 0.15833333333333333,
      "\u0394": 0.15833333333333333
    },
    {
      "task": "policy/table_extreme_transfer",
      "n": 120,
      "LightDec": 0.925,
      "proof_v2": 0.14166666666666666,
      "\u0394": 0.7833333333333334
    },
    {
      "task": "prompt_injections/detect",
      "n": 116,
      "LightDec": 0.5948275862068966,
      "proof_v2": 0.5431034482758621,
      "\u0394": 0.051724137931034475
    },
    {
      "task": "qasc/mcq",
      "n": 120,
      "LightDec": 0.9916666666666667,
      "proof_v2": 0.8583333333333333,
      "\u0394": 0.13333333333333341
    },
    {
      "task": "sciq/mcq",
      "n": 120,
      "LightDec": 0.9666666666666667,
      "proof_v2": 0.8666666666666667,
      "\u0394": 0.09999999999999998
    },
    {
      "task": "scitail/support",
      "n": 120,
      "LightDec": 0.9583333333333334,
      "proof_v2": 0.425,
      "\u0394": 0.5333333333333334
    },
    {
      "task": "snli/contradicts",
      "n": 120,
      "LightDec": 1.0,
      "proof_v2": 0.35833333333333334,
      "\u0394": 0.6416666666666666
    },
    {
      "task": "snli/must_be_true",
      "n": 120,
      "LightDec": 0.9916666666666667,
      "proof_v2": 0.925,
      "\u0394": 0.06666666666666665
    },
    {
      "task": "snli/nli",
      "n": 120,
      "LightDec": 0.9083333333333333,
      "proof_v2": 0.3416666666666667,
      "\u0394": 0.5666666666666667
    },
    {
      "task": "sst5/score",
      "n": 120,
      "LightDec": 0.38333333333333336,
      "proof_v2": 0.2833333333333333,
      "\u0394": 0.10000000000000003
    },
    {
      "task": "tev1_test/ag_news",
      "n": 120,
      "LightDec": 0.925,
      "proof_v2": 0.425,
      "\u0394": 0.5
    },
    {
      "task": "tev1_test/banking77",
      "n": 120,
      "LightDec": 0.7166666666666667,
      "proof_v2": 0.55,
      "\u0394": 0.16666666666666663
    },
    {
      "task": "tev1_test/boolq",
      "n": 120,
      "LightDec": 0.8416666666666667,
      "proof_v2": 0.5583333333333333,
      "\u0394": 0.2833333333333333
    },
    {
      "task": "tev1_test/mnli",
      "n": 120,
      "LightDec": 0.75,
      "proof_v2": 0.39166666666666666,
      "\u0394": 0.35833333333333334
    },
    {
      "task": "tev1_test/policy",
      "n": 120,
      "LightDec": 0.5,
      "proof_v2": 0.4083333333333333,
      "\u0394": 0.09166666666666667
    },
    {
      "task": "tev1_test/routing",
      "n": 120,
      "LightDec": 0.44166666666666665,
      "proof_v2": 0.2916666666666667,
      "\u0394": 0.14999999999999997
    },
    {
      "task": "tev1_test/sst5",
      "n": 120,
      "LightDec": 0.39166666666666666,
      "proof_v2": 0.275,
      "\u0394": 0.11666666666666664
    },
    {
      "task": "triage/support_email",
      "n": 120,
      "LightDec": 0.9583333333333334,
      "proof_v2": 0.39166666666666666,
      "\u0394": 0.5666666666666667
    },
    {
      "task": "typed_decisions/agent_trace_observability",
      "n": 120,
      "LightDec": 0.8083333333333333,
      "proof_v2": 0.3333333333333333,
      "\u0394": 0.47500000000000003
    },
    {
      "task": "typed_decisions/customer_service",
      "n": 120,
      "LightDec": 0.7416666666666667,
      "proof_v2": 0.2833333333333333,
      "\u0394": 0.45833333333333337
    },
    {
      "task": "typed_decisions/invoice_processing",
      "n": 120,
      "LightDec": 0.8166666666666667,
      "proof_v2": 0.475,
      "\u0394": 0.3416666666666667
    },
    {
      "task": "typed_decisions/security_incidents",
      "n": 120,
      "LightDec": 0.7583333333333333,
      "proof_v2": 0.5,
      "\u0394": 0.2583333333333333
    },
    {
      "task": "winogrande/blank",
      "n": 120,
      "LightDec": 0.6916666666666667,
      "proof_v2": 0.5583333333333333,
      "\u0394": 0.1333333333333333
    },
    {
      "task": "yelp/score",
      "n": 120,
      "LightDec": 0.625,
      "proof_v2": 0.325,
      "\u0394": 0.3
    }
  ],
  "head_to_head_error": null,
  "v3_benchmarks": {
    "tev1": {
      "per_task": [
        {
          "task": "tev1_test/ag_news",
          "n": 150,
          "acc": 0.92,
          "overlaps_LightDec_training_source": true
        },
        {
          "task": "tev1_test/banking77",
          "n": 200,
          "acc": 0.72,
          "overlaps_LightDec_training_source": false
        },
        {
          "task": "tev1_test/boolq",
          "n": 200,
          "acc": 0.835,
          "overlaps_LightDec_training_source": true
        },
        {
          "task": "tev1_test/mnli",
          "n": 300,
          "acc": 0.7233333333333334,
          "overlaps_LightDec_training_source": true
        },
        {
          "task": "tev1_test/policy",
          "n": 1200,
          "acc": 0.5291666666666667,
          "overlaps_LightDec_training_source": false
        },
        {
          "task": "tev1_test/routing",
          "n": 600,
          "acc": 0.4583333333333333,
          "overlaps_LightDec_training_source": false
        },
        {
          "task": "tev1_test/sst5",
          "n": 150,
          "acc": 0.3933333333333333,
          "overlaps_LightDec_training_source": false
        }
      ],
      "all": 0.5839285714285715,
      "no_overlap": 0.5176744186046511,
      "n": 2800,
      "published_tev1_4b": {
        "main decisions (1,000)": 0.88,
        "policy transfer (300)": 1.0
      }
    },
    "triage": {
      "emails": 101,
      "per_question": {
        "topic": 0.9900990099009901,
        "refund": 1.0,
        "breakage": 0.9900990099009901,
        "anger": 0.9207920792079208,
        "judgment": 0.8217821782178217
      },
      "pile_accuracy": 0.9405940594059405
    },
    "long_context": {
      "max_len": 2048,
      "per_task": [
        {
          "task": "long/contract_clause",
          "n": 150,
          "acc": 1.0,
          "chance": 0.19999999999999996
        },
        {
          "task": "long/email_thread",
          "n": 150,
          "acc": 1.0,
          "chance": 0.25
        },
        {
          "task": "long/service_log",
          "n": 150,
          "acc": 1.0,
          "chance": 0.19999999999999996
        }
      ]
    }
  },
  "architecture": "FalconDec",
  "latency": {
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
    "calls": [
      {
        "device": "cuda",
        "weights": "fp16",
        "questions": 1,
        "p50_ms": 10.05,
        "p95_ms": 10.43
      },
      {
        "device": "cuda",
        "weights": "fp16",
        "questions": 5,
        "p50_ms": 11.32,
        "p95_ms": 11.41
      },
      {
        "device": "cuda",
        "weights": "fp16",
        "questions": 10,
        "p50_ms": 12.57,
        "p95_ms": 12.69
      },
      {
        "device": "cpu fp32",
        "weights": "int8 file",
        "questions": 1,
        "p50_ms": 48.4,
        "p95_ms": 48.8
      }
    ],
    "throughput_per_s": 2588.8502560238694
  },
  "env": {
    "torch": "2.9.0+cu130",
    "transformers": "5.17.0",
    "python": "3.12.12",
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition"
  }
}