Hugging Face's logo

Falconsai
/
LightDec_V2

Zero-Shot Classification
Transformers
Safetensors
GGUF
English
unknown
decision-model
system-one
falcondec
lightdec_v2
calibrated-decisions
multiple-choice
intent-classification
customer-support
natural-language-inference
code
guardrails
agents
selective-prediction
falconsai
model-surgeon
attested-lineage
Model card Files
xet
 Community
1
LightDec_V2 / falcondec_report.json
RealFalconsAI's picture
RealFalconsAI
Upload 16 files
99347a4 verified
Raw History Blame Contribute Delete
45.5 kB
Invalid JSON:Unexpected token 'N', ..." (card)": NaN, "... is not valid JSON
{
"version": "1.0.0",
"notebook_version": "V3.7",
"mode": "scratch",
"parent_version": null,
"lineage": [
"jhu-clsp/ettin-encoder-150m"
],
"config": {
"MODE": "scratch",
"FINETUNE_FROM": "latest",
"MODEL_NAME": "LightDec_V2_Long",
"BACKBONE": "jhu-clsp/ettin-encoder-150m",
"BACKBONE_INIT": "pretrained",
"PRESET": "long",
"CUSTOM_DATA_JSONL": "",
"CUSTOM_WEIGHT": 3.0,
"INCLUDE_BUILDERS": [],
"EXCLUDE_BUILDERS": [
"mind2web"
],
"TEV1_DIR": "./tev1",
"TEV1_AUTOBUILD": true,
"TEV1_POLICY_CAP": 13500,
"TEV1_ROUTING_CAP": 6000,
"TEV1_RESEARCH_CAP": 1000,
"TRIAGE_EMAILS": 200,
"TEV1_RETRIES": 3,
"TEV1_STEP_TIMEOUT_MIN": 180,
"TEV1_STALL_MIN": 15,
"RETRY_FAILED_BUILDERS": true,
"LONG_TASK_TRAIN": 1500,
"MAX_LEN": 2048,
"LONG_MAX_LEN": 2048,
"LONG_OPTS_THRESHOLD": 24,
"HEAD_MAX_LEN": 192,
"MAX_TOK_PER_OPT": 24,
"MAX_OPTS_SINGLE_PASS": 96,
"INTERACT_LAYERS": 2,
"INTERACT_HEADS": 8,
"BATCH_SIZE": 128,
"TOKENS_PER_BATCH": 65536,
"GRAD_ACCUM": 1,
"LR_ENCODER": 8e-05,
"LR_HEAD": 0.0006,
"LLRD": 0.9,
"WEIGHT_DECAY": 0.01,
"WARMUP_FRAC": 0.06,
"EPOCHS": 0,
"EMA_DECAY": 0.999,
"GRAD_CLIP": 1.0,
"FINETUNE_LR_SCALE": 0.5,
"SPHERICAL_W": 0.5,
"BRIER_W": 0.0,
"RPS_W": 1.0,
"TASK_SAMPLING_ALPHA": 0.5,
"NOTA_PROB": 0.08,
"RLCD_EPOCHS": 0.0,
"RLCD_SAMPLES": 4,
"RLCD_SIGMA": 0.3,
"RLCD_CE_ANCHOR": 0.5,
"DEVICE": "cuda",
"SEED": 42,
"NUM_WORKERS": 8,
"USE_COMPILE": true,
"ATTN_IMPL": "auto",
"OUTPUT_ROOT": "./falcondec_runs",
"EXPORT_INT8": true,
"SIZE_TARGET_MB": 250,
"DEFER_THRESHOLD": 0.7,
"EVAL_BASELINE_PROOF_V2": true,
"BASELINE_MAX_PER_TASK": 120,
"RUN_CPU_BENCH": true,
"PUSH_TO_HUB": false,
"HUB_REPO": ""
},
"epochs": 10,
"best_epoch": 4,
"rlcd_kept": false,
"history": [
{
"epoch": 1,
"train_loss": 0.7058714814313827,
"train_acc": 0.7342094450703411,
"val_macro": 0.8188568656508199,
"val_micro": 0.820702479338843,
"val_nll": 0.4076740937270036
},
{
"epoch": 2,
"train_loss": 0.431469622218984,
"train_acc": 0.8473141073247583,
"val_macro": 0.8450872188411825,
"val_micro": 0.8444628099173553,
"val_nll": 0.3623910443100994
},
{
"epoch": 3,
"train_loss": 0.34276796075090776,
"train_acc": 0.8791940521680108,
"val_macro": 0.8501904675940947,
"val_micro": 0.8499173553719008,
"val_nll": 0.39227130460368825
},
{
"epoch": 4,
"train_loss": 0.2773282952523419,
"train_acc": 0.9039863196679513,
"val_macro": 0.8519172452047367,
"val_micro": 0.8519834710743802,
"val_nll": 0.4529550437184426
},
{
"epoch": 5,
"train_loss": 0.22030453415052323,
"train_acc": 0.925851485588573,
"val_macro": 0.8488092001804082,
"val_micro": 0.85,
"val_nll": 0.5369876145373608
},
{
"epoch": 6,
"train_loss": 0.16953414903032543,
"train_acc": 0.9449156076226879,
"val_macro": 0.8474279008050457,
"val_micro": 0.8488842975206612,
"val_nll": 0.6705887193589142
},
{
"epoch": 7,
"train_loss": 0.126976966611751,
"train_acc": 0.9602833527848939,
"val_macro": 0.8453322694829587,
"val_micro": 0.8476446280991735,
"val_nll": 0.8430423012207805
},
{
"epoch": 8,
"train_loss": 0.09470793687766035,
"train_acc": 0.9714013975720073,
"val_macro": 0.8451769009269958,
"val_micro": 0.8467355371900827,
"val_nll": 1.0336680755654943
},
{
"epoch": 9,
"train_loss": 0.0710737590306488,
"train_acc": 0.9794225926173407,
"val_macro": 0.8435159403610613,
"val_micro": 0.8458677685950413,
"val_nll": 1.210751206553627
},
{
"epoch": 10,
"train_loss": 0.05665995772408678,
"train_acc": 0.9843832093140154,
"val_macro": 0.844272704258198,
"val_micro": 0.8474380165289256,
"val_nll": 1.41872131482475
}
],
"train_minutes": 291.67019440333047,
"temperatures": [
[
2.37591814994812,
2.0426506996154785,
1.6455482244491577,
1.8417285680770874
],
[
2.4885058403015137,
2.4885058403015137,
2.4885058403015137,
2.4885058403015137
],
[
2.2925639152526855,
2.2925639152526855,
2.2925639152526855,
2.2925639152526855
]
],
"data_counts": {
"train": 1023814,
"validation": 24200,
"test": 31990
},
"skipped_builders": [],
"test_overall": {
"n": 31990.0,
"micro_acc": 0.784057517974367,
"macro_acc": 0.7887006006875437,
"nll": 0.5630830966790886,
"brier": 0.2954194093899411,
"ece": 0.02856345933167103,
"aurc": 0.06778106632931055,
"score_mae": 0.5060575584353655,
"coverage@0.7": 0.6797436698968428,
"acc_on_covered@0.7": 0.9059094044607956
},
"test_per_domain": {
"agentic": 0.821117397177404,
"classification": 0.6115,
"code": 0.9108344674425007,
"guardrails": 0.7984073149310548,
"intents": 0.8505,
"long_context": 1.0,
"policy": 0.8061818181818182,
"reasoning": 0.7180877154615182,
"support": 0.9815181518151815,
"tev1_benchmark": 0.6541666666666667,
"workflows": 0.773
},
"test_per_task": [
{
"task": "ag_news/topic",
"n": 500,
"acc": 0.9,
"chance": 0.25,
"nll": 0.2674782864307199,
"brier": 0.14296993909824685,
"heldout": false,
"domain": "classification",
"ece": 0.05165432274341582,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "agentharm/refuse",
"n": 416,
"acc": 0.6995192307692307,
"chance": 0.5,
"nll": 0.6008703433729422,
"brier": 0.4076762512706606,
"heldout": true,
"domain": "guardrails",
"ece": 0.08916855761064935,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "agenttrek/finish_now",
"n": 151,
"acc": 0.7880794701986755,
"chance": 0.5,
"nll": 0.445978463758558,
"brier": 0.30066953632150406,
"heldout": false,
"domain": "agentic",
"ece": 0.07512128787324918,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "agenttrek/next_action_type",
"n": 487,
"acc": 0.864476386036961,
"chance": 0.19054952576513148,
"nll": 0.3929848723152591,
"brier": 0.2184262054212701,
"heldout": false,
"domain": "agentic",
"ece": 0.09202881025827395,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "anli/nli",
"n": 498,
"acc": 0.4859437751004016,
"chance": 0.33333333333333326,
"nll": 1.069221544292677,
"brier": 0.6544603492386665,
"heldout": false,
"domain": "reasoning",
"ece": 0.14589204983299517,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "aqua_rat/math",
"n": 247,
"acc": 0.340080971659919,
"chance": 0.2,
"nll": 1.54525283087603,
"brier": 0.7739737508966391,
"heldout": false,
"domain": "reasoning",
"ece": 0.080350092668765,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "arc_challenge/mcq",
"n": 500,
"acc": 0.49,
"chance": 0.25006666666666666,
"nll": 1.3021090898208785,
"brier": 0.6752390805252713,
"heldout": true,
"domain": "reasoning",
"ece": 0.1680490539073944,
"proof_v2 (card)": 0.308,
"\u0394 vs proof_v2": 0.182
},
{
"task": "arc_easy/mcq",
"n": 500,
"acc": 0.612,
"chance": 0.24980000000000002,
"nll": 0.9830388274090365,
"brier": 0.5264415342806388,
"heldout": true,
"domain": "reasoning",
"ece": 0.1215095258355141,
"proof_v2 (card)": 0.425,
"\u0394 vs proof_v2": 0.187
},
{
"task": "banking77/intent",
"n": 500,
"acc": 0.922,
"chance": 0.3214666666666666,
"nll": 0.23128961177584006,
"brier": 0.11756106243365176,
"heldout": true,
"domain": "intents",
"ece": 0.037492330908775316,
"proof_v2 (card)": 0.883,
"\u0394 vs proof_v2": 0.039000000000000035
},
{
"task": "banking77/intent_77",
"n": 300,
"acc": 0.58,
"chance": 0.012987012987012986,
"nll": 2.159930980304489,
"brier": 0.6431857202493357,
"heldout": true,
"domain": "intents",
"ece": 0.24079640249411272,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "bigclonebench/clone",
"n": 500,
"acc": 0.962,
"chance": 0.5,
"nll": 0.1312969167594565,
"brier": 0.06444151702850223,
"heldout": false,
"domain": "code",
"ece": 0.03135248208045959,
"proof_v2 (card)": 0.383,
"\u0394 vs proof_v2": 0.579
},
{
"task": "bitext/category",
"n": 500,
"acc": 1.0,
"chance": 0.16128124098124094,
"nll": 0.0013673773490934594,
"brier": 0.0003828941821233322,
"heldout": false,
"domain": "support",
"ece": 0.001247308015823329,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "bitext/route",
"n": 500,
"acc": 1.0,
"chance": 0.19016313131313134,
"nll": 0.00031508885690954,
"brier": 4.903548307607296e-06,
"heldout": false,
"domain": "support",
"ece": 0.0003137041330337764,
"proof_v2 (card)": 0.958,
"\u0394 vs proof_v2": 0.04200000000000004
},
{
"task": "boolq/yes_no",
"n": 500,
"acc": 0.822,
"chance": 0.5,
"nll": 0.4842173567947466,
"brier": 0.2840138305161832,
"heldout": false,
"domain": "reasoning",
"ece": 0.0795972980260849,
"proof_v2 (card)": 0.717,
"\u0394 vs proof_v2": 0.10499999999999998
},
{
"task": "civil_comments/toxic",
"n": 500,
"acc": 0.926,
"chance": 0.5,
"nll": 0.2234244500696659,
"brier": 0.1209681824839501,
"heldout": false,
"domain": "guardrails",
"ece": 0.05123518884181974,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "clinc150/intent",
"n": 500,
"acc": 0.97,
"chance": 0.12404474984829318,
"nll": 0.09906445816022005,
"brier": 0.04996487012600131,
"heldout": false,
"domain": "intents",
"ece": 0.014944501101970714,
"proof_v2 (card)": 0.85,
"\u0394 vs proof_v2": 0.12
},
{
"task": "codexglue/code_to_doc",
"n": 504,
"acc": 0.9900793650793651,
"chance": 0.26233465608465617,
"nll": 0.032474144255576765,
"brier": 0.016158895201785088,
"heldout": false,
"domain": "code",
"ece": 0.009654374349685009,
"proof_v2 (card)": 0.969,
"\u0394 vs proof_v2": 0.021079365079365142
},
{
"task": "codexglue/doc_to_code",
"n": 504,
"acc": 0.9761904761904762,
"chance": 0.2786044973544973,
"nll": 0.05970113779317181,
"brier": 0.03072688989396561,
"heldout": false,
"domain": "code",
"ece": 0.012547755761752008,
"proof_v2 (card)": 0.961,
"\u0394 vs proof_v2": 0.015190476190476199
},
{
"task": "codexglue/func_name",
"n": 467,
"acc": 0.9550321199143469,
"chance": 0.25499643112062814,
"nll": 0.14961532151979784,
"brier": 0.07053876912275295,
"heldout": false,
"domain": "code",
"ece": 0.03948740715388316,
"proof_v2 (card)": 0.901,
"\u0394 vs proof_v2": 0.054032119914346866
},
{
"task": "codexglue/lang_id",
"n": 504,
"acc": 1.0,
"chance": 0.2354166666666667,
"nll": 0.0014352430686462487,
"brier": 1.2864369183890018e-05,
"heldout": false,
"domain": "code",
"ece": 0.0014309174721203188,
"proof_v2 (card)": 0.997,
"\u0394 vs proof_v2": 0.0030000000000000027
},
{
"task": "commonsense_qa/mcq",
"n": 493,
"acc": 0.6430020283975659,
"chance": 0.19999999999999998,
"nll": 0.9523430099929484,
"brier": 0.49435087368161623,
"heldout": false,
"domain": "reasoning",
"ece": 0.14308575696442233,
"proof_v2 (card)": 0.442,
"\u0394 vs proof_v2": 0.20100202839756592
},
{
"task": "counsel/critique_quality",
"n": 201,
"acc": 0.6268656716417911,
"chance": 0.3333333333333333,
"nll": 1.5466346353110312,
"brier": 0.6427543071457474,
"heldout": false,
"domain": "agentic",
"ece": 0.27062572442477023,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "counsel/step_has_error",
"n": 201,
"acc": 0.8308457711442786,
"chance": 0.5,
"nll": 0.8236887436017318,
"brier": 0.3201374357369142,
"heldout": false,
"domain": "agentic",
"ece": 0.14827988011326954,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "devign/vulnerability",
"n": 500,
"acc": 0.632,
"chance": 0.5,
"nll": 0.6074751503933221,
"brier": 0.42706214521965163,
"heldout": false,
"domain": "code",
"ece": 0.07018342161178594,
"proof_v2 (card)": 0.542,
"\u0394 vs proof_v2": 0.08999999999999997
},
{
"task": "emotion/6way",
"n": 500,
"acc": 0.472,
"chance": 0.16666666666666663,
"nll": 1.561316128242761,
"brier": 0.7356062387487318,
"heldout": true,
"domain": "classification",
"ece": 0.21054179659485817,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "gsm8k/math",
"n": 500,
"acc": 0.7,
"chance": 0.25,
"nll": 0.6923719964642078,
"brier": 0.4014884319624292,
"heldout": false,
"domain": "reasoning",
"ece": 0.08541783547401426,
"proof_v2 (card)": 0.275,
"\u0394 vs proof_v2": 0.42499999999999993
},
{
"task": "hellaswag/continuation",
"n": 500,
"acc": 0.576,
"chance": 0.25,
"nll": 0.968790470642969,
"brier": 0.5288133586177948,
"heldout": false,
"domain": "reasoning",
"ece": 0.07543198531866073,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "hotpotqa/comparison_yes_no",
"n": 26,
"acc": 0.9230769230769231,
"chance": 0.5,
"nll": 0.3400710394176153,
"brier": 0.15484679762361436,
"heldout": false,
"domain": "agentic",
"ece": 0.08232885369887719,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "hotpotqa/retrieve",
"n": 497,
"acc": 0.8933601609657947,
"chance": 0.16741720162243304,
"nll": 0.3457089332123268,
"brier": 0.16216258689991922,
"heldout": false,
"domain": "agentic",
"ece": 0.04268557528854612,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "humaneval/completion",
"n": 119,
"acc": 0.8487394957983193,
"chance": 0.39355742296918783,
"nll": 0.46361927195851294,
"brier": 0.2730045795813753,
"heldout": true,
"domain": "code",
"ece": 0.13989568708323633,
"proof_v2 (card)": 0.575,
"\u0394 vs proof_v2": 0.27373949579831935
},
{
"task": "jailbreak/detect",
"n": 262,
"acc": 0.9732824427480916,
"chance": 0.5,
"nll": 0.08469655378148465,
"brier": 0.044003931208438284,
"heldout": false,
"domain": "guardrails",
"ece": 0.030764950368240628,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "long/contract_clause",
"n": 150,
"acc": 1.0,
"chance": 0.19999999999999996,
"nll": 0.0002789537957808837,
"brier": 4.274148862877054e-07,
"heldout": false,
"domain": "long_context",
"ece": 0.00027882695198055973,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "long/email_thread",
"n": 150,
"acc": 1.0,
"chance": 0.25,
"nll": 0.0003266528345905802,
"brier": 2.717841913247229e-06,
"heldout": false,
"domain": "long_context",
"ece": 0.0003259424368540209,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "long/service_log",
"n": 150,
"acc": 1.0,
"chance": 0.19999999999999996,
"nll": 0.0010035930563268873,
"brier": 2.0987077842670875e-06,
"heldout": false,
"domain": "long_context",
"ece": 0.0010028688112894146,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "massive_en/intent",
"n": 500,
"acc": 0.93,
"chance": 0.12958727789085603,
"nll": 0.20636169074449573,
"brier": 0.10375277943487987,
"heldout": false,
"domain": "intents",
"ece": 0.0284088468849659,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "mbpp/bugspot",
"n": 256,
"acc": 0.85546875,
"chance": 0.40559895833333326,
"nll": 0.3667814989712497,
"brier": 0.21843909652329055,
"heldout": false,
"domain": "code",
"ece": 0.037241218378767364,
"proof_v2 (card)": 0.475,
"\u0394 vs proof_v2": 0.38046875
},
{
"task": "mbpp/solution",
"n": 500,
"acc": 0.978,
"chance": 0.25,
"nll": 0.05863042902501184,
"brier": 0.02981698831494921,
"heldout": false,
"domain": "code",
"ece": 0.01960155099630357,
"proof_v2 (card)": 0.992,
"\u0394 vs proof_v2": -0.014000000000000012
},
{
"task": "mmlu/mcq",
"n": 500,
"acc": 0.39,
"chance": 0.25,
"nll": 1.4543649319559335,
"brier": 0.7702575193705765,
"heldout": true,
"domain": "reasoning",
"ece": 0.15507307499647138,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "mnli/claim",
"n": 500,
"acc": 0.86,
"chance": 0.33333333333333326,
"nll": 0.4006310784481466,
"brier": 0.2131232579488494,
"heldout": false,
"domain": "reasoning",
"ece": 0.081218329668045,
"proof_v2 (card)": 0.492,
"\u0394 vs proof_v2": 0.368
},
{
"task": "openbookqa/mcq",
"n": 500,
"acc": 0.572,
"chance": 0.25,
"nll": 1.1920220527790952,
"brier": 0.6155284183368397,
"heldout": false,
"domain": "reasoning",
"ece": 0.21775564682483672,
"proof_v2 (card)": 0.292,
"\u0394 vs proof_v2": 0.27999999999999997
},
{
"task": "policy/access_control_transfer",
"n": 500,
"acc": 1.0,
"chance": 0.33333333333333326,
"nll": 0.002006467206090747,
"brier": 2.4728843418299015e-05,
"heldout": false,
"domain": "policy",
"ece": 0.001999327063560541,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "policy/count_threshold_transfer",
"n": 500,
"acc": 0.88,
"chance": 0.10537052392052391,
"nll": 0.319376940273738,
"brier": 0.1809248790341789,
"heldout": false,
"domain": "policy",
"ece": 0.04758520478010175,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "policy/free_shipping_transfer",
"n": 500,
"acc": 0.938,
"chance": 0.5,
"nll": 0.1287872482436942,
"brier": 0.08322479293591872,
"heldout": false,
"domain": "policy",
"ece": 0.041308605194091734,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "policy/invoice_overdue_transfer",
"n": 500,
"acc": 0.824,
"chance": 0.5,
"nll": 0.3965394942490384,
"brier": 0.2504247577332596,
"heldout": false,
"domain": "policy",
"ece": 0.01985593855381013,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "policy/invoice_total_transfer",
"n": 500,
"acc": 0.474,
"chance": 0.5,
"nll": 0.710681935429573,
"brier": 0.5164436840846652,
"heldout": false,
"domain": "policy",
"ece": 0.08196303224563599,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "policy/refund_approval_transfer",
"n": 500,
"acc": 0.962,
"chance": 0.33333333333333326,
"nll": 0.10006511305032836,
"brier": 0.057969574647252664,
"heldout": false,
"domain": "policy",
"ece": 0.018834196925163266,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "policy/return_window_transfer",
"n": 500,
"acc": 1.0,
"chance": 0.33333333333333326,
"nll": 0.02710788446944207,
"brier": 0.004185976644178241,
"heldout": false,
"domain": "policy",
"ece": 0.02567652928829193,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "policy/sla_urgency_transfer",
"n": 500,
"acc": 0.606,
"chance": 0.25,
"nll": 0.8170870101451874,
"brier": 0.5123369210022903,
"heldout": false,
"domain": "policy",
"ece": 0.13068743020296097,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "policy/table_compare_transfer",
"n": 500,
"acc": 0.928,
"chance": 0.5,
"nll": 0.2105378701629961,
"brier": 0.11898007306418568,
"heldout": false,
"domain": "policy",
"ece": 0.04929606854915622,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "policy/table_count_transfer",
"n": 500,
"acc": 0.324,
"chance": 0.12114228549228548,
"nll": 3.24442003638536,
"brier": 1.1488026091975905,
"heldout": false,
"domain": "policy",
"ece": 0.5407210965156555,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "policy/table_extreme_transfer",
"n": 500,
"acc": 0.932,
"chance": 0.12210815295815294,
"nll": 0.268062693382595,
"brier": 0.11322196827103552,
"heldout": false,
"domain": "policy",
"ece": 0.05259446609020234,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "prompt_injections/detect",
"n": 116,
"acc": 0.5948275862068966,
"chance": 0.5,
"nll": 1.1838965486195179,
"brier": 0.6826645118148511,
"heldout": true,
"domain": "guardrails",
"ece": 0.33272201953263114,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "qasc/mcq",
"n": 500,
"acc": 0.986,
"chance": 0.125,
"nll": 0.028506858559036514,
"brier": 0.017274368003708154,
"heldout": false,
"domain": "reasoning",
"ece": 0.007774531126022318,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "sciq/mcq",
"n": 498,
"acc": 0.9538152610441767,
"chance": 0.25,
"nll": 0.1506263591345834,
"brier": 0.06985463059035983,
"heldout": false,
"domain": "reasoning",
"ece": 0.020590302994452258,
"proof_v2 (card)": 0.692,
"\u0394 vs proof_v2": 0.26181526104417674
},
{
"task": "scitail/support",
"n": 500,
"acc": 0.962,
"chance": 0.5,
"nll": 0.11612921302905306,
"brier": 0.05990850379879065,
"heldout": false,
"domain": "reasoning",
"ece": 0.03015338957309728,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "snli/contradicts",
"n": 500,
"acc": 0.99,
"chance": 0.33333333333333326,
"nll": 0.042843449617153966,
"brier": 0.017027529190616314,
"heldout": false,
"domain": "reasoning",
"ece": 0.018511961221695038,
"proof_v2 (card)": 0.892,
"\u0394 vs proof_v2": 0.09799999999999998
},
{
"task": "snli/must_be_true",
"n": 500,
"acc": 0.986,
"chance": 0.33333333333333326,
"nll": 0.06753946821717545,
"brier": 0.02560326154020915,
"heldout": false,
"domain": "reasoning",
"ece": 0.043202445626258815,
"proof_v2 (card)": 0.908,
"\u0394 vs proof_v2": 0.07799999999999996
},
{
"task": "snli/nli",
"n": 988,
"acc": 0.8947368421052632,
"chance": 0.33333333333333326,
"nll": 0.3554420507991845,
"brier": 0.17963376213248355,
"heldout": false,
"domain": "reasoning",
"ece": 0.10791712172842222,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "sst5/score",
"n": 500,
"acc": 0.406,
"chance": 0.2,
"nll": 1.3002369542717933,
"brier": 0.6888741553408958,
"heldout": true,
"domain": "classification",
"ece": 0.068851743131876,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "tev1_test/ag_news",
"n": 150,
"acc": 0.92,
"chance": 0.25,
"nll": 0.2815313008365532,
"brier": 0.13456139206846243,
"heldout": true,
"domain": "tev1_benchmark",
"ece": 0.09770219445228576,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "tev1_test/banking77",
"n": 200,
"acc": 0.72,
"chance": 0.17791666666666664,
"nll": 0.777236735031438,
"brier": 0.3902663054271286,
"heldout": true,
"domain": "tev1_benchmark",
"ece": 0.13721376925706863,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "tev1_test/boolq",
"n": 200,
"acc": 0.835,
"chance": 0.5,
"nll": 0.40902234488283284,
"brier": 0.24753055362669119,
"heldout": true,
"domain": "tev1_benchmark",
"ece": 0.04955918818712238,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "tev1_test/mnli",
"n": 300,
"acc": 0.7233333333333334,
"chance": 0.3333333333333333,
"nll": 0.6736717029288412,
"brier": 0.39782655881245704,
"heldout": true,
"domain": "tev1_benchmark",
"ece": 0.07533363938331603,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "tev1_test/policy",
"n": 1200,
"acc": 0.5291666666666667,
"chance": 0.3333333333333333,
"nll": 0.8761954319352905,
"brier": 0.5462664224307302,
"heldout": true,
"domain": "tev1_benchmark",
"ece": 0.05868758827447889,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "tev1_test/routing",
"n": 600,
"acc": 0.4583333333333333,
"chance": 0.2,
"nll": 1.120521725914441,
"brier": 0.5789269511485964,
"heldout": true,
"domain": "tev1_benchmark",
"ece": 0.037736288358767814,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "tev1_test/sst5",
"n": 150,
"acc": 0.3933333333333333,
"chance": 0.19999999999999996,
"nll": 1.294863009850184,
"brier": 0.6810328679494898,
"heldout": true,
"domain": "tev1_benchmark",
"ece": 0.1081918982664744,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "triage/support_email",
"n": 505,
"acc": 0.9445544554455445,
"chance": 0.32333333333333325,
"nll": 0.15705669974757994,
"brier": 0.07848840114983771,
"heldout": false,
"domain": "support",
"ece": 0.045869263741049465,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "typed_decisions/agent_trace_observability",
"n": 500,
"acc": 0.734,
"chance": 0.3,
"nll": 0.8089503426551818,
"brier": 0.45125534421065666,
"heldout": false,
"domain": "workflows",
"ece": 0.22166652178764346,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "typed_decisions/customer_service",
"n": 500,
"acc": 0.764,
"chance": 0.28,
"nll": 0.7428050636351109,
"brier": 0.40177602517263195,
"heldout": false,
"domain": "workflows",
"ece": 0.20972245779633528,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "typed_decisions/invoice_processing",
"n": 500,
"acc": 0.826,
"chance": 0.35,
"nll": 0.5732684296518564,
"brier": 0.2997603036136731,
"heldout": false,
"domain": "workflows",
"ece": 0.19463082921504973,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "typed_decisions/security_incidents",
"n": 500,
"acc": 0.768,
"chance": 0.34,
"nll": 0.7502468670606613,
"brier": 0.4238190137148829,
"heldout": false,
"domain": "workflows",
"ece": 0.23508695220947262,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "winogrande/blank",
"n": 500,
"acc": 0.662,
"chance": 0.5,
"nll": 0.6651458515003323,
"brier": 0.45323706188140617,
"heldout": false,
"domain": "reasoning",
"ece": 0.1159557296037674,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
},
{
"task": "yelp/score",
"n": 500,
"acc": 0.668,
"chance": 0.2,
"nll": 0.7834262755662202,
"brier": 0.4423740240858843,
"heldout": false,
"domain": "classification",
"ece": 0.09068749487400055,
"proof_v2 (card)": NaN,
"\u0394 vs proof_v2": NaN
}
],
"head_to_head": [
{
"task": "ag_news/topic",
"n": 120,
"LightDec": 0.85,
"proof_v2": 0.45,
"\u0394": 0.39999999999999997
},
{
"task": "agentharm/refuse",
"n": 120,
"LightDec": 0.6666666666666666,
"proof_v2": 0.45,
"\u0394": 0.21666666666666662
},
{
"task": "agenttrek/finish_now",
"n": 120,
"LightDec": 0.7833333333333333,
"proof_v2": 0.25833333333333336,
"\u0394": 0.5249999999999999
},
{
"task": "agenttrek/next_action_type",
"n": 120,
"LightDec": 0.875,
"proof_v2": 0.075,
"\u0394": 0.8
},
{
"task": "anli/nli",
"n": 120,
"LightDec": 0.5916666666666667,
"proof_v2": 0.325,
"\u0394": 0.26666666666666666
},
{
"task": "aqua_rat/math",
"n": 120,
"LightDec": 0.35,
"proof_v2": 0.24166666666666667,
"\u0394": 0.10833333333333331
},
{
"task": "arc_challenge/mcq",
"n": 120,
"LightDec": 0.5333333333333333,
"proof_v2": 0.30833333333333335,
"\u0394": 0.22499999999999998
},
{
"task": "arc_easy/mcq",
"n": 120,
"LightDec": 0.65,
"proof_v2": 0.4583333333333333,
"\u0394": 0.1916666666666667
},
{
"task": "banking77/intent",
"n": 120,
"LightDec": 0.925,
"proof_v2": 0.8666666666666667,
"\u0394": 0.05833333333333335
},
{
"task": "bigclonebench/clone",
"n": 120,
"LightDec": 0.9666666666666667,
"proof_v2": 0.24166666666666667,
"\u0394": 0.725
},
{
"task": "bitext/category",
"n": 120,
"LightDec": 1.0,
"proof_v2": 0.8583333333333333,
"\u0394": 0.14166666666666672
},
{
"task": "bitext/route",
"n": 120,
"LightDec": 1.0,
"proof_v2": 0.925,
"\u0394": 0.07499999999999996
},
{
"task": "boolq/yes_no",
"n": 120,
"LightDec": 0.825,
"proof_v2": 0.6666666666666666,
"\u0394": 0.15833333333333333
},
{
"task": "civil_comments/toxic",
"n": 120,
"LightDec": 0.9333333333333333,
"proof_v2": 0.35833333333333334,
"\u0394": 0.575
},
{
"task": "clinc150/intent",
"n": 120,
"LightDec": 0.975,
"proof_v2": 0.6416666666666667,
"\u0394": 0.33333333333333326
},
{
"task": "codexglue/code_to_doc",
"n": 120,
"LightDec": 1.0,
"proof_v2": 0.9333333333333333,
"\u0394": 0.06666666666666665
},
{
"task": "codexglue/doc_to_code",
"n": 120,
"LightDec": 0.9833333333333333,
"proof_v2": 0.975,
"\u0394": 0.008333333333333304
},
{
"task": "codexglue/func_name",
"n": 120,
"LightDec": 0.9916666666666667,
"proof_v2": 0.9166666666666666,
"\u0394": 0.07500000000000007
},
{
"task": "codexglue/lang_id",
"n": 120,
"LightDec": 1.0,
"proof_v2": 1.0,
"\u0394": 0.0
},
{
"task": "commonsense_qa/mcq",
"n": 120,
"LightDec": 0.6833333333333333,
"proof_v2": 0.4083333333333333,
"\u0394": 0.275
},
{
"task": "counsel/critique_quality",
"n": 120,
"LightDec": 0.6083333333333333,
"proof_v2": 0.25833333333333336,
"\u0394": 0.3499999999999999
},
{
"task": "counsel/step_has_error",
"n": 120,
"LightDec": 0.825,
"proof_v2": 0.75,
"\u0394": 0.07499999999999996
},
{
"task": "devign/vulnerability",
"n": 120,
"LightDec": 0.6666666666666666,
"proof_v2": 0.5666666666666667,
"\u0394": 0.09999999999999998
},
{
"task": "emotion/6way",
"n": 120,
"LightDec": 0.49166666666666664,
"proof_v2": 0.36666666666666664,
"\u0394": 0.125
},
{
"task": "gsm8k/math",
"n": 120,
"LightDec": 0.7333333333333333,
"proof_v2": 0.125,
"\u0394": 0.6083333333333333
},
{
"task": "hellaswag/continuation",
"n": 120,
"LightDec": 0.6083333333333333,
"proof_v2": 0.2916666666666667,
"\u0394": 0.3166666666666666
},
{
"task": "hotpotqa/comparison_yes_no",
"n": 26,
"LightDec": 0.9230769230769231,
"proof_v2": 0.46153846153846156,
"\u0394": 0.46153846153846156
},
{
"task": "hotpotqa/retrieve",
"n": 120,
"LightDec": 0.875,
"proof_v2": 0.2916666666666667,
"\u0394": 0.5833333333333333
},
{
"task": "humaneval/completion",
"n": 119,
"LightDec": 0.8487394957983193,
"proof_v2": 0.5126050420168067,
"\u0394": 0.33613445378151263
},
{
"task": "jailbreak/detect",
"n": 120,
"LightDec": 0.9833333333333333,
"proof_v2": 0.5666666666666667,
"\u0394": 0.41666666666666663
},
{
"task": "long/contract_clause",
"n": 120,
"LightDec": 1.0,
"proof_v2": 0.23333333333333334,
"\u0394": 0.7666666666666666
},
{
"task": "long/email_thread",
"n": 120,
"LightDec": 1.0,
"proof_v2": 0.05,
"\u0394": 0.95
},
{
"task": "long/service_log",
"n": 120,
"LightDec": 1.0,
"proof_v2": 0.24166666666666667,
"\u0394": 0.7583333333333333
},
{
"task": "massive_en/intent",
"n": 120,
"LightDec": 0.95,
"proof_v2": 0.7416666666666667,
"\u0394": 0.20833333333333326
},
{
"task": "mbpp/bugspot",
"n": 120,
"LightDec": 0.8583333333333333,
"proof_v2": 0.575,
"\u0394": 0.2833333333333333
},
{
"task": "mbpp/solution",
"n": 120,
"LightDec": 1.0,
"proof_v2": 0.85,
"\u0394": 0.15000000000000002
},
{
"task": "mmlu/mcq",
"n": 120,
"LightDec": 0.4083333333333333,
"proof_v2": 0.30833333333333335,
"\u0394": 0.09999999999999998
},
{
"task": "mnli/claim",
"n": 120,
"LightDec": 0.875,
"proof_v2": 0.31666666666666665,
"\u0394": 0.5583333333333333
},
{
"task": "openbookqa/mcq",
"n": 120,
"LightDec": 0.65,
"proof_v2": 0.2833333333333333,
"\u0394": 0.3666666666666667
},
{
"task": "policy/access_control_transfer",
"n": 120,
"LightDec": 1.0,
"proof_v2": 0.23333333333333334,
"\u0394": 0.7666666666666666
},
{
"task": "policy/count_threshold_transfer",
"n": 120,
"LightDec": 0.8666666666666667,
"proof_v2": 0.10833333333333334,
"\u0394": 0.7583333333333333
},
{
"task": "policy/free_shipping_transfer",
"n": 120,
"LightDec": 0.925,
"proof_v2": 0.7083333333333334,
"\u0394": 0.21666666666666667
},
{
"task": "policy/invoice_overdue_transfer",
"n": 120,
"LightDec": 0.8083333333333333,
"proof_v2": 0.6583333333333333,
"\u0394": 0.15000000000000002
},
{
"task": "policy/invoice_total_transfer",
"n": 120,
"LightDec": 0.45,
"proof_v2": 0.525,
"\u0394": -0.07500000000000001
},
{
"task": "policy/refund_approval_transfer",
"n": 120,
"LightDec": 0.9583333333333334,
"proof_v2": 0.325,
"\u0394": 0.6333333333333333
},
{
"task": "policy/return_window_transfer",
"n": 120,
"LightDec": 1.0,
"proof_v2": 0.3416666666666667,
"\u0394": 0.6583333333333333
},
{
"task": "policy/sla_urgency_transfer",
"n": 120,
"LightDec": 0.5166666666666667,
"proof_v2": 0.44166666666666665,
"\u0394": 0.07500000000000007
},
{
"task": "policy/table_compare_transfer",
"n": 120,
"LightDec": 0.9166666666666666,
"proof_v2": 0.49166666666666664,
"\u0394": 0.425
},
{
"task": "policy/table_count_transfer",
"n": 120,
"LightDec": 0.31666666666666665,
"proof_v2": 0.15833333333333333,
"\u0394": 0.15833333333333333
},
{
"task": "policy/table_extreme_transfer",
"n": 120,
"LightDec": 0.925,
"proof_v2": 0.14166666666666666,
"\u0394": 0.7833333333333334
},
{
"task": "prompt_injections/detect",
"n": 116,
"LightDec": 0.5948275862068966,
"proof_v2": 0.5431034482758621,
"\u0394": 0.051724137931034475
},
{
"task": "qasc/mcq",
"n": 120,
"LightDec": 0.9916666666666667,
"proof_v2": 0.8583333333333333,
"\u0394": 0.13333333333333341
},
{
"task": "sciq/mcq",
"n": 120,
"LightDec": 0.9666666666666667,
"proof_v2": 0.8666666666666667,
"\u0394": 0.09999999999999998
},
{
"task": "scitail/support",
"n": 120,
"LightDec": 0.9583333333333334,
"proof_v2": 0.425,
"\u0394": 0.5333333333333334
},
{
"task": "snli/contradicts",
"n": 120,
"LightDec": 1.0,
"proof_v2": 0.35833333333333334,
"\u0394": 0.6416666666666666
},
{
"task": "snli/must_be_true",
"n": 120,
"LightDec": 0.9916666666666667,
"proof_v2": 0.925,
"\u0394": 0.06666666666666665
},
{
"task": "snli/nli",
"n": 120,
"LightDec": 0.9083333333333333,
"proof_v2": 0.3416666666666667,
"\u0394": 0.5666666666666667
},
{
"task": "sst5/score",
"n": 120,
"LightDec": 0.38333333333333336,
"proof_v2": 0.2833333333333333,
"\u0394": 0.10000000000000003
},
{
"task": "tev1_test/ag_news",
"n": 120,
"LightDec": 0.925,
"proof_v2": 0.425,
"\u0394": 0.5
},
{
"task": "tev1_test/banking77",
"n": 120,
"LightDec": 0.7166666666666667,
"proof_v2": 0.55,
"\u0394": 0.16666666666666663
},
{
"task": "tev1_test/boolq",
"n": 120,
"LightDec": 0.8416666666666667,
"proof_v2": 0.5583333333333333,
"\u0394": 0.2833333333333333
},
{
"task": "tev1_test/mnli",
"n": 120,
"LightDec": 0.75,
"proof_v2": 0.39166666666666666,
"\u0394": 0.35833333333333334
},
{
"task": "tev1_test/policy",
"n": 120,
"LightDec": 0.5,
"proof_v2": 0.4083333333333333,
"\u0394": 0.09166666666666667
},
{
"task": "tev1_test/routing",
"n": 120,
"LightDec": 0.44166666666666665,
"proof_v2": 0.2916666666666667,
"\u0394": 0.14999999999999997
},
{
"task": "tev1_test/sst5",
"n": 120,
"LightDec": 0.39166666666666666,
"proof_v2": 0.275,
"\u0394": 0.11666666666666664
},
{
"task": "triage/support_email",
"n": 120,
"LightDec": 0.9583333333333334,
"proof_v2": 0.39166666666666666,
"\u0394": 0.5666666666666667
},
{
"task": "typed_decisions/agent_trace_observability",
"n": 120,
"LightDec": 0.8083333333333333,
"proof_v2": 0.3333333333333333,
"\u0394": 0.47500000000000003
},
{
"task": "typed_decisions/customer_service",
"n": 120,
"LightDec": 0.7416666666666667,
"proof_v2": 0.2833333333333333,
"\u0394": 0.45833333333333337
},
{
"task": "typed_decisions/invoice_processing",
"n": 120,
"LightDec": 0.8166666666666667,
"proof_v2": 0.475,
"\u0394": 0.3416666666666667
},
{
"task": "typed_decisions/security_incidents",
"n": 120,
"LightDec": 0.7583333333333333,
"proof_v2": 0.5,
"\u0394": 0.2583333333333333
},
{
"task": "winogrande/blank",
"n": 120,
"LightDec": 0.6916666666666667,
"proof_v2": 0.5583333333333333,
"\u0394": 0.1333333333333333
},
{
"task": "yelp/score",
"n": 120,
"LightDec": 0.625,
"proof_v2": 0.325,
"\u0394": 0.3
}
],
"head_to_head_error": null,
"v3_benchmarks": {
"tev1": {
"per_task": [
{
"task": "tev1_test/ag_news",
"n": 150,
"acc": 0.92,
"overlaps_LightDec_training_source": true
},
{
"task": "tev1_test/banking77",
"n": 200,
"acc": 0.72,
"overlaps_LightDec_training_source": false
},
{
"task": "tev1_test/boolq",
"n": 200,
"acc": 0.835,
"overlaps_LightDec_training_source": true
},
{
"task": "tev1_test/mnli",
"n": 300,
"acc": 0.7233333333333334,
"overlaps_LightDec_training_source": true
},
{
"task": "tev1_test/policy",
"n": 1200,
"acc": 0.5291666666666667,
"overlaps_LightDec_training_source": false
},
{
"task": "tev1_test/routing",
"n": 600,
"acc": 0.4583333333333333,
"overlaps_LightDec_training_source": false
},
{
"task": "tev1_test/sst5",
"n": 150,
"acc": 0.3933333333333333,
"overlaps_LightDec_training_source": false
}
],
"all": 0.5839285714285715,
"no_overlap": 0.5176744186046511,
"n": 2800,
"published_tev1_4b": {
"main decisions (1,000)": 0.88,
"policy transfer (300)": 1.0
}
},
"triage": {
"emails": 101,
"per_question": {
"topic": 0.9900990099009901,
"refund": 1.0,
"breakage": 0.9900990099009901,
"anger": 0.9207920792079208,
"judgment": 0.8217821782178217
},
"pile_accuracy": 0.9405940594059405
},
"long_context": {
"max_len": 2048,
"per_task": [
{
"task": "long/contract_clause",
"n": 150,
"acc": 1.0,
"chance": 0.19999999999999996
},
{
"task": "long/email_thread",
"n": 150,
"acc": 1.0,
"chance": 0.25
},
{
"task": "long/service_log",
"n": 150,
"acc": 1.0,
"chance": 0.19999999999999996
}
]
}
},
"architecture": "FalconDec",
"latency": {
"gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
"calls": [
{
"device": "cuda",
"weights": "fp16",
"questions": 1,
"p50_ms": 10.05,
"p95_ms": 10.43
},
{
"device": "cuda",
"weights": "fp16",
"questions": 5,
"p50_ms": 11.32,
"p95_ms": 11.41
},
{
"device": "cuda",
"weights": "fp16",
"questions": 10,
"p50_ms": 12.57,
"p95_ms": 12.69
},
{
"device": "cpu fp32",
"weights": "int8 file",
"questions": 1,
"p50_ms": 48.4,
"p95_ms": 48.8
}
],
"throughput_per_s": 2588.8502560238694
},
"env": {
"torch": "2.9.0+cu130",
"transformers": "5.17.0",
"python": "3.12.12",
"gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition"
}
}