Initial WebGPU demo

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/cats.jpg filter=lfs diff=lfs merge=lfs -text
+examples/voice.flac filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,10 +1,21 @@
 ---
-title: Open D1 Webgpu
-emoji: 🏢
-colorFrom: red
-colorTo: yellow
+title: Open d1 WebGPU
+emoji: ⚖️
+colorFrom: yellow
+colorTo: red
 sdk: static
 pinned: false
+license: other
+short_description: Liquid AI d1 decision models running in the browser
+models:
+  - LiquidAI/d1-omni-600M
+  - LiquidAI/d1-3B
+  - onnx-community/d1-omni-600M-ONNX
+  - onnx-community/d1-3B-ONNX
+custom_headers:
+  cross-origin-embedder-policy: require-corp
+  cross-origin-opener-policy: same-origin
+  cross-origin-resource-policy: cross-origin
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Liquid AI's d1 decision models (d1-omni-600M and d1-3B) running locally in the browser on WebGPU, with a WASM fallback.

app.js

@@ -0,0 +1,300 @@
+const $ = (id) => document.getElementById(id);
+const TYPES = ["choice", "noul", "score"];
+
+const SUPPORT = [
+  ["refund", "noul", "Is the customer asking for a refund?", ""],
+  ["team", "choice", "Which team should handle this?", "billing: Charges, refunds, invoices | technical: App or site faults | fraud: Suspected unauthorised use"],
+  ["urgency", "score", "How urgent is this?", "Can wait | Today | Blocking the customer now"],
+];
+const CATS = [
+  ["cats", "choice", "How many cats are there?", "one: One | two: Two | more: Three or more"],
+  ["remote", "noul", "Is there a remote control in the picture?", ""],
+  ["mood", "choice", "What are the cats doing?", "sleeping: Lying down, resting or asleep | playing: Playing or fighting | eating: Eating or drinking"],
+];
+const VOICE = [
+  ["topic", "choice", "What is the speaker talking about?", "food: Food and meals | travel: Travel and transport | weather: The weather"],
+  ["question", "noul", "Is the speaker asking a question?", ""],
+];
+const MODERATION = [
+  ["toxic", "noul", "Is this comment insulting or abusive?", ""],
+  ["action", "choice", "What should a moderator do?", "keep: Leave it up | warn: Leave it, warn the author | remove: Take it down"],
+];
+const JUDGE = [
+  ["correct", "noul", "Does the answer correctly answer the question?", ""],
+  ["quality", "score", "Rate the answer's helpfulness.", "Useless | Poor | Okay | Good | Excellent"],
+];
+const EXAMPLES = [
+  { label: "Support ticket", model: "omni", state: "I was charged twice this month, please refund one of them.", q: SUPPORT },
+  { label: "Two cats", model: "omni", state: "", image: "examples/cats.jpg", q: CATS },
+  { label: "Voice note", model: "omni", state: "Voice note from a user.", audio: "examples/voice.flac", q: VOICE },
+  { label: "Moderation", model: "omni", state: "You are honestly the dumbest person on this forum, log off.", q: MODERATION },
+  { label: "LLM judge", model: "3b", state: '{"question": "What is the boiling point of water at sea level in Fahrenheit?", "answer": "Water boils at 212 °F at sea level."}', q: JUDGE },
+  { label: "Support ticket (3B)", model: "3b", state: "I was charged twice this month, please refund one of them.", q: SUPPORT },
+];
+
+// ------------------------------------------------------------------ state
+
+let model = "omni";
+let imageBlob = null, audioSamples = null, audioBlobUrl = null;
+const workers = {}, ready = {}, devices = {};
+let nextId = 1;
+const pending = new Map();
+
+// ------------------------------------------------------------------ questions editor
+
+function addRow([name, type, text, opts] = ["", "choice", "", ""]) {
+  const row = document.createElement("div");
+  row.className = "qrow";
+  row.innerHTML = `
+    <input type="text" class="qname" placeholder="name" aria-label="name" />
+    <select class="qtype" aria-label="type">${TYPES.map((t) => `<option>${t}</option>`).join("")}</select>
+    <input type="text" class="qtext" placeholder="question" aria-label="question" />
+    <input type="text" class="qopts" placeholder="options" aria-label="options" />
+    <button class="del" type="button" aria-label="Remove question">×</button>`;
+  row.querySelector(".qname").value = name;
+  row.querySelector(".qtype").value = type;
+  row.querySelector(".qtext").value = text;
+  row.querySelector(".qopts").value = opts;
+  row.querySelector(".del").onclick = () => row.remove();
+  $("questions").appendChild(row);
+}
+function setRows(rows) { $("questions").innerHTML = ""; rows.forEach(addRow); }
+
+function readQuestions() {
+  const out = [], seen = new Set();
+  [...$("questions").children].forEach((row, i) => {
+    let name = row.querySelector(".qname").value.trim();
+    const type = row.querySelector(".qtype").value;
+    const text = row.querySelector(".qtext").value.trim();
+    const parts = row.querySelector(".qopts").value.split("|").map((s) => s.trim()).filter(Boolean);
+    if (!name && !text) return;
+    name = name || `q${i + 1}`;
+    if (!text) throw new Error(`Question '${name}' has no question text.`);
+    if (seen.has(name)) throw new Error(`Two questions are named '${name}'.`);
+    seen.add(name);
+    const q = { name, type, instructions: text };
+    if (type === "choice") {
+      q.criteria = {};
+      for (const p of parts) {
+        const at = p.indexOf(":");
+        const key = (at < 0 ? p : p.slice(0, at)).trim();
+        const desc = at < 0 ? "" : p.slice(at + 1).trim();
+        q.criteria[key] = desc || key;
+      }
+      if (Object.keys(q.criteria).length < 2) throw new Error(`Question '${name}': a choice needs at least two options, e.g. a: ... | b: ...`);
+    } else if (type === "score") {
+      if (parts.length < 2 || parts.length > 10) throw new Error(`Question '${name}': a score needs 2 to 10 levels, lowest first, split by |.`);
+      q.criteria = parts;
+    } else if (parts.length === 2) {
+      q.criteria = { true: parts[0], false: parts[1] };
+    } else q.criteria = null;
+    out.push(q);
+  });
+  if (!out.length) throw new Error("Add at least one question.");
+  return out;
+}
+
+function parseState(text) {
+  text = text.trim();
+  if (!text) return null;
+  if (text[0] === "{" || text[0] === "[") { try { return JSON.parse(text); } catch {} }
+  return text;
+}
+
+// ------------------------------------------------------------------ media
+
+function setImage(blob) {
+  imageBlob = blob;
+  const img = $("imagePreview");
+  if (blob) { img.src = URL.createObjectURL(blob); img.hidden = false; $("imageHint").hidden = true; $("imageClear").hidden = false; }
+  else { img.hidden = true; img.removeAttribute("src"); $("imageHint").hidden = false; $("imageClear").hidden = true; }
+}
+
+async function decodeAudio(blob) {
+  const ctx = new AudioContext({ sampleRate: 16000 });
+  try {
+    const buf = await ctx.decodeAudioData(await blob.arrayBuffer());
+    const n = buf.length, mono = new Float32Array(n);
+    for (let c = 0; c < buf.numberOfChannels; c++) {
+      const ch = buf.getChannelData(c);
+      for (let i = 0; i < n; i++) mono[i] += ch[i] / buf.numberOfChannels;
+    }
+    return mono;
+  } finally { ctx.close(); }
+}
+
+async function setAudio(blob) {
+  if (audioBlobUrl) URL.revokeObjectURL(audioBlobUrl);
+  audioBlobUrl = null; audioSamples = null;
+  const el = $("audioPreview");
+  if (!blob) { el.hidden = true; el.removeAttribute("src"); $("audioHint").hidden = false; $("audioClear").hidden = true; return; }
+  audioSamples = await decodeAudio(blob);
+  audioBlobUrl = URL.createObjectURL(blob);
+  el.src = audioBlobUrl; el.hidden = false; $("audioHint").hidden = true; $("audioClear").hidden = false;
+}
+
+function wireDrop(drop, input, onFile) {
+  drop.addEventListener("click", (e) => { if (e.target === drop || e.target.classList.contains("hint")) input.click(); });
+  drop.addEventListener("keydown", (e) => { if (e.key === "Enter" || e.key === " ") { e.preventDefault(); input.click(); } });
+  input.addEventListener("change", () => input.files[0] && onFile(input.files[0]));
+  drop.addEventListener("dragover", (e) => { e.preventDefault(); drop.classList.add("over"); });
+  drop.addEventListener("dragleave", () => drop.classList.remove("over"));
+  drop.addEventListener("drop", (e) => { e.preventDefault(); drop.classList.remove("over"); const f = e.dataTransfer.files[0]; if (f) onFile(f); });
+}
+
+let recorder = null;
+async function toggleRecord() {
+  const btn = $("recordBtn");
+  if (recorder) { recorder.stop(); return; }
+  const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+  const chunks = [];
+  recorder = new MediaRecorder(stream);
+  recorder.ondataavailable = (e) => chunks.push(e.data);
+  recorder.onstop = async () => {
+    stream.getTracks().forEach((t) => t.stop());
+    recorder = null; btn.textContent = "Record"; btn.classList.remove("rec");
+    await setAudio(new Blob(chunks, { type: chunks[0]?.type || "audio/webm" }));
+  };
+  recorder.start();
+  btn.textContent = "Stop"; btn.classList.add("rec");
+  setTimeout(() => recorder && recorder.stop(), 30000);
+}
+
+// ------------------------------------------------------------------ workers
+
+function status(text, err = false) { const s = $("status"); s.textContent = text; s.classList.toggle("err", err); }
+function bar(frac) {
+  const b = $("bar");
+  if (frac === null) { b.hidden = true; return; }
+  b.hidden = false; b.firstElementChild.style.width = `${Math.round(frac * 100)}%`;
+}
+const gb = (n) => (n / 1e9).toFixed(2) + " GB";
+
+function getWorker(which) {
+  if (workers[which]) return ready[which];
+  // One model on the GPU at a time.
+  for (const k of Object.keys(workers)) if (k !== which) { workers[k].terminate(); delete workers[k]; delete ready[k]; }
+  const w = new Worker(which === "3b" ? "worker-3b.js" : "worker-omni.js", { type: "module" });
+  workers[which] = w;
+  const fileProgress = {};
+  ready[which] = new Promise((resolve, reject) => {
+    w.onmessage = ({ data }) => {
+      if (data.type === "progress") {
+        let loaded = data.loaded, total = data.total;
+        if (!data.aggregate) {
+          fileProgress[data.file] = [data.loaded, data.total];
+          loaded = 0; total = 0;
+          for (const [l, t] of Object.values(fileProgress)) { loaded += l; total += t; }
+        }
+        status(`Downloading ${which === "3b" ? "d1-3B" : "d1-omni-600M"} (${data.label}) · ${gb(loaded)}${total ? " / " + gb(total) : ""}`);
+        bar(total ? loaded / total : 0);
+      } else if (data.type === "ready") {
+        devices[which] = data.device; bar(null); resolve(w);
+      } else if (data.type === "result" || data.type === "error") {
+        const p = pending.get(data.id);
+        if (!p) { if (data.type === "error") reject(new Error(data.message)); return; }
+        pending.delete(data.id);
+        data.type === "result" ? p.resolve(data) : p.reject(new Error(data.message));
+      }
+    };
+    w.onerror = (e) => reject(new Error(e.message || "worker failed to start"));
+  });
+  status(`Loading ${which === "3b" ? "d1-3B" : "d1-omni-600M"}…`);
+  w.postMessage({ type: "init" });
+  return ready[which];
+}
+
+async function run(which, payload) {
+  const w = await getWorker(which);
+  const id = nextId++;
+  return new Promise((resolve, reject) => { pending.set(id, { resolve, reject }); w.postMessage({ type: "decide", id, ...payload }); });
+}
+
+// ------------------------------------------------------------------ results
+
+const esc = (s) => String(s).replace(/[&<>"']/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }[c]));
+
+function renderResult(questions, res) {
+  const cards = questions.map((q) => {
+    const a = res.answers[q.name];
+    let head, sub, rows, top;
+    if (a.type === "noul") {
+      head = a.noul >= 0.5 ? "yes" : "no"; top = head;
+      sub = `${(100 * Math.max(a.noul, 1 - a.noul)).toFixed(1)}% confident`;
+      rows = [["yes", a.noul], ["no", 1 - a.noul]];
+    } else if (a.type === "choice") {
+      head = a.choice; top = head; sub = `${(100 * a.confidence).toFixed(1)}% confident`;
+      rows = Object.entries(a.probabilities);
+    } else {
+      head = a.score.toFixed(2); sub = "expected level";
+      const best = Object.keys(a.probabilities).reduce((x, y) => (a.probabilities[y] > a.probabilities[x] ? y : x));
+      rows = Object.entries(a.probabilities).map(([k, v]) => [`${k} · ${a.legend[k]}`, v]);
+      top = `${best} · ${a.legend[best]}`;
+    }
+    const bars = rows.map(([lab, v]) => `<div class="row${lab === top ? " top" : ""}"><span class="lab">${esc(lab)}</span><span class="track"><span class="fill" style="width:${(100 * v).toFixed(1)}%"></span></span><span class="pct">${(100 * v).toFixed(1)}%</span></div>`).join("");
+    return `<div class="card"><div class="head"><span class="name">${esc(q.name)}</span><span class="q">${esc(q.instructions)}</span></div><div class="ans">${esc(head)}<span class="sub">${sub}</span></div>${bars}</div>`;
+  }).join("");
+  $("result").className = "";
+  $("result").innerHTML = `<div class="cards">${cards}</div><div class="took">took ${(res.ms / 1000).toFixed(2)} s</div>`;
+}
+
+// ------------------------------------------------------------------ actions
+
+function setModel(m) {
+  model = m;
+  document.querySelectorAll("#model button").forEach((b) => b.setAttribute("aria-checked", String(b.dataset.model === m)));
+  const omni = m === "omni";
+  $("audioDrop").hidden = !omni;
+  document.querySelector(".media").classList.toggle("single", !omni);
+  if (!omni) setAudio(null);
+}
+
+async function decide() {
+  const btn = $("run");
+  let questions;
+  try { questions = readQuestions(); } catch (e) { status(e.message, true); return; }
+  const state = parseState($("state").value);
+  const audio = model === "omni" ? audioSamples : null;
+  if (imageBlob && audio) { status("d1-omni takes an image or a voice clip in one request, not both. Remove one.", true); return; }
+  if (state === null && !imageBlob && !audio) { status("Give the model a state, an image or a voice clip.", true); return; }
+  btn.disabled = true;
+  try {
+    const res = await run(model, { state, questions, image: imageBlob, audio });
+    renderResult(questions, res);
+    status(`Ran on ${devices[model] === "webgpu" ? "WebGPU" : "WASM"}.`);
+    return res;
+  } catch (e) {
+    status(e.message, true); bar(null);
+  } finally { btn.disabled = false; }
+}
+
+async function loadExample(ex) {
+  setModel(ex.model);
+  $("state").value = ex.state;
+  setRows(ex.q);
+  setImage(ex.image ? await (await fetch(ex.image)).blob() : null);
+  await setAudio(ex.audio ? await (await fetch(ex.audio)).blob() : null);
+}
+
+// ------------------------------------------------------------------ boot
+
+document.querySelectorAll("#model button").forEach((b) => (b.onclick = () => setModel(b.dataset.model)));
+wireDrop($("imageDrop"), $("imageFile"), (f) => setImage(f));
+wireDrop($("audioDrop"), $("audioFile"), (f) => setAudio(f).catch((e) => status(`Could not read that clip: ${e.message}`, true)));
+$("imageClear").onclick = (e) => { e.stopPropagation(); setImage(null); };
+$("audioClear").onclick = (e) => { e.stopPropagation(); setAudio(null); };
+$("recordBtn").onclick = (e) => { e.stopPropagation(); toggleRecord().catch((err) => status(err.message, true)); };
+$("addQ").onclick = () => addRow();
+$("run").onclick = decide;
+EXAMPLES.forEach((ex) => {
+  const b = document.createElement("button");
+  b.type = "button"; b.textContent = ex.label; b.onclick = () => loadExample(ex);
+  $("examples").appendChild(b);
+});
+$("state").value = EXAMPLES[0].state;
+setRows(SUPPORT);
+setModel("omni");
+if (!navigator.gpu) status("WebGPU is not available here, so the models run on WASM (slower).");
+
+// Test hook: load an example and run it, resolving with the raw answers.
+window.d1 = { examples: EXAMPLES, loadExample, decide, setModel };

examples/voice.flac

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30885601173f96b0d8ddd020dc959b055c6c1582b85a33e3fcab8c4b08ed94c2
+size 183318

index.html

@@ -1,19 +1,85 @@
 <!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <title>Open d1 WebGPU</title>
+  <link rel="preconnect" href="https://fonts.googleapis.com" />
+  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
+  <link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=IBM+Plex+Sans:wght@400;500;600&display=swap" rel="stylesheet" />
+  <link rel="stylesheet" href="style.css" />
+</head>
+<body>
+  <main>
+    <header>
+      <h1>Open d1 <span class="tag">in your browser</span></h1>
+      <p class="lede">Liquid AI's decision models: a state and some typed questions in, calibrated answers out of one forward pass. Runs locally on WebGPU, with a WASM fallback.</p>
+      <p class="links">
+        <a href="https://huggingface.co/LiquidAI/d1-omni-600M" target="_blank" rel="noopener">d1-omni-600M</a> (text + image or voice) ·
+        <a href="https://huggingface.co/LiquidAI/d1-3B" target="_blank" rel="noopener">d1-3B</a> (text + image) ·
+        ONNX by <a href="https://huggingface.co/onnx-community" target="_blank" rel="noopener">onnx-community</a>
+      </p>
+    </header>
+
+    <div class="grid">
+      <section class="panel" id="inputs">
+        <div class="field">
+          <span class="label">Model</span>
+          <div class="seg" role="radiogroup" id="model">
+            <button role="radio" data-model="omni" aria-checked="true">d1-omni-600M <small>0.4 GB</small></button>
+            <button role="radio" data-model="3b" aria-checked="false">d1-3B <small>2.4 GB</small></button>
+          </div>
+        </div>
+
+        <label class="field">
+          <span class="label">State</span>
+          <textarea id="state" rows="4" placeholder="Text or JSON the questions are about. Can be empty when an image or clip is the whole state."></textarea>
+        </label>
+
+        <div class="media">
+          <div class="drop" id="imageDrop" tabindex="0">
+            <input type="file" id="imageFile" accept="image/*" hidden />
+            <img id="imagePreview" alt="" hidden />
+            <span class="hint" id="imageHint">Image · drop or click</span>
+            <button class="clear" id="imageClear" hidden aria-label="Remove image">×</button>
+          </div>
+          <div class="drop" id="audioDrop" tabindex="0">
+            <input type="file" id="audioFile" accept="audio/*" hidden />
+            <span class="hint" id="audioHint">Voice clip · drop or click</span>
+            <audio id="audioPreview" controls hidden></audio>
+            <div class="audio-actions">
+              <button class="small" id="recordBtn" type="button">Record</button>
+              <button class="clear" id="audioClear" hidden aria-label="Remove clip">×</button>
+            </div>
+          </div>
+        </div>
+
+        <div class="field">
+          <span class="label">Questions</span>
+          <div id="questions"></div>
+          <div class="qfoot">
+            <button class="small" id="addQ" type="button">Add question</button>
+            <span class="help"><b>choice</b>: <code>name: description | name: description</code> · <b>score</b>: levels lowest first, split by <code>|</code> · <b>noul</b> (yes/no): optional <code>yes means | no means</code></span>
+          </div>
+        </div>
+
+        <button class="primary" id="run" type="button">Decide</button>
+        <div class="status" id="status" aria-live="polite"></div>
+        <div class="bar" id="bar" hidden><span></span></div>
+      </section>
+
+      <section class="panel" id="outputs">
+        <div id="result" class="placeholder">Answers appear here.</div>
+      </section>
+    </div>
+
+    <section class="examples">
+      <span class="label">Examples</span>
+      <div id="examples"></div>
+    </section>
+
+    <footer><a href="https://x.com/realmrfakename" target="_blank" rel="noopener">Twitter / X</a></footer>
+  </main>
+  <script type="module" src="app.js"></script>
+</body>
 </html>

prompt.js

@@ -0,0 +1,163 @@
+// Ports of the two model repos' prompt.py files: questions in, token ids out, probabilities back.
+
+// Python's json.dumps(x, ensure_ascii=False) with its default ", " / ": " separators.
+export function pyDumps(x) {
+  if (x === null || x === undefined) return "null";
+  if (typeof x === "string") return JSON.stringify(x);
+  if (typeof x === "number" || typeof x === "boolean") return JSON.stringify(x);
+  if (Array.isArray(x)) return "[" + x.map(pyDumps).join(", ") + "]";
+  return "{" + Object.entries(x).map(([k, v]) => JSON.stringify(k) + ": " + pyDumps(v)).join(", ") + "}";
+}
+
+// ------------------------------------------------------------------ d1-omni-600M (prompt.py)
+
+export const QTYPES = { choice: 0, score: 1, noul: 2 };
+const DELIM = { state: "<|reserved_7|>", q: "<|reserved_8|>", opt: "<|reserved_9|>", opt_end: "<|reserved_10|>", decide: "<|reserved_11|>" };
+const MARKER = "<|mask|>";
+
+export const escapeSpecial = (t) => t.replace(/<\|([A-Za-z0-9_]+)\|>/g, "<¦$1¦>");
+const criterion = (v) => (typeof v === "string" ? v : pyDumps(v));
+export const nOptions = (q) => (q.type === "noul" ? 2 : q.type === "choice" ? Object.keys(q.criteria).length : q.criteria.length);
+
+function renderOptions(q, nounDefault, audio) {
+  if (q.type === "choice") {
+    const entries = Object.entries(q.criteria);
+    if (audio) return entries.map(([k, v], i) => `option_${String(i).padStart(3, "0")}: ${criterion(v == null || v === "" ? k : v)}`);
+    return entries.map(([k, v]) => (v == null || v === "" ? k : `${k}: ${criterion(v)}`));
+  }
+  if (q.type === "score") return q.criteria.map((c, i) => `level ${i}: ${criterion(c)}`);
+  if (audio) return ["false: no", "true: yes"];
+  const crit = q.criteria || nounDefault || {};
+  const f = crit.false ?? crit.no, t = crit.true ?? crit.yes;
+  return ["false: " + (f != null && f !== "" ? criterion(f) : "no, the statement does not hold"),
+          "true: " + (t != null && t !== "" ? criterion(t) : "yes, the statement holds")];
+}
+
+export function omniEncode(tok, state, q, maxLen, nounDefault = null, audio = false, perOption = 24) {
+  const id = (s) => {
+    const ids = tok.encode(s, { add_special_tokens: false });
+    if (ids.length !== 1) throw new Error(`not a single token: ${s}`);
+    return ids[0];
+  };
+  const enc = (s) => tok.encode(escapeSpecial(s), { add_special_tokens: false });
+  const opts = renderOptions(q, nounDefault, audio);
+  const budget = Math.max(96, Math.min(opts.length * perOption + 32, Math.floor(maxLen / 2)));
+  const per = Math.max(2, Math.floor((budget - 3 * opts.length) / opts.length));
+  let question = [id(DELIM.q), ...enc(q.instructions)].slice(0, Math.max(16, budget));
+  let markers = [];
+  for (const text of opts) {
+    markers.push(question.length + 1);
+    question = question.concat([id(DELIM.opt), id(MARKER)], enc(" " + text).slice(0, per), [id(DELIM.opt_end)]);
+  }
+  question.push(id(DELIM.decide));
+  const room = Math.max(0, maxLen - question.length - 2);
+  const stateIds = [id(DELIM.state), ...enc(typeof state === "string" ? state : pyDumps(state)).slice(0, room)];
+  const ids = [tok.bos_token_id ?? 1, ...stateIds, ...question].slice(0, maxLen);
+  markers = markers.map((m) => m + 1 + stateIds.length);
+  if (markers[markers.length - 1] >= maxLen) throw new Error("the options do not fit in the context");
+  return { ids, markers };
+}
+
+export function temperatureKey(q) {
+  const k = nOptions(q);
+  return `${q.type}:` + (k <= 2 ? "2" : k <= 5 ? "3-5" : k <= 10 ? "6-10" : "11+");
+}
+
+// ------------------------------------------------------------------ d1-3B (prompt.py)
+
+const FALLBACK_POOL = (() => {
+  const out = [];
+  for (let c = 65; c <= 90; c++) out.push(String.fromCharCode(c));
+  for (let i = 0; i < 100; i++) out.push(String(i).padStart(2, "0"));
+  for (let c = 97; c <= 122; c++) out.push(String.fromCharCode(c));
+  for (let i = 0; i < 200; i++) out.push(`#${i}`);
+  for (let a = 65; a <= 90; a++) for (let b = 65; b <= 90; b++) out.push(String.fromCharCode(a) + String.fromCharCode(b));
+  return out;
+})();
+
+function optionCodes(labels) {
+  const labs = labels.map((x) => String(x).trim());
+  if (labs.length && labs.every((k) => k.length === 1 && /\p{L}/u.test(k))) return labs;
+  if (labs.length <= 26) return labs.map((_, i) => String.fromCharCode(65 + i));
+  return labs.map((_, i) => String(i).padStart(2, "0"));
+}
+
+function aliases(tok, labels) {
+  const used = new Set(), out = [];
+  const take = (raw) => {
+    const e = tok.encode(raw, { add_special_tokens: false });
+    if (e.length !== 1 || used.has(e[0])) return false;
+    out.push([raw, e[0]]);
+    used.add(e[0]);
+    return true;
+  };
+  for (const code of optionCodes(labels)) {
+    if (take(code)) continue;
+    if (!FALLBACK_POOL.some(take)) throw new Error(`no single-token alias left for ${labels.length} options`);
+  }
+  return out;
+}
+
+function singleIds(tok, texts) {
+  const out = [], seen = new Set();
+  for (const t of texts) {
+    const e = tok.encode(t, { add_special_tokens: false });
+    if (e.length === 1 && !seen.has(e[0])) { out.push(e[0]); seen.add(e[0]); }
+  }
+  return out;
+}
+
+export function readoutIds3b(tok, q) {
+  if (q.type === "noul") return [singleIds(tok, ["yes", "Yes", "YES"]), singleIds(tok, ["no", "No", "NO"])];
+  if (q.type === "score") return q.criteria.map((_, i) => singleIds(tok, [String(i)]));
+  return aliases(tok, Object.keys(q.criteria)).map(([code, tid]) => [tid, ...singleIds(tok, [` ${code}`]).filter((i) => i !== tid)]);
+}
+
+function questionBlock3b(tok, q) {
+  if (q.type === "choice") {
+    const labels = Object.keys(q.criteria);
+    const codes = aliases(tok, labels);
+    const lines = labels.map((lab, i) => `${codes[i][0]} ${q.criteria[lab] || lab.replace(/_/g, " ")}`).join("\n");
+    return `${q.instructions}\n\nOptions:\n${lines}\n\nReply with the option code only.`;
+  }
+  if (q.type === "noul") {
+    const extra = q.criteria ? `\nYes: ${q.criteria.true}\nNo: ${q.criteria.false}` : "";
+    return `${q.instructions}${extra}\n\nReply with yes or no only.`;
+  }
+  const legend = q.criteria.map((name, i) => `${i} ${name}`).join("\n");
+  return `${q.instructions}\n\n${legend}\n\nReply with a single digit 0-${q.criteria.length - 1} only.`;
+}
+
+export function render3b(tok, state, q, imageMarkup = "") {
+  const bos = "<|startoftext|>";
+  let body = "";
+  if (state !== null && state !== undefined) {
+    const block = typeof state === "string" ? `${state}\n\n` : JSON.stringify(state, null, 2) + "\n\n";
+    body = `${block}\nQUESTION:\n`;
+  }
+  return `${bos}<|im_start|>user\n${imageMarkup}${body}${questionBlock3b(tok, q)}<|im_end|>\n<|im_start|>assistant\n`;
+}
+
+// ------------------------------------------------------------------ answers (api.py / prompt.py `answer`)
+
+export function softmax(z) {
+  const m = Math.max(...z);
+  const e = z.map((v) => Math.exp(v - m));
+  const s = e.reduce((a, b) => a + b, 0);
+  return e.map((v) => v / s);
+}
+
+export function answer(q, probs) {
+  if (q.type === "noul") return { type: "noul", noul: probs[0] };
+  let best = 0;
+  probs.forEach((p, i) => { if (p > probs[best]) best = i; });
+  if (q.type === "choice") {
+    const names = Object.keys(q.criteria);
+    return { type: "choice", choice: names[best], confidence: probs[best], probabilities: Object.fromEntries(names.map((n, i) => [n, probs[i]])) };
+  }
+  return {
+    type: "score", score: probs.reduce((a, p, i) => a + i * p, 0), confidence: probs[best],
+    probabilities: Object.fromEntries(probs.map((p, i) => [String(i), p])),
+    legend: Object.fromEntries(q.criteria.map((c, i) => [String(i), criterion(c)])),
+  };
+}

style.css

@@ -1,28 +1,134 @@
-body {
-	padding: 2rem;
-	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
+:root {
+  --bg: #f4f1ea;
+  --panel: #fbf9f4;
+  --ink: #1d1b17;
+  --muted: #6d675c;
+  --line: #ddd6c8;
+  --track: #e8e2d5;
+  --accent: #b4532a;
+  --accent-ink: #ffffff;
+  --bar-dim: #9c968a;
+  --field: #ffffff;
+  --radius: 6px;
+  --h: 34px;
+}
+@media (prefers-color-scheme: dark) {
+  :root:not([data-theme="light"]) {
+    --bg: #161512;
+    --panel: #1e1c18;
+    --ink: #ece7dc;
+    --muted: #9b9486;
+    --line: #34312a;
+    --track: #2c2924;
+    --accent: #d9774a;
+    --accent-ink: #161512;
+    --bar-dim: #6f695e;
+    --field: #191815;
+  }
+}
+:root[data-theme="dark"] {
+  --bg: #161512; --panel: #1e1c18; --ink: #ece7dc; --muted: #9b9486; --line: #34312a; --track: #2c2924;
+  --accent: #d9774a; --accent-ink: #161512; --bar-dim: #6f695e; --field: #191815;
 }
 
-h1 {
-	font-size: 16px;
-	margin-top: 0;
+* { box-sizing: border-box; }
+html, body { margin: 0; }
+body {
+  background: var(--bg);
+  color: var(--ink);
+  font: 15px/1.45 "IBM Plex Sans", system-ui, sans-serif;
 }
+main { max-width: 1180px; margin: 0 auto; padding: 24px 16px 32px; }
+a { color: var(--accent); }
+code, .mono { font-family: "IBM Plex Mono", ui-monospace, monospace; font-size: 12.5px; }
 
-p {
-	color: rgb(107, 114, 128);
-	font-size: 15px;
-	margin-bottom: 10px;
-	margin-top: 5px;
+header h1 { font-size: 28px; font-weight: 600; margin: 0 0 6px; letter-spacing: -0.01em; }
+.tag { font-size: 13px; font-weight: 500; color: var(--muted); border: 1px solid var(--line); border-radius: 4px; padding: 2px 6px; vertical-align: middle; }
+.lede { margin: 0 0 4px; color: var(--ink); max-width: 760px; }
+.links { margin: 0 0 18px; color: var(--muted); font-size: 14px; }
+
+.grid { display: grid; grid-template-columns: minmax(0, 1.15fr) minmax(0, 1fr); gap: 16px; align-items: start; }
+@media (max-width: 860px) { .grid { grid-template-columns: 1fr; } }
+.panel { background: var(--panel); border: 1px solid var(--line); border-radius: var(--radius); padding: 14px; }
+
+.field { display: block; margin-bottom: 12px; }
+.label { display: block; font-size: 12px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.06em; color: var(--muted); margin-bottom: 6px; }
+
+textarea, input[type="text"], select {
+  width: 100%; font: inherit; color: var(--ink); background: var(--field);
+  border: 1px solid var(--line); border-radius: var(--radius); padding: 7px 9px;
 }
+textarea { resize: vertical; }
+textarea:focus, input:focus, select:focus, button:focus-visible, .drop:focus-visible { outline: 2px solid var(--accent); outline-offset: -1px; }
+
+.seg { display: inline-flex; border: 1px solid var(--line); border-radius: var(--radius); overflow: hidden; }
+.seg button { height: var(--h); padding: 0 14px; border: 0; background: var(--field); color: var(--ink); font: inherit; cursor: pointer; }
+.seg button + button { border-left: 1px solid var(--line); }
+.seg button[aria-checked="true"] { background: var(--accent); color: var(--accent-ink); }
+.seg small { opacity: 0.75; margin-left: 4px; }
 
-.card {
-	max-width: 620px;
-	margin: 0 auto;
-	padding: 16px;
-	border: 1px solid lightgray;
-	border-radius: 16px;
+.media { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; margin-bottom: 12px; }
+.media.single { grid-template-columns: 1fr; }
+@media (max-width: 520px) { .media { grid-template-columns: 1fr; } }
+.drop {
+  position: relative; min-height: 120px; border: 1px dashed var(--line); border-radius: var(--radius);
+  background: var(--field); display: flex; flex-direction: column; align-items: center; justify-content: center;
+  gap: 8px; padding: 8px; cursor: pointer; overflow: hidden;
 }
+.drop.over { border-color: var(--accent); }
+.drop img { max-width: 100%; max-height: 200px; border-radius: 4px; }
+.drop audio { width: 100%; }
+.hint { color: var(--muted); font-size: 13px; }
+.audio-actions { display: flex; gap: 6px; align-items: center; }
+.clear {
+  position: absolute; top: 6px; right: 6px; width: 26px; height: 26px; border-radius: 4px;
+  border: 1px solid var(--line); background: var(--panel); color: var(--ink); cursor: pointer; font-size: 16px; line-height: 1;
+}
+.audio-actions .clear { position: static; }
 
-.card p:last-child {
-	margin-bottom: 0;
+.qrow { display: grid; grid-template-columns: 90px 92px minmax(0, 1.2fr) minmax(0, 1.5fr) 30px; gap: 6px; margin-bottom: 6px; }
+.qrow input, .qrow select { height: var(--h); padding: 0 8px; }
+.qrow .del { height: var(--h); border: 1px solid var(--line); background: var(--field); color: var(--muted); border-radius: var(--radius); cursor: pointer; }
+@media (max-width: 640px) {
+  .qrow { grid-template-columns: 1fr 1fr 30px; }
+  .qrow .qtext, .qrow .qopts { grid-column: 1 / -1; }
+  .qrow .del { grid-row: 1; grid-column: 3; }
 }
+.qfoot { display: flex; gap: 10px; align-items: center; flex-wrap: wrap; margin-top: 4px; }
+.help { color: var(--muted); font-size: 12.5px; }
+
+button.small { height: 28px; padding: 0 10px; border: 1px solid var(--line); background: var(--field); color: var(--ink); border-radius: var(--radius); font: inherit; font-size: 13px; cursor: pointer; }
+button.small.rec { border-color: var(--accent); color: var(--accent); }
+button.primary { width: 100%; height: 40px; border: 0; border-radius: var(--radius); background: var(--accent); color: var(--accent-ink); font: inherit; font-weight: 600; cursor: pointer; }
+button.primary:disabled { opacity: 0.6; cursor: progress; }
+
+.status { min-height: 20px; margin-top: 8px; font-size: 13px; color: var(--muted); }
+.status.err { color: var(--accent); }
+.bar { height: 4px; background: var(--track); border-radius: 2px; overflow: hidden; margin-top: 4px; }
+.bar span { display: block; height: 100%; width: 0; background: var(--accent); transition: width 0.15s linear; }
+
+.placeholder { color: var(--muted); font-size: 14px; }
+.cards { display: flex; flex-direction: column; gap: 10px; }
+.card { border: 1px solid var(--line); border-radius: var(--radius); padding: 12px 14px; background: var(--field); }
+.card .head { display: flex; gap: 10px; align-items: baseline; flex-wrap: wrap; margin-bottom: 2px; }
+.card .name { font-family: "IBM Plex Mono", monospace; font-size: 12px; color: var(--muted); }
+.card .q { font-size: 14px; }
+.card .ans { font-size: 22px; font-weight: 600; color: var(--accent); margin: 2px 0 8px; }
+.card .sub { font-size: 13px; font-weight: 400; color: var(--muted); margin-left: 6px; }
+.row { display: grid; grid-template-columns: minmax(70px, 38%) 1fr 56px; gap: 10px; align-items: center; font-size: 13px; color: var(--muted); padding: 2px 0; }
+.row.top { color: var(--ink); font-weight: 600; }
+.row .lab { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+.row .track { height: 8px; background: var(--track); border-radius: 4px; overflow: hidden; }
+.row .fill { display: block; height: 100%; background: var(--bar-dim); }
+.row.top .fill { background: var(--accent); }
+.row .pct { text-align: right; font-variant-numeric: tabular-nums; }
+.took { font-size: 12px; color: var(--muted); margin-top: 8px; }
+
+.examples { margin-top: 16px; }
+#examples { display: flex; flex-wrap: wrap; gap: 6px; }
+#examples button { height: 30px; padding: 0 12px; border: 1px solid var(--line); background: var(--panel); color: var(--ink); border-radius: var(--radius); font: inherit; font-size: 13px; cursor: pointer; }
+#examples button:hover { border-color: var(--accent); }
+
+footer { margin-top: 24px; font-size: 13px; }
+
+[hidden] { display: none !important; }

vision.js

@@ -0,0 +1,116 @@
+// d1-omni's vision.py `layout` and `preprocess`: LFM2-VL smart resize, optional 512 px tiles + thumbnail,
+// RGB in [-1, 1], 16 px patches padded to 1024.
+const TILE = 512, PATCH = 16, MAX_PATCHES = 1024;
+
+export function layout(width, height) {
+  const factor = 32, maximum = 256 * 1024, minimum = 64 * 1024;
+  let h = Math.max(factor, Math.round(height / factor) * factor);
+  let w = Math.max(factor, Math.round(width / factor) * factor);
+  if (h * w > maximum) {
+    const beta = Math.sqrt((height * width) / maximum);
+    h = Math.max(factor, Math.floor(height / beta / factor) * factor);
+    w = Math.max(factor, Math.floor(width / beta / factor) * factor);
+  } else if (h * w < minimum) {
+    const beta = Math.sqrt(minimum / (height * width));
+    h = Math.ceil((height * beta) / factor) * factor;
+    w = Math.ceil((width * beta) / factor) * factor;
+  }
+  const large = Math.max(16, Math.round(height / factor) * factor) * Math.max(16, Math.round(width / factor) * factor) > maximum * 2;
+  let grid = [1, 1];
+  if (large) {
+    const set = new Map();
+    for (let n = 2; n <= 10; n++) for (let x = 1; x <= n; x++) for (let y = 1; y <= n; y++) if (x * y >= 2 && x * y <= 10) set.set(`${x},${y}`, [x, y]);
+    // Python sorts a set by area; ties keep the set's order, which only matters for exact ties in `diff`.
+    const ratios = [...set.values()].sort((a, b) => a[0] * a[1] - b[0] * b[1]);
+    let best = Infinity;
+    for (const r of ratios) {
+      const diff = Math.abs(width / height - r[0] / r[1]);
+      if (diff < best || (diff === best && width * height > 0.5 * TILE * TILE * r[0] * r[1])) { grid = r; best = diff; }
+    }
+  }
+  return { grid, thumbnail: [h, w], tiled: large };
+}
+
+// PIL / torchvision antialiased bilinear (triangle filter), uint8 in and out, horizontal pass then vertical.
+function weights(inSize, outSize) {
+  const scale = inSize / outSize, support = Math.max(scale, 1);
+  const out = [];
+  for (let i = 0; i < outSize; i++) {
+    const center = (i + 0.5) * scale;
+    const lo = Math.max(0, Math.floor(center - support + 0.5));
+    const hi = Math.min(inSize, Math.floor(center + support + 0.5));
+    const ws = [];
+    let total = 0;
+    for (let j = lo; j < hi; j++) {
+      const x = (j - center + 0.5) / support;
+      const w = Math.max(0, 1 - Math.abs(x));
+      ws.push(w); total += w;
+    }
+    out.push({ lo, ws: ws.map((w) => (total ? w / total : 0)) });
+  }
+  return out;
+}
+
+function resizeRGB(src, sw, sh, dw, dh) {
+  const wx = weights(sw, dw), wy = weights(sh, dh);
+  const mid = new Uint8ClampedArray(dw * sh * 3);
+  for (let y = 0; y < sh; y++) for (let x = 0; x < dw; x++) {
+    const { lo, ws } = wx[x];
+    for (let c = 0; c < 3; c++) {
+      let s = 0;
+      for (let k = 0; k < ws.length; k++) s += ws[k] * src[(y * sw + lo + k) * 3 + c];
+      mid[(y * dw + x) * 3 + c] = Math.round(s);
+    }
+  }
+  const out = new Uint8ClampedArray(dw * dh * 3);
+  for (let y = 0; y < dh; y++) {
+    const { lo, ws } = wy[y];
+    for (let x = 0; x < dw; x++) for (let c = 0; c < 3; c++) {
+      let s = 0;
+      for (let k = 0; k < ws.length; k++) s += ws[k] * mid[((lo + k) * dw + x) * 3 + c];
+      out[(y * dw + x) * 3 + c] = Math.round(s);
+    }
+  }
+  return out;
+}
+
+// rgb: Uint8 RGB (no alpha), width, height -> {pixel_values Float32 [n,1024,768], mask, shapes, prefix}
+export function preprocess(rgb, width, height) {
+  const plan = layout(width, height);
+  const crops = [];
+  if (plan.tiled) {
+    const [gw, gh] = plan.grid;
+    const big = resizeRGB(rgb, width, height, gw * TILE, gh * TILE);
+    for (let r = 0; r < gh; r++) for (let c = 0; c < gw; c++) {
+      const tile = new Uint8ClampedArray(TILE * TILE * 3);
+      for (let y = 0; y < TILE; y++) {
+        const off = ((r * TILE + y) * gw * TILE + c * TILE) * 3;
+        tile.set(big.subarray(off, off + TILE * 3), y * TILE * 3);
+      }
+      crops.push({ data: tile, h: TILE, w: TILE });
+    }
+  }
+  const [th, tw] = plan.thumbnail;
+  crops.push({ data: resizeRGB(rgb, width, height, tw, th), h: th, w: tw });
+  const n = crops.length, D = 3 * PATCH * PATCH;
+  const pixels = new Float32Array(n * MAX_PATCHES * D);
+  const mask = new BigInt64Array(n * MAX_PATCHES);
+  const shapes = new BigInt64Array(n * 2);
+  let prefix = 0;
+  crops.forEach((crop, i) => {
+    const ph = crop.h / PATCH, pw = crop.w / PATCH;
+    shapes[2 * i] = BigInt(ph); shapes[2 * i + 1] = BigInt(pw);
+    prefix += (ph * pw) / 4;
+    for (let py = 0; py < ph; py++) for (let px = 0; px < pw; px++) {
+      const p = py * pw + px;
+      mask[i * MAX_PATCHES + p] = 1n;
+      const base = (i * MAX_PATCHES + p) * D;
+      for (let y = 0; y < PATCH; y++) for (let x = 0; x < PATCH; x++) {
+        const s = ((py * PATCH + y) * crop.w + px * PATCH + x) * 3;
+        const d = base + (y * PATCH + x) * 3;
+        for (let c = 0; c < 3; c++) pixels[d + c] = (crop.data[s + c] - 127.5) / 127.5;
+      }
+    }
+  });
+  return { pixels, mask, shapes, n, prefix };
+}

worker-3b.js

@@ -0,0 +1,69 @@
+// d1-3B in the browser: Transformers.js (WebGPU, WASM fallback) over onnx-community/d1-3B-ONNX.
+import { AutoModelForImageTextToText, AutoProcessor, RawImage, env } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.3.1/dist/transformers.min.js";
+import { answer, readoutIds3b, render3b, softmax } from "./prompt.js";
+
+env.allowLocalModels = false;
+const REPO = "onnx-community/d1-3B-ONNX";
+const post = (m) => self.postMessage(m);
+let processor = null, model = null, tok = null, device = "wasm";
+
+async function init() {
+  if (navigator.gpu) {
+    try { if (await navigator.gpu.requestAdapter()) device = "webgpu"; } catch {}
+  }
+  const files = {};
+  const progress_callback = (p) => {
+    if (p.status === "progress" && p.file && p.total) {
+      files[p.file] = [p.loaded, p.total];
+      let loaded = 0, total = 0;
+      for (const [l, t] of Object.values(files)) { loaded += l; total += t; }
+      post({ type: "progress", label: "text", file: p.file, loaded, total, aggregate: true });
+    }
+  };
+  processor = await AutoProcessor.from_pretrained(REPO);
+  tok = processor.tokenizer;
+  model = await AutoModelForImageTextToText.from_pretrained(REPO, {
+    device,
+    dtype: { embed_tokens: "q4", vision_encoder: "q8", decoder_model_merged: "q4" },
+    progress_callback,
+  });
+  post({ type: "ready", device });
+}
+
+async function loadImage(blob) {
+  let img = await RawImage.fromBlob(blob);
+  img = img.rgb();
+  const max = 1024 * 1024; // runner.py `cap_pixels`
+  if (img.width * img.height > max) {
+    const s = Math.sqrt(max / (img.width * img.height));
+    img = await img.resize(Math.max(1, Math.floor(img.width * s)), Math.max(1, Math.floor(img.height * s)), { resample: 3 });
+  }
+  return img;
+}
+
+async function decide({ state, questions, image }) {
+  const t0 = performance.now();
+  const img = image ? await loadImage(image) : null;
+  const answers = {}, probs = {};
+  for (const q of questions) {
+    const prompt = render3b(tok, state, q, img ? "<image>" : "");
+    const inputs = img ? await processor(img, prompt, { add_special_tokens: false }) : tok(prompt, { add_special_tokens: false });
+    const { logits } = await model(inputs);
+    const [, T, V] = logits.dims;
+    const row = logits.data.subarray((T - 1) * V, T * V);
+    const scores = readoutIds3b(tok, q).map((g) => Math.max(...g.map((i) => row[i])));
+    const p = softmax(scores);
+    answers[q.name] = answer(q, p);
+    probs[q.name] = p;
+  }
+  return { answers, probs, ms: performance.now() - t0 };
+}
+
+self.onmessage = async ({ data }) => {
+  try {
+    if (data.type === "init") await init();
+    else if (data.type === "decide") post({ type: "result", id: data.id, ...(await decide(data)) });
+  } catch (e) {
+    post({ type: "error", id: data.id, message: String(e && e.message ? e.message : e) });
+  }
+};

worker-omni.js

@@ -0,0 +1,162 @@
+// d1-omni-600M in the browser: onnxruntime-web (WebGPU, WASM fallback) over onnx-community/d1-omni-600M-ONNX.
+import * as ort from "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.31.0-dev.20260914-8d85527a0/dist/ort.webgpu.bundle.min.mjs";
+import { AutoTokenizer, env } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.3.1/dist/transformers.min.js";
+import { QTYPES, answer, nOptions, omniEncode, softmax, temperatureKey } from "./prompt.js";
+import { preprocess } from "./vision.js";
+
+ort.env.wasm.wasmPaths = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.31.0-dev.20260914-8d85527a0/dist/";
+ort.env.logLevel = "error";
+env.allowLocalModels = false;
+
+const REPO = "onnx-community/d1-omni-600M-ONNX";
+const BASE = `https://huggingface.co/${REPO}/resolve/main/`;
+const CACHE = "open-d1-omni-v1";
+const YES_NO = { false: "no", true: "yes" };
+const post = (m) => self.postMessage(m);
+
+let device = "wasm", tok = null, config = null;
+const sessions = {};
+
+async function fetchCached(path, label) {
+  const url = BASE + path;
+  let cache = null;
+  try { cache = await caches.open(CACHE); const hit = await cache.match(url); if (hit) return new Uint8Array(await hit.arrayBuffer()); } catch {}
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`${path}: HTTP ${res.status}`);
+  const total = Number(res.headers.get("content-length")) || 0;
+  const reader = res.body.getReader();
+  const chunks = [];
+  let loaded = 0, last = 0;
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value); loaded += value.length;
+    const now = performance.now();
+    if (now - last > 150) { last = now; post({ type: "progress", label, file: path, loaded, total }); }
+  }
+  post({ type: "progress", label, file: path, loaded, total: total || loaded });
+  const buf = new Uint8Array(loaded);
+  let off = 0;
+  for (const c of chunks) { buf.set(c, off); off += c.length; }
+  try { if (cache) await cache.put(url, new Response(buf, { headers: { "content-type": "application/octet-stream" } })); } catch {}
+  return buf;
+}
+
+async function session(name, label) {
+  if (sessions[name]) return sessions[name];
+  const graph = await fetchCached(`onnx/${name}.onnx`, label);
+  const data = await fetchCached(`onnx/${name}.onnx_data`, label);
+  const opts = { executionProviders: [device], graphOptimizationLevel: "all", externalData: [{ path: `${name}.onnx_data`, data }] };
+  sessions[name] = await ort.InferenceSession.create(graph, opts);
+  return sessions[name];
+}
+
+async function init() {
+  if (navigator.gpu) {
+    try { if (await navigator.gpu.requestAdapter()) device = "webgpu"; } catch {}
+  }
+  if (device === "wasm") ort.env.wasm.numThreads = self.crossOriginIsolated ? Math.min(8, navigator.hardwareConcurrency || 4) : 1;
+  config = await (await fetch(BASE + "config.json")).json();
+  tok = await AutoTokenizer.from_pretrained(REPO);
+  await session("embed_tokens_quantized", "text");
+  await session("decision_quantized", "text");
+  post({ type: "ready", device });
+}
+
+const i64 = (arr, dims) => new ort.Tensor("int64", BigInt64Array.from(arr, (v) => BigInt(v)), dims);
+
+async function embed(sessionName, ids) {
+  const s = sessions[sessionName];
+  const out = await s.run({ input_ids: i64(ids, [1, ids.length]) });
+  return out.inputs_embeds.data; // Float32Array [L*1024]
+}
+
+async function imagePrefix(blob) {
+  const vis = await session("vision_encoder_fp16", "image");
+  const bmp = await createImageBitmap(blob, { colorSpaceConversion: "none", premultiplyAlpha: "none", imageOrientation: "none" });
+  const canvas = new OffscreenCanvas(bmp.width, bmp.height);
+  const ctx = canvas.getContext("2d", { willReadFrequently: true });
+  ctx.drawImage(bmp, 0, 0);
+  const rgba = ctx.getImageData(0, 0, bmp.width, bmp.height).data;
+  const rgb = new Uint8ClampedArray(bmp.width * bmp.height * 3);
+  for (let i = 0, j = 0; i < rgba.length; i += 4, j += 3) { rgb[j] = rgba[i]; rgb[j + 1] = rgba[i + 1]; rgb[j + 2] = rgba[i + 2]; }
+  const x = preprocess(rgb, bmp.width, bmp.height);
+  const out = await vis.run({
+    pixel_values: new ort.Tensor("float32", x.pixels, [x.n, 1024, 768]),
+    pixel_attention_mask: new ort.Tensor("int64", x.mask, [x.n, 1024]),
+    spatial_shapes: new ort.Tensor("int64", x.shapes, [x.n, 2]),
+  });
+  const f = out.image_features;
+  return { data: f.data, len: f.dims[0] };
+}
+
+async function audioPrefix(samples) {
+  const aud = await session("audio_encoder_quantized", "voice");
+  await session("embed_tokens_fp16", "voice");
+  // audio.py `waveform`: cut to 30 s, zero-pad to 0.5 s
+  let x = samples.subarray(0, 30 * 16000);
+  if (x.length < 8000) { const y = new Float32Array(8000); y.set(x); x = y; }
+  const out = await aud.run({ input_values: new ort.Tensor("float32", Float32Array.from(x), [1, x.length]) });
+  const a = out.audio_embeds;
+  return { data: a.data, len: a.dims[1] };
+}
+
+async function decide({ state, questions, image, audio }) {
+  const t0 = performance.now();
+  let prefix = null, maxLen = config.max_length, noul = null, calibrate = true, spoken = false, embedName = "embed_tokens_quantized";
+  if (image) {
+    prefix = await imagePrefix(image);
+    maxLen = Math.min(config.image_text_length, config.max_length - prefix.len);
+    noul = YES_NO; calibrate = false;
+  } else if (audio) {
+    prefix = await audioPrefix(audio);
+    maxLen = Math.min(config.audio_text_length, config.max_length - prefix.len);
+    noul = YES_NO; calibrate = false; spoken = true; embedName = "embed_tokens_fp16";
+    if (state === null) state = {};
+  }
+  if (maxLen < 64) throw new Error("the media take too many positions");
+  if (state === null) state = "";
+  const P = prefix ? prefix.len : 0, D = 1024;
+  const rows = [];
+  for (const q of questions) rows.push(omniEncode(tok, state, q, maxLen, noul, spoken));
+  const embeds = [];
+  for (const r of rows) embeds.push(await embed(embedName, r.ids));
+  const B = rows.length, L = P + Math.max(...rows.map((r) => r.ids.length)), K = Math.max(...rows.map((r) => r.markers.length));
+  const h = new Float32Array(B * L * D), mask = [], mpos = [], mmask = [], qtype = [];
+  rows.forEach((r, b) => {
+    if (prefix) h.set(prefix.data, b * L * D);
+    h.set(embeds[b], (b * L + P) * D);
+    for (let t = 0; t < L; t++) mask.push(t < P + r.ids.length ? 1 : 0);
+    for (let k = 0; k < K; k++) { mpos.push(k < r.markers.length ? r.markers[k] : 0); mmask.push(k < r.markers.length ? 1 : 0); }
+    qtype.push(QTYPES[questions[b].type]);
+  });
+  const out = await sessions.decision_quantized.run({
+    inputs_embeds: new ort.Tensor("float32", h, [B, L, D]),
+    attention_mask: i64(mask, [B, L]),
+    prefix_len: i64(new Array(B).fill(P), [B]),
+    marker_pos: i64(mpos, [B, K]),
+    marker_mask: i64(mmask, [B, K]),
+    qtype: i64(qtype, [B]),
+  });
+  const logits = out.logits.data, Kout = out.logits.dims[1];
+  const temps = config.temperatures || {};
+  const answers = {}, probs = {};
+  questions.forEach((q, b) => {
+    let z = Array.from(logits.subarray(b * Kout, b * Kout + nOptions(q)));
+    if (calibrate) { const t = temps[temperatureKey(q)] ?? temps[q.type] ?? 1; z = z.map((v) => v / t); }
+    let p = softmax(z);
+    if (q.type === "noul") p = p.reverse(); // the model reads a noul as [false, true]
+    answers[q.name] = answer(q, p);
+    probs[q.name] = p;
+  });
+  return { answers, probs, ms: performance.now() - t0, prefix: P };
+}
+
+self.onmessage = async ({ data }) => {
+  try {
+    if (data.type === "init") await init();
+    else if (data.type === "decide") post({ type: "result", id: data.id, ...(await decide(data)) });
+  } catch (e) {
+    post({ type: "error", id: data.id, message: String(e && e.message ? e.message : e) });
+  }
+};