File size: 15,187 Bytes
8201728
5a7177d
 
 
8201728
 
 
 
38694da
8201728
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b65145a
8201728
 
b65145a
8201728
 
5a7177d
 
8201728
 
 
 
 
 
5a7177d
8201728
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f3d5640
 
 
 
 
8201728
 
 
 
 
 
ae9eca9
 
b65145a
 
 
 
 
 
 
8201728
 
 
 
 
 
ae9eca9
 
8201728
 
 
5a7177d
 
8201728
 
 
 
5a7177d
 
f3d5640
 
 
 
 
 
 
 
 
 
 
8201728
 
 
18d03ee
8201728
 
 
 
 
5a7177d
 
8201728
 
 
 
 
5a7177d
8201728
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5a7177d
8201728
 
 
 
 
 
 
 
 
 
5a7177d
8201728
 
 
 
5a7177d
 
8201728
 
 
 
5a7177d
 
8201728
 
 
 
 
 
 
 
 
fadb285
8201728
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ae9eca9
8201728
5a7177d
8201728
 
 
 
b65145a
8201728
 
 
 
ae9eca9
8201728
5a7177d
8201728
5a7177d
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
/* colgrep --agent terminal replay. Static: loads data/demo.json, replays recorded runs. */
(() => {
  const $ = (s) => document.querySelector(s);
  const esc = (s) => String(s ?? "").replace(/[&<>"]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
  const params = new URLSearchParams(location.search);
  const state = { data: null, repo: null, runId: 0, instant: false };
  const screen = $("#screen"), body = $("#term-body"), input = $("#cmd"), suggest = $("#suggest");

  const speed = () => 1;
  const wait = (ms) => new Promise((r) => setTimeout(r, state.instant || speed() === 0 ? 0 : ms / speed()));
  const scrollDown = () => { body.scrollTop = body.scrollHeight; };
  const line = (html, cls = "") => { const d = document.createElement("div"); d.className = `line ${cls}`; d.innerHTML = html; screen.appendChild(d); scrollDown(); return d; };

  async function typeCommand(text) {
    const el = line(`<span class="ps1"><span class="cwd">${esc(cwd())}</span> $</span> <span class="cmd"></span>`, "cmdline");
    const span = el.querySelector(".cmd");
    const html = text.replace(/^(colgrep --agent )("[^"]*")$/, (_, a, b) => `${esc(a)}<span class="q">${esc(b)}</span>`);
    if (state.instant || speed() === 0) { span.innerHTML = html; return; }
    for (let i = 1; i <= text.length; i++) { span.textContent = text.slice(0, i); await wait(28); }
    span.innerHTML = html;
  }

  const cwd = () => (state.repo ? `~/${state.repo.repo.split("/")[1]}` : "~");
  const setCwd = () => { $("#cwd").textContent = cwd(); };

  // ------------------------------------------------------------- suggestions
  function renderSuggest() {
    const d = state.data;
    if (!state.repo) {
      suggest.innerHTML = `<span class="lbl">repositories (click to cd):</span>` + d.repos.map((r, i) =>
        `<button type="button" class="chip repo" data-repo="${i}">cd ${esc(r.repo)}<span class="lang">${esc(r.language)}</span><span class="meta">${r.queries.length} question${r.queries.length > 1 ? "s" : ""}</span></button>`).join("");
    } else {
      suggest.innerHTML = `<span class="lbl">recorded questions in ${esc(state.repo.repo)}, ${esc(state.repo.language)} (click to run):</span>` + state.repo.queries.map((q, i) =>
        `<button type="button" class="chip" data-query="${i}" title="${esc(q.id)}">colgrep --agent "${esc(q.title)}"</button>`).join("") +
        `<button type="button" class="chip back" data-back="1">cd ..</button>`;
    }
  }
  suggest.addEventListener("click", (e) => {
    const b = e.target.closest("button"); if (!b) return;
    if (b.dataset.repo != null) run(`cd ${state.data.repos[+b.dataset.repo].repo}`);
    else if (b.dataset.query != null) run(`colgrep --agent "${state.repo.queries[+b.dataset.query].title}"`);
    else if (b.dataset.back) run("cd ..");
  });

  // ------------------------------------------------------------- commands
  async function run(cmd) {
    const id = ++state.runId;
    state.instant = false;
    $("#skip").hidden = true;
    input.value = "";
    input.disabled = true;
    try {
      await typeCommand(cmd);
      if (id !== state.runId) return;
      const m = /^colgrep\s+--agent\s+"?(.+?)"?\s*$/.exec(cmd);
      if (/^clear$/.test(cmd)) { screen.innerHTML = ""; }
      else if (/^help$/.test(cmd)) help();
      else if (/^cd\s+\.\.$/.test(cmd) || /^cd$/.test(cmd)) { state.repo = null; setCwd(); }
      else if (/^cd\s+/.test(cmd)) cd(cmd.replace(/^cd\s+/, "").trim());
      else if (/^ls$/.test(cmd)) ls();
      else if (m) await agent(m[1], id);
      else {
        const hit = matchAny(cmd);
        if (hit) { if (!state.repo || !state.repo.queries.includes(hit)) cd(state.data.repos.find((r) => r.queries.includes(hit)).repo); await agent(hit.title, id); }
        else line(`<span class="err">no recorded session matches "${esc(cmd)}".</span><span class="faint">  Nothing runs live here: this terminal replays ${state.data.repos.reduce((n, r) => n + r.queries.length, 0)} recorded runs. Pick one below, or type a few words of its title.</span>`);
      }
    } finally {
      if (id === state.runId) { input.disabled = false; $("#skip").hidden = true; renderSuggest(); scrollDown(); input.focus({ preventScroll: true }); }
    }
    const u = new URL(location);
    if (state.repo) u.searchParams.set("repo", state.repo.repo); else u.searchParams.delete("repo");
    history.replaceState(null, "", u);
  }

  function intro() {
    const n = state.data.repos.reduce((k, r) => k + r.queries.length, 0);
    return `<span class="dim">This terminal replays ${n} recorded runs of ${esc(state.data.model)} on ${state.data.repos.length} evaluation repositories, so only two commands do anything:</span>
<span class="dim">  <b>cd</b> &lt;repo&gt;                   pick a repository</span>
<span class="dim">  <b>colgrep --agent</b> "&lt;question&gt;"  replay the recorded run for one of its questions</span>`;
  }

  function help() {
    line(`<span class="dim">cd &lt;repo&gt;            enter one of the evaluation repositories
cd ..                leave it
ls                   list what is available here
colgrep --agent "…"  replay the recorded run for one of the listed questions
clear                clear the screen</span>`);
  }

  function ls() {
    if (!state.repo) line(`<span class="dim">${state.data.repos.map((r) => esc(r.repo.split("/")[1]) + "/").join("  ")}</span>`);
    else line(`<span class="dim">${state.repo.queries.map((q) => `${esc(q.id)}  ${esc(q.title)}`).join("\n")}</span>`);
  }

  function cd(name) {
    const r = state.data.repos.find((x) => x.repo === name || x.repo.split("/")[1] === name || x.repo.endsWith("/" + name));
    if (!r) { line(`<span class="err">cd: no such repository: ${esc(name)}</span>`); return; }
    state.repo = r; setCwd();
  }

  function matchAny(text) {
    const t = text.toLowerCase().replace(/^colgrep\s+--agent\s+/, "").replace(/^"|"$/g, "");
    const pool = (state.repo ? state.repo.queries : []).concat(state.data.repos.flatMap((r) => r.queries));
    return pool.find((q) => q.title.toLowerCase().includes(t) || q.id.toLowerCase().includes(t)) || null;
  }
  function filterChips(text) {
    const t = text.toLowerCase().replace(/^colgrep\s+--agent\s+/, "").replace(/^"|"$/g, "");
    suggest.querySelectorAll(".chip").forEach((c) => { c.classList.toggle("hidden", !!t && !c.textContent.toLowerCase().includes(t) && !(c.title || "").toLowerCase().includes(t)); });
  }
  input.addEventListener("input", () => filterChips(input.value.trim()));

  // ------------------------------------------------------------- the agent replay
  const benchShort = (b) => (b.includes("Lite") ? "SWE-bench Lite" : "Multi-SWE-bench Flash");
  const icon = { colgrep: "●", terminal: "■", finish: "◆" };

  function findQuery(text) {
    const t = text.toLowerCase();
    const pool = state.repo ? state.repo.queries : state.data.repos.flatMap((r) => r.queries);
    return pool.find((q) => q.title.toLowerCase() === t || q.id.toLowerCase() === t) ||
      pool.find((q) => q.title.toLowerCase().startsWith(t) || q.id.toLowerCase().includes(t)) || null;
  }

  async function agent(text, id) {
    const q = findQuery(text);
    if (!q) {
      line(`<span class="err">no recorded run for this question.</span><span class="faint">  Nothing runs live here: pick one of the questions below${state.repo ? "" : ", after choosing a repository"}.</span>`);
      return;
    }
    if (!state.repo || !state.repo.queries.includes(q)) { state.repo = state.data.repos.find((r) => r.queries.includes(q)); setCwd(); }
    $("#skip").hidden = false;
    const d = state.data;
    line(`◆ colgrep agent · ${esc(d.model)} · recorded run on ${esc(benchShort(q.benchmark))}, seed ${d.seed}, ${d.evaluation.max_turns}-turn budget\n◆ <a href="https://github.com/${esc(state.repo.repo)}/tree/${esc(q.base_commit)}" target="_blank" rel="noopener">${esc(state.repo.repo)} @ ${esc(q.base_commit.slice(0, 7))}</a>${q.n_files ? ` · ${q.n_files.toLocaleString()} files` : ""}`, "hdr");
    const issueLines = q.problem.split("\n").length;
    const qEl = line(`<span class="mark">❯</span> ${esc(q.title)} <button type="button" class="issue-toggle">[full issue text sent to the agent, ${issueLines} lines ▸]</button><div class="issue">${esc(q.problem)}</div>`, "question");
    qEl.querySelector(".issue-toggle").onclick = (e) => { const o = qEl.querySelector(".issue").classList.toggle("open"); e.target.textContent = `[full issue text sent to the agent, ${issueLines} lines ${o ? "▾" : "▸"}]`; };

    const steps = document.createElement("div"); steps.className = "steps"; screen.appendChild(steps);
    const goldFiles = q.gold_files;
    for (let i = 0; i < q.steps.length; i++) {
      const s = q.steps[i];
      await wait(i === 0 ? 500 : 250);
      if (id !== state.runId) return;
      const pending = document.createElement("div");
      pending.className = "step pending";
      pending.innerHTML = `<span class="n">${i + 1}</span><span class="tool"><span class="spinner">⠋</span></span><span class="text">thinking…</span>`;
      steps.appendChild(pending); scrollDown();
      const frames = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"; let f = 0;
      const spin = setInterval(() => { const sp = pending.querySelector(".spinner"); if (sp) sp.textContent = frames[++f % frames.length]; }, 80);
      await wait(s.tool === "colgrep" ? 900 : s.tool === "finish" ? 700 : 550);
      clearInterval(spin);
      if (id !== state.runId) { pending.remove(); return; }
      if (s.note) { const n = document.createElement("div"); n.className = "note"; n.textContent = `harness: ${s.note}`; steps.insertBefore(n, pending); }
      pending.className = `step ${s.tool}`;
      pending.tabIndex = 0;
      const hasOut = s.output && s.tool !== "finish";
      pending.innerHTML = `<span class="n">${i + 1}</span><span class="tool">${icon[s.tool]} ${s.tool}</span><span class="text">${esc(s.text)}${hasOut ? `<span class="caret">▸ output</span>` : ""}</span>`;
      if (hasOut) {
        const out = document.createElement("div"); out.className = "output";
        out.innerHTML = markGold(s.output, goldFiles);
        steps.appendChild(out);
        const toggle = () => { const o = out.classList.toggle("open"); pending.querySelector(".caret").textContent = o ? "▾ output" : "▸ output"; };
        pending.onclick = toggle;
        pending.onkeydown = (e) => { if (e.key === "Enter" || e.key === " ") { e.preventDefault(); toggle(); } };
      }
      scrollDown();
    }
    await wait(300);
    if (id !== state.runId) return;
    const n = q.locations.length;
    line(`<span class="ok">✓ ${n} location${n === 1 ? "" : "s"}</span><span class="dim"> · ${q.n_turns} turns · ${q.n_searches} search${q.n_searches === 1 ? "" : "es"} · ${q.n_terminal} terminal command${q.n_terminal === 1 ? "" : "s"}${q.forced_finish ? " · finish forced by the turn budget" : ""}</span>`, "summary");
    for (const l of q.locations) {
      await wait(200);
      if (id !== state.runId) return;
      const range = l.start === l.end ? `${l.start}` : `${l.start}-${l.end}`;
      const code = l.code ? `<pre class="code">${l.code.map((c) => c.gap ? `<span class="ln gap">      …</span>` : `<span class="ln ${c.in ? "in" : "ctx"}"><span class="no">${c.n}</span>${esc(c.t)}</span>`).join("")}</pre>` : `<span class="faint">(code not available)</span>`;
      line(`<div class="loc-head"><span class="path">${esc(l.file)}:${range}</span>${l.symbol ? `<span class="sym">${esc(l.symbol)}</span>` : ""}</div>${code}`, "loc");
    }
    await wait(300);
    if (id !== state.runId) return;
    line(referenceHtml(q), "ref");
    scrollDown();
  }

  function markGold(text, gold) {
    let h = esc(text);
    for (const g of gold) h = h.split(esc(g)).join(`<mark>${esc(g)}</mark>`);
    return h;
  }

  const overlaps = (a, b) => a.file === b.file && a.start <= b.end && b.start <= a.end;
  function referenceHtml(q) {
    const rows = q.locations.map((l) => {
      const inFile = q.gold_files.includes(l.file);
      const inLines = inFile && q.gold.some((g) => overlaps(l, g));
      const mark = inLines ? `<span class="ok">✓</span>` : inFile ? `<span class="gold">◐</span>` : `<span class="bad">✗</span>`;
      const why = inLines ? "overlaps a reference range" : inFile ? "right file, lines outside the reference ranges" : "not a file the fix touched";
      const range = l.start === l.end ? `${l.start}` : `${l.start}-${l.end}`;
      return `<span class="row">${mark} ${esc(l.file)}:${range}<span class="m faint">${why}</span></span>`;
    }).join("");
    const missed = q.gold_files.filter((g) => !q.locations.some((l) => l.file === g)).map((g) => `<span class="row"><span class="bad">✗</span> ${esc(g)}<span class="m faint">reference file not submitted</span></span>`).join("");
    const gold = q.gold.map((g) => `${esc(g.file)}:${g.start === g.end ? g.start : `${g.start}-${g.end}`}`).join(", ");
    const s = q.scores;
    const f = (x) => (x == null ? "n/a" : x.toFixed(2));
    return `<div class="ttl">Checked against the pull request that fixed <span class="faint">${esc(q.id)}</span></div>${rows}${missed}` +
      `<span class="row faint">reference ranges: ${gold}</span>` +
      `<span class="row">file F1 <b>${f(s.file_f1)}</b> · line F1 <b>${f(s.line_f1)}</b><span class="m faint">(line F1 counts exact line overlap)</span></span>` +
      (q.note ? `<div class="note-text">${esc(q.note)}</div>` : "");
  }

  // ------------------------------------------------------------- wiring
  $("#prompt").onsubmit = (e) => { e.preventDefault(); const v = input.value.trim(); if (v) run(v); };
  $("#skip").onclick = () => { state.instant = true; };
  $("#clear").onclick = () => { state.runId++; screen.innerHTML = ""; input.disabled = false; $("#skip").hidden = true; renderSuggest(); input.focus(); };
  body.addEventListener("click", (e) => { if (!e.target.closest("button, .step, .output, .issue, a, input, select") && !getSelection().toString()) input.focus({ preventScroll: true }); });
  input.addEventListener("keydown", (e) => {
    if (e.key === "Tab" && !e.shiftKey) {
      const v = input.value.trim(); if (!v) return;
      e.preventDefault();
      const opts = state.repo ? state.repo.queries.map((q) => `colgrep --agent "${q.title}"`) : state.data.repos.map((r) => `cd ${r.repo}`);
      const hit = opts.find((o) => o.toLowerCase().startsWith(v.toLowerCase())) || opts.find((o) => o.toLowerCase().includes(v.toLowerCase()));
      if (hit) input.value = hit;
    }
  });

  async function boot() {
    const r = await fetch("data/demo.json");
    state.data = await r.json();
    renderSuggest();
    line(intro(), "intro");
    const repo = params.get("repo"), query = params.get("query");
    if (query) {
      const q = state.data.repos.flatMap((r) => r.queries).find((x) => x.id === query);
      if (q) { await run(`cd ${state.data.repos.find((r) => r.queries.includes(q)).repo}`); await run(`colgrep --agent "${q.title}"`); return; }
    }
    if (repo) await run(`cd ${repo}`);
  }
  boot().catch((err) => line(`<span class="err">failed to load data: ${esc(err.message)}</span>`));
})();