Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions .github/workflows/pages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,13 @@ on:
- "self-test.html"
- "web/**"
- "answers/**"
- "task-review-*.js"
- "results/dataset/*.png"
- "src/vladbench/site.py"
- "results/rerun.json"
- "results/models.json"
- "results/audit/**"
- "original/all_task.json"
- "src/vladbench/**"
- "uv.lock"
- "pyproject.toml"
- ".github/workflows/pages.yml"
workflow_dispatch:

Expand All @@ -32,7 +36,7 @@ jobs:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v6
- name: Build the standalone site
run: uv run vladbench build --steps site --site dist/pages --article https://www.eventual.ai/blog/vladbench-reeval
run: uv run --group scripts vladbench build --steps site --site dist/pages --article https://www.eventual.ai/blog/vladbench-reeval
- uses: actions/configure-pages@v5
- uses: actions/upload-pages-artifact@v3
with:
Expand Down
10 changes: 4 additions & 6 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,8 @@ build/
dist/
data/
results/runs/
# task-review-*.js are the published data files, generated by scripts/build_review_results.py and
# scripts/build_variants.py and committed because the site is built from them in CI.
# Generated by scripts/export_parquet.py from results/. The two figures stay tracked because the site build copies them.
results/dataset/*
!results/dataset/cost-vs-score.png
!results/dataset/cost-vs-score-hero.png
# Generated companion data and plots; the site build recreates them from tracked inputs.
task-review-*.js
results/dataset/
results/logs/
.claude/launch.json
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,10 @@ To evaluate a model without changing the published results, place it in its own

```sh
uv run vladbench build # record, site data, answers, variants, parquet, figures, dataset card, site
uv run vladbench build --steps site --site dist/pages
uv run --group scripts vladbench build --steps site --site dist/pages
```

The Pages build recreates the six `task-review-*.js` data files and both charts from tracked inputs.
The `record`, `answers`, and `export` steps read the raw answer records, which are not in git; without them the build
skips those steps, says so, and starts from the committed `results/rerun.json`. The build refuses a model whose run is
partial or whose answers are newer than its score file.
Expand Down Expand Up @@ -155,7 +156,7 @@ The dataset, https://huggingface.co/datasets/Eventual-Inc/VLADBench-reeval, has
| `results/protocols/` | protocol files, one per condition |
| `results/models.json` | model registry |
| `results/scores-*.json`, `results/rerun.json` | score files and the record built from them |
| `answers/` | every model's answer and per-question mark, per task |
| `answers/` | every model's answer and per-question mark, per task; required by the interactive site |
| `original/` | the paper's scoring code and task catalog, byte for byte, with hashes |
| `docs/` | [protocol and limitations](docs/PROTOCOL.md), [reproduction](docs/REPRODUCTION.md), reviews, and plans |

Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ where = ["src"]

[dependency-groups]
# Tests, lint, and types. `uv sync` installs this group by default.
dev = ["pytest>=8", "ruff>=0.13", "ty>=0.0.83", "codespell>=2.4", "pyarrow>=17"]
dev = ["pytest>=8", "ruff>=0.13", "ty>=0.0.83", "codespell>=2.4", "pyarrow>=17", "matplotlib>=3.9"]
# The build and publish scripts under scripts/.
scripts = ["pyarrow>=17", "matplotlib>=3.9", "huggingface_hub>=0.34"]

Expand Down
Binary file removed results/dataset/cost-vs-score-hero.png
Binary file not shown.
Binary file removed results/dataset/cost-vs-score.png
Binary file not shown.
16 changes: 10 additions & 6 deletions src/vladbench/site.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,15 @@
import shutil

from . import paths
from .sitedata import FILES
from .sitedata import FILES, task_inputs, write_site_data
from .boxes import variants
from .figures import render_figures
from .record import load_record
from .registry import load_registry
from .sitedata import write_js

FULL_DATA = re.compile(r'<script src="task-review-data\.js(\?v=\d+)?" defer></script>')
SLIM_DATA = '<script src="task-review-tasks.js" defer></script>'
FIGURES = ("cost-vs-score.png", "cost-vs-score-hero.png")


def embed_html(results_html: str) -> str:
Expand Down Expand Up @@ -44,8 +48,8 @@ def build_site(out: Path, *, article: str | None = None, blog: str | None = "htt
shutil.copytree(paths.ROOT / "web", out / "web")
shutil.copytree(paths.ANSWERS, out / "answers") # fetched one task at a time when a row expands
(out / "web/config.js").write_text(config_js(article, blog))
for name, _ in FILES.values():
shutil.copy(paths.SITE_DATA / name, out / name)
for figure in FIGURES:
shutil.copy(paths.DATASET / figure, out / figure)
record = load_record()
write_site_data(record, task_inputs(), out)
write_js(out / FILES["variants"][0], FILES["variants"][1], variants(record, load_registry()))
render_figures(record, load_registry(), out)
return out
1 change: 0 additions & 1 deletion task-review-audit.js

This file was deleted.

1 change: 0 additions & 1 deletion task-review-data.js

This file was deleted.

1 change: 0 additions & 1 deletion task-review-published.js

This file was deleted.

1 change: 0 additions & 1 deletion task-review-results.js

This file was deleted.

1 change: 0 additions & 1 deletion task-review-tasks.js

This file was deleted.

1 change: 0 additions & 1 deletion task-review-variants.js

This file was deleted.

9 changes: 3 additions & 6 deletions tests/test_aggregate.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,11 @@
import unittest

from vladbench.aggregate import Point, TaskScore, Tie, frontier, group_means, task_wins, total
from vladbench.sitedata import task_inputs

ROOT = Path(__file__).resolve().parents[1]


def load_js(path: Path):
text = path.read_text()
return json.loads(text[text.index("=") + 1:].strip().rstrip(";"))


class TotalTests(unittest.TestCase):
TASKS: dict[str, TaskScore] = {"A": {"score": 90.0, "questions_scored": 300}, "B": {"score": 50.0, "questions_scored": 100}}
Expand All @@ -31,7 +28,7 @@ def test_no_tasks_is_an_error(self):
total(self.TASKS, skip={"A", "B"})

def test_reproduces_241_of_the_papers_250_group_averages(self):
published = load_js(ROOT / "task-review-published.js")
published = json.loads((ROOT / "results/audit/published-results-2025.json").read_text())
record = json.loads((ROOT / "results/rerun.json").read_text())
counts = {t: r["questions_scored"] for t, r in record["models"][0]["tasks"].items()}
within = checked = 0
Expand All @@ -55,7 +52,7 @@ def test_site_total_matches(self):
found = [re.search(rf"^function {name}\(.*?^}}", source, re.S | re.M) for name in ("composite", "modelMean")]
assert all(found), "web/results.js no longer defines composite and modelMean"
functions = "\n".join(m.group(0) for m in found if m)
script = (f"const state = {{weights: null}};\nconst TASKS = {json.dumps(load_js(ROOT / 'task-review-tasks.js'))};\n{functions}\n"
script = (f"const state = {{weights: null}};\nconst TASKS = {json.dumps(task_inputs())};\n{functions}\n"
f"const models = {json.dumps(json.loads((ROOT / 'results/rerun.json').read_text())['models'])};\n"
"console.log(JSON.stringify(Object.fromEntries(models.map((m) => [m.id, modelMean(m)]))));")
site = json.loads(subprocess.run(["node", "-"], input=script, capture_output=True, text=True, check=True).stdout) # stdin: the record exceeds the argument limit
Expand Down
9 changes: 5 additions & 4 deletions tests/test_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from vladbench.record import IncompleteRun, build_record, load_record
from vladbench.registry import load_registry
from vladbench.site import build_site
from vladbench.sitedata import FILES, site_file, task_inputs, write_js, write_site_data
from vladbench.sitedata import FILES, task_inputs, write_js, write_site_data
from vladbench.spec import load_spec

ROOT = Path(__file__).resolve().parents[1]
Expand All @@ -33,17 +33,18 @@ def same(self, ours: Path, committed: Path):
self.assertTrue(filecmp.cmp(ours, committed, shallow=False), f"{ours.name} differs from the committed {committed}")

def test_site_data(self):
site = build_site(self.tmp / "site-data-pages")
for path in write_site_data(self.record, task_inputs(), self.tmp):
with self.subTest(file=path.name):
self.same(path, site_file([k for k, (name, _) in FILES.items() if name == path.name][0]))
self.same(path, site / path.name)

def test_variants(self):
path = write_js(self.tmp / FILES["variants"][0], FILES["variants"][1], variants(self.record, load_registry()))
self.same(path, site_file("variants"))
self.same(path, build_site(self.tmp / "variant-pages") / path.name)

def test_site_bundle(self):
site = build_site(self.tmp / "pages")
for name in ("index.html", "results.html", "embed.html", "self-test.html", "rerun.json", "web/config.js", *(n for n, _ in FILES.values())):
for name in ("index.html", "results.html", "embed.html", "self-test.html", "rerun.json", "web/config.js", "cost-vs-score.png", "cost-vs-score-hero.png", *(n for n, _ in FILES.values())):
with self.subTest(file=name):
self.assertTrue((site / name).is_file())
self.assertNotIn("task-review-data.js", (site / "embed.html").read_text())
Expand Down
9 changes: 3 additions & 6 deletions tests/test_record.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,13 @@
import unittest

from vladbench.aggregate import total
from vladbench.boxes import BOX_TASKS
from vladbench.boxes import BOX_TASKS, variants
from vladbench.registry import load_registry
from vladbench.record import IncompleteRun, require_complete

ROOT = Path(__file__).resolve().parents[1]


def load_js(path: Path):
text = path.read_text()
return json.loads(text[text.index("=") + 1:].strip().rstrip(";"))


class CompletenessTests(unittest.TestCase):
SCORED_AT = datetime(2026, 9, 20, tzinfo=timezone.utc)
Expand All @@ -39,7 +36,7 @@ class PublishedNumbersTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.record = json.loads((ROOT / "results/rerun.json").read_text())
cls.variants = load_js(ROOT / "task-review-variants.js")
cls.variants = variants(cls.record, load_registry())

def test_task_scores_match_their_score_files(self):
for model in self.record["models"]:
Expand Down
6 changes: 5 additions & 1 deletion tests/test_review_results.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import json
import tempfile

from vladbench.sitedata import FILES, write_js
from pathlib import Path
import unittest

Expand Down Expand Up @@ -36,7 +39,8 @@ def test_summaries_track_their_score_files(self):
self.assertIsNotNone(model["usage"]["cost_usd"], f"{model['id']} has no billed cost")

def test_browser_asset_matches_json_record(self):
javascript = (ROOT / "task-review-results.js").read_text()
with tempfile.TemporaryDirectory() as tmp:
javascript = write_js(Path(tmp) / FILES["results"][0], FILES["results"][1], self.result).read_text()
prefix = "window.FULL_RESULTS = "
self.assertTrue(javascript.startswith(prefix))
self.assertTrue(javascript.endswith(";\n"))
Expand Down
Loading
Loading