Source files / site · raw

collect-data.sh

#!/usr/bin/env bash
# Refresh bench/data/ from the rig and from lean_bench before a site build.
#
#   site/collect-data.sh [rig-ssh-host] [lean_bench-results-dir]
#
# Defaults: tersler.local, ~/src/llm-benchmark/results. lean_bench results
# are slimmed to the scoring fields (no prompts or model answers).
set -euo pipefail
repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
rig=${1:-tersler.local}
lb=${2:-$HOME/src/llm-benchmark/results}
data=$repo/bench/data

mkdir -p "$data/lean_bench"
scp -q "$rig:/opt/tersler/bench/results.csv" "$rig:/opt/tersler/bench/concurrency.jsonl" "$rig:/opt/tersler/bench/speed.jsonl" \
    "$rig:/opt/tersler/bench/loadtest-*.csv" "$data/"
python3 -I - "$lb" "$data/lean_bench" <<'EOF'
import glob, json, os, sys
keep = ("id", "category", "family", "level", "correct", "completion_tokens", "seconds", "finish_reason")
for f in glob.glob(os.path.join(sys.argv[1], "*.jsonl")):
    with open(os.path.join(sys.argv[2], os.path.basename(f)), "w") as o:
        for line in open(f):
            r = json.loads(line)
            o.write(json.dumps({k: r.get(k) for k in keep}) + "\n")
EOF
echo "bench/data updated. New runs need a label in bench/data/notes.json to appear on the site."