Source files / scripts · raw

bench.sh

#!/usr/bin/env bash
# Benchmark a llama.cpp build with the configured model and flags, so builds,
# drivers and settings can be compared before switching.
#
#   tersler-bench                      # bench the active build
#   tersler-bench /opt/tersler/builds/<commit>   # bench a specific build
#   RUNS=5 MAX_TOKENS=1024 tersler-bench
#
# For the active build, uses the running tersler-llama server if it runs
# that build and model; otherwise starts a private llama-server on
# BENCH_PORT. Sends every prompt in bench/prompts/*.txt RUNS times (prompt
# cache off), and appends the medians to
# $JR_PREFIX/bench/results.csv (prompt + generation tok/s, as reported by
# llama-server's own timings).
set -euo pipefail
# shellcheck source=lib.sh
source "$(dirname "$(readlink -f "$0")")/lib.sh"
jr_load_config

build=$(readlink -f "${1:-$JR_PREFIX/current}")
RUNS=${RUNS:-3}
MAX_TOKENS=${MAX_TOKENS:-512}
BENCH_PORT=${BENCH_PORT:-18080}
PROMPTS=${PROMPTS:-$JR_PREFIX/bench/prompts}
[[ -d $PROMPTS ]] || PROMPTS=$(dirname "$(readlink -f "$0")")/../bench/prompts
RESULTS=${RESULTS:-$JR_PREFIX/bench/results.csv}
LOG=$(mktemp -t tersler-bench.XXXX.log)

[[ -x $build/bin/llama-server ]] || jr_die "no llama-server in $build"
command -v jq >/dev/null || jr_die "jq missing"

commit_full=$(jq -r '.commit // empty' "$build/build-info.json" 2>/dev/null || true)

# Reuse the running server if it is this build with this model: the build
# commit and model path come from its /props. (Flags changed in the config
# since it started are not detected; restart tersler-llama after edits.)
port=""
if props=$(curl -fs "http://127.0.0.1:$LLAMA_PORT/props" 2>/dev/null); then
    run_commit=$(jq -r '.build_info // "" | sub("^b[0-9]+-"; "")' <<<"$props")
    run_model=$(jq -r '.model_path // ""' <<<"$props")
    if [[ -n $commit_full && -n $run_commit && $commit_full == "$run_commit"* \
          && $run_model == "$(jr_model_path)" ]]; then
        port=$LLAMA_PORT
        jr_log "using the running server on :$port (build $run_commit, $(basename "$run_model"))"
    else
        jr_warn "the server on :$LLAMA_PORT runs build ${run_commit:-?} / $(basename "${run_model:-?}"), not $(basename "$build")."
        jr_warn "Two servers won't fit in VRAM:  sudo systemctl stop tersler-llama   and re-run"
        exit 1
    fi
elif ss -ltn | grep ":$LLAMA_PORT " >/dev/null; then
    jr_die "something listens on :$LLAMA_PORT but doesn't answer /props; stop it first"
fi

if [[ -z $port ]]; then
    port=$BENCH_PORT
    jr_log "starting $build on :$port (log: $LOG)"
    # installed as tersler-server next to us; server.sh in the repo
    here=$(dirname "$(readlink -f "$0")")
    server=$here/tersler-server; [[ -x $server ]] || server=$here/server.sh
    LLAMA_BIN=$build LLAMA_PORT=$port LLAMA_HOST=127.0.0.1 "$server" >"$LOG" 2>&1 &
    srv=$!
    trap 'kill $srv 2>/dev/null; wait $srv 2>/dev/null || true' EXIT

    for _ in $(seq 600); do
        curl -fs "http://127.0.0.1:$port/health" >/dev/null 2>&1 && break
        kill -0 $srv 2>/dev/null || { tail -20 "$LOG"; jr_die "server died during startup"; }
        sleep 1
    done
    curl -fs "http://127.0.0.1:$port/health" >/dev/null || jr_die "server not healthy after 10 min"
fi

median() { sort -n | awk '{a[NR]=$1} END {if (NR==0) print "NA"; else if (NR%2) print a[(NR+1)/2]; else printf "%.2f\n", (a[NR/2]+a[NR/2+1])/2}'; }

commit=$(jq -r '.commit[0:9]' "$build/build-info.json" 2>/dev/null || basename "$build")
driver=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | sed -n 1p)
plimit=$(nvidia-smi --query-gpu=power.limit --format=csv,noheader,nounits | sed -n 1p)
p2p=0; jr_p2p_enabled && p2p=1
[[ -f $RESULTS ]] || echo "date,commit,driver,power_limit_w,p2p,model,prompt,runs,max_tokens,pp_tok_s,tg_tok_s,draft_accept" > "$RESULTS"

for f in "$PROMPTS"/*.txt; do
    name=$(basename "$f" .txt)
    pp=(); tg=(); acc=()
    for r in $(seq "$RUNS"); do
        body=$(jq -n --rawfile p "$f" --argjson n "$MAX_TOKENS" \
            '{messages:[{role:"user",content:$p}], max_tokens:$n, stream:false, cache_prompt:false}')
        resp=$(curl -fs "http://127.0.0.1:$port/v1/chat/completions" \
               -H 'Content-Type: application/json' -d "$body") || { jr_warn "$name run $r failed"; continue; }
        pp+=("$(jq -r '.timings.prompt_per_second // empty' <<<"$resp")")
        tg+=("$(jq -r '.timings.predicted_per_second // empty' <<<"$resp")")
        acc+=("$(jq -r 'if .timings.draft_n then (.timings.draft_n_accepted / .timings.draft_n) else empty end' <<<"$resp")")
        printf '  %-12s run %d: pp %6.1f  tg %6.1f tok/s\n' "$name" "$r" "${pp[-1]:-0}" "${tg[-1]:-0}"
    done
    mp=$(printf '%s\n' "${pp[@]}" | grep . | median) || true
    mt=$(printf '%s\n' "${tg[@]}" | grep . | median) || true
    ma=$(printf '%s\n' "${acc[@]}" | grep . | median) || true
    echo "$(date -Is),$commit,$driver,$plimit,$p2p,$MODEL_FILE,$name,$RUNS,$MAX_TOKENS,$mp,$mt,${ma:-NA}" >> "$RESULTS"
    jr_log "$name: median pp $mp tok/s, tg $mt tok/s"
done
jr_log "results appended to $RESULTS"
column -s, -t < <(tail -n 8 "$RESULTS") || true