Source files / scripts · raw

loadtest.sh

#!/usr/bin/env bash
# Sustained-load thermal test: back-to-back long generations against the
# running tersler-llama server, logging GPU temps/power/clocks and the fan
# every few seconds.
#
#   tersler-loadtest [minutes]        default 20
#
# Writes $JR_PREFIX/bench/loadtest-<date>.csv and prints a summary.
set -euo pipefail
# shellcheck source=lib.sh
source "$(dirname "$(readlink -f "$0")")/lib.sh"
jr_load_config

minutes=${1:-20}
url=http://127.0.0.1:$LLAMA_PORT
out=$JR_PREFIX/bench/loadtest-$(date +%Y%m%d-%H%M).csv
curl -sf "$url/health" >/dev/null || jr_die "no server on $url (sudo systemctl start tersler-llama)"

fan=""
if [[ -n ${FAN_HWMON_NAME:-} && -n ${FAN_PWM:-} ]]; then
    for d in /sys/class/hwmon/hwmon*; do
        [[ $(cat "$d/name" 2>/dev/null) == "$FAN_HWMON_NAME" ]] && fan=$d
    done
fi
fan_rpm() {   # which fanN_input belongs to FAN_PWM is board-specific: FAN_INPUT in the config
    local f=${FAN_INPUT:-}
    [[ -n $fan && -n $f && -r $fan/${f}_input ]] && cat "$fan/${f}_input" || echo NA
}

echo "time_s,gpu,temp_c,power_w,sm_mhz,mem_mhz,util_pct,throttle,pwm,fan_rpm" > "$out"
t0=$(date +%s)
end=$(( t0 + minutes * 60 ))

# load: long answers, thinking off, so every second is decode
(
    prompts=(
        "Write a detailed, well-commented Python implementation of a B-tree with insert, delete and range queries, followed by unit tests."
        "Write a long technical article explaining how TCP congestion control evolved from Tahoe to BBR, with examples."
        "Implement a small Lisp interpreter in C, with a REPL, garbage collection and error handling. Explain each part."
    )
    i=0
    while (( $(date +%s) < end )); do
        p=${prompts[i % ${#prompts[@]}]}; i=$((i + 1))
        curl -s "$url/v1/chat/completions" -H 'Content-Type: application/json' \
            -d "$(jq -n --arg p "$p" '{messages:[{role:"user",content:$p}], max_tokens:4096,
                  chat_template_kwargs:{enable_thinking:false}}')" \
            | jq -r '.timings | "req: \(.predicted_n) tok at \(.predicted_per_second|floor) tok/s"' || true
    done
) &
load=$!
trap 'kill $load 2>/dev/null || true' EXIT

while (( $(date +%s) < end )); do
    now=$(( $(date +%s) - t0 ))
    pwm=NA; [[ -n $fan && -r $fan/$FAN_PWM ]] && pwm=$(cat "$fan/$FAN_PWM")
    rpm=$(fan_rpm)
    nvidia-smi --query-gpu=index,temperature.gpu,power.draw,clocks.sm,clocks.mem,utilization.gpu,clocks_throttle_reasons.active \
        --format=csv,noheader,nounits | while IFS=', ' read -r g t p sm mem u thr; do
        echo "$now,$g,$t,$p,$sm,$mem,$u,$thr,$pwm,$rpm" >> "$out"
    done
    sleep 5
done
wait "$load" 2>/dev/null || true

jr_log "log: $out"
python3 -I - "$out" <<'PY'
import csv, sys
rows = list(csv.DictReader(open(sys.argv[1])))
if not rows: sys.exit("no samples")
t_end = max(int(r["time_s"]) for r in rows)
# throttle reasons that mean heat: HW slowdown 0x8, SW thermal 0x20, HW thermal 0x40
# (0x4 = the power cap we set, 0x1 = idle: expected)
HEAT = 0x8 | 0x20 | 0x40
for g in sorted({r["gpu"] for r in rows}):
    rs = [r for r in rows if r["gpu"] == g]
    temps = [int(r["temp_c"]) for r in rs]
    late = [int(r["temp_c"]) for r in rs if int(r["time_s"]) >= t_end - 120]
    power = [float(r["power_w"]) for r in rs]
    sm = [int(r["sm_mhz"]) for r in rs if int(r["util_pct"]) > 50]
    hot = sum(1 for r in rs if int(r["throttle"], 16) & HEAT)
    print(f"GPU {g}: max {max(temps)} C, last 2 min avg {sum(late)/len(late):.1f} C, "
          f"power avg {sum(power)/len(power):.0f} W / max {max(power):.0f} W, "
          f"SM clock under load {min(sm, default=0)}-{max(sm, default=0)} MHz, "
          f"thermal throttle in {hot} of {len(rs)} samples")
pwm = [int(r["pwm"]) for r in rows if r["pwm"] != "NA"]
rpm = [int(r["fan_rpm"]) for r in rows if r["fan_rpm"] != "NA"]
if pwm: print(f"fan: pwm max {max(pwm)}/255" + (f", rpm max {max(rpm)}" if rpm else ""))
PY