RuView/scripts/run-ruforecast-benchmarks.sh

#!/usr/bin/env bash
# Reproducible CPU benchmark evidence runner for ADR-348.
#
# This script never downloads a model or dataset and never enables CUDA. The
# benchmark fixtures are generated by the checked-in Rust benches, so the
# source commit and Cargo.lock digest identify the fixture implementation.

set -Eeuo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd -P)"

TOOLCHAIN="${RUFORECAST_TOOLCHAIN:-1.92.0}"
THREADS="${RUFORECAST_THREADS:-1}"
WARM_UP_SECONDS="${RUFORECAST_WARM_UP_SECONDS:-3}"
MEASUREMENT_SECONDS="${RUFORECAST_MEASUREMENT_SECONDS:-10}"
SAMPLE_SIZE="${RUFORECAST_SAMPLE_SIZE:-30}"
CPUSET="${RUFORECAST_CPUSET:-}"
ALLOW_DIRTY="${RUFORECAST_ALLOW_DIRTY:-0}"
INCLUDE_LARGE="${RUFORECAST_BENCH_LARGE:-0}"

die() {
    printf 'ruforecast benchmark: %s\n' "$*" >&2
    exit 1
}

require_command() {
    command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
}

require_uint() {
    local name="$1"
    local value="$2"
    [[ "$value" =~ ^[0-9]+$ ]] || die "${name} must be an unsigned integer"
}

[[ "$TOOLCHAIN" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \
    || die "RUFORECAST_TOOLCHAIN must be an exact Rust release such as 1.92.0"
require_uint RUFORECAST_THREADS "$THREADS"
require_uint RUFORECAST_WARM_UP_SECONDS "$WARM_UP_SECONDS"
require_uint RUFORECAST_MEASUREMENT_SECONDS "$MEASUREMENT_SECONDS"
require_uint RUFORECAST_SAMPLE_SIZE "$SAMPLE_SIZE"
(( THREADS >= 1 )) || die "RUFORECAST_THREADS must be at least 1"
(( WARM_UP_SECONDS >= 1 )) || die "RUFORECAST_WARM_UP_SECONDS must be at least 1"
(( MEASUREMENT_SECONDS >= 1 )) || die "RUFORECAST_MEASUREMENT_SECONDS must be at least 1"
(( SAMPLE_SIZE >= 10 )) || die "RUFORECAST_SAMPLE_SIZE must be at least 10 for Criterion"
[[ "$ALLOW_DIRTY" == "0" || "$ALLOW_DIRTY" == "1" ]] \
    || die "RUFORECAST_ALLOW_DIRTY must be 0 or 1"
[[ "$INCLUDE_LARGE" == "0" || "$INCLUDE_LARGE" == "1" ]] \
    || die "RUFORECAST_BENCH_LARGE must be 0 or 1"

if [[ -n "$CPUSET" ]]; then
    [[ "$CPUSET" =~ ^[0-9,-]+$ ]] || die "RUFORECAST_CPUSET contains unsupported characters"
    require_command taskset
fi

for command_name in cargo rustc git jq sha256sum; do
    require_command "$command_name"
done

cd "$REPO_ROOT"

for manifest in \
    v2/crates/ruview-forecast-core/Cargo.toml \
    v2/crates/ruview-forecast-model/Cargo.toml \
    v2/crates/ruview-forecast-train/Cargo.toml; do
    [[ -f "$manifest" ]] || die "forecast crate is missing: $manifest"
done

WORKTREE_CLEAN=true
if [[ -n "$(git status --porcelain)" ]]; then
    WORKTREE_CLEAN=false
    [[ "$ALLOW_DIRTY" == "1" ]] \
        || die "worktree changes detected; commit them or set RUFORECAST_ALLOW_DIRTY=1 for diagnostic evidence"
fi

COMMIT="$(git rev-parse HEAD)"
SHORT_COMMIT="$(git rev-parse --short=12 HEAD)"
LOCK_SHA256="$(sha256sum v2/Cargo.lock | awk '{print $1}')"
RUN_UTC="$(date -u +'%Y%m%dT%H%M%SZ')"
SOURCE_DATE_EPOCH="$(git show -s --format=%ct HEAD)"

DEFAULT_OUTPUT="${REPO_ROOT}/target/ruforecast-evidence/${RUN_UTC}-${SHORT_COMMIT}"
OUTPUT_DIR="${RUFORECAST_OUTPUT_DIR:-$DEFAULT_OUTPUT}"
if [[ "$OUTPUT_DIR" != /* ]]; then
    OUTPUT_DIR="${REPO_ROOT}/${OUTPUT_DIR}"
fi
if [[ -d "$OUTPUT_DIR" && -n "$(find "$OUTPUT_DIR" -mindepth 1 -maxdepth 1 -print -quit)" ]]; then
    die "output directory is not empty: $OUTPUT_DIR"
fi
mkdir -p "$OUTPUT_DIR"

finalize_evidence() {
    local exit_status=$?
    local sums_status=0
    local sums_tmp=""
    trap - EXIT
    set +e

    if [[ -d "${OUTPUT_DIR:-}" ]]; then
        if (( exit_status == 0 )); then
            printf 'status=PASSED\nexit_code=0\n' >"${OUTPUT_DIR}/run-status.txt"
        else
            printf 'status=FAILED\nexit_code=%d\n' "$exit_status" \
                >"${OUTPUT_DIR}/run-status.txt"
        fi

        sums_tmp="$(mktemp "${OUTPUT_DIR}.SHA256SUMS.XXXXXX")" || sums_status=$?
        if (( sums_status == 0 )); then
            (
                cd "$OUTPUT_DIR" || exit
                find . -maxdepth 1 -type f ! -name SHA256SUMS -printf '%P\0' \
                    | sort -z \
                    | xargs -0 sha256sum
            ) >"$sums_tmp" || sums_status=$?
        fi
        if (( sums_status == 0 )); then
            mv "$sums_tmp" "${OUTPUT_DIR}/SHA256SUMS" || sums_status=$?
        elif [[ -n "$sums_tmp" ]]; then
            rm -f "$sums_tmp"
        fi
    fi

    if (( exit_status == 0 && sums_status != 0 )); then
        exit_status=$sums_status
    fi
    exit "$exit_status"
}
trap finalize_evidence EXIT

export RAYON_NUM_THREADS="$THREADS"
export OMP_NUM_THREADS="$THREADS"
export OPENBLAS_NUM_THREADS="$THREADS"
export MKL_NUM_THREADS="$THREADS"
export NUMEXPR_NUM_THREADS="$THREADS"
export CARGO_PROFILE_BENCH_DEBUG=0
export CARGO_PROFILE_RELEASE_DEBUG=0
export CARGO_INCREMENTAL=0
export RUST_BACKTRACE=1
export SOURCE_DATE_EPOCH
export TZ=UTC
export LC_ALL=C
export RUFORECAST_BENCH_LARGE="$INCLUDE_LARGE"

CARGO=(cargo "+${TOOLCHAIN}")
if [[ -n "$CPUSET" ]]; then
    CARGO=(taskset -c "$CPUSET" cargo "+${TOOLCHAIN}")
fi

"${CARGO[@]}" --version >"${OUTPUT_DIR}/cargo-version.txt"
if [[ -n "$CPUSET" ]]; then
    taskset -c "$CPUSET" rustc "+${TOOLCHAIN}" -vV >"${OUTPUT_DIR}/rustc-version.txt"
else
    rustc "+${TOOLCHAIN}" -vV >"${OUTPUT_DIR}/rustc-version.txt"
fi
if command -v lscpu >/dev/null 2>&1; then
    lscpu -J >"${OUTPUT_DIR}/cpu.json"
fi
uname -srmo >"${OUTPUT_DIR}/kernel.txt"
if [[ -r /etc/os-release ]]; then
    cp /etc/os-release "${OUTPUT_DIR}/os-release.txt"
fi
if command -v free >/dev/null 2>&1; then
    free -b >"${OUTPUT_DIR}/memory.txt"
fi

printf \
    'RUFORECAST_TOOLCHAIN=%q RUFORECAST_THREADS=%q RUFORECAST_CPUSET=%q RUFORECAST_WARM_UP_SECONDS=%q RUFORECAST_MEASUREMENT_SECONDS=%q RUFORECAST_SAMPLE_SIZE=%q RUFORECAST_ALLOW_DIRTY=%q RUFORECAST_BENCH_LARGE=%q scripts/run-ruforecast-benchmarks.sh\n' \
    "$TOOLCHAIN" \
    "$THREADS" \
    "$CPUSET" \
    "$WARM_UP_SECONDS" \
    "$MEASUREMENT_SECONDS" \
    "$SAMPLE_SIZE" \
    "$ALLOW_DIRTY" \
    "$INCLUDE_LARGE" \
    >"${OUTPUT_DIR}/reproducer.txt"

EVIDENCE_LABEL="MEASURED"
EVIDENCE_SCOPE="LOCAL_CPU_SYNTHETIC_FIXTURE"
if [[ "$WORKTREE_CLEAN" != "true" ]]; then
    EVIDENCE_LABEL="SYNTHETIC"
    EVIDENCE_SCOPE="DIRTY_WORKTREE_DIAGNOSTIC_ONLY"
fi

jq -n \
    --arg schema_version "1" \
    --arg evidence_label "$EVIDENCE_LABEL" \
    --arg evidence_scope "$EVIDENCE_SCOPE" \
    --arg commit "$COMMIT" \
    --arg lock_sha256 "$LOCK_SHA256" \
    --arg toolchain "$TOOLCHAIN" \
    --arg run_utc "$RUN_UTC" \
    --argjson source_date_epoch "$SOURCE_DATE_EPOCH" \
    --arg cpuset "$CPUSET" \
    --argjson include_large "$INCLUDE_LARGE" \
    --argjson worktree_clean "$WORKTREE_CLEAN" \
    --argjson threads "$THREADS" \
    --argjson warm_up_seconds "$WARM_UP_SECONDS" \
    --argjson measurement_seconds "$MEASUREMENT_SECONDS" \
    --argjson sample_size "$SAMPLE_SIZE" \
    '{
        schema_version: ($schema_version | tonumber),
        evidence_label: $evidence_label,
        evidence_scope: $evidence_scope,
        claim_status: "UNREVIEWED",
        input_class: "SYNTHETIC",
        backend: "burn-ndarray-cpu",
        cuda_enabled: false,
        source: {
            commit: $commit,
            cargo_lock_sha256: $lock_sha256,
            worktree_clean: $worktree_clean,
            source_date_epoch: $source_date_epoch
        },
        environment: {
            rust_toolchain: $toolchain,
            threads: $threads,
            cpuset: (if $cpuset == "" then null else $cpuset end)
        },
        criterion: {
            warm_up_seconds: $warm_up_seconds,
            measurement_seconds: $measurement_seconds,
            sample_size: $sample_size,
            include_large_profile: ($include_large == 1)
        },
        run_utc: $run_utc,
        caveat: "Criterion timing is host-specific. This report is not a RuView capability claim until reviewed against ADR-348."
    }' >"${OUTPUT_DIR}/metadata.json"

run_logged() {
    local log_name="$1"
    shift
    "$@" 2>&1 | tee "${OUTPUT_DIR}/${log_name}.log"
}

run_logged core-tests \
    "${CARGO[@]}" test --manifest-path v2/Cargo.toml --locked \
    -p ruview-forecast-core --no-default-features --lib --tests

run_logged model-cpu-tests \
    "${CARGO[@]}" test --manifest-path v2/Cargo.toml --locked \
    -p ruview-forecast-model --no-default-features --features cpu --lib --tests

run_logged train-cpu-tests \
    "${CARGO[@]}" test --manifest-path v2/Cargo.toml --locked \
    -p ruview-forecast-train --no-default-features --features cpu,cli --lib --bins

run_logged local-jsonl-training-smoke \
    "${CARGO[@]}" test --manifest-path v2/Cargo.toml --locked \
    -p ruview-forecast-train --no-default-features --features cpu \
    --test local_jsonl_smoke -- \
    --exact local_hash_addressed_jsonl_executes_one_real_optimizer_step

run_logged cli-smoke-idempotency \
    "${CARGO[@]}" test --manifest-path v2/Cargo.toml --locked \
    -p ruview-forecast-train --no-default-features --features cpu,cli \
    --test cli_smoke -- \
    --exact cli_smoke_trains_and_writes_the_complete_candidate_set

run_logged forecast-inference-compile \
    "${CARGO[@]}" bench --manifest-path v2/Cargo.toml --locked \
    -p ruview-forecast-model --no-default-features --features cpu \
    --bench forecast_inference --no-run

run_logged data-pipeline-compile \
    "${CARGO[@]}" bench --manifest-path v2/Cargo.toml --locked \
    -p ruview-forecast-train --no-default-features --features cpu \
    --bench data_pipeline --no-run

run_logged forecast-inference \
    "${CARGO[@]}" bench --manifest-path v2/Cargo.toml --locked \
    -p ruview-forecast-model --no-default-features --features cpu \
    --bench forecast_inference -- \
    --warm-up-time "$WARM_UP_SECONDS" \
    --measurement-time "$MEASUREMENT_SECONDS" \
    --sample-size "$SAMPLE_SIZE"

run_logged data-pipeline \
    "${CARGO[@]}" bench --manifest-path v2/Cargo.toml --locked \
    -p ruview-forecast-train --no-default-features --features cpu \
    --bench data_pipeline -- \
    --warm-up-time "$WARM_UP_SECONDS" \
    --measurement-time "$MEASUREMENT_SECONDS" \
    --sample-size "$SAMPLE_SIZE"

printf 'RuForecast CPU outputs collected; finalizing hashes in %s\n' "$OUTPUT_DIR"
printf 'Review metadata.json and the two Criterion logs before accepting or publishing evidence.\n'