diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 430d5e820a..c279099b88 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -364,20 +364,26 @@ jobs: run: | cpus=$(nproc) busy=$(awk '{ printf "%d", $1 + 0.5 }' /proc/loadavg) - mem_gb=$(awk '/^MemAvailable:/ { print int($2 / 1048576) }' /proc/meminfo) - jobs=$(( cpus - busy )) - jobs=$(( jobs < mem_gb / 3 ? jobs : mem_gb / 3 )) - jobs=$(( jobs > 8 ? 8 : jobs )) - jobs=$(( jobs < 1 ? 1 : jobs )) - echo "cpus=$cpus load=$busy mem_available_gb=$mem_gb -> build jobs $jobs" - echo "CMAKE_BUILD_PARALLEL_LEVEL=$jobs" >> "$GITHUB_ENV" - echo "CTEST_PARALLEL_LEVEL=$(( jobs > 4 ? 4 : jobs ))" >> "$GITHUB_ENV" + mem_mb=$(awk '/^MemTotal:/ { print int($2 / 1024) }' /proc/meminfo) + slots=$(( (mem_mb - 2048) / 1024 )) + slots=$(( slots < 1 ? 1 : slots )) + tests=$(( cpus - busy )) + tests=$(( tests < 1 ? 1 : tests )) + slot_dir=$CCACHE_DIR/compile-slots + mkdir -p -m 1777 "$slot_dir" + test -w "$slot_dir" + echo "cpus=$cpus load=$busy mem_total_mb=$mem_mb -> build jobs $cpus, cold compiles $slots per host, tests $tests" + echo "CMAKE_BUILD_PARALLEL_LEVEL=$cpus" >> "$GITHUB_ENV" + echo "CTEST_PARALLEL_LEVEL=$tests" >> "$GITHUB_ENV" + echo "CCACHE_PREFIX=$GITHUB_WORKSPACE/tools/compile_slot.sh" >> "$GITHUB_ENV" + echo "ANYPS5_COMPILE_SLOT_DIR=$slot_dir" >> "$GITHUB_ENV" + echo "ANYPS5_COMPILE_SLOTS=$slots" >> "$GITHUB_ENV" - run: cmake -S . -B build $CONFIGURE -DANYPS5_ENABLE_SPIRV_TOOLS=ON "-DCMAKE_EXE_LINKER_FLAGS=$LINKER_FLAGS" "-DCMAKE_SHARED_LINKER_FLAGS=$LINKER_FLAGS" "-DCMAKE_MODULE_LINKER_FLAGS=$LINKER_FLAGS" - name: Check that every test executable is registered with ctest run: python3 tools/check_test_registration.py build - run: cmake --build build -- -l "$(nproc)" - run: cmake --build build --target libs -- -l "$(nproc)" - - run: ctest --test-dir build --output-on-failure --timeout 120 --output-log ctest.log --output-junit ctest.xml + - run: ctest --test-dir build --test-load "$(nproc)" --output-on-failure --timeout 120 --output-log ctest.log --output-junit ctest.xml env: ANYPS5_REQUIRE_VULKAN: 1 GALLIUM_OVERRIDE_CPU_CAPS: avx diff --git a/tools/compile_slot.sh b/tools/compile_slot.sh new file mode 100755 index 0000000000..2f01329de4 --- /dev/null +++ b/tools/compile_slot.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -u +dir=${ANYPS5_COMPILE_SLOT_DIR:?} +slots=${ANYPS5_COMPILE_SLOTS:?} +while :; do + for ((i = 0; i < slots; i++)); do + [ -e "$dir/slot-$i" ] || : >> "$dir/slot-$i" + exec {fd}<"$dir/slot-$i" + if flock -n "$fd"; then + exec "$@" + fi + exec {fd}>&- + done + sleep 0.1 +done