Merge pull request #7080 from Rengundom/feat/trace-bench-fields

feat(trace): record host, title and per-frame benchmark data
This commit is contained in:
Dean authored and GitHub committed 2026-10-20 21:19:24 +13:00
commit f9012eed0d
14 files changed
+547 -2

No files matched your search

@@ -10,6 +10,10 @@
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <condition_variable>
#include <exception>
#include <functional>
#include <limits>
#include <map>
#include <memory>
#include <mutex>
@@ -18,6 +22,7 @@
#include <stdexcept>
#include <string>
#include <string_view>
#include <thread>
#include <utility>
#include <vector>
@@ -267,6 +272,18 @@ private:
Clock::duration folded{};
};
struct TraceHost {
std::string cpu;
std::uint32_t threads = 0;
std::uint64_t memoryMiB = 0;
};
struct TraceTitle {
std::string id;
std::string name;
std::string contentVersion;
};
struct TraceSummary {
std::string commit;
std::optional<bool> dirty;
@@ -281,6 +298,135 @@ struct TraceSummary {
double gpuBusyMs = 0;
std::string trace;
std::optional<std::pair<double, double>> interval;
std::optional<TraceHost> host;
std::optional<TraceTitle> title;
};
struct TraceBenchInterval {
double startS = 0;
double endS = 0;
std::uint64_t firstPresent = 0;
std::uint32_t width = 0;
std::uint32_t height = 0;
double gpuBusyMs = 0;
std::vector<std::uint32_t> frameUs;
};
using TraceBenchIntervals = std::vector<std::shared_ptr<const TraceBenchInterval>>;
class TraceBench {
public:
static constexpr std::size_t MaximumIntervals = 240;
static void Keep(TraceBenchIntervals& kept, std::shared_ptr<const TraceBenchInterval> interval, std::uint64_t& dropped) {
kept.push_back(std::move(interval));
if (kept.size() <= MaximumIntervals) return;
const auto excess = kept.size() - MaximumIntervals;
kept.erase(kept.begin(), kept.begin() + static_cast<std::ptrdiff_t>(excess));
dropped += excess;
}
static bool Parse(const char* bench, bool summaryEvery) {
if (bench == nullptr) return false;
if (std::string_view(bench) != "1") throw std::invalid_argument("APS5_TRACE_BENCH=" + std::string(bench) + " is not 1");
if (!summaryEvery) throw std::invalid_argument("APS5_TRACE_BENCH=1 needs APS5_TRACE_SUMMARY_EVERY");
return true;
}
};
struct TraceBenchSnapshot {
TraceSummary summary;
TraceBenchIntervals intervals;
std::uint64_t dropped = 0;
};
class TraceBenchWriter {
public:
explicit TraceBenchWriter(std::function<void(const TraceBenchSnapshot&)> writer) : writer(std::move(writer)) {
if (!this->writer) throw std::invalid_argument("invalid benchmark writer");
worker = std::thread([this] { run(); });
}
TraceBenchWriter(const TraceBenchWriter&) = delete;
TraceBenchWriter& operator=(const TraceBenchWriter&) = delete;
~TraceBenchWriter() {
Stop();
}
void Stop() {
std::lock_guard stopLock(stopMutex);
{
std::lock_guard lock(mutex);
stopping = true;
}
changed.notify_one();
if (worker.joinable()) worker.join();
}
void Write(TraceBenchSnapshot snapshot) {
{
std::lock_guard lock(mutex);
if (failure) std::rethrow_exception(failure);
if (stopping) throw std::runtime_error("benchmark writer is stopping");
if (pending) ++superseded;
pending = std::move(snapshot);
}
changed.notify_one();
}
void Flush() {
std::unique_lock lock(mutex);
idle.wait(lock, [&] { return failure || (!pending && !writing); });
if (failure) std::rethrow_exception(failure);
}
std::uint64_t Superseded() {
std::lock_guard lock(mutex);
return superseded;
}
private:
void run() noexcept {
try {
for (;;) {
TraceBenchSnapshot snapshot;
{
std::unique_lock lock(mutex);
changed.wait(lock, [&] { return stopping || pending.has_value(); });
if (!pending) return;
snapshot = std::move(*pending);
pending.reset();
writing = true;
}
writer(snapshot);
{
std::lock_guard lock(mutex);
writing = false;
}
idle.notify_all();
}
} catch (...) {
{
std::lock_guard lock(mutex);
failure = std::current_exception();
writing = false;
}
idle.notify_all();
}
}
std::function<void(const TraceBenchSnapshot&)> writer;
std::mutex stopMutex;
std::mutex mutex;
std::condition_variable changed;
std::condition_variable idle;
std::optional<TraceBenchSnapshot> pending;
std::uint64_t superseded = 0;
bool writing = false;
bool stopping = false;
std::exception_ptr failure;
std::thread worker;
};
struct TraceGpuRange {
@@ -429,6 +575,7 @@ public:
AppendString(output, summary.driver);
output += ",\"os\":";
AppendString(output, summary.os);
AppendMachine(output, summary);
output += ",\"resolution\":";
AppendString(output, std::to_string(summary.width) + "x" + std::to_string(summary.height));
output += ",\"window\":{\"spec\":";
@@ -502,7 +649,71 @@ public:
return output;
}
static std::vector<std::uint32_t> FrameMicroseconds(std::span<const Clock::time_point> presents) {
std::vector<std::uint32_t> frames;
for (std::size_t index = 1; index < presents.size(); ++index) {
const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(presents[index] - presents[index - 1]).count();
frames.push_back(static_cast<std::uint32_t>(std::clamp<std::int64_t>(elapsed, 0, std::numeric_limits<std::uint32_t>::max())));
}
return frames;
}
static std::string Bench(const TraceSummary& summary, std::span<const std::shared_ptr<const TraceBenchInterval>> intervals, std::uint64_t intervalsDropped) {
std::string output = "{\"format\":\"aps5-bench/1\",\"commit\":";
AppendString(output, summary.commit);
output += ",\"dirty\":";
output += summary.dirty.has_value() ? (*summary.dirty ? "true" : "false") : "null";
output += ",\"env\":{";
for (std::size_t index = 0; index < summary.environment.size(); ++index) {
if (index != 0) output += ',';
AppendString(output, summary.environment[index].first);
output += ':';
AppendString(output, summary.environment[index].second);
}
output += "},\"gpu\":";
AppendString(output, summary.gpu);
output += ",\"driver\":";
AppendString(output, summary.driver);
output += ",\"os\":";
AppendString(output, summary.os);
AppendMachine(output, summary);
output += ",\"intervals_dropped\":" + std::to_string(intervalsDropped) + ",\"intervals\":[";
for (std::size_t index = 0; index < intervals.size(); ++index) {
const auto& interval = *intervals[index];
if (index != 0) output += ',';
output += "\n{\"start_s\":" + number(interval.startS) + ",\"end_s\":" + number(interval.endS);
output += ",\"first_present\":" + std::to_string(interval.firstPresent);
output += ",\"resolution\":";
AppendString(output, std::to_string(interval.width) + "x" + std::to_string(interval.height));
output += ",\"gpu_busy_ms\":" + number(interval.gpuBusyMs) + ",\"frame_us\":[";
for (std::size_t frame = 0; frame < interval.frameUs.size(); ++frame) {
if (frame != 0) output += ',';
output += std::to_string(interval.frameUs[frame]);
}
output += "]}";
}
output += "\n]}\n";
return output;
}
private:
static void AppendMachine(std::string& output, const TraceSummary& summary) {
if (summary.host) {
output += ",\"cpu\":";
AppendString(output, summary.host->cpu);
output += ",\"cpu_threads\":" + std::to_string(summary.host->threads);
output += ",\"memory_mib\":" + std::to_string(summary.host->memoryMiB);
}
if (summary.title) {
output += ",\"title_id\":";
AppendString(output, summary.title->id);
output += ",\"title\":";
AppendString(output, summary.title->name);
output += ",\"content_version\":";
AppendString(output, summary.title->contentVersion);
}
}
static double microseconds(Clock::duration elapsed) {
return std::chrono::duration<double, std::micro>(elapsed).count();
}
@@ -521,7 +732,7 @@ private:
class PerformanceTrace {
public:
static bool Configured() {
static const bool configured = std::getenv("APS5_TRACE_FRAMES") != nullptr || std::getenv("APS5_TRACE_SECONDS") != nullptr || std::getenv("APS5_TRACE_SUMMARY_EVERY") != nullptr;
static const bool configured = std::getenv("APS5_TRACE_FRAMES") != nullptr || std::getenv("APS5_TRACE_SECONDS") != nullptr || std::getenv("APS5_TRACE_SUMMARY_EVERY") != nullptr || std::getenv("APS5_TRACE_BENCH") != nullptr;
return configured;
}
@@ -9,6 +9,7 @@ namespace AgcDriver {
class TraceFile {
public:
static void WriteGzip(const std::filesystem::path& path, std::string_view text);
static void ReplaceGzip(const std::filesystem::path& path, std::string_view text);
static void AppendLine(const std::filesystem::path& path, std::string_view line);
};
@@ -3,11 +3,15 @@
#include "prx/libSceAgcDriver/Execution/include/TraceFile.hpp"
#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
#include "prx/libSceAgcDriver/TraceRevision.hpp"
#include "prx/libkernel/AppMetadata/include/AppMetadata.hpp"
#include <array>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <limits>
#include <mutex>
#include <system_error>
#include <thread>
#ifdef _WIN32
#include <windows.h>
#else
@@ -15,6 +19,11 @@
#include <sys/utsname.h>
#include <unistd.h>
#endif
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
#elif defined(__x86_64__) || defined(__i386__)
#include <cpuid.h>
#endif
#ifndef _WIN32
extern char** environ;
@@ -46,6 +55,9 @@ struct Session {
double gpuBusyAtOpen = 0;
std::optional<TraceClockSample> gpuClock;
std::vector<Clock::time_point> presents;
bool bench = false;
TraceBenchIntervals benchIntervals;
std::uint64_t benchDropped = 0;
};
Session& session() {
@@ -63,6 +75,11 @@ std::unique_ptr<ProfileOutput>& summaryWriter() {
return output;
}
std::unique_ptr<TraceBenchWriter>& benchWriter() {
static std::unique_ptr<TraceBenchWriter> output;
return output;
}
std::filesystem::path traceDirectory() {
const char* directory = std::getenv("APS5_TRACE_DIR");
return std::filesystem::path(directory != nullptr ? directory : ".");
@@ -91,6 +108,64 @@ std::string operatingSystem() {
#endif
}
std::optional<std::string> processorName() {
std::array<std::uint32_t, 12> words{};
#if defined(_MSC_VER) && !defined(__clang__)
int registers[4]{};
__cpuid(registers, static_cast<int>(0x80000000u));
if (static_cast<std::uint32_t>(registers[0]) < 0x80000004u) return std::nullopt;
for (unsigned leaf = 0; leaf < 3; ++leaf) {
__cpuid(registers, static_cast<int>(0x80000002u + leaf));
std::memcpy(words.data() + leaf * 4, registers, sizeof registers);
}
#elif defined(__x86_64__) || defined(__i386__)
if (__get_cpuid_max(0x80000000u, nullptr) < 0x80000004u) return std::nullopt;
for (unsigned leaf = 0; leaf < 3; ++leaf) {
unsigned eax = 0, ebx = 0, ecx = 0, edx = 0;
if (__get_cpuid(0x80000002u + leaf, &eax, &ebx, &ecx, &edx) == 0) throw std::runtime_error("cpuid brand string leaf failed");
words[leaf * 4] = eax;
words[leaf * 4 + 1] = ebx;
words[leaf * 4 + 2] = ecx;
words[leaf * 4 + 3] = edx;
}
#else
return std::nullopt;
#endif
std::string name(reinterpret_cast<const char*>(words.data()), sizeof words);
name.resize(std::strlen(name.c_str()));
const auto first = name.find_first_not_of(' ');
if (first == std::string::npos) return std::nullopt;
return name.substr(first, name.find_last_not_of(' ') - first + 1);
}
std::uint64_t memoryMiB() {
#ifdef _WIN32
MEMORYSTATUSEX status{};
status.dwLength = sizeof(status);
if (::GlobalMemoryStatusEx(&status) == 0) throw std::system_error(static_cast<int>(::GetLastError()), std::system_category(), "GlobalMemoryStatusEx");
return status.ullTotalPhys / (1024 * 1024);
#else
const long pages = ::sysconf(_SC_PHYS_PAGES);
const long size = ::sysconf(_SC_PAGE_SIZE);
if (pages <= 0 || size <= 0) throw std::system_error(errno, std::generic_category(), "sysconf physical memory");
return static_cast<std::uint64_t>(pages) * static_cast<std::uint64_t>(size) / (1024 * 1024);
#endif
}
const std::optional<TraceHost>& host() {
static const auto value = []() -> std::optional<TraceHost> {
auto cpu = processorName();
if (!cpu) return std::nullopt;
return TraceHost{std::move(*cpu), std::thread::hardware_concurrency(), memoryMiB()};
}();
return value;
}
std::optional<TraceTitle> title() {
if (!HasAppParamJson_nid_no_patch()) return std::nullopt;
return TraceTitle{GetAppTitleId_nid_postfix().value, GetAppTitle_nid_postfix().value, GetAppContentVersion_nid_no_patch().value};
}
std::vector<std::pair<std::string, std::string>> environment() {
#ifdef _WIN32
char** entries = _environ;
@@ -120,6 +195,9 @@ TraceSummary identity(std::uint32_t width, std::uint32_t height, const VulkanDev
summary.os = operatingSystem();
summary.width = width;
summary.height = height;
summary.host = host();
static const auto running = title();
summary.title = running;
return summary;
}
@@ -185,6 +263,23 @@ void PerformanceTrace::FlushInterval(Clock::time_point now, std::uint32_t width,
summary.gpuBusyMs = VulkanDevice::PresentCounts().gpuBusyMs - state.intervalGpuBusy;
summary.interval = std::make_pair(seconds(state.intervalPresents.front() - state.first), seconds(now - state.first));
summaryWriter()->Write(TraceFormat::Summary(summary, snapshots, state.intervalPresents));
if (!state.bench) return;
TraceBench::Keep(state.benchIntervals, std::make_shared<const TraceBenchInterval>(TraceBenchInterval{summary.interval->first, summary.interval->second, summary.firstPresent, width, height, summary.gpuBusyMs, TraceFormat::FrameMicroseconds(state.intervalPresents)}), state.benchDropped);
if (!benchWriter()) {
const auto name = "aps5-bench-" + summary.commit + "-" + std::to_string(processId()) + ".json.gz";
benchWriter() = std::make_unique<TraceBenchWriter>([directory = traceDirectory(), name](const TraceBenchSnapshot& snapshot) {
try {
std::filesystem::create_directories(directory);
TraceFile::ReplaceGzip(directory / name, TraceFormat::Bench(snapshot.summary, snapshot.intervals, snapshot.dropped));
} catch (const std::exception& error) {
std::fprintf(stderr, "[trace] writing %s failed: %s\n", name.c_str(), error.what());
std::fflush(stderr);
throw;
}
});
ProfilePrint_nid_no_patch("[trace] benchmark frames go to %s, rewritten every interval\n", name.c_str());
}
benchWriter()->Write(TraceBenchSnapshot{std::move(summary), state.benchIntervals, state.benchDropped});
}
void PerformanceTrace::Present(std::uint32_t width, std::uint32_t height, const VulkanDevice* device) {
@@ -195,6 +290,7 @@ void PerformanceTrace::Present(std::uint32_t width, std::uint32_t height, const
if (!state.started) {
state.window = TraceWindow::Parse(std::getenv("APS5_TRACE_FRAMES"), std::getenv("APS5_TRACE_SECONDS"));
state.interval = TraceInterval::Parse(std::getenv("APS5_TRACE_SUMMARY_EVERY"));
state.bench = TraceBench::Parse(std::getenv("APS5_TRACE_BENCH"), state.interval.has_value());
state.first = now;
state.started = true;
}
@@ -287,6 +383,7 @@ void PerformanceTrace::Present(std::uint32_t width, std::uint32_t height, const
void PerformanceTrace::StopOutput() {
if (writer()) writer()->Stop();
if (summaryWriter()) summaryWriter()->Stop();
if (benchWriter()) benchWriter()->Stop();
}
}
@@ -6,7 +6,11 @@
#include <fstream>
#include <stdexcept>
#include <string>
#include <system_error>
#include <vector>
#ifdef _WIN32
#include <windows.h>
#endif
namespace AgcDriver {
@@ -50,6 +54,22 @@ void TraceFile::WriteGzip(const std::filesystem::path& path, std::string_view te
output.close();
}
void TraceFile::ReplaceGzip(const std::filesystem::path& path, std::string_view text) {
auto temporary = path;
temporary += ".tmp";
WriteGzip(temporary, text);
std::error_code error;
#ifdef _WIN32
if (::MoveFileExW(temporary.c_str(), path.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) == 0) error = std::error_code(static_cast<int>(::GetLastError()), std::system_category());
#else
std::filesystem::rename(temporary, path, error);
#endif
if (!error) return;
std::error_code removeError;
std::filesystem::remove(temporary, removeError);
throw std::system_error(error, "replacing " + path.string());
}
void TraceFile::AppendLine(const std::filesystem::path& path, std::string_view line) {
std::ofstream output;
output.exceptions(std::ios::badbit | std::ios::failbit);
@@ -1,9 +1,12 @@
#include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp"
#include "prx/libSceAgcDriver/Execution/include/TraceFile.hpp"
#include "zlib.h"
#include <atomic>
#include <condition_variable>
#include <fstream>
#include <iostream>
#include <iterator>
#include <mutex>
namespace {
@@ -346,6 +349,175 @@ void IntervalDroppedThreads() {
Check(line.find("\"queue worker 0x0\":{\"events_lost\":5,\"totals_full\":false}") != std::string::npos && line.find("events_full_after_s") == std::string::npos, "an interval without presents printed a loss time");
}
void SummaryMachineFields() {
const auto origin = Clock::now();
AgcDriver::TraceSummary summary;
summary.commit = "abc";
summary.dirty = false;
summary.gpu = "GPU";
summary.driver = "Driver 1";
summary.os = "OS 1";
summary.width = 1920;
summary.height = 1080;
summary.window = "summary_every 30";
summary.host = AgcDriver::TraceHost{"CPU \"X\"", 16, 32000};
summary.title = AgcDriver::TraceTitle{"PPSA00000", "Example", "01.002.000"};
const std::vector<Clock::time_point> presents{origin, origin + milliseconds(16)};
const auto line = TraceFormat::Summary(summary, {}, presents);
Check(line.find("\"os\":\"OS 1\",\"cpu\":\"CPU \\\"X\\\"\",\"cpu_threads\":16,\"memory_mib\":32000,\"title_id\":\"PPSA00000\",\"title\":\"Example\",\"content_version\":\"01.002.000\",\"resolution\":\"1920x1080\"") != std::string::npos, "the summary line did not name the machine and title");
summary.title.reset();
Check(TraceFormat::Summary(summary, {}, presents).find("title_id") == std::string::npos, "a summary without param.json named a title");
summary.host.reset();
const auto bare = TraceFormat::Summary(summary, {}, presents);
Check(bare.find("\"os\":\"OS 1\",\"resolution\":\"1920x1080\"") != std::string::npos && bare.find("cpu") == std::string::npos, "a summary on a host without a CPU name changed its fields");
}
void FrameMicroseconds() {
const auto origin = Clock::now();
const std::vector<Clock::time_point> presents{origin, origin + microseconds(16667), origin + microseconds(50000), origin + microseconds(50000)};
Check(TraceFormat::FrameMicroseconds(presents) == std::vector<std::uint32_t>{16667, 33333, 0}, "frame intervals in microseconds changed");
Check(TraceFormat::FrameMicroseconds({}).empty(), "no presents gave frame intervals");
}
void BenchFormat() {
AgcDriver::TraceSummary summary;
summary.commit = "abc";
summary.environment = {{"APS5_TRACE_BENCH", "1"}};
summary.gpu = "GPU";
summary.driver = "Driver 1";
summary.os = "OS 1";
summary.host = AgcDriver::TraceHost{"CPU", 8, 16000};
const AgcDriver::TraceBenchIntervals intervals{
std::make_shared<const AgcDriver::TraceBenchInterval>(AgcDriver::TraceBenchInterval{0, 30.5, 0, 1920, 1080, 120.25, {16667, 16666}}),
std::make_shared<const AgcDriver::TraceBenchInterval>(AgcDriver::TraceBenchInterval{30.5, 61, 1830, 2560, 1440, 98.5, {}}),
};
const std::string expected =
"{\"format\":\"aps5-bench/1\",\"commit\":\"abc\",\"dirty\":null,\"env\":{\"APS5_TRACE_BENCH\":\"1\"},\"gpu\":\"GPU\",\"driver\":\"Driver 1\",\"os\":\"OS 1\","
"\"cpu\":\"CPU\",\"cpu_threads\":8,\"memory_mib\":16000,\"intervals_dropped\":3,\"intervals\":[\n"
"{\"start_s\":0.000,\"end_s\":30.500,\"first_present\":0,\"resolution\":\"1920x1080\",\"gpu_busy_ms\":120.250,\"frame_us\":[16667,16666]},\n"
"{\"start_s\":30.500,\"end_s\":61.000,\"first_present\":1830,\"resolution\":\"2560x1440\",\"gpu_busy_ms\":98.500,\"frame_us\":[]}\n]}\n";
Check(TraceFormat::Bench(summary, intervals, 3) == expected, "benchmark file changed");
}
void BenchRotation() {
AgcDriver::TraceBenchIntervals kept;
std::uint64_t dropped = 0;
for (std::size_t index = 0; index < AgcDriver::TraceBench::MaximumIntervals; ++index) AgcDriver::TraceBench::Keep(kept, std::make_shared<const AgcDriver::TraceBenchInterval>(AgcDriver::TraceBenchInterval{static_cast<double>(index)}), dropped);
Check(kept.size() == AgcDriver::TraceBench::MaximumIntervals && dropped == 0, "intervals were dropped before the file was full");
for (std::size_t index = 0; index < 5; ++index) AgcDriver::TraceBench::Keep(kept, std::make_shared<const AgcDriver::TraceBenchInterval>(AgcDriver::TraceBenchInterval{static_cast<double>(AgcDriver::TraceBench::MaximumIntervals + index)}), dropped);
Check(kept.size() == AgcDriver::TraceBench::MaximumIntervals && dropped == 5, "a full file did not drop one interval per new one");
Check(kept.front()->startS == 5 && kept.back()->startS == static_cast<double>(AgcDriver::TraceBench::MaximumIntervals + 4), "the file did not keep the newest intervals");
const auto file = TraceFormat::Bench({}, kept, dropped);
Check(file.find("\"intervals_dropped\":5,") != std::string::npos && file.find("{\"start_s\":5.000,") != std::string::npos && file.find("{\"start_s\":4.000,") == std::string::npos, "the file does not match the kept intervals");
}
void BenchWriterKeepsNewest() {
std::mutex mutex;
std::condition_variable changed;
bool entered = false;
bool release = false;
std::vector<std::uint64_t> written;
AgcDriver::TraceBenchWriter writer([&](const AgcDriver::TraceBenchSnapshot& snapshot) {
std::unique_lock lock(mutex);
written.push_back(snapshot.dropped);
entered = true;
changed.notify_all();
changed.wait(lock, [&] { return release; });
});
writer.Write({{}, {}, 1});
{
std::unique_lock lock(mutex);
changed.wait(lock, [&] { return entered; });
}
writer.Write({{}, {}, 2});
writer.Write({{}, {}, 3});
writer.Write({{}, {}, 4});
{
std::lock_guard lock(mutex);
release = true;
}
changed.notify_all();
writer.Flush();
writer.Stop();
Check(written == std::vector<std::uint64_t>{1, 4}, "a benchmark snapshot was written after a newer one replaced it");
Check(writer.Superseded() == 2, "replaced benchmark snapshots were not counted");
}
void BenchWriterStopWrites() {
std::vector<std::uint64_t> written;
{
AgcDriver::TraceBenchWriter writer([&](const AgcDriver::TraceBenchSnapshot& snapshot) { written.push_back(snapshot.dropped); });
writer.Write({{}, {}, 7});
}
Check(written == std::vector<std::uint64_t>{7}, "stopping the benchmark writer lost its pending snapshot");
}
std::string ReadGzip(const std::filesystem::path& path) {
std::ifstream input(path, std::ios::binary);
const std::vector<unsigned char> bytes((std::istreambuf_iterator<char>(input)), std::istreambuf_iterator<char>());
Check(bytes.size() > 18, "the gzip file is truncated");
std::size_t size = 0;
for (unsigned index = 0; index < 4; ++index) size |= static_cast<std::size_t>(bytes[bytes.size() - 4 + index]) << (8 * index);
z_stream stream{};
Check(inflateInit2(&stream, -MAX_WBITS) == Z_OK, "inflateInit2 failed");
std::string inflated(size, 0);
stream.next_in = const_cast<Bytef*>(bytes.data() + 10);
stream.avail_in = static_cast<uInt>(bytes.size() - 18);
stream.next_out = reinterpret_cast<Bytef*>(inflated.data());
stream.avail_out = static_cast<uInt>(inflated.size());
const int status = inflate(&stream, Z_FINISH);
inflateEnd(&stream);
Check(status == Z_STREAM_END && stream.avail_out == 0, "the gzip file does not inflate to its size");
return inflated;
}
void ReplaceGzipFile() {
const auto directory = std::filesystem::temp_directory_path() / ("aps5-bench-test-" + std::to_string(Clock::now().time_since_epoch().count()));
std::filesystem::create_directories(directory);
const auto path = directory / "aps5-bench.json.gz";
auto temporary = path;
temporary += ".tmp";
AgcDriver::TraceFile::ReplaceGzip(path, "{\"first\":1}\n");
AgcDriver::TraceFile::ReplaceGzip(path, "{\"second\":2}\n");
Check(ReadGzip(path) == "{\"second\":2}\n" && !std::filesystem::exists(temporary), "the benchmark file was not replaced");
const auto blocked = directory / "aps5-bench-blocked.json.gz";
auto blockedTemporary = blocked;
blockedTemporary += ".tmp";
std::filesystem::create_directory(blocked);
bool failed = false;
try { AgcDriver::TraceFile::ReplaceGzip(blocked, "{\"third\":3}\n"); } catch (const std::system_error&) { failed = true; }
const bool blockedKept = std::filesystem::is_directory(blocked);
const bool temporaryLeft = std::filesystem::exists(blockedTemporary);
const auto kept = ReadGzip(path);
#ifndef _WIN32
// A directory at <name>.tmp makes the write itself fail, which is the case ReplaceGzip exists
// for: the previous file must survive, so an in-place rewrite fails this check. The failure is
// a std::ios_base::failure thrown inside libstdc++'s ofstream, which MinGW builds could not
// unwind to this catch, so the case runs on the other platforms only.
std::filesystem::create_directory(temporary);
bool writeFailed = false;
try { AgcDriver::TraceFile::ReplaceGzip(path, "{\"third\":3}\n"); } catch (const std::exception&) { writeFailed = true; }
const auto keptAfterWrite = ReadGzip(path);
#endif
std::filesystem::remove_all(directory);
Check(failed && blockedKept && !temporaryLeft, "a replace that could not move its file over the destination did not fail cleanly or left its temporary file");
Check(kept == "{\"second\":2}\n", "a failed replace of another file damaged the benchmark file");
#ifndef _WIN32
Check(writeFailed && keptAfterWrite == "{\"second\":2}\n", "a failed rewrite damaged the previous benchmark file");
#endif
}
void BenchSwitch() {
Check(!AgcDriver::TraceBench::Parse(nullptr, false), "the benchmark file was enabled without its switch");
Check(AgcDriver::TraceBench::Parse("1", true), "APS5_TRACE_BENCH=1 was not accepted");
for (const auto& [text, every] : {std::pair{"1", false}, std::pair{"0", true}, std::pair{"yes", true}, std::pair{"", true}}) {
bool rejected = false;
try { AgcDriver::TraceBench::Parse(text, every); }
catch (const std::invalid_argument&) { rejected = true; }
Check(rejected, "a malformed or unsupported APS5_TRACE_BENCH was accepted");
}
}
void GzipFile() {
std::string text;
for (int index = 0; index < 20000; ++index) text += "{\"name\":\"Driver.Packet\",\"ph\":\"X\",\"ts\":" + std::to_string(index) + "}\n";
@@ -413,7 +585,15 @@ int main() {
TotalsSince();
IntervalSummaryFields();
IntervalDroppedThreads();
SummaryMachineFields();
FrameMicroseconds();
BenchFormat();
BenchRotation();
BenchWriterKeepsNewest();
BenchWriterStopWrites();
BenchSwitch();
GzipFile();
ReplaceGzipFile();
AppendLines();
std::cout << "performance trace tests passed\n";
return 0;
@@ -11,6 +11,10 @@ struct AppTitleId {
char value[12];
};
struct AppContentVersion {
char value[16];
};
struct AppIconData {
const std::uint8_t* bytes;
std::uint64_t size;
@@ -20,6 +24,8 @@ extern "C" {
AppTitle GetAppTitle_nid_postfix();
AppTitleId GetAppTitleId_nid_postfix();
bool HasAppParamJson_nid_no_patch();
AppContentVersion GetAppContentVersion_nid_no_patch();
bool HasAppIcon_nid_postfix();
AppIconData GetAppIconData_nid_postfix();
std::uint64_t GetAppDownloadDataSizeMiB_nid_postfix();
@@ -9,6 +9,7 @@
struct ParsedParamJson {
std::string title;
std::string titleId;
std::string contentVersion;
std::uint64_t downloadDataSizeMiB = 0;
std::uint32_t attribute3 = 0;
std::optional<std::uint64_t> flexibleMemorySizeBytes;
@@ -20,6 +20,7 @@ bool g_iconAttempted = false;
bool g_iconLoaded = false;
char g_title[128] = {};
char g_titleId[12] = {};
char g_contentVersion[16] = {};
std::uint64_t g_downloadDataSizeMiB = 0;
std::uint32_t g_attribute3 = 0;
std::int32_t g_userDefinedParams[4] = {};
@@ -47,6 +48,7 @@ void ensureTitleLoaded() {
const auto parsed = parseParams(ResolvePath_nid_no_patch(AppMetadataParamJsonGuestPath), ResolvePath_nid_no_patch(AppMetadataParamSfoGuestPath));
copyToFixedBuffer(g_title, sizeof(g_title), parsed.title);
copyToFixedBuffer(g_titleId, sizeof(g_titleId), parsed.titleId);
if (!parsed.contentVersion.empty() && parsed.contentVersion.size() < sizeof(g_contentVersion)) copyToFixedBuffer(g_contentVersion, sizeof(g_contentVersion), parsed.contentVersion);
g_downloadDataSizeMiB = parsed.downloadDataSizeMiB;
g_attribute3 = parsed.attribute3;
std::memcpy(g_userDefinedParams, parsed.userDefinedParams, sizeof(g_userDefinedParams));
@@ -133,6 +135,17 @@ AppTitleId GetAppTitleId_nid_postfix() {
return result;
}
bool HasAppParamJson_nid_no_patch() {
return std::filesystem::exists(ResolvePath_nid_no_patch(AppMetadataParamJsonGuestPath));
}
AppContentVersion GetAppContentVersion_nid_no_patch() {
ensureTitleLoaded();
AppContentVersion result{};
std::memcpy(result.value, g_contentVersion, sizeof(result.value));
return result;
}
bool HasAppIcon_nid_postfix() {
ensureIconLoaded();
return g_iconLoaded;
@@ -322,6 +322,7 @@ ParsedParamJson parseParamJson(const std::filesystem::path& paramJsonPath) {
ParsedParamJson result;
result.title = title;
result.titleId = titleId;
if (const JsonValue* contentVersion = findObjectMember(root, "contentVersion"); contentVersion != nullptr && contentVersion->type == JsonType::String) result.contentVersion = contentVersion->stringValue;
if (const JsonValue* attribute = findObjectMember(root, "attribute3")) {
const auto& number = attribute->numberText;
if (attribute->type != JsonType::Number || number.find_first_of("-.eE") != std::string::npos) throw std::runtime_error("param.json attribute3 is not an unsigned integer");
+7
View File
@@ -81,6 +81,12 @@ static void CheckUserDefinedParams(const std::filesystem::path& path) {
}
}
static void CheckContentVersion(const std::filesystem::path& path) {
Require(ParseText(path, R"({"titleId":"PPSA00000","contentVersion":"01.002.000","localizedParameters":{"en-US":{"titleName":"Example"}}})").contentVersion == "01.002.000", "Incorrect content version");
Require(ParseText(path, R"({"titleId":"PPSA00000","localizedParameters":{"en-US":{"titleName":"Example"}}})").contentVersion.empty(), "A missing content version was not empty");
Require(ParseText(path, R"({"titleId":"PPSA00000","contentVersion":1,"localizedParameters":{"en-US":{"titleName":"Example"}}})").contentVersion.empty(), "A content version that is not a string was kept");
}
static void CheckNesting(const std::filesystem::path& path) {
const std::string prefix = R"({"titleId":"PPSA00000","localizedParameters":{"en-US":{"titleName":"Example"}},"extra":)";
const auto nested = [&](std::size_t arrays) {
@@ -226,6 +232,7 @@ int main() {
}
CheckLanguages(path);
CheckUserDefinedParams(path);
CheckContentVersion(path);
CheckOutputPermission(path);
CheckNesting(path);
} catch (const std::exception& error) {
+3 -1
View File
@@ -202,7 +202,7 @@ Set `APS5_TRACE_EVENTS=<count>` to change the queue-worker and presenter buffers
Inside the window the queue workers and the presenter record every `PerformanceTimer` scope and stage as a complete event named after its frame timing key: `Driver.WorkerWait`, `Driver.Packet`, `Driver.Packet.draw`, `Graphics.Draw`, `Driver.Present`, `Driver.Present.inflight_wait`, `Vulkan.Retire` and so on. A slice shorter than 20 us is only counted: its time is in the thread's counter `<thread> folded` (milliseconds per present) and in the summary. A thread writes a present's counter sample at its first slice after the next present, so the last present of the window has no sample; its time is still in the summary. By default, each queue worker and presenter allocates a buffer for 262,144 events when it starts; a full buffer stops that thread's recording, sets `trace_dropped` and puts an instant event `trace buffer full` on the thread's track at the start of the first lost slice (slices are appended when their outer scope ends, so that can precede the last recorded slice by up to one outer-scope length), with the number of lost slices in `args.events_lost`. When the window closes, the presenter formats the trace once and a `ProfileOutput` writer thread writes `aps5-trace-<commit>-<pid>.json.gz` (Chrome trace-event JSON, opened by ui.perfetto.dev or chrome://tracing) and appends one line to `aps5-summary.jsonl`, both in `APS5_TRACE_DIR` or the working directory. A window that is still open when the title exits writes nothing.
The summary line has `commit` and `dirty` of the build, `env` (every `APS5_*` variable; a value containing `/` or `\` is written as `<path>`), `gpu`, `driver`, `os`, `resolution` (the video output's), `window`, `presents`, `frame_ms` (p50, p90, p99 and max of the intervals between presents), `gpu_busy_ms_per_present` (the batch timestamps behind `[present]`), `worker_busy_pct` (share of the window each queue worker spends in `Driver.Packet`, `Driver.Suspend`, `Driver.EndSubmission`, `Driver.Rewind` and `Driver.Completion`), `top_phases_ms_per_frame` (the eight largest scopes and stages, without `Driver.WorkerWait`), `trace_dropped`, `trace_dropped_threads` and `trace`. `trace_dropped` is `true` when any thread lost a slice, any thread's totals table (512 scope and stage entries) was full, or the `gpu` track was full. `trace_dropped_threads` is present only then and has one entry per such thread name (threads that share a name, such as the shader prepare threads, share one entry: `events_lost` is summed, `events_full_after_s` is the earliest, `totals_full` is true if any is): `events_full_after_s` (seconds from the first present of the window to the start of the first lost slice; absent for `gpu` and, in every summary line (windowed `APS5_TRACE_FRAMES`/`APS5_TRACE_SECONDS` and `APS5_TRACE_SUMMARY_EVERY`), whenever the first lost slice starts outside the line's span of presents, including a slice whose scope began before the window opened or one lost after the window's last present), `events_lost` (slices not recorded; absent when 0; in `APS5_TRACE_SUMMARY_EVERY` lines it counts the thread buffer's losses since it started, not only those of the line's span) and `totals_full` (`true` when the thread's totals table overflowed, which makes the summary numbers incomplete; a thread with `events_lost` only has complete totals). A trace holds timings, scope names, counts and thread names; no guest memory, paths or strings of the title.
The summary line has `commit` and `dirty` of the build, `env` (every `APS5_*` variable; a value containing `/` or `\` is written as `<path>`), `gpu`, `driver`, `os`, `cpu` (the CPU brand string), `cpu_threads` and `memory_mib` (physical memory) of the host (all three absent on a host that is not x86 or reports no brand string), `title_id`, `title` and `content_version` from `app0/sce_sys/param.json` (only when the title has one; `content_version` is empty when the file has no `contentVersion`, or one that is not a string of at most 15 characters), `resolution` (the video output's), `window`, `presents`, `frame_ms` (p50, p90, p99 and max of the intervals between presents), `gpu_busy_ms_per_present` (the batch timestamps behind `[present]`), `worker_busy_pct` (share of the window each queue worker spends in `Driver.Packet`, `Driver.Suspend`, `Driver.EndSubmission`, `Driver.Rewind` and `Driver.Completion`), `top_phases_ms_per_frame` (the eight largest scopes and stages, without `Driver.WorkerWait`), `trace_dropped`, `trace_dropped_threads` and `trace`. `trace_dropped` is `true` when any thread lost a slice, any thread's totals table (512 scope and stage entries) was full, or the `gpu` track was full. `trace_dropped_threads` is present only then and has one entry per such thread name (threads that share a name, such as the shader prepare threads, share one entry: `events_lost` is summed, `events_full_after_s` is the earliest, `totals_full` is true if any is): `events_full_after_s` (seconds from the first present of the window to the start of the first lost slice; absent for `gpu` and, in every summary line (windowed `APS5_TRACE_FRAMES`/`APS5_TRACE_SECONDS` and `APS5_TRACE_SUMMARY_EVERY`), whenever the first lost slice starts outside the line's span of presents, including a slice whose scope began before the window opened or one lost after the window's last present), `events_lost` (slices not recorded; absent when 0; in `APS5_TRACE_SUMMARY_EVERY` lines it counts the thread buffer's losses since it started, not only those of the line's span) and `totals_full` (`true` when the thread's totals table overflowed, which makes the summary numbers incomplete; a thread with `events_lost` only has complete totals). A trace holds timings, scope names, counts and thread names; no guest memory, paths or strings of the title.
The shader preparation threads are traced as well, `shader prepare`, with a buffer for 32,768 events each. A wait for the GPU mutex on a traced thread is a `GpuMutex.wait.<site>` slice (`dispatch`, `draw`, `present`, `flush` and so on, as on the `[lock]` line).
@@ -218,6 +218,8 @@ Set `APS5_TRACE_SUMMARY_EVERY=<seconds>` (a whole number above 0; a malformed va
The switch combines with `APS5_TRACE_FRAMES` or `APS5_TRACE_SECONDS`. The intervals run for the whole session and the window records its full trace on top: the event buffers are allocated and filled only while the window is open, the window's own summary line (with `trace` set and no `interval_*` fields) holds only the time inside the window, and the interval lines continue unchanged around it. Without a window switch no buffer is allocated at all. The two switches do not share a clock origin: `interval_*` is relative to the first present, as the window's `APS5_TRACE_SECONDS` start is.
Set `APS5_TRACE_BENCH=1` together with `APS5_TRACE_SUMMARY_EVERY` to also keep every interval's frame times for a benchmark report. At each interval the presenter hands the kept intervals to a writer thread, which formats and compresses the file and rewrites `aps5-bench-<commit>-<pid>.json.gz` in `APS5_TRACE_DIR` or the working directory: one JSON object with `format` (`aps5-bench/1`), the identity fields of the summary line (`commit`, `dirty`, `env`, `gpu`, `driver`, `os`, `cpu`, `cpu_threads`, `memory_mib` and, with `param.json`, `title_id`, `title` and `content_version`), `intervals_dropped` and `intervals`. Each interval has `start_s`, `end_s`, `first_present`, `resolution`, `gpu_busy_ms` (the interval's total) and `frame_us`, the whole microseconds between consecutive presents. The file keeps the last 240 intervals; `intervals_dropped` counts older ones it no longer holds. Any other value, or the switch without `APS5_TRACE_SUMMARY_EVERY`, fails the first present. The file is written to `<name>.tmp` and then moved over the previous one, so a crash or kill during a rewrite leaves the last complete report. Only the newest report waits for the writer: one handed over while an older one is still pending replaces it. The frame times cost 4 bytes per present in memory.
`python tools/trace_digest.py aps5-summary.jsonl` prints the interval lines of such a file as one row per interval (frame p50, p90, p99, max, GPU busy per present, the busiest queue worker and the largest phase) and a text sparkline of the frame p99 and of the GPU busy time; lines of window captures in the file are skipped, and a change of commit, environment, GPU, driver or OS between intervals is reported. `--json` prints the rows as JSON.
With `APS5_TRACE_SUMMARY_EVERY` set, the recorder writes a timestamp pair per batch for the whole run and every `PerformanceTimer` scope adds its time to the thread's totals, two clock reads per scope. A thread's totals are read while it runs, so a scope that straddles an interval boundary is counted in one interval or the other.
+1
View File
@@ -380,6 +380,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti
| `APS5_THREAD_RT` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/ThreadPriority.cpp` | 10 | realtime priority the raised threads ask rtkit for (clamped to 1..99) |
| `APS5_THREAD_RTTIME_US` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/ThreadPriority.cpp` | 200000 | hard RLIMIT_RTTIME the realtime plan lowers to, in microseconds |
| `APS5_TRACE_BARRIERS` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | prints that many simulated hazard barriers before stopping |
| `APS5_TRACE_BENCH` | string | `core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp` | | `1` with `APS5_TRACE_SUMMARY_EVERY` also rewrites `aps5-bench-<commit>-<pid>.json.gz` with every interval's frame times; any other value fails the first present |
| `APS5_TRACE_BIGBUF` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | | lists every copied guest region of at least 1 MiB |
| `APS5_TRACE_CAPSYNC` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp` | | prints every unlocked hook wait |
| `APS5_TRACE_CONTEXT_STATE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp` | | logs context-state operations and the context registers written |
+1
View File
@@ -207,6 +207,7 @@
- [Thick SW_4KB_S_X volumes](../../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp) use AMD addrlib's GFX10 non-RB+ 16-pipe `GFX10_SW_4K_S3_X_PATINFO` equations for 1, 2, 4, 8 and 16-byte elements, including packed mip tails. The layout follows [AMD PAL c5e80007](https://github.com/GPUOpen-Drivers/pal/blob/c5e800072a32f68b6ccc4422936d96167c6e0728/src/core/imported/addrlib/src/gfx10/gfx10addrlib.cpp); it has not been measured on console hardware. Other thick swizzle modes outside the supported families still throw. Mipmapped 3D color targets remain refused by the color-register decoder.
- [Viewport depth range remap](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - on a device without `VK_EXT_depth_range_unrestricted` a viewport depth range outside [0, 1] (Minecraft PPSA17221 draws its world with `PA_CL_VPORT_ZSCALE` 1, `ZOFFSET` 0 in negative-one-to-one clip space) is folded into the vertex shader: `RemapPositionDepth` scales clip z so z/w is the console's depth, the pipeline's depth clamp applies the console's `PA_SC_VPORT_ZMIN/ZMAX` clipped to [0, 1] (exact for a normalized depth target or none; Minecraft clamps its float depth target to [-1, 1], so a depth below 0 is stored as 0 there, which only a later test or read at exactly 0 can tell, and the first such draw is reported once), and two extra `gl_ClipDistance` elements restore the near and far planes depth clamping removes. The mode is a pipeline specialization constant (`PipelineSpecialization::DepthRemapBase`); scale and offset are read from the draw's push block (two words at byte 104, `DepthRemapPushOffsetBytes`, so a remapped draw's stage push data ends before it), so draws that differ only in them share one vertex module and one pipeline. The remap serves the vertex path only; mesh and tessellation draws with such a range still throw. Whether the console clamps before or after its depth bias and how it treats a clamp range narrower than the transform's are not measured.
- [Benchmark rewrite failure test off Windows](../../../core/libs/prx/libSceAgcDriver/tests/PerformanceTrace.cpp) (libSceAgcDriver) - `ReplaceGzipFile` checks that a write failing at `<name>.tmp` keeps the previous `aps5-bench.json.gz` only on non-Windows builds: the `std::ios_base::failure` thrown inside the MinGW ofstream did not unwind to the test's catch, so Windows CI covers only the failed move.
- [Sampled array windows](../../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp) (libSceAgcDriver) - a sampled 2D array, 1D array or cube view whose base array slice is past 0 is built from its viewed slices only (`ViewedArrayWindow`), at the guest address of its base slice. The window is taken only when the surface is thin, has no DCC metadata, and, for a 64 KiB XOR swizzle, when the base slice keeps every slice bit of the swizzle equation at zero (R_X 1-byte elements: a base slice that is a multiple of 16). Any other view still builds slices 0 to its last slice. A window and a whole-array view of the same surface are separate images over overlapping memory, kept coherent by the existing overlap rules, at the cost of the memory and uploads of both. Ghost of Yotei (PPSA26344) samples a 256x256 cube array at slices 4299..7232, built at 1467 MiB instead of about 3.8 GB.
- [2D reads of 2D arrays](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - a sampled image whose shader shape is 2D and whose T# is a 2D array is built from its base slice alone: the T# is narrowed to that slice (`FirstLayerDescriptor`) before its decode, cache key and image, in stage A records, stage B bindings, revalidation and image tables. A whole-array read of the same T# keeps its own cache entry, so both images exist when a title uses both. Storage images read or written through a 2D shape are not narrowed and still build every slice the T# names. Ghost of Yotei (PPSA26344) reads a 3328x1872 R_X T# of 1537 slices this way: the whole array was a 12.4 GiB image that did not fit the device-local budget of an RTX 4090.
+4
View File
@@ -296,6 +296,10 @@ GPU results in storage images and render targets stay on the GPU until something
A title's own CPU reads of that memory are not seen, so a title that waits for a GPU label and then reads the results reads stale memory. `APS5_STORE_AT_LABEL=1` stores every pending result before each label the GPU writes (`RELEASE_MEM`, `WRITE_DATA`, `DMA_DATA`), at the cost of a write-back of every pending image per label. Any other value stops the title at its first label.
### Benchmark reports
To measure how a title runs on a machine, start it with `APS5_TRACE_SUMMARY_EVERY=30 APS5_TRACE_BENCH=1`, play the scene for a few minutes and quit. Every 30 seconds the runtime rewrites `aps5-bench-<commit>-<pid>.json.gz` in `APS5_TRACE_DIR` or the working directory and appends a line to `aps5-summary.jsonl`. The file holds the time between every two presents, the commit, GPU, driver, OS, CPU name, thread count and physical memory, and, when `app0/sce_sys/param.json` is present, the title ID, name and content version. It holds no game data. The first interval includes shader compilation; compare later intervals. The interval still open when the title exits is not written. The fields are described under [summaries for a long session](../dev/BUILD.md#summaries-for-a-long-session).
### Tracing system library calls
Windows only. An executable converted with `--trace-imports` records the calls it makes to the libraries named in `APS5_TRACE_CALLS`, a comma-separated list such as `libkernel,libSceSaveData`, or `all`. Without the variable nothing is recorded. The crash and abort reports then end with the last 64 calls: sequence number, thread, library, import name as the executable imports it (a NID) and the caller's address. They are followed by the 32 most called imports since start-up, with their number of calls.