diff --git a/ps5-kstuff/debug-reader.c b/ps5-kstuff/debug-reader.c index 979a3da..505e370 100644 --- a/ps5-kstuff/debug-reader.c +++ b/ps5-kstuff/debug-reader.c @@ -131,6 +131,15 @@ static void print_metrics(const struct kstuff_metrics* metrics) PRINT_FIELD("mailbox_unhandled", metrics->mailbox_unhandled); tee_putc('\n'); + tee_printf("uelf_main"); + PRINT_FIELD("entries", metrics->uelf_main_entries); + PRINT_FIELD("trap_read_fail", metrics->uelf_main_trap_read_failures); + PRINT_FIELD("jr_read_fail", metrics->uelf_main_just_return_read_failures); + PRINT_FIELD("trap_write_fail", metrics->uelf_main_trap_write_failures); + PRINT_FIELD("gadget", metrics->run_gadget_calls); + PRINT_FIELD("gadget_fail", metrics->run_gadget_failures); + tee_putc('\n'); + tee_printf("decrypt_regs"); PRINT_FIELD("rax", metrics->debug_reg_decrypt_rax); PRINT_FIELD("rcx", metrics->debug_reg_decrypt_rcx); @@ -198,6 +207,19 @@ static void print_metrics(const struct kstuff_metrics* metrics) PRINT_FIELD("out_max", metrics->copy_to_cycles_max); tee_putc('\n'); + tee_printf("optimization_cycles"); + PRINT_FIELD("main_total", metrics->uelf_main_cycles_total); + PRINT_FIELD("main_max", metrics->uelf_main_cycles_max); + PRINT_FIELD("gadget_total", metrics->run_gadget_cycles_total); + PRINT_FIELD("gadget_max", metrics->run_gadget_cycles_max); + PRINT_FIELD("scalar_in_total", metrics->scalar_copy_from_cycles_total); + PRINT_FIELD("scalar_in_max", metrics->scalar_copy_from_cycles_max); + PRINT_FIELD("scalar_out_total", metrics->scalar_copy_to_cycles_total); + PRINT_FIELD("scalar_out_max", metrics->scalar_copy_to_cycles_max); + PRINT_FIELD("crypto_snap_total", metrics->crypto_message_snapshot_cycles_total); + PRINT_FIELD("crypto_snap_max", metrics->crypto_message_snapshot_cycles_max); + tee_putc('\n'); + tee_printf("fself_cache"); PRINT_FIELD("hdr_hit", metrics->fself_header_cache_hits); PRINT_FIELD("hdr_miss", metrics->fself_header_cache_misses); @@ -252,6 +274,8 @@ static void print_metrics(const struct kstuff_metrics* metrics) PRINT_FIELD("hmac_msg", metrics->crypto_messages_hmac); PRINT_FIELD("other_msg", metrics->crypto_messages_other); PRINT_FIELD("emu_msg", metrics->crypto_emulated_messages); + PRINT_FIELD("snap", metrics->crypto_message_snapshot_reads); + PRINT_FIELD("snap_fail", metrics->crypto_message_snapshot_failures); tee_putc('\n'); tee_printf("xts"); @@ -319,6 +343,26 @@ static void print_metrics(const struct kstuff_metrics* metrics) PRINT_FIELD("out", metrics->copy_to_calls); PRINT_FIELD("out_bytes", metrics->copy_to_bytes); PRINT_FIELD("out_fail", metrics->copy_to_failures); + PRINT_FIELD("scalar_in", metrics->scalar_copy_from_calls); + PRINT_FIELD("scalar_out", metrics->scalar_copy_to_calls); + tee_putc('\n'); + + tee_printf("mapping_cache"); + PRINT_FIELD("trap_hit", metrics->trap_mapping_hits); + PRINT_FIELD("trap_miss", metrics->trap_mapping_misses); + PRINT_FIELD("trap_fallback", metrics->trap_mapping_fallbacks); + PRINT_FIELD("jr_hit", metrics->just_return_mapping_hits); + PRINT_FIELD("jr_miss", metrics->just_return_mapping_misses); + PRINT_FIELD("jr_fallback", metrics->just_return_mapping_fallbacks); + PRINT_FIELD("pcpu_hit", metrics->pcpu_mapping_hits); + PRINT_FIELD("pcpu_miss", metrics->pcpu_mapping_misses); + PRINT_FIELD("pcpu_fallback", metrics->pcpu_mapping_fallbacks); + PRINT_FIELD("tss_hit", metrics->tss_mapping_hits); + PRINT_FIELD("tss_miss", metrics->tss_mapping_misses); + PRINT_FIELD("tss_fallback", metrics->tss_mapping_fallbacks); + PRINT_FIELD("wrmsr_hit", metrics->wrmsr_args_mapping_hits); + PRINT_FIELD("wrmsr_miss", metrics->wrmsr_args_mapping_misses); + PRINT_FIELD("wrmsr_fallback", metrics->wrmsr_args_mapping_fallbacks); tee_putc('\n'); tee_printf("obs"); diff --git a/ps5-kstuff/uelf/fakekeys.c b/ps5-kstuff/uelf/fakekeys.c index 49232a4..e76778f 100644 --- a/ps5-kstuff/uelf/fakekeys.c +++ b/ps5-kstuff/uelf/fakekeys.c @@ -3,22 +3,6 @@ #include "fakekeys.h" #include "shared_area.h" -int has_fake_key(int key_id) -{ - if(key_id < 0 || key_id >= 63) - { - METRIC_INC(fake_key_has_misses); - return 0; - } - if(__atomic_load_n(&shared_area.ready_mask, __ATOMIC_ACQUIRE) & (1ull << key_id)) - { - METRIC_INC(fake_key_has_hits); - return 1; - } - METRIC_INC(fake_key_has_misses); - return 0; -} - int register_fake_key(const uint8_t key_data[32]) { uint64_t mask, mask1; diff --git a/ps5-kstuff/uelf/fakekeys.h b/ps5-kstuff/uelf/fakekeys.h index 6f6b621..876ce13 100644 --- a/ps5-kstuff/uelf/fakekeys.h +++ b/ps5-kstuff/uelf/fakekeys.h @@ -1,7 +1,6 @@ #pragma once #include -int has_fake_key(int key_id); int register_fake_key(const uint8_t key_data[32]); int unregister_fake_key(int key_id); int get_fake_key(int key_id, uint8_t key_data[32]); diff --git a/ps5-kstuff/uelf/fpkg.c b/ps5-kstuff/uelf/fpkg.c index 7f7e3a7..cc76368 100644 --- a/ps5-kstuff/uelf/fpkg.c +++ b/ps5-kstuff/uelf/fpkg.c @@ -37,6 +37,11 @@ enum crypto_message_kind CRYPTO_MESSAGE_XTS, }; +enum { + CRYPTO_MESSAGE_DATA_QWORDS = 21, + CRYPTO_MESSAGE_NEXT_OFFSET = 320, +}; + struct crypto_message_info { enum crypto_message_kind kind; @@ -60,37 +65,50 @@ static int crypto_request_emulated(uint64_t* regs, uint64_t msg, uint32_t status return 1; } -static int read_crypto_message(uint64_t msg, uint64_t msg_data[21]) +static int read_crypto_message(uint64_t msg, + uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS], + uint64_t* next_msg) { - return copy_from_kernel(msg_data, msg, sizeof(uint64_t) * 21); + METRIC_INC(crypto_message_snapshot_reads); + METRIC_TIME_START(start_cycles); + int error = copy_from_kernel(msg_data, msg, + sizeof(uint64_t) * CRYPTO_MESSAGE_DATA_QWORDS); + if(!error) + error = copy_u64_from_kernel(next_msg, msg + CRYPTO_MESSAGE_NEXT_OFFSET); + if(error) + METRIC_INC(crypto_message_snapshot_failures); + METRIC_TIME(crypto_message_snapshot_cycles_total, + crypto_message_snapshot_cycles_max, start_cycles); + return error; } -static int is_hmac_message(const uint64_t msg_data[21]) +static int is_hmac_message(const uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS]) { return (msg_data[0] & 0x7fffffff) == 0x9132000; } -static int is_xts_message(const uint64_t msg_data[21]) +static int is_xts_message(const uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS]) { return (msg_data[0] & 0x7ffff7ff) == 0x2108000; } -static struct crypto_message_result unhandled_crypto_message(uint64_t msg) +static struct crypto_message_result unhandled_crypto_message(uint64_t next_msg) { return (struct crypto_message_result){ .emulated_messages = 0, - .next_msg = kpeek64(msg + 320), + .next_msg = next_msg, .status = ENOSYS, .total_messages = 1, }; } -static struct crypto_message_info inspect_crypto_message(const uint64_t msg_data[21]) +static struct crypto_message_info inspect_crypto_message( + const uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS], uint8_t key[32]) { if(is_xts_message(msg_data)) { int key_idx = HANDLE_TO_IDX(msg_data[5]); - if(key_idx < 0 || !has_fake_key(key_idx)) + if(key_idx < 0 || !get_fake_key(key_idx, key)) { METRIC_INC(fpkg_reject_xts_non_fake); } @@ -110,7 +128,7 @@ static struct crypto_message_info inspect_crypto_message(const uint64_t msg_data { METRIC_INC(fpkg_reject_hmac_bad_shape); } - else if(key_idx < 0 || !has_fake_key(key_idx)) + else if(key_idx < 0 || !get_fake_key(key_idx, key)) { METRIC_INC(fpkg_reject_hmac_non_fake); } @@ -130,15 +148,13 @@ static struct crypto_message_info inspect_crypto_message(const uint64_t msg_data }; } -static struct crypto_message_result handle_hmac_message(uint64_t msg, const uint64_t msg_data[21], - int key_idx, +static struct crypto_message_result handle_hmac_message(uint64_t msg, + const uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS], + uint64_t next_msg, + int key_idx, const uint8_t key[32], struct crypto_request_cache* cache) { - struct crypto_message_result result = unhandled_crypto_message(msg); - uint8_t key[32]; - if(!get_fake_key(key_idx, key)) - return result; - + struct crypto_message_result result = unhandled_crypto_message(next_msg); uint8_t hash[32] = {0}; if(pfs_hmac_virtual_fpu_held(cache, hash, key_idx, key, msg_data[2], msg_data[1])) { @@ -157,14 +173,11 @@ static struct crypto_message_result handle_hmac_message(uint64_t msg, const uint return result; } -static struct crypto_message_result handle_xts_message(uint64_t msg, const uint64_t msg_data[21], - int key_idx, - struct crypto_request_cache* cache) +static struct crypto_message_result handle_xts_message( + const uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS], uint64_t next_msg, + int key_idx, const uint8_t key[32], struct crypto_request_cache* cache) { - struct crypto_message_result result = unhandled_crypto_message(msg); - uint8_t key[32]; - if(!get_fake_key(key_idx, key)) - return result; + struct crypto_message_result result = unhandled_crypto_message(next_msg); METRIC_INC(xts_run_messages_total); uint64_t src = msg_data[2]; @@ -209,14 +222,16 @@ static int handle_crypto_request(uint64_t* regs) for (uint64_t msg = start; msg && !total_status;) { - uint64_t msg_data[21]; - if(read_crypto_message(msg, msg_data)) + uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS]; + uint64_t next_msg; + if(read_crypto_message(msg, msg_data, &next_msg)) { total_status = -1; break; } - struct crypto_message_info msg_info = inspect_crypto_message(msg_data); + uint8_t key[32]; + struct crypto_message_info msg_info = inspect_crypto_message(msg_data, key); METRIC_INC(crypto_messages_total); if(msg_info.kind == CRYPTO_MESSAGE_XTS) METRIC_INC(crypto_messages_xts); @@ -236,11 +251,14 @@ static int handle_crypto_request(uint64_t* regs) struct crypto_message_result result; if(msg_info.kind == CRYPTO_MESSAGE_XTS) - result = handle_xts_message(msg, msg_data, msg_info.key_idx, &s_crypto_request_cache); + result = handle_xts_message(msg_data, next_msg, msg_info.key_idx, key, + &s_crypto_request_cache); else if(msg_info.kind == CRYPTO_MESSAGE_HMAC) - result = handle_hmac_message(msg, msg_data, msg_info.key_idx, &s_crypto_request_cache); + result = handle_hmac_message(msg, msg_data, next_msg, + msg_info.key_idx, key, + &s_crypto_request_cache); else - result = unhandled_crypto_message(msg); + result = unhandled_crypto_message(next_msg); int status = result.status; total += result.total_messages; diff --git a/ps5-kstuff/uelf/main.c b/ps5-kstuff/uelf/main.c index 40e5975..e4b7891 100644 --- a/ps5-kstuff/uelf/main.c +++ b/ps5-kstuff/uelf/main.c @@ -466,17 +466,32 @@ from_userspace: void main(uint64_t just_return) { + METRIC_INC(uelf_main_entries); + METRIC_TIME_START(start_cycles); +#define RETURN_UELF_MAIN() do { \ + METRIC_TIME(uelf_main_cycles_total, uelf_main_cycles_max, start_cycles); \ + return; \ +} while(0) uint64_t regs[NREGS + 1]; if(copy_from_trap_frame_cached(regs, sizeof(regs))) - return; + { + METRIC_INC(uelf_main_trap_read_failures); + RETURN_UELF_MAIN(); + } uint64_t jr_frame[5]; if(copy_from_just_return_cached(jr_frame, just_return, sizeof(jr_frame))) - return; + { + METRIC_INC(uelf_main_just_return_read_failures); + RETURN_UELF_MAIN(); + } have_error_code = jr_frame[0]; regs[RDX] = jr_frame[2]; regs[RCX] = jr_frame[3]; regs[RAX] = jr_frame[4]; intno = regs[NREGS]; handle(regs); - copy_to_trap_frame_cached(regs, NREGS * sizeof(uint64_t)); + if(copy_to_trap_frame_cached(regs, NREGS * sizeof(uint64_t))) + METRIC_INC(uelf_main_trap_write_failures); + METRIC_TIME(uelf_main_cycles_total, uelf_main_cycles_max, start_cycles); +#undef RETURN_UELF_MAIN } diff --git a/ps5-kstuff/uelf/pfs_crypto.c b/ps5-kstuff/uelf/pfs_crypto.c index b548cca..48b9b13 100644 --- a/ps5-kstuff/uelf/pfs_crypto.c +++ b/ps5-kstuff/uelf/pfs_crypto.c @@ -82,20 +82,24 @@ static int hmac_sha256_cache_entry_matches(const struct hmac_sha256_cache_entry* } static struct hmac_sha256_cache_entry* select_hmac_sha256_cache_entry(struct crypto_request_cache* cache, - int key_id, const uint8_t* key) + int key_id, const uint8_t* key, + int* matched) { + struct hmac_sha256_cache_entry* first_empty = NULL; for(size_t i = 0; i < PFS_HMAC_SHA256_CACHE_SLOTS; i++) { struct hmac_sha256_cache_entry* entry = &cache->hmac[i]; if(hmac_sha256_cache_entry_matches(entry, key_id, key)) + { + *matched = 1; return entry; + } + if(!entry->valid && !first_empty) + first_empty = entry; } - for(size_t i = 0; i < PFS_HMAC_SHA256_CACHE_SLOTS; i++) - { - struct hmac_sha256_cache_entry* entry = &cache->hmac[i]; - if(!entry->valid) - return entry; - } + *matched = 0; + if(first_empty) + return first_empty; struct hmac_sha256_cache_entry* entry = &cache->hmac[cache->hmac_next_slot]; cache->hmac_next_slot++; if(cache->hmac_next_slot == PFS_HMAC_SHA256_CACHE_SLOTS) @@ -106,8 +110,10 @@ static struct hmac_sha256_cache_entry* select_hmac_sha256_cache_entry(struct cry static int get_hmac_sha256_cache_entry(struct crypto_request_cache* cache, int key_id, const uint8_t* key, const struct hmac_sha256_cache_entry** out) { - struct hmac_sha256_cache_entry* entry = select_hmac_sha256_cache_entry(cache, key_id, key); - if(hmac_sha256_cache_entry_matches(entry, key_id, key)) + int matched; + struct hmac_sha256_cache_entry* entry = select_hmac_sha256_cache_entry(cache, key_id, key, + &matched); + if(matched) { METRIC_INC(hmac_cache_hits); *out = entry; @@ -213,20 +219,23 @@ static int xts_key_cache_entry_matches(const struct xts_key_cache_entry* entry, } static struct xts_key_cache_entry* select_xts_key_cache_entry(struct crypto_request_cache* cache, int key_id, - const uint8_t* key) + const uint8_t* key, int* matched) { + struct xts_key_cache_entry* first_empty = NULL; for(size_t i = 0; i < PFS_XTS_KEY_CACHE_SLOTS; i++) { struct xts_key_cache_entry* entry = &cache->xts[i]; if(xts_key_cache_entry_matches(entry, key_id, key)) + { + *matched = 1; return entry; + } + if(!entry->valid && !first_empty) + first_empty = entry; } - for(size_t i = 0; i < PFS_XTS_KEY_CACHE_SLOTS; i++) - { - struct xts_key_cache_entry* entry = &cache->xts[i]; - if(!entry->valid) - return entry; - } + *matched = 0; + if(first_empty) + return first_empty; struct xts_key_cache_entry* entry = &cache->xts[cache->xts_next_slot]; cache->xts_next_slot++; if(cache->xts_next_slot == PFS_XTS_KEY_CACHE_SLOTS) @@ -237,8 +246,9 @@ static struct xts_key_cache_entry* select_xts_key_cache_entry(struct crypto_requ static int get_xts_key_cache_entry(struct crypto_request_cache* cache, int key_id, const uint8_t* key, const struct xts_key_cache_entry** out) { - struct xts_key_cache_entry* entry = select_xts_key_cache_entry(cache, key_id, key); - if(xts_key_cache_entry_matches(entry, key_id, key)) + int matched; + struct xts_key_cache_entry* entry = select_xts_key_cache_entry(cache, key_id, key, &matched); + if(matched) { METRIC_INC(xts_cache_hits); *out = entry; diff --git a/ps5-kstuff/uelf/shared_area.h b/ps5-kstuff/uelf/shared_area.h index 0c78048..04f215e 100644 --- a/ps5-kstuff/uelf/shared_area.h +++ b/ps5-kstuff/uelf/shared_area.h @@ -221,6 +221,47 @@ struct kstuff_metrics uint64_t log_word_writes; uint64_t log_msg_writes; uint64_t log_msg_bytes; + + /* Keep new metrics appended so existing field offsets remain stable. */ + uint64_t uelf_main_entries; + uint64_t uelf_main_trap_read_failures; + uint64_t uelf_main_just_return_read_failures; + uint64_t uelf_main_trap_write_failures; + uint64_t uelf_main_cycles_total; + uint64_t uelf_main_cycles_max; + + uint64_t run_gadget_calls; + uint64_t run_gadget_failures; + uint64_t run_gadget_cycles_total; + uint64_t run_gadget_cycles_max; + + uint64_t scalar_copy_from_calls; + uint64_t scalar_copy_from_cycles_total; + uint64_t scalar_copy_from_cycles_max; + uint64_t scalar_copy_to_calls; + uint64_t scalar_copy_to_cycles_total; + uint64_t scalar_copy_to_cycles_max; + + uint64_t crypto_message_snapshot_reads; + uint64_t crypto_message_snapshot_failures; + uint64_t crypto_message_snapshot_cycles_total; + uint64_t crypto_message_snapshot_cycles_max; + + uint64_t trap_mapping_hits; + uint64_t trap_mapping_misses; + uint64_t trap_mapping_fallbacks; + uint64_t just_return_mapping_hits; + uint64_t just_return_mapping_misses; + uint64_t just_return_mapping_fallbacks; + uint64_t pcpu_mapping_hits; + uint64_t pcpu_mapping_misses; + uint64_t pcpu_mapping_fallbacks; + uint64_t tss_mapping_hits; + uint64_t tss_mapping_misses; + uint64_t tss_mapping_fallbacks; + uint64_t wrmsr_args_mapping_hits; + uint64_t wrmsr_args_mapping_misses; + uint64_t wrmsr_args_mapping_fallbacks; }; struct kstuff_word_log_entry @@ -299,14 +340,14 @@ extern struct shared_area_layout shared_area; #define METRIC_MAX(field, value) do { } while(0) #endif -_Static_assert(sizeof(struct kstuff_metrics) == 1512, "unexpected metrics size"); +_Static_assert(sizeof(struct kstuff_metrics) == 1792, "unexpected metrics size"); _Static_assert(sizeof(struct kstuff_word_log) == 264, "unexpected word log size"); _Static_assert(sizeof(struct kstuff_ioctl_com_entry) == 24, "unexpected ioctl com entry size"); _Static_assert(sizeof(struct kstuff_ioctl_com_table) == 3088, "unexpected ioctl com table size"); _Static_assert(sizeof(struct kstuff_msg_log) == 504, "unexpected message log size"); -_Static_assert(sizeof(struct kstuff_snapshot) == 5384, "unexpected snapshot size"); +_Static_assert(sizeof(struct kstuff_snapshot) == 5664, "unexpected snapshot size"); #if KSTUFF_OBS -_Static_assert(sizeof(struct shared_area_layout) == 7416, "unexpected shared_area size"); +_Static_assert(sizeof(struct shared_area_layout) == 7696, "unexpected shared_area size"); #else _Static_assert(sizeof(struct shared_area_layout) == 2048, "unexpected non-OBS shared_area size"); #endif diff --git a/ps5-kstuff/uelf/utils.c b/ps5-kstuff/uelf/utils.c index 2e928f7..15a27cb 100644 --- a/ps5-kstuff/uelf/utils.c +++ b/ps5-kstuff/uelf/utils.c @@ -115,6 +115,29 @@ int copy_to_kernel(uint64_t dst, const void* src, uint64_t sz) return 0; } +#if KSTUFF_OBS +#define METRIC_SCALAR_COPY_FROM_TIME(start) do { \ + uint64_t _metric_elapsed = uelf_rdtsc() - (uint64_t)(start); \ + METRIC_INC(scalar_copy_from_calls); \ + METRIC_ADD(copy_from_cycles_total, _metric_elapsed); \ + METRIC_MAX(copy_from_cycles_max, _metric_elapsed); \ + METRIC_ADD(scalar_copy_from_cycles_total, _metric_elapsed); \ + METRIC_MAX(scalar_copy_from_cycles_max, _metric_elapsed); \ +} while(0) + +#define METRIC_SCALAR_COPY_TO_TIME(start) do { \ + uint64_t _metric_elapsed = uelf_rdtsc() - (uint64_t)(start); \ + METRIC_INC(scalar_copy_to_calls); \ + METRIC_ADD(copy_to_cycles_total, _metric_elapsed); \ + METRIC_MAX(copy_to_cycles_max, _metric_elapsed); \ + METRIC_ADD(scalar_copy_to_cycles_total, _metric_elapsed); \ + METRIC_MAX(scalar_copy_to_cycles_max, _metric_elapsed); \ +} while(0) +#else +#define METRIC_SCALAR_COPY_FROM_TIME(start) do { } while(0) +#define METRIC_SCALAR_COPY_TO_TIME(start) do { } while(0) +#endif + int copy_u16_from_kernel(uint16_t* dst, uint64_t src) { METRIC_TIME_START(start_cycles); @@ -125,7 +148,7 @@ int copy_u16_from_kernel(uint16_t* dst, uint64_t src) { METRIC_INC(copy_from_failures); log_word((uint64_t)__builtin_return_address(0)); - METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles); + METRIC_SCALAR_COPY_FROM_TIME(start_cycles); return EFAULT; } if(phys_end - phys >= sizeof(*dst)) @@ -134,10 +157,10 @@ int copy_u16_from_kernel(uint16_t* dst, uint64_t src) { METRIC_INC(copy_from_failures); log_word((uint64_t)__builtin_return_address(0)); - METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles); + METRIC_SCALAR_COPY_FROM_TIME(start_cycles); return EFAULT; } - METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles); + METRIC_SCALAR_COPY_FROM_TIME(start_cycles); return 0; } @@ -151,7 +174,7 @@ int copy_u32_from_kernel(uint32_t* dst, uint64_t src) { METRIC_INC(copy_from_failures); log_word((uint64_t)__builtin_return_address(0)); - METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles); + METRIC_SCALAR_COPY_FROM_TIME(start_cycles); return EFAULT; } if(phys_end - phys >= sizeof(*dst)) @@ -160,10 +183,10 @@ int copy_u32_from_kernel(uint32_t* dst, uint64_t src) { METRIC_INC(copy_from_failures); log_word((uint64_t)__builtin_return_address(0)); - METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles); + METRIC_SCALAR_COPY_FROM_TIME(start_cycles); return EFAULT; } - METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles); + METRIC_SCALAR_COPY_FROM_TIME(start_cycles); return 0; } @@ -177,7 +200,7 @@ int copy_u64_from_kernel(uint64_t* dst, uint64_t src) { METRIC_INC(copy_from_failures); log_word((uint64_t)__builtin_return_address(0)); - METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles); + METRIC_SCALAR_COPY_FROM_TIME(start_cycles); return EFAULT; } if(phys_end - phys >= sizeof(*dst)) @@ -186,10 +209,10 @@ int copy_u64_from_kernel(uint64_t* dst, uint64_t src) { METRIC_INC(copy_from_failures); log_word((uint64_t)__builtin_return_address(0)); - METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles); + METRIC_SCALAR_COPY_FROM_TIME(start_cycles); return EFAULT; } - METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles); + METRIC_SCALAR_COPY_FROM_TIME(start_cycles); return 0; } @@ -203,7 +226,7 @@ int copy_u16_to_kernel(uint64_t dst, uint16_t value) { METRIC_INC(copy_to_failures); log_word((uint64_t)__builtin_return_address(0)); - METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles); + METRIC_SCALAR_COPY_TO_TIME(start_cycles); return EFAULT; } if(phys_end - phys >= sizeof(value)) @@ -212,10 +235,10 @@ int copy_u16_to_kernel(uint64_t dst, uint16_t value) { METRIC_INC(copy_to_failures); log_word((uint64_t)__builtin_return_address(0)); - METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles); + METRIC_SCALAR_COPY_TO_TIME(start_cycles); return EFAULT; } - METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles); + METRIC_SCALAR_COPY_TO_TIME(start_cycles); return 0; } @@ -229,7 +252,7 @@ int copy_u32_to_kernel(uint64_t dst, uint32_t value) { METRIC_INC(copy_to_failures); log_word((uint64_t)__builtin_return_address(0)); - METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles); + METRIC_SCALAR_COPY_TO_TIME(start_cycles); return EFAULT; } if(phys_end - phys >= sizeof(value)) @@ -238,10 +261,10 @@ int copy_u32_to_kernel(uint64_t dst, uint32_t value) { METRIC_INC(copy_to_failures); log_word((uint64_t)__builtin_return_address(0)); - METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles); + METRIC_SCALAR_COPY_TO_TIME(start_cycles); return EFAULT; } - METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles); + METRIC_SCALAR_COPY_TO_TIME(start_cycles); return 0; } @@ -255,7 +278,7 @@ int copy_u64_to_kernel(uint64_t dst, uint64_t value) { METRIC_INC(copy_to_failures); log_word((uint64_t)__builtin_return_address(0)); - METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles); + METRIC_SCALAR_COPY_TO_TIME(start_cycles); return EFAULT; } if(phys_end - phys >= sizeof(value)) @@ -264,26 +287,51 @@ int copy_u64_to_kernel(uint64_t dst, uint64_t value) { METRIC_INC(copy_to_failures); log_word((uint64_t)__builtin_return_address(0)); - METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles); + METRIC_SCALAR_COPY_TO_TIME(start_cycles); return EFAULT; } - METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles); + METRIC_SCALAR_COPY_TO_TIME(start_cycles); return 0; } +#undef METRIC_SCALAR_COPY_FROM_TIME +#undef METRIC_SCALAR_COPY_TO_TIME + uint64_t yield(void); struct kernel_mapping_cache { uint64_t physical_address; int valid; +#if KSTUFF_OBS + uint64_t* hits; + uint64_t* misses; + uint64_t* fallbacks; +#endif }; -static struct kernel_mapping_cache s_trap_frame_mapping; -static struct kernel_mapping_cache s_just_return_mapping; -static struct kernel_mapping_cache s_pcpu_mapping; -static struct kernel_mapping_cache s_tss_rsp0_mapping; -static struct kernel_mapping_cache s_wrmsr_args_mapping; +#if KSTUFF_OBS +#define DEFINE_MAPPING_CACHE(name, metric_prefix) \ + static struct kernel_mapping_cache name = { \ + .hits = &shared_area.metrics.metric_prefix##_hits, \ + .misses = &shared_area.metrics.metric_prefix##_misses, \ + .fallbacks = &shared_area.metrics.metric_prefix##_fallbacks, \ + } +#define OBSERVE_MAPPING_CACHE(cache, field) \ + __atomic_fetch_add((cache)->field, 1, __ATOMIC_RELAXED) +#else +#define DEFINE_MAPPING_CACHE(name, metric_prefix) \ + static struct kernel_mapping_cache name +#define OBSERVE_MAPPING_CACHE(cache, field) do { (void)(cache); } while(0) +#endif + +DEFINE_MAPPING_CACHE(s_trap_frame_mapping, trap_mapping); +DEFINE_MAPPING_CACHE(s_just_return_mapping, just_return_mapping); +DEFINE_MAPPING_CACHE(s_pcpu_mapping, pcpu_mapping); +DEFINE_MAPPING_CACHE(s_tss_rsp0_mapping, tss_mapping); +DEFINE_MAPPING_CACHE(s_wrmsr_args_mapping, wrmsr_args_mapping); + +#undef DEFINE_MAPPING_CACHE extern char tss[]; extern uint64_t wrmsr_args; @@ -311,13 +359,20 @@ static int get_cached_physical_address(struct kernel_mapping_cache* cache, { if(__atomic_load_n(&cache->valid, __ATOMIC_ACQUIRE)) { + OBSERVE_MAPPING_CACHE(cache, hits); *physical_address = cache->physical_address; return 1; } - return initialize_kernel_mapping_cache(cache, address, size, - physical_address); + OBSERVE_MAPPING_CACHE(cache, misses); + int initialized = initialize_kernel_mapping_cache(cache, address, size, + physical_address); + if(!initialized) + OBSERVE_MAPPING_CACHE(cache, fallbacks); + return initialized; } +#undef OBSERVE_MAPPING_CACHE + static __attribute__((noinline, cold)) int copy_from_kernel_uncached( void* dst, uint64_t src, uint64_t size) { @@ -425,18 +480,28 @@ __attribute__((noinline)) int copy_to_wrmsr_args_cached(const uint64_t args[3]) __attribute__((noinline)) int run_gadget_checked(uint64_t* regs) { + METRIC_INC(run_gadget_calls); + METRIC_TIME_START(start_cycles); +#define RETURN_RUN_GADGET(value) do { \ + int _run_gadget_result = (value); \ + if(_run_gadget_result) \ + METRIC_INC(run_gadget_failures); \ + METRIC_TIME(run_gadget_cycles_total, run_gadget_cycles_max, start_cycles); \ + return _run_gadget_result; \ +} while(0) if(copy_to_trap_frame_cached(regs, NREGS*8)) - return EFAULT; + RETURN_RUN_GADGET(EFAULT); uint64_t just_return = yield(); uint64_t jr_frame[5]; if(copy_from_trap_frame_cached(regs, NREGS*8)) - return EFAULT; + RETURN_RUN_GADGET(EFAULT); if(copy_from_just_return_cached(jr_frame, just_return, sizeof(jr_frame))) - return EFAULT; + RETURN_RUN_GADGET(EFAULT); regs[RDX] = jr_frame[2]; regs[RCX] = jr_frame[3]; regs[RAX] = jr_frame[4]; - return 0; + RETURN_RUN_GADGET(0); +#undef RETURN_RUN_GADGET } extern char dr2gpr_start[];