starred/kstuff-lite
1
0 Fork 0
mirror of https://github.com/EchoStretch/kstuff-lite.git synced 2026-10-22 03:32:30 -05:00
Code Issues Packages Projects Releases 10 Wiki Activity

perf: streamline crypto dispatch and observability

Browse Source
This commit is contained in:
Roman Tarasov committed 2026-08-26 03:02:20 +03:00
1 parent 64a5a39982
commit f2fa607e8c
8 files changed
+275 -99
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

ps5-kstuff/debug-reader.c
+44
View File
@@ -131,6 +131,15 @@ static void print_metrics(const struct kstuff_metrics* metrics)
PRINT_FIELD("mailbox_unhandled", metrics->mailbox_unhandled);
tee_putc('\n');
tee_printf("uelf_main");
PRINT_FIELD("entries", metrics->uelf_main_entries);
PRINT_FIELD("trap_read_fail", metrics->uelf_main_trap_read_failures);
PRINT_FIELD("jr_read_fail", metrics->uelf_main_just_return_read_failures);
PRINT_FIELD("trap_write_fail", metrics->uelf_main_trap_write_failures);
PRINT_FIELD("gadget", metrics->run_gadget_calls);
PRINT_FIELD("gadget_fail", metrics->run_gadget_failures);
tee_putc('\n');
tee_printf("decrypt_regs");
PRINT_FIELD("rax", metrics->debug_reg_decrypt_rax);
PRINT_FIELD("rcx", metrics->debug_reg_decrypt_rcx);
@@ -198,6 +207,19 @@ static void print_metrics(const struct kstuff_metrics* metrics)
PRINT_FIELD("out_max", metrics->copy_to_cycles_max);
tee_putc('\n');
tee_printf("optimization_cycles");
PRINT_FIELD("main_total", metrics->uelf_main_cycles_total);
PRINT_FIELD("main_max", metrics->uelf_main_cycles_max);
PRINT_FIELD("gadget_total", metrics->run_gadget_cycles_total);
PRINT_FIELD("gadget_max", metrics->run_gadget_cycles_max);
PRINT_FIELD("scalar_in_total", metrics->scalar_copy_from_cycles_total);
PRINT_FIELD("scalar_in_max", metrics->scalar_copy_from_cycles_max);
PRINT_FIELD("scalar_out_total", metrics->scalar_copy_to_cycles_total);
PRINT_FIELD("scalar_out_max", metrics->scalar_copy_to_cycles_max);
PRINT_FIELD("crypto_snap_total", metrics->crypto_message_snapshot_cycles_total);
PRINT_FIELD("crypto_snap_max", metrics->crypto_message_snapshot_cycles_max);
tee_putc('\n');
tee_printf("fself_cache");
PRINT_FIELD("hdr_hit", metrics->fself_header_cache_hits);
PRINT_FIELD("hdr_miss", metrics->fself_header_cache_misses);
@@ -252,6 +274,8 @@ static void print_metrics(const struct kstuff_metrics* metrics)
PRINT_FIELD("hmac_msg", metrics->crypto_messages_hmac);
PRINT_FIELD("other_msg", metrics->crypto_messages_other);
PRINT_FIELD("emu_msg", metrics->crypto_emulated_messages);
PRINT_FIELD("snap", metrics->crypto_message_snapshot_reads);
PRINT_FIELD("snap_fail", metrics->crypto_message_snapshot_failures);
tee_putc('\n');
tee_printf("xts");
@@ -319,6 +343,26 @@ static void print_metrics(const struct kstuff_metrics* metrics)
PRINT_FIELD("out", metrics->copy_to_calls);
PRINT_FIELD("out_bytes", metrics->copy_to_bytes);
PRINT_FIELD("out_fail", metrics->copy_to_failures);
PRINT_FIELD("scalar_in", metrics->scalar_copy_from_calls);
PRINT_FIELD("scalar_out", metrics->scalar_copy_to_calls);
tee_putc('\n');
tee_printf("mapping_cache");
PRINT_FIELD("trap_hit", metrics->trap_mapping_hits);
PRINT_FIELD("trap_miss", metrics->trap_mapping_misses);
PRINT_FIELD("trap_fallback", metrics->trap_mapping_fallbacks);
PRINT_FIELD("jr_hit", metrics->just_return_mapping_hits);
PRINT_FIELD("jr_miss", metrics->just_return_mapping_misses);
PRINT_FIELD("jr_fallback", metrics->just_return_mapping_fallbacks);
PRINT_FIELD("pcpu_hit", metrics->pcpu_mapping_hits);
PRINT_FIELD("pcpu_miss", metrics->pcpu_mapping_misses);
PRINT_FIELD("pcpu_fallback", metrics->pcpu_mapping_fallbacks);
PRINT_FIELD("tss_hit", metrics->tss_mapping_hits);
PRINT_FIELD("tss_miss", metrics->tss_mapping_misses);
PRINT_FIELD("tss_fallback", metrics->tss_mapping_fallbacks);
PRINT_FIELD("wrmsr_hit", metrics->wrmsr_args_mapping_hits);
PRINT_FIELD("wrmsr_miss", metrics->wrmsr_args_mapping_misses);
PRINT_FIELD("wrmsr_fallback", metrics->wrmsr_args_mapping_fallbacks);
tee_putc('\n');
tee_printf("obs");
ps5-kstuff/uelf/fakekeys.c
-16
View File
@@ -3,22 +3,6 @@
#include "fakekeys.h"
#include "shared_area.h"
int has_fake_key(int key_id)
{
if(key_id < 0 || key_id >= 63)
{
METRIC_INC(fake_key_has_misses);
return 0;
}
if(__atomic_load_n(&shared_area.ready_mask, __ATOMIC_ACQUIRE) & (1ull << key_id))
{
METRIC_INC(fake_key_has_hits);
return 1;
}
METRIC_INC(fake_key_has_misses);
return 0;
}
int register_fake_key(const uint8_t key_data[32])
{
uint64_t mask, mask1;
ps5-kstuff/uelf/fakekeys.h
-1
View File
@@ -1,7 +1,6 @@
#pragma once
#include <stdint.h>
int has_fake_key(int key_id);
int register_fake_key(const uint8_t key_data[32]);
int unregister_fake_key(int key_id);
int get_fake_key(int key_id, uint8_t key_data[32]);
ps5-kstuff/uelf/fpkg.c
+47 -29
View File
@@ -37,6 +37,11 @@ enum crypto_message_kind
CRYPTO_MESSAGE_XTS,
};
enum {
CRYPTO_MESSAGE_DATA_QWORDS = 21,
CRYPTO_MESSAGE_NEXT_OFFSET = 320,
};
struct crypto_message_info
{
enum crypto_message_kind kind;
@@ -60,37 +65,50 @@ static int crypto_request_emulated(uint64_t* regs, uint64_t msg, uint32_t status
return 1;
}
static int read_crypto_message(uint64_t msg, uint64_t msg_data[21])
static int read_crypto_message(uint64_t msg,
uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS],
uint64_t* next_msg)
{
return copy_from_kernel(msg_data, msg, sizeof(uint64_t) * 21);
METRIC_INC(crypto_message_snapshot_reads);
METRIC_TIME_START(start_cycles);
int error = copy_from_kernel(msg_data, msg,
sizeof(uint64_t) * CRYPTO_MESSAGE_DATA_QWORDS);
if(!error)
error = copy_u64_from_kernel(next_msg, msg + CRYPTO_MESSAGE_NEXT_OFFSET);
if(error)
METRIC_INC(crypto_message_snapshot_failures);
METRIC_TIME(crypto_message_snapshot_cycles_total,
crypto_message_snapshot_cycles_max, start_cycles);
return error;
}
static int is_hmac_message(const uint64_t msg_data[21])
static int is_hmac_message(const uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS])
{
return (msg_data[0] & 0x7fffffff) == 0x9132000;
}
static int is_xts_message(const uint64_t msg_data[21])
static int is_xts_message(const uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS])
{
return (msg_data[0] & 0x7ffff7ff) == 0x2108000;
}
static struct crypto_message_result unhandled_crypto_message(uint64_t msg)
static struct crypto_message_result unhandled_crypto_message(uint64_t next_msg)
{
return (struct crypto_message_result){
.emulated_messages = 0,
.next_msg = kpeek64(msg + 320),
.next_msg = next_msg,
.status = ENOSYS,
.total_messages = 1,
};
}
static struct crypto_message_info inspect_crypto_message(const uint64_t msg_data[21])
static struct crypto_message_info inspect_crypto_message(
const uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS], uint8_t key[32])
{
if(is_xts_message(msg_data))
{
int key_idx = HANDLE_TO_IDX(msg_data[5]);
if(key_idx < 0 || !has_fake_key(key_idx))
if(key_idx < 0 || !get_fake_key(key_idx, key))
{
METRIC_INC(fpkg_reject_xts_non_fake);
}
@@ -110,7 +128,7 @@ static struct crypto_message_info inspect_crypto_message(const uint64_t msg_data
{
METRIC_INC(fpkg_reject_hmac_bad_shape);
}
else if(key_idx < 0 || !has_fake_key(key_idx))
else if(key_idx < 0 || !get_fake_key(key_idx, key))
{
METRIC_INC(fpkg_reject_hmac_non_fake);
}
@@ -130,15 +148,13 @@ static struct crypto_message_info inspect_crypto_message(const uint64_t msg_data
};
}
static struct crypto_message_result handle_hmac_message(uint64_t msg, const uint64_t msg_data[21],
int key_idx,
static struct crypto_message_result handle_hmac_message(uint64_t msg,
const uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS],
uint64_t next_msg,
int key_idx, const uint8_t key[32],
struct crypto_request_cache* cache)
{
struct crypto_message_result result = unhandled_crypto_message(msg);
uint8_t key[32];
if(!get_fake_key(key_idx, key))
return result;
struct crypto_message_result result = unhandled_crypto_message(next_msg);
uint8_t hash[32] = {0};
if(pfs_hmac_virtual_fpu_held(cache, hash, key_idx, key, msg_data[2], msg_data[1]))
{
@@ -157,14 +173,11 @@ static struct crypto_message_result handle_hmac_message(uint64_t msg, const uint
return result;
}
static struct crypto_message_result handle_xts_message(uint64_t msg, const uint64_t msg_data[21],
int key_idx,
struct crypto_request_cache* cache)
static struct crypto_message_result handle_xts_message(
const uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS], uint64_t next_msg,
int key_idx, const uint8_t key[32], struct crypto_request_cache* cache)
{
struct crypto_message_result result = unhandled_crypto_message(msg);
uint8_t key[32];
if(!get_fake_key(key_idx, key))
return result;
struct crypto_message_result result = unhandled_crypto_message(next_msg);
METRIC_INC(xts_run_messages_total);
uint64_t src = msg_data[2];
@@ -209,14 +222,16 @@ static int handle_crypto_request(uint64_t* regs)
for (uint64_t msg = start; msg && !total_status;)
{
uint64_t msg_data[21];
if(read_crypto_message(msg, msg_data))
uint64_t msg_data[CRYPTO_MESSAGE_DATA_QWORDS];
uint64_t next_msg;
if(read_crypto_message(msg, msg_data, &next_msg))
{
total_status = -1;
break;
}
struct crypto_message_info msg_info = inspect_crypto_message(msg_data);
uint8_t key[32];
struct crypto_message_info msg_info = inspect_crypto_message(msg_data, key);
METRIC_INC(crypto_messages_total);
if(msg_info.kind == CRYPTO_MESSAGE_XTS)
METRIC_INC(crypto_messages_xts);
@@ -236,11 +251,14 @@ static int handle_crypto_request(uint64_t* regs)
struct crypto_message_result result;
if(msg_info.kind == CRYPTO_MESSAGE_XTS)
result = handle_xts_message(msg, msg_data, msg_info.key_idx, &s_crypto_request_cache);
result = handle_xts_message(msg_data, next_msg, msg_info.key_idx, key,
&s_crypto_request_cache);
else if(msg_info.kind == CRYPTO_MESSAGE_HMAC)
result = handle_hmac_message(msg, msg_data, msg_info.key_idx, &s_crypto_request_cache);
result = handle_hmac_message(msg, msg_data, next_msg,
msg_info.key_idx, key,
&s_crypto_request_cache);
else
result = unhandled_crypto_message(msg);
result = unhandled_crypto_message(next_msg);
int status = result.status;
total += result.total_messages;
ps5-kstuff/uelf/main.c
+18 -3
View File
@@ -466,17 +466,32 @@ from_userspace:
void main(uint64_t just_return)
{
METRIC_INC(uelf_main_entries);
METRIC_TIME_START(start_cycles);
#define RETURN_UELF_MAIN() do { \
METRIC_TIME(uelf_main_cycles_total, uelf_main_cycles_max, start_cycles); \
return; \
} while(0)
uint64_t regs[NREGS + 1];
if(copy_from_trap_frame_cached(regs, sizeof(regs)))
return;
{
METRIC_INC(uelf_main_trap_read_failures);
RETURN_UELF_MAIN();
}
uint64_t jr_frame[5];
if(copy_from_just_return_cached(jr_frame, just_return, sizeof(jr_frame)))
return;
{
METRIC_INC(uelf_main_just_return_read_failures);
RETURN_UELF_MAIN();
}
have_error_code = jr_frame[0];
regs[RDX] = jr_frame[2];
regs[RCX] = jr_frame[3];
regs[RAX] = jr_frame[4];
intno = regs[NREGS];
handle(regs);
copy_to_trap_frame_cached(regs, NREGS * sizeof(uint64_t));
if(copy_to_trap_frame_cached(regs, NREGS * sizeof(uint64_t)))
METRIC_INC(uelf_main_trap_write_failures);
METRIC_TIME(uelf_main_cycles_total, uelf_main_cycles_max, start_cycles);
#undef RETURN_UELF_MAIN
}
ps5-kstuff/uelf/pfs_crypto.c
+28 -18
View File
@@ -82,20 +82,24 @@ static int hmac_sha256_cache_entry_matches(const struct hmac_sha256_cache_entry*
}
static struct hmac_sha256_cache_entry* select_hmac_sha256_cache_entry(struct crypto_request_cache* cache,
int key_id, const uint8_t* key)
int key_id, const uint8_t* key,
int* matched)
{
struct hmac_sha256_cache_entry* first_empty = NULL;
for(size_t i = 0; i < PFS_HMAC_SHA256_CACHE_SLOTS; i++)
{
struct hmac_sha256_cache_entry* entry = &cache->hmac[i];
if(hmac_sha256_cache_entry_matches(entry, key_id, key))
{
*matched = 1;
return entry;
}
if(!entry->valid && !first_empty)
first_empty = entry;
}
for(size_t i = 0; i < PFS_HMAC_SHA256_CACHE_SLOTS; i++)
{
struct hmac_sha256_cache_entry* entry = &cache->hmac[i];
if(!entry->valid)
return entry;
}
*matched = 0;
if(first_empty)
return first_empty;
struct hmac_sha256_cache_entry* entry = &cache->hmac[cache->hmac_next_slot];
cache->hmac_next_slot++;
if(cache->hmac_next_slot == PFS_HMAC_SHA256_CACHE_SLOTS)
@@ -106,8 +110,10 @@ static struct hmac_sha256_cache_entry* select_hmac_sha256_cache_entry(struct cry
static int get_hmac_sha256_cache_entry(struct crypto_request_cache* cache, int key_id, const uint8_t* key,
const struct hmac_sha256_cache_entry** out)
{
struct hmac_sha256_cache_entry* entry = select_hmac_sha256_cache_entry(cache, key_id, key);
if(hmac_sha256_cache_entry_matches(entry, key_id, key))
int matched;
struct hmac_sha256_cache_entry* entry = select_hmac_sha256_cache_entry(cache, key_id, key,
&matched);
if(matched)
{
METRIC_INC(hmac_cache_hits);
*out = entry;
@@ -213,20 +219,23 @@ static int xts_key_cache_entry_matches(const struct xts_key_cache_entry* entry,
}
static struct xts_key_cache_entry* select_xts_key_cache_entry(struct crypto_request_cache* cache, int key_id,
const uint8_t* key)
const uint8_t* key, int* matched)
{
struct xts_key_cache_entry* first_empty = NULL;
for(size_t i = 0; i < PFS_XTS_KEY_CACHE_SLOTS; i++)
{
struct xts_key_cache_entry* entry = &cache->xts[i];
if(xts_key_cache_entry_matches(entry, key_id, key))
{
*matched = 1;
return entry;
}
if(!entry->valid && !first_empty)
first_empty = entry;
}
for(size_t i = 0; i < PFS_XTS_KEY_CACHE_SLOTS; i++)
{
struct xts_key_cache_entry* entry = &cache->xts[i];
if(!entry->valid)
return entry;
}
*matched = 0;
if(first_empty)
return first_empty;
struct xts_key_cache_entry* entry = &cache->xts[cache->xts_next_slot];
cache->xts_next_slot++;
if(cache->xts_next_slot == PFS_XTS_KEY_CACHE_SLOTS)
@@ -237,8 +246,9 @@ static struct xts_key_cache_entry* select_xts_key_cache_entry(struct crypto_requ
static int get_xts_key_cache_entry(struct crypto_request_cache* cache, int key_id, const uint8_t* key,
const struct xts_key_cache_entry** out)
{
struct xts_key_cache_entry* entry = select_xts_key_cache_entry(cache, key_id, key);
if(xts_key_cache_entry_matches(entry, key_id, key))
int matched;
struct xts_key_cache_entry* entry = select_xts_key_cache_entry(cache, key_id, key, &matched);
if(matched)
{
METRIC_INC(xts_cache_hits);
*out = entry;
ps5-kstuff/uelf/shared_area.h
+44 -3
View File
@@ -221,6 +221,47 @@ struct kstuff_metrics
uint64_t log_word_writes;
uint64_t log_msg_writes;
uint64_t log_msg_bytes;
/* Keep new metrics appended so existing field offsets remain stable. */
uint64_t uelf_main_entries;
uint64_t uelf_main_trap_read_failures;
uint64_t uelf_main_just_return_read_failures;
uint64_t uelf_main_trap_write_failures;
uint64_t uelf_main_cycles_total;
uint64_t uelf_main_cycles_max;
uint64_t run_gadget_calls;
uint64_t run_gadget_failures;
uint64_t run_gadget_cycles_total;
uint64_t run_gadget_cycles_max;
uint64_t scalar_copy_from_calls;
uint64_t scalar_copy_from_cycles_total;
uint64_t scalar_copy_from_cycles_max;
uint64_t scalar_copy_to_calls;
uint64_t scalar_copy_to_cycles_total;
uint64_t scalar_copy_to_cycles_max;
uint64_t crypto_message_snapshot_reads;
uint64_t crypto_message_snapshot_failures;
uint64_t crypto_message_snapshot_cycles_total;
uint64_t crypto_message_snapshot_cycles_max;
uint64_t trap_mapping_hits;
uint64_t trap_mapping_misses;
uint64_t trap_mapping_fallbacks;
uint64_t just_return_mapping_hits;
uint64_t just_return_mapping_misses;
uint64_t just_return_mapping_fallbacks;
uint64_t pcpu_mapping_hits;
uint64_t pcpu_mapping_misses;
uint64_t pcpu_mapping_fallbacks;
uint64_t tss_mapping_hits;
uint64_t tss_mapping_misses;
uint64_t tss_mapping_fallbacks;
uint64_t wrmsr_args_mapping_hits;
uint64_t wrmsr_args_mapping_misses;
uint64_t wrmsr_args_mapping_fallbacks;
};
struct kstuff_word_log_entry
@@ -299,14 +340,14 @@ extern struct shared_area_layout shared_area;
#define METRIC_MAX(field, value) do { } while(0)
#endif
_Static_assert(sizeof(struct kstuff_metrics) == 1512, "unexpected metrics size");
_Static_assert(sizeof(struct kstuff_metrics) == 1792, "unexpected metrics size");
_Static_assert(sizeof(struct kstuff_word_log) == 264, "unexpected word log size");
_Static_assert(sizeof(struct kstuff_ioctl_com_entry) == 24, "unexpected ioctl com entry size");
_Static_assert(sizeof(struct kstuff_ioctl_com_table) == 3088, "unexpected ioctl com table size");
_Static_assert(sizeof(struct kstuff_msg_log) == 504, "unexpected message log size");
_Static_assert(sizeof(struct kstuff_snapshot) == 5384, "unexpected snapshot size");
_Static_assert(sizeof(struct kstuff_snapshot) == 5664, "unexpected snapshot size");
#if KSTUFF_OBS
_Static_assert(sizeof(struct shared_area_layout) == 7416, "unexpected shared_area size");
_Static_assert(sizeof(struct shared_area_layout) == 7696, "unexpected shared_area size");
#else
_Static_assert(sizeof(struct shared_area_layout) == 2048, "unexpected non-OBS shared_area size");
#endif
ps5-kstuff/uelf/utils.c
+94 -29
View File
@@ -115,6 +115,29 @@ int copy_to_kernel(uint64_t dst, const void* src, uint64_t sz)
return 0;
}
#if KSTUFF_OBS
#define METRIC_SCALAR_COPY_FROM_TIME(start) do { \
uint64_t _metric_elapsed = uelf_rdtsc() - (uint64_t)(start); \
METRIC_INC(scalar_copy_from_calls); \
METRIC_ADD(copy_from_cycles_total, _metric_elapsed); \
METRIC_MAX(copy_from_cycles_max, _metric_elapsed); \
METRIC_ADD(scalar_copy_from_cycles_total, _metric_elapsed); \
METRIC_MAX(scalar_copy_from_cycles_max, _metric_elapsed); \
} while(0)
#define METRIC_SCALAR_COPY_TO_TIME(start) do { \
uint64_t _metric_elapsed = uelf_rdtsc() - (uint64_t)(start); \
METRIC_INC(scalar_copy_to_calls); \
METRIC_ADD(copy_to_cycles_total, _metric_elapsed); \
METRIC_MAX(copy_to_cycles_max, _metric_elapsed); \
METRIC_ADD(scalar_copy_to_cycles_total, _metric_elapsed); \
METRIC_MAX(scalar_copy_to_cycles_max, _metric_elapsed); \
} while(0)
#else
#define METRIC_SCALAR_COPY_FROM_TIME(start) do { } while(0)
#define METRIC_SCALAR_COPY_TO_TIME(start) do { } while(0)
#endif
int copy_u16_from_kernel(uint16_t* dst, uint64_t src)
{
METRIC_TIME_START(start_cycles);
@@ -125,7 +148,7 @@ int copy_u16_from_kernel(uint16_t* dst, uint64_t src)
{
METRIC_INC(copy_from_failures);
log_word((uint64_t)__builtin_return_address(0));
METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles);
METRIC_SCALAR_COPY_FROM_TIME(start_cycles);
return EFAULT;
}
if(phys_end - phys >= sizeof(*dst))
@@ -134,10 +157,10 @@ int copy_u16_from_kernel(uint16_t* dst, uint64_t src)
{
METRIC_INC(copy_from_failures);
log_word((uint64_t)__builtin_return_address(0));
METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles);
METRIC_SCALAR_COPY_FROM_TIME(start_cycles);
return EFAULT;
}
METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles);
METRIC_SCALAR_COPY_FROM_TIME(start_cycles);
return 0;
}
@@ -151,7 +174,7 @@ int copy_u32_from_kernel(uint32_t* dst, uint64_t src)
{
METRIC_INC(copy_from_failures);
log_word((uint64_t)__builtin_return_address(0));
METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles);
METRIC_SCALAR_COPY_FROM_TIME(start_cycles);
return EFAULT;
}
if(phys_end - phys >= sizeof(*dst))
@@ -160,10 +183,10 @@ int copy_u32_from_kernel(uint32_t* dst, uint64_t src)
{
METRIC_INC(copy_from_failures);
log_word((uint64_t)__builtin_return_address(0));
METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles);
METRIC_SCALAR_COPY_FROM_TIME(start_cycles);
return EFAULT;
}
METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles);
METRIC_SCALAR_COPY_FROM_TIME(start_cycles);
return 0;
}
@@ -177,7 +200,7 @@ int copy_u64_from_kernel(uint64_t* dst, uint64_t src)
{
METRIC_INC(copy_from_failures);
log_word((uint64_t)__builtin_return_address(0));
METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles);
METRIC_SCALAR_COPY_FROM_TIME(start_cycles);
return EFAULT;
}
if(phys_end - phys >= sizeof(*dst))
@@ -186,10 +209,10 @@ int copy_u64_from_kernel(uint64_t* dst, uint64_t src)
{
METRIC_INC(copy_from_failures);
log_word((uint64_t)__builtin_return_address(0));
METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles);
METRIC_SCALAR_COPY_FROM_TIME(start_cycles);
return EFAULT;
}
METRIC_TIME(copy_from_cycles_total, copy_from_cycles_max, start_cycles);
METRIC_SCALAR_COPY_FROM_TIME(start_cycles);
return 0;
}
@@ -203,7 +226,7 @@ int copy_u16_to_kernel(uint64_t dst, uint16_t value)
{
METRIC_INC(copy_to_failures);
log_word((uint64_t)__builtin_return_address(0));
METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles);
METRIC_SCALAR_COPY_TO_TIME(start_cycles);
return EFAULT;
}
if(phys_end - phys >= sizeof(value))
@@ -212,10 +235,10 @@ int copy_u16_to_kernel(uint64_t dst, uint16_t value)
{
METRIC_INC(copy_to_failures);
log_word((uint64_t)__builtin_return_address(0));
METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles);
METRIC_SCALAR_COPY_TO_TIME(start_cycles);
return EFAULT;
}
METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles);
METRIC_SCALAR_COPY_TO_TIME(start_cycles);
return 0;
}
@@ -229,7 +252,7 @@ int copy_u32_to_kernel(uint64_t dst, uint32_t value)
{
METRIC_INC(copy_to_failures);
log_word((uint64_t)__builtin_return_address(0));
METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles);
METRIC_SCALAR_COPY_TO_TIME(start_cycles);
return EFAULT;
}
if(phys_end - phys >= sizeof(value))
@@ -238,10 +261,10 @@ int copy_u32_to_kernel(uint64_t dst, uint32_t value)
{
METRIC_INC(copy_to_failures);
log_word((uint64_t)__builtin_return_address(0));
METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles);
METRIC_SCALAR_COPY_TO_TIME(start_cycles);
return EFAULT;
}
METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles);
METRIC_SCALAR_COPY_TO_TIME(start_cycles);
return 0;
}
@@ -255,7 +278,7 @@ int copy_u64_to_kernel(uint64_t dst, uint64_t value)
{
METRIC_INC(copy_to_failures);
log_word((uint64_t)__builtin_return_address(0));
METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles);
METRIC_SCALAR_COPY_TO_TIME(start_cycles);
return EFAULT;
}
if(phys_end - phys >= sizeof(value))
@@ -264,26 +287,51 @@ int copy_u64_to_kernel(uint64_t dst, uint64_t value)
{
METRIC_INC(copy_to_failures);
log_word((uint64_t)__builtin_return_address(0));
METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles);
METRIC_SCALAR_COPY_TO_TIME(start_cycles);
return EFAULT;
}
METRIC_TIME(copy_to_cycles_total, copy_to_cycles_max, start_cycles);
METRIC_SCALAR_COPY_TO_TIME(start_cycles);
return 0;
}
#undef METRIC_SCALAR_COPY_FROM_TIME
#undef METRIC_SCALAR_COPY_TO_TIME
uint64_t yield(void);
struct kernel_mapping_cache
{
uint64_t physical_address;
int valid;
#if KSTUFF_OBS
uint64_t* hits;
uint64_t* misses;
uint64_t* fallbacks;
#endif
};
static struct kernel_mapping_cache s_trap_frame_mapping;
static struct kernel_mapping_cache s_just_return_mapping;
static struct kernel_mapping_cache s_pcpu_mapping;
static struct kernel_mapping_cache s_tss_rsp0_mapping;
static struct kernel_mapping_cache s_wrmsr_args_mapping;
#if KSTUFF_OBS
#define DEFINE_MAPPING_CACHE(name, metric_prefix) \
static struct kernel_mapping_cache name = { \
.hits = &shared_area.metrics.metric_prefix##_hits, \
.misses = &shared_area.metrics.metric_prefix##_misses, \
.fallbacks = &shared_area.metrics.metric_prefix##_fallbacks, \
}
#define OBSERVE_MAPPING_CACHE(cache, field) \
__atomic_fetch_add((cache)->field, 1, __ATOMIC_RELAXED)
#else
#define DEFINE_MAPPING_CACHE(name, metric_prefix) \
static struct kernel_mapping_cache name
#define OBSERVE_MAPPING_CACHE(cache, field) do { (void)(cache); } while(0)
#endif
DEFINE_MAPPING_CACHE(s_trap_frame_mapping, trap_mapping);
DEFINE_MAPPING_CACHE(s_just_return_mapping, just_return_mapping);
DEFINE_MAPPING_CACHE(s_pcpu_mapping, pcpu_mapping);
DEFINE_MAPPING_CACHE(s_tss_rsp0_mapping, tss_mapping);
DEFINE_MAPPING_CACHE(s_wrmsr_args_mapping, wrmsr_args_mapping);
#undef DEFINE_MAPPING_CACHE
extern char tss[];
extern uint64_t wrmsr_args;
@@ -311,13 +359,20 @@ static int get_cached_physical_address(struct kernel_mapping_cache* cache,
{
if(__atomic_load_n(&cache->valid, __ATOMIC_ACQUIRE))
{
OBSERVE_MAPPING_CACHE(cache, hits);
*physical_address = cache->physical_address;
return 1;
}
return initialize_kernel_mapping_cache(cache, address, size,
physical_address);
OBSERVE_MAPPING_CACHE(cache, misses);
int initialized = initialize_kernel_mapping_cache(cache, address, size,
physical_address);
if(!initialized)
OBSERVE_MAPPING_CACHE(cache, fallbacks);
return initialized;
}
#undef OBSERVE_MAPPING_CACHE
static __attribute__((noinline, cold)) int copy_from_kernel_uncached(
void* dst, uint64_t src, uint64_t size)
{
@@ -425,18 +480,28 @@ __attribute__((noinline)) int copy_to_wrmsr_args_cached(const uint64_t args[3])
__attribute__((noinline)) int run_gadget_checked(uint64_t* regs)
{
METRIC_INC(run_gadget_calls);
METRIC_TIME_START(start_cycles);
#define RETURN_RUN_GADGET(value) do { \
int _run_gadget_result = (value); \
if(_run_gadget_result) \
METRIC_INC(run_gadget_failures); \
METRIC_TIME(run_gadget_cycles_total, run_gadget_cycles_max, start_cycles); \
return _run_gadget_result; \
} while(0)
if(copy_to_trap_frame_cached(regs, NREGS*8))
return EFAULT;
RETURN_RUN_GADGET(EFAULT);
uint64_t just_return = yield();
uint64_t jr_frame[5];
if(copy_from_trap_frame_cached(regs, NREGS*8))
return EFAULT;
RETURN_RUN_GADGET(EFAULT);
if(copy_from_just_return_cached(jr_frame, just_return, sizeof(jr_frame)))
return EFAULT;
RETURN_RUN_GADGET(EFAULT);
regs[RDX] = jr_frame[2];
regs[RCX] = jr_frame[3];
regs[RAX] = jr_frame[4];
return 0;
RETURN_RUN_GADGET(0);
#undef RETURN_RUN_GADGET
}
extern char dr2gpr_start[];