diff options
Diffstat (limited to 'mm/kfence/core.c')
| -rw-r--r-- | mm/kfence/core.c | 70 |
1 files changed, 49 insertions, 21 deletions
diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 7d01a2c76e80..dad3c0eb70a0 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -297,20 +297,13 @@ metadata_update_state(struct kfence_metadata *meta, enum kfence_object_state nex WRITE_ONCE(meta->state, next); } -/* Write canary byte to @addr. */ -static inline bool set_canary_byte(u8 *addr) -{ - *addr = KFENCE_CANARY_PATTERN(addr); - return true; -} - /* Check canary byte at @addr. */ static inline bool check_canary_byte(u8 *addr) { struct kfence_metadata *meta; unsigned long flags; - if (likely(*addr == KFENCE_CANARY_PATTERN(addr))) + if (likely(*addr == KFENCE_CANARY_PATTERN_U8(addr))) return true; atomic_long_inc(&counters[KFENCE_COUNTER_BUGS]); @@ -323,15 +316,31 @@ static inline bool check_canary_byte(u8 *addr) return false; } -/* __always_inline this to ensure we won't do an indirect call to fn. */ -static __always_inline void for_each_canary(const struct kfence_metadata *meta, bool (*fn)(u8 *)) +static inline void set_canary(const struct kfence_metadata *meta) { const unsigned long pageaddr = ALIGN_DOWN(meta->addr, PAGE_SIZE); - unsigned long addr; + unsigned long addr = pageaddr; + + /* + * The canary may be written to part of the object memory, but it does + * not affect it. The user should initialize the object before using it. + */ + for (; addr < meta->addr; addr += sizeof(u64)) + *((u64 *)addr) = KFENCE_CANARY_PATTERN_U64; + + addr = ALIGN_DOWN(meta->addr + meta->size, sizeof(u64)); + for (; addr - pageaddr < PAGE_SIZE; addr += sizeof(u64)) + *((u64 *)addr) = KFENCE_CANARY_PATTERN_U64; +} + +static inline void check_canary(const struct kfence_metadata *meta) +{ + const unsigned long pageaddr = ALIGN_DOWN(meta->addr, PAGE_SIZE); + unsigned long addr = pageaddr; /* - * We'll iterate over each canary byte per-side until fn() returns - * false. However, we'll still iterate over the canary bytes to the + * We'll iterate over each canary byte per-side until a corrupted byte + * is found. However, we'll still iterate over the canary bytes to the * right of the object even if there was an error in the canary bytes to * the left of the object. Specifically, if check_canary_byte() * generates an error, showing both sides might give more clues as to @@ -339,16 +348,35 @@ static __always_inline void for_each_canary(const struct kfence_metadata *meta, */ /* Apply to left of object. */ - for (addr = pageaddr; addr < meta->addr; addr++) { - if (!fn((u8 *)addr)) + for (; meta->addr - addr >= sizeof(u64); addr += sizeof(u64)) { + if (unlikely(*((u64 *)addr) != KFENCE_CANARY_PATTERN_U64)) break; } - /* Apply to right of object. */ - for (addr = meta->addr + meta->size; addr < pageaddr + PAGE_SIZE; addr++) { - if (!fn((u8 *)addr)) + /* + * If the canary is corrupted in a certain 64 bytes, or the canary + * memory cannot be completely covered by multiple consecutive 64 bytes, + * it needs to be checked one by one. + */ + for (; addr < meta->addr; addr++) { + if (unlikely(!check_canary_byte((u8 *)addr))) break; } + + /* Apply to right of object. */ + for (addr = meta->addr + meta->size; addr % sizeof(u64) != 0; addr++) { + if (unlikely(!check_canary_byte((u8 *)addr))) + return; + } + for (; addr - pageaddr < PAGE_SIZE; addr += sizeof(u64)) { + if (unlikely(*((u64 *)addr) != KFENCE_CANARY_PATTERN_U64)) { + + for (; addr - pageaddr < PAGE_SIZE; addr++) { + if (!check_canary_byte((u8 *)addr)) + return; + } + } + } } static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t gfp, @@ -434,7 +462,7 @@ static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t g #endif /* Memory initialization. */ - for_each_canary(meta, set_canary_byte); + set_canary(meta); /* * We check slab_want_init_on_alloc() ourselves, rather than letting @@ -495,7 +523,7 @@ static void kfence_guarded_free(void *addr, struct kfence_metadata *meta, bool z alloc_covered_add(meta->alloc_stack_hash, -1); /* Check canary bytes for memory corruption. */ - for_each_canary(meta, check_canary_byte); + check_canary(meta); /* * Clear memory if init-on-free is set. While we protect the page, the @@ -751,7 +779,7 @@ static void kfence_check_all_canary(void) struct kfence_metadata *meta = &kfence_metadata[i]; if (meta->state == KFENCE_OBJECT_ALLOCATED) - for_each_canary(meta, check_canary_byte); + check_canary(meta); } } |