diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c index e3535e2a86f8..59d42dfbb273 100644 --- a/sys/kern/kern_malloc.c +++ b/sys/kern/kern_malloc.c @@ -110,7 +110,7 @@ dtrace_malloc_probe_func_t __read_mostly dtrace_malloc_probe; #define MALLOC_DEBUG 1 #endif -#if defined(KASAN) || defined(DEBUG_REDZONE) +#if defined(KASAN) || defined(DEBUG_REDZONE) || defined(__CHERI_PURE_CAPABILITY__) #define DEBUG_REDZONE_ARG_DEF , unsigned long osize #define DEBUG_REDZONE_ARG , osize #else @@ -633,14 +633,24 @@ malloc_large(size_t size, struct malloc_type *mtp, struct domainset *policy, size = roundup(size, PAGE_SIZE); va = kmem_malloc_domainset(policy, size, flags); if (va != NULL) { - /* Use low bits unused for slab pointers. */ - vsetzoneslab((uintptr_t)va, NULL, MALLOC_LARGE_SLAB(size)); + /* + * Use low bits unused for slab pointers. + * XXX-AM: Abuse the zone pointer to stash the original pointer. + * On CHERI systems, this is necessary to recover the bounds of + * the original allocation. + */ + vsetzoneslab((uintptr_t)va, va, MALLOC_LARGE_SLAB(size)); uma_total_inc(size); #ifdef __CHERI_PURE_CAPABILITY__ - KASSERT(cheri_getlen(va) <= CHERI_REPRESENTABLE_LENGTH(size), + va = cheri_setbounds(va, osize); + KASSERT(cheri_getlen(va) <= CHERI_REPRESENTABLE_LENGTH(osize), ("Invalid bounds: expected %zx found %zx", - (size_t)CHERI_REPRESENTABLE_LENGTH(size), + (size_t)CHERI_REPRESENTABLE_LENGTH(osize), (size_t)cheri_getlen(va))); + if ((flags & M_ZERO) == 0 && osize < cheri_getlen(va)) { + bzero((void *)((uintptr_t)va + osize), + cheri_getlen(va) - osize); + } #endif } malloc_type_allocated(mtp, va, va == NULL ? 0 : size); @@ -659,10 +669,32 @@ malloc_large(size_t size, struct malloc_type *mtp, struct domainset *policy, static void free_large(void *addr, size_t size) { - kmem_free(addr, size); uma_total_dec(size); } + +#ifdef __CHERI_PURE_CAPABILITY__ +/* + * Recover original bounds for a malloc_large allocation. + * + * Error conditions: + * - Clear the capability tag if the given capability is not a subset of + * the saved object capability. + */ +static void * +malloc_large_grow_bounds(void *saved_ptr, void *addr) +{ + KASSERT(cheri_is_subset(saved_ptr, addr), + ("Unexpected malloc_large grow bounds: pointer %#p is " + "not derived from %#p", addr, saved_ptr)); + + addr = cheri_setaddress(saved_ptr, (vm_offset_t)addr); + addr = cheri_setboundsexact(addr, cheri_getlen(saved_ptr)); + KASSERT(cheri_gettag(addr), + ("Failed to recover malloc_large bounds for %#p", addr)); + return (addr); +} +#endif #undef IS_MALLOC_LARGE #undef MALLOC_LARGE_SLAB @@ -680,7 +712,7 @@ void * int indx; caddr_t va; uma_zone_t zone; -#if defined(DEBUG_REDZONE) || defined(KASAN) +#if defined(DEBUG_REDZONE) || defined(KASAN) || defined(__CHERI_PURE_CAPABILITY__) unsigned long osize = size; #endif @@ -722,10 +754,17 @@ void * kasan_mark((void *)va, osize, size, KASAN_MALLOC_REDZONE); #endif #ifdef __CHERI_PURE_CAPABILITY__ - KASSERT(cheri_getlen(va) <= CHERI_REPRESENTABLE_LENGTH(size), - ("Invalid bounds: expected %zx found %zx", - (size_t)CHERI_REPRESENTABLE_LENGTH(size), - (size_t)cheri_getlen(va))); + /* Intentionally inexect bounds allow for non-representable sizes */ + va = cheri_setbounds(va, osize); + KASSERT(cheri_gettag(va), + ("Invalid malloc: %#p requested size %zx", va, osize)); + KASSERT(cheri_getlen(va) <= CHERI_REPRESENTABLE_LENGTH(osize), + ("Invalid malloc: %#p expected length %zx", va, + (size_t)CHERI_REPRESENTABLE_LENGTH(osize))); + if (va != NULL && (flags & M_ZERO) == 0 && osize < cheri_getlen(va)) { + bzero((void *)((uintptr_t)va + osize), + cheri_getlen(va) - osize); + } #endif return ((void *) va); } @@ -765,7 +804,7 @@ malloc_domainset(size_t size, struct malloc_type *mtp, struct domainset *ds, caddr_t va; int domain; int indx; -#if defined(KASAN) || defined(DEBUG_REDZONE) +#if defined(KASAN) || defined(DEBUG_REDZONE) || defined(__CHERI_PURE_CAPABILITY__) unsigned long osize = size; #endif @@ -781,6 +820,11 @@ malloc_domainset(size_t size, struct malloc_type *mtp, struct domainset *ds, return (malloc_large(size, mtp, DOMAINSET_RR(), flags DEBUG_REDZONE_ARG)); + /* XXX-AM: see malloc() */ + if (size != CHERI_REPRESENTABLE_LENGTH(size)) { + flags |= M_ZERO; + } + vm_domainset_iter_policy_init(&di, ds, &domain, &flags); do { va = malloc_domain(&size, &indx, mtp, domain, flags); @@ -803,6 +847,15 @@ malloc_domainset(size_t size, struct malloc_type *mtp, struct domainset *ds, kmsan_mark(va, size, KMSAN_STATE_UNINIT); kmsan_orig(va, size, KMSAN_TYPE_MALLOC, KMSAN_RET_ADDR); } +#endif +#ifdef __CHERI_PURE_CAPABILITY__ + /* Intentionally inexect bounds allow for non-representable sizes */ + va = cheri_setbounds(va, osize); + KASSERT(cheri_gettag(va), + ("Invalid malloc: %#p requested size %zx", va, osize)); + KASSERT(cheri_getlen(va) == CHERI_REPRESENTABLE_LENGTH(osize), + ("Invalid malloc: %#p expected length %zx", va, + (size_t)CHERI_REPRESENTABLE_LENGTH(osize))); #endif return (va); } @@ -821,7 +874,7 @@ void * malloc_domainset_exec(size_t size, struct malloc_type *mtp, struct domainset *ds, int flags) { -#if defined(DEBUG_REDZONE) || defined(KASAN) +#if defined(DEBUG_REDZONE) || defined(KASAN) || defined(__CHERI_PURE_CAPABILITY__) unsigned long osize = size; #endif #ifdef MALLOC_DEBUG @@ -996,11 +1049,10 @@ _free(void *addr, struct malloc_type *mtp, bool dozero) case __predict_true(SLAB_COOKIE_SLAB_PTR): size = zone->uz_size; #ifdef __CHERI_PURE_CAPABILITY__ - if (__predict_false(cheri_getlen(addr) != - CHERI_REPRESENTABLE_LENGTH(size))) - panic("Invalid bounds: expected %zx found %zx", - (size_t)CHERI_REPRESENTABLE_LENGTH(size), - cheri_getlen(addr)); + addr = uma_zgrow_bounds(zone, addr); + KASSERT(cheri_getlen(addr) == size, + ("vtozoneslab disagrees with uma_zgrow_bounds: %zx != %zx", + cheri_getlen(addr), size)); #endif #if defined(INVARIANTS) && !defined(KASAN) free_save_type(addr, mtp, size); @@ -1014,11 +1066,10 @@ _free(void *addr, struct malloc_type *mtp, bool dozero) case SLAB_COOKIE_MALLOC_LARGE: size = malloc_large_size(slab); #ifdef __CHERI_PURE_CAPABILITY__ - if (__predict_false(cheri_getlen(addr) != - CHERI_REPRESENTABLE_LENGTH(size))) - panic("Invalid bounds: expected %zx found %zx", - (size_t)CHERI_REPRESENTABLE_LENGTH(size), - cheri_getlen(addr)); + addr = malloc_large_grow_bounds(zone, addr); + KASSERT(cheri_getlen(addr) == size, + ("malloc_large object bounds mismatch: %zx != %zx", + cheri_getlen(addr), size)); #endif if (dozero) { kasan_mark(addr, size, size, 0); @@ -1074,6 +1125,9 @@ realloc(void *addr, size_t size, struct malloc_type *mtp, int flags) #endif unsigned long alloc; void *newaddr; +#ifdef __CHERI_PURE_CAPABILITY__ + size_t olength; +#endif KASSERT(mtp->ks_version == M_VERSION, ("realloc: bad malloc type version")); @@ -1088,6 +1142,7 @@ realloc(void *addr, size_t size, struct malloc_type *mtp, int flags) panic("Expect valid capability"); if (__predict_false(cheri_getsealed(addr))) panic("Expect unsealed capability"); + olength = cheri_getlen(addr); #endif /* @@ -1113,9 +1168,25 @@ realloc(void *addr, size_t size, struct malloc_type *mtp, int flags) switch (GET_SLAB_COOKIE(slab)) { case __predict_true(SLAB_COOKIE_SLAB_PTR): alloc = zone->uz_size; +#ifdef __CHERI_PURE_CAPABILITY__ + if (size > olength) { + addr = uma_zgrow_bounds(zone, addr); + KASSERT(cheri_getlen(addr) == alloc, + ("realloc mismatch uma_zgrow_bounds: %zx != %zx", + cheri_getlen(addr), alloc)); + } +#endif break; case SLAB_COOKIE_MALLOC_LARGE: alloc = malloc_large_size(slab); +#ifdef __CHERI_PURE_CAPABILITY__ + if (size > olength) { + addr = malloc_large_grow_bounds(zone, addr); + KASSERT(cheri_getlen(addr) == size, + ("realloc large object bounds mismatch: %zx != %zx", + cheri_getlen(addr), size)); + } +#endif break; default: #ifdef INVARIANTS @@ -1129,7 +1200,19 @@ realloc(void *addr, size_t size, struct malloc_type *mtp, int flags) if (size <= alloc && (size > (alloc >> REALLOC_FRACTION) || alloc == MINALLOCSIZE)) { kasan_mark((void *)addr, size, alloc, KASAN_MALLOC_REDZONE); +#ifdef __CHERI_PURE_CAPABILITY__ + addr = cheri_setbounds(addr, size); + /* + * Zero only the non-representable portion of allocation + * that was not reachable from the original capability. + */ + if (size > olength) { + bzero((void *)((uintptr_t)addr + olength), + cheri_getlen(addr) - olength); + } +#else return (addr); +#endif } #endif /* !DEBUG_REDZONE */ @@ -1142,7 +1225,16 @@ realloc(void *addr, size_t size, struct malloc_type *mtp, int flags) * valid before performing the copy. */ kasan_mark(addr, alloc, alloc, 0); +#ifdef __CHERI_PURE_CAPABILITY__ + /* + * We have unbounded addr here, need to avoid copying + * past the original length. + * XXX-AM: is it worth it re-setting bounds on addr? + */ + bcopy(addr, newaddr, min(size, olength)); +#else bcopy(addr, newaddr, min(size, alloc)); +#endif free(addr, mtp); return (newaddr); } diff --git a/sys/vm/uma.h b/sys/vm/uma.h index 2fdc58065746..f7806d34826d 100644 --- a/sys/vm/uma.h +++ b/sys/vm/uma.h @@ -414,6 +414,31 @@ uma_zfree_pcpu(uma_zone_t zone, void *item) uma_zfree_pcpu_arg(zone, item, NULL); } +#ifdef __CHERI_PURE_CAPABILITY__ +/* + * Reset bounds on an item to the full item size. + * + * This is intended for nested allocators to recover items capabilities before + * free-ing them. + * Note that UMA maintains the invariant that a full item capability must be + * passed to uma_zfree functions. + * XXX-AM: Should we instead relax that invariant (optionally via a zone flag), + * or add a different set of uma_zfree_narrow functions that allow for partial + * item capabilities? + * Given that this is a privileged operation, I would like to maintain the + * intentionality of the operation here. + * + * Error conditions: + * - Clear the item capability tag if the item does not belong + * to the given zone. + * - Clear the item capability tag if the item does not belong + * to the slab from vtoslab. + */ +void *uma_zgrow_bounds(uma_zone_t zone, void *item); +#else +#define uma_zgrow_bounds(zone, item) (item) +#endif + /* * Wait until the specified zone can allocate an item. */ diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index 63497891070d..d06bfd0882af 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -5036,6 +5036,40 @@ zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) zone_free_limit(zone, 1); } +#ifdef __CHERI_PURE_CAPABILITY__ +/* See uma.h */ +void * +uma_zgrow_bounds(uma_zone_t zone, void *item) +{ + uma_keg_t keg = zone->uz_keg; + uma_slab_t slab; + int index; + + KASSERT((zone->uz_flags & UMA_ZFLAG_VTOSLAB) != 0, + ("Purecap kernel UMA zone missing UMA_ZFLAG_VTOSLAB")); + KASSERT((zone->uz_flags & (UMA_ZONE_PCPU | UMA_ZONE_SMR)) == 0, + ("Can not grow bounds on PCPU and SMR zones")); + + /* + * XXX-AM: It should be safe to only check the index range + * with INVARIANTS. If an item does not belong to the slab the computed + * index will be garbage but it will fail setbounds with the slab_data() + * capability. + */ + slab = vtoslab((vm_offset_t)item); + index = slab_item_index(slab, keg, item); + KASSERT(index >= 0 && index < keg->uk_ipers, + ("Invalid item index %d for slab %#p from item %#p for zone %s", + index, slab, item, zone->uz_name)); + item = cheri_setboundsexact(slab_item(slab, keg, index), + keg->uk_size); + KASSERT(cheri_gettag(item), + ("Failed to recover item %#p bounds for zone %s", + item, zone->uz_name)); + return (item); +} +#endif + /* See uma.h */ int uma_zone_set_max(uma_zone_t zone, int nitems) diff --git a/sys/vm/uma_int.h b/sys/vm/uma_int.h index c71a433e7d6f..46965f824373 100644 --- a/sys/vm/uma_int.h +++ b/sys/vm/uma_int.h @@ -403,6 +403,7 @@ struct uma_hash_slab { typedef struct uma_hash_slab * uma_hash_slab_t; +#ifdef _KERNEL static inline uma_hash_slab_t slab_tohashslab(uma_slab_t slab) { @@ -437,6 +438,7 @@ slab_item_index(uma_slab_t slab, uma_keg_t keg, void *item) data = (uintptr_t)slab_data(slab, keg); return (((ptraddr_t)item - (ptraddr_t)data) / keg->uk_rsize); } +#endif /* _KERNEL */ STAILQ_HEAD(uma_bucketlist, uma_bucket);