diff options
| author | riastradh <riastradh@NetBSD.org> | 2021-12-19 12:07:55 +0000 |
|---|---|---|
| committer | riastradh <riastradh@NetBSD.org> | 2021-12-19 12:07:55 +0000 |
| commit | ed314ebd4ef538a14a8987dc94bcd4ceab090d79 (patch) | |
| tree | f46ecce10cf3e20a845ba5edf5e9e3002b3564d0 /sys/external/bsd/common/include/linux | |
| parent | 3a9e48730c2a1846dfd4f94e584c7d17aabb2d22 (diff) | |
linux: Use kmem directly for Linux kmalloc.
Take advantage of this to do LOCKDEBUG_MEM_CHECK at the point of
kfree_rcu rather than in the RCU GC thread.
Diffstat (limited to 'sys/external/bsd/common/include/linux')
| -rw-r--r-- | sys/external/bsd/common/include/linux/slab.h | 65 |
1 files changed, 49 insertions, 16 deletions
diff --git a/sys/external/bsd/common/include/linux/slab.h b/sys/external/bsd/common/include/linux/slab.h index 6f1a95f2353..215e2b844ab 100644 --- a/sys/external/bsd/common/include/linux/slab.h +++ b/sys/external/bsd/common/include/linux/slab.h @@ -1,4 +1,4 @@ -/* $NetBSD: slab.h,v 1.7 2021/12/19 12:00:48 riastradh Exp $ */ +/* $NetBSD: slab.h,v 1.8 2021/12/19 12:07:55 riastradh Exp $ */ /*- * Copyright (c) 2013 The NetBSD Foundation, Inc. @@ -33,7 +33,6 @@ #define _LINUX_SLAB_H_ #include <sys/kmem.h> -#include <sys/malloc.h> #include <machine/limits.h> @@ -44,10 +43,12 @@ #define ARCH_KMALLOC_MINALIGN 4 /* XXX ??? */ -/* XXX Should use kmem, but Linux kfree doesn't take the size. */ +struct linux_malloc { + size_t lm_size; +} __aligned(ALIGNBYTES + 1); static inline int -linux_gfp_to_malloc(gfp_t gfp) +linux_gfp_to_kmem(gfp_t gfp) { int flags = 0; @@ -62,7 +63,6 @@ linux_gfp_to_malloc(gfp_t gfp) } if (ISSET(gfp, __GFP_ZERO)) { - flags |= M_ZERO; gfp &= ~__GFP_ZERO; } @@ -75,31 +75,43 @@ linux_gfp_to_malloc(gfp_t gfp) ((gfp & ~__GFP_WAIT) == (GFP_KERNEL & ~__GFP_WAIT))); if (ISSET(gfp, __GFP_WAIT)) { - flags |= M_WAITOK; + flags |= KM_SLEEP; gfp &= ~__GFP_WAIT; } else { - flags |= M_NOWAIT; + flags |= KM_NOSLEEP; } return flags; } /* - * XXX vmalloc and kmalloc both use malloc(9). If you change this, be - * sure to update vmalloc in <linux/vmalloc.h> and kvfree in - * <linux/mm.h>. + * XXX vmalloc and kmalloc both use this. If you change that, be sure + * to update vmalloc in <linux/vmalloc.h> and kvfree in <linux/mm.h>. */ static inline void * kmalloc(size_t size, gfp_t gfp) { - return malloc(size, M_TEMP, linux_gfp_to_malloc(gfp)); + struct linux_malloc *lm; + int kmflags = linux_gfp_to_kmem(gfp); + + KASSERTMSG(size < SIZE_MAX - sizeof(*lm), "size=%zu", size); + + if (gfp & __GFP_ZERO) + lm = kmem_intr_zalloc(sizeof(*lm) + size, kmflags); + else + lm = kmem_intr_alloc(sizeof(*lm) + size, kmflags); + if (lm == NULL) + return NULL; + + lm->lm_size = size; + return lm + 1; } static inline void * kzalloc(size_t size, gfp_t gfp) { - return malloc(size, M_TEMP, (linux_gfp_to_malloc(gfp) | M_ZERO)); + return kmalloc(size, gfp | __GFP_ZERO); } static inline void * @@ -107,7 +119,7 @@ kmalloc_array(size_t n, size_t size, gfp_t gfp) { if ((size != 0) && (n > (SIZE_MAX / size))) return NULL; - return malloc((n * size), M_TEMP, linux_gfp_to_malloc(gfp)); + return kmalloc(n * size, gfp); } static inline void * @@ -119,14 +131,35 @@ kcalloc(size_t n, size_t size, gfp_t gfp) static inline void * krealloc(void *ptr, size_t size, gfp_t gfp) { - return realloc(ptr, size, M_TEMP, linux_gfp_to_malloc(gfp)); + struct linux_malloc *olm, *nlm; + int kmflags = linux_gfp_to_kmem(gfp); + + if (gfp & __GFP_ZERO) + nlm = kmem_intr_zalloc(sizeof(*nlm) + size, kmflags); + else + nlm = kmem_intr_alloc(sizeof(*nlm) + size, kmflags); + if (nlm == NULL) + return NULL; + + nlm->lm_size = size; + if (ptr) { + olm = (struct linux_malloc *)ptr - 1; + memcpy(nlm + 1, olm + 1, MIN(nlm->lm_size, olm->lm_size)); + kmem_intr_free(olm, sizeof(*olm) + olm->lm_size); + } + return nlm + 1; } static inline void kfree(void *ptr) { - if (ptr != NULL) - free(ptr, M_TEMP); + struct linux_malloc *lm; + + if (ptr == NULL) + return; + + lm = (struct linux_malloc *)ptr - 1; + kmem_intr_free(lm, sizeof(*lm) + lm->lm_size); } #define SLAB_HWCACHE_ALIGN __BIT(0) |
