diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/Makefile | 3 | ||||
| -rw-r--r-- | mm/mempolicy.c | 8 | ||||
| -rw-r--r-- | mm/slab.c | 406 | ||||
| -rw-r--r-- | mm/slab.h | 33 | ||||
| -rw-r--r-- | mm/slab_common.c | 120 | ||||
| -rw-r--r-- | mm/slob.c | 152 | ||||
| -rw-r--r-- | mm/slub.c | 436 |
7 files changed, 559 insertions, 599 deletions
diff --git a/mm/Makefile b/mm/Makefile index 2e2fbbefb99f..8e81fe263c94 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -16,7 +16,8 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ readahead.o swap.o truncate.o vmscan.o shmem.o \ prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \ page_isolation.o mm_init.o mmu_context.o percpu.o \ - compaction.o $(mmu-y) + compaction.o slab_common.o $(mmu-y) + obj-y += init-mm.o ifdef CONFIG_NO_BOOTMEM diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 1d771e4200d2..bd92431d4c49 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -1602,8 +1602,14 @@ static unsigned interleave_nodes(struct mempolicy *policy) * task can change it's policy. The system default policy requires no * such protection. */ -unsigned slab_node(struct mempolicy *policy) +unsigned slab_node(void) { + struct mempolicy *policy; + + if (in_interrupt()) + return numa_node_id(); + + policy = current->mempolicy; if (!policy || policy->flags & MPOL_F_LOCAL) return numa_node_id(); diff --git a/mm/slab.c b/mm/slab.c index e901a36e2520..1fcf3ac94b6c 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -68,7 +68,7 @@ * Further notes from the original documentation: * * 11 April '97. Started multi-threading - markhe - * The global cache-chain is protected by the mutex 'cache_chain_mutex'. + * The global cache-chain is protected by the mutex 'slab_mutex'. * The sem is only needed when accessing/extending the cache-chain, which * can never happen inside an interrupt (kmem_cache_create(), * kmem_cache_shrink() and kmem_cache_reap()). @@ -87,6 +87,7 @@ */ #include <linux/slab.h> +#include "slab.h" #include <linux/mm.h> #include <linux/poison.h> #include <linux/swap.h> @@ -424,8 +425,8 @@ static void kmem_list3_init(struct kmem_list3 *parent) * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1: * redzone word. * cachep->obj_offset: The real object. - * cachep->buffer_size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long] - * cachep->buffer_size - 1* BYTES_PER_WORD: last caller address + * cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long] + * cachep->size - 1* BYTES_PER_WORD: last caller address * [BYTES_PER_WORD long] */ static int obj_offset(struct kmem_cache *cachep) @@ -433,11 +434,6 @@ static int obj_offset(struct kmem_cache *cachep) return cachep->obj_offset; } -static int obj_size(struct kmem_cache *cachep) -{ - return cachep->obj_size; -} - static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp) { BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); @@ -449,23 +445,22 @@ static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp) { BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); if (cachep->flags & SLAB_STORE_USER) - return (unsigned long long *)(objp + cachep->buffer_size - + return (unsigned long long *)(objp + cachep->size - sizeof(unsigned long long) - REDZONE_ALIGN); - return (unsigned long long *) (objp + cachep->buffer_size - + return (unsigned long long *) (objp + cachep->size - sizeof(unsigned long long)); } static void **dbg_userword(struct kmem_cache *cachep, void *objp) { BUG_ON(!(cachep->flags & SLAB_STORE_USER)); - return (void **)(objp + cachep->buffer_size - BYTES_PER_WORD); + return (void **)(objp + cachep->size - BYTES_PER_WORD); } #else #define obj_offset(x) 0 -#define obj_size(cachep) (cachep->buffer_size) #define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) #define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) #define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) @@ -475,7 +470,7 @@ static void **dbg_userword(struct kmem_cache *cachep, void *objp) #ifdef CONFIG_TRACING size_t slab_buffer_size(struct kmem_cache *cachep) { - return cachep->buffer_size; + return cachep->size; } EXPORT_SYMBOL(slab_buffer_size); #endif @@ -489,56 +484,37 @@ EXPORT_SYMBOL(slab_buffer_size); static int slab_max_order = SLAB_MAX_ORDER_LO; static bool slab_max_order_set __initdata; -/* - * Functions for storing/retrieving the cachep and or slab from the page - * allocator. These are used to find the slab an obj belongs to. With kfree(), - * these are used to find the cache which an obj belongs to. - */ -static inline void page_set_cache(struct page *page, struct kmem_cache *cache) -{ - page->lru.next = (struct list_head *)cache; -} - static inline struct kmem_cache *page_get_cache(struct page *page) { page = compound_head(page); BUG_ON(!PageSlab(page)); - return (struct kmem_cache *)page->lru.next; -} - -static inline void page_set_slab(struct page *page, struct slab *slab) -{ - page->lru.prev = (struct list_head *)slab; -} - -static inline struct slab *page_get_slab(struct page *page) -{ - BUG_ON(!PageSlab(page)); - return (struct slab *)page->lru.prev; + return page->slab_cache; } static inline struct kmem_cache *virt_to_cache(const void *obj) { struct page *page = virt_to_head_page(obj); - return page_get_cache(page); + return page->slab_cache; } static inline struct slab *virt_to_slab(const void *obj) { struct page *page = virt_to_head_page(obj); - return page_get_slab(page); + + VM_BUG_ON(!PageSlab(page)); + return page->slab_page; } static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab, unsigned int idx) { - return slab->s_mem + cache->buffer_size * idx; + return slab->s_mem + cache->size * idx; } /* - * We want to avoid an expensive divide : (offset / cache->buffer_size) - * Using the fact that buffer_size is a constant for a particular cache, - * we can replace (offset / cache->buffer_size) by + * We want to avoid an expensive divide : (offset / cache->size) + * Using the fact that size is a constant for a particular cache, + * we can replace (offset / cache->size) by * reciprocal_divide(offset, cache->reciprocal_buffer_size) */ static inline unsigned int obj_to_index(const struct kmem_cache *cache, @@ -584,33 +560,12 @@ static struct kmem_cache cache_cache = { .batchcount = 1, .limit = BOOT_CPUCACHE_ENTRIES, .shared = 1, - .buffer_size = sizeof(struct kmem_cache), + .size = sizeof(struct kmem_cache), .name = "kmem_cache", }; #define BAD_ALIEN_MAGIC 0x01020304ul -/* - * chicken and egg problem: delay the per-cpu array allocation - * until the general caches are up. - */ -static enum { - NONE, - PARTIAL_AC, - PARTIAL_L3, - EARLY, - LATE, - FULL -} g_cpucache_up; - -/* - * used by boot code to determine if it can use slab based allocator - */ -int slab_is_available(void) -{ - return g_cpucache_up >= EARLY; -} - #ifdef CONFIG_LOCKDEP /* @@ -676,7 +631,7 @@ static void init_node_lock_keys(int q) { struct cache_sizes *s = malloc_sizes; - if (g_cpucache_up < LATE) + if (slab_state < UP) return; for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) { @@ -716,12 +671,6 @@ static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) } #endif -/* - * Guard access to the cache-chain. - */ -static DEFINE_MUTEX(cache_chain_mutex); -static struct list_head cache_chain; - static DEFINE_PER_CPU(struct delayed_work, slab_reap_work); static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) @@ -1145,7 +1094,7 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) * When hotplugging memory or a cpu, existing nodelists are not replaced if * already in use. * - * Must hold cache_chain_mutex. + * Must hold slab_mutex. */ static int init_cache_nodelists_node(int node) { @@ -1153,7 +1102,7 @@ static int init_cache_nodelists_node(int node) struct kmem_list3 *l3; const int memsize = sizeof(struct kmem_list3); - list_for_each_entry(cachep, &cache_chain, next) { + list_for_each_entry(cachep, &slab_caches, list) { /* * Set up the size64 kmemlist for cpu before we can * begin anything. Make sure some other cpu on this @@ -1169,7 +1118,7 @@ static int init_cache_nodelists_node(int node) /* * The l3s don't come and go as CPUs come and - * go. cache_chain_mutex is sufficient + * go. slab_mutex is sufficient * protection here. */ cachep->nodelists[node] = l3; @@ -1191,7 +1140,7 @@ static void __cpuinit cpuup_canceled(long cpu) int node = cpu_to_mem(cpu); const struct cpumask *mask = cpumask_of_node(node); - list_for_each_entry(cachep, &cache_chain, next) { + list_for_each_entry(cachep, &slab_caches, list) { struct array_cache *nc; struct array_cache *shared; struct array_cache **alien; @@ -1241,7 +1190,7 @@ free_array_cache: * the respective cache's slabs, now we can go ahead and * shrink each nodelist to its limit. */ - list_for_each_entry(cachep, &cache_chain, next) { + list_for_each_entry(cachep, &slab_caches, list) { l3 = cachep->nodelists[node]; if (!l3) continue; @@ -1270,7 +1219,7 @@ static int __cpuinit cpuup_prepare(long cpu) * Now we can go ahead with allocating the shared arrays and * array caches */ - list_for_each_entry(cachep, &cache_chain, next) { + list_for_each_entry(cachep, &slab_caches, list) { struct array_cache *nc; struct array_cache *shared = NULL; struct array_cache **alien = NULL; @@ -1338,9 +1287,9 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb, switch (action) { case CPU_UP_PREPARE: case CPU_UP_PREPARE_FROZEN: - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); err = cpuup_prepare(cpu); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); break; case CPU_ONLINE: case CPU_ONLINE_FROZEN: @@ -1350,7 +1299,7 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb, case CPU_DOWN_PREPARE: case CPU_DOWN_PREPARE_FROZEN: /* - * Shutdown cache reaper. Note that the cache_chain_mutex is + * Shutdown cache reaper. Note that the slab_mutex is * held so that if cache_reap() is invoked it cannot do * anything expensive but will only modify reap_work * and reschedule the timer. @@ -1377,9 +1326,9 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb, #endif case CPU_UP_CANCELED: case CPU_UP_CANCELED_FROZEN: - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); cpuup_canceled(cpu); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); break; } return notifier_from_errno(err); @@ -1395,14 +1344,14 @@ static struct notifier_block __cpuinitdata cpucache_notifier = { * Returns -EBUSY if all objects cannot be drained so that the node is not * removed. * - * Must hold cache_chain_mutex. + * Must hold slab_mutex. */ static int __meminit drain_cache_nodelists_node(int node) { struct kmem_cache *cachep; int ret = 0; - list_for_each_entry(cachep, &cache_chain, next) { + list_for_each_entry(cachep, &slab_caches, list) { struct kmem_list3 *l3; l3 = cachep->nodelists[node]; @@ -1433,14 +1382,14 @@ static int __meminit slab_memory_callback(struct notifier_block *self, switch (action) { case MEM_GOING_ONLINE: - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); ret = init_cache_nodelists_node(nid); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); break; case MEM_GOING_OFFLINE: - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); ret = drain_cache_nodelists_node(nid); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); break; case MEM_ONLINE: case MEM_OFFLINE: @@ -1544,8 +1493,8 @@ void __init kmem_cache_init(void) node = numa_mem_id(); /* 1) create the cache_cache */ - INIT_LIST_HEAD(&cache_chain); - list_add(&cache_cache.next, &cache_chain); + INIT_LIST_HEAD(&slab_caches); + list_add(&cache_cache.list, &slab_caches); cache_cache.colour_off = cache_line_size(); cache_cache.array[smp_processor_id()] = &initarray_cache.cache; cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node]; @@ -1553,18 +1502,16 @@ void __init kmem_cache_init(void) /* * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids */ - cache_cache.buffer_size = offsetof(struct kmem_cache, array[nr_cpu_ids]) + + cache_cache.size = offsetof(struct kmem_cache, array[nr_cpu_ids]) + nr_node_ids * sizeof(struct kmem_list3 *); -#if DEBUG - cache_cache.obj_size = cache_cache.buffer_size; -#endif - cache_cache.buffer_size = ALIGN(cache_cache.buffer_size, + cache_cache.object_size = cache_cache.size; + cache_cache.size = ALIGN(cache_cache.size, cache_line_size()); cache_cache.reciprocal_buffer_size = - reciprocal_value(cache_cache.buffer_size); + reciprocal_value(cache_cache.size); for (order = 0; order < MAX_ORDER; order++) { - cache_estimate(order, cache_cache.buffer_size, + cache_estimate(order, cache_cache.size, cache_line_size(), 0, &left_over, &cache_cache.num); if (cache_cache.num) break; @@ -1585,7 +1532,7 @@ void __init kmem_cache_init(void) * bug. */ - sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name, + sizes[INDEX_AC].cs_cachep = __kmem_cache_create(names[INDEX_AC].name, sizes[INDEX_AC].cs_size, ARCH_KMALLOC_MINALIGN, ARCH_KMALLOC_FLAGS|SLAB_PANIC, @@ -1593,7 +1540,7 @@ void __init kmem_cache_init(void) if (INDEX_AC != INDEX_L3) { sizes[INDEX_L3].cs_cachep = - kmem_cache_create(names[INDEX_L3].name, + __kmem_cache_create(names[INDEX_L3].name, sizes[INDEX_L3].cs_size, ARCH_KMALLOC_MINALIGN, ARCH_KMALLOC_FLAGS|SLAB_PANIC, @@ -1611,14 +1558,14 @@ void __init kmem_cache_init(void) * allow tighter packing of the smaller caches. */ if (!sizes->cs_cachep) { - sizes->cs_cachep = kmem_cache_create(names->name, + sizes->cs_cachep = __kmem_cache_create(names->name, sizes->cs_size, ARCH_KMALLOC_MINALIGN, ARCH_KMALLOC_FLAGS|SLAB_PANIC, NULL); } #ifdef CONFIG_ZONE_DMA - sizes->cs_dmacachep = kmem_cache_create( + sizes->cs_dmacachep = __kmem_cache_create( names->name_dma, sizes->cs_size, ARCH_KMALLOC_MINALIGN, @@ -1676,27 +1623,27 @@ void __init kmem_cache_init(void) } } - g_cpucache_up = EARLY; + slab_state = UP; } void __init kmem_cache_init_late(void) { struct kmem_cache *cachep; - g_cpucache_up = LATE; + slab_state = UP; /* Annotate slab for lockdep -- annotate the malloc caches */ init_lock_keys(); /* 6) resize the head arrays to their final sizes */ - mutex_lock(&cache_chain_mutex); - list_for_each_entry(cachep, &cache_chain, next) + mutex_lock(&slab_mutex); + list_for_each_entry(cachep, &slab_caches, list) if (enable_cpucache(cachep, GFP_NOWAIT)) BUG(); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); /* Done! */ - g_cpucache_up = FULL; + slab_state = FULL; /* * Register a cpu startup notifier callback that initializes @@ -1727,6 +1674,9 @@ static int __init cpucache_init(void) */ for_each_online_cpu(cpu) start_cpu_timer(cpu); + + /* Done! */ + slab_state = FULL; return 0; } __initcall(cpucache_init); @@ -1743,7 +1693,7 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) "SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n", nodeid, gfpflags); printk(KERN_WARNING " cache: %s, object size: %d, order: %d\n", - cachep->name, cachep->buffer_size, cachep->gfporder); + cachep->name, cachep->size, cachep->gfporder); for_each_online_node(node) { unsigned long active_objs = 0, num_objs = 0, free_objects = 0; @@ -1798,7 +1748,7 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid) flags |= __GFP_COMP; #endif - flags |= cachep->gfpflags; + flags |= cachep->allocflags; if (cachep->flags & SLAB_RECLAIM_ACCOUNT) flags |= __GFP_RECLAIMABLE; @@ -1874,7 +1824,7 @@ static void kmem_rcu_free(struct rcu_head *head) static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr, unsigned long caller) { - int size = obj_size(cachep); + int size = cachep->object_size; addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)]; @@ -1906,7 +1856,7 @@ static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr, static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val) { - int size = obj_size(cachep); + int size = cachep->object_size; addr = &((char *)addr)[obj_offset(cachep)]; memset(addr, val, size); @@ -1966,7 +1916,7 @@ static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines) printk("\n"); } realobj = (char *)objp + obj_offset(cachep); - size = obj_size(cachep); + size = cachep->object_size; for (i = 0; i < size && lines; i += 16, lines--) { int limit; limit = 16; @@ -1983,7 +1933,7 @@ static void check_poison_obj(struct kmem_cache *cachep, void *objp) int lines = 0; realobj = (char *)objp + obj_offset(cachep); - size = obj_size(cachep); + size = cachep->object_size; for (i = 0; i < size; i++) { char exp = POISON_FREE; @@ -2047,10 +1997,10 @@ static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slab if (cachep->flags & SLAB_POISON) { #ifdef CONFIG_DEBUG_PAGEALLOC - if (cachep->buffer_size % PAGE_SIZE == 0 && + if (cachep->size % PAGE_SIZE == 0 && OFF_SLAB(cachep)) kernel_map_pages(virt_to_page(objp), - cachep->buffer_size / PAGE_SIZE, 1); + cachep->size / PAGE_SIZE, 1); else check_poison_obj(cachep, objp); #else @@ -2194,10 +2144,10 @@ static size_t calculate_slab_order(struct kmem_cache *cachep, static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) { - if (g_cpucache_up == FULL) + if (slab_state >= FULL) return enable_cpucache(cachep, gfp); - if (g_cpucache_up == NONE) { + if (slab_state == DOWN) { /* * Note: the first kmem_cache_create must create the cache * that's used by kmalloc(24), otherwise the creation of @@ -2212,16 +2162,16 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) */ set_up_list3s(cachep, SIZE_AC); if (INDEX_AC == INDEX_L3) - g_cpucache_up = PARTIAL_L3; + slab_state = PARTIAL_L3; else - g_cpucache_up = PARTIAL_AC; + slab_state = PARTIAL_ARRAYCACHE; } else { cachep->array[smp_processor_id()] = kmalloc(sizeof(struct arraycache_init), gfp); - if (g_cpucache_up == PARTIAL_AC) { + if (slab_state == PARTIAL_ARRAYCACHE) { set_up_list3s(cachep, SIZE_L3); - g_cpucache_up = PARTIAL_L3; + slab_state = PARTIAL_L3; } else { int node; for_each_online_node(node) { @@ -2247,7 +2197,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) } /** - * kmem_cache_create - Create a cache. + * __kmem_cache_create - Create a cache. * @name: A string which is used in /proc/slabinfo to identify this cache. * @size: The size of objects to be created in this cache. * @align: The required alignment for the objects. @@ -2274,59 +2224,14 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) * as davem. */ struct kmem_cache * -kmem_cache_create (const char *name, size_t size, size_t align, +__kmem_cache_create (const char *name, size_t size, size_t align, unsigned long flags, void (*ctor)(void *)) { size_t left_over, slab_size, ralign; - struct kmem_cache *cachep = NULL, *pc; + struct kmem_cache *cachep = NULL; gfp_t gfp; - /* - * Sanity checks... these are all serious usage bugs. - */ - if (!name || in_interrupt() || (size < BYTES_PER_WORD) || - size > KMALLOC_MAX_SIZE) { - printk(KERN_ERR "%s: Early error in slab %s\n", __func__, - name); - BUG(); - } - - /* - * We use cache_chain_mutex to ensure a consistent view of - * cpu_online_mask as well. Please see cpuup_callback - */ - if (slab_is_available()) { - get_online_cpus(); - mutex_lock(&cache_chain_mutex); - } - - list_for_each_entry(pc, &cache_chain, next) { - char tmp; - int res; - - /* - * This happens when the module gets unloaded and doesn't - * destroy its slab cache and no-one else reuses the vmalloc - * area of the module. Print a warning. - */ - res = probe_kernel_address(pc->name, tmp); - if (res) { - printk(KERN_ERR - "SLAB: cache with size %d has lost its name\n", - pc->buffer_size); - continue; - } - - if (!strcmp(pc->name, name)) { - printk(KERN_ERR - "kmem_cache_create: duplicate cache %s\n", name); - dump_stack(); - goto oops; - } - } - #if DEBUG - WARN_ON(strchr(name, ' ')); /* It confuses parsers */ #if FORCED_DEBUG /* * Enable redzoning and last user accounting, except for caches with @@ -2415,11 +2320,12 @@ kmem_cache_create (const char *name, size_t size, size_t align, /* Get cache's description obj. */ cachep = kmem_cache_zalloc(&cache_cache, gfp); if (!cachep) - goto oops; + return NULL; cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids]; + cachep->object_size = size; + cachep->align = align; #if DEBUG - cachep->obj_size = size; /* * Both debugging options require word-alignment which is calculated @@ -2442,7 +2348,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, } #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) if (size >= malloc_sizes[INDEX_L3 + 1].cs_size - && cachep->obj_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) { + && cachep->object_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) { cachep->obj_offset += PAGE_SIZE - ALIGN(size, align); size = PAGE_SIZE; } @@ -2471,8 +2377,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, printk(KERN_ERR "kmem_cache_create: couldn't create cache %s.\n", name); kmem_cache_free(&cache_cache, cachep); - cachep = NULL; - goto oops; + return NULL; } slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab), align); @@ -2508,10 +2413,10 @@ kmem_cache_create (const char *name, size_t size, size_t align, cachep->colour = left_over / cachep->colour_off; cachep->slab_size = slab_size; cachep->flags = flags; - cachep->gfpflags = 0; + cachep->allocflags = 0; if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA)) - cachep->gfpflags |= GFP_DMA; - cachep->buffer_size = size; + cachep->allocflags |= GFP_DMA; + cachep->size = size; cachep->reciprocal_buffer_size = reciprocal_value(size); if (flags & CFLGS_OFF_SLAB) { @@ -2530,8 +2435,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, if (setup_cpu_cache(cachep, gfp)) { __kmem_cache_destroy(cachep); - cachep = NULL; - goto oops; + return NULL; } if (flags & SLAB_DEBUG_OBJECTS) { @@ -2545,18 +2449,9 @@ kmem_cache_create (const char *name, size_t size, size_t align, } /* cache setup completed, link it into the list */ - list_add(&cachep->next, &cache_chain); -oops: - if (!cachep && (flags & SLAB_PANIC)) - panic("kmem_cache_create(): failed to create slab `%s'\n", - name); - if (slab_is_available()) { - mutex_unlock(&cache_chain_mutex); - put_online_cpus(); - } + list_add(&cachep->list, &slab_caches); return cachep; } -EXPORT_SYMBOL(kmem_cache_create); #if DEBUG static void check_irq_off(void) @@ -2671,7 +2566,7 @@ out: return nr_freed; } -/* Called with cache_chain_mutex held to protect against cpu hotplug */ +/* Called with slab_mutex held to protect against cpu hotplug */ static int __cache_shrink(struct kmem_cache *cachep) { int ret = 0, i = 0; @@ -2706,9 +2601,9 @@ int kmem_cache_shrink(struct kmem_cache *cachep) BUG_ON(!cachep || in_interrupt()); get_online_cpus(); - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); ret = __cache_shrink(cachep); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); put_online_cpus(); return ret; } @@ -2736,15 +2631,15 @@ void kmem_cache_destroy(struct kmem_cache *cachep) /* Find the cache in the chain of caches. */ get_online_cpus(); - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); /* * the chain is never empty, cache_cache is never destroyed */ - list_del(&cachep->next); + list_del(&cachep->list); if (__cache_shrink(cachep)) { slab_error(cachep, "Can't free all objects"); - list_add(&cachep->next, &cache_chain); - mutex_unlock(&cache_chain_mutex); + list_add(&cachep->list, &slab_caches); + mutex_unlock(&slab_mutex); put_online_cpus(); return; } @@ -2753,7 +2648,7 @@ void kmem_cache_destroy(struct kmem_cache *cachep) rcu_barrier(); __kmem_cache_destroy(cachep); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); put_online_cpus(); } EXPORT_SYMBOL(kmem_cache_destroy); @@ -2840,10 +2735,10 @@ static void cache_init_objs(struct kmem_cache *cachep, slab_error(cachep, "constructor overwrote the" " start of an object"); } - if ((cachep->buffer_size % PAGE_SIZE) == 0 && + if ((cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep) && cachep->flags & SLAB_POISON) kernel_map_pages(virt_to_page(objp), - cachep->buffer_size / PAGE_SIZE, 0); + cachep->size / PAGE_SIZE, 0); #else if (cachep->ctor) cachep->ctor(objp); @@ -2857,9 +2752,9 @@ static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags) { if (CONFIG_ZONE_DMA_FLAG) { if (flags & GFP_DMA) - BUG_ON(!(cachep->gfpflags & GFP_DMA)); + BUG_ON(!(cachep->allocflags & GFP_DMA)); else - BUG_ON(cachep->gfpflags & GFP_DMA); + BUG_ON(cachep->allocflags & GFP_DMA); } } @@ -2918,8 +2813,8 @@ static void slab_map_pages(struct kmem_cache *cache, struct slab *slab, nr_pages <<= cache->gfporder; do { - page_set_cache(page, cache); - page_set_slab(page, slab); + page->slab_cache = cache; + page->slab_page = slab; page++; } while (--nr_pages); } @@ -3057,7 +2952,7 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, kfree_debugcheck(objp); page = virt_to_head_page(objp); - slabp = page_get_slab(page); + slabp = page->slab_page; if (cachep->flags & SLAB_RED_ZONE) { verify_redzone_free(cachep, objp); @@ -3077,10 +2972,10 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, #endif if (cachep->flags & SLAB_POISON) { #ifdef CONFIG_DEBUG_PAGEALLOC - if ((cachep->buffer_size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) { + if ((cachep->size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) { store_stackinfo(cachep, objp, (unsigned long)caller); kernel_map_pages(virt_to_page(objp), - cachep->buffer_size / PAGE_SIZE, 0); + cachep->size / PAGE_SIZE, 0); } else { poison_obj(cachep, objp, POISON_FREE); } @@ -3230,9 +3125,9 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, return objp; if (cachep->flags & SLAB_POISON) { #ifdef CONFIG_DEBUG_PAGEALLOC - if ((cachep->buffer_size % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) + if ((cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) kernel_map_pages(virt_to_page(objp), - cachep->buffer_size / PAGE_SIZE, 1); + cachep->size / PAGE_SIZE, 1); else check_poison_obj(cachep, objp); #else @@ -3261,8 +3156,8 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, struct slab *slabp; unsigned objnr; - slabp = page_get_slab(virt_to_head_page(objp)); - objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size; + slabp = virt_to_head_page(objp)->slab_page; + objnr = (unsigned)(objp - slabp->s_mem) / cachep->size; slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE; } #endif @@ -3285,7 +3180,7 @@ static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags) if (cachep == &cache_cache) return false; - return should_failslab(obj_size(cachep), flags, cachep->flags); + return should_failslab(cachep->object_size, flags, cachep->flags); } static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) @@ -3336,7 +3231,7 @@ static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags) if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD)) nid_alloc = cpuset_slab_spread_node(); else if (current->mempolicy) - nid_alloc = slab_node(current->mempolicy); + nid_alloc = slab_node(); if (nid_alloc != nid_here) return ____cache_alloc_node(cachep, flags, nid_alloc); return NULL; @@ -3368,7 +3263,7 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags) retry_cpuset: cpuset_mems_cookie = get_mems_allowed(); - zonelist = node_zonelist(slab_node(current->mempolicy), flags); + zonelist = node_zonelist(slab_node(), flags); retry: /* @@ -3545,14 +3440,14 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, out: local_irq_restore(save_flags); ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller); - kmemleak_alloc_recursive(ptr, obj_size(cachep), 1, cachep->flags, + kmemleak_alloc_recursive(ptr, cachep->object_size, 1, cachep->flags, flags); if (likely(ptr)) - kmemcheck_slab_alloc(cachep, flags, ptr, obj_size(cachep)); + kmemcheck_slab_alloc(cachep, flags, ptr, cachep->object_size); if (unlikely((flags & __GFP_ZERO) && ptr)) - memset(ptr, 0, obj_size(cachep)); + memset(ptr, 0, cachep->object_size); return ptr; } @@ -3607,15 +3502,15 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller) objp = __do_cache_alloc(cachep, flags); local_irq_restore(save_flags); objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller); - kmemleak_alloc_recursive(objp, obj_size(cachep), 1, cachep->flags, + kmemleak_alloc_recursive(objp, cachep->object_size, 1, cachep->flags, flags); prefetchw(objp); if (likely(objp)) - kmemcheck_slab_alloc(cachep, flags, objp, obj_size(cachep)); + kmemcheck_slab_alloc(cachep, flags, objp, cachep->object_size); if (unlikely((flags & __GFP_ZERO) && objp)) - memset(objp, 0, obj_size(cachep)); + memset(objp, 0, cachep->object_size); return objp; } @@ -3731,7 +3626,7 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp, kmemleak_free_recursive(objp, cachep->flags); objp = cache_free_debugcheck(cachep, objp, caller); - kmemcheck_slab_free(cachep, objp, obj_size(cachep)); + kmemcheck_slab_free(cachep, objp, cachep->object_size); /* * Skip calling cache_free_alien() when the platform is not numa. @@ -3766,7 +3661,7 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0)); trace_kmem_cache_alloc(_RET_IP_, ret, - obj_size(cachep), cachep->buffer_size, flags); + cachep->object_size, cachep->size, flags); return ret; } @@ -3794,7 +3689,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) __builtin_return_address(0)); trace_kmem_cache_alloc_node(_RET_IP_, ret, - obj_size(cachep), cachep->buffer_size, + cachep->object_size, cachep->size, flags, nodeid); return ret; @@ -3876,7 +3771,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, ret = __cache_alloc(cachep, flags, caller); trace_kmalloc((unsigned long) caller, ret, - size, cachep->buffer_size, flags); + size, cachep->size, flags); return ret; } @@ -3916,9 +3811,9 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) unsigned long flags; local_irq_save(flags); - debug_check_no_locks_freed(objp, obj_size(cachep)); + debug_check_no_locks_freed(objp, cachep->object_size); if (!(cachep->flags & SLAB_DEBUG_OBJECTS)) - debug_check_no_obj_freed(objp, obj_size(cachep)); + debug_check_no_obj_freed(objp, cachep->object_size); __cache_free(cachep, objp, __builtin_return_address(0)); local_irq_restore(flags); @@ -3947,8 +3842,9 @@ void kfree(const void *objp) local_irq_save(flags); kfree_debugcheck(objp); c = virt_to_cache(objp); - debug_check_no_locks_freed(objp, obj_size(c)); - debug_check_no_obj_freed(objp, obj_size(c)); + debug_check_no_locks_freed(objp, c->object_size); + + debug_check_no_obj_freed(objp, c->object_size); __cache_free(c, (void *)objp, __builtin_return_address(0)); local_irq_restore(flags); } @@ -3956,7 +3852,7 @@ EXPORT_SYMBOL(kfree); unsigned int kmem_cache_size(struct kmem_cache *cachep) { - return obj_size(cachep); + return cachep->object_size; } EXPORT_SYMBOL(kmem_cache_size); @@ -4030,7 +3926,7 @@ static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp) return 0; fail: - if (!cachep->next.next) { + if (!cachep->list.next) { /* Cache is not active yet. Roll back what we did */ node--; while (node >= 0) { @@ -4065,7 +3961,7 @@ static void do_ccupdate_local(void *info) new->new[smp_processor_id()] = old; } -/* Always called with the cache_chain_mutex held */ +/* Always called with the slab_mutex held */ static int do_tune_cpucache(struct kmem_cache *cachep, int limit, int batchcount, int shared, gfp_t gfp) { @@ -4109,7 +4005,7 @@ static int do_tune_cpucache(struct kmem_cache *cachep, int limit, return alloc_kmemlist(cachep, gfp); } -/* Called with cache_chain_mutex held always */ +/* Called with slab_mutex held always */ static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) { int err; @@ -4124,13 +4020,13 @@ static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) * The numbers are guessed, we should auto-tune as described by * Bonwick. */ - if (cachep->buffer_size > 131072) + if (cachep->size > 131072) limit = 1; - else if (cachep->buffer_size > PAGE_SIZE) + else if (cachep->size > PAGE_SIZE) limit = 8; - else if (cachep->buffer_size > 1024) + else if (cachep->size > 1024) limit = 24; - else if (cachep->buffer_size > 256) + else if (cachep->size > 256) limit = 54; else limit = 120; @@ -4145,7 +4041,7 @@ static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) * to a larger limit. Thus disabled by default. */ shared = 0; - if (cachep->buffer_size <= PAGE_SIZE && num_possible_cpus() > 1) + if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1) shared = 8; #if DEBUG @@ -4211,11 +4107,11 @@ static void cache_reap(struct work_struct *w) int node = numa_mem_id(); struct delayed_work *work = to_delayed_work(w); - if (!mutex_trylock(&cache_chain_mutex)) + if (!mutex_trylock(&slab_mutex)) /* Give up. Setup the next iteration. */ goto out; - list_for_each_entry(searchp, &cache_chain, next) { + list_for_each_entry(searchp, &slab_caches, list) { check_irq_on(); /* @@ -4253,7 +4149,7 @@ next: cond_resched(); } check_irq_on(); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); next_reap_node(); out: /* Set up the next iteration */ @@ -4289,26 +4185,26 @@ static void *s_start(struct seq_file *m, loff_t *pos) { loff_t n = *pos; - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); if (!n) print_slabinfo_header(m); - return seq_list_start(&cache_chain, *pos); + return seq_list_start(&slab_caches, *pos); } static void *s_next(struct seq_file *m, void *p, loff_t *pos) { - return seq_list_next(p, &cache_chain, pos); + return seq_list_next(p, &slab_caches, pos); } static void s_stop(struct seq_file *m, void *p) { - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); } static int s_show(struct seq_file *m, void *p) { - struct kmem_cache *cachep = list_entry(p, struct kmem_cache, next); + struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list); struct slab *slabp; unsigned long active_objs; unsigned long num_objs; @@ -4364,7 +4260,7 @@ static int s_show(struct seq_file *m, void *p) printk(KERN_ERR "slab: cache %s error: %s\n", name, error); seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", - name, active_objs, num_objs, cachep->buffer_size, + name, active_objs, num_objs, cachep->size, cachep->num, (1 << cachep->gfporder)); seq_printf(m, " : tunables %4u %4u %4u", cachep->limit, cachep->batchcount, cachep->shared); @@ -4454,9 +4350,9 @@ static ssize_t slabinfo_write(struct file *file, const char __user *buffer, return -EINVAL; /* Find the cache in the chain of caches. */ - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); res = -EINVAL; - list_for_each_entry(cachep, &cache_chain, next) { + list_for_each_entry(cachep, &slab_caches, list) { if (!strcmp(cachep->name, kbuf)) { if (limit < 1 || batchcount < 1 || batchcount > limit || shared < 0) { @@ -4469,7 +4365,7 @@ static ssize_t slabinfo_write(struct file *file, const char __user *buffer, break; } } - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); if (res >= 0) res = count; return res; @@ -4492,8 +4388,8 @@ static const struct file_operations proc_slabinfo_operations = { static void *leaks_start(struct seq_file *m, loff_t *pos) { - mutex_lock(&cache_chain_mutex); - return seq_list_start(&cache_chain, *pos); + mutex_lock(&slab_mutex); + return seq_list_start(&slab_caches, *pos); } static inline int add_caller(unsigned long *n, unsigned long v) @@ -4532,7 +4428,7 @@ static void handle_slab(unsigned long *n, struct kmem_cache *c, struct slab *s) int i; if (n[0] == n[1]) return; - for (i = 0, p = s->s_mem; i < c->num; i++, p += c->buffer_size) { + for (i = 0, p = s->s_mem; i < c->num; i++, p += c->size) { if (slab_bufctl(s)[i] != BUFCTL_ACTIVE) continue; if (!add_caller(n, (unsigned long)*dbg_userword(c, p))) @@ -4558,7 +4454,7 @@ static void show_symbol(struct seq_file *m, unsigned long address) static int leaks_show(struct seq_file *m, void *p) { - struct kmem_cache *cachep = list_entry(p, struct kmem_cache, next); + struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list); struct slab *slabp; struct kmem_list3 *l3; const char *name; @@ -4592,17 +4488,17 @@ static int leaks_show(struct seq_file *m, void *p) name = cachep->name; if (n[0] == n[1]) { /* Increase the buffer size */ - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); m->private = kzalloc(n[0] * 4 * sizeof(unsigned long), GFP_KERNEL); if (!m->private) { /* Too bad, we are really out */ m->private = n; - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); return -ENOMEM; } *(unsigned long *)m->private = n[0] * 2; kfree(n); - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); /* Now make sure this entry will be retried */ m->count = m->size; return 0; @@ -4677,6 +4573,6 @@ size_t ksize(const void *objp) if (unlikely(objp == ZERO_SIZE_PTR)) return 0; - return obj_size(virt_to_cache(objp)); + return virt_to_cache(objp)->object_size; } EXPORT_SYMBOL(ksize); diff --git a/mm/slab.h b/mm/slab.h new file mode 100644 index 000000000000..db7848caaa25 --- /dev/null +++ b/mm/slab.h @@ -0,0 +1,33 @@ +#ifndef MM_SLAB_H +#define MM_SLAB_H +/* + * Internal slab definitions + */ + +/* + * State of the slab allocator. + * + * This is used to describe the states of the allocator during bootup. + * Allocators use this to gradually bootstrap themselves. Most allocators + * have the problem that the structures used for managing slab caches are + * allocated from slab caches themselves. + */ +enum slab_state { + DOWN, /* No slab functionality yet */ + PARTIAL, /* SLUB: kmem_cache_node available */ + PARTIAL_ARRAYCACHE, /* SLAB: kmalloc size for arraycache available */ + PARTIAL_L3, /* SLAB: kmalloc size for l3 struct available */ + UP, /* Slab caches usable but not all extras yet */ + FULL /* Everything is working */ +}; + +extern enum slab_state slab_state; + +/* The slab cache mutex protects the management structures during changes */ +extern struct mutex slab_mutex; +extern struct list_head slab_caches; + +struct kmem_cache *__kmem_cache_create(const char *name, size_t size, + size_t align, unsigned long flags, void (*ctor)(void *)); + +#endif diff --git a/mm/slab_common.c b/mm/slab_common.c new file mode 100644 index 000000000000..aa3ca5bb01b5 --- /dev/null +++ b/mm/slab_common.c @@ -0,0 +1,120 @@ +/* + * Slab allocator functions that are independent of the allocator strategy + * + * (C) 2012 Christoph Lameter <cl@linux.com> + */ +#include <linux/slab.h> + +#include <linux/mm.h> +#include <linux/poison.h> +#include <linux/interrupt.h> +#include <linux/memory.h> +#include <linux/compiler.h> +#include <linux/module.h> +#include <linux/cpu.h> +#include <linux/uaccess.h> +#include <asm/cacheflush.h> +#include <asm/tlbflush.h> +#include <asm/page.h> + +#include "slab.h" + +enum slab_state slab_state; +LIST_HEAD(slab_caches); +DEFINE_MUTEX(slab_mutex); + +/* + * kmem_cache_create - Create a cache. + * @name: A string which is used in /proc/slabinfo to identify this cache. + * @size: The size of objects to be created in this cache. + * @align: The required alignment for the objects. + * @flags: SLAB flags + * @ctor: A constructor for the objects. + * + * Returns a ptr to the cache on success, NULL on failure. + * Cannot be called within a interrupt, but can be interrupted. + * The @ctor is run when new pages are allocated by the cache. + * + * The flags are + * + * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) + * to catch references to uninitialised memory. + * + * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check + * for buffer overruns. + * + * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware + * cacheline. This can be beneficial if you're counting cycles as closely + * as davem. + */ + +struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align, + unsigned long flags, void (*ctor)(void *)) +{ + struct kmem_cache *s = NULL; + +#ifdef CONFIG_DEBUG_VM + if (!name || in_interrupt() || size < sizeof(void *) || + size > KMALLOC_MAX_SIZE) { + printk(KERN_ERR "kmem_cache_create(%s) integrity check" + " failed\n", name); + goto out; + } +#endif + + get_online_cpus(); + mutex_lock(&slab_mutex); + +#ifdef CONFIG_DEBUG_VM + list_for_each_entry(s, &slab_caches, list) { + char tmp; + int res; + + /* + * This happens when the module gets unloaded and doesn't + * destroy its slab cache and no-one else reuses the vmalloc + * area of the module. Print a warning. + */ + res = probe_kernel_address(s->name, tmp); + if (res) { + printk(KERN_ERR + "Slab cache with size %d has lost its name\n", + s->object_size); + continue; + } + + if (!strcmp(s->name, name)) { + printk(KERN_ERR "kmem_cache_create(%s): Cache name" + " already exists.\n", + name); + dump_stack(); + s = NULL; + goto oops; + } + } + + WARN_ON(strchr(name, ' ')); /* It confuses parsers */ +#endif + + s = __kmem_cache_create(name, size, align, flags, ctor); + +#ifdef CONFIG_DEBUG_VM +oops: +#endif + mutex_unlock(&slab_mutex); + put_online_cpus(); + +#ifdef CONFIG_DEBUG_VM +out: +#endif + if (!s && (flags & SLAB_PANIC)) + panic("kmem_cache_create: Failed to create slab '%s'\n", name); + + return s; +} +EXPORT_SYMBOL(kmem_cache_create); + +int slab_is_available(void) +{ + return slab_state >= UP; +} diff --git a/mm/slob.c b/mm/slob.c index 8105be42cad1..45d4ca79933a 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -59,6 +59,8 @@ #include <linux/kernel.h> #include <linux/slab.h> +#include "slab.h" + #include <linux/mm.h> #include <linux/swap.h> /* struct reclaim_state */ #include <linux/cache.h> @@ -92,36 +94,6 @@ struct slob_block { typedef struct slob_block slob_t; /* - * We use struct page fields to manage some slob allocation aspects, - * however to avoid the horrible mess in include/linux/mm_types.h, we'll - * just define our own struct page type variant here. - */ -struct slob_page { - union { - struct { - unsigned long flags; /* mandatory */ - atomic_t _count; /* mandatory */ - slobidx_t units; /* free units left in page */ - unsigned long pad[2]; - slob_t *free; /* first free slob_t in page */ - struct list_head list; /* linked list of free pages */ - }; - struct page page; - }; -}; -static inline void struct_slob_page_wrong_size(void) -{ BUILD_BUG_ON(sizeof(struct slob_page) != sizeof(struct page)); } - -/* - * free_slob_page: call before a slob_page is returned to the page allocator. - */ -static inline void free_slob_page(struct slob_page *sp) -{ - reset_page_mapcount(&sp->page); - sp->page.mapping = NULL; -} - -/* * All partially free slob pages go on these lists. */ #define SLOB_BREAK1 256 @@ -131,46 +103,23 @@ static LIST_HEAD(free_slob_medium); static LIST_HEAD(free_slob_large); /* - * is_slob_page: True for all slob pages (false for bigblock pages) - */ -static inline int is_slob_page(struct slob_page *sp) -{ - return PageSlab((struct page *)sp); -} - -static inline void set_slob_page(struct slob_page *sp) -{ - __SetPageSlab((struct page *)sp); -} - -static inline void clear_slob_page(struct slob_page *sp) -{ - __ClearPageSlab((struct page *)sp); -} - -static inline struct slob_page *slob_page(const void *addr) -{ - return (struct slob_page *)virt_to_page(addr); -} - -/* * slob_page_free: true for pages on free_slob_pages list. */ -static inline int slob_page_free(struct slob_page *sp) +static inline int slob_page_free(struct page *sp) { - return PageSlobFree((struct page *)sp); + return PageSlobFree(sp); } -static void set_slob_page_free(struct slob_page *sp, struct list_head *list) +static void set_slob_page_free(struct page *sp, struct list_head *list) { list_add(&sp->list, list); - __SetPageSlobFree((struct page *)sp); + __SetPageSlobFree(sp); } -static inline void clear_slob_page_free(struct slob_page *sp) +static inline void clear_slob_page_free(struct page *sp) { list_del(&sp->list); - __ClearPageSlobFree((struct page *)sp); + __ClearPageSlobFree(sp); } #define SLOB_UNIT sizeof(slob_t) @@ -267,12 +216,12 @@ static void slob_free_pages(void *b, int order) /* * Allocate a slob block within a given slob_page sp. */ -static void *slob_page_alloc(struct slob_page *sp, size_t size, int align) +static void *slob_page_alloc(struct page *sp, size_t size, int align) { slob_t *prev, *cur, *aligned = NULL; int delta = 0, units = SLOB_UNITS(size); - for (prev = NULL, cur = sp->free; ; prev = cur, cur = slob_next(cur)) { + for (prev = NULL, cur = sp->freelist; ; prev = cur, cur = slob_next(cur)) { slobidx_t avail = slob_units(cur); if (align) { @@ -296,12 +245,12 @@ static void *slob_page_alloc(struct slob_page *sp, size_t size, int align) if (prev) set_slob(prev, slob_units(prev), next); else - sp->free = next; + sp->freelist = next; } else { /* fragment */ if (prev) set_slob(prev, slob_units(prev), cur + units); else - sp->free = cur + units; + sp->freelist = cur + units; set_slob(cur + units, avail - units, next); } @@ -320,7 +269,7 @@ static void *slob_page_alloc(struct slob_page *sp, size_t size, int align) */ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) { - struct slob_page *sp; + struct page *sp; struct list_head *prev; struct list_head *slob_list; slob_t *b = NULL; @@ -341,7 +290,7 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) * If there's a node specification, search for a partial * page with a matching node id in the freelist. */ - if (node != -1 && page_to_nid(&sp->page) != node) + if (node != -1 && page_to_nid(sp) != node) continue; #endif /* Enough room on this page? */ @@ -369,12 +318,12 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) b = slob_new_pages(gfp & ~__GFP_ZERO, 0, node); if (!b) return NULL; - sp = slob_page(b); - set_slob_page(sp); + sp = virt_to_page(b); + __SetPageSlab(sp); spin_lock_irqsave(&slob_lock, flags); sp->units = SLOB_UNITS(PAGE_SIZE); - sp->free = b; + sp->freelist = b; INIT_LIST_HEAD(&sp->list); set_slob(b, SLOB_UNITS(PAGE_SIZE), b + SLOB_UNITS(PAGE_SIZE)); set_slob_page_free(sp, slob_list); @@ -392,7 +341,7 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) */ static void slob_free(void *block, int size) { - struct slob_page *sp; + struct page *sp; slob_t *prev, *next, *b = (slob_t *)block; slobidx_t units; unsigned long flags; @@ -402,7 +351,7 @@ static void slob_free(void *block, int size) return; BUG_ON(!size); - sp = slob_page(block); + sp = virt_to_page(block); units = SLOB_UNITS(size); spin_lock_irqsave(&slob_lock, flags); @@ -412,8 +361,8 @@ static void slob_free(void *block, int size) if (slob_page_free(sp)) clear_slob_page_free(sp); spin_unlock_irqrestore(&slob_lock, flags); - clear_slob_page(sp); - free_slob_page(sp); + __ClearPageSlab(sp); + reset_page_mapcount(sp); slob_free_pages(b, 0); return; } @@ -421,7 +370,7 @@ static void slob_free(void *block, int size) if (!slob_page_free(sp)) { /* This slob page is about to become partially free. Easy! */ sp->units = units; - sp->free = b; + sp->freelist = b; set_slob(b, units, (void *)((unsigned long)(b + SLOB_UNITS(PAGE_SIZE)) & PAGE_MASK)); @@ -441,15 +390,15 @@ static void slob_free(void *block, int size) */ sp->units += units; - if (b < sp->free) { - if (b + units == sp->free) { - units += slob_units(sp->free); - sp->free = slob_next(sp->free); + if (b < (slob_t *)sp->freelist) { + if (b + units == sp->freelist) { + units += slob_units(sp->freelist); + sp->freelist = slob_next(sp->freelist); } - set_slob(b, units, sp->free); - sp->free = b; + set_slob(b, units, sp->freelist); + sp->freelist = b; } else { - prev = sp->free; + prev = sp->freelist; next = slob_next(prev); while (b > next) { prev = next; @@ -522,7 +471,7 @@ EXPORT_SYMBOL(__kmalloc_node); void kfree(const void *block) { - struct slob_page *sp; + struct page *sp; trace_kfree(_RET_IP_, block); @@ -530,43 +479,36 @@ void kfree(const void *block) return; kmemleak_free(block); - sp = slob_page(block); - if (is_slob_page(sp)) { + sp = virt_to_page(block); + if (PageSlab(sp)) { int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); unsigned int *m = (unsigned int *)(block - align); slob_free(m, *m + align); } else - put_page(&sp->page); + put_page(sp); } EXPORT_SYMBOL(kfree); /* can't use ksize for kmem_cache_alloc memory, only kmalloc */ size_t ksize(const void *block) { - struct slob_page *sp; + struct page *sp; BUG_ON(!block); if (unlikely(block == ZERO_SIZE_PTR)) return 0; - sp = slob_page(block); - if (is_slob_page(sp)) { + sp = virt_to_page(block); + if (PageSlab(sp)) { int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); unsigned int *m = (unsigned int *)(block - align); return SLOB_UNITS(*m) * SLOB_UNIT; } else - return sp->page.private; + return sp->private; } EXPORT_SYMBOL(ksize); -struct kmem_cache { - unsigned int size, align; - unsigned long flags; - const char *name; - void (*ctor)(void *); -}; - -struct kmem_cache *kmem_cache_create(const char *name, size_t size, +struct kmem_cache *__kmem_cache_create(const char *name, size_t size, size_t align, unsigned long flags, void (*ctor)(void *)) { struct kmem_cache *c; @@ -589,13 +531,12 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, c->align = ARCH_SLAB_MINALIGN; if (c->align < align) c->align = align; - } else if (flags & SLAB_PANIC) - panic("Cannot create slab cache %s\n", name); - kmemleak_alloc(c, sizeof(struct kmem_cache), 1, GFP_KERNEL); + kmemleak_alloc(c, sizeof(struct kmem_cache), 1, GFP_KERNEL); + c->refcount = 1; + } return c; } -EXPORT_SYMBOL(kmem_cache_create); void kmem_cache_destroy(struct kmem_cache *c) { @@ -678,19 +619,12 @@ int kmem_cache_shrink(struct kmem_cache *d) } EXPORT_SYMBOL(kmem_cache_shrink); -static unsigned int slob_ready __read_mostly; - -int slab_is_available(void) -{ - return slob_ready; -} - void __init kmem_cache_init(void) { - slob_ready = 1; + slab_state = UP; } void __init kmem_cache_init_late(void) { - /* Nothing to do */ + slab_state = FULL; } diff --git a/mm/slub.c b/mm/slub.c index 8c691fa1cf3c..e517d435e5dc 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -16,6 +16,7 @@ #include <linux/interrupt.h> #include <linux/bitops.h> #include <linux/slab.h> +#include "slab.h" #include <linux/proc_fs.h> #include <linux/seq_file.h> #include <linux/kmemcheck.h> @@ -35,13 +36,13 @@ /* * Lock order: - * 1. slub_lock (Global Semaphore) + * 1. slab_mutex (Global Mutex) * 2. node->list_lock * 3. slab_lock(page) (Only on some arches and for debugging) * - * slub_lock + * slab_mutex * - * The role of the slub_lock is to protect the list of all the slabs + * The role of the slab_mutex is to protect the list of all the slabs * and to synchronize major metadata changes to slab cache structures. * * The slab_lock is only used for debugging and on arches that do not @@ -182,17 +183,6 @@ static int kmem_size = sizeof(struct kmem_cache); static struct notifier_block slab_notifier; #endif -static enum { - DOWN, /* No slab functionality available */ - PARTIAL, /* Kmem_cache_node works */ - UP, /* Everything works but does not show up in sysfs */ - SYSFS /* Sysfs up */ -} slab_state = DOWN; - -/* A list of all slab caches on the system */ -static DECLARE_RWSEM(slub_lock); -static LIST_HEAD(slab_caches); - /* * Tracking user of a slab. */ @@ -237,11 +227,6 @@ static inline void stat(const struct kmem_cache *s, enum stat_item si) * Core slab cache functions *******************************************************************/ -int slab_is_available(void) -{ - return slab_state >= UP; -} - static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) { return s->node[node]; @@ -311,7 +296,7 @@ static inline size_t slab_ksize(const struct kmem_cache *s) * and whatever may come after it. */ if (s->flags & (SLAB_RED_ZONE | SLAB_POISON)) - return s->objsize; + return s->object_size; #endif /* @@ -609,11 +594,11 @@ static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) if (p > addr + 16) print_section("Bytes b4 ", p - 16, 16); - print_section("Object ", p, min_t(unsigned long, s->objsize, + print_section("Object ", p, min_t(unsigned long, s->object_size, PAGE_SIZE)); if (s->flags & SLAB_RED_ZONE) - print_section("Redzone ", p + s->objsize, - s->inuse - s->objsize); + print_section("Redzone ", p + s->object_size, + s->inuse - s->object_size); if (s->offset) off = s->offset + sizeof(void *); @@ -655,12 +640,12 @@ static void init_object(struct kmem_cache *s, void *object, u8 val) u8 *p = object; if (s->flags & __OBJECT_POISON) { - memset(p, POISON_FREE, s->objsize - 1); - p[s->objsize - 1] = POISON_END; + memset(p, POISON_FREE, s->object_size - 1); + p[s->object_size - 1] = POISON_END; } if (s->flags & SLAB_RED_ZONE) - memset(p + s->objsize, val, s->inuse - s->objsize); + memset(p + s->object_size, val, s->inuse - s->object_size); } static void restore_bytes(struct kmem_cache *s, char *message, u8 data, @@ -705,10 +690,10 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page, * Poisoning uses 0x6b (POISON_FREE) and the last byte is * 0xa5 (POISON_END) * - * object + s->objsize + * object + s->object_size * Padding to reach word boundary. This is also used for Redzoning. * Padding is extended by another word if Redzoning is enabled and - * objsize == inuse. + * object_size == inuse. * * We fill with 0xbb (RED_INACTIVE) for inactive objects and with * 0xcc (RED_ACTIVE) for objects in use. @@ -727,7 +712,7 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page, * object + s->size * Nothing is used beyond s->size. * - * If slabcaches are merged then the objsize and inuse boundaries are mostly + * If slabcaches are merged then the object_size and inuse boundaries are mostly * ignored. And therefore no slab options that rely on these boundaries * may be used with merged slabcaches. */ @@ -787,25 +772,25 @@ static int check_object(struct kmem_cache *s, struct page *page, void *object, u8 val) { u8 *p = object; - u8 *endobject = object + s->objsize; + u8 *endobject = object + s->object_size; if (s->flags & SLAB_RED_ZONE) { if (!check_bytes_and_report(s, page, object, "Redzone", - endobject, val, s->inuse - s->objsize)) + endobject, val, s->inuse - s->object_size)) return 0; } else { - if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) { + if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { check_bytes_and_report(s, page, p, "Alignment padding", - endobject, POISON_INUSE, s->inuse - s->objsize); + endobject, POISON_INUSE, s->inuse - s->object_size); } } if (s->flags & SLAB_POISON) { if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) && (!check_bytes_and_report(s, page, p, "Poison", p, - POISON_FREE, s->objsize - 1) || + POISON_FREE, s->object_size - 1) || !check_bytes_and_report(s, page, p, "Poison", - p + s->objsize - 1, POISON_END, 1))) + p + s->object_size - 1, POISON_END, 1))) return 0; /* * check_pad_bytes cleans up on its own. @@ -926,7 +911,7 @@ static void trace(struct kmem_cache *s, struct page *page, void *object, page->freelist); if (!alloc) - print_section("Object ", (void *)object, s->objsize); + print_section("Object ", (void *)object, s->object_size); dump_stack(); } @@ -942,14 +927,14 @@ static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) lockdep_trace_alloc(flags); might_sleep_if(flags & __GFP_WAIT); - return should_failslab(s->objsize, flags, s->flags); + return should_failslab(s->object_size, flags, s->flags); } static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, void *object) { flags &= gfp_allowed_mask; kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); - kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, flags); + kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags); } static inline void slab_free_hook(struct kmem_cache *s, void *x) @@ -966,13 +951,13 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x) unsigned long flags; local_irq_save(flags); - kmemcheck_slab_free(s, x, s->objsize); - debug_check_no_locks_freed(x, s->objsize); + kmemcheck_slab_free(s, x, s->object_size); + debug_check_no_locks_freed(x, s->object_size); local_irq_restore(flags); } #endif if (!(s->flags & SLAB_DEBUG_OBJECTS)) - debug_check_no_obj_freed(x, s->objsize); + debug_check_no_obj_freed(x, s->object_size); } /* @@ -1207,7 +1192,7 @@ out: __setup("slub_debug", setup_slub_debug); -static unsigned long kmem_cache_flags(unsigned long objsize, +static unsigned long kmem_cache_flags(unsigned long object_size, unsigned long flags, const char *name, void (*ctor)(void *)) { @@ -1237,7 +1222,7 @@ static inline int check_object(struct kmem_cache *s, struct page *page, static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page) {} static inline void remove_full(struct kmem_cache *s, struct page *page) {} -static inline unsigned long kmem_cache_flags(unsigned long objsize, +static inline unsigned long kmem_cache_flags(unsigned long object_size, unsigned long flags, const char *name, void (*ctor)(void *)) { @@ -1314,13 +1299,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) stat(s, ORDER_FALLBACK); } - if (flags & __GFP_WAIT) - local_irq_disable(); - - if (!page) - return NULL; - - if (kmemcheck_enabled + if (kmemcheck_enabled && page && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { int pages = 1 << oo_order(oo); @@ -1336,6 +1315,11 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) kmemcheck_mark_unallocated_pages(page, pages); } + if (flags & __GFP_WAIT) + local_irq_disable(); + if (!page) + return NULL; + page->objects = oo_objects(oo); mod_zone_page_state(page_zone(page), (s->flags & SLAB_RECLAIM_ACCOUNT) ? @@ -1490,12 +1474,12 @@ static inline void remove_partial(struct kmem_cache_node *n, } /* - * Lock slab, remove from the partial list and put the object into the - * per cpu freelist. + * Remove slab from the partial list, freeze it and + * return the pointer to the freelist. * * Returns a list of objects or NULL if it fails. * - * Must hold list_lock. + * Must hold list_lock since we modify the partial list. */ static inline void *acquire_slab(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page, @@ -1510,26 +1494,27 @@ static inline void *acquire_slab(struct kmem_cache *s, * The old freelist is the list of objects for the * per cpu allocation list. */ - do { - freelist = page->freelist; - counters = page->counters; - new.counters = counters; - if (mode) { - new.inuse = page->objects; - new.freelist = NULL; - } else { - new.freelist = freelist; - } + freelist = page->freelist; + counters = page->counters; + new.counters = counters; + if (mode) { + new.inuse = page->objects; + new.freelist = NULL; + } else { + new.freelist = freelist; + } - VM_BUG_ON(new.frozen); - new.frozen = 1; + VM_BUG_ON(new.frozen); + new.frozen = 1; - } while (!__cmpxchg_double_slab(s, page, + if (!__cmpxchg_double_slab(s, page, freelist, counters, new.freelist, new.counters, - "lock and freeze")); + "acquire_slab")) + return NULL; remove_partial(n, page); + WARN_ON(!freelist); return freelist; } @@ -1563,7 +1548,6 @@ static void *get_partial_node(struct kmem_cache *s, if (!object) { c->page = page; - c->node = page_to_nid(page); stat(s, ALLOC_FROM_PARTIAL); object = t; available = page->objects - page->inuse; @@ -1617,7 +1601,7 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, do { cpuset_mems_cookie = get_mems_allowed(); - zonelist = node_zonelist(slab_node(current->mempolicy), flags); + zonelist = node_zonelist(slab_node(), flags); for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { struct kmem_cache_node *n; @@ -1731,14 +1715,12 @@ void init_kmem_cache_cpus(struct kmem_cache *s) /* * Remove the cpu slab */ -static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) +static void deactivate_slab(struct kmem_cache *s, struct page *page, void *freelist) { enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE }; - struct page *page = c->page; struct kmem_cache_node *n = get_node(s, page_to_nid(page)); int lock = 0; enum slab_modes l = M_NONE, m = M_NONE; - void *freelist; void *nextfree; int tail = DEACTIVATE_TO_HEAD; struct page new; @@ -1749,11 +1731,6 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) tail = DEACTIVATE_TO_TAIL; } - c->tid = next_tid(c->tid); - c->page = NULL; - freelist = c->freelist; - c->freelist = NULL; - /* * Stage one: Free all available per cpu objects back * to the page freelist while it is still frozen. Leave the @@ -1879,21 +1856,31 @@ redo: } } -/* Unfreeze all the cpu partial slabs */ +/* + * Unfreeze all the cpu partial slabs. + * + * This function must be called with interrupt disabled. + */ static void unfreeze_partials(struct kmem_cache *s) { - struct kmem_cache_node *n = NULL; + struct kmem_cache_node *n = NULL, *n2 = NULL; struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab); struct page *page, *discard_page = NULL; while ((page = c->partial)) { - enum slab_modes { M_PARTIAL, M_FREE }; - enum slab_modes l, m; struct page new; struct page old; c->partial = page->next; - l = M_FREE; + + n2 = get_node(s, page_to_nid(page)); + if (n != n2) { + if (n) + spin_unlock(&n->list_lock); + + n = n2; + spin_lock(&n->list_lock); + } do { @@ -1906,43 +1893,17 @@ static void unfreeze_partials(struct kmem_cache *s) new.frozen = 0; - if (!new.inuse && (!n || n->nr_partial > s->min_partial)) - m = M_FREE; - else { - struct kmem_cache_node *n2 = get_node(s, - page_to_nid(page)); - - m = M_PARTIAL; - if (n != n2) { - if (n) - spin_unlock(&n->list_lock); - - n = n2; - spin_lock(&n->list_lock); - } - } - - if (l != m) { - if (l == M_PARTIAL) { - remove_partial(n, page); - stat(s, FREE_REMOVE_PARTIAL); - } else { - add_partial(n, page, - DEACTIVATE_TO_TAIL); - stat(s, FREE_ADD_PARTIAL); - } - - l = m; - } - - } while (!cmpxchg_double_slab(s, page, + } while (!__cmpxchg_double_slab(s, page, old.freelist, old.counters, new.freelist, new.counters, "unfreezing slab")); - if (m == M_FREE) { + if (unlikely(!new.inuse && n->nr_partial > s->min_partial)) { page->next = discard_page; discard_page = page; + } else { + add_partial(n, page, DEACTIVATE_TO_TAIL); + stat(s, FREE_ADD_PARTIAL); } } @@ -2011,7 +1972,11 @@ int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) { stat(s, CPUSLAB_FLUSH); - deactivate_slab(s, c); + deactivate_slab(s, c->page, c->freelist); + + c->tid = next_tid(c->tid); + c->page = NULL; + c->freelist = NULL; } /* @@ -2055,10 +2020,10 @@ static void flush_all(struct kmem_cache *s) * Check if the objects in a per cpu structure fit numa * locality expectations. */ -static inline int node_match(struct kmem_cache_cpu *c, int node) +static inline int node_match(struct page *page, int node) { #ifdef CONFIG_NUMA - if (node != NUMA_NO_NODE && c->node != node) + if (node != NUMA_NO_NODE && page_to_nid(page) != node) return 0; #endif return 1; @@ -2101,10 +2066,10 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) "SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n", nid, gfpflags); printk(KERN_WARNING " cache: %s, object size: %d, buffer size: %d, " - "default order: %d, min order: %d\n", s->name, s->objsize, + "default order: %d, min order: %d\n", s->name, s->object_size, s->size, oo_order(s->oo), oo_order(s->min)); - if (oo_order(s->min) > get_order(s->objsize)) + if (oo_order(s->min) > get_order(s->object_size)) printk(KERN_WARNING " %s debugging increased min order, use " "slub_debug=O to disable.\n", s->name); @@ -2130,10 +2095,16 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, int node, struct kmem_cache_cpu **pc) { - void *object; - struct kmem_cache_cpu *c; - struct page *page = new_slab(s, flags, node); + void *freelist; + struct kmem_cache_cpu *c = *pc; + struct page *page; + + freelist = get_partial(s, flags, node, c); + if (freelist) + return freelist; + + page = new_slab(s, flags, node); if (page) { c = __this_cpu_ptr(s->cpu_slab); if (c->page) @@ -2143,17 +2114,16 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, * No other reference to the page yet so we can * muck around with it freely without cmpxchg */ - object = page->freelist; + freelist = page->freelist; page->freelist = NULL; stat(s, ALLOC_SLAB); - c->node = page_to_nid(page); c->page = page; *pc = c; } else - object = NULL; + freelist = NULL; - return object; + return freelist; } /* @@ -2163,6 +2133,8 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, * The page is still frozen if the return value is not NULL. * * If this function returns NULL then the page has been unfrozen. + * + * This function must be called with interrupt disabled. */ static inline void *get_freelist(struct kmem_cache *s, struct page *page) { @@ -2173,13 +2145,14 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page) do { freelist = page->freelist; counters = page->counters; + new.counters = counters; VM_BUG_ON(!new.frozen); new.inuse = page->objects; new.frozen = freelist != NULL; - } while (!cmpxchg_double_slab(s, page, + } while (!__cmpxchg_double_slab(s, page, freelist, counters, NULL, new.counters, "get_freelist")); @@ -2206,7 +2179,8 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page) static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long addr, struct kmem_cache_cpu *c) { - void **object; + void *freelist; + struct page *page; unsigned long flags; local_irq_save(flags); @@ -2219,25 +2193,29 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, c = this_cpu_ptr(s->cpu_slab); #endif - if (!c->page) + page = c->page; + if (!page) goto new_slab; redo: - if (unlikely(!node_match(c, node))) { + + if (unlikely(!node_match(page, node))) { stat(s, ALLOC_NODE_MISMATCH); - deactivate_slab(s, c); + deactivate_slab(s, page, c->freelist); + c->page = NULL; + c->freelist = NULL; goto new_slab; } /* must check again c->freelist in case of cpu migration or IRQ */ - object = c->freelist; - if (object) + freelist = c->freelist; + if (freelist) goto load_freelist; stat(s, ALLOC_SLOWPATH); - object = get_freelist(s, c->page); + freelist = get_freelist(s, page); - if (!object) { + if (!freelist) { c->page = NULL; stat(s, DEACTIVATE_BYPASS); goto new_slab; @@ -2246,50 +2224,50 @@ redo: stat(s, ALLOC_REFILL); load_freelist: - c->freelist = get_freepointer(s, object); + /* + * freelist is pointing to the list of objects to be used. + * page is pointing to the page from which the objects are obtained. + * That page must be frozen for per cpu allocations to work. + */ + VM_BUG_ON(!c->page->frozen); + c->freelist = get_freepointer(s, freelist); c->tid = next_tid(c->tid); local_irq_restore(flags); - return object; + return freelist; new_slab: if (c->partial) { - c->page = c->partial; - c->partial = c->page->next; - c->node = page_to_nid(c->page); + page = c->page = c->partial; + c->partial = page->next; stat(s, CPU_PARTIAL_ALLOC); c->freelist = NULL; goto redo; } - /* Then do expensive stuff like retrieving pages from the partial lists */ - object = get_partial(s, gfpflags, node, c); + freelist = new_slab_objects(s, gfpflags, node, &c); - if (unlikely(!object)) { + if (unlikely(!freelist)) { + if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) + slab_out_of_memory(s, gfpflags, node); - object = new_slab_objects(s, gfpflags, node, &c); - - if (unlikely(!object)) { - if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) - slab_out_of_memory(s, gfpflags, node); - - local_irq_restore(flags); - return NULL; - } + local_irq_restore(flags); + return NULL; } + page = c->page; if (likely(!kmem_cache_debug(s))) goto load_freelist; /* Only entered in the debug case */ - if (!alloc_debug_processing(s, c->page, object, addr)) + if (!alloc_debug_processing(s, page, freelist, addr)) goto new_slab; /* Slab failed checks. Next slab needed */ - c->freelist = get_freepointer(s, object); - deactivate_slab(s, c); - c->node = NUMA_NO_NODE; + deactivate_slab(s, page, get_freepointer(s, freelist)); + c->page = NULL; + c->freelist = NULL; local_irq_restore(flags); - return object; + return freelist; } /* @@ -2307,6 +2285,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, { void **object; struct kmem_cache_cpu *c; + struct page *page; unsigned long tid; if (slab_pre_alloc_hook(s, gfpflags)) @@ -2332,7 +2311,8 @@ redo: barrier(); object = c->freelist; - if (unlikely(!object || !node_match(c, node))) + page = c->page; + if (unlikely(!object || !node_match(page, node))) object = __slab_alloc(s, gfpflags, node, addr, c); @@ -2364,7 +2344,7 @@ redo: } if (unlikely(gfpflags & __GFP_ZERO) && object) - memset(object, 0, s->objsize); + memset(object, 0, s->object_size); slab_post_alloc_hook(s, gfpflags, object); @@ -2375,7 +2355,7 @@ void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) { void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_); - trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags); + trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, s->size, gfpflags); return ret; } @@ -2405,7 +2385,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) void *ret = slab_alloc(s, gfpflags, node, _RET_IP_); trace_kmem_cache_alloc_node(_RET_IP_, ret, - s->objsize, s->size, gfpflags, node); + s->object_size, s->size, gfpflags, node); return ret; } @@ -2900,7 +2880,7 @@ static void set_min_partial(struct kmem_cache *s, unsigned long min) static int calculate_sizes(struct kmem_cache *s, int forced_order) { unsigned long flags = s->flags; - unsigned long size = s->objsize; + unsigned long size = s->object_size; unsigned long align = s->align; int order; @@ -2929,7 +2909,7 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order) * end of the object and the free pointer. If not then add an * additional word to have some bytes to store Redzone information. */ - if ((flags & SLAB_RED_ZONE) && size == s->objsize) + if ((flags & SLAB_RED_ZONE) && size == s->object_size) size += sizeof(void *); #endif @@ -2977,7 +2957,7 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order) * user specified and the dynamic determination of cache line size * on bootup. */ - align = calculate_alignment(flags, align, s->objsize); + align = calculate_alignment(flags, align, s->object_size); s->align = align; /* @@ -3025,7 +3005,7 @@ static int kmem_cache_open(struct kmem_cache *s, memset(s, 0, kmem_size); s->name = name; s->ctor = ctor; - s->objsize = size; + s->object_size = size; s->align = align; s->flags = kmem_cache_flags(size, flags, name, ctor); s->reserved = 0; @@ -3040,7 +3020,7 @@ static int kmem_cache_open(struct kmem_cache *s, * Disable debugging flags that store metadata if the min slab * order increased. */ - if (get_order(s->size) > get_order(s->objsize)) { + if (get_order(s->size) > get_order(s->object_size)) { s->flags &= ~DEBUG_METADATA_FLAGS; s->offset = 0; if (!calculate_sizes(s, -1)) @@ -3114,7 +3094,7 @@ error: */ unsigned int kmem_cache_size(struct kmem_cache *s) { - return s->objsize; + return s->object_size; } EXPORT_SYMBOL(kmem_cache_size); @@ -3192,11 +3172,11 @@ static inline int kmem_cache_close(struct kmem_cache *s) */ void kmem_cache_destroy(struct kmem_cache *s) { - down_write(&slub_lock); + mutex_lock(&slab_mutex); s->refcount--; if (!s->refcount) { list_del(&s->list); - up_write(&slub_lock); + mutex_unlock(&slab_mutex); if (kmem_cache_close(s)) { printk(KERN_ERR "SLUB %s: %s called for cache that " "still has objects.\n", s->name, __func__); @@ -3206,7 +3186,7 @@ void kmem_cache_destroy(struct kmem_cache *s) rcu_barrier(); sysfs_slab_remove(s); } else - up_write(&slub_lock); + mutex_unlock(&slab_mutex); } EXPORT_SYMBOL(kmem_cache_destroy); @@ -3268,7 +3248,7 @@ static struct kmem_cache *__init create_kmalloc_cache(const char *name, /* * This function is called with IRQs disabled during early-boot on - * single CPU so there's no need to take slub_lock here. + * single CPU so there's no need to take slab_mutex here. */ if (!kmem_cache_open(s, name, size, ARCH_KMALLOC_MINALIGN, flags, NULL)) @@ -3553,10 +3533,10 @@ static int slab_mem_going_offline_callback(void *arg) { struct kmem_cache *s; - down_read(&slub_lock); + mutex_lock(&slab_mutex); list_for_each_entry(s, &slab_caches, list) kmem_cache_shrink(s); - up_read(&slub_lock); + mutex_unlock(&slab_mutex); return 0; } @@ -3577,7 +3557,7 @@ static void slab_mem_offline_callback(void *arg) if (offline_node < 0) return; - down_read(&slub_lock); + mutex_lock(&slab_mutex); list_for_each_entry(s, &slab_caches, list) { n = get_node(s, offline_node); if (n) { @@ -3593,7 +3573,7 @@ static void slab_mem_offline_callback(void *arg) kmem_cache_free(kmem_cache_node, n); } } - up_read(&slub_lock); + mutex_unlock(&slab_mutex); } static int slab_mem_going_online_callback(void *arg) @@ -3616,7 +3596,7 @@ static int slab_mem_going_online_callback(void *arg) * allocate a kmem_cache_node structure in order to bring the node * online. */ - down_read(&slub_lock); + mutex_lock(&slab_mutex); list_for_each_entry(s, &slab_caches, list) { /* * XXX: kmem_cache_alloc_node will fallback to other nodes @@ -3632,7 +3612,7 @@ static int slab_mem_going_online_callback(void *arg) s->node[nid] = n; } out: - up_read(&slub_lock); + mutex_unlock(&slab_mutex); return ret; } @@ -3843,11 +3823,11 @@ void __init kmem_cache_init(void) if (s && s->size) { char *name = kasprintf(GFP_NOWAIT, - "dma-kmalloc-%d", s->objsize); + "dma-kmalloc-%d", s->object_size); BUG_ON(!name); kmalloc_dma_caches[i] = create_kmalloc_cache(name, - s->objsize, SLAB_CACHE_DMA); + s->object_size, SLAB_CACHE_DMA); } } #endif @@ -3924,16 +3904,12 @@ static struct kmem_cache *find_mergeable(size_t size, return NULL; } -struct kmem_cache *kmem_cache_create(const char *name, size_t size, +struct kmem_cache *__kmem_cache_create(const char *name, size_t size, size_t align, unsigned long flags, void (*ctor)(void *)) { struct kmem_cache *s; char *n; - if (WARN_ON(!name)) - return NULL; - - down_write(&slub_lock); s = find_mergeable(size, align, flags, name, ctor); if (s) { s->refcount++; @@ -3941,49 +3917,42 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, * Adjust the object sizes so that we clear * the complete object on kzalloc. */ - s->objsize = max(s->objsize, (int)size); + s->object_size = max(s->object_size, (int)size); s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *))); if (sysfs_slab_alias(s, name)) { s->refcount--; - goto err; + return NULL; } - up_write(&slub_lock); return s; } n = kstrdup(name, GFP_KERNEL); if (!n) - goto err; + return NULL; s = kmalloc(kmem_size, GFP_KERNEL); if (s) { if (kmem_cache_open(s, n, size, align, flags, ctor)) { + int r; + list_add(&s->list, &slab_caches); - up_write(&slub_lock); - if (sysfs_slab_add(s)) { - down_write(&slub_lock); - list_del(&s->list); - kfree(n); - kfree(s); - goto err; - } - return s; + mutex_unlock(&slab_mutex); + r = sysfs_slab_add(s); + mutex_lock(&slab_mutex); + + if (!r) + return s; + + list_del(&s->list); + kmem_cache_close(s); } kfree(s); } kfree(n); -err: - up_write(&slub_lock); - - if (flags & SLAB_PANIC) - panic("Cannot create slabcache %s\n", name); - else - s = NULL; - return s; + return NULL; } -EXPORT_SYMBOL(kmem_cache_create); #ifdef CONFIG_SMP /* @@ -4002,13 +3971,13 @@ static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb, case CPU_UP_CANCELED_FROZEN: case CPU_DEAD: case CPU_DEAD_FROZEN: - down_read(&slub_lock); + mutex_lock(&slab_mutex); list_for_each_entry(s, &slab_caches, list) { local_irq_save(flags); __flush_cpu_slab(s, cpu); local_irq_restore(flags); } - up_read(&slub_lock); + mutex_unlock(&slab_mutex); break; default: break; @@ -4500,30 +4469,31 @@ static ssize_t show_slab_objects(struct kmem_cache *s, for_each_possible_cpu(cpu) { struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); - int node = ACCESS_ONCE(c->node); + int node; struct page *page; - if (node < 0) - continue; page = ACCESS_ONCE(c->page); - if (page) { - if (flags & SO_TOTAL) - x = page->objects; - else if (flags & SO_OBJECTS) - x = page->inuse; - else - x = 1; + if (!page) + continue; - total += x; - nodes[node] += x; - } - page = c->partial; + node = page_to_nid(page); + if (flags & SO_TOTAL) + x = page->objects; + else if (flags & SO_OBJECTS) + x = page->inuse; + else + x = 1; + total += x; + nodes[node] += x; + + page = ACCESS_ONCE(c->partial); if (page) { x = page->pobjects; total += x; nodes[node] += x; } + per_cpu[node]++; } } @@ -4623,7 +4593,7 @@ SLAB_ATTR_RO(align); static ssize_t object_size_show(struct kmem_cache *s, char *buf) { - return sprintf(buf, "%d\n", s->objsize); + return sprintf(buf, "%d\n", s->object_size); } SLAB_ATTR_RO(object_size); @@ -5286,7 +5256,7 @@ static int sysfs_slab_add(struct kmem_cache *s) const char *name; int unmergeable; - if (slab_state < SYSFS) + if (slab_state < FULL) /* Defer until later */ return 0; @@ -5331,7 +5301,7 @@ static int sysfs_slab_add(struct kmem_cache *s) static void sysfs_slab_remove(struct kmem_cache *s) { - if (slab_state < SYSFS) + if (slab_state < FULL) /* * Sysfs has not been setup yet so no need to remove the * cache from sysfs. @@ -5359,7 +5329,7 @@ static int sysfs_slab_alias(struct kmem_cache *s, const char *name) { struct saved_alias *al; - if (slab_state == SYSFS) { + if (slab_state == FULL) { /* * If we have a leftover link then remove it. */ @@ -5383,16 +5353,16 @@ static int __init slab_sysfs_init(void) struct kmem_cache *s; int err; - down_write(&slub_lock); + mutex_lock(&slab_mutex); slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj); if (!slab_kset) { - up_write(&slub_lock); + mutex_unlock(&slab_mutex); printk(KERN_ERR "Cannot register slab subsystem.\n"); return -ENOSYS; } - slab_state = SYSFS; + slab_state = FULL; list_for_each_entry(s, &slab_caches, list) { err = sysfs_slab_add(s); @@ -5408,11 +5378,11 @@ static int __init slab_sysfs_init(void) err = sysfs_slab_alias(al->s, al->name); if (err) printk(KERN_ERR "SLUB: Unable to add boot slab alias" - " %s to sysfs\n", s->name); + " %s to sysfs\n", al->name); kfree(al); } - up_write(&slub_lock); + mutex_unlock(&slab_mutex); resiliency_test(); return 0; } @@ -5427,7 +5397,7 @@ __initcall(slab_sysfs_init); static void print_slabinfo_header(struct seq_file *m) { seq_puts(m, "slabinfo - version: 2.1\n"); - seq_puts(m, "# name <active_objs> <num_objs> <objsize> " + seq_puts(m, "# name <active_objs> <num_objs> <object_size> " "<objperslab> <pagesperslab>"); seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>"); seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>"); @@ -5438,7 +5408,7 @@ static void *s_start(struct seq_file *m, loff_t *pos) { loff_t n = *pos; - down_read(&slub_lock); + mutex_lock(&slab_mutex); if (!n) print_slabinfo_header(m); @@ -5452,7 +5422,7 @@ static void *s_next(struct seq_file *m, void *p, loff_t *pos) static void s_stop(struct seq_file *m, void *p) { - up_read(&slub_lock); + mutex_unlock(&slab_mutex); } static int s_show(struct seq_file *m, void *p) |