slqb.c

/*
 * SLQB: A slab allocator that focuses on per-CPU scaling, and good performance
 * with order-0 allocations. Fastpaths emphasis is placed on local allocaiton
 * and freeing, but with a secondary goal of good remote freeing (freeing on
 * another CPU from that which allocated).
 *
 * Using ideas and code from mm/slab.c, mm/slob.c, and mm/slub.c.
 */

#include <linux/mm.h>
#include <linux/swap.h> /* struct reclaim_state */
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/slab.h>
#include <linux/seq_file.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/mempolicy.h>
#include <linux/ctype.h>
#include <linux/kallsyms.h>
#include <linux/memory.h>
#include <linux/fault-inject.h>

/*
 * TODO
 * - fix up releasing of offlined data structures. Not a big deal because
 *   they don't get cumulatively leaked with successive online/offline cycles
 * - allow OOM conditions to flush back per-CPU pages to common lists to be
 *   reused by other CPUs.
 * - investiage performance with memoryless nodes. Perhaps CPUs can be given
 *   a default closest home node via which it can use fastpath functions.
 *   Perhaps it is not a big problem.
 */

/*
 * slqb_page overloads struct page, and is used to manage some slob allocation
 * aspects, however to avoid the horrible mess in include/linux/mm_types.h,
 * we'll just define our own struct slqb_page type variant here.
 */
struct slqb_page {
	union {
		struct {
			unsigned long	flags;		/* mandatory */
			atomic_t	_count;		/* mandatory */
			unsigned int	inuse;		/* Nr of objects */
			struct kmem_cache_list *list;	/* Pointer to list */
			void		 **freelist;	/* LIFO freelist */
			union {
				struct list_head lru;	/* misc. list */
				struct rcu_head rcu_head; /* for rcu freeing */
			};
		};
		struct page page;
	};
};
static inline void struct_slqb_page_wrong_size(void)
{ BUILD_BUG_ON(sizeof(struct slqb_page) != sizeof(struct page)); }

#define PG_SLQB_BIT (1 << PG_slab)

/*
 * slqb_min_order: minimum allocation order for slabs
 */
static int slqb_min_order;

/*
 * slqb_min_objects: minimum number of objects per slab. Increasing this
 * will increase the allocation order for slabs with larger objects
 */
static int slqb_min_objects = 1;

#ifdef CONFIG_NUMA
static inline int slab_numa(struct kmem_cache *s)
{
	return s->flags & SLAB_NUMA;
}
#else
static inline int slab_numa(struct kmem_cache *s)
{
	return 0;
}
#endif

static inline int slab_hiwater(struct kmem_cache *s)
{
	return s->hiwater;
}

static inline int slab_freebatch(struct kmem_cache *s)
{
	return s->freebatch;
}

/*
 * Lock order:
 * kmem_cache_node->list_lock
 *   kmem_cache_remote_free->lock
 *
 * Data structures:
 * SLQB is primarily per-cpu. For each kmem_cache, each CPU has:
 *
 * - A LIFO list of node-local objects. Allocation and freeing of node local
 *   objects goes first to this list.
 *
 * - 2 Lists of slab pages, free and partial pages. If an allocation misses
 *   the object list, it tries from the partial list, then the free list.
 *   After freeing an object to the object list, if it is over a watermark,
 *   some objects are freed back to pages. If an allocation misses these lists,
 *   a new slab page is allocated from the page allocator. If the free list
 *   reaches a watermark, some of its pages are returned to the page allocator.
 *
 * - A remote free queue, where objects freed that did not come from the local
 *   node are queued to. When this reaches a watermark, the objects are
 *   flushed.
 *
 * - A remotely freed queue, where objects allocated from this CPU are flushed
 *   to from other CPUs' remote free queues. kmem_cache_remote_free->lock is
 *   used to protect access to this queue.
 *
 *   When the remotely freed queue reaches a watermark, a flag is set to tell
 *   the owner CPU to check it. The owner CPU will then check the queue on the
 *   next allocation that misses the object list. It will move all objects from
 *   this list onto the object list and then allocate one.
 *
 *   This system of remote queueing is intended to reduce lock and remote
 *   cacheline acquisitions, and give a cooling off period for remotely freed
 *   objects before they are re-allocated.
 *
 * node specific allocations from somewhere other than the local node are
 * handled by a per-node list which is the same as the above per-CPU data
 * structures except for the following differences:
 *
 * - kmem_cache_node->list_lock is used to protect access for multiple CPUs to
 *   allocate from a given node.
 *
 * - There is no remote free queue. Nodes don't free objects, CPUs do.
 */

static inline void slqb_stat_inc(struct kmem_cache_list *list,
				enum stat_item si)
{
#ifdef CONFIG_SLQB_STATS
	list->stats[si]++;
#endif
}

static inline void slqb_stat_add(struct kmem_cache_list *list,
				enum stat_item si, unsigned long nr)
{
#ifdef CONFIG_SLQB_STATS
	list->stats[si] += nr;
#endif
}

static inline int slqb_page_to_nid(struct slqb_page *page)
{
	return page_to_nid(&page->page);
}

static inline void *slqb_page_address(struct slqb_page *page)
{
	return page_address(&page->page);
}

static inline struct zone *slqb_page_zone(struct slqb_page *page)
{
	return page_zone(&page->page);
}

static inline int virt_to_nid(const void *addr)
{
	return page_to_nid(virt_to_page(addr));
}

static inline struct slqb_page *virt_to_head_slqb_page(const void *addr)
{
	struct page *p;

	p = virt_to_head_page(addr);
	return (struct slqb_page *)p;
}

static inline void __free_slqb_pages(struct slqb_page *page, unsigned int order,
					int pages)
{
	struct page *p = &page->page;

	reset_page_mapcount(p);
	p->mapping = NULL;
	VM_BUG_ON(!(p->flags & PG_SLQB_BIT));
	p->flags &= ~PG_SLQB_BIT;

	if (current->reclaim_state)
		current->reclaim_state->reclaimed_slab += pages;
	__free_pages(p, order);
}

#ifdef CONFIG_SLQB_DEBUG
static inline int slab_debug(struct kmem_cache *s)
{
	return s->flags &
			(SLAB_DEBUG_FREE |
			 SLAB_RED_ZONE |
			 SLAB_POISON |
			 SLAB_STORE_USER |
			 SLAB_TRACE);
}
static inline int slab_poison(struct kmem_cache *s)
{
	return s->flags & SLAB_POISON;
}
#else
static inline int slab_debug(struct kmem_cache *s)
{
	return 0;
}
static inline int slab_poison(struct kmem_cache *s)
{
	return 0;
}
#endif

#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \
				SLAB_POISON | SLAB_STORE_USER)

/* Internal SLQB flags */
#define __OBJECT_POISON		0x80000000 /* Poison object */

/* Not all arches define cache_line_size */
#ifndef cache_line_size
#define cache_line_size()	L1_CACHE_BYTES
#endif

#ifdef CONFIG_SMP
static struct notifier_block slab_notifier;
#endif

/*
 * slqb_lock protects slab_caches list and serialises hotplug operations.
 * hotplug operations take lock for write, other operations can hold off
 * hotplug by taking it for read (or write).
 */
static DECLARE_RWSEM(slqb_lock);

/*
 * A list of all slab caches on the system
 */
static LIST_HEAD(slab_caches);

/*
 * Tracking user of a slab.
 */
struct track {
	unsigned long addr;	/* Called from address */
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
};

enum track_item { TRACK_ALLOC, TRACK_FREE };

static struct kmem_cache kmem_cache_cache;

#ifdef CONFIG_SLQB_SYSFS
static int sysfs_slab_add(struct kmem_cache *s);
static void sysfs_slab_remove(struct kmem_cache *s);
#else
static inline int sysfs_slab_add(struct kmem_cache *s)
{
	return 0;
}
static inline void sysfs_slab_remove(struct kmem_cache *s)
{
	kmem_cache_free(&kmem_cache_cache, s);
}
#endif

/********************************************************************
 * 			Core slab cache functions
 *******************************************************************/

static int __slab_is_available __read_mostly;
int slab_is_available(void)
{
	return __slab_is_available;
}

static inline struct kmem_cache_cpu *get_cpu_slab(struct kmem_cache *s, int cpu)
{
#ifdef CONFIG_SMP
	VM_BUG_ON(!s->cpu_slab[cpu]);
	return s->cpu_slab[cpu];
#else
	return &s->cpu_slab;
#endif
}

static inline int check_valid_pointer(struct kmem_cache *s,
				struct slqb_page *page, const void *object)
{
	void *base;

	base = slqb_page_address(page);
	if (object < base || object >= base + s->objects * s->size ||
		(object - base) % s->size) {
		return 0;
	}

	return 1;
}

static inline void *get_freepointer(struct kmem_cache *s, void *object)
{
	return *(void **)(object + s->offset);
}

static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
{
	*(void **)(object + s->offset) = fp;
}

/* Loop over all objects in a slab */
#define for_each_object(__p, __s, __addr) \
	for (__p = (__addr); __p < (__addr) + (__s)->objects * (__s)->size;\
			__p += (__s)->size)

/* Scan freelist */
#define for_each_free_object(__p, __s, __free) \
	for (__p = (__free); (__p) != NULL; __p = get_freepointer((__s),\
		__p))

#ifdef CONFIG_SLQB_DEBUG
/*
 * Debug settings:
 */
#ifdef CONFIG_SLQB_DEBUG_ON
static int slqb_debug __read_mostly = DEBUG_DEFAULT_FLAGS;
#else
static int slqb_debug __read_mostly;
#endif

static char *slqb_debug_slabs;

/*
 * Object debugging
 */
static void print_section(char *text, u8 *addr, unsigned int length)
{
	int i, offset;
	int newline = 1;
	char ascii[17];

	ascii[16] = 0;

	for (i = 0; i < length; i++) {
		if (newline) {
			printk(KERN_ERR "%8s 0x%p: ", text, addr + i);
			newline = 0;
		}
		printk(KERN_CONT " %02x", addr[i]);
		offset = i % 16;
		ascii[offset] = isgraph(addr[i]) ? addr[i] : '.';
		if (offset == 15) {
			printk(KERN_CONT " %s\n", ascii);
			newline = 1;
		}
	}
	if (!newline) {
		i %= 16;
		while (i < 16) {
			printk(KERN_CONT "   ");
			ascii[i] = ' ';
			i++;
		}
		printk(KERN_CONT " %s\n", ascii);
	}
}

static struct track *get_track(struct kmem_cache *s, void *object,
	enum track_item alloc)
{
	struct track *p;

	if (s->offset)
		p = object + s->offset + sizeof(void *);
	else
		p = object + s->inuse;

	return p + alloc;
}

static void set_track(struct kmem_cache *s, void *object,
				enum track_item alloc, unsigned long addr)
{
	struct track *p;

	if (s->offset)
		p = object + s->offset + sizeof(void *);
	else
		p = object + s->inuse;

	p += alloc;
	if (addr) {
		p->addr = addr;
		p->cpu = raw_smp_processor_id();
		p->pid = current ? current->pid : -1;
		p->when = jiffies;
	} else
		memset(p, 0, sizeof(struct track));
}

static void init_tracking(struct kmem_cache *s, void *object)
{
	if (!(s->flags & SLAB_STORE_USER))
		return;

	set_track(s, object, TRACK_FREE, 0UL);
	set_track(s, object, TRACK_ALLOC, 0UL);
}

static void print_track(const char *s, struct track *t)
{
	if (!t->addr)
		return;

	printk(KERN_ERR "INFO: %s in ", s);
	__print_symbol("%s", (unsigned long)t->addr);
	printk(" age=%lu cpu=%u pid=%d\n", jiffies - t->when, t->cpu, t->pid);
}

static void print_tracking(struct kmem_cache *s, void *object)
{
	if (!(s->flags & SLAB_STORE_USER))
		return;

	print_track("Allocated", get_track(s, object, TRACK_ALLOC));
	print_track("Freed", get_track(s, object, TRACK_FREE));
}

static void print_page_info(struct slqb_page *page)
{
	printk(KERN_ERR "INFO: Slab 0x%p used=%u fp=0x%p flags=0x%04lx\n",
		page, page->inuse, page->freelist, page->flags);

}

#define MAX_ERR_STR 100
static void slab_bug(struct kmem_cache *s, char *fmt, ...)
{
	va_list args;
	char buf[MAX_ERR_STR];

	va_start(args, fmt);
	vsnprintf(buf, sizeof(buf), fmt, args);
	va_end(args);
	printk(KERN_ERR "========================================"
			"=====================================\n");
	printk(KERN_ERR "BUG %s: %s\n", s->name, buf);
	printk(KERN_ERR "----------------------------------------"
			"-------------------------------------\n\n");
}

static void slab_fix(struct kmem_cache *s, char *fmt, ...)
{
	va_list args;
	char buf[100];

	va_start(args, fmt);
	vsnprintf(buf, sizeof(buf), fmt, args);
	va_end(args);
	printk(KERN_ERR "FIX %s: %s\n", s->name, buf);
}

static void print_trailer(struct kmem_cache *s, struct slqb_page *page, u8 *p)
{
	unsigned int off;	/* Offset of last byte */
	u8 *addr = slqb_page_address(page);

	print_tracking(s, p);

	print_page_info(page);

	printk(KERN_ERR "INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
			p, p - addr, get_freepointer(s, p));

	if (p > addr + 16)
		print_section("Bytes b4", p - 16, 16);

	print_section("Object", p, min(s->objsize, 128));

	if (s->flags & SLAB_RED_ZONE)
		print_section("Redzone", p + s->objsize, s->inuse - s->objsize);

	if (s->offset)
		off = s->offset + sizeof(void *);
	else
		off = s->inuse;

	if (s->flags & SLAB_STORE_USER)
		off += 2 * sizeof(struct track);

	if (off != s->size) {
		/* Beginning of the filler is the free pointer */
		print_section("Padding", p + off, s->size - off);
	}

	dump_stack();
}

static void object_err(struct kmem_cache *s, struct slqb_page *page,
			u8 *object, char *reason)
{
	slab_bug(s, reason);
	print_trailer(s, page, object);
}

static void slab_err(struct kmem_cache *s, struct slqb_page *page,
			char *fmt, ...)
{
	slab_bug(s, fmt);
	print_page_info(page);
	dump_stack();
}

static void init_object(struct kmem_cache *s, void *object, int active)
{
	u8 *p = object;

	if (s->flags & __OBJECT_POISON) {
		memset(p, POISON_FREE, s->objsize - 1);
		p[s->objsize - 1] = POISON_END;
	}

	if (s->flags & SLAB_RED_ZONE) {
		memset(p + s->objsize,
			active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE,
			s->inuse - s->objsize);
	}
}

static u8 *check_bytes(u8 *start, unsigned int value, unsigned int bytes)
{
	while (bytes) {
		if (*start != (u8)value)
			return start;
		start++;
		bytes--;
	}
	return NULL;
}

static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
				void *from, void *to)
{
	slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
	memset(from, data, to - from);
}

static int check_bytes_and_report(struct kmem_cache *s, struct slqb_page *page,
			u8 *object, char *what,
			u8 *start, unsigned int value, unsigned int bytes)
{
	u8 *fault;
	u8 *end;

	fault = check_bytes(start, value, bytes);
	if (!fault)
		return 1;

	end = start + bytes;
	while (end > fault && end[-1] == value)
		end--;

	slab_bug(s, "%s overwritten", what);
	printk(KERN_ERR "INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
					fault, end - 1, fault[0], value);
	print_trailer(s, page, object);

	restore_bytes(s, what, value, fault, end);
	return 0;
}

/*
 * Object layout:
 *
 * object address
 * 	Bytes of the object to be managed.
 * 	If the freepointer may overlay the object then the free
 * 	pointer is the first word of the object.
 *
 * 	Poisoning uses 0x6b (POISON_FREE) and the last byte is
 * 	0xa5 (POISON_END)
 *
 * object + s->objsize
 * 	Padding to reach word boundary. This is also used for Redzoning.
 * 	Padding is extended by another word if Redzoning is enabled and
 * 	objsize == inuse.
 *
 * 	We fill with 0xbb (RED_INACTIVE) for inactive objects and with
 * 	0xcc (RED_ACTIVE) for objects in use.
 *
 * object + s->inuse
 * 	Meta data starts here.
 *
 * 	A. Free pointer (if we cannot overwrite object on free)
 * 	B. Tracking data for SLAB_STORE_USER
 * 	C. Padding to reach required alignment boundary or at mininum
 * 		one word if debuggin is on to be able to detect writes
 * 		before the word boundary.
 *
 *	Padding is done using 0x5a (POISON_INUSE)
 *
 * object + s->size
 * 	Nothing is used beyond s->size.
 */

static int check_pad_bytes(struct kmem_cache *s, struct slqb_page *page, u8 *p)
{
	unsigned long off = s->inuse;	/* The end of info */

	if (s->offset) {
		/* Freepointer is placed after the object. */
		off += sizeof(void *);
	}

	if (s->flags & SLAB_STORE_USER) {
		/* We also have user information there */
		off += 2 * sizeof(struct track);
	}

	if (s->size == off)
		return 1;

	return check_bytes_and_report(s, page, p, "Object padding",
				p + off, POISON_INUSE, s->size - off);
}

static int slab_pad_check(struct kmem_cache *s, struct slqb_page *page)
{
	u8 *start;
	u8 *fault;
	u8 *end;
	int length;
	int remainder;

	if (!(s->flags & SLAB_POISON))
		return 1;

	start = slqb_page_address(page);
	end = start + (PAGE_SIZE << s->order);
	length = s->objects * s->size;
	remainder = end - (start + length);
	if (!remainder)
		return 1;

	fault = check_bytes(start + length, POISON_INUSE, remainder);
	if (!fault)
		return 1;

	while (end > fault && end[-1] == POISON_INUSE)
		end--;

	slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
	print_section("Padding", start, length);

	restore_bytes(s, "slab padding", POISON_INUSE, start, end);
	return 0;
}

static int check_object(struct kmem_cache *s, struct slqb_page *page,
					void *object, int active)
{
	u8 *p = object;
	u8 *endobject = object + s->objsize;

	if (s->flags & SLAB_RED_ZONE) {
		unsigned int red =
			active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE;

		if (!check_bytes_and_report(s, page, object, "Redzone",
			endobject, red, s->inuse - s->objsize))
			return 0;
	} else {
		if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) {
			check_bytes_and_report(s, page, p, "Alignment padding",
				endobject, POISON_INUSE, s->inuse - s->objsize);
		}
	}

	if (s->flags & SLAB_POISON) {
		if (!active && (s->flags & __OBJECT_POISON)) {
			if (!check_bytes_and_report(s, page, p, "Poison", p,
					POISON_FREE, s->objsize - 1))
				return 0;

			if (!check_bytes_and_report(s, page, p, "Poison",
					p + s->objsize - 1, POISON_END, 1))
				return 0;
		}

		/*
		 * check_pad_bytes cleans up on its own.
		 */
		check_pad_bytes(s, page, p);
	}

	return 1;
}

static int check_slab(struct kmem_cache *s, struct slqb_page *page)
{
	if (!(page->flags & PG_SLQB_BIT)) {
		slab_err(s, page, "Not a valid slab page");
		return 0;
	}
	if (page->inuse == 0) {
		slab_err(s, page, "inuse before free / after alloc", s->name);
		return 0;
	}
	if (page->inuse > s->objects) {
		slab_err(s, page, "inuse %u > max %u",
			s->name, page->inuse, s->objects);
		return 0;
	}
	/* Slab_pad_check fixes things up after itself */
	slab_pad_check(s, page);
	return 1;
}

static void trace(struct kmem_cache *s, struct slqb_page *page,
			void *object, int alloc)
{
	if (s->flags & SLAB_TRACE) {
		printk(KERN_INFO "TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
			s->name,
			alloc ? "alloc" : "free",
			object, page->inuse,
			page->freelist);

		if (!alloc)
			print_section("Object", (void *)object, s->objsize);

		dump_stack();
	}
}

static void setup_object_debug(struct kmem_cache *s, struct slqb_page *page,
				void *object)
{
	if (!slab_debug(s))
		return;

	if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
		return;

	init_object(s, object, 0);
	init_tracking(s, object);
}

static int alloc_debug_processing(struct kmem_cache *s,
					void *object, unsigned long addr)
{
	struct slqb_page *page;
	page = virt_to_head_slqb_page(object);

	if (!check_slab(s, page))
		goto bad;

	if (!check_valid_pointer(s, page, object)) {
		object_err(s, page, object, "Freelist Pointer check fails");
		goto bad;
	}

	if (object && !check_object(s, page, object, 0))
		goto bad;

	/* Success perform special debug activities for allocs */
	if (s->flags & SLAB_STORE_USER)
		set_track(s, object, TRACK_ALLOC, addr);
	trace(s, page, object, 1);
	init_object(s, object, 1);
	return 1;

bad:
	return 0;
}

static int free_debug_processing(struct kmem_cache *s,
					void *object, unsigned long addr)
{
	struct slqb_page *page;
	page = virt_to_head_slqb_page(object);

	if (!check_slab(s, page))
		goto fail;

	if (!check_valid_pointer(s, page, object)) {
		slab_err(s, page, "Invalid object pointer 0x%p", object);
		goto fail;
	}

	if (!check_object(s, page, object, 1))
		return 0;

	/* Special debug activities for freeing objects */
	if (s->flags & SLAB_STORE_USER)
		set_track(s, object, TRACK_FREE, addr);
	trace(s, page, object, 0);
	init_object(s, object, 0);
	return 1;

fail:
	slab_fix(s, "Object at 0x%p not freed", object);
	return 0;
}

static int __init setup_slqb_debug(char *str)
{
	slqb_debug = DEBUG_DEFAULT_FLAGS;
	if (*str++ != '=' || !*str) {
		/*
		 * No options specified. Switch on full debugging.
		 */
		goto out;
	}

	if (*str == ',') {
		/*
		 * No options but restriction on slabs. This means full
		 * debugging for slabs matching a pattern.
		 */
		goto check_slabs;
	}

	slqb_debug = 0;
	if (*str == '-') {
		/*
		 * Switch off all debugging measures.
		 */
		goto out;
	}

	/*
	 * Determine which debug features should be switched on
	 */
	for (; *str && *str != ','; str++) {
		switch (tolower(*str)) {
		case 'f':
			slqb_debug |= SLAB_DEBUG_FREE;
			break;
		case 'z':
			slqb_debug |= SLAB_RED_ZONE;
			break;
		case 'p':
			slqb_debug |= SLAB_POISON;
			break;
		case 'u':
			slqb_debug |= SLAB_STORE_USER;
			break;
		case 't':
			slqb_debug |= SLAB_TRACE;
			break;
		case 'a':
			slqb_debug |= SLAB_FAILSLAB;
			break;
		default:
			printk(KERN_ERR "slqb_debug option '%c' "
				"unknown. skipped\n", *str);
		}
	}

check_slabs:
	if (*str == ',')
		slqb_debug_slabs = str + 1;
out:
	return 1;
}
__setup("slqb_debug", setup_slqb_debug);

static int __init setup_slqb_min_order(char *str)
{
	get_option(&str, &slqb_min_order);
	slqb_min_order = min(slqb_min_order, MAX_ORDER - 1);

	return 1;
}
__setup("slqb_min_order=", setup_slqb_min_order);

static int __init setup_slqb_min_objects(char *str)
{
	get_option(&str, &slqb_min_objects);

	return 1;
}

__setup("slqb_min_objects=", setup_slqb_min_objects);

static unsigned long kmem_cache_flags(unsigned long objsize,
				unsigned long flags, const char *name,
				void (*ctor)(void *))
{
	/*
	 * Enable debugging if selected on the kernel commandline.
	 */
	if (slqb_debug && (!slqb_debug_slabs ||
	    strncmp(slqb_debug_slabs, name,
		strlen(slqb_debug_slabs)) == 0))
			flags |= slqb_debug;

	if (num_possible_nodes() > 1)
		flags |= SLAB_NUMA;

	return flags;
}
#else
static inline void setup_object_debug(struct kmem_cache *s,
			struct slqb_page *page, void *object)
{
}

static inline int alloc_debug_processing(struct kmem_cache *s,
			void *object, unsigned long addr)
{
	return 0;
}

static inline int free_debug_processing(struct kmem_cache *s,
			void *object, unsigned long addr)
{
	return 0;
}

static inline int slab_pad_check(struct kmem_cache *s, struct slqb_page *page)
{
	return 1;
}

static inline int check_object(struct kmem_cache *s, struct slqb_page *page,
			void *object, int active)
{
	return 1;
}

static inline void add_full(struct kmem_cache_node *n, struct slqb_page *page)
{
}

static inline unsigned long kmem_cache_flags(unsigned long objsize,
	unsigned long flags, const char *name, void (*ctor)(void *))
{
	if (num_possible_nodes() > 1)
		flags |= SLAB_NUMA;
	return flags;
}

static const int slqb_debug;
#endif

/*
 * allocate a new slab (return its corresponding struct slqb_page)
 */
static struct slqb_page *allocate_slab(struct kmem_cache *s,
					gfp_t flags, int node)
{
	struct slqb_page *page;
	int pages = 1 << s->order;

	flags |= s->allocflags;

	page = (struct slqb_page *)alloc_pages_node(node, flags, s->order);
	if (!page)
		return NULL;

	mod_zone_page_state(slqb_page_zone(page),
		(s->flags & SLAB_RECLAIM_ACCOUNT) ?
		NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
		pages);

	return page;
}

/*
 * Called once for each object on a new slab page
 */
static void setup_object(struct kmem_cache *s,
				struct slqb_page *page, void *object)
{
	setup_object_debug(s, page, object);
	if (unlikely(s->ctor))
		s->ctor(object);
}

/*
 * Allocate a new slab, set up its object list.
 */
static struct slqb_page *new_slab_page(struct kmem_cache *s,
				gfp_t flags, int node, unsigned int colour)
{
	struct slqb_page *page;
	void *start;
	void *last;
	void *p;

	BUG_ON(flags & GFP_SLAB_BUG_MASK);

	page = allocate_slab(s,
		flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
	if (!page)
		goto out;

	page->flags |= PG_SLQB_BIT;

	start = page_address(&page->page);

	if (unlikely(slab_poison(s)))
		memset(start, POISON_INUSE, PAGE_SIZE << s->order);

	start += colour;

	last = start;
	for_each_object(p, s, start) {
		setup_object(s, page, p);
		set_freepointer(s, last, p);
		last = p;
	}
	set_freepointer(s, last, NULL);

	page->freelist = start;
	page->inuse = 0;
out:
	return page;
}

/*
 * Free a slab page back to the page allocator
 */
static void __free_slab(struct kmem_cache *s, struct slqb_page *page)
{
	int pages = 1 << s->order;

	if (unlikely(slab_debug(s))) {
		void *p;

		slab_pad_check(s, page);
		for_each_free_object(p, s, page->freelist)
			check_object(s, page, p, 0);
	}

	mod_zone_page_state(slqb_page_zone(page),
		(s->flags & SLAB_RECLAIM_ACCOUNT) ?
		NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
		-pages);

	__free_slqb_pages(page, s->order, pages);
}

static void rcu_free_slab(struct rcu_head *h)
{
	struct slqb_page *page;

	page = container_of(h, struct slqb_page, rcu_head);
	__free_slab(page->list->cache, page);
}

static void free_slab(struct kmem_cache *s, struct slqb_page *page)
{
	VM_BUG_ON(page->inuse);
	if (unlikely(s->flags & SLAB_DESTROY_BY_RCU))
		call_rcu(&page->rcu_head, rcu_free_slab);
	else
		__free_slab(s, page);
}

/*
 * Return an object to its slab.
 *
 * Caller must be the owner CPU in the case of per-CPU list, or hold the node's
 * list_lock in the case of per-node list.
 */
static int free_object_to_page(struct kmem_cache *s,
			struct kmem_cache_list *l, struct slqb_page *page,
			void *object)
{
	VM_BUG_ON(page->list != l);

	set_freepointer(s, object, page->freelist);
	page->freelist = object;
	page->inuse--;

	if (!page->inuse) {
		if (likely(s->objects > 1)) {
			l->nr_partial--;
			list_del(&page->lru);
		}
		l->nr_slabs--;
		free_slab(s, page);
		slqb_stat_inc(l, FLUSH_SLAB_FREE);
		return 1;

	} else if (page->inuse + 1 == s->objects) {
		l->nr_partial++;
		list_add(&page->lru, &l->partial);
		slqb_stat_inc(l, FLUSH_SLAB_PARTIAL);
		return 0;
	}
	return 0;
}

#ifdef CONFIG_SMP
static void slab_free_to_remote(struct kmem_cache *s, struct slqb_page *page,
				void *object, struct kmem_cache_cpu *c);
#endif

/*
 * Flush the LIFO list of objects on a list. They are sent back to their pages
 * in case the pages also belong to the list, or to our CPU's remote-free list
 * in the case they do not.
 *
 * Doesn't flush the entire list. flush_free_list_all does.
 *
 * Caller must be the owner CPU in the case of per-CPU list, or hold the node's
 * list_lock in the case of per-node list.
 */
static void flush_free_list(struct kmem_cache *s, struct kmem_cache_list *l)
{
	void **head;
	int nr;
	int locked = 0;

	nr = l->freelist.nr;
	if (unlikely(!nr))
		return;

	nr = min(slab_freebatch(s), nr);

	slqb_stat_inc(l, FLUSH_FREE_LIST);
	slqb_stat_add(l, FLUSH_FREE_LIST_OBJECTS, nr);

	l->freelist.nr -= nr;
	head = l->freelist.head;

	do {
		struct slqb_page *page;
		void **object;

		object = head;
		VM_BUG_ON(!object);
		head = get_freepointer(s, object);
		page = virt_to_head_slqb_page(object);

#ifdef CONFIG_SMP
		if (page->list != l) {
			struct kmem_cache_cpu *c;

			if (locked) {
				spin_unlock(&l->page_lock);
				locked = 0;
			}

			c = get_cpu_slab(s, smp_processor_id());

			slab_free_to_remote(s, page, object, c);
			slqb_stat_inc(l, FLUSH_FREE_LIST_REMOTE);
		} else
#endif
		{
			if (!locked) {
				spin_lock(&l->page_lock);
				locked = 1;
			}
			free_object_to_page(s, l, page, object);
		}

		nr--;
	} while (nr);

	if (locked)
		spin_unlock(&l->page_lock);

	l->freelist.head = head;
	if (!l->freelist.nr)
		l->freelist.tail = NULL;
}

static void flush_free_list_all(struct kmem_cache *s, struct kmem_cache_list *l)
{
	while (l->freelist.nr)
		flush_free_list(s, l);
}

#ifdef CONFIG_SMP
/*
 * If enough objects have been remotely freed back to this list,
 * remote_free_check will be set. In which case, we'll eventually come here
 * to take those objects off our remote_free list and onto our LIFO freelist.
 *
 * Caller must be the owner CPU in the case of per-CPU list, or hold the node's
 * list_lock in the case of per-node list.
 */
static void claim_remote_free_list(struct kmem_cache *s,
					struct kmem_cache_list *l)
{
	void **head, **tail;
	int nr;

	if (!l->remote_free.list.nr)
		return;

	spin_lock(&l->remote_free.lock);

	l->remote_free_check = 0;
	head = l->remote_free.list.head;
	l->remote_free.list.head = NULL;
	tail = l->remote_free.list.tail;
	l->remote_free.list.tail = NULL;
	nr = l->remote_free.list.nr;
	l->remote_free.list.nr = 0;

	spin_unlock(&l->remote_free.lock);

	VM_BUG_ON(!nr);

	if (!l->freelist.nr) {
		/* Get head hot for likely subsequent allocation or flush */
		prefetchw(head);
		l->freelist.head = head;
	} else
		set_freepointer(s, l->freelist.tail, head);
	l->freelist.tail = tail;

	l->freelist.nr += nr;

	slqb_stat_inc(l, CLAIM_REMOTE_LIST);
	slqb_stat_add(l, CLAIM_REMOTE_LIST_OBJECTS, nr);
}
#else
static inline void claim_remote_free_list(struct kmem_cache *s,
					struct kmem_cache_list *l)
{
}
#endif

/*
 * Allocation fastpath. Get an object from the list's LIFO freelist, or
 * return NULL if it is empty.
 *
 * Caller must be the owner CPU in the case of per-CPU list, or hold the node's
 * list_lock in the case of per-node list.
 */
static __always_inline void *__cache_list_get_object(struct kmem_cache *s,
						struct kmem_cache_list *l)
{
	void *object;

	object = l->freelist.head;
	if (likely(object)) {
		void *next = get_freepointer(s, object);

		VM_BUG_ON(!l->freelist.nr);
		l->freelist.nr--;
		l->freelist.head = next;

		return object;
	}
	VM_BUG_ON(l->freelist.nr);

#ifdef CONFIG_SMP
	if (unlikely(l->remote_free_check)) {
		claim_remote_free_list(s, l);

		if (l->freelist.nr > slab_hiwater(s))
			flush_free_list(s, l);

		/* repetition here helps gcc :( */
		object = l->freelist.head;
		if (likely(object)) {
			void *next = get_freepointer(s, object);

			VM_BUG_ON(!l->freelist.nr);
			l->freelist.nr--;
			l->freelist.head = next;

			return object;
		}
		VM_BUG_ON(l->freelist.nr);
	}
#endif

	return NULL;
}

/*
 * Slow(er) path. Get a page from this list's existing pages. Will be a
 * new empty page in the case that __slab_alloc_page has just been called
 * (empty pages otherwise never get queued up on the lists), or a partial page
 * already on the list.
 *
 * Caller must be the owner CPU in the case of per-CPU list, or hold the node's
 * list_lock in the case of per-node list.
 */
static noinline void *__cache_list_get_page(struct kmem_cache *s,
				struct kmem_cache_list *l)
{
	struct slqb_page *page;
	void *object;

	if (unlikely(!l->nr_partial))
		return NULL;

	page = list_first_entry(&l->partial, struct slqb_page, lru);
	VM_BUG_ON(page->inuse == s->objects);
	if (page->inuse + 1 == s->objects) {
		l->nr_partial--;
		list_del(&page->lru);
	}

	VM_BUG_ON(!page->freelist);

	page->inuse++;

	object = page->freelist;
	page->freelist = get_freepointer(s, object);
	if (page->freelist)
		prefetchw(page->freelist);
	VM_BUG_ON((page->inuse == s->objects) != (page->freelist == NULL));
	slqb_stat_inc(l, ALLOC_SLAB_FILL);

	return object;
}

static void *cache_list_get_page(struct kmem_cache *s,
				struct kmem_cache_list *l)
{
	void *object;

	if (unlikely(!l->nr_partial))
		return NULL;

	spin_lock(&l->page_lock);
	object = __cache_list_get_page(s, l);
	spin_unlock(&l->page_lock);

	return object;
}

/*
 * Allocation slowpath. Allocate a new slab page from the page allocator, and
 * put it on the list's partial list. Must be followed by an allocation so
 * that we don't have dangling empty pages on the partial list.
 *
 * Returns 0 on allocation failure.
 *
 * Must be called with interrupts disabled.
 */
static noinline void *__slab_alloc_page(struct kmem_cache *s,
				gfp_t gfpflags, int node)
{
	struct slqb_page *page;
	struct kmem_cache_list *l;
	struct kmem_cache_cpu *c;
	unsigned int colour;
	void *object;

	c = get_cpu_slab(s, smp_processor_id());
	colour = c->colour_next;
	c->colour_next += s->colour_off;
	if (c->colour_next >= s->colour_range)
		c->colour_next = 0;

	/* Caller handles __GFP_ZERO */
	gfpflags &= ~__GFP_ZERO;

	if (gfpflags & __GFP_WAIT)
		local_irq_enable();
	page = new_slab_page(s, gfpflags, node, colour);
	if (gfpflags & __GFP_WAIT)
		local_irq_disable();
	if (unlikely(!page))
		return page;

	if (!NUMA_BUILD || likely(slqb_page_to_nid(page) == numa_node_id())) {
		struct kmem_cache_cpu *c;
		int cpu = smp_processor_id();

		c = get_cpu_slab(s, cpu);
		l = &c->list;
		page->list = l;

		spin_lock(&l->page_lock);
		l->nr_slabs++;
		l->nr_partial++;
		list_add(&page->lru, &l->partial);
		slqb_stat_inc(l, ALLOC);
		slqb_stat_inc(l, ALLOC_SLAB_NEW);
		object = __cache_list_get_page(s, l);
		spin_unlock(&l->page_lock);
	} else {
#ifdef CONFIG_NUMA
		struct kmem_cache_node *n;

		n = s->node_slab[slqb_page_to_nid(page)];
		l = &n->list;
		page->list = l;

		spin_lock(&n->list_lock);
		spin_lock(&l->page_lock);
		l->nr_slabs++;
		l->nr_partial++;
		list_add(&page->lru, &l->partial);
		slqb_stat_inc(l, ALLOC);
		slqb_stat_inc(l, ALLOC_SLAB_NEW);
		object = __cache_list_get_page(s, l);
		spin_unlock(&l->page_lock);
		spin_unlock(&n->list_lock);
#endif
	}
	VM_BUG_ON(!object);
	return object;
}

#ifdef CONFIG_NUMA
static noinline int alternate_nid(struct kmem_cache *s,
				gfp_t gfpflags, int node)
{
	if (in_interrupt() || (gfpflags & __GFP_THISNODE))
		return node;
	if (cpuset_do_slab_mem_spread() && (s->flags & SLAB_MEM_SPREAD))
		return cpuset_mem_spread_node();
	else if (current->mempolicy)
		return slab_node(current->mempolicy);
	return node;
}

/*
 * Allocate an object from a remote node. Return NULL if none could be found
 * (in which case, caller should allocate a new slab)
 *
 * Must be called with interrupts disabled.
 */
static void *__remote_slab_alloc_node(struct kmem_cache *s,
				gfp_t gfpflags, int node)
{
	struct kmem_cache_node *n;
	struct kmem_cache_list *l;
	void *object;

	n = s->node_slab[node];
	if (unlikely(!n)) /* node has no memory */
		return NULL;
	l = &n->list;

	spin_lock(&n->list_lock);

	object = __cache_list_get_object(s, l);
	if (unlikely(!object)) {
		object = cache_list_get_page(s, l);
		if (unlikely(!object)) {
			spin_unlock(&n->list_lock);
			return __slab_alloc_page(s, gfpflags, node);
		}
	}
	if (likely(object))
		slqb_stat_inc(l, ALLOC);
	spin_unlock(&n->list_lock);
	return object;
}

static noinline void *__remote_slab_alloc(struct kmem_cache *s,
				gfp_t gfpflags, int node)
{
	void *object;
	struct zonelist *zonelist;
	struct zoneref *z;
	struct zone *zone;
	enum zone_type high_zoneidx = gfp_zone(gfpflags);

	object = __remote_slab_alloc_node(s, gfpflags, node);
	if (likely(object || (gfpflags & __GFP_THISNODE)))
		return object;

	zonelist = node_zonelist(slab_node(current->mempolicy), gfpflags);
	for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
		if (!cpuset_zone_allowed_hardwall(zone, gfpflags))
			continue;

		node = zone_to_nid(zone);
		object = __remote_slab_alloc_node(s, gfpflags, node);
		if (likely(object))
			return object;
	}
	return NULL;
}
#endif

/*
 * Main allocation path. Return an object, or NULL on allocation failure.
 *
 * Must be called with interrupts disabled.
 */
static __always_inline void *__slab_alloc(struct kmem_cache *s,
				gfp_t gfpflags, int node)
{
	void *object;
	struct kmem_cache_cpu *c;
	struct kmem_cache_list *l;

#ifdef CONFIG_NUMA
	if (unlikely(node != -1) && unlikely(node != numa_node_id())) {
try_remote:
		return __remote_slab_alloc(s, gfpflags, node);
	}
#endif

	c = get_cpu_slab(s, smp_processor_id());
	VM_BUG_ON(!c);
	l = &c->list;
	object = __cache_list_get_object(s, l);
	if (unlikely(!object)) {
#ifdef CONFIG_NUMA
		int thisnode = numa_node_id();

		/*
		 * If the local node is memoryless, try remote alloc before
		 * trying the page allocator. Otherwise, what happens is
		 * objects are always freed to remote lists but the allocation
		 * side always allocates a new page with only one object
		 * used in each page
		 */
		if (unlikely(!node_state(thisnode, N_HIGH_MEMORY)))
			object = __remote_slab_alloc(s, gfpflags, thisnode);
#endif

		if (!object) {
			object = cache_list_get_page(s, l);
			if (unlikely(!object)) {
				object = __slab_alloc_page(s, gfpflags, node);
#ifdef CONFIG_NUMA
				if (unlikely(!object)) {
					node = numa_node_id();
					goto try_remote;
				}
#endif
				return object;
			}
		}
	}
	if (likely(object))
		slqb_stat_inc(l, ALLOC);
	return object;
}

/*
 * Perform some interrupts-on processing around the main allocation path
 * (debug checking and memset()ing).
 */
static __always_inline void *slab_alloc(struct kmem_cache *s,
				gfp_t gfpflags, int node, unsigned long addr)
{
	void *object;
	unsigned long flags;

	gfpflags &= gfp_allowed_mask;

	lockdep_trace_alloc(gfpflags);
	might_sleep_if(gfpflags & __GFP_WAIT);

	if (should_failslab(s->objsize, gfpflags, s->flags))
		return NULL;

again:
	local_irq_save(flags);
	object = __slab_alloc(s, gfpflags, node);
	local_irq_restore(flags);

	if (unlikely(slab_debug(s)) && likely(object)) {
		if (unlikely(!alloc_debug_processing(s, object, addr)))
			goto again;
	}

	if (unlikely(gfpflags & __GFP_ZERO) && likely(object))
		memset(object, 0, s->objsize);

	return object;
}

static __always_inline void *__kmem_cache_alloc(struct kmem_cache *s,
				gfp_t gfpflags, unsigned long caller)
{
	int node = -1;

#ifdef CONFIG_NUMA
	if (unlikely(current->flags & (PF_SPREAD_SLAB | PF_MEMPOLICY)))
		node = alternate_nid(s, gfpflags, node);
#endif
	return slab_alloc(s, gfpflags, node, caller);
}

void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
{
	return __kmem_cache_alloc(s, gfpflags, _RET_IP_);
}
EXPORT_SYMBOL(kmem_cache_alloc);

#ifdef CONFIG_NUMA
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
{
	return slab_alloc(s, gfpflags, node, _RET_IP_);
}
EXPORT_SYMBOL(kmem_cache_alloc_node);
#endif

#ifdef CONFIG_SMP
/*
 * Flush this CPU's remote free list of objects back to the list from where
 * they originate. They end up on that list's remotely freed list, and
 * eventually we set it's remote_free_check if there are enough objects on it.
 *
 * This seems convoluted, but it keeps is from stomping on the target CPU's
 * fastpath cachelines.
 *
 * Must be called with interrupts disabled.
 */
static void flush_remote_free_cache(struct kmem_cache *s,
				struct kmem_cache_cpu *c)
{
	struct kmlist *src;
	struct kmem_cache_list *dst;
	unsigned int nr;
	int set;

	src = &c->rlist;
	nr = src->nr;
	if (unlikely(!nr))
		return;

#ifdef CONFIG_SLQB_STATS
	{
		struct kmem_cache_list *l = &c->list;

		slqb_stat_inc(l, FLUSH_RFREE_LIST);
		slqb_stat_add(l, FLUSH_RFREE_LIST_OBJECTS, nr);
	}
#endif

	dst = c->remote_cache_list;

	/*
	 * Less common case, dst is filling up so free synchronously.
	 * No point in having remote CPU free thse as it will just
	 * free them back to the page list anyway.
	 */
	if (unlikely(dst->remote_free.list.nr > (slab_hiwater(s) >> 1))) {
		void **head;

		head = src->head;
		spin_lock(&dst->page_lock);
		do {
			struct slqb_page *page;
			void **object;

			object = head;
			VM_BUG_ON(!object);
			head = get_freepointer(s, object);
			page = virt_to_head_slqb_page(object);

			free_object_to_page(s, dst, page, object);
			nr--;
		} while (nr);
		spin_unlock(&dst->page_lock);

		src->head = NULL;
		src->tail = NULL;
		src->nr = 0;

		return;
	}

	spin_lock(&dst->remote_free.lock);

	if (!dst->remote_free.list.head)
		dst->remote_free.list.head = src->head;
	else
		set_freepointer(s, dst->remote_free.list.tail, src->head);
	dst->remote_free.list.tail = src->tail;

	src->head = NULL;
	src->tail = NULL;
	src->nr = 0;

	if (dst->remote_free.list.nr < slab_freebatch(s))
		set = 1;
	else
		set = 0;

	dst->remote_free.list.nr += nr;

	if (unlikely(dst->remote_free.list.nr >= slab_freebatch(s) && set))
		dst->remote_free_check = 1;

	spin_unlock(&dst->remote_free.lock);
}

/*
 * Free an object to this CPU's remote free list.
 *
 * Must be called with interrupts disabled.
 */
static noinline void slab_free_to_remote(struct kmem_cache *s,
				struct slqb_page *page, void *object,
				struct kmem_cache_cpu *c)
{
	struct kmlist *r;

	/*
	 * Our remote free list corresponds to a different list. Must
	 * flush it and switch.
	 */
	if (page->list != c->remote_cache_list) {
		flush_remote_free_cache(s, c);
		c->remote_cache_list = page->list;
	}

	r = &c->rlist;
	if (!r->head)
		r->head = object;
	else
		set_freepointer(s, r->tail, object);
	set_freepointer(s, object, NULL);
	r->tail = object;
	r->nr++;

	if (unlikely(r->nr >= slab_freebatch(s)))
		flush_remote_free_cache(s, c);
}
#endif

/*
 * Main freeing path. Return an object, or NULL on allocation failure.
 *
 * Must be called with interrupts disabled.
 */
static __always_inline void __slab_free(struct kmem_cache *s,
				struct slqb_page *page, void *object)
{
	struct kmem_cache_cpu *c;
	struct kmem_cache_list *l;
	int thiscpu = smp_processor_id();

	c = get_cpu_slab(s, thiscpu);
	l = &c->list;

	slqb_stat_inc(l, FREE);

	if (!NUMA_BUILD || !slab_numa(s) ||
			likely(slqb_page_to_nid(page) == numa_node_id())) {
		/*
		 * Freeing fastpath. Collects all local-node objects, not
		 * just those allocated from our per-CPU list. This allows
		 * fast transfer of objects from one CPU to another within
		 * a given node.
		 */
		set_freepointer(s, object, l->freelist.head);
		l->freelist.head = object;
		if (!l->freelist.nr)
			l->freelist.tail = object;
		l->freelist.nr++;

		if (unlikely(l->freelist.nr > slab_hiwater(s)))
			flush_free_list(s, l);

	} else {
#ifdef CONFIG_SMP
		/*
		 * Freeing an object that was allocated on a remote node.
		 */
		slab_free_to_remote(s, page, object, c);
		slqb_stat_inc(l, FREE_REMOTE);
#endif
	}
}

/*
 * Perform some interrupts-on processing around the main freeing path
 * (debug checking).
 */
static __always_inline void slab_free(struct kmem_cache *s,
				struct slqb_page *page, void *object)
{
	unsigned long flags;

	prefetchw(object);

	debug_check_no_locks_freed(object, s->objsize);
	if (likely(object) && unlikely(slab_debug(s))) {
		if (unlikely(!free_debug_processing(s, object, _RET_IP_)))
			return;
	}

	local_irq_save(flags);
	__slab_free(s, page, object);
	local_irq_restore(flags);
}

void kmem_cache_free(struct kmem_cache *s, void *object)
{
	struct slqb_page *page = NULL;

	if (slab_numa(s))
		page = virt_to_head_slqb_page(object);
	slab_free(s, page, object);
}
EXPORT_SYMBOL(kmem_cache_free);

/*
 * Calculate the order of allocation given an slab object size.
 *
 * Order 0 allocations are preferred since order 0 does not cause fragmentation
 * in the page allocator, and they have fastpaths in the page allocator. But
 * also minimise external fragmentation with large objects.
 */
static int slab_order(int size, int max_order, int frac)
{
	int order;

	if (fls(size - 1) <= PAGE_SHIFT)
		order = 0;
	else
		order = fls(size - 1) - PAGE_SHIFT;
	if (order < slqb_min_order)
		order = slqb_min_order;

	while (order <= max_order) {
		unsigned long slab_size = PAGE_SIZE << order;
		unsigned long objects;
		unsigned long waste;

		objects = slab_size / size;
		if (!objects)
			goto next;

		if (order < MAX_ORDER && objects < slqb_min_objects) {
			/*
			 * if we don't have enough objects for min_objects,
			 * then try the next size up. Unless we have reached
			 * our maximum possible page size.
			 */
			goto next;
		}

		waste = slab_size - (objects * size);

		if (waste * frac <= slab_size)
			break;

next:
		order++;
	}

	return order;
}

static int calculate_order(int size)
{
	int order;

	/*
	 * Attempt to find best configuration for a slab. This
	 * works by first attempting to generate a layout with
	 * the best configuration and backing off gradually.
	 */
	order = slab_order(size, 1, 4);
	if (order <= 1)
		return order;

	/*
	 * This size cannot fit in order-1. Allow bigger orders, but
	 * forget about trying to save space.
	 */
	order = slab_order(size, MAX_ORDER - 1, 0);
	if (order < MAX_ORDER)
		return order;

	return -ENOSYS;
}

/*
 * Figure out what the alignment of the objects will be.
 */
static unsigned long calculate_alignment(unsigned long flags,
				unsigned long align, unsigned long size)
{
	/*
	 * If the user wants hardware cache aligned objects then follow that
	 * suggestion if the object is sufficiently large.
	 *
	 * The hardware cache alignment cannot override the specified
	 * alignment though. If that is greater then use it.
	 */
	if (flags & SLAB_HWCACHE_ALIGN) {
		unsigned long ralign = cache_line_size();

		while (size <= ralign / 2)
			ralign /= 2;
		align = max(align, ralign);
	}

	if (align < ARCH_SLAB_MINALIGN)
		align = ARCH_SLAB_MINALIGN;

	return ALIGN(align, sizeof(void *));
}

static void init_kmem_cache_list(struct kmem_cache *s,
				struct kmem_cache_list *l)
{
	l->cache		= s;
	l->freelist.nr		= 0;
	l->freelist.head	= NULL;
	l->freelist.tail	= NULL;
	l->nr_partial		= 0;
	l->nr_slabs		= 0;
	INIT_LIST_HEAD(&l->partial);
	spin_lock_init(&l->page_lock);

#ifdef CONFIG_SMP
	l->remote_free_check	= 0;
	spin_lock_init(&l->remote_free.lock);
	l->remote_free.list.nr	= 0;
	l->remote_free.list.head = NULL;
	l->remote_free.list.tail = NULL;
#endif

#ifdef CONFIG_SLQB_STATS
	memset(l->stats, 0, sizeof(l->stats));
#endif
}

static void init_kmem_cache_cpu(struct kmem_cache *s,
				struct kmem_cache_cpu *c)
{
	init_kmem_cache_list(s, &c->list);

	c->colour_next		= 0;
#ifdef CONFIG_SMP
	c->rlist.nr		= 0;
	c->rlist.head		= NULL;
	c->rlist.tail		= NULL;
	c->remote_cache_list	= NULL;
#endif
}

#ifdef CONFIG_NUMA
static void init_kmem_cache_node(struct kmem_cache *s,
				struct kmem_cache_node *n)
{
	spin_lock_init(&n->list_lock);
	init_kmem_cache_list(s, &n->list);
}
#endif

/* Initial slabs. */
#ifdef CONFIG_SMP
static DEFINE_PER_CPU(struct kmem_cache_cpu, kmem_cache_cpus);
#endif
#ifdef CONFIG_NUMA
/* XXX: really need a DEFINE_PER_NODE for per-node data because a static
 *      array is wasteful */
static struct kmem_cache_node kmem_cache_nodes[MAX_NUMNODES];
#endif

#ifdef CONFIG_SMP
static struct kmem_cache kmem_cpu_cache;
static DEFINE_PER_CPU(struct kmem_cache_cpu, kmem_cpu_cpus);
#ifdef CONFIG_NUMA
static struct kmem_cache_node kmem_cpu_nodes[MAX_NUMNODES]; /* XXX per-nid */
#endif
#endif

#ifdef CONFIG_NUMA
static struct kmem_cache kmem_node_cache;
#ifdef CONFIG_SMP
static DEFINE_PER_CPU(struct kmem_cache_cpu, kmem_node_cpus);
#endif
static struct kmem_cache_node kmem_node_nodes[MAX_NUMNODES]; /*XXX per-nid */
#endif

#ifdef CONFIG_SMP
static struct kmem_cache_cpu *alloc_kmem_cache_cpu(struct kmem_cache *s,
				int cpu)
{
	struct kmem_cache_cpu *c;
	int node;

	node = cpu_to_node(cpu);

	c = kmem_cache_alloc_node(&kmem_cpu_cache, GFP_KERNEL, node);
	if (!c)
		return NULL;

	init_kmem_cache_cpu(s, c);
	return c;
}

static void free_kmem_cache_cpus(struct kmem_cache *s)
{
	int cpu;

	for_each_online_cpu(cpu) {
		struct kmem_cache_cpu *c;

		c = s->cpu_slab[cpu];
		if (c) {
			kmem_cache_free(&kmem_cpu_cache, c);
			s->cpu_slab[cpu] = NULL;
		}
	}
}

static int alloc_kmem_cache_cpus(struct kmem_cache *s)
{
	int cpu;

	for_each_online_cpu(cpu) {
		struct kmem_cache_cpu *c;

		c = s->cpu_slab[cpu];
		if (c)
			continue;

		c = alloc_kmem_cache_cpu(s, cpu);
		if (!c) {
			free_kmem_cache_cpus(s);
			return 0;
		}
		s->cpu_slab[cpu] = c;
	}
	return 1;
}

#else
static inline void free_kmem_cache_cpus(struct kmem_cache *s)
{
}

static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
{
	init_kmem_cache_cpu(s, &s->cpu_slab);
	return 1;
}
#endif

#ifdef CONFIG_NUMA
static void free_kmem_cache_nodes(struct kmem_cache *s)
{
	int node;

	for_each_node_state(node, N_NORMAL_MEMORY) {
		struct kmem_cache_node *n;

		n = s->node_slab[node];
		if (n) {
			kmem_cache_free(&kmem_node_cache, n);
			s->node_slab[node] = NULL;
		}
	}
}

static int alloc_kmem_cache_nodes(struct kmem_cache *s)
{
	int node;

	for_each_node_state(node, N_NORMAL_MEMORY) {
		struct kmem_cache_node *n;

		n = kmem_cache_alloc_node(&kmem_node_cache, GFP_KERNEL, node);
		if (!n) {
			free_kmem_cache_nodes(s);
			return 0;
		}
		init_kmem_cache_node(s, n);
		s->node_slab[node] = n;
	}
	return 1;
}
#else
static void free_kmem_cache_nodes(struct kmem_cache *s)
{
}

static int alloc_kmem_cache_nodes(struct kmem_cache *s)
{
	return 1;
}
#endif

/*
 * calculate_sizes() determines the order and the distribution of data within
 * a slab object.
 */
static int calculate_sizes(struct kmem_cache *s)
{
	unsigned long flags = s->flags;
	unsigned long size = s->objsize;
	unsigned long align = s->align;

	/*
	 * Determine if we can poison the object itself. If the user of
	 * the slab may touch the object after free or before allocation
	 * then we should never poison the object itself.
	 */
	if (slab_poison(s) && !(flags & SLAB_DESTROY_BY_RCU) && !s->ctor)
		s->flags |= __OBJECT_POISON;
	else
		s->flags &= ~__OBJECT_POISON;

	/*
	 * Round up object size to the next word boundary. We can only
	 * place the free pointer at word boundaries and this determines
	 * the possible location of the free pointer.
	 */
	size = ALIGN(size, sizeof(void *));

#ifdef CONFIG_SLQB_DEBUG
	/*
	 * If we are Redzoning then check if there is some space between the
	 * end of the object and the free pointer. If not then add an
	 * additional word to have some bytes to store Redzone information.
	 */
	if ((flags & SLAB_RED_ZONE) && size == s->objsize)
		size += sizeof(void *);
#endif

	/*
	 * With that we have determined the number of bytes in actual use
	 * by the object. This is the potential offset to the free pointer.
	 */
	s->inuse = size;

	if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) || s->ctor)) {
		/*
		 * Relocate free pointer after the object if it is not
		 * permitted to overwrite the first word of the object on
		 * kmem_cache_free.
		 *
		 * This is the case if we do RCU, have a constructor or
		 * destructor or are poisoning the objects.
		 */
		s->offset = size;
		size += sizeof(void *);
	}

#ifdef CONFIG_SLQB_DEBUG
	if (flags & SLAB_STORE_USER) {
		/*
		 * Need to store information about allocs and frees after
		 * the object.
		 */
		size += 2 * sizeof(struct track);
	}

	if (flags & SLAB_RED_ZONE) {
		/*
		 * Add some empty padding so that we can catch
		 * overwrites from earlier objects rather than let
		 * tracking information or the free pointer be
		 * corrupted if an user writes before the start
		 * of the object.
		 */
		size += sizeof(void *);
	}
#endif

	/*
	 * Determine the alignment based on various parameters that the
	 * user specified and the dynamic determination of cache line size
	 * on bootup.
	 */
	align = calculate_alignment(flags, align, s->objsize);

	/*
	 * SLQB stores one object immediately after another beginning from
	 * offset 0. In order to align the objects we have to simply size
	 * each object to conform to the alignment.
	 */
	size = ALIGN(size, align);
	s->size = size;
	s->order = calculate_order(size);

	if (s->order < 0)
		return 0;

	s->allocflags = 0;
	if (s->order)
		s->allocflags |= __GFP_COMP;

	if (s->flags & SLAB_CACHE_DMA)
		s->allocflags |= SLQB_DMA;

	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		s->allocflags |= __GFP_RECLAIMABLE;

	/*
	 * Determine the number of objects per slab
	 */
	s->objects = (PAGE_SIZE << s->order) / size;

	s->freebatch = max(4UL*PAGE_SIZE / size,
				min(256UL, 64*PAGE_SIZE / size));
	if (!s->freebatch)
		s->freebatch = 1;
	s->hiwater = s->freebatch << 2;

	return !!s->objects;

}

#ifdef CONFIG_SMP
/*
 * Per-cpu allocator can't be used because it always uses slab allocator,
 * and it can't do per-node allocations.
 */
static void *kmem_cache_dyn_array_alloc(int ids)
{
	size_t size = sizeof(void *) * ids;

	BUG_ON(!size);

	if (unlikely(!slab_is_available())) {
		static void *nextmem;
		static size_t nextleft;
		void *ret;

		/*
		 * Special case for setting up initial caches. These will
		 * never get freed by definition so we can do it rather
		 * simply.
		 */
		if (size > nextleft) {
			nextmem = alloc_pages_exact(size, GFP_KERNEL);
			if (!nextmem)
				return NULL;
			nextleft = roundup(size, PAGE_SIZE);
		}

		ret = nextmem;
		nextleft -= size;
		nextmem += size;
		memset(ret, 0, size);
		return ret;
	} else {
		return kzalloc(size, GFP_KERNEL);
	}
}

static void kmem_cache_dyn_array_free(void *array)
{
	if (unlikely(!slab_is_available()))
		return; /* error case without crashing here (will panic soon) */
	kfree(array);
}
#endif

/*
 * Except in early boot, this should be called with slqb_lock held for write
 * to lock out hotplug, and protect list modifications.
 */
static int kmem_cache_open(struct kmem_cache *s,
			const char *name, size_t size, size_t align,
			unsigned long flags, void (*ctor)(void *), int alloc)
{
	unsigned int left_over;

	memset(s, 0, sizeof(struct kmem_cache));
	s->name = name;
	s->ctor = ctor;
	s->objsize = size;
	s->align = align;
	s->flags = kmem_cache_flags(size, flags, name, ctor);

	if (!calculate_sizes(s))
		goto error;

	if (!slab_debug(s)) {
		left_over = (PAGE_SIZE << s->order) - (s->objects * s->size);
		s->colour_off = max(cache_line_size(), s->align);
		s->colour_range = left_over;
	} else {
		s->colour_off = 0;
		s->colour_range = 0;
	}

#ifdef CONFIG_SMP
	s->cpu_slab = kmem_cache_dyn_array_alloc(nr_cpu_ids);
	if (!s->cpu_slab)
		goto error;
# ifdef CONFIG_NUMA
	s->node_slab = kmem_cache_dyn_array_alloc(nr_node_ids);
	if (!s->node_slab)
		goto error_cpu_array;
# endif
#endif

	if (likely(alloc)) {
		if (!alloc_kmem_cache_nodes(s))
			goto error_node_array;

		if (!alloc_kmem_cache_cpus(s))
			goto error_nodes;
	}

	sysfs_slab_add(s);
	list_add(&s->list, &slab_caches);

	return 1;

error_nodes:
	free_kmem_cache_nodes(s);
error_node_array:
#if defined(CONFIG_NUMA) && defined(CONFIG_SMP)
	kmem_cache_dyn_array_free(s->node_slab);
error_cpu_array:
#endif
#ifdef CONFIG_SMP
	kmem_cache_dyn_array_free(s->cpu_slab);
#endif
error:
	if (flags & SLAB_PANIC)
		panic("%s: failed to create slab `%s'\n", __func__, name);
	return 0;
}

/**
 * kmem_ptr_validate - check if an untrusted pointer might be a slab entry.
 * @s: the cache we're checking against
 * @ptr: pointer to validate
 *
 * This verifies that the untrusted pointer looks sane;
 * it is _not_ a guarantee that the pointer is actually
 * part of the slab cache in question, but it at least
 * validates that the pointer can be dereferenced and
 * looks half-way sane.
 *
 * Currently only used for dentry validation.
 */
int kmem_ptr_validate(struct kmem_cache *s, const void *ptr)
{
	unsigned long addr = (unsigned long)ptr;
	struct slqb_page *page;

	if (unlikely(addr < PAGE_OFFSET))
		goto out;
	if (unlikely(addr > (unsigned long)high_memory - s->size))
		goto out;
	if (unlikely(!IS_ALIGNED(addr, s->align)))
		goto out;
	if (unlikely(!kern_addr_valid(addr)))
		goto out;
	if (unlikely(!kern_addr_valid(addr + s->size - 1)))
		goto out;
	if (unlikely(!pfn_valid(addr >> PAGE_SHIFT)))
		goto out;
	page = virt_to_head_slqb_page(ptr);
	if (unlikely(!(page->flags & PG_SLQB_BIT)))
		goto out;
	if (unlikely(page->list->cache != s)) /* XXX: ouch, racy */
		goto out;
	return 1;
out:
	return 0;
}
EXPORT_SYMBOL(kmem_ptr_validate);

/*
 * Determine the size of a slab object
 */
unsigned int kmem_cache_size(struct kmem_cache *s)
{
	return s->objsize;
}
EXPORT_SYMBOL(kmem_cache_size);

const char *kmem_cache_name(struct kmem_cache *s)
{
	return s->name;
}
EXPORT_SYMBOL(kmem_cache_name);

/*
 * Release all resources used by a slab cache. No more concurrency on the
 * slab, so we can touch remote kmem_cache_cpu structures.
 */
void kmem_cache_destroy(struct kmem_cache *s)
{
#ifdef CONFIG_NUMA
	int node;
#endif
	int cpu;

	down_write(&slqb_lock);
	list_del(&s->list);

	local_irq_disable();
#ifdef CONFIG_SMP
	for_each_online_cpu(cpu) {
		struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
		struct kmem_cache_list *l = &c->list;

		flush_free_list_all(s, l);
		flush_remote_free_cache(s, c);
	}
#endif

	for_each_online_cpu(cpu) {
		struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
		struct kmem_cache_list *l = &c->list;

		claim_remote_free_list(s, l);
		flush_free_list_all(s, l);

		WARN_ON(l->freelist.nr);
		WARN_ON(l->nr_slabs);
		WARN_ON(l->nr_partial);
	}

	free_kmem_cache_cpus(s);

#ifdef CONFIG_NUMA
	for_each_node_state(node, N_NORMAL_MEMORY) {
		struct kmem_cache_node *n;
		struct kmem_cache_list *l;

		n = s->node_slab[node];
		if (!n)
			continue;
		l = &n->list;

		claim_remote_free_list(s, l);
		flush_free_list_all(s, l);

		WARN_ON(l->freelist.nr);
		WARN_ON(l->nr_slabs);
		WARN_ON(l->nr_partial);
	}

	free_kmem_cache_nodes(s);
#endif
	local_irq_enable();

	sysfs_slab_remove(s);
	up_write(&slqb_lock);
}
EXPORT_SYMBOL(kmem_cache_destroy);

/********************************************************************
 *		Kmalloc subsystem
 *******************************************************************/

struct kmem_cache kmalloc_caches[KMALLOC_SHIFT_SLQB_HIGH + 1] __cacheline_aligned;
EXPORT_SYMBOL(kmalloc_caches);

#ifdef CONFIG_ZONE_DMA
struct kmem_cache kmalloc_caches_dma[KMALLOC_SHIFT_SLQB_HIGH + 1] __cacheline_aligned;
EXPORT_SYMBOL(kmalloc_caches_dma);
#endif

#ifndef ARCH_KMALLOC_FLAGS
#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
#endif

static struct kmem_cache *open_kmalloc_cache(struct kmem_cache *s,
				const char *name, int size, gfp_t gfp_flags)
{
	unsigned int flags = ARCH_KMALLOC_FLAGS | SLAB_PANIC;

	if (gfp_flags & SLQB_DMA)
		flags |= SLAB_CACHE_DMA;

	kmem_cache_open(s, name, size, ARCH_KMALLOC_MINALIGN, flags, NULL, 1);

	return s;
}

/*
 * Conversion table for small slabs sizes / 8 to the index in the
 * kmalloc array. This is necessary for slabs < 192 since we have non power
 * of two cache sizes there. The size of larger slabs can be determined using
 * fls.
 */
static s8 size_index[24] __cacheline_aligned = {
	3,	/* 8 */
	4,	/* 16 */
	5,	/* 24 */
	5,	/* 32 */
	6,	/* 40 */
	6,	/* 48 */
	6,	/* 56 */
	6,	/* 64 */
#if L1_CACHE_BYTES < 64
	1,	/* 72 */
	1,	/* 80 */
	1,	/* 88 */
	1,	/* 96 */
#else
	7,
	7,
	7,
	7,
#endif
	7,	/* 104 */
	7,	/* 112 */
	7,	/* 120 */
	7,	/* 128 */
#if L1_CACHE_BYTES < 128
	2,	/* 136 */
	2,	/* 144 */
	2,	/* 152 */
	2,	/* 160 */
	2,	/* 168 */
	2,	/* 176 */
	2,	/* 184 */
	2	/* 192 */
#else
	-1,
	-1,
	-1,
	-1,
	-1,
	-1,
	-1,
	-1
#endif
};

static struct kmem_cache *get_slab(size_t size, gfp_t flags)
{
	int index;

	if (unlikely(size <= KMALLOC_MIN_SIZE)) {
		if (unlikely(!size))
			return ZERO_SIZE_PTR;

		index = KMALLOC_SHIFT_LOW;
		goto got_index;
	}

#if L1_CACHE_BYTES >= 128
	if (size <= 128) {
#else
	if (size <= 192) {
#endif
		index = size_index[(size - 1) / 8];
	} else {
		if (unlikely(size > 1UL << KMALLOC_SHIFT_SLQB_HIGH))
			return NULL;

		index = fls(size - 1);
	}

got_index:
	if (unlikely((flags & SLQB_DMA)))
		return &kmalloc_caches_dma[index];
	else
		return &kmalloc_caches[index];
}

void *__kmalloc(size_t size, gfp_t flags)
{
	struct kmem_cache *s;

	s = get_slab(size, flags);
	if (unlikely(ZERO_OR_NULL_PTR(s)))
		return s;

	return __kmem_cache_alloc(s, flags, _RET_IP_);
}
EXPORT_SYMBOL(__kmalloc);

#ifdef CONFIG_NUMA
void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
	struct kmem_cache *s;

	s = get_slab(size, flags);
	if (unlikely(ZERO_OR_NULL_PTR(s)))
		return s;

	return kmem_cache_alloc_node(s, flags, node);
}
EXPORT_SYMBOL(__kmalloc_node);
#endif

size_t ksize(const void *object)
{
	struct slqb_page *page;
	struct kmem_cache *s;

	BUG_ON(!object);
	if (unlikely(object == ZERO_SIZE_PTR))
		return 0;

	page = virt_to_head_slqb_page(object);
	BUG_ON(!(page->flags & PG_SLQB_BIT));

	s = page->list->cache;

	/*
	 * Debugging requires use of the padding between object
	 * and whatever may come after it.
	 */
	if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
		return s->objsize;

	/*
	 * If we have the need to store the freelist pointer
	 * back there or track user information then we can
	 * only use the space before that information.
	 */
	if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER))
		return s->inuse;

	/*
	 * Else we can use all the padding etc for the allocation
	 */
	return s->size;
}
EXPORT_SYMBOL(ksize);

void kfree(const void *object)
{
	struct kmem_cache *s;
	struct slqb_page *page;

	if (unlikely(ZERO_OR_NULL_PTR(object)))
		return;

	page = virt_to_head_slqb_page(object);
	s = page->list->cache;

	slab_free(s, page, (void *)object);
}
EXPORT_SYMBOL(kfree);

static void kmem_cache_trim_percpu(void *arg)
{
	int cpu = smp_processor_id();
	struct kmem_cache *s = arg;
	struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
	struct kmem_cache_list *l = &c->list;

	claim_remote_free_list(s, l);
	flush_free_list(s, l);
#ifdef CONFIG_SMP
	flush_remote_free_cache(s, c);
#endif
}

int kmem_cache_shrink(struct kmem_cache *s)
{
#ifdef CONFIG_NUMA
	int node;
#endif

	on_each_cpu(kmem_cache_trim_percpu, s, 1);

#ifdef CONFIG_NUMA
	for_each_node_state(node, N_NORMAL_MEMORY) {
		struct kmem_cache_node *n;
		struct kmem_cache_list *l;

		n = s->node_slab[node];
		if (!n)
			continue;
		l = &n->list;

		spin_lock_irq(&n->list_lock);
		claim_remote_free_list(s, l);
		flush_free_list(s, l);
		spin_unlock_irq(&n->list_lock);
	}
#endif

	return 0;
}
EXPORT_SYMBOL(kmem_cache_shrink);

#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
static void kmem_cache_reap_percpu(void *arg)
{
	int cpu = smp_processor_id();
	struct kmem_cache *s;
	long phase = (long)arg;

	list_for_each_entry(s, &slab_caches, list) {
		struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
		struct kmem_cache_list *l = &c->list;

		if (phase == 0) {
			flush_free_list_all(s, l);
			flush_remote_free_cache(s, c);
		}

		if (phase == 1) {
			claim_remote_free_list(s, l);
			flush_free_list_all(s, l);
		}
	}
}

static void kmem_cache_reap(void)
{
	struct kmem_cache *s;
	int node;

	down_read(&slqb_lock);
	on_each_cpu(kmem_cache_reap_percpu, (void *)0, 1);
	on_each_cpu(kmem_cache_reap_percpu, (void *)1, 1);

	list_for_each_entry(s, &slab_caches, list) {
		for_each_node_state(node, N_NORMAL_MEMORY) {
			struct kmem_cache_node *n;
			struct kmem_cache_list *l;

			n = s->node_slab[node];
			if (!n)
				continue;
			l = &n->list;

			spin_lock_irq(&n->list_lock);
			claim_remote_free_list(s, l);
			flush_free_list_all(s, l);
			spin_unlock_irq(&n->list_lock);
		}
	}
	up_read(&slqb_lock);
}
#endif

static void cache_trim_worker(struct work_struct *w)
{
	struct delayed_work *work =
		container_of(w, struct delayed_work, work);
	struct kmem_cache *s;

	if (!down_read_trylock(&slqb_lock))
		goto out;

	list_for_each_entry(s, &slab_caches, list) {
#ifdef CONFIG_NUMA
		int node = numa_node_id();
		struct kmem_cache_node *n = s->node_slab[node];

		if (n) {
			struct kmem_cache_list *l = &n->list;

			spin_lock_irq(&n->list_lock);
			claim_remote_free_list(s, l);
			flush_free_list(s, l);
			spin_unlock_irq(&n->list_lock);
		}
#endif

		local_irq_disable();
		kmem_cache_trim_percpu(s);
		local_irq_enable();
	}

	up_read(&slqb_lock);
out:
	schedule_delayed_work(work, round_jiffies_relative(3*HZ));
}

static DEFINE_PER_CPU(struct delayed_work, slqb_cache_trim_work);

static void __cpuinit start_cpu_timer(int cpu)
{
	struct delayed_work *cache_trim_work = &per_cpu(slqb_cache_trim_work,
			cpu);

	/*
	 * When this gets called from do_initcalls via cpucache_init(),
	 * init_workqueues() has already run, so keventd will be setup
	 * at that time.
	 */
	if (keventd_up() && cache_trim_work->work.func == NULL) {
		INIT_DELAYED_WORK(cache_trim_work, cache_trim_worker);
		schedule_delayed_work_on(cpu, cache_trim_work,
					__round_jiffies_relative(HZ, cpu));
	}
}

static int __init cpucache_init(void)
{
	int cpu;

	for_each_online_cpu(cpu)
		start_cpu_timer(cpu);

	return 0;
}
device_initcall(cpucache_init);

#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
static void slab_mem_going_offline_callback(void *arg)
{
	kmem_cache_reap();
}

static void slab_mem_offline_callback(void *arg)
{
	/* XXX: should release structures, see CPU offline comment */
}

static int slab_mem_going_online_callback(void *arg)
{
	struct kmem_cache *s;
	struct kmem_cache_node *n;
	struct memory_notify *marg = arg;
	int nid = marg->status_change_nid;
	int ret = 0;

	/*
	 * If the node's memory is already available, then kmem_cache_node is
	 * already created. Nothing to do.
	 */
	if (nid < 0)
		return 0;

	/*
	 * We are bringing a node online. No memory is availabe yet. We must
	 * allocate a kmem_cache_node structure in order to bring the node
	 * online.
	 */
	down_write(&slqb_lock);
	list_for_each_entry(s, &slab_caches, list) {
		/*
		 * XXX: kmem_cache_alloc_node will fallback to other nodes
		 *      since memory is not yet available from the node that
		 *      is brought up.
		 */
		if (s->node_slab[nid]) /* could be lefover from last online */
			continue;
		n = kmem_cache_alloc(&kmem_node_cache, GFP_KERNEL);
		if (!n) {
			ret = -ENOMEM;
			goto out;
		}
		init_kmem_cache_node(s, n);
		s->node_slab[nid] = n;
	}
out:
	up_write(&slqb_lock);
	return ret;
}

static int slab_memory_callback(struct notifier_block *self,
				unsigned long action, void *arg)
{
	int ret = 0;

	switch (action) {
	case MEM_GOING_ONLINE:
		ret = slab_mem_going_online_callback(arg);
		break;
	case MEM_GOING_OFFLINE:
		slab_mem_going_offline_callback(arg);
		break;
	case MEM_OFFLINE:
	case MEM_CANCEL_ONLINE:
		slab_mem_offline_callback(arg);
		break;
	case MEM_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}

	if (ret)
		ret = notifier_from_errno(ret);
	else
		ret = NOTIFY_OK;
	return ret;
}

#endif /* CONFIG_MEMORY_HOTPLUG */

/********************************************************************
 *			Basic setup of slabs
 *******************************************************************/

void __init kmem_cache_init(void)
{
	int i;
	unsigned int flags = SLAB_HWCACHE_ALIGN|SLAB_PANIC;

	/*
	 * All the ifdefs are rather ugly here, but it's just the setup code,
	 * so it doesn't have to be too readable :)
	 */

	/*
	 * No need to take slqb_lock here: there should be no concurrency
	 * anyway, and spin_unlock_irq in rwsem code could enable interrupts
	 * too early.
	 */
	kmem_cache_open(&kmem_cache_cache, "kmem_cache",
			sizeof(struct kmem_cache), 0, flags, NULL, 0);
#ifdef CONFIG_SMP
	kmem_cache_open(&kmem_cpu_cache, "kmem_cache_cpu",
			sizeof(struct kmem_cache_cpu), 0, flags, NULL, 0);
#endif
#ifdef CONFIG_NUMA
	kmem_cache_open(&kmem_node_cache, "kmem_cache_node",
			sizeof(struct kmem_cache_node), 0, flags, NULL, 0);
#endif

#ifdef CONFIG_SMP
	for_each_possible_cpu(i) {
		struct kmem_cache_cpu *c;

		c = &per_cpu(kmem_cache_cpus, i);
		init_kmem_cache_cpu(&kmem_cache_cache, c);
		kmem_cache_cache.cpu_slab[i] = c;

		c = &per_cpu(kmem_cpu_cpus, i);
		init_kmem_cache_cpu(&kmem_cpu_cache, c);
		kmem_cpu_cache.cpu_slab[i] = c;

#ifdef CONFIG_NUMA
		c = &per_cpu(kmem_node_cpus, i);
		init_kmem_cache_cpu(&kmem_node_cache, c);
		kmem_node_cache.cpu_slab[i] = c;
#endif
	}
#else
	init_kmem_cache_cpu(&kmem_cache_cache, &kmem_cache_cache.cpu_slab);
#endif

#ifdef CONFIG_NUMA
	for_each_node_state(i, N_NORMAL_MEMORY) {
		struct kmem_cache_node *n;

		n = &kmem_cache_nodes[i];
		init_kmem_cache_node(&kmem_cache_cache, n);
		kmem_cache_cache.node_slab[i] = n;
#ifdef CONFIG_SMP
		n = &kmem_cpu_nodes[i];
		init_kmem_cache_node(&kmem_cpu_cache, n);
		kmem_cpu_cache.node_slab[i] = n;
#endif
		n = &kmem_node_nodes[i];
		init_kmem_cache_node(&kmem_node_cache, n);
		kmem_node_cache.node_slab[i] = n;
	}
#endif

	/* Caches that are not of the two-to-the-power-of size */
	if (L1_CACHE_BYTES < 64 && KMALLOC_MIN_SIZE <= 64) {
		open_kmalloc_cache(&kmalloc_caches[1],
				"kmalloc-96", 96, GFP_KERNEL);
#ifdef CONFIG_ZONE_DMA
		open_kmalloc_cache(&kmalloc_caches_dma[1],
				"kmalloc_dma-96", 96, GFP_KERNEL|SLQB_DMA);
#endif
	}
	if (L1_CACHE_BYTES < 128 && KMALLOC_MIN_SIZE <= 128) {
		open_kmalloc_cache(&kmalloc_caches[2],
				"kmalloc-192", 192, GFP_KERNEL);
#ifdef CONFIG_ZONE_DMA
		open_kmalloc_cache(&kmalloc_caches_dma[2],
				"kmalloc_dma-192", 192, GFP_KERNEL|SLQB_DMA);
#endif
	}

	for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_SLQB_HIGH; i++) {
		open_kmalloc_cache(&kmalloc_caches[i],
				"kmalloc", 1 << i, GFP_KERNEL);
#ifdef CONFIG_ZONE_DMA
		open_kmalloc_cache(&kmalloc_caches_dma[i],
				"kmalloc_dma", 1 << i, GFP_KERNEL|SLQB_DMA);
#endif
	}

	/*
	 * Patch up the size_index table if we have strange large alignment
	 * requirements for the kmalloc array. This is only the case for
	 * mips it seems. The standard arches will not generate any code here.
	 *
	 * Largest permitted alignment is 256 bytes due to the way we
	 * handle the index determination for the smaller caches.
	 *
	 * Make sure that nothing crazy happens if someone starts tinkering
	 * around with ARCH_KMALLOC_MINALIGN
	 */
	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
		(KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));

	for (i = 8; i < KMALLOC_MIN_SIZE; i += 8)
		size_index[(i - 1) / 8] = KMALLOC_SHIFT_LOW;

	/* Provide the correct kmalloc names now that the caches are up */
	for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_SLQB_HIGH; i++) {
		kmalloc_caches[i].name =
			kasprintf(GFP_KERNEL, "kmalloc-%d", 1 << i);
#ifdef CONFIG_ZONE_DMA
		kmalloc_caches_dma[i].name =
			kasprintf(GFP_KERNEL, "kmalloc_dma-%d", 1 << i);
#endif
	}

#ifdef CONFIG_SMP
	register_cpu_notifier(&slab_notifier);
#endif
#ifdef CONFIG_NUMA
	hotplug_memory_notifier(slab_memory_callback, 1);
#endif
	/*
	 * smp_init() has not yet been called, so no worries about memory
	 * ordering with __slab_is_available.
	 */
	__slab_is_available = 1;
}

void __init kmem_cache_init_late(void)
{
}

/*
 * Some basic slab creation sanity checks
 */
static int kmem_cache_create_ok(const char *name, size_t size,
		size_t align, unsigned long flags)
{
	struct kmem_cache *tmp;

	/*
	 * Sanity checks... these are all serious usage bugs.
	 */
	if (!name || in_interrupt() || (size < sizeof(void *))) {
		printk(KERN_ERR "kmem_cache_create(): early error in slab %s\n",
				name);
		dump_stack();

		return 0;
	}

	list_for_each_entry(tmp, &slab_caches, list) {
		char x;
		int res;

		/*
		 * This happens when the module gets unloaded and doesn't
		 * destroy its slab cache and no-one else reuses the vmalloc
		 * area of the module.  Print a warning.
		 */
		res = probe_kernel_address(tmp->name, x);
		if (res) {
			printk(KERN_ERR
			       "SLAB: cache with size %d has lost its name\n",
			       tmp->size);
			continue;
		}

		if (!strcmp(tmp->name, name)) {
			printk(KERN_ERR
			       "SLAB: duplicate cache %s\n", name);
			dump_stack();

			return 0;
		}
	}

	WARN_ON(strchr(name, ' '));	/* It confuses parsers */
	if (flags & SLAB_DESTROY_BY_RCU)
		WARN_ON(flags & SLAB_POISON);

	return 1;
}

struct kmem_cache *kmem_cache_create(const char *name, size_t size,
		size_t align, unsigned long flags, void (*ctor)(void *))
{
	struct kmem_cache *s;

	down_write(&slqb_lock);
	if (!kmem_cache_create_ok(name, size, align, flags))
		goto err;

	s = kmem_cache_alloc(&kmem_cache_cache, GFP_KERNEL);
	if (!s)
		goto err;

	if (kmem_cache_open(s, name, size, align, flags, ctor, 1)) {
		up_write(&slqb_lock);
		return s;
	}

	kmem_cache_free(&kmem_cache_cache, s);

err:
	up_write(&slqb_lock);
	if (flags & SLAB_PANIC)
		panic("%s: failed to create slab `%s'\n", __func__, name);

	return NULL;
}
EXPORT_SYMBOL(kmem_cache_create);

#ifdef CONFIG_SMP
/*
 * Use the cpu notifier to insure that the cpu slabs are flushed when
 * necessary.
 */
static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb,
				unsigned long action, void *hcpu)
{
	long cpu = (long)hcpu;
	struct kmem_cache *s;

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
		down_write(&slqb_lock);
		list_for_each_entry(s, &slab_caches, list) {
			if (s->cpu_slab[cpu]) /* could be lefover last online */
				continue;
			s->cpu_slab[cpu] = alloc_kmem_cache_cpu(s, cpu);
			if (!s->cpu_slab[cpu]) {
				up_read(&slqb_lock);
				return NOTIFY_BAD;
			}
		}
		up_write(&slqb_lock);
		break;

	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
	case CPU_DOWN_FAILED:
	case CPU_DOWN_FAILED_FROZEN:
		start_cpu_timer(cpu);
		break;

	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
		cancel_delayed_work_sync(&per_cpu(slqb_cache_trim_work,
					cpu));
		per_cpu(slqb_cache_trim_work, cpu).work.func = NULL;
		break;

	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
		/*
		 * XXX: Freeing here doesn't work because objects can still be
		 * on this CPU's list. periodic timer needs to check if a CPU
		 * is offline and then try to cleanup from there. Same for node
		 * offline.
		 */
	default:
		break;
	}
	return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata slab_notifier = {
	.notifier_call = slab_cpuup_callback
};

#endif

#ifdef CONFIG_SLQB_DEBUG
void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
{
	struct kmem_cache *s;
	int node = -1;

	s = get_slab(size, flags);
	if (unlikely(ZERO_OR_NULL_PTR(s)))
		return s;

#ifdef CONFIG_NUMA
	if (unlikely(current->flags & (PF_SPREAD_SLAB | PF_MEMPOLICY)))
		node = alternate_nid(s, flags, node);
#endif
	return slab_alloc(s, flags, node, caller);
}

void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node,
				unsigned long caller)
{
	struct kmem_cache *s;

	s = get_slab(size, flags);
	if (unlikely(ZERO_OR_NULL_PTR(s)))
		return s;

	return slab_alloc(s, flags, node, caller);
}
#endif

#if defined(CONFIG_SLQB_SYSFS) || defined(CONFIG_SLABINFO)
struct stats_gather {
	struct kmem_cache *s;
	spinlock_t lock;
	unsigned long nr_slabs;
	unsigned long nr_partial;
	unsigned long nr_inuse;
	unsigned long nr_objects;

#ifdef CONFIG_SLQB_STATS
	unsigned long stats[NR_SLQB_STAT_ITEMS];
#endif
};

static void __gather_stats(void *arg)
{
	unsigned long nr_slabs;
	unsigned long nr_partial;
	unsigned long nr_inuse;
	struct stats_gather *gather = arg;
	int cpu = smp_processor_id();
	struct kmem_cache *s = gather->s;
	struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
	struct kmem_cache_list *l = &c->list;
	struct slqb_page *page;
#ifdef CONFIG_SLQB_STATS
	int i;
#endif

	spin_lock(&l->page_lock);
	nr_slabs = l->nr_slabs;
	nr_partial = l->nr_partial;
	nr_inuse = (nr_slabs - nr_partial) * s->objects;

	list_for_each_entry(page, &l->partial, lru) {
		nr_inuse += page->inuse;
	}
	spin_unlock(&l->page_lock);

	spin_lock(&gather->lock);
	gather->nr_slabs += nr_slabs;
	gather->nr_partial += nr_partial;
	gather->nr_inuse += nr_inuse;
#ifdef CONFIG_SLQB_STATS
	for (i = 0; i < NR_SLQB_STAT_ITEMS; i++)
		gather->stats[i] += l->stats[i];
#endif
	spin_unlock(&gather->lock);
}

/* must be called with slqb_lock held */
static void gather_stats_locked(struct kmem_cache *s,
				struct stats_gather *stats)
{
#ifdef CONFIG_NUMA
	int node;
#endif

	memset(stats, 0, sizeof(struct stats_gather));
	stats->s = s;
	spin_lock_init(&stats->lock);

	on_each_cpu(__gather_stats, stats, 1);

#ifdef CONFIG_NUMA
	for_each_online_node(node) {
		struct kmem_cache_node *n = s->node_slab[node];
		struct kmem_cache_list *l = &n->list;
		struct slqb_page *page;
		unsigned long flags;
#ifdef CONFIG_SLQB_STATS
		int i;
#endif

		spin_lock_irqsave(&n->list_lock, flags);
#ifdef CONFIG_SLQB_STATS
		for (i = 0; i < NR_SLQB_STAT_ITEMS; i++)
			stats->stats[i] += l->stats[i];
#endif
		stats->nr_slabs += l->nr_slabs;
		stats->nr_partial += l->nr_partial;
		stats->nr_inuse += (l->nr_slabs - l->nr_partial) * s->objects;

		list_for_each_entry(page, &l->partial, lru) {
			stats->nr_inuse += page->inuse;
		}
		spin_unlock_irqrestore(&n->list_lock, flags);
	}
#endif

	stats->nr_objects = stats->nr_slabs * s->objects;
}

#ifdef CONFIG_SLQB_SYSFS
static void gather_stats(struct kmem_cache *s, struct stats_gather *stats)
{
	down_read(&slqb_lock); /* hold off hotplug */
	gather_stats_locked(s, stats);
	up_read(&slqb_lock);
}
#endif
#endif

/*
 * The /proc/slabinfo ABI
 */
#ifdef CONFIG_SLABINFO
#include <linux/proc_fs.h>
ssize_t slabinfo_write(struct file *file, const char __user * buffer,
		       size_t count, loff_t *ppos)
{
	return -EINVAL;
}

static void print_slabinfo_header(struct seq_file *m)
{
	seq_puts(m, "slabinfo - version: 2.1\n");
	seq_puts(m, "# name	    <active_objs> <num_objs> <objsize> "
		 "<objperslab> <pagesperslab>");
	seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
	seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
	seq_putc(m, '\n');
}

static void *s_start(struct seq_file *m, loff_t *pos)
{
	loff_t n = *pos;

	down_read(&slqb_lock);
	if (!n)
		print_slabinfo_header(m);

	return seq_list_start(&slab_caches, *pos);
}

static void *s_next(struct seq_file *m, void *p, loff_t *pos)
{
	return seq_list_next(p, &slab_caches, pos);
}

static void s_stop(struct seq_file *m, void *p)
{
	up_read(&slqb_lock);
}

static int s_show(struct seq_file *m, void *p)
{
	struct stats_gather stats;
	struct kmem_cache *s;

	s = list_entry(p, struct kmem_cache, list);

	gather_stats_locked(s, &stats);

	seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", s->name, stats.nr_inuse,
			stats.nr_objects, s->size, s->objects, (1 << s->order));
	seq_printf(m, " : tunables %4u %4u %4u", slab_hiwater(s),
			slab_freebatch(s), 0);
	seq_printf(m, " : slabdata %6lu %6lu %6lu", stats.nr_slabs,
			stats.nr_slabs, 0UL);
	seq_putc(m, '\n');
	return 0;
}

static const struct seq_operations slabinfo_op = {
	.start = s_start,
	.next = s_next,
	.stop = s_stop,
	.show = s_show,
};

static int slabinfo_open(struct inode *inode, struct file *file)
{
	return seq_open(file, &slabinfo_op);
}

static const struct file_operations proc_slabinfo_operations = {
	.open		= slabinfo_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release,
};

static int __init slab_proc_init(void)
{
	proc_create("slabinfo", S_IWUSR|S_IRUGO, NULL,
			&proc_slabinfo_operations);
	return 0;
}
module_init(slab_proc_init);
#endif /* CONFIG_SLABINFO */

#ifdef CONFIG_SLQB_SYSFS
/*
 * sysfs API
 */
#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
#define to_slab(n) container_of(n, struct kmem_cache, kobj);

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
};

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = __ATTR_RO(_name)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0644, _name##_show, _name##_store)

static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->size);
}
SLAB_ATTR_RO(slab_size);

static ssize_t align_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->align);
}
SLAB_ATTR_RO(align);

static ssize_t object_size_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->objsize);
}
SLAB_ATTR_RO(object_size);

static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->objects);
}
SLAB_ATTR_RO(objs_per_slab);

static ssize_t order_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->order);
}
SLAB_ATTR_RO(order);

static ssize_t ctor_show(struct kmem_cache *s, char *buf)
{
	if (s->ctor) {
		int n = sprint_symbol(buf, (unsigned long)s->ctor);

		return n + sprintf(buf + n, "\n");
	}
	return 0;
}
SLAB_ATTR_RO(ctor);

static ssize_t slabs_show(struct kmem_cache *s, char *buf)
{
	struct stats_gather stats;

	gather_stats(s, &stats);

	return sprintf(buf, "%lu\n", stats.nr_slabs);
}
SLAB_ATTR_RO(slabs);

static ssize_t objects_show(struct kmem_cache *s, char *buf)
{
	struct stats_gather stats;

	gather_stats(s, &stats);

	return sprintf(buf, "%lu\n", stats.nr_inuse);
}
SLAB_ATTR_RO(objects);

static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
{
	struct stats_gather stats;

	gather_stats(s, &stats);

	return sprintf(buf, "%lu\n", stats.nr_objects);
}
SLAB_ATTR_RO(total_objects);

#ifdef CONFIG_FAILSLAB
static ssize_t failslab_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
}

static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
							size_t length)
{
	s->flags &= ~SLAB_FAILSLAB;
	if (buf[0] == '1')
		s->flags |= SLAB_FAILSLAB;
	return length;
}
SLAB_ATTR(failslab);
#endif

static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
}
SLAB_ATTR_RO(reclaim_account);

static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
}
SLAB_ATTR_RO(hwcache_align);

#ifdef CONFIG_ZONE_DMA
static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
}
SLAB_ATTR_RO(cache_dma);
#endif

static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
}
SLAB_ATTR_RO(destroy_by_rcu);

static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
}
SLAB_ATTR_RO(red_zone);

static ssize_t poison_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
}
SLAB_ATTR_RO(poison);

static ssize_t store_user_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
}
SLAB_ATTR_RO(store_user);

static ssize_t hiwater_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	long hiwater;
	int err;

	err = strict_strtol(buf, 10, &hiwater);
	if (err)
		return err;

	if (hiwater < 0)
		return -EINVAL;

	s->hiwater = hiwater;

	return length;
}

static ssize_t hiwater_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", slab_hiwater(s));
}
SLAB_ATTR(hiwater);

static ssize_t freebatch_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	long freebatch;
	int err;

	err = strict_strtol(buf, 10, &freebatch);
	if (err)
		return err;

	if (freebatch <= 0 || freebatch - 1 > s->hiwater)
		return -EINVAL;

	s->freebatch = freebatch;

	return length;
}

static ssize_t freebatch_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", slab_freebatch(s));
}
SLAB_ATTR(freebatch);

#ifdef CONFIG_SLQB_STATS
static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
{
	struct stats_gather stats;
	int len;
#ifdef CONFIG_SMP
	int cpu;
#endif

	gather_stats(s, &stats);

	len = sprintf(buf, "%lu", stats.stats[si]);

#ifdef CONFIG_SMP
	for_each_online_cpu(cpu) {
		struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
		struct kmem_cache_list *l = &c->list;

		if (len < PAGE_SIZE - 20)
			len += sprintf(buf+len, " C%d=%lu", cpu, l->stats[si]);
	}
#endif
	return len + sprintf(buf + len, "\n");
}

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
SLAB_ATTR_RO(text);						\

STAT_ATTR(ALLOC, alloc);
STAT_ATTR(ALLOC_SLAB_FILL, alloc_slab_fill);
STAT_ATTR(ALLOC_SLAB_NEW, alloc_slab_new);
STAT_ATTR(FREE, free);
STAT_ATTR(FREE_REMOTE, free_remote);
STAT_ATTR(FLUSH_FREE_LIST, flush_free_list);
STAT_ATTR(FLUSH_FREE_LIST_OBJECTS, flush_free_list_objects);
STAT_ATTR(FLUSH_FREE_LIST_REMOTE, flush_free_list_remote);
STAT_ATTR(FLUSH_SLAB_PARTIAL, flush_slab_partial);
STAT_ATTR(FLUSH_SLAB_FREE, flush_slab_free);
STAT_ATTR(FLUSH_RFREE_LIST, flush_rfree_list);
STAT_ATTR(FLUSH_RFREE_LIST_OBJECTS, flush_rfree_list_objects);
STAT_ATTR(CLAIM_REMOTE_LIST, claim_remote_list);
STAT_ATTR(CLAIM_REMOTE_LIST_OBJECTS, claim_remote_list_objects);
#endif

static struct attribute *slab_attrs[] = {
	&slab_size_attr.attr,
	&object_size_attr.attr,
	&objs_per_slab_attr.attr,
	&order_attr.attr,
	&objects_attr.attr,
	&total_objects_attr.attr,
	&slabs_attr.attr,
	&ctor_attr.attr,
	&align_attr.attr,
	&hwcache_align_attr.attr,
	&reclaim_account_attr.attr,
	&destroy_by_rcu_attr.attr,
	&red_zone_attr.attr,
	&poison_attr.attr,
	&store_user_attr.attr,
	&hiwater_attr.attr,
	&freebatch_attr.attr,
#ifdef CONFIG_ZONE_DMA
	&cache_dma_attr.attr,
#endif
#ifdef CONFIG_SLQB_STATS
	&alloc_attr.attr,
	&alloc_slab_fill_attr.attr,
	&alloc_slab_new_attr.attr,
	&free_attr.attr,
	&free_remote_attr.attr,
	&flush_free_list_attr.attr,
	&flush_free_list_objects_attr.attr,
	&flush_free_list_remote_attr.attr,
	&flush_slab_partial_attr.attr,
	&flush_slab_free_attr.attr,
	&flush_rfree_list_attr.attr,
	&flush_rfree_list_objects_attr.attr,
	&claim_remote_list_attr.attr,
	&claim_remote_list_objects_attr.attr,
#endif
#ifdef CONFIG_FAILSLAB
	&failslab_attr.attr,
#endif

	NULL
};

static struct attribute_group slab_attr_group = {
	.attrs = slab_attrs,
};

static ssize_t slab_attr_show(struct kobject *kobj,
				struct attribute *attr, char *buf)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;
	int err;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->show)
		return -EIO;

	err = attribute->show(s, buf);

	return err;
}

static ssize_t slab_attr_store(struct kobject *kobj,
			struct attribute *attr, const char *buf, size_t len)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;
	int err;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->store)
		return -EIO;

	err = attribute->store(s, buf, len);

	return err;
}

static void kmem_cache_release(struct kobject *kobj)
{
	struct kmem_cache *s = to_slab(kobj);

	kmem_cache_free(&kmem_cache_cache, s);
}

static struct sysfs_ops slab_sysfs_ops = {
	.show = slab_attr_show,
	.store = slab_attr_store,
};

static struct kobj_type slab_ktype = {
	.sysfs_ops = &slab_sysfs_ops,
	.release = kmem_cache_release
};

static int uevent_filter(struct kset *kset, struct kobject *kobj)
{
	struct kobj_type *ktype = get_ktype(kobj);

	if (ktype == &slab_ktype)
		return 1;
	return 0;
}

static struct kset_uevent_ops slab_uevent_ops = {
	.filter = uevent_filter,
};

static struct kset *slab_kset;

static int sysfs_available __read_mostly;

static int sysfs_slab_add(struct kmem_cache *s)
{
	int err;

	if (!sysfs_available)
		return 0;

	s->kobj.kset = slab_kset;
	err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, s->name);
	if (err) {
		kobject_put(&s->kobj);
		return err;
	}

	err = sysfs_create_group(&s->kobj, &slab_attr_group);
	if (err)
		return err;

	kobject_uevent(&s->kobj, KOBJ_ADD);

	return 0;
}

static void sysfs_slab_remove(struct kmem_cache *s)
{
	kobject_uevent(&s->kobj, KOBJ_REMOVE);
	kobject_del(&s->kobj);
	kobject_put(&s->kobj);
}

static int __init slab_sysfs_init(void)
{
	struct kmem_cache *s;
	int err;

	slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
	if (!slab_kset) {
		printk(KERN_ERR "Cannot register slab subsystem.\n");
		return -ENOSYS;
	}

	down_write(&slqb_lock);

	sysfs_available = 1;

	list_for_each_entry(s, &slab_caches, list) {
		err = sysfs_slab_add(s);
		if (err)
			printk(KERN_ERR "SLQB: Unable to add boot slab %s"
						" to sysfs\n", s->name);
	}

	up_write(&slqb_lock);

	return 0;
}
device_initcall(slab_sysfs_init);

#endif