From 50d5c41cd151b21ac1dfc98f048210456ccacc20 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 1 Jun 2011 12:25:45 -0500 Subject: slub: Do not use frozen page flag but a bit in the page counters Do not use a page flag for the frozen bit. It needs to be part of the state that is handled with cmpxchg_double(). So use a bit in the counter struct in the page struct for that purpose. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/mm_types.h | 5 +++-- include/linux/page-flags.h | 5 ----- 2 files changed, 3 insertions(+), 7 deletions(-) (limited to 'include') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 027935c86c68..e5fb2a70518b 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -41,8 +41,9 @@ struct page { * & limit reverse map searches. */ struct { /* SLUB */ - u16 inuse; - u16 objects; + unsigned inuse:16; + unsigned objects:15; + unsigned frozen:1; }; }; union { diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 6081493db68f..20791f18f5cf 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -124,9 +124,6 @@ enum pageflags { /* SLOB */ PG_slob_free = PG_private, - - /* SLUB */ - PG_slub_frozen = PG_active, }; #ifndef __GENERATING_BOUNDS_H @@ -212,8 +209,6 @@ PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked) __PAGEFLAG(SlobFree, slob_free) -__PAGEFLAG(SlubFrozen, slub_frozen) - /* * Private page markings that may be used by the filesystem that owns the page * for its own purposes. -- cgit v1.2.3 From fc9bb8c768abe7ae10861c3510e01a95f98d5933 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 1 Jun 2011 12:25:48 -0500 Subject: mm: Rearrange struct page We need to be able to use cmpxchg_double on the freelist and object count field in struct page. Rearrange the fields in struct page according to doubleword entities so that the freelist pointer comes before the counters. Do the rearranging with a future in mind where we use more doubleword atomics to avoid locking of updates to flags/mapping or lru pointers. Create another union to allow access to counters in struct page as a single unsigned long value. The doublewords must be properly aligned for cmpxchg_double to work. Sadly this increases the size of page struct by one word on some architectures. But as a resultpage structs are now cacheline aligned on x86_64. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/mm_types.h | 87 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 28 deletions(-) (limited to 'include') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index e5fb2a70518b..3d76a433d52f 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -30,24 +30,60 @@ struct address_space; * moment. Note that we have no way to track which tasks are using * a page, though if it is a pagecache page, rmap structures can tell us * who is mapping it. + * + * The objects in struct page are organized in double word blocks in + * order to allows us to use atomic double word operations on portions + * of struct page. That is currently only used by slub but the arrangement + * allows the use of atomic double word operations on the flags/mapping + * and lru list pointers also. */ struct page { + /* First double word block */ unsigned long flags; /* Atomic flags, some possibly * updated asynchronously */ - atomic_t _count; /* Usage count, see below. */ + struct address_space *mapping; /* If low bit clear, points to + * inode address_space, or NULL. + * If page mapped as anonymous + * memory, low bit is set, and + * it points to anon_vma object: + * see PAGE_MAPPING_ANON below. + */ + /* Second double word */ union { - atomic_t _mapcount; /* Count of ptes mapped in mms, - * to show when page is mapped - * & limit reverse map searches. + struct { + pgoff_t index; /* Our offset within mapping. */ + atomic_t _mapcount; /* Count of ptes mapped in mms, + * to show when page is mapped + * & limit reverse map searches. + */ + atomic_t _count; /* Usage count, see below. */ + }; + + struct { /* SLUB cmpxchg_double area */ + void *freelist; + union { + unsigned long counters; + struct { + unsigned inuse:16; + unsigned objects:15; + unsigned frozen:1; + /* + * Kernel may make use of this field even when slub + * uses the rest of the double word! */ - struct { /* SLUB */ - unsigned inuse:16; - unsigned objects:15; - unsigned frozen:1; + atomic_t _count; + }; + }; }; }; + + /* Third double word block */ + struct list_head lru; /* Pageout list, eg. active_list + * protected by zone->lru_lock ! + */ + + /* Remainder is not double word aligned */ union { - struct { unsigned long private; /* Mapping-private opaque data: * usually used for buffer_heads * if PagePrivate set; used for @@ -55,27 +91,13 @@ struct page { * indicates order in the buddy * system if PG_buddy is set. */ - struct address_space *mapping; /* If low bit clear, points to - * inode address_space, or NULL. - * If page mapped as anonymous - * memory, low bit is set, and - * it points to anon_vma object: - * see PAGE_MAPPING_ANON below. - */ - }; #if USE_SPLIT_PTLOCKS - spinlock_t ptl; + spinlock_t ptl; #endif - struct kmem_cache *slab; /* SLUB: Pointer to slab */ - struct page *first_page; /* Compound tail pages */ + struct kmem_cache *slab; /* SLUB: Pointer to slab */ + struct page *first_page; /* Compound tail pages */ }; - union { - pgoff_t index; /* Our offset within mapping. */ - void *freelist; /* SLUB: freelist req. slab lock */ - }; - struct list_head lru; /* Pageout list, eg. active_list - * protected by zone->lru_lock ! - */ + /* * On machines where all RAM is mapped into kernel address space, * we can simply calculate the virtual address. On machines with @@ -101,7 +123,16 @@ struct page { */ void *shadow; #endif -}; +} +/* + * If another subsystem starts using the double word pairing for atomic + * operations on struct page then it must change the #if to ensure + * proper alignment of the page struct. + */ +#if defined(CONFIG_SLUB) && defined(CONFIG_CMPXCHG_LOCAL) + __attribute__((__aligned__(2*sizeof(unsigned long)))) +#endif +; typedef unsigned long __nocast vm_flags_t; -- cgit v1.2.3 From b789ef518b2a7231b0668c813f677cee528a9d3f Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 1 Jun 2011 12:25:49 -0500 Subject: slub: Add cmpxchg_double_slab() Add a function that operates on the second doubleword in the page struct and manipulates the object counters, the freelist and the frozen attribute. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index c8668d161dd8..b42715294147 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -33,6 +33,7 @@ enum stat_item { DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */ ORDER_FALLBACK, /* Number of times fallback was necessary */ CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */ + CMPXCHG_DOUBLE_FAIL, /* Number of times that cmpxchg double did not match */ NR_SLUB_STAT_ITEMS }; struct kmem_cache_cpu { -- cgit v1.2.3 From e36a2652d7d1ad97f7636a39bdd8654d296cc36b Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 1 Jun 2011 12:25:57 -0500 Subject: slub: Add statistics for the case that the current slab does not match the node Slub reloads the per cpu slab if the page does not satisfy the NUMA condition. Track those reloads since doing so has a performance impact. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index b42715294147..5b228b785377 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -24,6 +24,7 @@ enum stat_item { ALLOC_FROM_PARTIAL, /* Cpu slab acquired from partial list */ ALLOC_SLAB, /* Cpu slab acquired from page allocator */ ALLOC_REFILL, /* Refill cpu slab from slab freelist */ + ALLOC_NODE_MISMATCH, /* Switching cpu slab */ FREE_SLAB, /* Slab freed to the page allocator */ CPUSLAB_FLUSH, /* Abandoning of the cpu slab */ DEACTIVATE_FULL, /* Cpu slab was full when deactivated */ -- cgit v1.2.3 From 03e404af26dc2ea0d278d7a342de0aab394793ce Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 1 Jun 2011 12:25:58 -0500 Subject: slub: fast release on full slab Make deactivation occur implicitly while checking out the current freelist. This avoids one cmpxchg operation on a slab that is now fully in use. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 5b228b785377..71441f89729b 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -32,6 +32,7 @@ enum stat_item { DEACTIVATE_TO_HEAD, /* Cpu slab was moved to the head of partials */ DEACTIVATE_TO_TAIL, /* Cpu slab was moved to the tail of partials */ DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */ + DEACTIVATE_BYPASS, /* Implicit deactivation */ ORDER_FALLBACK, /* Number of times fallback was necessary */ CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */ CMPXCHG_DOUBLE_FAIL, /* Number of times that cmpxchg double did not match */ -- cgit v1.2.3 From ea6bd8ee1a2ccdffc38b2b1fcfe941addfafaade Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 6 Jul 2011 09:46:18 -0500 Subject: SLUB: Fix build breakage in linux/mm_types.h On Wed, 6 Jul 2011, Jonathan Cameron wrote: > Getting: > > CHK include/linux/version.h > CHK include/generated/utsrelease.h > make[1]: `include/generated/mach-types.h' is up to date. > CC arch/arm/kernel/asm-offsets.s > In file included from include/linux/sched.h:64:0, > from arch/arm/kernel/asm-offsets.c:13: > include/linux/mm_types.h:74:15: error: duplicate member '_count' > make[1]: *** [arch/arm/kernel/asm-offsets.s] Error 1 > make: *** [prepare0] Error 2 > > Issue looks to have been introduced by > > mm: Rearrange struct page > > fc9bb8c768abe7ae10861c3510e01a95f98d5933 > > Guessing it's a known issue, but just thought I'd flag it up in case > it's something very specific about my build. > > gcc-2.6 armv7a > > Reverting that patch works, but given I don't know the history, I'm > not proposing doing that in general! Well _count exists in two unionized structs but always has the same offset within the larger struct. Maybe ARM creates different offsets there for some reason? The following is a patch to restructure the union / structs combo in such a way that only a single definition of _count Reported-by: Jonathan Cameron Tested-by: Piotr Hosowicz Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/mm_types.h | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) (limited to 'include') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 3d76a433d52f..6dc6413f5deb 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -49,32 +49,27 @@ struct page { * see PAGE_MAPPING_ANON below. */ /* Second double word */ - union { - struct { + struct { + union { pgoff_t index; /* Our offset within mapping. */ + void *freelist; /* slub first free object */ + }; + + union { atomic_t _mapcount; /* Count of ptes mapped in mms, * to show when page is mapped * & limit reverse map searches. */ - atomic_t _count; /* Usage count, see below. */ - }; - struct { /* SLUB cmpxchg_double area */ - void *freelist; - union { - unsigned long counters; - struct { + /* Used for cmpxchg_double in slub */ + unsigned long counters; + struct { unsigned inuse:16; unsigned objects:15; unsigned frozen:1; - /* - * Kernel may make use of this field even when slub - * uses the rest of the double word! - */ - atomic_t _count; - }; }; }; + atomic_t _count; /* Usage count, see below. */ }; /* Third double word block */ -- cgit v1.2.3 From 3adf004d82c03ca9d57cbe960657757a71e4c2e7 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Mon, 18 Jul 2011 15:16:55 +0300 Subject: Revert "SLUB: Fix build breakage in linux/mm_types.h" This reverts commit ea6bd8ee1a2ccdffc38b2b1fcfe941addfafaade. --- include/linux/mm_types.h | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'include') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 6dc6413f5deb..3d76a433d52f 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -49,27 +49,32 @@ struct page { * see PAGE_MAPPING_ANON below. */ /* Second double word */ - struct { - union { + union { + struct { pgoff_t index; /* Our offset within mapping. */ - void *freelist; /* slub first free object */ - }; - - union { atomic_t _mapcount; /* Count of ptes mapped in mms, * to show when page is mapped * & limit reverse map searches. */ + atomic_t _count; /* Usage count, see below. */ + }; - /* Used for cmpxchg_double in slub */ - unsigned long counters; - struct { + struct { /* SLUB cmpxchg_double area */ + void *freelist; + union { + unsigned long counters; + struct { unsigned inuse:16; unsigned objects:15; unsigned frozen:1; + /* + * Kernel may make use of this field even when slub + * uses the rest of the double word! + */ + atomic_t _count; + }; }; }; - atomic_t _count; /* Usage count, see below. */ }; /* Third double word block */ -- cgit v1.2.3 From 013e896373fc53f3d3c039364a25ccbd1fc0729a Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 14 Jul 2011 12:48:14 -0500 Subject: Avoid duplicate _count variables in page_struct Restructure the union / struct cascade in struct page so that we only have one definition of _count. Tested-by: Hugh Dickins Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/mm_types.h | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) (limited to 'include') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 3d76a433d52f..774b8952deb4 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -49,30 +49,31 @@ struct page { * see PAGE_MAPPING_ANON below. */ /* Second double word */ - union { - struct { + struct { + union { pgoff_t index; /* Our offset within mapping. */ - atomic_t _mapcount; /* Count of ptes mapped in mms, + void *freelist; /* slub first free object */ + }; + + union { + /* Used for cmpxchg_double in slub */ + unsigned long counters; + + struct { + + union { + atomic_t _mapcount; /* Count of ptes mapped in mms, * to show when page is mapped * & limit reverse map searches. */ - atomic_t _count; /* Usage count, see below. */ - }; - struct { /* SLUB cmpxchg_double area */ - void *freelist; - union { - unsigned long counters; - struct { - unsigned inuse:16; - unsigned objects:15; - unsigned frozen:1; - /* - * Kernel may make use of this field even when slub - * uses the rest of the double word! - */ - atomic_t _count; + struct { + unsigned inuse:16; + unsigned objects:15; + unsigned frozen:1; + }; }; + atomic_t _count; /* Usage count, see below. */ }; }; }; -- cgit v1.2.3