Skip to content

Commit c389a25

Browse files
kiryltorvalds
authored andcommitted
mm, thp: do not access mm->pmd_huge_pte directly
Currently mm->pmd_huge_pte protected by page table lock. It will not work with split lock. We have to have per-pmd pmd_huge_pte for proper access serialization. For now, let's just introduce wrapper to access mm->pmd_huge_pte. Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Tested-by: Alex Thorlton <athorlton@sgi.com> Cc: Alex Thorlton <athorlton@sgi.com> Cc: Ingo Molnar <mingo@redhat.com> Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> Cc: "Eric W . Biederman" <ebiederm@xmission.com> Cc: "Paul E . McKenney" <paulmck@linux.vnet.ibm.com> Cc: Al Viro <viro@zeniv.linux.org.uk> Cc: Andi Kleen <ak@linux.intel.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Dave Hansen <dave.hansen@intel.com> Cc: Dave Jones <davej@redhat.com> Cc: David Howells <dhowells@redhat.com> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Kees Cook <keescook@chromium.org> Cc: Mel Gorman <mgorman@suse.de> Cc: Michael Kerrisk <mtk.manpages@gmail.com> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Rik van Riel <riel@redhat.com> Cc: Robin Holt <robinmholt@gmail.com> Cc: Sedat Dilek <sedat.dilek@gmail.com> Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Hugh Dickins <hughd@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent 117b079 commit c389a25

File tree

4 files changed

+19
-18
lines changed

4 files changed

+19
-18
lines changed

arch/s390/mm/pgtable.c

+6-6
Original file line numberDiff line numberDiff line change
@@ -1244,11 +1244,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
12441244
assert_spin_locked(&mm->page_table_lock);
12451245

12461246
/* FIFO */
1247-
if (!mm->pmd_huge_pte)
1247+
if (!pmd_huge_pte(mm, pmdp))
12481248
INIT_LIST_HEAD(lh);
12491249
else
1250-
list_add(lh, (struct list_head *) mm->pmd_huge_pte);
1251-
mm->pmd_huge_pte = pgtable;
1250+
list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
1251+
pmd_huge_pte(mm, pmdp) = pgtable;
12521252
}
12531253

12541254
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
@@ -1260,12 +1260,12 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
12601260
assert_spin_locked(&mm->page_table_lock);
12611261

12621262
/* FIFO */
1263-
pgtable = mm->pmd_huge_pte;
1263+
pgtable = pmd_huge_pte(mm, pmdp);
12641264
lh = (struct list_head *) pgtable;
12651265
if (list_empty(lh))
1266-
mm->pmd_huge_pte = NULL;
1266+
pmd_huge_pte(mm, pmdp) = NULL;
12671267
else {
1268-
mm->pmd_huge_pte = (pgtable_t) lh->next;
1268+
pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
12691269
list_del(lh);
12701270
}
12711271
ptep = (pte_t *) pgtable;

arch/sparc/mm/tlb.c

+6-6
Original file line numberDiff line numberDiff line change
@@ -196,11 +196,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
196196
assert_spin_locked(&mm->page_table_lock);
197197

198198
/* FIFO */
199-
if (!mm->pmd_huge_pte)
199+
if (!pmd_huge_pte(mm, pmdp))
200200
INIT_LIST_HEAD(lh);
201201
else
202-
list_add(lh, (struct list_head *) mm->pmd_huge_pte);
203-
mm->pmd_huge_pte = pgtable;
202+
list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
203+
pmd_huge_pte(mm, pmdp) = pgtable;
204204
}
205205

206206
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
@@ -211,12 +211,12 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
211211
assert_spin_locked(&mm->page_table_lock);
212212

213213
/* FIFO */
214-
pgtable = mm->pmd_huge_pte;
214+
pgtable = pmd_huge_pte(mm, pmdp);
215215
lh = (struct list_head *) pgtable;
216216
if (list_empty(lh))
217-
mm->pmd_huge_pte = NULL;
217+
pmd_huge_pte(mm, pmdp) = NULL;
218218
else {
219-
mm->pmd_huge_pte = (pgtable_t) lh->next;
219+
pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
220220
list_del(lh);
221221
}
222222
pte_val(pgtable[0]) = 0;

include/linux/mm.h

+1
Original file line numberDiff line numberDiff line change
@@ -1383,6 +1383,7 @@ static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
13831383
return &mm->page_table_lock;
13841384
}
13851385

1386+
#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
13861387

13871388
static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
13881389
{

mm/pgtable-generic.c

+6-6
Original file line numberDiff line numberDiff line change
@@ -154,11 +154,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
154154
assert_spin_locked(&mm->page_table_lock);
155155

156156
/* FIFO */
157-
if (!mm->pmd_huge_pte)
157+
if (!pmd_huge_pte(mm, pmdp))
158158
INIT_LIST_HEAD(&pgtable->lru);
159159
else
160-
list_add(&pgtable->lru, &mm->pmd_huge_pte->lru);
161-
mm->pmd_huge_pte = pgtable;
160+
list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
161+
pmd_huge_pte(mm, pmdp) = pgtable;
162162
}
163163
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
164164
#endif
@@ -173,11 +173,11 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
173173
assert_spin_locked(&mm->page_table_lock);
174174

175175
/* FIFO */
176-
pgtable = mm->pmd_huge_pte;
176+
pgtable = pmd_huge_pte(mm, pmdp);
177177
if (list_empty(&pgtable->lru))
178-
mm->pmd_huge_pte = NULL;
178+
pmd_huge_pte(mm, pmdp) = NULL;
179179
else {
180-
mm->pmd_huge_pte = list_entry(pgtable->lru.next,
180+
pmd_huge_pte(mm, pmdp) = list_entry(pgtable->lru.next,
181181
struct page, lru);
182182
list_del(&pgtable->lru);
183183
}

0 commit comments

Comments
 (0)