Skip to content

Commit 6a37627

Browse files
frankjaafrankjaa
committed
s390/mm: Add gmap pmd invalidation and clearing
If the host invalidates a pmd, we also have to invalidate the corresponding gmap pmds, as well as flush them from the TLB. This is necessary, as we don't share the pmd tables between host and guest as we do with ptes. The clearing part of these three new functions sets a guest pmd entry to _SEGMENT_ENTRY_EMPTY, so the guest will fault on it and we will re-link it. Flushing the gmap is not necessary in the host's lazy local and csp cases. Both purge the TLB completely. Signed-off-by: Janosch Frank <frankja@linux.vnet.ibm.com> Reviewed-by: Martin Schwidefsky <schwidefsky@de.ibm.com> Acked-by: David Hildenbrand <david@redhat.com>
1 parent 7c4b13a commit 6a37627

File tree

3 files changed

+143
-3
lines changed

3 files changed

+143
-3
lines changed

‎arch/s390/include/asm/pgtable.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1118,6 +1118,10 @@ int set_pgste_bits(struct mm_struct *mm, unsigned long addr,
11181118
int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep);
11191119
int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
11201120
unsigned long *oldpte, unsigned long *oldpgste);
1121+
void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr);
1122+
void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr);
1123+
void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr);
1124+
void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr);
11211125

11221126
/*
11231127
* Certain architectures need to do special things when PTEs

‎arch/s390/mm/gmap.c‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2221,6 +2221,131 @@ void ptep_notify(struct mm_struct *mm, unsigned long vmaddr,
22212221
}
22222222
EXPORT_SYMBOL_GPL(ptep_notify);
22232223

2224+
static void pmdp_notify_gmap(struct gmap *gmap, pmd_t *pmdp,
2225+
unsigned long gaddr)
2226+
{
2227+
pmd_val(*pmdp) &= ~_SEGMENT_ENTRY_GMAP_IN;
2228+
gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1);
2229+
}
2230+
2231+
static void gmap_pmdp_clear(struct mm_struct *mm, unsigned long vmaddr,
2232+
int purge)
2233+
{
2234+
pmd_t *pmdp;
2235+
struct gmap *gmap;
2236+
unsigned long gaddr;
2237+
2238+
rcu_read_lock();
2239+
list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
2240+
spin_lock(&gmap->guest_table_lock);
2241+
pmdp = (pmd_t *)radix_tree_delete(&gmap->host_to_guest,
2242+
vmaddr >> PMD_SHIFT);
2243+
if (pmdp) {
2244+
gaddr = __gmap_segment_gaddr((unsigned long *)pmdp);
2245+
pmdp_notify_gmap(gmap, pmdp, gaddr);
2246+
WARN_ON(pmd_val(*pmdp) & ~_SEGMENT_ENTRY_HARDWARE_BITS_LARGE);
2247+
if (purge)
2248+
__pmdp_csp(pmdp);
2249+
pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
2250+
}
2251+
spin_unlock(&gmap->guest_table_lock);
2252+
}
2253+
rcu_read_unlock();
2254+
}
2255+
2256+
/**
2257+
* gmap_pmdp_invalidate - invalidate all affected guest pmd entries without
2258+
* flushing
2259+
* @mm: pointer to the process mm_struct
2260+
* @vmaddr: virtual address in the process address space
2261+
*/
2262+
void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr)
2263+
{
2264+
gmap_pmdp_clear(mm, vmaddr, 0);
2265+
}
2266+
EXPORT_SYMBOL_GPL(gmap_pmdp_invalidate);
2267+
2268+
/**
2269+
* gmap_pmdp_csp - csp all affected guest pmd entries
2270+
* @mm: pointer to the process mm_struct
2271+
* @vmaddr: virtual address in the process address space
2272+
*/
2273+
void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr)
2274+
{
2275+
gmap_pmdp_clear(mm, vmaddr, 1);
2276+
}
2277+
EXPORT_SYMBOL_GPL(gmap_pmdp_csp);
2278+
2279+
/**
2280+
* gmap_pmdp_idte_local - invalidate and clear a guest pmd entry
2281+
* @mm: pointer to the process mm_struct
2282+
* @vmaddr: virtual address in the process address space
2283+
*/
2284+
void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr)
2285+
{
2286+
unsigned long *entry, gaddr;
2287+
struct gmap *gmap;
2288+
pmd_t *pmdp;
2289+
2290+
rcu_read_lock();
2291+
list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
2292+
spin_lock(&gmap->guest_table_lock);
2293+
entry = radix_tree_delete(&gmap->host_to_guest,
2294+
vmaddr >> PMD_SHIFT);
2295+
if (entry) {
2296+
pmdp = (pmd_t *)entry;
2297+
gaddr = __gmap_segment_gaddr(entry);
2298+
pmdp_notify_gmap(gmap, pmdp, gaddr);
2299+
WARN_ON(*entry & ~_SEGMENT_ENTRY_HARDWARE_BITS_LARGE);
2300+
if (MACHINE_HAS_TLB_GUEST)
2301+
__pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE,
2302+
gmap->asce, IDTE_LOCAL);
2303+
else if (MACHINE_HAS_IDTE)
2304+
__pmdp_idte(gaddr, pmdp, 0, 0, IDTE_LOCAL);
2305+
*entry = _SEGMENT_ENTRY_EMPTY;
2306+
}
2307+
spin_unlock(&gmap->guest_table_lock);
2308+
}
2309+
rcu_read_unlock();
2310+
}
2311+
EXPORT_SYMBOL_GPL(gmap_pmdp_idte_local);
2312+
2313+
/**
2314+
* gmap_pmdp_idte_global - invalidate and clear a guest pmd entry
2315+
* @mm: pointer to the process mm_struct
2316+
* @vmaddr: virtual address in the process address space
2317+
*/
2318+
void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr)
2319+
{
2320+
unsigned long *entry, gaddr;
2321+
struct gmap *gmap;
2322+
pmd_t *pmdp;
2323+
2324+
rcu_read_lock();
2325+
list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
2326+
spin_lock(&gmap->guest_table_lock);
2327+
entry = radix_tree_delete(&gmap->host_to_guest,
2328+
vmaddr >> PMD_SHIFT);
2329+
if (entry) {
2330+
pmdp = (pmd_t *)entry;
2331+
gaddr = __gmap_segment_gaddr(entry);
2332+
pmdp_notify_gmap(gmap, pmdp, gaddr);
2333+
WARN_ON(*entry & ~_SEGMENT_ENTRY_HARDWARE_BITS_LARGE);
2334+
if (MACHINE_HAS_TLB_GUEST)
2335+
__pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE,
2336+
gmap->asce, IDTE_GLOBAL);
2337+
else if (MACHINE_HAS_IDTE)
2338+
__pmdp_idte(gaddr, pmdp, 0, 0, IDTE_GLOBAL);
2339+
else
2340+
__pmdp_csp(pmdp);
2341+
*entry = _SEGMENT_ENTRY_EMPTY;
2342+
}
2343+
spin_unlock(&gmap->guest_table_lock);
2344+
}
2345+
rcu_read_unlock();
2346+
}
2347+
EXPORT_SYMBOL_GPL(gmap_pmdp_idte_global);
2348+
22242349
static inline void thp_split_mm(struct mm_struct *mm)
22252350
{
22262351
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

‎arch/s390/mm/pgtable.c‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -347,18 +347,27 @@ static inline void pmdp_idte_local(struct mm_struct *mm,
347347
mm->context.asce, IDTE_LOCAL);
348348
else
349349
__pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
350+
if (mm_has_pgste(mm))
351+
gmap_pmdp_idte_local(mm, addr);
350352
}
351353

352354
static inline void pmdp_idte_global(struct mm_struct *mm,
353355
unsigned long addr, pmd_t *pmdp)
354356
{
355-
if (MACHINE_HAS_TLB_GUEST)
357+
if (MACHINE_HAS_TLB_GUEST) {
356358
__pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
357359
mm->context.asce, IDTE_GLOBAL);
358-
else if (MACHINE_HAS_IDTE)
360+
if (mm_has_pgste(mm))
361+
gmap_pmdp_idte_global(mm, addr);
362+
} else if (MACHINE_HAS_IDTE) {
359363
__pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
360-
else
364+
if (mm_has_pgste(mm))
365+
gmap_pmdp_idte_global(mm, addr);
366+
} else {
361367
__pmdp_csp(pmdp);
368+
if (mm_has_pgste(mm))
369+
gmap_pmdp_csp(mm, addr);
370+
}
362371
}
363372

364373
static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
@@ -392,6 +401,8 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
392401
cpumask_of(smp_processor_id()))) {
393402
pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
394403
mm->context.flush_mm = 1;
404+
if (mm_has_pgste(mm))
405+
gmap_pmdp_invalidate(mm, addr);
395406
} else {
396407
pmdp_idte_global(mm, addr, pmdp);
397408
}

0 commit comments

Comments
 (0)