Skip to content

Commit

Permalink
x86/resctrl: Prepare for different scope for control/monitor operations
Browse files Browse the repository at this point in the history 
Resctrl assumes that control and monitor operations on a resource are
performed at the same scope.

Prepare for systems that use different scope (specifically Intel needs
to split the RDT_RESOURCE_L3 resource to use L3 scope for cache control
and NODE scope for cache occupancy and memory bandwidth monitoring).

Create separate domain lists for control and monitor operations.

Note that errors during initialization of either control or monitor
functions on a domain would previously result in that domain being
excluded from both control and monitor operations. Now the domains are
allocated independently it is no longer required to disable both control
and monitor operations if either fail.

Signed-off-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Reinette Chatre <reinette.chatre@intel.com>
Tested-by: Babu Moger <babu.moger@amd.com>
Link: https://lore.kernel.org/r/20240628215619.76401-4-tony.luck@intel.com
  • Loading branch information
@aegl @bp3tk0v
aegl authored and bp3tk0v committed Jul 2, 2024
1 parent c103d4d commit cd84f72
Show file tree
Hide file tree
Showing 7 changed files with 240 additions and 96 deletions.
224 changes: 171 additions & 53 deletions arch/x86/kernel/cpu/resctrl/core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,19 @@ static void mba_wrmsr_intel(struct msr_param *m);
static void cat_wrmsr(struct msr_param *m);
static void mba_wrmsr_amd(struct msr_param *m);

#define domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.domains)
#define ctrl_domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.ctrl_domains)
#define mon_domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.mon_domains)

struct rdt_hw_resource rdt_resources_all[] = {
[RDT_RESOURCE_L3] =
{
.r_resctrl = {
.rid = RDT_RESOURCE_L3,
.name = "L3",
.scope = RESCTRL_L3_CACHE,
.domains = domain_init(RDT_RESOURCE_L3),
.ctrl_scope = RESCTRL_L3_CACHE,
.mon_scope = RESCTRL_L3_CACHE,
.ctrl_domains = ctrl_domain_init(RDT_RESOURCE_L3),
.mon_domains = mon_domain_init(RDT_RESOURCE_L3),
.parse_ctrlval = parse_cbm,
.format_str = "%d=%0*x",
.fflags = RFTYPE_RES_CACHE,
Expand All @@ -82,8 +85,8 @@ struct rdt_hw_resource rdt_resources_all[] = {
.r_resctrl = {
.rid = RDT_RESOURCE_L2,
.name = "L2",
.scope = RESCTRL_L2_CACHE,
.domains = domain_init(RDT_RESOURCE_L2),
.ctrl_scope = RESCTRL_L2_CACHE,
.ctrl_domains = ctrl_domain_init(RDT_RESOURCE_L2),
.parse_ctrlval = parse_cbm,
.format_str = "%d=%0*x",
.fflags = RFTYPE_RES_CACHE,
Expand All @@ -96,8 +99,8 @@ struct rdt_hw_resource rdt_resources_all[] = {
.r_resctrl = {
.rid = RDT_RESOURCE_MBA,
.name = "MB",
.scope = RESCTRL_L3_CACHE,
.domains = domain_init(RDT_RESOURCE_MBA),
.ctrl_scope = RESCTRL_L3_CACHE,
.ctrl_domains = ctrl_domain_init(RDT_RESOURCE_MBA),
.parse_ctrlval = parse_bw,
.format_str = "%d=%*u",
.fflags = RFTYPE_RES_MB,
Expand All @@ -108,8 +111,8 @@ struct rdt_hw_resource rdt_resources_all[] = {
.r_resctrl = {
.rid = RDT_RESOURCE_SMBA,
.name = "SMBA",
.scope = RESCTRL_L3_CACHE,
.domains = domain_init(RDT_RESOURCE_SMBA),
.ctrl_scope = RESCTRL_L3_CACHE,
.ctrl_domains = ctrl_domain_init(RDT_RESOURCE_SMBA),
.parse_ctrlval = parse_bw,
.format_str = "%d=%*u",
.fflags = RFTYPE_RES_MB,
Expand Down Expand Up @@ -349,13 +352,28 @@ static void cat_wrmsr(struct msr_param *m)
wrmsrl(hw_res->msr_base + i, hw_dom->ctrl_val[i]);
}

struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r)
struct rdt_domain *get_ctrl_domain_from_cpu(int cpu, struct rdt_resource *r)
{
struct rdt_domain *d;

lockdep_assert_cpus_held();

list_for_each_entry(d, &r->domains, hdr.list) {
list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
/* Find the domain that contains this CPU */
if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask))
return d;
}

return NULL;
}

struct rdt_domain *get_mon_domain_from_cpu(int cpu, struct rdt_resource *r)
{
struct rdt_domain *d;

lockdep_assert_cpus_held();

list_for_each_entry(d, &r->mon_domains, hdr.list) {
/* Find the domain that contains this CPU */
if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask))
return d;
Expand All @@ -379,26 +397,26 @@ void rdt_ctrl_update(void *arg)
}

/*
* rdt_find_domain - Find a domain in a resource that matches input resource id
* rdt_find_domain - Search for a domain id in a resource domain list.
*
* Search resource r's domain list to find the resource id. If the resource
* id is found in a domain, return the domain. Otherwise, if requested by
* caller, return the first domain whose id is bigger than the input id.
* The domain list is sorted by id in ascending order.
* Search the domain list to find the domain id. If the domain id is
* found, return the domain. NULL otherwise. If the domain id is not
* found (and NULL returned) then the first domain with id bigger than
* the input id can be returned to the caller via @pos.
*/
struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id,
struct list_head **pos)
struct rdt_domain_hdr *rdt_find_domain(struct list_head *h, int id,
struct list_head **pos)
{
struct rdt_domain *d;
struct rdt_domain_hdr *d;
struct list_head *l;

list_for_each(l, &r->domains) {
d = list_entry(l, struct rdt_domain, hdr.list);
list_for_each(l, h) {
d = list_entry(l, struct rdt_domain_hdr, list);
/* When id is found, return its domain. */
if (id == d->hdr.id)
if (id == d->id)
return d;
/* Stop searching when finding id's position in sorted list. */
if (id < d->hdr.id)
if (id < d->id)
break;
}

Expand Down Expand Up @@ -494,38 +512,29 @@ static int get_domain_id_from_scope(int cpu, enum resctrl_scope scope)
return -EINVAL;
}

/*
* domain_add_cpu - Add a cpu to a resource's domain list.
*
* If an existing domain in the resource r's domain list matches the cpu's
* resource id, add the cpu in the domain.
*
* Otherwise, a new domain is allocated and inserted into the right position
* in the domain list sorted by id in ascending order.
*
* The order in the domain list is visible to users when we print entries
* in the schemata file and schemata input is validated to have the same order
* as this list.
*/
static void domain_add_cpu(int cpu, struct rdt_resource *r)
static void domain_add_cpu_ctrl(int cpu, struct rdt_resource *r)
{
int id = get_domain_id_from_scope(cpu, r->scope);
int id = get_domain_id_from_scope(cpu, r->ctrl_scope);
struct list_head *add_pos = NULL;
struct rdt_hw_domain *hw_dom;
struct rdt_domain_hdr *hdr;
struct rdt_domain *d;
int err;

lockdep_assert_held(&domain_list_lock);

if (id < 0) {
pr_warn_once("Can't find domain id for CPU:%d scope:%d for resource %s\n",
cpu, r->scope, r->name);
pr_warn_once("Can't find control domain id for CPU:%d scope:%d for resource %s\n",
cpu, r->ctrl_scope, r->name);
return;
}

d = rdt_find_domain(r, id, &add_pos);
hdr = rdt_find_domain(&r->ctrl_domains, id, &add_pos);
if (hdr) {
if (WARN_ON_ONCE(hdr->type != RESCTRL_CTRL_DOMAIN))
return;
d = container_of(hdr, struct rdt_domain, hdr);

if (d) {
cpumask_set_cpu(cpu, &d->hdr.cpu_mask);
if (r->cache.arch_has_per_cpu_cfg)
rdt_domain_reconfigure_cdp(r);
Expand All @@ -538,54 +547,116 @@ static void domain_add_cpu(int cpu, struct rdt_resource *r)

d = &hw_dom->d_resctrl;
d->hdr.id = id;
d->hdr.type = RESCTRL_CTRL_DOMAIN;
cpumask_set_cpu(cpu, &d->hdr.cpu_mask);

rdt_domain_reconfigure_cdp(r);

if (r->alloc_capable && domain_setup_ctrlval(r, d)) {
if (domain_setup_ctrlval(r, d)) {
domain_free(hw_dom);
return;
}

if (r->mon_capable && arch_domain_mbm_alloc(r->num_rmid, hw_dom)) {
list_add_tail_rcu(&d->hdr.list, add_pos);

err = resctrl_online_ctrl_domain(r, d);
if (err) {
list_del_rcu(&d->hdr.list);
synchronize_rcu();
domain_free(hw_dom);
}
}

static void domain_add_cpu_mon(int cpu, struct rdt_resource *r)
{
int id = get_domain_id_from_scope(cpu, r->mon_scope);
struct list_head *add_pos = NULL;
struct rdt_hw_domain *hw_dom;
struct rdt_domain_hdr *hdr;
struct rdt_domain *d;
int err;

lockdep_assert_held(&domain_list_lock);

if (id < 0) {
pr_warn_once("Can't find monitor domain id for CPU:%d scope:%d for resource %s\n",
cpu, r->mon_scope, r->name);
return;
}

hdr = rdt_find_domain(&r->mon_domains, id, &add_pos);
if (hdr) {
if (WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN))
return;
d = container_of(hdr, struct rdt_domain, hdr);

cpumask_set_cpu(cpu, &d->hdr.cpu_mask);
return;
}

hw_dom = kzalloc_node(sizeof(*hw_dom), GFP_KERNEL, cpu_to_node(cpu));
if (!hw_dom)
return;

d = &hw_dom->d_resctrl;
d->hdr.id = id;
d->hdr.type = RESCTRL_MON_DOMAIN;
cpumask_set_cpu(cpu, &d->hdr.cpu_mask);

if (arch_domain_mbm_alloc(r->num_rmid, hw_dom)) {
domain_free(hw_dom);
return;
}

list_add_tail_rcu(&d->hdr.list, add_pos);

err = resctrl_online_domain(r, d);
err = resctrl_online_mon_domain(r, d);
if (err) {
list_del_rcu(&d->hdr.list);
synchronize_rcu();
domain_free(hw_dom);
}
}

static void domain_remove_cpu(int cpu, struct rdt_resource *r)
static void domain_add_cpu(int cpu, struct rdt_resource *r)
{
if (r->alloc_capable)
domain_add_cpu_ctrl(cpu, r);
if (r->mon_capable)
domain_add_cpu_mon(cpu, r);
}

static void domain_remove_cpu_ctrl(int cpu, struct rdt_resource *r)
{
int id = get_domain_id_from_scope(cpu, r->scope);
int id = get_domain_id_from_scope(cpu, r->ctrl_scope);
struct rdt_hw_domain *hw_dom;
struct rdt_domain_hdr *hdr;
struct rdt_domain *d;

lockdep_assert_held(&domain_list_lock);

if (id < 0) {
pr_warn_once("Can't find domain id for CPU:%d scope:%d for resource %s\n",
cpu, r->scope, r->name);
pr_warn_once("Can't find control domain id for CPU:%d scope:%d for resource %s\n",
cpu, r->ctrl_scope, r->name);
return;
}

d = rdt_find_domain(r, id, NULL);
if (!d) {
pr_warn("Couldn't find domain with id=%d for CPU %d\n", id, cpu);
hdr = rdt_find_domain(&r->ctrl_domains, id, NULL);
if (!hdr) {
pr_warn("Can't find control domain for id=%d for CPU %d for resource %s\n",
id, cpu, r->name);
return;
}

if (WARN_ON_ONCE(hdr->type != RESCTRL_CTRL_DOMAIN))
return;

d = container_of(hdr, struct rdt_domain, hdr);
hw_dom = resctrl_to_arch_dom(d);

cpumask_clear_cpu(cpu, &d->hdr.cpu_mask);
if (cpumask_empty(&d->hdr.cpu_mask)) {
resctrl_offline_domain(r, d);
resctrl_offline_ctrl_domain(r, d);
list_del_rcu(&d->hdr.list);
synchronize_rcu();

Expand All @@ -601,6 +672,53 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r)
}
}

static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r)
{
int id = get_domain_id_from_scope(cpu, r->mon_scope);
struct rdt_hw_domain *hw_dom;
struct rdt_domain_hdr *hdr;
struct rdt_domain *d;

lockdep_assert_held(&domain_list_lock);

if (id < 0) {
pr_warn_once("Can't find monitor domain id for CPU:%d scope:%d for resource %s\n",
cpu, r->mon_scope, r->name);
return;
}

hdr = rdt_find_domain(&r->mon_domains, id, NULL);
if (!hdr) {
pr_warn("Can't find monitor domain for id=%d for CPU %d for resource %s\n",
id, cpu, r->name);
return;
}

if (WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN))
return;

d = container_of(hdr, struct rdt_domain, hdr);
hw_dom = resctrl_to_arch_dom(d);

cpumask_clear_cpu(cpu, &d->hdr.cpu_mask);
if (cpumask_empty(&d->hdr.cpu_mask)) {
resctrl_offline_mon_domain(r, d);
list_del_rcu(&d->hdr.list);
synchronize_rcu();
domain_free(hw_dom);

return;
}
}

static void domain_remove_cpu(int cpu, struct rdt_resource *r)
{
if (r->alloc_capable)
domain_remove_cpu_ctrl(cpu, r);
if (r->mon_capable)
domain_remove_cpu_mon(cpu, r);
}

static void clear_closid_rmid(int cpu)
{
struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state);
Expand Down
Loading

0 comments on commit cd84f72

Please sign in to comment.