Skip to content

Commit 25d8c0d

Browse files
jrfastabdavem330
jrfastab
authored and
davem330
committed
net: rcu-ify tcf_proto
rcu'ify tcf_proto this allows calling tc_classify() without holding any locks. Updaters are protected by RTNL. This patch prepares the core net_sched infrastracture for running the classifier/action chains without holding the qdisc lock however it does nothing to ensure cls_xxx and act_xxx types also work without locking. Additional patches are required to address the fall out. Signed-off-by: John Fastabend <john.r.fastabend@intel.com> Acked-by: Eric Dumazet <edumazet@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent 46e5da4 commit 25d8c0d

17 files changed

+121
-88
lines changed

‎include/net/sch_generic.h‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ struct Qdisc_class_ops {
143143
void (*walk)(struct Qdisc *, struct qdisc_walker * arg);
144144

145145
/* Filter manipulation */
146-
struct tcf_proto ** (*tcf_chain)(struct Qdisc *, unsigned long);
146+
struct tcf_proto __rcu ** (*tcf_chain)(struct Qdisc *, unsigned long);
147147
unsigned long (*bind_tcf)(struct Qdisc *, unsigned long,
148148
u32 classid);
149149
void (*unbind_tcf)(struct Qdisc *, unsigned long);
@@ -212,8 +212,8 @@ struct tcf_proto_ops {
212212

213213
struct tcf_proto {
214214
/* Fast access part */
215-
struct tcf_proto *next;
216-
void *root;
215+
struct tcf_proto __rcu *next;
216+
void __rcu *root;
217217
int (*classify)(struct sk_buff *,
218218
const struct tcf_proto *,
219219
struct tcf_result *);
@@ -225,6 +225,7 @@ struct tcf_proto {
225225
struct Qdisc *q;
226226
void *data;
227227
const struct tcf_proto_ops *ops;
228+
struct rcu_head rcu;
228229
};
229230

230231
struct qdisc_skb_cb {
@@ -378,7 +379,7 @@ struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
378379
void __qdisc_calculate_pkt_len(struct sk_buff *skb,
379380
const struct qdisc_size_table *stab);
380381
void tcf_destroy(struct tcf_proto *tp);
381-
void tcf_destroy_chain(struct tcf_proto **fl);
382+
void tcf_destroy_chain(struct tcf_proto __rcu **fl);
382383

383384
/* Reset all TX qdiscs greater then index of a device. */
384385
static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i)

‎net/sched/cls_api.c‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -117,15 +117,15 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n)
117117
{
118118
struct net *net = sock_net(skb->sk);
119119
struct nlattr *tca[TCA_MAX + 1];
120-
spinlock_t *root_lock;
121120
struct tcmsg *t;
122121
u32 protocol;
123122
u32 prio;
124123
u32 nprio;
125124
u32 parent;
126125
struct net_device *dev;
127126
struct Qdisc *q;
128-
struct tcf_proto **back, **chain;
127+
struct tcf_proto __rcu **back;
128+
struct tcf_proto __rcu **chain;
129129
struct tcf_proto *tp;
130130
const struct tcf_proto_ops *tp_ops;
131131
const struct Qdisc_class_ops *cops;
@@ -197,7 +197,9 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n)
197197
goto errout;
198198

199199
/* Check the chain for existence of proto-tcf with this priority */
200-
for (back = chain; (tp = *back) != NULL; back = &tp->next) {
200+
for (back = chain;
201+
(tp = rtnl_dereference(*back)) != NULL;
202+
back = &tp->next) {
201203
if (tp->prio >= prio) {
202204
if (tp->prio == prio) {
203205
if (!nprio ||
@@ -209,8 +211,6 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n)
209211
}
210212
}
211213

212-
root_lock = qdisc_root_sleeping_lock(q);
213-
214214
if (tp == NULL) {
215215
/* Proto-tcf does not exist, create new one */
216216

@@ -259,7 +259,8 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n)
259259
}
260260
tp->ops = tp_ops;
261261
tp->protocol = protocol;
262-
tp->prio = nprio ? : TC_H_MAJ(tcf_auto_prio(*back));
262+
tp->prio = nprio ? :
263+
TC_H_MAJ(tcf_auto_prio(rtnl_dereference(*back)));
263264
tp->q = q;
264265
tp->classify = tp_ops->classify;
265266
tp->classid = parent;
@@ -280,9 +281,9 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n)
280281

281282
if (fh == 0) {
282283
if (n->nlmsg_type == RTM_DELTFILTER && t->tcm_handle == 0) {
283-
spin_lock_bh(root_lock);
284-
*back = tp->next;
285-
spin_unlock_bh(root_lock);
284+
struct tcf_proto *next = rtnl_dereference(tp->next);
285+
286+
RCU_INIT_POINTER(*back, next);
286287

287288
tfilter_notify(net, skb, n, tp, fh, RTM_DELTFILTER);
288289
tcf_destroy(tp);
@@ -322,10 +323,8 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n)
322323
n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE);
323324
if (err == 0) {
324325
if (tp_created) {
325-
spin_lock_bh(root_lock);
326-
tp->next = *back;
327-
*back = tp;
328-
spin_unlock_bh(root_lock);
326+
RCU_INIT_POINTER(tp->next, rtnl_dereference(*back));
327+
rcu_assign_pointer(*back, tp);
329328
}
330329
tfilter_notify(net, skb, n, tp, fh, RTM_NEWTFILTER);
331330
} else {
@@ -420,7 +419,7 @@ static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
420419
int s_t;
421420
struct net_device *dev;
422421
struct Qdisc *q;
423-
struct tcf_proto *tp, **chain;
422+
struct tcf_proto *tp, __rcu **chain;
424423
struct tcmsg *tcm = nlmsg_data(cb->nlh);
425424
unsigned long cl = 0;
426425
const struct Qdisc_class_ops *cops;
@@ -454,7 +453,8 @@ static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
454453

455454
s_t = cb->args[0];
456455

457-
for (tp = *chain, t = 0; tp; tp = tp->next, t++) {
456+
for (tp = rtnl_dereference(*chain), t = 0;
457+
tp; tp = rtnl_dereference(tp->next), t++) {
458458
if (t < s_t)
459459
continue;
460460
if (TC_H_MAJ(tcm->tcm_info) &&

‎net/sched/sch_api.c‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1781,7 +1781,7 @@ int tc_classify_compat(struct sk_buff *skb, const struct tcf_proto *tp,
17811781
__be16 protocol = skb->protocol;
17821782
int err;
17831783

1784-
for (; tp; tp = tp->next) {
1784+
for (; tp; tp = rcu_dereference_bh(tp->next)) {
17851785
if (tp->protocol != protocol &&
17861786
tp->protocol != htons(ETH_P_ALL))
17871787
continue;
@@ -1833,15 +1833,15 @@ void tcf_destroy(struct tcf_proto *tp)
18331833
{
18341834
tp->ops->destroy(tp);
18351835
module_put(tp->ops->owner);
1836-
kfree(tp);
1836+
kfree_rcu(tp, rcu);
18371837
}
18381838

1839-
void tcf_destroy_chain(struct tcf_proto **fl)
1839+
void tcf_destroy_chain(struct tcf_proto __rcu **fl)
18401840
{
18411841
struct tcf_proto *tp;
18421842

1843-
while ((tp = *fl) != NULL) {
1844-
*fl = tp->next;
1843+
while ((tp = rtnl_dereference(*fl)) != NULL) {
1844+
RCU_INIT_POINTER(*fl, tp->next);
18451845
tcf_destroy(tp);
18461846
}
18471847
}

‎net/sched/sch_atm.c‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141

4242
struct atm_flow_data {
4343
struct Qdisc *q; /* FIFO, TBF, etc. */
44-
struct tcf_proto *filter_list;
44+
struct tcf_proto __rcu *filter_list;
4545
struct atm_vcc *vcc; /* VCC; NULL if VCC is closed */
4646
void (*old_pop)(struct atm_vcc *vcc,
4747
struct sk_buff *skb); /* chaining */
@@ -273,7 +273,7 @@ static int atm_tc_change(struct Qdisc *sch, u32 classid, u32 parent,
273273
error = -ENOBUFS;
274274
goto err_out;
275275
}
276-
flow->filter_list = NULL;
276+
RCU_INIT_POINTER(flow->filter_list, NULL);
277277
flow->q = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, classid);
278278
if (!flow->q)
279279
flow->q = &noop_qdisc;
@@ -311,7 +311,7 @@ static int atm_tc_delete(struct Qdisc *sch, unsigned long arg)
311311
pr_debug("atm_tc_delete(sch %p,[qdisc %p],flow %p)\n", sch, p, flow);
312312
if (list_empty(&flow->list))
313313
return -EINVAL;
314-
if (flow->filter_list || flow == &p->link)
314+
if (rcu_access_pointer(flow->filter_list) || flow == &p->link)
315315
return -EBUSY;
316316
/*
317317
* Reference count must be 2: one for "keepalive" (set at class
@@ -345,7 +345,8 @@ static void atm_tc_walk(struct Qdisc *sch, struct qdisc_walker *walker)
345345
}
346346
}
347347

348-
static struct tcf_proto **atm_tc_find_tcf(struct Qdisc *sch, unsigned long cl)
348+
static struct tcf_proto __rcu **atm_tc_find_tcf(struct Qdisc *sch,
349+
unsigned long cl)
349350
{
350351
struct atm_qdisc_data *p = qdisc_priv(sch);
351352
struct atm_flow_data *flow = (struct atm_flow_data *)cl;
@@ -369,11 +370,12 @@ static int atm_tc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
369370
flow = NULL;
370371
if (TC_H_MAJ(skb->priority) != sch->handle ||
371372
!(flow = (struct atm_flow_data *)atm_tc_get(sch, skb->priority))) {
373+
struct tcf_proto *fl;
374+
372375
list_for_each_entry(flow, &p->flows, list) {
373-
if (flow->filter_list) {
374-
result = tc_classify_compat(skb,
375-
flow->filter_list,
376-
&res);
376+
fl = rcu_dereference_bh(flow->filter_list);
377+
if (fl) {
378+
result = tc_classify_compat(skb, fl, &res);
377379
if (result < 0)
378380
continue;
379381
flow = (struct atm_flow_data *)res.class;
@@ -544,7 +546,7 @@ static int atm_tc_init(struct Qdisc *sch, struct nlattr *opt)
544546
if (!p->link.q)
545547
p->link.q = &noop_qdisc;
546548
pr_debug("atm_tc_init: link (%p) qdisc %p\n", &p->link, p->link.q);
547-
p->link.filter_list = NULL;
549+
RCU_INIT_POINTER(p->link.filter_list, NULL);
548550
p->link.vcc = NULL;
549551
p->link.sock = NULL;
550552
p->link.classid = sch->handle;

‎net/sched/sch_cbq.c‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ struct cbq_class {
133133
struct gnet_stats_rate_est64 rate_est;
134134
struct tc_cbq_xstats xstats;
135135

136-
struct tcf_proto *filter_list;
136+
struct tcf_proto __rcu *filter_list;
137137

138138
int refcnt;
139139
int filters;
@@ -221,6 +221,7 @@ cbq_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
221221
struct cbq_class **defmap;
222222
struct cbq_class *cl = NULL;
223223
u32 prio = skb->priority;
224+
struct tcf_proto *fl;
224225
struct tcf_result res;
225226

226227
/*
@@ -235,11 +236,12 @@ cbq_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
235236
int result = 0;
236237
defmap = head->defaults;
237238

239+
fl = rcu_dereference_bh(head->filter_list);
238240
/*
239241
* Step 2+n. Apply classifier.
240242
*/
241-
if (!head->filter_list ||
242-
(result = tc_classify_compat(skb, head->filter_list, &res)) < 0)
243+
result = tc_classify_compat(skb, fl, &res);
244+
if (!fl || result < 0)
243245
goto fallback;
244246

245247
cl = (void *)res.class;
@@ -1954,7 +1956,8 @@ static int cbq_delete(struct Qdisc *sch, unsigned long arg)
19541956
return 0;
19551957
}
19561958

1957-
static struct tcf_proto **cbq_find_tcf(struct Qdisc *sch, unsigned long arg)
1959+
static struct tcf_proto __rcu **cbq_find_tcf(struct Qdisc *sch,
1960+
unsigned long arg)
19581961
{
19591962
struct cbq_sched_data *q = qdisc_priv(sch);
19601963
struct cbq_class *cl = (struct cbq_class *)arg;

‎net/sched/sch_choke.c‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ struct choke_sched_data {
5757

5858
/* Variables */
5959
struct red_vars vars;
60-
struct tcf_proto *filter_list;
60+
struct tcf_proto __rcu *filter_list;
6161
struct {
6262
u32 prob_drop; /* Early probability drops */
6363
u32 prob_mark; /* Early probability marks */
@@ -193,9 +193,11 @@ static bool choke_classify(struct sk_buff *skb,
193193
{
194194
struct choke_sched_data *q = qdisc_priv(sch);
195195
struct tcf_result res;
196+
struct tcf_proto *fl;
196197
int result;
197198

198-
result = tc_classify(skb, q->filter_list, &res);
199+
fl = rcu_dereference_bh(q->filter_list);
200+
result = tc_classify(skb, fl, &res);
199201
if (result >= 0) {
200202
#ifdef CONFIG_NET_CLS_ACT
201203
switch (result) {
@@ -249,19 +251,19 @@ static bool choke_match_random(const struct choke_sched_data *q,
249251
return false;
250252

251253
oskb = choke_peek_random(q, pidx);
252-
if (q->filter_list)
254+
if (rcu_access_pointer(q->filter_list))
253255
return choke_get_classid(nskb) == choke_get_classid(oskb);
254256

255257
return choke_match_flow(oskb, nskb);
256258
}
257259

258260
static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch)
259261
{
262+
int ret = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
260263
struct choke_sched_data *q = qdisc_priv(sch);
261264
const struct red_parms *p = &q->parms;
262-
int ret = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
263265

264-
if (q->filter_list) {
266+
if (rcu_access_pointer(q->filter_list)) {
265267
/* If using external classifiers, get result and record it. */
266268
if (!choke_classify(skb, sch, &ret))
267269
goto other_drop; /* Packet was eaten by filter */
@@ -554,7 +556,8 @@ static unsigned long choke_bind(struct Qdisc *sch, unsigned long parent,
554556
return 0;
555557
}
556558

557-
static struct tcf_proto **choke_find_tcf(struct Qdisc *sch, unsigned long cl)
559+
static struct tcf_proto __rcu **choke_find_tcf(struct Qdisc *sch,
560+
unsigned long cl)
558561
{
559562
struct choke_sched_data *q = qdisc_priv(sch);
560563

‎net/sched/sch_drr.c‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ struct drr_class {
3535

3636
struct drr_sched {
3737
struct list_head active;
38-
struct tcf_proto *filter_list;
38+
struct tcf_proto __rcu *filter_list;
3939
struct Qdisc_class_hash clhash;
4040
};
4141

@@ -184,7 +184,8 @@ static void drr_put_class(struct Qdisc *sch, unsigned long arg)
184184
drr_destroy_class(sch, cl);
185185
}
186186

187-
static struct tcf_proto **drr_tcf_chain(struct Qdisc *sch, unsigned long cl)
187+
static struct tcf_proto __rcu **drr_tcf_chain(struct Qdisc *sch,
188+
unsigned long cl)
188189
{
189190
struct drr_sched *q = qdisc_priv(sch);
190191

@@ -319,6 +320,7 @@ static struct drr_class *drr_classify(struct sk_buff *skb, struct Qdisc *sch,
319320
struct drr_sched *q = qdisc_priv(sch);
320321
struct drr_class *cl;
321322
struct tcf_result res;
323+
struct tcf_proto *fl;
322324
int result;
323325

324326
if (TC_H_MAJ(skb->priority ^ sch->handle) == 0) {
@@ -328,7 +330,8 @@ static struct drr_class *drr_classify(struct sk_buff *skb, struct Qdisc *sch,
328330
}
329331

330332
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
331-
result = tc_classify(skb, q->filter_list, &res);
333+
fl = rcu_dereference_bh(q->filter_list);
334+
result = tc_classify(skb, fl, &res);
332335
if (result >= 0) {
333336
#ifdef CONFIG_NET_CLS_ACT
334337
switch (result) {

‎net/sched/sch_dsmark.c‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737

3838
struct dsmark_qdisc_data {
3939
struct Qdisc *q;
40-
struct tcf_proto *filter_list;
40+
struct tcf_proto __rcu *filter_list;
4141
u8 *mask; /* "owns" the array */
4242
u8 *value;
4343
u16 indices;
@@ -186,8 +186,8 @@ static void dsmark_walk(struct Qdisc *sch, struct qdisc_walker *walker)
186186
}
187187
}
188188

189-
static inline struct tcf_proto **dsmark_find_tcf(struct Qdisc *sch,
190-
unsigned long cl)
189+
static inline struct tcf_proto __rcu **dsmark_find_tcf(struct Qdisc *sch,
190+
unsigned long cl)
191191
{
192192
struct dsmark_qdisc_data *p = qdisc_priv(sch);
193193
return &p->filter_list;
@@ -229,7 +229,8 @@ static int dsmark_enqueue(struct sk_buff *skb, struct Qdisc *sch)
229229
skb->tc_index = TC_H_MIN(skb->priority);
230230
else {
231231
struct tcf_result res;
232-
int result = tc_classify(skb, p->filter_list, &res);
232+
struct tcf_proto *fl = rcu_dereference_bh(p->filter_list);
233+
int result = tc_classify(skb, fl, &res);
233234

234235
pr_debug("result %d class 0x%04x\n", result, res.classid);
235236

0 commit comments

Comments
 (0)