diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c index 1cc2cd894f877c..28dc04c848ce2e 100644 --- a/drivers/net/bonding/bond_alb.c +++ b/drivers/net/bonding/bond_alb.c @@ -1299,8 +1299,8 @@ void bond_alb_deinitialize(struct bonding *bond) rlb_deinitialize(bond); } -static netdev_tx_t bond_do_alb_xmit(struct sk_buff *skb, struct bonding *bond, - struct slave *tx_slave) +static bool bond_do_alb_xmit(struct sk_buff *skb, struct bonding *bond, + struct slave *tx_slave) { struct alb_bond_info *bond_info = &(BOND_ALB_INFO(bond)); struct ethhdr *eth_data = eth_hdr(skb); @@ -1319,7 +1319,7 @@ static netdev_tx_t bond_do_alb_xmit(struct sk_buff *skb, struct bonding *bond, } bond_dev_queue_xmit(bond, skb, tx_slave->dev); - goto out; + return true; } if (tx_slave && bond->params.tlb_dynamic_lb) { @@ -1330,11 +1330,11 @@ static netdev_tx_t bond_do_alb_xmit(struct sk_buff *skb, struct bonding *bond, /* no suitable interface, frame not sent */ bond_tx_drop(bond->dev, skb); -out: - return NETDEV_TX_OK; + + return false; } -netdev_tx_t bond_tlb_xmit(struct sk_buff *skb, struct net_device *bond_dev) +bool bond_tlb_xmit(struct sk_buff *skb, struct net_device *bond_dev) { struct bonding *bond = netdev_priv(bond_dev); struct ethhdr *eth_data; @@ -1372,7 +1372,7 @@ netdev_tx_t bond_tlb_xmit(struct sk_buff *skb, struct net_device *bond_dev) return bond_do_alb_xmit(skb, bond, tx_slave); } -netdev_tx_t bond_alb_xmit(struct sk_buff *skb, struct net_device *bond_dev) +bool bond_alb_xmit(struct sk_buff *skb, struct net_device *bond_dev) { struct bonding *bond = netdev_priv(bond_dev); struct ethhdr *eth_data; diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 1e9d5d35fc7866..3e52fd25f78374 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -251,8 +251,6 @@ static struct flow_dissector flow_keys_bonding __read_mostly; static int bond_init(struct net_device *bond_dev); static void bond_uninit(struct net_device *bond_dev); -static void bond_get_stats(struct net_device *bond_dev, - struct rtnl_link_stats64 *stats); static void bond_slave_arr_handler(struct work_struct *work); static bool bond_time_in_interval(struct bonding *bond, unsigned long last_act, int mod); @@ -1258,8 +1256,10 @@ static rx_handler_result_t bond_handle_frame(struct sk_buff **pskb) */ if (bond_should_deliver_exact_match(skb, slave, bond)) { if (is_link_local_ether_addr(eth_hdr(skb)->h_dest)) - return RX_HANDLER_PASS; - return RX_HANDLER_EXACT; + ret = RX_HANDLER_PASS; + else + ret = RX_HANDLER_EXACT; + goto out; } skb->dev = bond->dev; @@ -1277,6 +1277,23 @@ static rx_handler_result_t bond_handle_frame(struct sk_buff **pskb) bond->dev->addr_len); } +out: + if (ret == RX_HANDLER_ANOTHER) { + struct bond_pcpu_stats *pcpu_stats; + + pcpu_stats = this_cpu_ptr(bond->pcpu_stats); + u64_stats_update_begin(&pcpu_stats->syncp); + pcpu_stats->rx_packets++; + pcpu_stats->rx_bytes += skb->len; + if (skb->pkt_type == PACKET_MULTICAST) + pcpu_stats->rx_multicast++; + u64_stats_update_end(&pcpu_stats->syncp); + } else if (ret == RX_HANDLER_EXACT) { + this_cpu_inc(bond->pcpu_stats->rx_nohandler); + } else if (ret == RX_HANDLER_PASS) { + this_cpu_inc(bond->pcpu_stats->rx_dropped); + } + return ret; } @@ -1608,8 +1625,6 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev, } slave_dev->priv_flags |= IFF_BONDING; - /* initialize slave stats */ - dev_get_stats(new_slave->dev, &new_slave->slave_stats); if (bond_is_lb(bond)) { /* bond_alb_init_slave() must be called before all other stages since @@ -1944,9 +1959,6 @@ static int __bond_release_one(struct net_device *bond_dev, bond_sysfs_slave_del(slave); - /* recompute stats just before removing the slave */ - bond_get_stats(bond->dev, &bond->bond_stats); - bond_upper_dev_unlink(bond, slave); /* unregister rx_handler early so bond_handle_frame wouldn't be called * for this slave anymore. @@ -3497,60 +3509,39 @@ static int bond_close(struct net_device *bond_dev) return 0; } -/* fold stats, assuming all rtnl_link_stats64 fields are u64, but - * that some drivers can provide 32bit values only. - */ -static void bond_fold_stats(struct rtnl_link_stats64 *_res, - const struct rtnl_link_stats64 *_new, - const struct rtnl_link_stats64 *_old) -{ - const u64 *new = (const u64 *)_new; - const u64 *old = (const u64 *)_old; - u64 *res = (u64 *)_res; - int i; - - for (i = 0; i < sizeof(*_res) / sizeof(u64); i++) { - u64 nv = new[i]; - u64 ov = old[i]; - s64 delta = nv - ov; - - /* detects if this particular field is 32bit only */ - if (((nv | ov) >> 32) == 0) - delta = (s64)(s32)((u32)nv - (u32)ov); - - /* filter anomalies, some drivers reset their stats - * at down/up events. - */ - if (delta > 0) - res[i] += delta; - } -} - static void bond_get_stats(struct net_device *bond_dev, struct rtnl_link_stats64 *stats) { + u64 rx_packets, rx_bytes, rx_multicast, tx_packets, tx_bytes; + u32 rx_dropped = 0, tx_dropped = 0, rx_nohandler = 0; struct bonding *bond = netdev_priv(bond_dev); - struct rtnl_link_stats64 temp; - struct list_head *iter; - struct slave *slave; - - spin_lock(&bond->stats_lock); - memcpy(stats, &bond->bond_stats, sizeof(*stats)); - - rcu_read_lock(); - bond_for_each_slave_rcu(bond, slave, iter) { - const struct rtnl_link_stats64 *new = - dev_get_stats(slave->dev, &temp); - - bond_fold_stats(stats, new, &slave->slave_stats); - - /* save off the slave stats for the next run */ - memcpy(&slave->slave_stats, new, sizeof(*new)); - } - rcu_read_unlock(); + struct bond_pcpu_stats *p; + unsigned int start; + int i; - memcpy(&bond->bond_stats, stats, sizeof(*stats)); - spin_unlock(&bond->stats_lock); + for_each_possible_cpu(i) { + p = per_cpu_ptr(bond->pcpu_stats, i); + do { + start = u64_stats_fetch_begin_irq(&p->syncp); + rx_packets = p->rx_packets; + rx_bytes = p->rx_bytes; + rx_multicast = p->rx_multicast; + tx_packets = p->tx_packets; + tx_bytes = p->tx_bytes; + } while (u64_stats_fetch_retry_irq(&p->syncp, start)); + + stats->rx_packets += rx_packets; + stats->rx_bytes += rx_bytes; + stats->multicast += rx_multicast; + stats->tx_packets += tx_packets; + stats->tx_bytes += tx_bytes; + rx_dropped += p->rx_dropped; + tx_dropped += p->tx_dropped; + rx_nohandler += p->rx_nohandler; + } + stats->rx_dropped = rx_dropped; + stats->tx_dropped = tx_dropped; + stats->rx_nohandler = rx_nohandler; } static int bond_do_ioctl(struct net_device *bond_dev, struct ifreq *ifr, int cmd) @@ -3885,7 +3876,8 @@ static int bond_set_mac_address(struct net_device *bond_dev, void *addr) * it fails, it tries to find the first available slave for transmission. * The skb is consumed in all cases, thus the function is void. */ -static void bond_xmit_slave_id(struct bonding *bond, struct sk_buff *skb, int slave_id) +static bool bond_xmit_slave_id(struct bonding *bond, struct sk_buff *skb, + int slave_id) { struct list_head *iter; struct slave *slave; @@ -3896,7 +3888,7 @@ static void bond_xmit_slave_id(struct bonding *bond, struct sk_buff *skb, int sl if (--i < 0) { if (bond_slave_can_tx(slave)) { bond_dev_queue_xmit(bond, skb, slave->dev); - return; + return true; } } } @@ -3908,11 +3900,12 @@ static void bond_xmit_slave_id(struct bonding *bond, struct sk_buff *skb, int sl break; if (bond_slave_can_tx(slave)) { bond_dev_queue_xmit(bond, skb, slave->dev); - return; + return true; } } /* no slave that can tx has been found */ bond_tx_drop(bond->dev, skb); + return false; } /** @@ -3948,8 +3941,8 @@ static u32 bond_rr_gen_slave_id(struct bonding *bond) return slave_id; } -static netdev_tx_t bond_xmit_roundrobin(struct sk_buff *skb, - struct net_device *bond_dev) +static bool bond_xmit_roundrobin(struct sk_buff *skb, + struct net_device *bond_dev) { struct bonding *bond = netdev_priv(bond_dev); struct slave *slave; @@ -3972,11 +3965,12 @@ static netdev_tx_t bond_xmit_roundrobin(struct sk_buff *skb, iph = ip_hdr(skb); if (iph->protocol == IPPROTO_IGMP) { slave = rcu_dereference(bond->curr_active_slave); - if (slave) + if (slave) { bond_dev_queue_xmit(bond, skb, slave->dev); - else - bond_xmit_slave_id(bond, skb, 0); - return NETDEV_TX_OK; + return true; + } else { + return bond_xmit_slave_id(bond, skb, 0); + } } } @@ -3984,29 +3978,28 @@ static netdev_tx_t bond_xmit_roundrobin(struct sk_buff *skb, slave_cnt = READ_ONCE(bond->slave_cnt); if (likely(slave_cnt)) { slave_id = bond_rr_gen_slave_id(bond); - bond_xmit_slave_id(bond, skb, slave_id % slave_cnt); - } else { - bond_tx_drop(bond_dev, skb); + return bond_xmit_slave_id(bond, skb, slave_id % slave_cnt); } - return NETDEV_TX_OK; + bond_tx_drop(bond_dev, skb); + return false; } /* In active-backup mode, we know that bond->curr_active_slave is always valid if * the bond has a usable interface. */ -static netdev_tx_t bond_xmit_activebackup(struct sk_buff *skb, - struct net_device *bond_dev) +static bool bond_xmit_activebackup(struct sk_buff *skb, + struct net_device *bond_dev) { struct bonding *bond = netdev_priv(bond_dev); struct slave *slave; slave = rcu_dereference(bond->curr_active_slave); - if (slave) + if (slave) { bond_dev_queue_xmit(bond, skb, slave->dev); - else - bond_tx_drop(bond_dev, skb); - - return NETDEV_TX_OK; + return true; + } + bond_tx_drop(bond_dev, skb); + return false; } /* Use this to update slave_array when (a) it's not appropriate to update @@ -4137,8 +4130,8 @@ int bond_update_slave_arr(struct bonding *bond, struct slave *skipslave) * usable slave array is formed in the control path. The xmit function * just calculates hash and sends the packet out. */ -static netdev_tx_t bond_3ad_xor_xmit(struct sk_buff *skb, - struct net_device *dev) +static bool bond_3ad_xor_xmit(struct sk_buff *skb, + struct net_device *dev) { struct bonding *bond = netdev_priv(dev); struct slave *slave; @@ -4150,20 +4143,20 @@ static netdev_tx_t bond_3ad_xor_xmit(struct sk_buff *skb, if (likely(count)) { slave = slaves->arr[bond_xmit_hash(bond, skb) % count]; bond_dev_queue_xmit(bond, skb, slave->dev); - } else { - bond_tx_drop(dev, skb); + return true; } - - return NETDEV_TX_OK; + bond_tx_drop(dev, skb); + return false; } /* in broadcast mode, we send everything to all usable interfaces. */ -static netdev_tx_t bond_xmit_broadcast(struct sk_buff *skb, - struct net_device *bond_dev) +static bool bond_xmit_broadcast(struct sk_buff *skb, + struct net_device *bond_dev) { struct bonding *bond = netdev_priv(bond_dev); struct slave *slave = NULL; struct list_head *iter; + int ret = 0; bond_for_each_slave_rcu(bond, slave, iter) { if (bond_is_last_slave(bond, slave)) @@ -4177,14 +4170,18 @@ static netdev_tx_t bond_xmit_broadcast(struct sk_buff *skb, continue; } bond_dev_queue_xmit(bond, skb2, slave->dev); + ret++; } } - if (slave && bond_slave_is_up(slave) && slave->link == BOND_LINK_UP) + + if (slave && bond_slave_is_up(slave) && slave->link == BOND_LINK_UP) { bond_dev_queue_xmit(bond, skb, slave->dev); - else - bond_tx_drop(bond_dev, skb); + ret++; + } - return NETDEV_TX_OK; + if (!ret) + bond_tx_drop(bond_dev, skb); + return !!ret; } /*------------------------- Device initialization ---------------------------*/ @@ -4237,13 +4234,13 @@ static u16 bond_select_queue(struct net_device *dev, struct sk_buff *skb, return txq; } -static netdev_tx_t __bond_start_xmit(struct sk_buff *skb, struct net_device *dev) +static bool __bond_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct bonding *bond = netdev_priv(dev); if (bond_should_override_tx_queue(bond) && !bond_slave_override(bond, skb)) - return NETDEV_TX_OK; + return true; switch (BOND_MODE(bond)) { case BOND_MODE_ROUNDROBIN: @@ -4264,14 +4261,14 @@ static netdev_tx_t __bond_start_xmit(struct sk_buff *skb, struct net_device *dev netdev_err(dev, "Unknown bonding mode %d\n", BOND_MODE(bond)); WARN_ON_ONCE(1); bond_tx_drop(dev, skb); - return NETDEV_TX_OK; + return false; } } static netdev_tx_t bond_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct bonding *bond = netdev_priv(dev); - netdev_tx_t ret = NETDEV_TX_OK; + int len = skb->len; /* If we risk deadlock from transmitting this in the * netpoll path, tell netpoll to queue the frame for later tx @@ -4280,13 +4277,24 @@ static netdev_tx_t bond_start_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_BUSY; rcu_read_lock(); - if (bond_has_slaves(bond)) - ret = __bond_start_xmit(skb, dev); - else + if (bond_has_slaves(bond)) { + if (__bond_start_xmit(skb, dev)) { + struct bond_pcpu_stats *pcpu_stats; + + pcpu_stats = this_cpu_ptr(bond->pcpu_stats); + u64_stats_update_begin(&pcpu_stats->syncp); + pcpu_stats->tx_packets++; + pcpu_stats->tx_bytes += len; + u64_stats_update_end(&pcpu_stats->syncp); + } else { + this_cpu_inc(bond->pcpu_stats->tx_dropped); + } + } else { bond_tx_drop(dev, skb); + } rcu_read_unlock(); - return ret; + return NETDEV_TX_OK; } static int bond_ethtool_get_link_ksettings(struct net_device *bond_dev, @@ -4368,8 +4376,10 @@ static const struct device_type bond_type = { static void bond_destructor(struct net_device *bond_dev) { struct bonding *bond = netdev_priv(bond_dev); + if (bond->wq) destroy_workqueue(bond->wq); + free_percpu(bond->pcpu_stats); } void bond_setup(struct net_device *bond_dev) @@ -4445,7 +4455,6 @@ static void bond_uninit(struct net_device *bond_dev) list_del(&bond->bond_list); - lockdep_unregister_key(&bond->stats_lock_key); bond_debug_unregister(bond); } @@ -4845,13 +4854,13 @@ static int bond_init(struct net_device *bond_dev) netdev_dbg(bond_dev, "Begin bond_init\n"); - bond->wq = alloc_ordered_workqueue(bond_dev->name, WQ_MEM_RECLAIM); - if (!bond->wq) + bond->pcpu_stats = netdev_alloc_pcpu_stats(struct bond_pcpu_stats); + if (!bond->pcpu_stats) return -ENOMEM; - spin_lock_init(&bond->stats_lock); - lockdep_register_key(&bond->stats_lock_key); - lockdep_set_class(&bond->stats_lock, &bond->stats_lock_key); + bond->wq = alloc_ordered_workqueue(bond_dev->name, WQ_MEM_RECLAIM); + if (!bond->wq) + goto err_alloc_workqueue; list_add_tail(&bond->bond_list, &bn->dev_list); @@ -4865,6 +4874,9 @@ static int bond_init(struct net_device *bond_dev) eth_hw_addr_random(bond_dev); return 0; +err_alloc_workqueue: + free_percpu(bond->pcpu_stats); + return -ENOMEM; } unsigned int bond_get_num_tx_queues(void) diff --git a/include/net/bond_alb.h b/include/net/bond_alb.h index b3504fcd773dcf..89c0545458cd1a 100644 --- a/include/net/bond_alb.h +++ b/include/net/bond_alb.h @@ -156,8 +156,8 @@ int bond_alb_init_slave(struct bonding *bond, struct slave *slave); void bond_alb_deinit_slave(struct bonding *bond, struct slave *slave); void bond_alb_handle_link_change(struct bonding *bond, struct slave *slave, char link); void bond_alb_handle_active_change(struct bonding *bond, struct slave *new_slave); -int bond_alb_xmit(struct sk_buff *skb, struct net_device *bond_dev); -int bond_tlb_xmit(struct sk_buff *skb, struct net_device *bond_dev); +bool bond_alb_xmit(struct sk_buff *skb, struct net_device *bond_dev); +bool bond_tlb_xmit(struct sk_buff *skb, struct net_device *bond_dev); void bond_alb_monitor(struct work_struct *); int bond_alb_set_mac_address(struct net_device *bond_dev, void *addr); void bond_alb_clear_vlan(struct bonding *bond, unsigned short vlan_id); diff --git a/include/net/bonding.h b/include/net/bonding.h index 3d56b026bb9e77..53106e59ff9cb6 100644 --- a/include/net/bonding.h +++ b/include/net/bonding.h @@ -111,6 +111,18 @@ static inline int is_netpoll_tx_blocked(struct net_device *dev) #define is_netpoll_tx_blocked(dev) (0) #endif +struct bond_pcpu_stats { + u64 rx_packets; + u64 rx_bytes; + u64 rx_multicast; + u64 tx_packets; + u64 tx_bytes; + struct u64_stats_sync syncp; + u32 rx_dropped; + u32 tx_dropped; + u32 rx_nohandler; +}; + struct bond_params { int mode; int xmit_policy; @@ -177,7 +189,6 @@ struct slave { #endif struct delayed_work notify_work; struct kobject kobj; - struct rtnl_link_stats64 slave_stats; }; struct bond_up_slave { @@ -213,7 +224,6 @@ struct bonding { * ALB mode (6) - to sync the use and modifications of its hash table */ spinlock_t mode_lock; - spinlock_t stats_lock; u8 send_peer_notif; u8 igmp_retrans; #ifdef CONFIG_PROC_FS @@ -236,8 +246,7 @@ struct bonding { /* debugging support via debugfs */ struct dentry *debug_dir; #endif /* CONFIG_DEBUG_FS */ - struct rtnl_link_stats64 bond_stats; - struct lock_class_key stats_lock_key; + struct bond_pcpu_stats __percpu *pcpu_stats; }; #define bond_slave_get_rcu(dev) \