Skip to content

xdp: introduce bulking for page_pool tx return path #273

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
sudo: required
language: bash
dist: bionic
services:
- docker

env:
global:
- PROJECT_NAME='libbpf'
- AUTHOR_EMAIL="$(git log -1 --pretty=\"%aE\")"
- REPO_ROOT="$TRAVIS_BUILD_DIR"
- CI_ROOT="$REPO_ROOT/travis-ci"
- VMTEST_ROOT="$CI_ROOT/vmtest"

addons:
apt:
packages:
- qemu-kvm
- zstd
- binutils-dev
- elfutils
- libcap-dev
- libelf-dev
- libdw-dev
- python3-docutils

jobs:
include:
- stage: Builds & Tests
name: Kernel LATEST + selftests
language: bash
env: KERNEL=LATEST
script: $CI_ROOT/vmtest/run_vmtest.sh || travis_terminate 1
5 changes: 4 additions & 1 deletion drivers/net/ethernet/marvell/mvneta.c
Original file line number Diff line number Diff line change
Expand Up @@ -1834,8 +1834,10 @@ static void mvneta_txq_bufs_free(struct mvneta_port *pp,
struct netdev_queue *nq, bool napi)
{
unsigned int bytes_compl = 0, pkts_compl = 0;
struct xdp_frame_bulk bq;
int i;

bq.xa = NULL;
for (i = 0; i < num; i++) {
struct mvneta_tx_buf *buf = &txq->buf[txq->txq_get_index];
struct mvneta_tx_desc *tx_desc = txq->descs +
Expand All @@ -1857,9 +1859,10 @@ static void mvneta_txq_bufs_free(struct mvneta_port *pp,
if (napi && buf->type == MVNETA_TYPE_XDP_TX)
xdp_return_frame_rx_napi(buf->xdpf);
else
xdp_return_frame(buf->xdpf);
xdp_return_frame_bulk(buf->xdpf, &bq);
}
}
xdp_flush_frame_bulk(&bq);

netdev_tx_completed_queue(nq, pkts_compl, bytes_compl);
}
Expand Down
5 changes: 4 additions & 1 deletion drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -2440,8 +2440,10 @@ static void mvpp2_txq_bufs_free(struct mvpp2_port *port,
struct mvpp2_tx_queue *txq,
struct mvpp2_txq_pcpu *txq_pcpu, int num)
{
struct xdp_frame_bulk bq;
int i;

bq.xa = NULL;
for (i = 0; i < num; i++) {
struct mvpp2_txq_pcpu_buf *tx_buf =
txq_pcpu->buffs + txq_pcpu->txq_get_index;
Expand All @@ -2454,10 +2456,11 @@ static void mvpp2_txq_bufs_free(struct mvpp2_port *port,
dev_kfree_skb_any(tx_buf->skb);
else if (tx_buf->type == MVPP2_TYPE_XDP_TX ||
tx_buf->type == MVPP2_TYPE_XDP_NDO)
xdp_return_frame(tx_buf->xdpf);
xdp_return_frame_bulk(tx_buf->xdpf, &bq);

mvpp2_txq_inc_get(txq_pcpu);
}
xdp_flush_frame_bulk(&bq);
}

static inline struct mvpp2_rx_queue *mvpp2_get_rx_queue(struct mvpp2_port *port,
Expand Down
5 changes: 4 additions & 1 deletion drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
Original file line number Diff line number Diff line change
Expand Up @@ -369,8 +369,10 @@ static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
bool recycle)
{
struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo;
struct xdp_frame_bulk bq;
u16 i;

bq.xa = NULL;
for (i = 0; i < wi->num_pkts; i++) {
struct mlx5e_xdp_info xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);

Expand All @@ -379,7 +381,7 @@ static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
/* XDP_TX from the XSK RQ and XDP_REDIRECT */
dma_unmap_single(sq->pdev, xdpi.frame.dma_addr,
xdpi.frame.xdpf->len, DMA_TO_DEVICE);
xdp_return_frame(xdpi.frame.xdpf);
xdp_return_frame_bulk(xdpi.frame.xdpf, &bq);
break;
case MLX5E_XDP_XMIT_MODE_PAGE:
/* XDP_TX from the regular RQ */
Expand All @@ -393,6 +395,7 @@ static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
WARN_ON_ONCE(true);
}
}
xdp_flush_frame_bulk(&bq);
}

bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
Expand Down
26 changes: 26 additions & 0 deletions include/net/page_pool.h
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,8 @@ struct page_pool *page_pool_create(const struct page_pool_params *params);
void page_pool_destroy(struct page_pool *pool);
void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *));
void page_pool_release_page(struct page_pool *pool, struct page *page);
void page_pool_put_page_bulk(struct page_pool *pool, void **data,
int count);
#else
static inline void page_pool_destroy(struct page_pool *pool)
{
Expand All @@ -165,6 +167,11 @@ static inline void page_pool_release_page(struct page_pool *pool,
struct page *page)
{
}

static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data,
int count)
{
}
#endif

void page_pool_put_page(struct page_pool *pool, struct page *page,
Expand Down Expand Up @@ -215,4 +222,23 @@ static inline void page_pool_nid_changed(struct page_pool *pool, int new_nid)
if (unlikely(pool->p.nid != new_nid))
page_pool_update_nid(pool, new_nid);
}

static inline void page_pool_ring_lock(struct page_pool *pool)
__acquires(&pool->ring.producer_lock)
{
if (in_serving_softirq())
spin_lock(&pool->ring.producer_lock);
else
spin_lock_bh(&pool->ring.producer_lock);
}

static inline void page_pool_ring_unlock(struct page_pool *pool)
__releases(&pool->ring.producer_lock)
{
if (in_serving_softirq())
spin_unlock(&pool->ring.producer_lock);
else
spin_unlock_bh(&pool->ring.producer_lock);
}

#endif /* _NET_PAGE_POOL_H */
9 changes: 9 additions & 0 deletions include/net/xdp.h
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,12 @@ struct xdp_frame {
struct net_device *dev_rx; /* used by cpumap */
};

#define XDP_BULK_QUEUE_SIZE 16
struct xdp_frame_bulk {
void *q[XDP_BULK_QUEUE_SIZE];
int count;
void *xa;
};

static inline struct skb_shared_info *
xdp_get_shared_info_from_frame(struct xdp_frame *frame)
Expand Down Expand Up @@ -194,6 +200,9 @@ struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp)
void xdp_return_frame(struct xdp_frame *xdpf);
void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
void xdp_return_buff(struct xdp_buff *xdp);
void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq);
void xdp_return_frame_bulk(struct xdp_frame *xdpf,
struct xdp_frame_bulk *bq);

/* When sending xdp_frame into the network stack, then there is no
* return point callback, which is needed to release e.g. DMA-mapping
Expand Down
33 changes: 33 additions & 0 deletions net/core/page_pool.c
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
#include <linux/device.h>

#include <net/page_pool.h>
#include <net/xdp.h>

#include <linux/dma-direction.h>
#include <linux/dma-mapping.h>
#include <linux/page-flags.h>
Expand Down Expand Up @@ -408,6 +410,37 @@ void page_pool_put_page(struct page_pool *pool, struct page *page,
}
EXPORT_SYMBOL(page_pool_put_page);

void page_pool_put_page_bulk(struct page_pool *pool, void **data,
int count)
{
struct page *page_ring[XDP_BULK_QUEUE_SIZE];
int i, len = 0;

for (i = 0; i < count; i++) {
struct page *page = virt_to_head_page(data[i]);

if (unlikely(page_ref_count(page) != 1 ||
!pool_page_reusable(pool, page))) {
page_pool_release_page(pool, page);
put_page(page);
continue;
}

if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
page_pool_dma_sync_for_device(pool, page, -1);

page_ring[len++] = page;
}

page_pool_ring_lock(pool);
for (i = 0; i < len; i++) {
if (__ptr_ring_produce(&pool->ring, page_ring[i]))
page_pool_return_page(pool, page_ring[i]);
}
page_pool_ring_unlock(pool);
}
EXPORT_SYMBOL(page_pool_put_page_bulk);

static void page_pool_empty_ring(struct page_pool *pool)
{
struct page *page;
Expand Down
46 changes: 46 additions & 0 deletions net/core/xdp.c
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,52 @@ void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
}
EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);

void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq)
{
struct xdp_mem_allocator *xa = bq->xa;

if (unlikely(!xa || !bq->count))
return;

page_pool_put_page_bulk(xa->page_pool, bq->q, bq->count);
bq->count = 0;
}
EXPORT_SYMBOL_GPL(xdp_flush_frame_bulk);

void xdp_return_frame_bulk(struct xdp_frame *xdpf,
struct xdp_frame_bulk *bq)
{
struct xdp_mem_info *mem = &xdpf->mem;
struct xdp_mem_allocator *xa, *nxa;

if (mem->type != MEM_TYPE_PAGE_POOL) {
__xdp_return(xdpf->data, &xdpf->mem, false);
return;
}

rcu_read_lock();

xa = bq->xa;
if (unlikely(!xa || mem->id != xa->mem.id)) {
nxa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
if (unlikely(!xa)) {
bq->count = 0;
bq->xa = nxa;
xa = nxa;
}
}

if (mem->id != xa->mem.id || bq->count == XDP_BULK_QUEUE_SIZE)
xdp_flush_frame_bulk(bq);

bq->q[bq->count++] = xdpf->data;
if (mem->id != xa->mem.id)
bq->xa = nxa;

rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);

void xdp_return_buff(struct xdp_buff *xdp)
{
__xdp_return(xdp->data, &xdp->rxq->mem, true);
Expand Down
84 changes: 84 additions & 0 deletions travis-ci/managers/debian.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
#!/bin/bash

PHASES=(${@:-SETUP RUN RUN_ASAN CLEANUP})
DEBIAN_RELEASE="${DEBIAN_RELEASE:-testing}"
CONT_NAME="${CONT_NAME:-libbpf-debian-$DEBIAN_RELEASE}"
ENV_VARS="${ENV_VARS:-}"
DOCKER_RUN="${DOCKER_RUN:-docker run}"
REPO_ROOT="${REPO_ROOT:-$PWD}"
ADDITIONAL_DEPS=(clang pkg-config gcc-8)
CFLAGS="-g -O2 -Werror -Wall"

function info() {
echo -e "\033[33;1m$1\033[0m"
}

function error() {
echo -e "\033[31;1m$1\033[0m"
}

function docker_exec() {
docker exec $ENV_VARS -it $CONT_NAME "$@"
}

set -e

source "$(dirname $0)/travis_wait.bash"

for phase in "${PHASES[@]}"; do
case $phase in
SETUP)
info "Setup phase"
info "Using Debian $DEBIAN_RELEASE"

sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
docker --version

docker pull debian:$DEBIAN_RELEASE
info "Starting container $CONT_NAME"
$DOCKER_RUN -v $REPO_ROOT:/build:rw \
-w /build --privileged=true --name $CONT_NAME \
-dit --net=host debian:$DEBIAN_RELEASE /bin/bash
docker_exec bash -c "echo deb-src http://deb.debian.org/debian $DEBIAN_RELEASE main >>/etc/apt/sources.list"
docker_exec apt-get -y update
docker_exec apt-get -y build-dep libelf-dev
docker_exec apt-get -y install libelf-dev
docker_exec apt-get -y install "${ADDITIONAL_DEPS[@]}"
;;
RUN|RUN_CLANG|RUN_GCC8|RUN_ASAN|RUN_CLANG_ASAN|RUN_GCC8_ASAN)
if [[ "$phase" = *"CLANG"* ]]; then
ENV_VARS="-e CC=clang -e CXX=clang++"
CC="clang"
elif [[ "$phase" = *"GCC8"* ]]; then
ENV_VARS="-e CC=gcc-8 -e CXX=g++-8"
CC="gcc-8"
else
CFLAGS="${CFLAGS} -Wno-stringop-truncation"
fi
if [[ "$phase" = *"ASAN"* ]]; then
CFLAGS="${CFLAGS} -fsanitize=address,undefined"
fi
docker_exec mkdir build install
docker_exec ${CC:-cc} --version
info "build"
docker_exec make -j$((4*$(nproc))) CFLAGS="${CFLAGS}" -C ./src -B OBJDIR=../build
info "ldd build/libbpf.so:"
docker_exec ldd build/libbpf.so
if ! docker_exec ldd build/libbpf.so | grep -q libelf; then
error "No reference to libelf.so in libbpf.so!"
exit 1
fi
info "install"
docker_exec make -j$((4*$(nproc))) -C src OBJDIR=../build DESTDIR=../install install
docker_exec rm -rf build install
;;
CLEANUP)
info "Cleanup phase"
docker stop $CONT_NAME
docker rm -f $CONT_NAME
;;
*)
echo >&2 "Unknown phase '$phase'"
exit 1
esac
done
Loading