Add tool: sysctl.

This commit contains an ipc library implemented by dpdk rte_ring and
sysctl tool ported from FreeBSD.

With this commit we can get and set FreeBSD kernel state in runtime.

app/nginx-1.11.10/auto/make

@@ -33,7 +33,7 @@ CORE_LIBS+=" -g -Wl,--no-as-needed -fvisibility=default -pthread -lm -lrt"
 CORE_LIBS+=" -Wl,--whole-archive -lrte_pmd_vmxnet3_uio -lrte_pmd_i40e -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring"
 CORE_LIBS+=" -Wl,--whole-archive -lrte_hash -lrte_kvargs -Wl,-lrte_mbuf -lethdev -lrte_eal -Wl,-lrte_mempool"
 CORE_LIBS+=" -lrte_ring -lrte_cmdline -lrte_cfgfile -lrte_kni -lrte_timer -Wl,-lrte_pmd_virtio"
-CORE_LIBS+=" -Wl,--no-whole-archive -lrt -lm -ldl -lm -lcrypto"
+CORE_LIBS+=" -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto"
 
 cat << END                                                     > $NGX_MAKEFILE
 

example/Makefile

@@ -1,9 +1,19 @@
+TOPDIR=..
+
+ifeq ($(FF_PATH),)
+	FF_PATH=${TOPDIR}
+endif
+
+ifeq ($(FF_DPDK),)
+	FF_DPDK=${TOPDIR}/dpdk/x86_64-native-linuxapp-gcc
+endif
+
 LIBS+= -L${FF_PATH}/lib -L${FF_DPDK}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive
 LIBS+= -g -Wl,--no-as-needed -fvisibility=default -pthread -lm -lrt
 LIBS+= -Wl,--whole-archive -lrte_pmd_vmxnet3_uio -lrte_pmd_i40e -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring
 LIBS+= -Wl,--whole-archive -lrte_hash -lrte_kvargs -Wl,-lrte_mbuf -lethdev -lrte_eal -Wl,-lrte_mempool
 LIBS+= -lrte_ring -lrte_cmdline -lrte_cfgfile -lrte_kni -lrte_timer -Wl,-lrte_pmd_virtio
-LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lm -lcrypto
+LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto
 
 TARGET="helloworld"
 all:

lib/ff_dpdk_if.c

@@ -55,11 +55,15 @@
 #include "ff_config.h"
 #include "ff_veth.h"
 #include "ff_host_interface.h"
+#include "ff_msg.h"
+#include "ff_api.h"
 
 #define MEMPOOL_CACHE_SIZE 256
 
 #define ARP_RING_SIZE 2048
 
+#define MSG_RING_SIZE 32
+
 /*
  * Configurable number of RX/TX ring descriptors
  */
@@ -153,6 +157,16 @@ static struct rte_mempool *pktmbuf_pool[NB_SOCKETS];
 
 static struct rte_ring **arp_ring[RTE_MAX_LCORE];
 
+struct ff_msg_ring {
+    char ring_name[2][RTE_RING_NAMESIZE];
+    /* ring[0] for lcore recv msg, other send */
+    /* ring[1] for lcore send msg, other read */
+    struct rte_ring *ring[2];
+} __rte_cache_aligned;
+
+static struct ff_msg_ring msg_ring[RTE_MAX_LCORE];
+static struct rte_mempool *message_pool;
+
 struct ff_dpdk_if_context {
     void *sc;
     void *ifp;
@@ -441,6 +455,25 @@ init_mem_pool(void)
     return 0;
 }
 
+static struct rte_ring *
+create_ring(const char *name, unsigned count, int socket_id, unsigned flags)
+{
+    struct rte_ring *ring;
+
+    if (name == NULL)
+        return NULL;
+
+    /* If already create, just attached it */
+    if (likely((ring = rte_ring_lookup(name)) != NULL))
+        return ring;
+
+    if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+        return rte_ring_create(name, count, socket_id, flags);
+    } else {
+        return rte_ring_lookup(name);
+    }
+}
+
 static int
 init_arp_ring(void)
 {
@@ -472,28 +505,73 @@ init_arp_ring(void)
         uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[j].port_id;
 
         for(i = 0; i < nb_procs; ++i) {
-            snprintf(name_buf, RTE_RING_NAMESIZE, "ring_%d_%d", i, port_id);
-            if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
-                arp_ring[i][port_id] = rte_ring_create(name_buf,
-                    ARP_RING_SIZE, socketid,
-                    RING_F_SC_DEQ);
-                if (rte_ring_lookup(name_buf) != arp_ring[i][port_id])
-                    rte_panic("lookup arp ring:%s failed!\n", name_buf);
-            } else {
-                arp_ring[i][port_id] = rte_ring_lookup(name_buf);
-            }
+            snprintf(name_buf, RTE_RING_NAMESIZE, "arp_ring_%d_%d", i, port_id);
+            arp_ring[i][port_id] = create_ring(name_buf, ARP_RING_SIZE,
+                socketid, RING_F_SC_DEQ);
 
             if (arp_ring[i][port_id] == NULL)
-                rte_panic("create arp ring::%s failed!\n", name_buf);
+                rte_panic("create ring:%s failed!\n", name_buf);
 
-            printf("create arp ring:%s success, %u ring entries are now free!\n",
+            printf("create ring:%s success, %u ring entries are now free!\n",
                 name_buf, rte_ring_free_count(arp_ring[i][port_id]));
         }
     }
 
     return 0;
 }
 
+static void
+ff_msg_init(struct rte_mempool *mp,
+    __attribute__((unused)) void *opaque_arg,
+    void *obj, __attribute__((unused)) unsigned i)
+{
+    struct ff_msg *msg = (struct ff_msg *)obj;
+    msg->buf_addr = (char *)msg + sizeof(struct ff_msg);
+    msg->buf_len = mp->elt_size - sizeof(struct ff_msg);
+}
+
+static int
+init_msg_ring(void)
+{
+    uint16_t i;
+    uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs;
+    unsigned socketid = lcore_conf.socket_id;
+
+    /* Create message buffer pool */
+    if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+        message_pool = rte_mempool_create(FF_MSG_POOL,
+           MSG_RING_SIZE * 2 * nb_procs,
+           MAX_MSG_BUF_SIZE, MSG_RING_SIZE / 2, 0,
+           NULL, NULL, ff_msg_init, NULL,
+           socketid, 0);
+    } else {
+        message_pool = rte_mempool_lookup(FF_MSG_POOL);
+    }
+
+    if (message_pool == NULL) {
+        rte_panic("Create msg mempool failed\n");
+    }
+
+    for(i = 0; i < nb_procs; ++i) {
+        snprintf(msg_ring[i].ring_name[0], RTE_RING_NAMESIZE,
+            "%s%u", FF_MSG_RING_IN, i);
+        snprintf(msg_ring[i].ring_name[1], RTE_RING_NAMESIZE,
+            "%s%u", FF_MSG_RING_OUT, i);
+
+        msg_ring[i].ring[0] = create_ring(msg_ring[i].ring_name[0],
+            MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ);
+        if (msg_ring[i].ring[0] == NULL)
+            rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]);
+
+        msg_ring[i].ring[1] = create_ring(msg_ring[i].ring_name[1],
+            MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ);
+        if (msg_ring[i].ring[1] == NULL)
+            rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]);
+    }
+
+    return 0;
+}
+
 static int
 init_kni(void)
 {
@@ -730,6 +808,8 @@ ff_dpdk_init(int argc, char **argv)
 
     init_arp_ring();
 
+    init_msg_ring();
+
     enable_kni = ff_global_cfg.kni.enable;
     if (enable_kni) {
         init_kni();
@@ -872,12 +952,61 @@ process_arp_ring(uint8_t port_id, uint16_t queue_id,
     struct rte_mbuf **pkts_burst, const struct ff_dpdk_if_context *ctx)
 {
     /* read packet from ring buf and to process */
-    uint16_t nb_tx;
-    nb_tx = rte_ring_dequeue_burst(arp_ring[queue_id][port_id],
+    uint16_t nb_rb;
+    nb_rb = rte_ring_dequeue_burst(arp_ring[queue_id][port_id],
         (void **)pkts_burst, MAX_PKT_BURST);
 
-    if(nb_tx > 0) {
-        process_packets(port_id, queue_id, pkts_burst, nb_tx, ctx, 1);
+    if(nb_rb > 0) {
+        process_packets(port_id, queue_id, pkts_burst, nb_rb, ctx, 1);
+    }
+
+    return 0;
+}
+
+static inline void
+handle_sysctl_msg(struct ff_msg *msg, uint16_t proc_id)
+{
+    int ret = ff_sysctl(msg->sysctl.name, msg->sysctl.namelen,
+        msg->sysctl.old, msg->sysctl.oldlenp, msg->sysctl.new,
+        msg->sysctl.newlen);
+
+    if (ret < 0) {
+        msg->result = errno;
+    } else {
+        msg->result = 0;
+    }
+
+    rte_ring_enqueue(msg_ring[proc_id].ring[1], msg);
+}
+
+static inline void
+handle_default_msg(struct ff_msg *msg, uint16_t proc_id)
+{
+    msg->result = EINVAL;
+    rte_ring_enqueue(msg_ring[proc_id].ring[1], msg);
+}
+
+static inline void
+handle_msg(struct ff_msg *msg, uint16_t proc_id)
+{
+    switch (msg->msg_type) {
+        case FF_SYSCTL:
+            handle_sysctl_msg(msg, proc_id);
+            break;
+        default:
+            handle_default_msg(msg, proc_id);
+            break;
+    }
+}
+
+static inline int
+process_msg_ring(uint16_t proc_id)
+{
+    void *msg;
+    int ret = rte_ring_dequeue(msg_ring[proc_id].ring[0], &msg);
+
+    if (unlikely(ret == 0)) {
+        handle_msg((struct ff_msg *)msg, proc_id);
     }
 
     return 0;
@@ -1104,6 +1233,8 @@ main_loop(void *arg)
             }
         }
 
+        process_msg_ring(qconf->proc_id);
+
         if (likely(lr->loop != NULL)) {
             lr->loop(lr->arg);
         }

lib/ff_dpdk_kni.c

@@ -383,16 +383,16 @@ ff_kni_alloc(uint8_t port_id, unsigned socket_id,
     if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
         kni_rp[port_id] = rte_ring_create(ring_name, KNI_QUEUE_SIZE, 
             socket_id, RING_F_SC_DEQ);
+
+        if (rte_ring_lookup(ring_name) != kni_rp[port_id])
+            rte_panic("lookup kni ring failed!\n");
     } else {
         kni_rp[port_id] = rte_ring_lookup(ring_name);
     }
 
     if (kni_rp[port_id] == NULL)
         rte_panic("create kni ring failed!\n");
 
-    if (rte_ring_lookup(ring_name) != kni_rp[port_id])
-        rte_panic("lookup kni ring failed!\n");
-
     printf("create kni ring success, %u ring entries are now free!\n",
         rte_ring_free_count(kni_rp[port_id]));
 }

lib/ff_msg.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2017 THL A29 Limited, a Tencent company.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef _FF_MSG_H_
+#define _FF_MSG_H_
+
+#include <rte_memory.h>
+
+#define FF_MSG_RING_IN  "ff_msg_ring_in_"
+#define FF_MSG_RING_OUT "ff_msg_ring_out_"
+#define FF_MSG_POOL     "ff_msg_pool"
+
+/* MSG TYPE: sysctl, sysctlbyname, etc.. */
+enum FF_MSG_TYPE {
+    FF_UNKNOWN = 0,
+    FF_SYSCTL,
+};
+
+struct ff_sysctl_args {
+    int *name;
+    unsigned namelen;
+    void *old;
+    size_t *oldlenp;
+    void *new;
+    size_t newlen;
+};
+
+#define MAX_MSG_BUF_SIZE 10240
+
+/* structure of ipc msg */
+struct ff_msg {
+    enum FF_MSG_TYPE msg_type;
+    /* Result of msg processing */
+    int result;
+    /* Length of segment buffer. */
+    uint16_t buf_len;
+    /* Address of segment buffer. */
+    char *buf_addr;
+
+    union {
+        struct ff_sysctl_args sysctl;
+    };
+} __attribute__((packed)) __rte_cache_aligned;
+
+#endif

tools/Makefile

@@ -0,0 +1,7 @@
+SUBDIRS=ipc sysctl
+
+all:
+	for d in $(SUBDIRS); do ( cd $$d; $(MAKE) all ) ; done
+
+clean:
+	for d in $(SUBDIRS); do ( cd $$d; $(MAKE) clean ) ; done