Cleanup multi-poller

Author: huaicheng

hw/femu/bbssd/ftl.c

@@ -876,7 +876,7 @@ static void *ftl_thread(void *arg)
     ssd->to_poller = n->to_poller;
 
     while (1) {
-        for (i = 1; i <= n->num_poller; i++) {
+        for (i = 1; i <= n->nr_pollers; i++) {
             if (!ssd->to_ftl[i] || !femu_ring_count(ssd->to_ftl[i]))
                 continue;
 

hw/femu/femu.c

@@ -21,12 +21,12 @@ static void nvme_clear_ctrl(FemuCtrl *n, bool shutdown)
         nvme_clear_virq(n);
     }
 
-    for (int i = 0; i <= n->num_io_queues; i++) {
+    for (int i = 0; i <= n->nr_io_queues; i++) {
         if (n->sq[i] != NULL) {
             nvme_free_sq(n->sq[i], n);
         }
     }
-    for (int i = 0; i <= n->num_io_queues; i++) {
+    for (int i = 0; i <= n->nr_io_queues; i++) {
         if (n->cq[i] != NULL) {
             nvme_free_cq(n->cq[i], n);
         }
@@ -293,7 +293,7 @@ static const MemoryRegionOps nvme_mmio_ops = {
 static int nvme_check_constraints(FemuCtrl *n)
 {
     if ((n->num_namespaces == 0 || n->num_namespaces > NVME_MAX_NUM_NAMESPACES)
-        || (n->num_io_queues < 1 || n->num_io_queues > NVME_MAX_QS) ||
+        || (n->nr_io_queues < 1 || n->nr_io_queues > NVME_MAX_QS) ||
         (n->db_stride > NVME_MAX_STRIDE) ||
         (n->max_q_ents < 1) ||
         (n->max_sqes > NVME_MAX_QUEUE_ES || n->max_cqes > NVME_MAX_QUEUE_ES ||
@@ -429,14 +429,14 @@ static void nvme_init_ctrl(FemuCtrl *n)
     n->features.temp_thresh     = 0x14d;
     n->features.err_rec         = 0;
     n->features.volatile_wc     = n->vwc;
-    n->features.num_io_queues   = ((n->num_io_queues - 1) | ((n->num_io_queues -
+    n->features.nr_io_queues   = ((n->nr_io_queues - 1) | ((n->nr_io_queues -
                                                               1) << 16));
     n->features.int_coalescing  = n->intc_thresh | (n->intc_time << 8);
     n->features.write_atomicity = 0;
     n->features.async_config    = 0x0;
     n->features.sw_prog_marker  = 0;
 
-    for (i = 0; i <= n->num_io_queues; i++) {
+    for (i = 0; i <= n->nr_io_queues; i++) {
         n->features.int_vector_config[i] = i | (n->intc << 16);
     }
 
@@ -487,7 +487,7 @@ static void nvme_init_pci(FemuCtrl *n)
                           n->reg_size);
     pci_register_bar(&n->parent_obj, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
                      PCI_BASE_ADDRESS_MEM_TYPE_64, &n->iomem);
-    if (msix_init_exclusive_bar(&n->parent_obj, n->num_io_queues + 1, 4, NULL)) {
+    if (msix_init_exclusive_bar(&n->parent_obj, n->nr_io_queues + 1, 4, NULL)) {
         return;
     }
     msi_init(&n->parent_obj, 0x50, 32, true, false, NULL);
@@ -540,16 +540,16 @@ static void femu_realize(PCIDevice *pci_dev, Error **errp)
 
     n->completed = 0;
     n->start_time = time(NULL);
-    n->reg_size = pow2ceil(0x1004 + 2 * (n->num_io_queues + 1) * 4);
+    n->reg_size = pow2ceil(0x1004 + 2 * (n->nr_io_queues + 1) * 4);
     n->ns_size = bs_size / (uint64_t)n->num_namespaces;
 
-    /* Coperd: [1..num_io_queues] are used as IO queues */
-    n->sq = g_malloc0(sizeof(*n->sq) * (n->num_io_queues + 1));
-    n->cq = g_malloc0(sizeof(*n->cq) * (n->num_io_queues + 1));
+    /* Coperd: [1..nr_io_queues] are used as IO queues */
+    n->sq = g_malloc0(sizeof(*n->sq) * (n->nr_io_queues + 1));
+    n->cq = g_malloc0(sizeof(*n->cq) * (n->nr_io_queues + 1));
     n->namespaces = g_malloc0(sizeof(*n->namespaces) * n->num_namespaces);
     n->elpes = g_malloc0(sizeof(*n->elpes) * (n->elpe + 1));
     n->aer_reqs = g_malloc0(sizeof(*n->aer_reqs) * (n->aerl + 1));
-    n->features.int_vector_config = g_malloc0(sizeof(*n->features.int_vector_config) * (n->num_io_queues + 1));
+    n->features.int_vector_config = g_malloc0(sizeof(*n->features.int_vector_config) * (n->nr_io_queues + 1));
 
     nvme_init_pci(n);
     nvme_init_ctrl(n);
@@ -564,13 +564,14 @@ static void femu_realize(PCIDevice *pci_dev, Error **errp)
 
 static void nvme_destroy_poller(FemuCtrl *n)
 {
+    int i;
     femu_debug("Destroying NVMe poller !!\n");
 
-    for (int i = 1; i <= n->num_poller; i++) {
+    for (i = 1; i <= n->nr_pollers; i++) {
         qemu_thread_join(&n->poller[i]);
     }
 
-    for (int i = 1; i <= n->num_poller; i++) {
+    for (i = 1; i <= n->nr_pollers; i++) {
         pqueue_free(n->pq[i]);
         femu_ring_free(n->to_poller[i]);
         femu_ring_free(n->to_ftl[i]);
@@ -610,7 +611,7 @@ static Property femu_props[] = {
     DEFINE_PROP_STRING("serial", FemuCtrl, serial),
     DEFINE_PROP_UINT32("devsz_mb", FemuCtrl, memsz, 1024), /* in MB */
     DEFINE_PROP_UINT32("namespaces", FemuCtrl, num_namespaces, 1),
-    DEFINE_PROP_UINT32("queues", FemuCtrl, num_io_queues, 8),
+    DEFINE_PROP_UINT32("queues", FemuCtrl, nr_io_queues, 8),
     DEFINE_PROP_UINT32("entries", FemuCtrl, max_q_ents, 0x7ff),
     DEFINE_PROP_UINT8("multipoller_enabled", FemuCtrl, multipoller_enabled, 0),
     DEFINE_PROP_UINT8("max_cqes", FemuCtrl, max_cqes, 0x4),

hw/femu/intr.c

@@ -50,7 +50,7 @@ static void nvme_clear_guest_notifier(FemuCtrl *n)
 {
     NvmeCQueue *cq;
 
-    for (uint32_t qid = 1; qid <= n->num_io_queues; qid++) {
+    for (uint32_t qid = 1; qid <= n->nr_io_queues; qid++) {
         cq = n->cq[qid];
         if (!cq) {
             break;
@@ -74,7 +74,7 @@ static int nvme_vector_unmask(PCIDevice *dev, unsigned vector, MSIMessage msg)
     EventNotifier *e;
     int ret;
 
-    for (uint32_t qid = 1; qid <= n->num_io_queues; qid++) {
+    for (uint32_t qid = 1; qid <= n->nr_io_queues; qid++) {
         cq = n->cq[qid];
         if (!cq) {
             continue;
@@ -110,7 +110,7 @@ static void nvme_vector_mask(PCIDevice *dev, unsigned vector)
     EventNotifier *e;
     int ret;
 
-    for (uint32_t qid = 1; qid <= n->num_io_queues; qid++) {
+    for (uint32_t qid = 1; qid <= n->nr_io_queues; qid++) {
         cq = n->cq[qid];
         if (!cq) {
             continue;
@@ -135,7 +135,7 @@ static void nvme_vector_poll(PCIDevice *dev, unsigned int vector_start, unsigned
     EventNotifier *e;
     uint32_t vector;
 
-    for (uint32_t qid = 1; qid <= n->num_io_queues; qid++) {
+    for (uint32_t qid = 1; qid <= n->nr_io_queues; qid++) {
         cq = n->cq[qid];
         if (!cq) {
             continue;

hw/femu/nvme-admin.c

@@ -158,7 +158,7 @@ static uint16_t nvme_create_cq(FemuCtrl *n, NvmeCmd *cmd)
     if (!prp1) {
         return NVME_INVALID_FIELD | NVME_DNR;
     }
-    if (vector > n->num_io_queues) {
+    if (vector > n->nr_io_queues) {
         return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
     }
     if (!(NVME_CQ_FLAGS_PC(qflags)) && NVME_CAP_CQR(n->bar.cap)) {
@@ -206,18 +206,96 @@ static uint16_t nvme_del_cq(FemuCtrl *n, NvmeCmd *cmd)
     return NVME_SUCCESS;
 }
 
+static int cmp_pri(pqueue_pri_t next, pqueue_pri_t curr)
+{
+    return (next > curr);
+}
+
+static pqueue_pri_t get_pri(void *a)
+{
+    return ((NvmeRequest *)a)->expire_time;
+}
+
+static void set_pri(void *a, pqueue_pri_t pri)
+{
+    ((NvmeRequest *)a)->expire_time = pri;
+}
+
+static size_t get_pos(void *a)
+{
+    return ((NvmeRequest *)a)->pos;
+}
+
+static void set_pos(void *a, size_t pos)
+{
+    ((NvmeRequest *)a)->pos = pos;
+}
+
+static void nvme_init_poller(FemuCtrl *n)
+{
+    int i;
+
+    n->should_isr = g_malloc0(sizeof(bool) * (n->nr_io_queues + 1));
+
+    n->nr_pollers = n->multipoller_enabled ? n->nr_io_queues : 1;
+    /* Coperd: we put NvmeRequest into these rings */
+    n->to_ftl = g_malloc0(sizeof(struct rte_ring *) * (n->nr_pollers + 1));
+    for (i = 1; i <= n->nr_pollers; i++) {
+        n->to_ftl[i] = femu_ring_create(FEMU_RING_TYPE_MP_SC, FEMU_MAX_INF_REQS);
+        if (!n->to_ftl[i]) {
+            femu_err("Failed to create ring (n->to_ftl) ...\n");
+            abort();
+        }
+        assert(rte_ring_empty(n->to_ftl[i]));
+    }
+
+    n->to_poller = g_malloc0(sizeof(struct rte_ring *) * (n->nr_pollers + 1));
+    for (i = 1; i <= n->nr_pollers; i++) {
+        n->to_poller[i] = femu_ring_create(FEMU_RING_TYPE_MP_SC, FEMU_MAX_INF_REQS);
+        if (!n->to_poller[i]) {
+            femu_err("Failed to create ring (n->to_poller) ...\n");
+            abort();
+        }
+        assert(rte_ring_empty(n->to_poller[i]));
+    }
+
+    n->pq = g_malloc0(sizeof(pqueue_t *) * (n->nr_pollers + 1));
+    for (i = 1; i <= n->nr_pollers; i++) {
+        n->pq[i] = pqueue_init(FEMU_MAX_INF_REQS, cmp_pri, get_pri, set_pri,
+                               get_pos, set_pos);
+        if (!n->pq[i]) {
+            femu_err("Failed to create pqueue (n->pq) ...\n");
+            abort();
+        }
+    }
+
+    n->poller = g_malloc0(sizeof(QemuThread) * (n->nr_pollers + 1));
+    NvmePollerThreadArgument *args = malloc(sizeof(NvmePollerThreadArgument) *
+                                            (n->nr_pollers + 1));
+    for (i = 1; i <= n->nr_pollers; i++) {
+        args[i].n = n;
+        args[i].index = i;
+        qemu_thread_create(&n->poller[i], "femu-nvme-poller", nvme_poller,
+                &args[i], QEMU_THREAD_JOINABLE);
+        femu_debug("femu-nvme-poller [%d] created ...\n", i - 1);
+    }
+}
+
 static uint16_t nvme_set_db_memory(FemuCtrl *n, const NvmeCmd *cmd)
 {
     uint64_t dbs_addr = le64_to_cpu(cmd->dptr.prp1);
     uint64_t eis_addr = le64_to_cpu(cmd->dptr.prp2);
     uint8_t stride = n->db_stride;
     int dbbuf_entry_sz = 1 << (2 + stride);
     AddressSpace *as = pci_get_address_space(&n->parent_obj);
+    int i;
+
+
     dma_addr_t dbs_tlen = n->page_size, eis_tlen = n->page_size;
 
     /* Addresses should not be NULL and should be page aligned. */
-    if (dbs_addr == 0 || dbs_addr & (n->page_size - 1) ||
-            eis_addr == 0 || eis_addr & (n->page_size - 1)) {
+    if (dbs_addr == 0 || dbs_addr & (n->page_size - 1) || eis_addr == 0 ||
+            eis_addr & (n->page_size - 1)) {
         return NVME_INVALID_FIELD | NVME_DNR;
     }
 
@@ -226,7 +304,7 @@ static uint16_t nvme_set_db_memory(FemuCtrl *n, const NvmeCmd *cmd)
     n->dbs_addr_hva = (uint64_t)dma_memory_map(as, dbs_addr, &dbs_tlen, 0, MEMTXATTRS_UNSPECIFIED);
     n->eis_addr_hva = (uint64_t)dma_memory_map(as, eis_addr, &eis_tlen, 0, MEMTXATTRS_UNSPECIFIED);
 
-    for (int i = 1; i <= n->num_io_queues; i++) {
+    for (i = 1; i <= n->nr_io_queues; i++) {
         NvmeSQueue *sq = n->sq[i];
         NvmeCQueue *cq = n->cq[i];
 
@@ -252,8 +330,8 @@ static uint16_t nvme_set_db_memory(FemuCtrl *n, const NvmeCmd *cmd)
 
     assert(n->dataplane_started == false);
     if (!n->poller_on) {
-        /* Coperd: make sure this only run once across all controller resets */
-        nvme_create_poller(n);
+        /* Coperd: make sure this only runs once across all controller resets */
+        nvme_init_poller(n);
         n->poller_on = true;
     }
     n->dataplane_started = true;
@@ -570,8 +648,8 @@ static uint16_t nvme_get_feature(FemuCtrl *n, NvmeCmd *cmd, NvmeCqe *cqe)
                 MIN(sizeof(*rt), (dw11 & 0x3f) * sizeof(*rt)),
                 prp1, prp2);
     case NVME_NUMBER_OF_QUEUES:
-        cqe->n.result = cpu_to_le32((n->num_io_queues - 1) |
-                ((n->num_io_queues - 1) << 16));
+        cqe->n.result = cpu_to_le32((n->nr_io_queues - 1) |
+                ((n->nr_io_queues - 1) << 16));
         break;
     case NVME_TEMPERATURE_THRESHOLD:
         cqe->n.result = cpu_to_le32(n->features.temp_thresh);
@@ -586,7 +664,7 @@ static uint16_t nvme_get_feature(FemuCtrl *n, NvmeCmd *cmd, NvmeCqe *cqe)
         cqe->n.result = cpu_to_le32(n->features.int_coalescing);
         break;
     case NVME_INTERRUPT_VECTOR_CONF:
-        if ((dw11 & 0xffff) > n->num_io_queues) {
+        if ((dw11 & 0xffff) > n->nr_io_queues) {
             return NVME_INVALID_FIELD | NVME_DNR;
         }
         cqe->n.result = cpu_to_le32(n->features.int_vector_config[dw11 & 0xffff]);
@@ -633,9 +711,9 @@ static uint16_t nvme_set_feature(FemuCtrl *n, NvmeCmd *cmd, NvmeCqe *cqe)
                 MIN(sizeof(*rt), (dw11 & 0x3f) * sizeof(*rt)),
                 prp1, prp2);
     case NVME_NUMBER_OF_QUEUES:
-        /* Coperd: num_io_queues is 0-based */
-        cqe->n.result = cpu_to_le32((n->num_io_queues - 1) |
-                ((n->num_io_queues - 1) << 16));
+        /* Coperd: nr_io_queues is 0-based */
+        cqe->n.result = cpu_to_le32((n->nr_io_queues - 1) |
+                ((n->nr_io_queues - 1) << 16));
         break;
     case NVME_TEMPERATURE_THRESHOLD:
         n->features.temp_thresh = dw11;
@@ -656,7 +734,7 @@ static uint16_t nvme_set_feature(FemuCtrl *n, NvmeCmd *cmd, NvmeCqe *cqe)
         n->features.int_coalescing = dw11;
         break;
     case NVME_INTERRUPT_VECTOR_CONF:
-        if ((dw11 & 0xffff) > n->num_io_queues) {
+        if ((dw11 & 0xffff) > n->nr_io_queues) {
             return NVME_INVALID_FIELD | NVME_DNR;
         }
         n->features.int_vector_config[dw11 & 0xffff] = dw11 & 0x1ffff;