feat(baremetal NIC): initial push for baremetal NIC

src/IOBuf.h

@@ -7,10 +7,10 @@
 
 #include <algorithm>
 #include <cassert>
+#include <cstring>
 #include <forward_list>
 #include <memory>
 #include <vector>
-#include <cstring>
 
 #include <boost/iterator/iterator_facade.hpp>
 
@@ -64,6 +64,7 @@ class IOBuf {
   }
 
   void TrimEnd(size_t amount) { length_ -= amount; }
+  void SetLength(size_t amount) { length_ = amount; }
 
   bool IsChained() const { return Next() != this; }
 
@@ -319,11 +320,34 @@ class MutIOBuf : public IOBuf {
 
     template <typename T> T& GetNoAdvance() {
       assert(p_->Length() > 0);
+
+#ifdef __EBBRT_ENABLE_BAREMETAL_NIC__
+      if (p_->Length() - offset_ < sizeof(T)) {
+        // request straddles buffers, allocate a new chunk of memory to copy it
+        // into (so it is contiguous)
+        chunk_list.emplace_front();
+        auto& chunk = chunk_list.front();
+        chunk.reserve(sizeof(T));
+        auto p = p_;
+        auto len = sizeof(T);
+        auto offset = offset_;
+        while (len > 0) {
+          auto remainder = std::min(p->Length() - offset, len);
+          auto data = p->Data() + offset;
+          chunk.insert(chunk.end(), data, data + remainder);
+          p = p->Next();
+          offset = 0;
+          len -= remainder;
+        }	
+        return *reinterpret_cast<T*>(Data());
+      }
+#else
+
       if (p_->Length() - offset_ < sizeof(T)) {
         throw std::runtime_error(
             "MutDataPointer::Get(): request straddles buffer");
       }
-
+#endif
       return *reinterpret_cast<T*>(Data());
     }
 
@@ -354,6 +378,7 @@ class MutIOBuf : public IOBuf {
    private:
     MutIOBuf* p_{nullptr};
     size_t offset_{0};
+    std::forward_list<std::vector<char>> chunk_list;
   };
 
   MutDataPointer GetMutDataPointer() { return MutDataPointer(this); }

src/native/EventManager.cc

@@ -16,6 +16,19 @@
 #include "Trace.h"
 #include "VMem.h"
 
+uint32_t nsleep_states[16];
+uint32_t sleep_state[16];
+/*uint32_t processCnt[16];
+uint32_t swEventCnt[16];
+uint32_t idleEventCnt[16];
+uint32_t processInterruptCntAll[16];
+uint32_t processInterruptCntA[16];
+uint32_t processInterruptCntB[16];
+uint32_t processInterruptCntC[16];
+uint32_t passTokenCnt[16];
+uint32_t receiveTokenCnt[16];
+uint32_t genFireCnt[16];*/
+
 namespace {
 struct InterruptHandler {
   ebbrt::RcuHListHook hook;
@@ -105,11 +118,13 @@ SwitchStack(uintptr_t first_param, uintptr_t stack, void (*func)(uintptr_t));
 
 void ebbrt::EventManager::StartProcessingEvents() {
   auto stack_top = (active_event_context_.stack + kStackPages).ToAddr();
+  //ebbrt::kprintf_force("StartProcessingEvents()\n");
   SwitchStack(reinterpret_cast<uintptr_t>(this), stack_top, CallProcess);
 }
 
 void ebbrt::EventManager::CallProcess(uintptr_t mgr) {
   auto pmgr = reinterpret_cast<EventManager*>(mgr);
+  //ebbrt::kprintf_force("CallProcess()\n");
   pmgr->Process();
 }
 
@@ -129,12 +144,17 @@ template <typename F> void ebbrt::EventManager::InvokeFunction(F&& f) {
 
 void ebbrt::EventManager::Process() {
   auto stack_top = (active_event_context_.stack + kStackPages).ToAddr();
+  uint32_t mycpu = static_cast<uint32_t>(Cpu::GetMine());
   Cpu::GetMine().SetEventStack(stack_top);
+  uint32_t ecx, edx, eax;
+  ecx = edx = eax = 0;
+
 // process an interrupt without halting
 // the sti instruction starts processing interrupts *after* the next
 // instruction is executed (to allow for a halt for example). The nop gives us
 // a one instruction window to process an interrupt (before the cli)
 process:
+  //processCnt[mycpu]++;
   asm volatile("sti;"
                "nop;"
                "cli;");
@@ -145,17 +165,38 @@ void ebbrt::EventManager::Process() {
     auto f = std::move(tasks_.front());
     tasks_.pop_front();
     InvokeFunction(f);
+    //swEventCnt[mycpu]++;
     // if we had a task to execute, then we go to the top again
     goto process;
   }
 
   if (idle_callback_) {
+    //idleEventCnt[mycpu]++;
     InvokeFunction(*idle_callback_);
     goto process;
   }
 
+  nsleep_states[mycpu] ++;  
+  asm volatile(".byte 0x0f, 0x01, 0xc8;"
+	       :: "a" ((void*)&flags), "c" (ecx), "d"(edx));
+
+  // https://elixir.bootlin.com/linux/v4.15.1/source/arch/x86/include/asm/mwait.h#L100
+  // https://elixir.bootlin.com/linux/v5.5.1/source/drivers/idle/intel_idle.c
+
+  // sandy bridge
+  // C1 0x00
+  // C1E 0x01
+  // C3 0x10
+  // C6 0x20
+  // C7 0x30
+  ecx = 1; /* break on interrupt flag */
+  eax = sleep_state[mycpu]; /* we always pick the deepest sleep state */
+
+  asm volatile("sti; .byte 0x0f, 0x01, 0xc9;"
+	       :: "a" (eax), "c" (ecx));
+
   asm volatile("sti;"
-               "hlt;");
+	       "hlt;");
   kabort("Woke up from halt?!?!");
 }
 
@@ -175,7 +216,7 @@ void ebbrt::EventManager::FreeStack(Pfn stack) { free_stacks_.push(stack); }
 static_assert(ebbrt::Cpu::kMaxCpus <= 256, "adjust event id calculation");
 
 ebbrt::EventManager::EventManager(const RepMap& rm)
-    : reps_(rm), next_event_id_(Cpu::GetMine() << 24),
+  : reps_(rm), next_event_id_(Cpu::GetMine() << 24),
       active_event_context_(next_event_id_++, AllocateStack()) {}
 
 void ebbrt::EventManager::Spawn(MovableFunction<void()> func,
@@ -204,6 +245,7 @@ void ebbrt::EventManager::CallSync(uintptr_t mgr) {
   // "fresh" event. Therefore if the sync_contexts_ stack is empty, we just go
   // back to the event loop
   if (unlikely(pmgr->sync_contexts_.empty())) {
+    //ebbrt::kprintf_force("CallSync, Process()\n");
     pmgr->Process();
   } else {
     // save this stack
@@ -327,19 +369,27 @@ uint8_t ebbrt::EventManager::AllocateVector(MovableFunction<void()> func) {
 }
 
 void ebbrt::EventManager::ProcessInterrupt(int num) {
+  //uint32_t mycpu = static_cast<uint32_t>(Cpu::GetMine());  
   apic::Eoi();
+  //processInterruptCntAll[mycpu]++;
+
   if (num == 32) {
     // pull all remote tasks onto our queue
     std::lock_guard<ebbrt::SpinLock> l(remote_.lock);
     tasks_.splice(tasks_.end(), std::move(remote_.tasks));
+    //processInterruptCntA[mycpu]++;
+
   } else if (num == 33) {
+    //processInterruptCntB[mycpu]++;
     ReceiveToken();
   } else {
+    //processInterruptCntC[mycpu]++;
     auto ih = vec_data->map.find(num);
     kassert(ih != nullptr);
     auto& f = ih->func;
     InvokeFunction(f);
   }
+  //ebbrt::kprintf_force("ProcessInterrupt %d\n", num);OA
   Process();
 }
 
@@ -378,7 +428,9 @@ ebbrt::EventManager::EventContext::EventContext(uint32_t event_id, Pfn stack)
 
 void ebbrt::EventManager::PassToken() {
   size_t my_cpu_index = Cpu::GetMine();
+  //uint32_t mycpu = static_cast<uint32_t>(Cpu::GetMine());  
   if (Cpu::Count() > 1) {
+    //passTokenCnt[mycpu] ++;
     auto next_cpu_index = (my_cpu_index + 1) % Cpu::Count();
     auto next_cpu = Cpu::GetByIndex(next_cpu_index);
     kassert(next_cpu != nullptr);
@@ -389,14 +441,18 @@ void ebbrt::EventManager::PassToken() {
 }
 
 void ebbrt::EventManager::ReceiveToken() {
+  //uint32_t mycpu = static_cast<uint32_t>(Cpu::GetMine());  
   pending_generation_ = generation_++;
-
+  //receiveTokenCnt[mycpu] ++;  
   StartTimer();
 }
 
 // Check Generation
 void ebbrt::EventManager::Fire() {
-  if (generation_count_[pending_generation_ % 2] == 0) {
+  //uint32_t mycpu = static_cast<uint32_t>(Cpu::GetMine());
+  //genFireCnt[mycpu] ++;
+
+  if (generation_count_[pending_generation_ % 2] == 0) {    
     // generation complete
     PassToken();
     // temporarily store tasks that have now lived at least one entire
@@ -410,12 +466,15 @@ void ebbrt::EventManager::Fire() {
       tasks.pop();
     }
   } else {
+    //ebbrt::kprintf("EventManager()::Fire()\n");
     StartTimer();
   }
 }
 
 void ebbrt::EventManager::StartTimer() {
-  timer->Start(*this, std::chrono::milliseconds(1),
+  //timer->Start(*this, std::chrono::milliseconds(2),
+  //             /* repeat = */ false);
+  timer->Start(*this, std::chrono::microseconds(1500),
                /* repeat = */ false);
 }
 

src/native/EventManager.h

@@ -23,6 +23,23 @@
 #include "Trans.h"
 #include "VMemAllocator.h"
 
+extern uint32_t nsleep_states[16];
+extern uint32_t sleep_state[16];
+/*extern uint32_t processCnt[16];
+extern uint32_t swEventCnt[16];
+extern uint32_t idleEventCnt[16];
+extern uint32_t processInterruptCntAll[16];
+extern uint32_t processInterruptCntA[16];
+extern uint32_t processInterruptCntB[16];
+extern uint32_t processInterruptCntC[16];
+extern uint32_t passTokenCnt[16];
+extern uint32_t receiveTokenCnt[16];
+extern uint32_t genFireCnt[16];
+extern uint32_t timerCnt[16];
+extern uint32_t fireCntA[16];
+extern uint32_t fireCntB[16];
+*/
+
 namespace ebbrt {
 
 class EventManager : Timer::Hook {
@@ -112,7 +129,8 @@ class EventManager : Timer::Hook {
   size_t pending_generation_ = 0;
   std::queue<MovableFunction<void()>> prev_rcu_tasks_;
   std::queue<MovableFunction<void()>> curr_rcu_tasks_;
-
+  unsigned long flags;
+
   struct RemoteData : CacheAligned {
     ebbrt::SpinLock lock;
     std::list<MovableFunction<void()>> tasks;

src/native/GeneralPurposeAllocator.h

@@ -16,6 +16,26 @@
 
 namespace ebbrt {
 
+// handler used in Pci.cc code to handle faults on multicores when mapping
+// device
+class MulticorePciFaultHandler : public ebbrt::VMemAllocator::PageFaultHandler {
+  ebbrt::Pfn vpage_;
+  ebbrt::Pfn ppage_;
+  size_t size_;
+
+ public:
+  void SetMap(ebbrt::Pfn va, ebbrt::Pfn pa, size_t s) {
+    vpage_ = va;
+    ppage_ = pa;
+    size_ = s;
+  }
+
+  void HandleFault(ebbrt::idt::ExceptionFrame* ef,
+                   uintptr_t faulted_address) override {
+    ebbrt::vmem::MapMemory(vpage_, ppage_, size_);
+  }
+};
+
 // page fault handler for mapping in physical pages
 // to virtual pages on all cores
 class LargeRegionFaultHandler : public ebbrt::VMemAllocator::PageFaultHandler {