Improve BCS waiting logic to satisfy AUB/TBX mode

BartoszDunajski · Compute-Runtime-Automation · commit a046de5689c6 · 2019-10-22T13:36:43.000+02:00
Change-Id: I52b44959b8bdc1cc66f136a4785233b95870fd0b
Signed-off-by: Dunajski, Bartosz &lt;bartosz.dunajski@intel.com&gt;
diff --git a/runtime/command_queue/command_queue.cpp b/runtime/command_queue/command_queue.cpp
@@ -149,7 +149,7 @@ void CommandQueue::waitUntilComplete(uint32_t taskCountToWait, FlushStamp flushS
     getGpgpuCommandStreamReceiver().waitForTaskCountAndCleanAllocationList(taskCountToWait, TEMPORARY_ALLOCATION);
 
     if (auto bcsCsr = getBcsCommandStreamReceiver()) {
-        auto bcsTaskCount = *bcsCsr->getTagAddress();
+        bcsCsr->waitForTaskCountWithKmdNotifyFallback(bcsTaskCount, 0, false, false);
         bcsCsr->waitForTaskCountAndCleanAllocationList(bcsTaskCount, TEMPORARY_ALLOCATION);
     }
 
diff --git a/runtime/command_queue/command_queue.h b/runtime/command_queue/command_queue.h
@@ -406,6 +406,8 @@ class CommandQueue : public BaseObject<_cl_command_queue> {
 
     bool isMultiEngineQueue() const { return this->multiEngineQueue; }
 
+    void updateBcsTaskCount(uint32_t newBcsTaskCount) { this->bcsTaskCount = newBcsTaskCount; }
+
     // taskCount of last task
     uint32_t taskCount = 0;
 
@@ -455,6 +457,7 @@ class CommandQueue : public BaseObject<_cl_command_queue> {
     QueuePriority priority = QueuePriority::MEDIUM;
     QueueThrottle throttle = QueueThrottle::MEDIUM;
     uint64_t sliceCount = QueueSliceCount::defaultSliceCount;
+    uint32_t bcsTaskCount = 0;
 
     bool perfCountersEnabled = false;
 
diff --git a/runtime/command_queue/enqueue_common.h b/runtime/command_queue/enqueue_common.h
@@ -865,7 +865,7 @@ CompletionStamp CommandQueueHw<GfxFamily>::enqueueCommandWithoutKernel(
         enqueueProperties.blitProperties->csrDependencies.makeResident(*bcsCsr);
         previousTimestampPacketNodes->makeResident(*bcsCsr);
         timestampPacketContainer->makeResident(*bcsCsr);
-        bcsCsr->blitBuffer(*enqueueProperties.blitProperties);
+        this->bcsTaskCount = bcsCsr->blitBuffer(*enqueueProperties.blitProperties);
     }
 
     DispatchFlags dispatchFlags(
diff --git a/runtime/command_stream/command_stream_receiver.h b/runtime/command_stream/command_stream_receiver.h
@@ -176,7 +176,7 @@ class CommandStreamReceiver {
         this->latestSentTaskCount = latestSentTaskCount;
     }
 
-    virtual void blitBuffer(const BlitProperties &blitProperites) = 0;
+    virtual uint32_t blitBuffer(const BlitProperties &blitProperites) = 0;
 
     ScratchSpaceController *getScratchSpaceController() const {
         return scratchSpaceController.get();
diff --git a/runtime/command_stream/command_stream_receiver_hw.h b/runtime/command_stream/command_stream_receiver_hw.h
@@ -73,7 +73,7 @@ class CommandStreamReceiverHw : public CommandStreamReceiver {
         return CommandStreamReceiverType::CSR_HW;
     }
 
-    void blitBuffer(const BlitProperties &blitProperites) override;
+    uint32_t blitBuffer(const BlitProperties &blitProperites) override;
 
     bool isMultiOsContextCapable() const override;
 
diff --git a/runtime/command_stream/command_stream_receiver_hw_base.inl b/runtime/command_stream/command_stream_receiver_hw_base.inl
@@ -795,7 +795,7 @@ bool CommandStreamReceiverHw<GfxFamily>::detectInitProgrammingFlagsRequired(cons
 }
 
 template <typename GfxFamily>
-void CommandStreamReceiverHw<GfxFamily>::blitBuffer(const BlitProperties &blitProperites) {
+uint32_t CommandStreamReceiverHw<GfxFamily>::blitBuffer(const BlitProperties &blitProperites) {
     using MI_BATCH_BUFFER_END = typename GfxFamily::MI_BATCH_BUFFER_END;
     using MI_FLUSH_DW = typename GfxFamily::MI_FLUSH_DW;
 
@@ -845,6 +845,8 @@ void CommandStreamReceiverHw<GfxFamily>::blitBuffer(const BlitProperties &blitPr
         waitForTaskCountWithKmdNotifyFallback(newTaskCount, flushStampToWait, false, false);
         internalAllocationStorage->cleanAllocationList(newTaskCount, TEMPORARY_ALLOCATION);
     }
+
+    return newTaskCount;
 }
 
 template <typename GfxFamily>
diff --git a/runtime/helpers/task_information.cpp b/runtime/helpers/task_information.cpp
@@ -257,7 +257,9 @@ void CommandWithoutKernel::dispatchBlitOperation() {
     blitProperties.csrDependencies.push_back(barrierTimestampPacketNodes.get());
     blitProperties.outputTimestampPacket = currentTimestampPacketNodes.get();
 
-    bcsCsr->blitBuffer(blitProperties);
+    auto bcsTaskCount = bcsCsr->blitBuffer(blitProperties);
+
+    commandQueue.updateBcsTaskCount(bcsTaskCount);
 }
 
 CompletionStamp &CommandWithoutKernel::submit(uint32_t taskLevel, bool terminated) {
diff --git a/unit_tests/kernel/kernel_tests.cpp b/unit_tests/kernel/kernel_tests.cpp
@@ -489,7 +489,7 @@ class CommandStreamReceiverMock : public CommandStreamReceiver {
 
     void waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool quickKmdSleep, bool forcePowerSavingMode) override {
     }
-    void blitBuffer(const BlitProperties &blitProperites) override{};
+    uint32_t blitBuffer(const BlitProperties &blitProperites) override { return taskCount; };
 
     CompletionStamp flushTask(
         LinearStream &commandStream,
diff --git a/unit_tests/libult/ult_aub_command_stream_receiver.h b/unit_tests/libult/ult_aub_command_stream_receiver.h
@@ -38,9 +38,9 @@ class UltAubCommandStreamReceiver : public AUBCommandStreamReceiverHw<GfxFamily>
         return csr;
     }
 
-    void blitBuffer(const BlitProperties &blitProperites) override {
+    uint32_t blitBuffer(const BlitProperties &blitProperites) override {
         blitBufferCalled++;
-        BaseClass::blitBuffer(blitProperites);
+        return BaseClass::blitBuffer(blitProperites);
     }
 
     uint32_t blitBufferCalled = 0;
diff --git a/unit_tests/libult/ult_command_stream_receiver.h b/unit_tests/libult/ult_command_stream_receiver.h
@@ -173,9 +173,9 @@ class UltCommandStreamReceiver : public CommandStreamReceiverHw<GfxFamily>, publ
         return CommandStreamReceiverHw<GfxFamily>::obtainUniqueOwnership();
     }
 
-    void blitBuffer(const BlitProperties &blitProperites) override {
+    uint32_t blitBuffer(const BlitProperties &blitProperites) override {
         blitBufferCalled++;
-        CommandStreamReceiverHw<GfxFamily>::blitBuffer(blitProperites);
+        return CommandStreamReceiverHw<GfxFamily>::blitBuffer(blitProperites);
     }
 
     bool createPerDssBackedBuffer(Device &device) override {
@@ -210,4 +210,4 @@ class UltCommandStreamReceiver : public CommandStreamReceiverHw<GfxFamily>, publ
     DispatchFlags recordedDispatchFlags;
     bool multiOsContextCapable = false;
 };
-} // namespace NEO
+} // namespace NEO
diff --git a/unit_tests/mem_obj/buffer_tests.cpp b/unit_tests/mem_obj/buffer_tests.cpp
@@ -718,6 +718,31 @@ struct BcsBufferTests : public ::testing::Test {
         std::unique_ptr<CommandStreamReceiver> bcsCsr;
     };
 
+    template <typename FamilyType>
+    class MyMockCsr : public UltCommandStreamReceiver<FamilyType> {
+      public:
+        using UltCommandStreamReceiver<FamilyType>::UltCommandStreamReceiver;
+
+        void waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait,
+                                                   bool useQuickKmdSleep, bool forcePowerSavingMode) override {
+            EXPECT_EQ(this->latestFlushedTaskCount, taskCountToWait);
+            EXPECT_EQ(0u, flushStampToWait);
+            EXPECT_FALSE(useQuickKmdSleep);
+            EXPECT_FALSE(forcePowerSavingMode);
+            waitForTaskCountWithKmdNotifyFallbackCalled++;
+        }
+
+        void waitForTaskCountAndCleanAllocationList(uint32_t requiredTaskCount, uint32_t allocationUsage) override {
+            EXPECT_EQ(1u, waitForTaskCountWithKmdNotifyFallbackCalled);
+            EXPECT_EQ(this->latestFlushedTaskCount, requiredTaskCount);
+            waitForTaskCountAndCleanAllocationListCalled++;
+        }
+
+        uint32_t waitForTaskCountAndCleanAllocationListCalled = 0;
+        uint32_t waitForTaskCountWithKmdNotifyFallbackCalled = 0;
+        CommandStreamReceiver *gpgpuCsr = nullptr;
+    };
+
     template <typename FamilyType>
     void SetUpT() {
         if (is32bit) {
@@ -1122,21 +1147,7 @@ HWTEST_TEMPLATED_F(BcsBufferTests, givenInputAndOutputTimestampPacketWhenBlitCal
 }
 
 HWTEST_TEMPLATED_F(BcsBufferTests, givenBlockingEnqueueWhenUsingBcsThenCallWait) {
-    class MyMockCsr : public UltCommandStreamReceiver<FamilyType> {
-      public:
-        using UltCommandStreamReceiver<FamilyType>::UltCommandStreamReceiver;
-
-        void waitForTaskCountAndCleanAllocationList(uint32_t requiredTaskCount, uint32_t allocationUsage) override {
-            EXPECT_TRUE(gpgpuCsr->getTemporaryAllocations().peekIsEmpty());
-            EXPECT_EQ(*this->getTagAddress(), requiredTaskCount);
-            waitForTaskCountAndCleanAllocationListCalled++;
-        }
-
-        uint32_t waitForTaskCountAndCleanAllocationListCalled = 0;
-        CommandStreamReceiver *gpgpuCsr = nullptr;
-    };
-
-    auto myMockCsr = new MyMockCsr(*device->getExecutionEnvironment());
+    auto myMockCsr = new MyMockCsr<FamilyType>(*device->getExecutionEnvironment());
     myMockCsr->taskCount = 1234;
     myMockCsr->initializeTagAllocation();
     myMockCsr->setupContext(*bcsMockContext->bcsOsContext);
@@ -1163,6 +1174,37 @@ HWTEST_TEMPLATED_F(BcsBufferTests, givenBlockingEnqueueWhenUsingBcsThenCallWait)
     EXPECT_EQ(1u, myMockCsr->waitForTaskCountAndCleanAllocationListCalled);
 }
 
+HWTEST_TEMPLATED_F(BcsBufferTests, givenBlockedEnqueueWhenUsingBcsThenWaitForValidTaskCountOnBlockingCall) {
+    auto myMockCsr = new MyMockCsr<FamilyType>(*device->getExecutionEnvironment());
+    myMockCsr->taskCount = 1234;
+    myMockCsr->initializeTagAllocation();
+    myMockCsr->setupContext(*bcsMockContext->bcsOsContext);
+    bcsMockContext->bcsCsr.reset(myMockCsr);
+
+    EngineControl bcsEngineControl = {myMockCsr, bcsMockContext->bcsOsContext.get()};
+
+    auto cmdQ = clUniquePtr(new MockCommandQueueHw<FamilyType>(bcsMockContext.get(), device.get(), nullptr));
+    cmdQ->bcsEngine = &bcsEngineControl;
+    auto &gpgpuCsr = cmdQ->getGpgpuCommandStreamReceiver();
+    myMockCsr->gpgpuCsr = &gpgpuCsr;
+
+    cl_int retVal = CL_SUCCESS;
+    auto buffer = clUniquePtr<Buffer>(Buffer::create(bcsMockContext.get(), CL_MEM_READ_WRITE, 1, nullptr, retVal));
+    buffer->forceDisallowCPUCopy = true;
+    void *hostPtr = reinterpret_cast<void *>(0x12340000);
+
+    UserEvent userEvent;
+    cl_event waitlist = &userEvent;
+
+    cmdQ->enqueueWriteBuffer(buffer.get(), false, 0, 1, hostPtr, nullptr, 1, &waitlist, nullptr);
+
+    userEvent.setStatus(CL_COMPLETE);
+    EXPECT_EQ(0u, myMockCsr->waitForTaskCountAndCleanAllocationListCalled);
+
+    cmdQ->finish();
+    EXPECT_EQ(1u, myMockCsr->waitForTaskCountAndCleanAllocationListCalled);
+}
+
 TEST_F(RenderCompressedBuffersCopyHostMemoryTests, givenNonRenderCompressedBufferWhenCopyFromHostPtrIsRequiredThenDontCallWriteBuffer) {
     hwInfo->capabilityTable.ftrRenderCompressedBuffers = false;
 
diff --git a/unit_tests/mocks/mock_csr.h b/unit_tests/mocks/mock_csr.h
@@ -214,10 +214,11 @@ class MockCsrHw2 : public CommandStreamReceiverHw<GfxFamily> {
         return completionStamp;
     }
 
-    void blitBuffer(const BlitProperties &blitProperites) override {
+    uint32_t blitBuffer(const BlitProperties &blitProperites) override {
         if (!skipBlitCalls) {
-            CommandStreamReceiverHw<GfxFamily>::blitBuffer(blitProperites);
+            return CommandStreamReceiverHw<GfxFamily>::blitBuffer(blitProperites);
         }
+        return taskCount;
     }
 
     bool skipBlitCalls = false;
@@ -286,7 +287,7 @@ class MockCommandStreamReceiver : public CommandStreamReceiver {
     void waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool quickKmdSleep, bool forcePowerSavingMode) override {
     }
 
-    void blitBuffer(const BlitProperties &blitProperites) override{};
+    uint32_t blitBuffer(const BlitProperties &blitProperites) override { return taskCount; };
 
     void setOSInterface(OSInterface *osInterface);
 

Original file line number	Diff line number	Diff line change
`@@ -149,7 +149,7 @@ void CommandQueue::waitUntilComplete(uint32_t taskCountToWait, FlushStamp flushS`
`149`	`149`	`getGpgpuCommandStreamReceiver().waitForTaskCountAndCleanAllocationList(taskCountToWait, TEMPORARY_ALLOCATION);`
`150`	`150`
`151`	`151`	`if (auto bcsCsr = getBcsCommandStreamReceiver()) {`
`152`		`- auto bcsTaskCount = *bcsCsr->getTagAddress();`
	`152`	`+ bcsCsr->waitForTaskCountWithKmdNotifyFallback(bcsTaskCount, 0, false, false);`
`153`	`153`	`bcsCsr->waitForTaskCountAndCleanAllocationList(bcsTaskCount, TEMPORARY_ALLOCATION);`
`154`	`154`	`}`
`155`	`155`
Original file line number	Diff line number	Diff line change
`@@ -865,7 +865,7 @@ CompletionStamp CommandQueueHw<GfxFamily>::enqueueCommandWithoutKernel(`
`865`	`865`	`enqueueProperties.blitProperties->csrDependencies.makeResident(*bcsCsr);`
`866`	`866`	`previousTimestampPacketNodes->makeResident(*bcsCsr);`
`867`	`867`	`timestampPacketContainer->makeResident(*bcsCsr);`
`868`		`- bcsCsr->blitBuffer(*enqueueProperties.blitProperties);`
	`868`	`+ this->bcsTaskCount = bcsCsr->blitBuffer(*enqueueProperties.blitProperties);`
`869`	`869`	`}`
`870`	`870`
`871`	`871`	`DispatchFlags dispatchFlags(`
Original file line number	Diff line number	Diff line change
`@@ -176,7 +176,7 @@ class CommandStreamReceiver {`
`176`	`176`	`this->latestSentTaskCount = latestSentTaskCount;`
`177`	`177`	`}`
`178`	`178`
`179`		`- virtual void blitBuffer(const BlitProperties &blitProperites) = 0;`
	`179`	`+ virtual uint32_t blitBuffer(const BlitProperties &blitProperites) = 0;`
`180`	`180`
`181`	`181`	`ScratchSpaceController *getScratchSpaceController() const {`
`182`	`182`	`return scratchSpaceController.get();`
Original file line number	Diff line number	Diff line change
`@@ -73,7 +73,7 @@ class CommandStreamReceiverHw : public CommandStreamReceiver {`
`73`	`73`	`return CommandStreamReceiverType::CSR_HW;`
`74`	`74`	`}`
`75`	`75`
`76`		`- void blitBuffer(const BlitProperties &blitProperites) override;`
	`76`	`+ uint32_t blitBuffer(const BlitProperties &blitProperites) override;`
`77`	`77`
`78`	`78`	`bool isMultiOsContextCapable() const override;`
`79`	`79`
Original file line number	Diff line number	Diff line change
`@@ -795,7 +795,7 @@ bool CommandStreamReceiverHw<GfxFamily>::detectInitProgrammingFlagsRequired(cons`
`795`	`795`	`}`
`796`	`796`
`797`	`797`	`template <typename GfxFamily>`
`798`		`-void CommandStreamReceiverHw<GfxFamily>::blitBuffer(const BlitProperties &blitProperites) {`
	`798`	`+uint32_t CommandStreamReceiverHw<GfxFamily>::blitBuffer(const BlitProperties &blitProperites) {`
`799`	`799`	`using MI_BATCH_BUFFER_END = typename GfxFamily::MI_BATCH_BUFFER_END;`
`800`	`800`	`using MI_FLUSH_DW = typename GfxFamily::MI_FLUSH_DW;`
`801`	`801`
`@@ -845,6 +845,8 @@ void CommandStreamReceiverHw<GfxFamily>::blitBuffer(const BlitProperties &blitPr`
`845`	`845`	`waitForTaskCountWithKmdNotifyFallback(newTaskCount, flushStampToWait, false, false);`
`846`	`846`	`internalAllocationStorage->cleanAllocationList(newTaskCount, TEMPORARY_ALLOCATION);`
`847`	`847`	`}`
	`848`	`+`
	`849`	`+ return newTaskCount;`
`848`	`850`	`}`
`849`	`851`
`850`	`852`	`template <typename GfxFamily>`
Original file line number	Diff line number	Diff line change
`@@ -257,7 +257,9 @@ void CommandWithoutKernel::dispatchBlitOperation() {`
`257`	`257`	`blitProperties.csrDependencies.push_back(barrierTimestampPacketNodes.get());`
`258`	`258`	`blitProperties.outputTimestampPacket = currentTimestampPacketNodes.get();`
`259`	`259`
`260`		`- bcsCsr->blitBuffer(blitProperties);`
	`260`	`+ auto bcsTaskCount = bcsCsr->blitBuffer(blitProperties);`
	`261`	`+`
	`262`	`+ commandQueue.updateBcsTaskCount(bcsTaskCount);`
`261`	`263`	`}`
`262`	`264`
`263`	`265`	`CompletionStamp &CommandWithoutKernel::submit(uint32_t taskLevel, bool terminated) {`
Original file line number	Diff line number	Diff line change
`@@ -489,7 +489,7 @@ class CommandStreamReceiverMock : public CommandStreamReceiver {`
`489`	`489`
`490`	`490`	`void waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool quickKmdSleep, bool forcePowerSavingMode) override {`
`491`	`491`	`}`
`492`		`- void blitBuffer(const BlitProperties &blitProperites) override{};`
	`492`	`+ uint32_t blitBuffer(const BlitProperties &blitProperites) override { return taskCount; };`
`493`	`493`
`494`	`494`	`CompletionStamp flushTask(`
`495`	`495`	`LinearStream &commandStream,`
Original file line number	Diff line number	Diff line change
`@@ -38,9 +38,9 @@ class UltAubCommandStreamReceiver : public AUBCommandStreamReceiverHw<GfxFamily>`
`38`	`38`	`return csr;`
`39`	`39`	`}`
`40`	`40`
`41`		`- void blitBuffer(const BlitProperties &blitProperites) override {`
	`41`	`+ uint32_t blitBuffer(const BlitProperties &blitProperites) override {`
`42`	`42`	`blitBufferCalled++;`
`43`		`- BaseClass::blitBuffer(blitProperites);`
	`43`	`+ return BaseClass::blitBuffer(blitProperites);`
`44`	`44`	`}`
`45`	`45`
`46`	`46`	`uint32_t blitBufferCalled = 0;`