Skip to content

Commit dc4de3c

Browse files
DziubanMaciejIntelCompute-Runtime-Automation
DziubanMaciejIntel
authored and
Compute-Runtime-Automation
committed
Move trimResidency functions to WddmResidencyController
Change-Id: I046fd34d5336b767ed38eda31e58e4a35ceee5f8 Signed-off-by: Maciej Dziuban <maciej.dziuban@intel.com>
1 parent a30c70d commit dc4de3c

17 files changed

+621
-573
lines changed

‎runtime/os_interface/windows/os_context_win.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ OsContextWin::OsContextImpl(Wddm &wddm, uint32_t osContextId) : wddm(wddm), resi
2323
return;
2424
}
2525
}
26-
initialized = wddmInterface->createMonitoredFence(*this);
26+
initialized = wddmInterface->createMonitoredFence(this->residencyController);
2727
};
2828
OsContextWin::~OsContextImpl() {
2929
wddm.getWddmInterface()->destroyHwQueue(hwQueueHandle);

‎runtime/os_interface/windows/wddm/wddm.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -768,13 +768,13 @@ bool Wddm::waitOnGPU(D3DKMT_HANDLE context) {
768768
return status == STATUS_SUCCESS;
769769
}
770770

771-
bool Wddm::waitFromCpu(uint64_t lastFenceValue, OsContextWin &osContext) {
771+
bool Wddm::waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence) {
772772
NTSTATUS status = STATUS_SUCCESS;
773773

774-
if (lastFenceValue > *osContext.getResidencyController().getMonitoredFence().cpuAddress) {
774+
if (lastFenceValue > *monitoredFence.cpuAddress) {
775775
D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU waitFromCpu = {0};
776776
waitFromCpu.ObjectCount = 1;
777-
waitFromCpu.ObjectHandleArray = &osContext.getResidencyController().getMonitoredFence().fenceHandle;
777+
waitFromCpu.ObjectHandleArray = &monitoredFence.fenceHandle;
778778
waitFromCpu.FenceValueArray = &lastFenceValue;
779779
waitFromCpu.hDevice = device;
780780
waitFromCpu.hAsyncEvent = NULL;

‎runtime/os_interface/windows/wddm/wddm.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ class Wddm {
7676
MOCKABLE_VIRTUAL bool queryAdapterInfo();
7777

7878
MOCKABLE_VIRTUAL bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, OsContextWin &osContext);
79-
MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue, OsContextWin &osContext);
79+
MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence);
8080

8181
NTSTATUS escape(D3DKMT_ESCAPE &escapeCommand);
8282
VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmMemoryManager *memoryManager);

‎runtime/os_interface/windows/wddm/wddm_interface.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ bool WddmInterface20::createHwQueue(PreemptionMode preemptionMode, OsContextWin
1818
}
1919
void WddmInterface20::destroyHwQueue(D3DKMT_HANDLE hwQueue) {}
2020

21-
bool WddmInterface::createMonitoredFence(OsContextWin &osContext) {
21+
bool WddmInterface::createMonitoredFence(WddmResidencyController &residencyController) {
2222
NTSTATUS Status;
2323
D3DKMT_CREATESYNCHRONIZATIONOBJECT2 CreateSynchronizationObject = {0};
2424
CreateSynchronizationObject.hDevice = wddm.getDevice();
@@ -29,9 +29,9 @@ bool WddmInterface::createMonitoredFence(OsContextWin &osContext) {
2929

3030
DEBUG_BREAK_IF(STATUS_SUCCESS != Status);
3131

32-
osContext.getResidencyController().resetMonitoredFenceParams(CreateSynchronizationObject.hSyncObject,
33-
reinterpret_cast<uint64_t *>(CreateSynchronizationObject.Info.MonitoredFence.FenceValueCPUVirtualAddress),
34-
CreateSynchronizationObject.Info.MonitoredFence.FenceValueGPUVirtualAddress);
32+
residencyController.resetMonitoredFenceParams(CreateSynchronizationObject.hSyncObject,
33+
reinterpret_cast<uint64_t *>(CreateSynchronizationObject.Info.MonitoredFence.FenceValueCPUVirtualAddress),
34+
CreateSynchronizationObject.Info.MonitoredFence.FenceValueGPUVirtualAddress);
3535

3636
return Status == STATUS_SUCCESS;
3737
}

‎runtime/os_interface/windows/wddm/wddm_interface.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
namespace OCLRT {
1616
class Gdi;
1717
class Wddm;
18+
class WddmResidencyController;
1819

1920
using OsContextWin = OsContext::OsContextImpl;
2021

@@ -25,7 +26,7 @@ class WddmInterface {
2526
WddmInterface() = delete;
2627
virtual bool createHwQueue(PreemptionMode preemptionMode, OsContextWin &osContext) = 0;
2728
virtual void destroyHwQueue(D3DKMT_HANDLE hwQueue) = 0;
28-
bool createMonitoredFence(OsContextWin &osContext);
29+
bool createMonitoredFence(WddmResidencyController &residencyController);
2930
virtual const bool hwQueuesSupported() = 0;
3031
virtual bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, OsContextWin &osContext) = 0;
3132
Wddm &wddm;

‎runtime/os_interface/windows/wddm_device_command_stream.inl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ MemoryManager *WddmCommandStreamReceiver<GfxFamily>::createMemoryManager(bool en
158158

159159
template <typename GfxFamily>
160160
bool WddmCommandStreamReceiver<GfxFamily>::waitForFlushStamp(FlushStamp &flushStampToWait, OsContext &osContext) {
161-
return wddm->waitFromCpu(flushStampToWait, *osContext.get());
161+
return wddm->waitFromCpu(flushStampToWait, osContext.get()->getResidencyController().getMonitoredFence());
162162
}
163163

164164
template <typename GfxFamily>

‎runtime/os_interface/windows/wddm_memory_manager.cpp‎

Lines changed: 2 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ void APIENTRY WddmMemoryManager::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *t
6363
}
6464

6565
wddmMemMngr->getRegisteredOsContext(0)->get()->getResidencyController().acquireTrimCallbackLock();
66-
wddmMemMngr->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim);
66+
wddmMemMngr->getRegisteredOsContext(0)->get()->getResidencyController().trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim);
6767
wddmMemMngr->getRegisteredOsContext(0)->get()->getResidencyController().releaseTrimCallbackLock();
6868
}
6969

@@ -526,7 +526,7 @@ bool WddmMemoryManager::makeResidentResidencyAllocations(ResidencyContainer &all
526526
uint64_t bytesToTrim = 0;
527527
while ((result = wddm->makeResident(handlesForResidency.get(), totalHandlesCount, false, &bytesToTrim)) == false) {
528528
this->memoryBudgetExhausted = true;
529-
bool trimmingDone = trimResidencyToBudget(bytesToTrim);
529+
bool trimmingDone = this->getRegisteredOsContext(0u)->get()->getResidencyController().trimResidencyToBudget(bytesToTrim);
530530
bool cantTrimFurther = !trimmingDone;
531531
if (cantTrimFurther) {
532532
result = wddm->makeResident(handlesForResidency.get(), totalHandlesCount, true, &bytesToTrim);
@@ -571,152 +571,6 @@ void WddmMemoryManager::makeNonResidentEvictionAllocations(ResidencyContainer &e
571571
osContext.get()->getResidencyController().releaseLock();
572572
}
573573

574-
void WddmMemoryManager::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes) {
575-
OsContext &osContext = *getRegisteredOsContext(0);
576-
if (flags.PeriodicTrim) {
577-
bool periodicTrimDone = false;
578-
D3DKMT_HANDLE fragmentEvictHandles[3] = {0};
579-
uint64_t sizeToTrim = 0;
580-
581-
osContext.get()->getResidencyController().acquireLock();
582-
583-
WddmAllocation *wddmAllocation = nullptr;
584-
while ((wddmAllocation = osContext.get()->getResidencyController().getTrimCandidateHead()) != nullptr) {
585-
586-
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", osContext.get()->getResidencyController().getLastTrimFenceValue());
587-
588-
// allocation was not used from last periodic trim
589-
if (wddmAllocation->getResidencyData().getFenceValueForContextId(0) <= osContext.get()->getResidencyController().getLastTrimFenceValue()) {
590-
591-
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation: handle =", wddmAllocation->handle, "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(0));
592-
593-
uint32_t fragmentsToEvict = 0;
594-
595-
if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
596-
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation: handle =", wddmAllocation->handle, "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(0));
597-
wddm->evict(&wddmAllocation->handle, 1, sizeToTrim);
598-
}
599-
600-
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
601-
if (wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(0) <= osContext.get()->getResidencyController().getLastTrimFenceValue()) {
602-
603-
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle, "lastFence =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(0));
604-
605-
fragmentEvictHandles[fragmentsToEvict++] = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle;
606-
wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->resident = false;
607-
}
608-
}
609-
610-
if (fragmentsToEvict != 0) {
611-
wddm->evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim);
612-
}
613-
614-
wddmAllocation->getResidencyData().resident = false;
615-
616-
osContext.get()->getResidencyController().removeFromTrimCandidateList(wddmAllocation, false);
617-
} else {
618-
periodicTrimDone = true;
619-
break;
620-
}
621-
}
622-
623-
if (osContext.get()->getResidencyController().checkTrimCandidateListCompaction()) {
624-
osContext.get()->getResidencyController().compactTrimCandidateList();
625-
}
626-
627-
osContext.get()->getResidencyController().releaseLock();
628-
}
629-
630-
if (flags.TrimToBudget) {
631-
632-
osContext.get()->getResidencyController().acquireLock();
633-
634-
trimResidencyToBudget(bytes);
635-
636-
osContext.get()->getResidencyController().releaseLock();
637-
}
638-
639-
if (flags.PeriodicTrim || flags.RestartPeriodicTrim) {
640-
const auto newPeriodicTrimFenceValue = *osContext.get()->getResidencyController().getMonitoredFence().cpuAddress;
641-
osContext.get()->getResidencyController().setLastTrimFenceValue(newPeriodicTrimFenceValue);
642-
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", newPeriodicTrimFenceValue);
643-
}
644-
}
645-
646-
bool WddmMemoryManager::trimResidencyToBudget(uint64_t bytes) {
647-
bool trimToBudgetDone = false;
648-
D3DKMT_HANDLE fragmentEvictHandles[3] = {0};
649-
uint64_t numberOfBytesToTrim = bytes;
650-
WddmAllocation *wddmAllocation = nullptr;
651-
auto &osContext = *getRegisteredOsContext(0);
652-
653-
trimToBudgetDone = (numberOfBytesToTrim == 0);
654-
655-
while (!trimToBudgetDone) {
656-
uint64_t lastFence = 0;
657-
wddmAllocation = osContext.get()->getResidencyController().getTrimCandidateHead();
658-
659-
if (wddmAllocation == nullptr) {
660-
break;
661-
}
662-
663-
lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(0);
664-
auto &monitoredFence = osContext.get()->getResidencyController().getMonitoredFence();
665-
666-
if (lastFence <= monitoredFence.lastSubmittedFence) {
667-
uint32_t fragmentsToEvict = 0;
668-
uint64_t sizeEvicted = 0;
669-
uint64_t sizeToTrim = 0;
670-
671-
if (lastFence > *monitoredFence.cpuAddress) {
672-
wddm->waitFromCpu(lastFence, *osContext.get());
673-
}
674-
675-
if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
676-
wddm->evict(&wddmAllocation->handle, 1, sizeToTrim);
677-
678-
sizeEvicted = wddmAllocation->getAlignedSize();
679-
} else {
680-
auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData;
681-
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
682-
if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(0) <= monitoredFence.lastSubmittedFence) {
683-
fragmentEvictHandles[fragmentsToEvict++] = fragmentStorageData[allocationId].osHandleStorage->handle;
684-
}
685-
}
686-
687-
if (fragmentsToEvict != 0) {
688-
wddm->evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim);
689-
690-
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
691-
if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(0) <= monitoredFence.lastSubmittedFence) {
692-
fragmentStorageData[allocationId].residency->resident = false;
693-
sizeEvicted += fragmentStorageData[allocationId].fragmentSize;
694-
}
695-
}
696-
}
697-
}
698-
699-
if (sizeEvicted >= numberOfBytesToTrim) {
700-
numberOfBytesToTrim = 0;
701-
} else {
702-
numberOfBytesToTrim -= sizeEvicted;
703-
}
704-
705-
wddmAllocation->getResidencyData().resident = false;
706-
osContext.get()->getResidencyController().removeFromTrimCandidateList(wddmAllocation, false);
707-
trimToBudgetDone = (numberOfBytesToTrim == 0);
708-
} else {
709-
trimToBudgetDone = true;
710-
}
711-
}
712-
713-
if (bytes > numberOfBytesToTrim && osContext.get()->getResidencyController().checkTrimCandidateListCompaction()) {
714-
osContext.get()->getResidencyController().compactTrimCandidateList();
715-
}
716-
717-
return numberOfBytesToTrim == 0;
718-
}
719-
720574
bool WddmMemoryManager::mapAuxGpuVA(GraphicsAllocation *graphicsAllocation) {
721575
return wddm->updateAuxTable(graphicsAllocation->getGpuAddress(), graphicsAllocation->gmm, true);
722576
}

‎runtime/os_interface/windows/wddm_memory_manager.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,6 @@ class WddmMemoryManager : public MemoryManager {
7979
AlignedMallocRestrictions *getAlignedMallocRestrictions() override;
8080

8181
protected:
82-
void trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes);
83-
bool trimResidencyToBudget(uint64_t bytes);
8482
VOID *trimCallbackHandle = nullptr;
8583

8684
GraphicsAllocation *createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle);

0 commit comments

Comments
 (0)