Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Vulkan] Region allocator fixes for memory requirements and allocations #8087

Merged
merged 7 commits into from
Feb 14, 2024
Merged

[Vulkan] Region allocator fixes for memory requirements and allocations #8087

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c651e28
Add region allocator tests that check alignment, nearest_multiple and
Feb 9, 2024
dcc9f63
Fix can_split() routine to use conformed sizes so that split allocation
Feb 9, 2024
d6de5b3
Fix aligned_offset() routine to check for zero alignment (which means no
Feb 9, 2024
be2cea5
Fix ifdef for internal debugging
Feb 9, 2024
41a8934
Clean up debug internal log messages
Feb 9, 2024
221fb80
Use memory_requirements to determine nearest_multiple during
Feb 9, 2024
5919b14
Formatting pass
Feb 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 63 additions & 63 deletions src/runtime/internal/block_allocator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ BlockAllocator *BlockAllocator::create(void *user_context, const Config &cfg, co
allocators.system.allocate(user_context, sizeof(BlockAllocator)));

if (result == nullptr) {
error(user_context) << "BlockAllocator: Failed to create instance! Out of memory!\n";
error(user_context) << "BlockAllocator: Failed to create instance! Out of memory\n";
return nullptr;
}

Expand Down Expand Up @@ -160,12 +160,12 @@ MemoryRegion *BlockAllocator::reserve(void *user_context, const MemoryRequest &r
<< "dedicated=" << (request.dedicated ? "true" : "false") << " "
<< "usage=" << halide_memory_usage_name(request.properties.usage) << " "
<< "caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(request.properties.visibility) << ") ...\n";
<< "visibility=" << halide_memory_visibility_name(request.properties.visibility) << ") ...";
#endif
BlockEntry *block_entry = reserve_block_entry(user_context, request.properties, request.size, request.dedicated);
if (block_entry == nullptr) {
error(user_context) << "BlockAllocator: Failed to allocate new empty block of requested size ("
<< (int32_t)(request.size) << " bytes)!\n";
<< (int32_t)(request.size) << " bytes)\n";
return nullptr;
}

Expand All @@ -180,7 +180,7 @@ MemoryRegion *BlockAllocator::reserve(void *user_context, const MemoryRequest &r
block_entry = create_block_entry(user_context, request.properties, request.size, request.dedicated);
if (block_entry == nullptr) {
error(user_context) << "BlockAllocator: Out of memory! Failed to allocate empty block of size ("
<< (int32_t)(request.size) << " bytes)!\n";
<< (int32_t)(request.size) << " bytes)\n";
return nullptr;
}

Expand Down Expand Up @@ -288,7 +288,7 @@ MemoryRegion *BlockAllocator::reserve_memory_region(void *user_context, RegionAl
if (result == nullptr) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: Failed to allocate region of size ("
<< (int32_t)(request.size) << " bytes)!\n";
<< (int32_t)(request.size) << " bytes)\n";
#endif
// allocator has enough free space, but not enough contiguous space
// -- collect and try to reallocate
Expand All @@ -302,38 +302,38 @@ MemoryRegion *BlockAllocator::reserve_memory_region(void *user_context, RegionAl
bool BlockAllocator::is_block_suitable_for_request(void *user_context, const BlockResource *block, const MemoryProperties &properties, size_t size, bool dedicated) const {
if (!is_compatible_block(block, properties)) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: skipping block ... incompatible properties!\n"
<< " block_resource=" << (void *)block << "\n"
<< " block_size=" << (uint32_t)block->memory.size << "\n"
<< " block_reserved=" << (uint32_t)block->reserved << "\n"
<< " block_usage=" << halide_memory_usage_name(block->memory.properties.usage) << "\n"
<< " block_caching=" << halide_memory_caching_name(block->memory.properties.caching) << "\n"
<< " block_visibility=" << halide_memory_visibility_name(block->memory.properties.visibility) << "\n";
debug(user_context) << " request_size=" << (uint32_t)size << "\n"
<< " request_usage=" << halide_memory_usage_name(properties.usage) << "\n"
<< " request_caching=" << halide_memory_caching_name(properties.caching) << "\n"
<< " request_visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
debug(user_context) << "BlockAllocator: skipping block ... incompatible properties! ("
<< "block_resource=" << (void *)block << " "
<< "block_size=" << (uint32_t)block->memory.size << " "
<< "block_reserved=" << (uint32_t)block->reserved << " "
<< "block_usage=" << halide_memory_usage_name(block->memory.properties.usage) << " "
<< "block_caching=" << halide_memory_caching_name(block->memory.properties.caching) << " "
<< "block_visibility=" << halide_memory_visibility_name(block->memory.properties.visibility) << " "
<< "request_size=" << (uint32_t)size << " "
<< "request_usage=" << halide_memory_usage_name(properties.usage) << " "
<< "request_caching=" << halide_memory_caching_name(properties.caching) << " "
<< "request_visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
#endif
// skip blocks that are using incompatible memory
return false;
}

if (dedicated && (block->reserved > 0)) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: skipping block ... can be used for dedicated allocation!\n"
<< " block_resource=" << (void *)block << "\n"
<< " block_size=" << (uint32_t)block->memory.size << "\n"
<< " block_reserved=" << (uint32_t)block->reserved << "\n";
debug(user_context) << "BlockAllocator: skipping block ... can be used for dedicated allocation! ("
<< "block_resource=" << (void *)block << " "
<< "block_size=" << (uint32_t)block->memory.size << " "
<< "block_reserved=" << (uint32_t)block->reserved << ")";
#endif
// skip blocks that can't be dedicated to a single allocation
return false;

} else if (block->memory.dedicated && (block->reserved > 0)) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: skipping block ... already dedicated to an allocation!\n"
<< " block_resource=" << (void *)block << "\n"
<< " block_size=" << (uint32_t)block->memory.size << "\n"
<< " block_reserved=" << (uint32_t)block->reserved << "\n";
debug(user_context) << "BlockAllocator: skipping block ... already dedicated to an allocation! ("
<< "block_resource=" << (void *)block << " "
<< "block_size=" << (uint32_t)block->memory.size << " "
<< "block_reserved=" << (uint32_t)block->reserved << ")";
#endif
// skip dedicated blocks that are already allocated
return false;
Expand All @@ -355,16 +355,16 @@ BlockAllocator::find_block_entry(void *user_context, const MemoryProperties &pro
const BlockResource *block = static_cast<BlockResource *>(block_entry->value);
if (is_block_suitable_for_request(user_context, block, properties, size, dedicated)) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: found suitable block ...\n"
<< " user_context=" << (void *)(user_context) << "\n"
<< " block_resource=" << (void *)block << "\n"
<< " block_size=" << (uint32_t)block->memory.size << "\n"
<< " block_reserved=" << (uint32_t)block->reserved << "\n"
<< " request_size=" << (uint32_t)size << "\n"
<< " dedicated=" << (dedicated ? "true" : "false") << "\n"
<< " usage=" << halide_memory_usage_name(properties.usage) << "\n"
<< " caching=" << halide_memory_caching_name(properties.caching) << "\n"
<< " visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
debug(user_context) << "BlockAllocator: found suitable block ("
<< "user_context=" << (void *)(user_context) << " "
<< "block_resource=" << (void *)block << " "
<< "block_size=" << (uint32_t)block->memory.size << " "
<< "block_reserved=" << (uint32_t)block->reserved << " "
<< "request_size=" << (uint32_t)size << " "
<< "dedicated=" << (dedicated ? "true" : "false") << " "
<< "usage=" << halide_memory_usage_name(properties.usage) << " "
<< "caching=" << halide_memory_caching_name(properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
#endif
return block_entry;
}
Expand All @@ -373,13 +373,13 @@ BlockAllocator::find_block_entry(void *user_context, const MemoryProperties &pro

if (block_entry == nullptr) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: couldn't find suitable block!\n"
<< " user_context=" << (void *)(user_context) << "\n"
<< " request_size=" << (uint32_t)size << "\n"
<< " dedicated=" << (dedicated ? "true" : "false") << "\n"
<< " usage=" << halide_memory_usage_name(properties.usage) << "\n"
<< " caching=" << halide_memory_caching_name(properties.caching) << "\n"
<< " visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
debug(user_context) << "BlockAllocator: couldn't find suitable block! ("
<< "user_context=" << (void *)(user_context) << " "
<< "request_size=" << (uint32_t)size << " "
<< "dedicated=" << (dedicated ? "true" : "false") << " "
<< "usage=" << halide_memory_usage_name(properties.usage) << " "
<< "caching=" << halide_memory_caching_name(properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
#endif
}
return block_entry;
Expand All @@ -388,22 +388,22 @@ BlockAllocator::find_block_entry(void *user_context, const MemoryProperties &pro
BlockAllocator::BlockEntry *
BlockAllocator::reserve_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: reserving block ... !\n"
<< " requested_size=" << (uint32_t)size << "\n"
<< " requested_is_dedicated=" << (dedicated ? "true" : "false") << "\n"
<< " requested_usage=" << halide_memory_usage_name(properties.usage) << "\n"
<< " requested_caching=" << halide_memory_caching_name(properties.caching) << "\n"
<< " requested_visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
debug(user_context) << "BlockAllocator: reserving block ... ! ("
<< "requested_size=" << (uint32_t)size << " "
<< "requested_is_dedicated=" << (dedicated ? "true" : "false") << " "
<< "requested_usage=" << halide_memory_usage_name(properties.usage) << " "
<< "requested_caching=" << halide_memory_caching_name(properties.caching) << " "
<< "requested_visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
#endif
BlockEntry *block_entry = find_block_entry(user_context, properties, size, dedicated);
if (block_entry == nullptr) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: creating block ... !\n"
<< " requested_size=" << (uint32_t)size << "\n"
<< " requested_is_dedicated=" << (dedicated ? "true" : "false") << "\n"
<< " requested_usage=" << halide_memory_usage_name(properties.usage) << "\n"
<< " requested_caching=" << halide_memory_caching_name(properties.caching) << "\n"
<< " requested_visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
debug(user_context) << "BlockAllocator: creating block ... ! ("
<< "requested_size=" << (uint32_t)size << " "
<< "requested_is_dedicated=" << (dedicated ? "true" : "false") << " "
<< "requested_usage=" << halide_memory_usage_name(properties.usage) << " "
<< "requested_caching=" << halide_memory_caching_name(properties.caching) << " "
<< "requested_visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
#endif
block_entry = create_block_entry(user_context, properties, size, dedicated);
}
Expand All @@ -422,14 +422,14 @@ BlockAllocator::create_region_allocator(void *user_context, BlockResource *block
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: Creating region allocator ("
<< "user_context=" << (void *)(user_context) << " "
<< "block_resource=" << (void *)(block) << ")...\n";
<< "block_resource=" << (void *)(block) << ")...";
#endif
halide_abort_if_false(user_context, block != nullptr);
RegionAllocator *region_allocator = RegionAllocator::create(
user_context, block, {allocators.system, allocators.region});

if (region_allocator == nullptr) {
error(user_context) << "BlockAllocator: Failed to create new region allocator!\n";
error(user_context) << "BlockAllocator: Failed to create new region allocator\n";
return nullptr;
}

Expand All @@ -440,7 +440,7 @@ int BlockAllocator::destroy_region_allocator(void *user_context, RegionAllocator
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: Destroying region allocator ("
<< "user_context=" << (void *)(user_context) << " "
<< "region_allocator=" << (void *)(region_allocator) << ")...\n";
<< "region_allocator=" << (void *)(region_allocator) << ")...";
#endif
if (region_allocator == nullptr) {
return 0;
Expand All @@ -459,21 +459,21 @@ BlockAllocator::create_block_entry(void *user_context, const MemoryProperties &p

if (config.maximum_block_count && (block_count() >= config.maximum_block_count)) {
error(user_context) << "BlockAllocator: No free blocks found! Maximum block count reached ("
<< (int32_t)(config.maximum_block_count) << ")!\n";
<< (int32_t)(config.maximum_block_count) << ")\n";
return nullptr;
}

BlockEntry *block_entry = block_list.append(user_context);
if (block_entry == nullptr) {
debug(user_context) << "BlockAllocator: Failed to allocate new block entry!\n";
debug(user_context) << "BlockAllocator: Failed to allocate new block entry\n";
return nullptr;
}

#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: Creating block entry ("
<< "block_entry=" << (void *)(block_entry) << " "
<< "block=" << (void *)(block_entry->value) << " "
<< "allocator=" << (void *)(allocators.block.allocate) << ")...\n";
<< "allocator=" << (void *)(allocators.block.allocate) << ")...";
#endif

BlockResource *block = static_cast<BlockResource *>(block_entry->value);
Expand All @@ -492,7 +492,7 @@ int BlockAllocator::release_block_entry(void *user_context, BlockAllocator::Bloc
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: Releasing block entry ("
<< "block_entry=" << (void *)(block_entry) << " "
<< "block=" << (void *)(block_entry->value) << ")...\n";
<< "block=" << (void *)(block_entry->value) << ")...";
#endif
BlockResource *block = static_cast<BlockResource *>(block_entry->value);
if (block->allocator) {
Expand All @@ -506,7 +506,7 @@ int BlockAllocator::destroy_block_entry(void *user_context, BlockAllocator::Bloc
debug(user_context) << "BlockAllocator: Destroying block entry ("
<< "block_entry=" << (void *)(block_entry) << " "
<< "block=" << (void *)(block_entry->value) << " "
<< "deallocator=" << (void *)(allocators.block.deallocate) << ")...\n";
<< "deallocator=" << (void *)(allocators.block.deallocate) << ")...";
#endif
BlockResource *block = static_cast<BlockResource *>(block_entry->value);
if (block->allocator) {
Expand All @@ -520,7 +520,7 @@ int BlockAllocator::destroy_block_entry(void *user_context, BlockAllocator::Bloc

int BlockAllocator::alloc_memory_block(void *user_context, BlockResource *block) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: Allocating block (ptr=" << (void *)block << " allocator=" << (void *)allocators.block.allocate << ")...\n";
debug(user_context) << "BlockAllocator: Allocating block (ptr=" << (void *)block << " allocator=" << (void *)allocators.block.allocate << ")...";
#endif
halide_abort_if_false(user_context, allocators.block.allocate != nullptr);
MemoryBlock *memory_block = &(block->memory);
Expand All @@ -531,7 +531,7 @@ int BlockAllocator::alloc_memory_block(void *user_context, BlockResource *block)

int BlockAllocator::free_memory_block(void *user_context, BlockResource *block) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: Deallocating block (ptr=" << (void *)block << " allocator=" << (void *)allocators.block.deallocate << ")...\n";
debug(user_context) << "BlockAllocator: Deallocating block (ptr=" << (void *)block << " allocator=" << (void *)allocators.block.deallocate << ")...";
#endif
halide_abort_if_false(user_context, allocators.block.deallocate != nullptr);
MemoryBlock *memory_block = &(block->memory);
Expand Down
2 changes: 1 addition & 1 deletion src/runtime/internal/memory_arena.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ void *MemoryArena::create_entry(void *user_context, Block *block, uint32_t index
void *entry_ptr = lookup_entry(user_context, block, index);
block->free_index = block->indices[index];
block->status[index] = AllocationStatus::InUse;
#if DEBUG_RUNTIME_INTERNAL
#ifdef DEBUG_RUNTIME_INTERNAL
memset(entry_ptr, 0, config.entry_size);
#endif
return entry_ptr;
Expand Down
2 changes: 1 addition & 1 deletion src/runtime/internal/memory_resources.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ ALWAYS_INLINE bool is_power_of_two_alignment(size_t x) {
// -- Alignment must be power of two!
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment) {
halide_abort_if_false(nullptr, is_power_of_two_alignment(alignment));
return (offset + (alignment - 1)) & ~(alignment - 1);
return (alignment == 0) ? (offset) : (offset + (alignment - 1)) & ~(alignment - 1);
}

// Returns a suitable alignment such that requested alignment is a suitable
Expand Down
Loading
Loading