Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add file mapping for windows platform. #12183

Merged
merged 7 commits into from
Jul 18, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add unit test for file mapping for windows. Also add an error message…
… for mis-aligned offset
  • Loading branch information
Ting Cao committed Jul 15, 2022
commit e6e0da56672f3851e236faef8b974d6e541acb40
11 changes: 11 additions & 0 deletions onnxruntime/core/platform/windows/env.cc
Original file line number Diff line number Diff line change
Expand Up @@ -400,9 +400,20 @@ class WindowsEnv : public Env {
SYSTEM_INFO sysinfo;
GetSystemInfo(&sysinfo);
static const long page_size = sysinfo.dwPageSize;
static const long allocation_granularity = sysinfo.dwAllocationGranularity;
const FileOffsetType offset_to_page = offset % static_cast<FileOffsetType>(page_size);
const size_t mapped_length = length + offset_to_page;
const FileOffsetType mapped_offset = offset - offset_to_page;
if (mapped_offset % allocation_granularity != 0) {
const auto error_code = GetLastError();
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
"mapped offset must be a multiple of the allocation granularity",
" , mapped_offset = ", mapped_offset,
" , allocation_granularity = ", allocation_granularity,
" , errcode = ", error_code,
" - ", std::system_category().message(error_code));
}

void* const mapped_base = MapViewOfFile(file_mapping_handle.get(),
FILE_MAP_READ,
0,
Expand Down
59 changes: 59 additions & 0 deletions onnxruntime/test/platform/file_io_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@

#ifndef _WIN32
#include <unistd.h> // for sysconf() and _SC_PAGESIZE
#else
#include <Windows.h>
#endif

#include "gsl/gsl"
Expand Down Expand Up @@ -61,7 +63,11 @@ std::vector<char> GenerateData(size_t length, uint32_t seed = 0) {
}

void WriteDataToFile(gsl::span<const char> data, const PathString& path) {
#ifndef _WIN32
std::ofstream out{path, std::ios_base::out | std::ios_base::trunc};
#else
std::ofstream out{path, std::ios_base::out | std::ios_base::trunc | std::ios_base::binary};
#endif
out.write(data.data(), data.size());
}

Expand Down Expand Up @@ -144,6 +150,59 @@ TEST(FileIoTest, MapFileIntoMemory) {
ASSERT_FALSE(Env::Default().MapFileIntoMemory(tmp.path.c_str(), -1, 0, mapped_memory).IsOK());
}
}
#else
TEST(FileIoTest, MapFileIntoMemory) {
SYSTEM_INFO sysinfo;
GetSystemInfo(&sysinfo);
static const auto page_size = sysinfo.dwPageSize;
static const auto allocation_granularity = sysinfo.dwAllocationGranularity;
ASSERT_GT(page_size, 0);

TempFilePath tmp(ORT_TSTR("map_file_test_"));
const auto expected_data = GenerateData(page_size * 3 / 2);
WriteDataToFile(gsl::make_span(expected_data), tmp.path);

const auto offsets_and_lengths = GenerateValidOffsetLengthPairs(
0, expected_data.size(), page_size / 10);

for (const auto& offset_and_length : offsets_and_lengths) {
const auto offset = offset_and_length.first;
const auto length = offset_and_length.second;

// The offset must be a multiple of the allocation granularity
if (offset % allocation_granularity != 0) {
continue;
}

Env::MappedMemoryPtr mapped_memory{};
auto status = Env::Default().MapFileIntoMemory(
tmp.path.c_str(), offset, length, mapped_memory);
ASSERT_TRUE(status.IsOK())
<< "MapFileIntoMemory failed for offset " << offset << " and length " << length
<< " with error: " << status.ErrorMessage();

auto mapped_span = gsl::make_span(mapped_memory.get(), length);

auto expected_data_span = gsl::make_span(expected_data.data() + offset, length);

ASSERT_EQ(mapped_span, expected_data_span);
}

{
Env::MappedMemoryPtr mapped_memory{};

// invalid - offset is not a multiple of the allocation granularity
ASSERT_FALSE(Env::Default().MapFileIntoMemory(
tmp.path.c_str(), allocation_granularity * 3 / 2, page_size / 10, mapped_memory).IsOK());
}

{
Env::MappedMemoryPtr mapped_memory{};

// invalid - negative offset
ASSERT_FALSE(Env::Default().MapFileIntoMemory(tmp.path.c_str(), -1, 0, mapped_memory).IsOK());
}
}
#endif

} // namespace test
Expand Down