Skip to content

Commit

Permalink
Support read/write memory-mapped files.
Browse files Browse the repository at this point in the history 
See also: chromium-dev thread "Support for Writable Memory Mapped Files", specifically the last few messages starting with "And... we're back!".

BUG=546019

Review-Url: https://codereview.chromium.org/1798203002
Cr-Commit-Position: refs/heads/master@{#394868}
  • Loading branch information
bcwhite authored and Commit bot committed May 19, 2016
1 parent c13ff44 commit f0dfab0
Show file tree
Hide file tree
Showing 5 changed files with 213 additions and 50 deletions.
47 changes: 40 additions & 7 deletions base/files/memory_mapped_file.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,30 +30,63 @@ MemoryMappedFile::~MemoryMappedFile() {
}

#if !defined(OS_NACL)
bool MemoryMappedFile::Initialize(const FilePath& file_name) {
bool MemoryMappedFile::Initialize(const FilePath& file_name, Access access) {
if (IsValid())
return false;

file_.Initialize(file_name, File::FLAG_OPEN | File::FLAG_READ);
uint32_t flags = 0;
switch (access) {
case READ_ONLY:
flags = File::FLAG_OPEN | File::FLAG_READ;
break;
case READ_WRITE:
flags = File::FLAG_OPEN | File::FLAG_READ | File::FLAG_WRITE;
break;
case READ_WRITE_EXTEND:
// Can't open with "extend" because no maximum size is known.
NOTREACHED();
}
file_.Initialize(file_name, flags);

if (!file_.IsValid()) {
DLOG(ERROR) << "Couldn't open " << file_name.AsUTF8Unsafe();
return false;
}

if (!MapFileRegionToMemory(Region::kWholeFile)) {
if (!MapFileRegionToMemory(Region::kWholeFile, access)) {
CloseHandles();
return false;
}

return true;
}

bool MemoryMappedFile::Initialize(File file) {
return Initialize(std::move(file), Region::kWholeFile);
bool MemoryMappedFile::Initialize(File file, Access access) {
DCHECK_NE(READ_WRITE_EXTEND, access);
return Initialize(std::move(file), Region::kWholeFile, access);
}

bool MemoryMappedFile::Initialize(File file, const Region& region) {
bool MemoryMappedFile::Initialize(File file,
const Region& region,
Access access) {
switch (access) {
case READ_WRITE_EXTEND:
// Ensure that the extended size is within limits of File.
if (region.size > std::numeric_limits<int64_t>::max() - region.offset) {
DLOG(ERROR) << "Region bounds exceed maximum for base::File.";
return false;
}
// Fall through.
case READ_ONLY:
case READ_WRITE:
// Ensure that the region values are valid.
if (region.offset < 0 || region.size < 0) {
DLOG(ERROR) << "Region bounds are not valid.";
return false;
}
break;
}

if (IsValid())
return false;

Expand All @@ -64,7 +97,7 @@ bool MemoryMappedFile::Initialize(File file, const Region& region) {

file_ = std::move(file);

if (!MapFileRegionToMemory(region)) {
if (!MapFileRegionToMemory(region, access)) {
CloseHandles();
return false;
}
Expand Down
74 changes: 53 additions & 21 deletions base/files/memory_mapped_file.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,29 @@ class FilePath;

class BASE_EXPORT MemoryMappedFile {
public:
enum Access {
// Mapping a file into memory effectively allows for file I/O on any thread.
// The accessing thread could be paused while data from the file is paged
// into memory. Worse, a corrupted filesystem could cause a SEGV within the
// program instead of just an I/O error.
READ_ONLY,

// This provides read/write access to a file and must be used with care of
// the additional subtleties involved in doing so. Though the OS will do
// the writing of data on its own time, too many dirty pages can cause
// the OS to pause the thread while it writes them out. The pause can
// be as much as 1s on some systems.
READ_WRITE,

// This provides read/write access but with the ability to write beyond
// the end of the existing file up to a maximum size specified as the
// "region". Depending on the OS, the file may or may not be immediately
// extended to the maximum size though it won't be loaded in RAM until
// needed. Note, however, that the maximum size will still be reserved
// in the process address space.
READ_WRITE_EXTEND,
};

// The default constructor sets all members to invalid/null values.
MemoryMappedFile();
~MemoryMappedFile();
Expand All @@ -41,27 +64,37 @@ class BASE_EXPORT MemoryMappedFile {
int64_t size;
};

// Opens an existing file and maps it into memory. Access is restricted to
// read only. If this object already points to a valid memory mapped file
// then this method will fail and return false. If it cannot open the file,
// the file does not exist, or the memory mapping fails, it will return false.
// Later we may want to allow the user to specify access.
bool Initialize(const FilePath& file_name);

// As above, but works with an already-opened file. MemoryMappedFile takes
// ownership of |file| and closes it when done.
bool Initialize(File file);

// As above, but works with a region of an already-opened file.
bool Initialize(File file, const Region& region);

#if defined(OS_WIN)
// Opens an existing file and maps it as an image section. Please refer to
// the Initialize function above for additional information.
bool InitializeAsImageSection(const FilePath& file_name);
#endif // OS_WIN
// Opens an existing file and maps it into memory. |access| can be read-only
// or read/write but not read/write+extend. If this object already points
// to a valid memory mapped file then this method will fail and return
// false. If it cannot open the file, the file does not exist, or the
// memory mapping fails, it will return false.
bool Initialize(const FilePath& file_name, Access access);
bool Initialize(const FilePath& file_name) {
return Initialize(file_name, READ_ONLY);
}

// As above, but works with an already-opened file. |access| can be read-only
// or read/write but not read/write+extend. MemoryMappedFile takes ownership
// of |file| and closes it when done. |file| must have been opened with
// permissions suitable for |access|. If the memory mapping fails, it will
// return false.
bool Initialize(File file, Access access);
bool Initialize(File file) {
return Initialize(std::move(file), READ_ONLY);
}

// As above, but works with a region of an already-opened file. All forms of
// |access| are allowed. If READ_WRITE_EXTEND is specified then |region|
// provides the maximum size of the file. If the memory mapping fails, it
// return false.
bool Initialize(File file, const Region& region, Access access);
bool Initialize(File file, const Region& region) {
return Initialize(std::move(file), region, READ_ONLY);
}

const uint8_t* data() const { return data_; }
uint8_t* data() { return data_; }
size_t length() const { return length_; }

// Is file_ a valid file handle that points to an open, memory mapped file?
Expand All @@ -82,7 +115,7 @@ class BASE_EXPORT MemoryMappedFile {

// Map the file to memory, set data_ to that memory address. Return true on
// success, false on any kind of failure. This is a helper for Initialize().
bool MapFileRegionToMemory(const Region& region);
bool MapFileRegionToMemory(const Region& region, Access access);

// Closes all open handles.
void CloseHandles();
Expand All @@ -93,7 +126,6 @@ class BASE_EXPORT MemoryMappedFile {

#if defined(OS_WIN)
win::ScopedHandle file_mapping_;
bool image_; // Map as an image.
#endif

DISALLOW_COPY_AND_ASSIGN(MemoryMappedFile);
Expand Down
21 changes: 19 additions & 2 deletions base/files/memory_mapped_file_posix.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ MemoryMappedFile::MemoryMappedFile() : data_(NULL), length_(0) {

#if !defined(OS_NACL)
bool MemoryMappedFile::MapFileRegionToMemory(
const MemoryMappedFile::Region& region) {
const MemoryMappedFile::Region& region,
Access access) {
ThreadRestrictions::AssertIOAllowed();

off_t map_start = 0;
Expand Down Expand Up @@ -65,7 +66,23 @@ bool MemoryMappedFile::MapFileRegionToMemory(
length_ = static_cast<size_t>(region.size);
}

data_ = static_cast<uint8_t*>(mmap(NULL, map_size, PROT_READ, MAP_SHARED,
int flags = 0;
switch (access) {
case READ_ONLY:
flags |= PROT_READ;
break;
case READ_WRITE:
flags |= PROT_READ | PROT_WRITE;
break;
case READ_WRITE_EXTEND:
// POSIX won't auto-extend the file when it is written so it must first
// be explicitly extended to the maximum size. Zeros will fill the new
// space.
file_.SetLength(std::max(file_.GetLength(), region.offset + region.size));
flags |= PROT_READ | PROT_WRITE;
break;
}
data_ = static_cast<uint8_t*>(mmap(NULL, map_size, flags, MAP_SHARED,
file_.GetPlatformFile(), map_start));
if (data_ == MAP_FAILED) {
DPLOG(ERROR) << "mmap " << file_.GetPlatformFile();
Expand Down
84 changes: 76 additions & 8 deletions base/files/memory_mapped_file_unittest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ TEST_F(MemoryMappedFileTest, MapWholeFileByPath) {
MemoryMappedFile map;
map.Initialize(temp_file_path());
ASSERT_EQ(kFileSize, map.length());
ASSERT_TRUE(map.data() != NULL);
ASSERT_TRUE(map.data() != nullptr);
EXPECT_TRUE(map.IsValid());
ASSERT_TRUE(CheckBufferContents(map.data(), kFileSize, 0));
}
Expand All @@ -76,7 +76,7 @@ TEST_F(MemoryMappedFileTest, MapWholeFileByFD) {
MemoryMappedFile map;
map.Initialize(File(temp_file_path(), File::FLAG_OPEN | File::FLAG_READ));
ASSERT_EQ(kFileSize, map.length());
ASSERT_TRUE(map.data() != NULL);
ASSERT_TRUE(map.data() != nullptr);
EXPECT_TRUE(map.IsValid());
ASSERT_TRUE(CheckBufferContents(map.data(), kFileSize, 0));
}
Expand All @@ -87,7 +87,7 @@ TEST_F(MemoryMappedFileTest, MapSmallFile) {
MemoryMappedFile map;
map.Initialize(temp_file_path());
ASSERT_EQ(kFileSize, map.length());
ASSERT_TRUE(map.data() != NULL);
ASSERT_TRUE(map.data() != nullptr);
EXPECT_TRUE(map.IsValid());
ASSERT_TRUE(CheckBufferContents(map.data(), kFileSize, 0));
}
Expand All @@ -100,7 +100,7 @@ TEST_F(MemoryMappedFileTest, MapWholeFileUsingRegion) {
File file(temp_file_path(), File::FLAG_OPEN | File::FLAG_READ);
map.Initialize(std::move(file), MemoryMappedFile::Region::kWholeFile);
ASSERT_EQ(kFileSize, map.length());
ASSERT_TRUE(map.data() != NULL);
ASSERT_TRUE(map.data() != nullptr);
EXPECT_TRUE(map.IsValid());
ASSERT_TRUE(CheckBufferContents(map.data(), kFileSize, 0));
}
Expand All @@ -115,7 +115,7 @@ TEST_F(MemoryMappedFileTest, MapPartialRegionAtBeginning) {
MemoryMappedFile::Region region = {0, kPartialSize};
map.Initialize(std::move(file), region);
ASSERT_EQ(kPartialSize, map.length());
ASSERT_TRUE(map.data() != NULL);
ASSERT_TRUE(map.data() != nullptr);
EXPECT_TRUE(map.IsValid());
ASSERT_TRUE(CheckBufferContents(map.data(), kPartialSize, 0));
}
Expand All @@ -131,7 +131,7 @@ TEST_F(MemoryMappedFileTest, MapPartialRegionAtEnd) {
MemoryMappedFile::Region region = {kOffset, kPartialSize};
map.Initialize(std::move(file), region);
ASSERT_EQ(kPartialSize, map.length());
ASSERT_TRUE(map.data() != NULL);
ASSERT_TRUE(map.data() != nullptr);
EXPECT_TRUE(map.IsValid());
ASSERT_TRUE(CheckBufferContents(map.data(), kPartialSize, kOffset));
}
Expand All @@ -148,7 +148,7 @@ TEST_F(MemoryMappedFileTest, MapSmallPartialRegionInTheMiddle) {
MemoryMappedFile::Region region = {kOffset, kPartialSize};
map.Initialize(std::move(file), region);
ASSERT_EQ(kPartialSize, map.length());
ASSERT_TRUE(map.data() != NULL);
ASSERT_TRUE(map.data() != nullptr);
EXPECT_TRUE(map.IsValid());
ASSERT_TRUE(CheckBufferContents(map.data(), kPartialSize, kOffset));
}
Expand All @@ -165,11 +165,79 @@ TEST_F(MemoryMappedFileTest, MapLargePartialRegionInTheMiddle) {
MemoryMappedFile::Region region = {kOffset, kPartialSize};
map.Initialize(std::move(file), region);
ASSERT_EQ(kPartialSize, map.length());
ASSERT_TRUE(map.data() != NULL);
ASSERT_TRUE(map.data() != nullptr);
EXPECT_TRUE(map.IsValid());
ASSERT_TRUE(CheckBufferContents(map.data(), kPartialSize, kOffset));
}

TEST_F(MemoryMappedFileTest, WriteableFile) {
const size_t kFileSize = 127;
CreateTemporaryTestFile(kFileSize);

{
MemoryMappedFile map;
map.Initialize(temp_file_path(), MemoryMappedFile::READ_WRITE);
ASSERT_EQ(kFileSize, map.length());
ASSERT_TRUE(map.data() != nullptr);
EXPECT_TRUE(map.IsValid());
ASSERT_TRUE(CheckBufferContents(map.data(), kFileSize, 0));

uint8_t* bytes = map.data();
bytes[0] = 'B';
bytes[1] = 'a';
bytes[2] = 'r';
bytes[kFileSize - 1] = '!';
EXPECT_FALSE(CheckBufferContents(map.data(), kFileSize, 0));
EXPECT_TRUE(CheckBufferContents(map.data() + 3, kFileSize - 4, 3));
}

int64_t file_size;
ASSERT_TRUE(GetFileSize(temp_file_path(), &file_size));
EXPECT_EQ(static_cast<int64_t>(kFileSize), file_size);

std::string contents;
ASSERT_TRUE(ReadFileToString(temp_file_path(), &contents));
EXPECT_EQ("Bar", contents.substr(0, 3));
EXPECT_EQ("!", contents.substr(kFileSize - 1, 1));
}

TEST_F(MemoryMappedFileTest, ExtendableFile) {
const size_t kFileSize = 127;
const size_t kFileExtend = 100;
CreateTemporaryTestFile(kFileSize);

{
File file(temp_file_path(),
File::FLAG_OPEN | File::FLAG_READ | File::FLAG_WRITE);
MemoryMappedFile::Region region = {0, kFileSize + kFileExtend};
MemoryMappedFile map;
map.Initialize(std::move(file), region,
MemoryMappedFile::READ_WRITE_EXTEND);
EXPECT_EQ(kFileSize + kFileExtend, map.length());
ASSERT_TRUE(map.data() != nullptr);
EXPECT_TRUE(map.IsValid());
ASSERT_TRUE(CheckBufferContents(map.data(), kFileSize, 0));

uint8_t* bytes = map.data();
EXPECT_EQ(0, bytes[kFileSize + 0]);
EXPECT_EQ(0, bytes[kFileSize + 1]);
EXPECT_EQ(0, bytes[kFileSize + 2]);
bytes[kFileSize + 0] = 'B';
bytes[kFileSize + 1] = 'A';
bytes[kFileSize + 2] = 'Z';
EXPECT_TRUE(CheckBufferContents(map.data(), kFileSize, 0));
}

int64_t file_size;
ASSERT_TRUE(GetFileSize(temp_file_path(), &file_size));
EXPECT_LE(static_cast<int64_t>(kFileSize + 3), file_size);
EXPECT_GE(static_cast<int64_t>(kFileSize + kFileExtend), file_size);

std::string contents;
ASSERT_TRUE(ReadFileToString(temp_file_path(), &contents));
EXPECT_EQ("BAZ", contents.substr(kFileSize, 3));
}

} // namespace

} // namespace base
Loading

0 comments on commit f0dfab0

Please sign in to comment.