diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index 841b68a9fb2b7..0fc8c29a5b769 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -158,6 +158,8 @@ ArchiveBuilder::ArchiveBuilder() : _rw_region("rw", MAX_SHARED_DELTA), _ro_region("ro", MAX_SHARED_DELTA), _ptrmap(mtClassShared), + _rw_ptrmap(mtClassShared), + _ro_ptrmap(mtClassShared), _rw_src_objs(), _ro_src_objs(), _src_obj_table(INITIAL_TABLE_SIZE, MAX_TABLE_SIZE), @@ -1275,8 +1277,11 @@ void ArchiveBuilder::write_archive(FileMapInfo* mapinfo, ArchiveHeapInfo* heap_i write_region(mapinfo, MetaspaceShared::rw, &_rw_region, /*read_only=*/false,/*allow_exec=*/false); write_region(mapinfo, MetaspaceShared::ro, &_ro_region, /*read_only=*/true, /*allow_exec=*/false); + // Split pointer map into read-write and read-only bitmaps + ArchivePtrMarker::initialize_rw_ro_maps(&_rw_ptrmap, &_ro_ptrmap); + size_t bitmap_size_in_bytes; - char* bitmap = mapinfo->write_bitmap_region(ArchivePtrMarker::ptrmap(), heap_info, + char* bitmap = mapinfo->write_bitmap_region(ArchivePtrMarker::rw_ptrmap(), ArchivePtrMarker::ro_ptrmap(), heap_info, bitmap_size_in_bytes); if (heap_info->is_used()) { diff --git a/src/hotspot/share/cds/archiveBuilder.hpp b/src/hotspot/share/cds/archiveBuilder.hpp index 4f811a0c51265..a80370d39761e 100644 --- a/src/hotspot/share/cds/archiveBuilder.hpp +++ b/src/hotspot/share/cds/archiveBuilder.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -204,7 +204,14 @@ class ArchiveBuilder : public StackObj { DumpRegion _rw_region; DumpRegion _ro_region; - CHeapBitMap _ptrmap; // bitmap used by ArchivePtrMarker + + // Combined bitmap to track pointers in both RW and RO regions. This is updated + // as objects are copied into RW and RO. + CHeapBitMap _ptrmap; + + // _ptrmap is split into these two bitmaps which are written into the archive. + CHeapBitMap _rw_ptrmap; // marks pointers in the RW region + CHeapBitMap _ro_ptrmap; // marks pointers in the RO region SourceObjList _rw_src_objs; // objs to put in rw region SourceObjList _ro_src_objs; // objs to put in ro region diff --git a/src/hotspot/share/cds/archiveHeapLoader.cpp b/src/hotspot/share/cds/archiveHeapLoader.cpp index fe30be1642796..2ef502a3643d3 100644 --- a/src/hotspot/share/cds/archiveHeapLoader.cpp +++ b/src/hotspot/share/cds/archiveHeapLoader.cpp @@ -442,7 +442,7 @@ void ArchiveHeapLoader::patch_native_pointers() { FileMapRegion* r = FileMapInfo::current_info()->region_at(MetaspaceShared::hp); if (r->mapped_base() != nullptr && r->has_ptrmap()) { log_info(cds, heap)("Patching native pointers in heap region"); - BitMapView bm = r->ptrmap_view(); + BitMapView bm = FileMapInfo::current_info()->ptrmap_view(MetaspaceShared::hp); PatchNativePointers patcher((Metadata**)r->mapped_base() + FileMapInfo::current_info()->heap_ptrmap_start_pos()); bm.iterate(&patcher); } diff --git a/src/hotspot/share/cds/archiveUtils.cpp b/src/hotspot/share/cds/archiveUtils.cpp index b14dfc8c33e6a..5ba36960c55b1 100644 --- a/src/hotspot/share/cds/archiveUtils.cpp +++ b/src/hotspot/share/cds/archiveUtils.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -46,12 +46,16 @@ #include "utilities/globalDefinitions.hpp" CHeapBitMap* ArchivePtrMarker::_ptrmap = nullptr; +CHeapBitMap* ArchivePtrMarker::_rw_ptrmap = nullptr; +CHeapBitMap* ArchivePtrMarker::_ro_ptrmap = nullptr; VirtualSpace* ArchivePtrMarker::_vs; bool ArchivePtrMarker::_compacted; void ArchivePtrMarker::initialize(CHeapBitMap* ptrmap, VirtualSpace* vs) { assert(_ptrmap == nullptr, "initialize only once"); + assert(_rw_ptrmap == nullptr, "initialize only once"); + assert(_ro_ptrmap == nullptr, "initialize only once"); _vs = vs; _compacted = false; _ptrmap = ptrmap; @@ -67,6 +71,37 @@ void ArchivePtrMarker::initialize(CHeapBitMap* ptrmap, VirtualSpace* vs) { _ptrmap->initialize(estimated_archive_size / sizeof(intptr_t)); } +void ArchivePtrMarker::initialize_rw_ro_maps(CHeapBitMap* rw_ptrmap, CHeapBitMap* ro_ptrmap) { + address* rw_bottom = (address*)ArchiveBuilder::current()->rw_region()->base(); + address* ro_bottom = (address*)ArchiveBuilder::current()->ro_region()->base(); + + _rw_ptrmap = rw_ptrmap; + _ro_ptrmap = ro_ptrmap; + + size_t rw_size = ArchiveBuilder::current()->rw_region()->used() / sizeof(address); + size_t ro_size = ArchiveBuilder::current()->ro_region()->used() / sizeof(address); + // ro_start is the first bit in _ptrmap that covers the pointer that would sit at ro_bottom. + // E.g., if rw_bottom = (address*)100 + // ro_bottom = (address*)116 + // then for 64-bit platform: + // ro_start = ro_bottom - rw_bottom = (116 - 100) / sizeof(address) = 2; + size_t ro_start = ro_bottom - rw_bottom; + + // Note: ptrmap is big enough only to cover the last pointer in ro_region. + // See ArchivePtrMarker::compact() + _rw_ptrmap->initialize(rw_size); + _ro_ptrmap->initialize(_ptrmap->size() - ro_start); + + for (size_t rw_bit = 0; rw_bit < _rw_ptrmap->size(); rw_bit++) { + _rw_ptrmap->at_put(rw_bit, _ptrmap->at(rw_bit)); + } + + for(size_t ro_bit = ro_start; ro_bit < _ptrmap->size(); ro_bit++) { + _ro_ptrmap->at_put(ro_bit-ro_start, _ptrmap->at(ro_bit)); + } + assert(_ptrmap->size() - ro_start == _ro_ptrmap->size(), "must be"); +} + void ArchivePtrMarker::mark_pointer(address* ptr_loc) { assert(_ptrmap != nullptr, "not initialized"); assert(!_compacted, "cannot mark anymore"); diff --git a/src/hotspot/share/cds/archiveUtils.hpp b/src/hotspot/share/cds/archiveUtils.hpp index 2c965f8fe9cf1..efe5a468b93b4 100644 --- a/src/hotspot/share/cds/archiveUtils.hpp +++ b/src/hotspot/share/cds/archiveUtils.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -42,6 +42,8 @@ class VirtualSpace; // fixed, but _ptr_end can be expanded as more objects are dumped. class ArchivePtrMarker : AllStatic { static CHeapBitMap* _ptrmap; + static CHeapBitMap* _rw_ptrmap; + static CHeapBitMap* _ro_ptrmap; static VirtualSpace* _vs; // Once _ptrmap is compacted, we don't allow bit marking anymore. This is to @@ -53,6 +55,7 @@ class ArchivePtrMarker : AllStatic { public: static void initialize(CHeapBitMap* ptrmap, VirtualSpace* vs); + static void initialize_rw_ro_maps(CHeapBitMap* rw_ptrmap, CHeapBitMap* ro_ptrmap); static void mark_pointer(address* ptr_loc); static void clear_pointer(address* ptr_loc); static void compact(address relocatable_base, address relocatable_end); @@ -73,8 +76,18 @@ class ArchivePtrMarker : AllStatic { return _ptrmap; } + static CHeapBitMap* rw_ptrmap() { + return _rw_ptrmap; + } + + static CHeapBitMap* ro_ptrmap() { + return _ro_ptrmap; + } + static void reset_map_and_vs() { _ptrmap = nullptr; + _rw_ptrmap = nullptr; + _ro_ptrmap = nullptr; _vs = nullptr; } }; diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp index 2b35e64c26480..c81838ed2eff8 100644 --- a/src/hotspot/share/cds/filemap.cpp +++ b/src/hotspot/share/cds/filemap.cpp @@ -294,7 +294,6 @@ void FileMapHeader::print(outputStream* st) { st->print_cr("- allow_archiving_with_java_agent:%d", _allow_archiving_with_java_agent); st->print_cr("- use_optimized_module_handling: %d", _use_optimized_module_handling); st->print_cr("- has_full_module_graph %d", _has_full_module_graph); - st->print_cr("- ptrmap_size_in_bits: " SIZE_FORMAT, _ptrmap_size_in_bits); } void SharedClassPathEntry::init_as_non_existent(const char* path, TRAPS) { @@ -1453,22 +1452,6 @@ void FileMapRegion::init_ptrmap(size_t offset, size_t size_in_bits) { _ptrmap_size_in_bits = size_in_bits; } -BitMapView FileMapRegion::bitmap_view(bool is_oopmap) { - char* bitmap_base = FileMapInfo::current_info()->map_bitmap_region(); - bitmap_base += is_oopmap ? _oopmap_offset : _ptrmap_offset; - size_t size_in_bits = is_oopmap ? _oopmap_size_in_bits : _ptrmap_size_in_bits; - return BitMapView((BitMap::bm_word_t*)(bitmap_base), size_in_bits); -} - -BitMapView FileMapRegion::oopmap_view() { - return bitmap_view(true); -} - -BitMapView FileMapRegion::ptrmap_view() { - assert(has_ptrmap(), "must be"); - return bitmap_view(false); -} - bool FileMapRegion::check_region_crc(char* base) const { // This function should be called after the region has been properly // loaded into memory via FileMapInfo::map_region() or FileMapInfo::read_region(). @@ -1497,6 +1480,27 @@ static const char* region_name(int region_index) { return names[region_index]; } +BitMapView FileMapInfo::bitmap_view(int region_index, bool is_oopmap) { + FileMapRegion* r = region_at(region_index); + char* bitmap_base = is_static() ? FileMapInfo::current_info()->map_bitmap_region() : FileMapInfo::dynamic_info()->map_bitmap_region(); + bitmap_base += is_oopmap ? r->oopmap_offset() : r->ptrmap_offset(); + size_t size_in_bits = is_oopmap ? r->oopmap_size_in_bits() : r->ptrmap_size_in_bits(); + + log_debug(cds, reloc)("mapped %s relocation %smap @ " INTPTR_FORMAT " (" SIZE_FORMAT " bits)", + region_name(region_index), is_oopmap ? "oop" : "ptr", + p2i(bitmap_base), size_in_bits); + + return BitMapView((BitMap::bm_word_t*)(bitmap_base), size_in_bits); +} + +BitMapView FileMapInfo::oopmap_view(int region_index) { + return bitmap_view(region_index, /*is_oopmap*/true); + } + +BitMapView FileMapInfo::ptrmap_view(int region_index) { + return bitmap_view(region_index, /*is_oopmap*/false); +} + void FileMapRegion::print(outputStream* st, int region_index) { st->print_cr("============ region ============= %d \"%s\"", region_index, region_name(region_index)); st->print_cr("- crc: 0x%08x", _crc); @@ -1510,6 +1514,8 @@ void FileMapRegion::print(outputStream* st, int region_index) { st->print_cr("- used: " SIZE_FORMAT, _used); st->print_cr("- oopmap_offset: " SIZE_FORMAT_X, _oopmap_offset); st->print_cr("- oopmap_size_in_bits: " SIZE_FORMAT, _oopmap_size_in_bits); + st->print_cr("- ptrmap_offset: " SIZE_FORMAT_X, _ptrmap_offset); + st->print_cr("- ptrmap_size_in_bits: " SIZE_FORMAT, _ptrmap_size_in_bits); st->print_cr("- mapped_base: " INTPTR_FORMAT, p2i(_mapped_base)); } @@ -1586,9 +1592,9 @@ size_t FileMapInfo::remove_bitmap_leading_zeros(CHeapBitMap* map) { return old_zeros; } -char* FileMapInfo::write_bitmap_region(const CHeapBitMap* ptrmap, ArchiveHeapInfo* heap_info, +char* FileMapInfo::write_bitmap_region(const CHeapBitMap* rw_ptrmap, const CHeapBitMap* ro_ptrmap, ArchiveHeapInfo* heap_info, size_t &size_in_bytes) { - size_in_bytes = ptrmap->size_in_bytes(); + size_in_bytes = rw_ptrmap->size_in_bytes() + ro_ptrmap->size_in_bytes(); if (heap_info->is_used()) { // Remove leading zeros @@ -1602,14 +1608,19 @@ char* FileMapInfo::write_bitmap_region(const CHeapBitMap* ptrmap, ArchiveHeapInf size_in_bytes += heap_info->ptrmap()->size_in_bytes(); } - // The bitmap region contains up to 3 parts: - // ptrmap: metaspace pointers inside the ro/rw regions + // The bitmap region contains up to 4 parts: + // rw_ptrmap: metaspace pointers inside the read-write region + // ro_ptrmap: metaspace pointers inside the read-only region // heap_info->oopmap(): Java oop pointers in the heap region // heap_info->ptrmap(): metaspace pointers in the heap region char* buffer = NEW_C_HEAP_ARRAY(char, size_in_bytes, mtClassShared); size_t written = 0; - written = write_bitmap(ptrmap, buffer, written); - header()->set_ptrmap_size_in_bits(ptrmap->size()); + + region_at(MetaspaceShared::rw)->init_ptrmap(0, rw_ptrmap->size()); + written = write_bitmap(rw_ptrmap, buffer, written); + + region_at(MetaspaceShared::ro)->init_ptrmap(written, ro_ptrmap->size()); + written = write_bitmap(ro_ptrmap, buffer, written); if (heap_info->is_used()) { FileMapRegion* r = region_at(MetaspaceShared::hp); @@ -1904,15 +1915,19 @@ bool FileMapInfo::relocate_pointers_in_core_regions(intx addr_delta) { if (bitmap_base == nullptr) { return false; // OOM, or CRC check failure } else { - size_t ptrmap_size_in_bits = header()->ptrmap_size_in_bits(); - log_debug(cds, reloc)("mapped relocation bitmap @ " INTPTR_FORMAT " (" SIZE_FORMAT " bits)", - p2i(bitmap_base), ptrmap_size_in_bits); + BitMapView rw_ptrmap = ptrmap_view(MetaspaceShared::rw); + BitMapView ro_ptrmap = ptrmap_view(MetaspaceShared::ro); - BitMapView ptrmap((BitMap::bm_word_t*)bitmap_base, ptrmap_size_in_bits); + FileMapRegion* rw_region = first_core_region(); + FileMapRegion* ro_region = last_core_region(); - // Patch all pointers in the mapped region that are marked by ptrmap. - address patch_base = (address)mapped_base(); - address patch_end = (address)mapped_end(); + // Patch all pointers inside the RW region + address rw_patch_base = (address)rw_region->mapped_base(); + address rw_patch_end = (address)rw_region->mapped_end(); + + // Patch all pointers inside the RO region + address ro_patch_base = (address)ro_region->mapped_base(); + address ro_patch_end = (address)ro_region->mapped_end(); // the current value of the pointers to be patched must be within this // range (i.e., must be between the requested base address and the address of the current archive). @@ -1925,9 +1940,12 @@ bool FileMapInfo::relocate_pointers_in_core_regions(intx addr_delta) { address valid_new_base = (address)header()->mapped_base_address(); address valid_new_end = (address)mapped_end(); - SharedDataRelocator patcher((address*)patch_base, (address*)patch_end, valid_old_base, valid_old_end, + SharedDataRelocator rw_patcher((address*)rw_patch_base, (address*)rw_patch_end, valid_old_base, valid_old_end, + valid_new_base, valid_new_end, addr_delta); + SharedDataRelocator ro_patcher((address*)ro_patch_base, (address*)ro_patch_end, valid_old_base, valid_old_end, valid_new_base, valid_new_end, addr_delta); - ptrmap.iterate(&patcher); + rw_ptrmap.iterate(&rw_patcher); + ro_ptrmap.iterate(&ro_patcher); // The MetaspaceShared::bm region will be unmapped in MetaspaceShared::initialize_shared_spaces(). diff --git a/src/hotspot/share/cds/filemap.hpp b/src/hotspot/share/cds/filemap.hpp index e7c131ee5bb64..6d106ef099d7e 100644 --- a/src/hotspot/share/cds/filemap.hpp +++ b/src/hotspot/share/cds/filemap.hpp @@ -131,7 +131,6 @@ class SharedPathTable { class FileMapRegion: private CDSFileMapRegion { - BitMapView bitmap_view(bool is_oopmap); public: void assert_is_heap_region() const { assert(_is_heap_region, "must be heap region"); @@ -158,6 +157,8 @@ class FileMapRegion: private CDSFileMapRegion { bool mapped_from_file() const { return _mapped_from_file != 0; } size_t oopmap_offset() const { assert_is_heap_region(); return _oopmap_offset; } size_t oopmap_size_in_bits() const { assert_is_heap_region(); return _oopmap_size_in_bits; } + size_t ptrmap_offset() const { return _ptrmap_offset; } + size_t ptrmap_size_in_bits() const { return _ptrmap_size_in_bits; } void set_file_offset(size_t s) { _file_offset = s; } void set_read_only(bool v) { _read_only = v; } @@ -167,8 +168,6 @@ class FileMapRegion: private CDSFileMapRegion { bool allow_exec, int crc); void init_oopmap(size_t offset, size_t size_in_bits); void init_ptrmap(size_t offset, size_t size_in_bits); - BitMapView oopmap_view(); - BitMapView ptrmap_view(); bool has_ptrmap() { return _ptrmap_size_in_bits != 0; } bool check_region_crc(char* base) const; @@ -225,7 +224,6 @@ class FileMapHeader: private CDSFileMapHeaderBase { bool _use_optimized_module_handling;// No module-relation VM options were specified, so we can skip // some expensive operations. bool _has_full_module_graph; // Does this CDS archive contain the full archived module graph? - size_t _ptrmap_size_in_bits; // Size of pointer relocation bitmap size_t _heap_roots_offset; // Offset of the HeapShared::roots() object, from the bottom // of the archived heap objects, in bytes. size_t _heap_oopmap_start_pos; // The first bit in the oopmap corresponds to this position in the heap. @@ -267,7 +265,6 @@ class FileMapHeader: private CDSFileMapHeaderBase { char* mapped_base_address() const { return _mapped_base_address; } bool has_platform_or_app_classes() const { return _has_platform_or_app_classes; } bool has_non_jar_in_classpath() const { return _has_non_jar_in_classpath; } - size_t ptrmap_size_in_bits() const { return _ptrmap_size_in_bits; } bool compressed_oops() const { return _compressed_oops; } bool compressed_class_pointers() const { return _compressed_class_ptrs; } size_t heap_roots_offset() const { return _heap_roots_offset; } @@ -282,7 +279,6 @@ class FileMapHeader: private CDSFileMapHeaderBase { void set_has_platform_or_app_classes(bool v) { _has_platform_or_app_classes = v; } void set_cloned_vtables(char* p) { set_as_offset(p, &_cloned_vtables_offset); } void set_serialized_data(char* p) { set_as_offset(p, &_serialized_data_offset); } - void set_ptrmap_size_in_bits(size_t s) { _ptrmap_size_in_bits = s; } void set_mapped_base_address(char* p) { _mapped_base_address = p; } void set_heap_roots_offset(size_t n) { _heap_roots_offset = n; } void set_heap_oopmap_start_pos(size_t n) { _heap_oopmap_start_pos = n; } @@ -443,7 +439,7 @@ class FileMapInfo : public CHeapObj { void write_region(int region, char* base, size_t size, bool read_only, bool allow_exec); size_t remove_bitmap_leading_zeros(CHeapBitMap* map); - char* write_bitmap_region(const CHeapBitMap* ptrmap, ArchiveHeapInfo* heap_info, + char* write_bitmap_region(const CHeapBitMap* rw_ptrmap, const CHeapBitMap* ro_ptrmap, ArchiveHeapInfo* heap_info, size_t &size_in_bytes); size_t write_heap_region(ArchiveHeapInfo* heap_info); void write_bytes(const void* buffer, size_t count); @@ -526,6 +522,10 @@ class FileMapInfo : public CHeapObj { return header()->region_at(i); } + BitMapView bitmap_view(int region_index, bool is_oopmap); + BitMapView oopmap_view(int region_index); + BitMapView ptrmap_view(int region_index); + void print(outputStream* st) const; const char* vm_version() {