Closed
Description
Something similar happened here #54886, but then we found a workaround right away.
This time I couldn't localize the problem, and don't see any easy solutions besides disabling compressed pointers completely.
error from flutter build:
Exhausted heap space, trying to allocate 65584 bytes.
Error: Out of memory
trace
local unopt profile build from these commits
engine: flutter/engine@3.19.6...2ZeroSix:engine:ee0d52dd5f559d8500e3e2bc230b41f074dd8848
sdk: 3.3.4...2ZeroSix:sdk:0fe1ac45725932b52d8803c9064513c96ba32efe
./flutter/tools/gn --android --android-cpu arm64 --runtime-mode=profile --no-prebuilt-dart-sdk --unopt
ninja -C ./out/android_profile_unopt_arm64
./out/android_profile_unopt_arm64/clang_x64/gen_snapshot_arm64 --deterministic --verbose --snapshot_kind=app-aot-elf out/android_profile_unopt_arm64/test/arm64/app.so --strip app.dill
gen_snapshot_arm64!dart::Heap::AllocateOld(dart::Thread*, long, bool) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/heap/heap.cc:177)
gen_snapshot_arm64!dart::Object::Allocate(long, long, dart::Heap::Space, bool, unsigned long, unsigned long) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/object.cc:2838)
gen_snapshot_arm64!dart::Array::ObjectPtrType dart::Object::AllocateVariant<dart::Array>(long, dart::Heap::Space, long) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/object.h:749)
gen_snapshot_arm64!dart::Array::NewUninitialized(long, long, dart::Heap::Space) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/object.cc:25136)
gen_snapshot_arm64!dart::Array::New(long, long, dart::Heap::Space) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/object.cc:25148)
gen_snapshot_arm64!dart::Array::New(long, dart::Heap::Space) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/object.h:11063)
gen_snapshot_arm64!dart::ArrayStorageTraits::New(dart::Zone*, long, dart::Heap::Space) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/hash_table.h:31)
gen_snapshot_arm64!dart::HashSet<dart::UnorderedHashTable<dart::CanonicalTypeArgumentsTraits, 0l, dart::ArrayStorageTraits>, dart::ArrayStorageTraits>::Storage::ArrayPtr dart::HashTables::New<dart::HashSet<dart::UnorderedHashTable<dart::CanonicalTypeArgumentsTraits, 0l, dart::ArrayStorageTraits>, dart::ArrayStorageTraits>>(long, dart::Heap::Space) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/hash_table.h:579)
gen_snapshot_arm64!void dart::HashTables::EnsureLoadFactor<dart::HashSet<dart::UnorderedHashTable<dart::CanonicalTypeArgumentsTraits, 0l, dart::ArrayStorageTraits>, dart::ArrayStorageTraits>>(double, dart::HashSet<dart::UnorderedHashTable<dart::CanonicalTypeArgumentsTraits, 0l, dart::ArrayStorageTraits>, dart::ArrayStorageTraits> const&) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/hash_table.h:644)
gen_snapshot_arm64!dart::HashSet<dart::UnorderedHashTable<dart::CanonicalTypeArgumentsTraits, 0l, dart::ArrayStorageTraits>, dart::ArrayStorageTraits>::EnsureCapacity() const (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/hash_table.h:863)
gen_snapshot_arm64!dart::HashSet<dart::UnorderedHashTable<dart::CanonicalTypeArgumentsTraits, 0l, dart::ArrayStorageTraits>, dart::ArrayStorageTraits>::Insert(dart::Object const&) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/hash_table.h:802)
gen_snapshot_arm64!dart::TypeArguments::Canonicalize(dart::Thread*) const (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/object.cc:7837)
gen_snapshot_arm64!dart::Type::Canonicalize(dart::Thread*) const (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/object.cc:22397)
gen_snapshot_arm64!dart::ClassFinalizer::FinalizeTypesInClass(dart::Class const&) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/class_finalizer.cc:590)
gen_snapshot_arm64!dart::ClassFinalizer::ProcessPendingClasses() (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/class_finalizer.cc:218)
gen_snapshot_arm64!dart::kernel::KernelLoader::LoadProgram(bool) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/kernel_loader.cc:518)
gen_snapshot_arm64!dart::BootstrapFromKernel(dart::Thread*, unsigned char const*, long) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/bootstrap.cc:151)
gen_snapshot_arm64!dart::Bootstrap::DoBootstrapping(unsigned char const*, long) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/bootstrap.cc:193)
gen_snapshot_arm64!dart::Object::Init(dart::IsolateGroup*, unsigned char const*, long) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/object.cc:2502)
gen_snapshot_arm64!dart::Dart::InitIsolateGroupFromSnapshot(dart::Thread*, unsigned char const*, unsigned char const*, unsigned char const*, long) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/dart.cc:827)
gen_snapshot_arm64!dart::Dart::InitializeIsolateGroup(dart::Thread*, unsigned char const*, unsigned char const*, unsigned char const*, long) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/dart.cc:929)
gen_snapshot_arm64!dart::CreateIsolate(dart::IsolateGroup*, bool, char const*, void*, char**) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/dart_api_impl.cc:1276)
gen_snapshot_arm64!::Dart_CreateIsolateGroupFromKernel(const char *, const char *, const uint8_t *, intptr_t, Dart_IsolateFlags *, void *, void *, char **) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/vm/dart_api_impl.cc:1409)
gen_snapshot_arm64!dart::bin::CreateIsolateAndSnapshot(dart::bin::CommandLineOptions const&) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/bin/gen_snapshot.cc:758)
gen_snapshot_arm64!dart::bin::main(int, char**) (/Users/2zerosix/Projects/oss/engine/src/third_party/dart/runtime/bin/gen_snapshot.cc:954)
dyld!start (Unknown Source:0)
At first, the error occurred during refactoring of a large module into a separate package, at some point refactoring introduced a lot of cycles with long export-import chains, but as soon as we got rid of them the problem disappeared.
We postponed actual investigation for a while, and now it's happening on our main branch.
I'm almost sure there is some problem with our exports overusage, but I'm not sure how to find what exactly is consuming all the memory.
I've tried --trace_timeline --timeline_recorder=file --timeline_streams=all --timeline_dir=$(pwd) but it just produced an empty file.
xctrace gave some information
4326.75 MB 86.1% 4433245 dart::kernel::KernelLoader::LoadLibrary(long)
4255.69 MB 84.7% 4363786 dart::kernel::KernelLoader::LoadLibraryImportsAndExports(dart::Library*, dart::Class const&)
1878.74 MB 37.4% 1925892 dart::GrowableObjectArray::New(long, dart::Heap::Space)
480.48 MB 9.5% 493724 dart::Library::AddImport(dart::Namespace const&) const
474.28 MB 9.4% 485661 dart::Object::HandleImpl(dart::Zone*, dart::ObjectPtr, long)
469.83 MB 9.3% 482678 dart::Namespace::New(dart::Library const&, dart::Array const&, dart::Array const&, dart::Library const&)
469.59 MB 9.3% 480856 dart::kernel::TranslationHelper::DartSymbolPlain(dart::kernel::StringIndex) const
469.46 MB 9.3% 480726 dart::Object::ZoneHandleImpl(dart::Zone*, dart::ObjectPtr, long)
12.67 MB 0.2% 13001 dart::kernel::KernelLoader::LookupLibrary(dart::kernel::NameIndex)
207.00 KB 0.0% 207 dart::Library::AddObject(dart::Object const&, dart::String const&) const
71.00 KB 0.0% 71 0x1011b2fdb
69.00 KB 0.0% 69 0x1011af9bc
66.00 KB 0.0% 66 0x1011b2f74
62.00 KB 0.0% 62 0x1011b2ff6
55.00 KB 0.0% 55 dart::GrowableObjectArray::Add(dart::Object const&, dart::Heap::Space) const
50.00 KB 0.0% 50 dart::kernel::TranslationHelper::DartSymbolObfuscate(dart::kernel::StringIndex) const
34.00 KB 0.0% 34 dart::Library::AddExport(dart::Namespace const&) const
18.72 KB 0.0% 599 dart::Array::New(long, long, dart::Heap::Space)
16.00 KB 0.0% 16 0x1011aebd9
10.00 KB 0.0% 10 0x1011aeb6d
9.00 KB 0.0% 9 0x1011aec18
49.65 MB 0.9% 47867 dart::kernel::KernelLoader::LoadClass(dart::Library const&, dart::Class const&, long, dart::Class*)
4.71 MB 0.0% 4826 dart::Object::HandleImpl(dart::Zone*, dart::ObjectPtr, long)
4.33 MB 0.0% 4086 dart::Class::New(dart::Library const&, dart::String const&, dart::Script const&, dart::TokenPosition, bool)
4.12 MB 0.0% 4231 dart::kernel::KernelLoader::LookupLibrary(dart::kernel::NameIndex)
2.66 MB 0.0% 2729 dart::GrowableObjectArray::Add(dart::Object const&, dart::Heap::Space) const
1.53 MB 0.0% 1562 dart::KernelProgramInfo::ScriptAt(long) const
1.37 MB 0.0% 1411 dart::TypedDataBase::ViewFromTo(long, long, dart::Heap::Space) const
1.30 MB 0.0% 1335 dart::kernel::TranslationHelper::DartSymbolObfuscate(dart::kernel::StringIndex) const
707.00 KB 0.0% 707 dart::KernelProgramInfo::KernelLibraryEndOffset(long) const
701.00 KB 0.0% 701 0x1011a160d
2.00 KB 0.0% 2 0x10114718d
64 Bytes 0.0% 2 dart::KernelProgramInfo::InsertLibrary(dart::Thread*, dart::Smi const&, dart::Library const&) const
6.68 MB 0.1% 62892 dart::ClassFinalizer::ProcessPendingClasses()
4.83 MB 0.0% 3938 dart::ClassFinalizer::FinalizeTypesInClass(dart::Class const&)
2.05 MB 0.0% 1097 dart::Type::Canonicalize(dart::Thread*) const
1.42 MB 0.0% 1452 dart::ClassFinalizer::RegisterClassInHierarchy(dart::Zone*, dart::Class const&)
807.00 KB 0.0% 807 dart::ClassFinalizer::FinalizeTypesInClass(dart::Class const&)
270.00 KB 0.0% 270 dart::ClassFinalizer::FinalizeTypeParameters(dart::Zone*, dart::TypeParameters const&, dart::ClassFinalizer::FinalizationKind)
145.00 KB 0.0% 145 dart::Object::HandleImpl(dart::Zone*, dart::ObjectPtr, long)
135.00 KB 0.0% 135 dart::ClassFinalizer::FinalizeType(dart::AbstractType const&, dart::ClassFinalizer::FinalizationKind)
16.00 KB 0.0% 16 0x101191cb0
14.00 KB 0.0% 14 0x101191cba
2.00 KB 0.0% 2 dart::Class::SuperClass(dart::ClassTable*) const
1.84 MB 0.0% 58946 dart::Class::SetUserVisibleNameInClassTable()
4.09 KB 0.0% 7 dart::Object::Print() const
512 Bytes 0.0% 1 dart::Log::Print(char const*, ...)
15.47 MB 0.3% 16389 dart::kernel::KernelLoader::InitializeFields(dart::DirectChainedHashMap<dart::kernel::UriToSourceTableTrait>*)
Steps to reproduce
I couldn't reproduce it without internal code yet, so I don't know if this is still an issue in the main sdk branch.
Workaround
Same as in previous issue, error goes away when excluding excess code from the build.
Our current hotfix is to exclude one large module that wasn't yet enabled in production.