[vm, gc] Produce a proper error message when crashing due to lack of memory at isolate startup.

Bug: flutter/flutter#35651
Change-Id: I2639baa4daaeb1d70d71a83da69e2c51800bdbf5
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/108365
Reviewed-by: Siva Annamalai <asiva@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>

Author: rmacnak-google

runtime/vm/heap/heap.cc

@@ -439,6 +439,13 @@ void Heap::NotifyLowMemory() {
 
 void Heap::EvacuateNewSpace(Thread* thread, GCReason reason) {
   ASSERT((reason != kOldSpace) && (reason != kPromotion));
+  if (thread->isolate() == Dart::vm_isolate()) {
+    // The vm isolate cannot safely collect garbage due to unvisited read-only
+    // handles and slots bootstrapped with RAW_NULL. Ignore GC requests to
+    // trigger a nice out-of-memory message instead of a crash in the middle of
+    // visiting pointers.
+    return;
+  }
   if (BeginNewSpaceGC(thread)) {
     RecordBeforeGC(kScavenge, reason);
     VMTagScope tagScope(thread, reason == kIdle ? VMTag::kGCIdleTagId
@@ -454,6 +461,13 @@ void Heap::EvacuateNewSpace(Thread* thread, GCReason reason) {
 
 void Heap::CollectNewSpaceGarbage(Thread* thread, GCReason reason) {
   ASSERT((reason != kOldSpace) && (reason != kPromotion));
+  if (thread->isolate() == Dart::vm_isolate()) {
+    // The vm isolate cannot safely collect garbage due to unvisited read-only
+    // handles and slots bootstrapped with RAW_NULL. Ignore GC requests to
+    // trigger a nice out-of-memory message instead of a crash in the middle of
+    // visiting pointers.
+    return;
+  }
   if (BeginNewSpaceGC(thread)) {
     RecordBeforeGC(kScavenge, reason);
     {
@@ -484,6 +498,13 @@ void Heap::CollectOldSpaceGarbage(Thread* thread,
   if (FLAG_use_compactor) {
     type = kMarkCompact;
   }
+  if (thread->isolate() == Dart::vm_isolate()) {
+    // The vm isolate cannot safely collect garbage due to unvisited read-only
+    // handles and slots bootstrapped with RAW_NULL. Ignore GC requests to
+    // trigger a nice out-of-memory message instead of a crash in the middle of
+    // visiting pointers.
+    return;
+  }
   if (BeginOldSpaceGC(thread)) {
     RecordBeforeGC(type, reason);
     VMTagScope tagScope(thread, reason == kIdle ? VMTag::kGCIdleTagId

runtime/vm/object.cc

@@ -2156,12 +2156,17 @@ RawObject* Object::Allocate(intptr_t cls_id, intptr_t size, Heap::Space space) {
     address = heap->Allocate(size, space);
   }
   if (UNLIKELY(address == 0)) {
-    // Use the preallocated out of memory exception to avoid calling
-    // into dart code or allocating any code.
-    const Instance& exception =
-        Instance::Handle(thread->isolate()->object_store()->out_of_memory());
-    Exceptions::Throw(thread, exception);
-    UNREACHABLE();
+    if (thread->top_exit_frame_info() != 0) {
+      // Use the preallocated out of memory exception to avoid calling
+      // into dart code or allocating any code.
+      const Instance& exception =
+          Instance::Handle(thread->isolate()->object_store()->out_of_memory());
+      Exceptions::Throw(thread, exception);
+      UNREACHABLE();
+    } else {
+      // No Dart to propagate an exception to.
+      OUT_OF_MEMORY();
+    }
   }
 #ifndef PRODUCT
   ClassTable* class_table = thread->isolate()->class_table();