[vm/isolates] Introduce fast isolate spawn in AOT.

Speed up is achieved by sharing most of the dart code, object store
and class table between isolates in single isolate group. So
instead of bootstrapping isolate from the snapshot, isolate is
initialized by setting pointers to existing data structures already
set up for first isolate, and only few isolate-specific structures (moved
to newly introducted isolate_object_store) are created.

To allow for safe cross-isolate switchable call site, type test cache
mutations additional synchronization via RunWithStoppedMutators(that
relies on safepoints) was added.
Besides switchable call sites, no other mutation to the dart code is
done in AOT, which allows such sharing.

Bug: #37835
Bug: #36097
Change-Id: I64c86525f4ef9cb30567a49a106bfe700355942b
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/136780
Commit-Queue: Alexander Aprelev <aam@google.com>
Reviewed-by: Ryan Macnak <rmacnak@google.com>

Author: aam

runtime/lib/isolate.cc

@@ -167,7 +167,11 @@ class SpawnIsolateTask : public ThreadPool::Task {
         return;
       }
 
+#if defined(DART_PRECOMPILED_RUNTIME)
+      isolate = CreateWithinExistingIsolateGroupAOT(group, name, &error);
+#else
       isolate = CreateWithinExistingIsolateGroup(group, name, &error);
+#endif
       parent_isolate_->DecrementSpawnCount();
       parent_isolate_ = nullptr;
       if (isolate == nullptr) {

runtime/observatory/tests/service/get_retaining_path_rpc_test.dart

@@ -193,7 +193,7 @@ var tests = <IsolateTest>[
       'limit': 100,
     };
     var result = await isolate.invokeRpcNoUpgrade('getRetainingPath', params);
-    expect(result['gcRootType'], 'object store');
+    expect(result['gcRootType'], 'isolate_object store');
     expect(result['elements'].length, 0);
   },
 ];

runtime/vm/class_finalizer.cc

@@ -1519,7 +1519,7 @@ void ClassFinalizer::RemapClassIds(intptr_t* old_to_new_cid) {
     // The [HeapIterationScope] also safepoints all threads.
     HeapIterationScope his(T);
 
-    IG->class_table()->Remap(old_to_new_cid);
+    IG->shared_class_table()->Remap(old_to_new_cid);
     IG->ForEachIsolate(
         [&](Isolate* I) {
           I->set_remapping_cids(true);

runtime/vm/class_table.cc

@@ -31,7 +31,8 @@ SharedClassTable::SharedClassTable()
         calloc(capacity_, sizeof(RelaxedAtomic<intptr_t>))));
   } else {
     // Duplicate the class table from the VM isolate.
-    auto vm_shared_class_table = Dart::vm_isolate()->group()->class_table();
+    auto vm_shared_class_table =
+        Dart::vm_isolate()->group()->shared_class_table();
     capacity_ = vm_shared_class_table->capacity_;
     // Note that [calloc] will zero-initialize the memory.
     RelaxedAtomic<intptr_t>* table = reinterpret_cast<RelaxedAtomic<intptr_t>*>(
@@ -71,6 +72,13 @@ SharedClassTable::~SharedClassTable() {
   NOT_IN_PRODUCT(free(trace_allocation_table_.load()));
 }
 
+void ClassTable::set_table(RawClass** table) {
+  Isolate* isolate = Isolate::Current();
+  ASSERT(isolate != nullptr);
+  table_.store(table);
+  isolate->set_class_table_table(table);
+}
+
 ClassTable::ClassTable(SharedClassTable* shared_class_table)
     : top_(kNumPredefinedCids),
       capacity_(0),
@@ -81,6 +89,9 @@ ClassTable::ClassTable(SharedClassTable* shared_class_table)
     ASSERT(kInitialCapacity >= kNumPredefinedCids);
     capacity_ = kInitialCapacity;
     // Note that [calloc] will zero-initialize the memory.
+    // Don't use set_table because caller is supposed to set up isolates
+    // cached copy when constructing ClassTable. Isolate::Current might not
+    // be available at this point yet.
     table_.store(static_cast<RawClass**>(calloc(capacity_, sizeof(RawClass*))));
   } else {
     // Duplicate the class table from the VM isolate.
@@ -100,6 +111,9 @@ ClassTable::ClassTable(SharedClassTable* shared_class_table)
     table[kDynamicCid] = vm_class_table->At(kDynamicCid);
     table[kVoidCid] = vm_class_table->At(kVoidCid);
     table[kNeverCid] = vm_class_table->At(kNeverCid);
+    // Don't use set_table because caller is supposed to set up isolates
+    // cached copy when constructing ClassTable. Isolate::Current might not
+    // be available at this point yet.
     table_.store(table);
   }
 }
@@ -226,7 +240,7 @@ void ClassTable::Grow(intptr_t new_capacity) {
     new_table[i] = 0;
   }
   old_class_tables_->Add(old_table);
-  table_.store(new_table);
+  set_table(new_table);
 
   capacity_ = new_capacity;
 }
@@ -374,6 +388,19 @@ void ClassTable::CopySizesFromClassObjects() {
   }
 }
 
+void ClassTable::CopyFrom(ClassTable* class_table) {
+  capacity_ = class_table->Capacity();
+  top_ = class_table->NumCids();
+  auto old_table = class_table->table_.load();
+  auto new_table =
+      static_cast<RawClass**>(malloc(capacity_ * sizeof(RawClass*)));  // NOLINT
+  for (intptr_t i = 0; i < capacity_; i++) {
+    new_table[i] = old_table[i];
+  }
+  old_class_tables_->Add(table_.load());
+  set_table(new_table);
+}
+
 void ClassTable::Validate() {
   Class& cls = Class::Handle();
   for (intptr_t cid = kNumPredefinedCids; cid < top_; cid++) {

runtime/vm/class_table.h

@@ -340,6 +340,8 @@ class ClassTable {
   void AllocateIndex(intptr_t index);
   void Unregister(intptr_t index);
 
+  void CopyFrom(ClassTable* class_table);
+
   void Remap(intptr_t* old_to_new_cids);
 
   void VisitObjectPointers(ObjectPointerVisitor* visitor);
@@ -353,14 +355,6 @@ class ClassTable {
 
   void Print();
 
-  // Used by the generated code.
-  static intptr_t table_offset() { return OFFSET_OF(ClassTable, table_); }
-
-  // Used by the generated code.
-  static intptr_t shared_class_table_offset() {
-    return OFFSET_OF(ClassTable, shared_class_table_);
-  }
-
 #ifndef PRODUCT
   // Describes layout of heap stats for code generation. See offset_extractor.cc
   struct ArrayLayout {
@@ -385,16 +379,21 @@ class ClassTable {
   friend class MarkingWeakVisitor;
   friend class Scavenger;
   friend class ScavengerWeakVisitor;
+  friend class Dart;
   friend Isolate* CreateWithinExistingIsolateGroup(IsolateGroup* group,
                                                    const char* name,
                                                    char** error);
+  friend class Isolate;  // for table()
   static const int kInitialCapacity = SharedClassTable::kInitialCapacity;
   static const int kCapacityIncrement = SharedClassTable::kCapacityIncrement;
 
   void AddOldTable(RawClass** old_table);
 
   void Grow(intptr_t index);
 
+  RawClass** table() { return table_.load(); }
+  void set_table(RawClass** table);
+
   intptr_t top_;
   intptr_t capacity_;
 

runtime/vm/clustered_snapshot.cc

@@ -2,6 +2,8 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+#include <memory>
+
 #include "vm/clustered_snapshot.h"
 
 #include "platform/assert.h"
@@ -231,7 +233,8 @@ class ClassSerializationCluster : public SerializationCluster {
   UnboxedFieldBitmap CalculateTargetUnboxedFieldsBitmap(Serializer* s,
                                                         intptr_t class_id) {
     const auto unboxed_fields_bitmap_host =
-        s->isolate()->group()->class_table()->GetUnboxedFieldsMapAt(class_id);
+        s->isolate()->group()->shared_class_table()->GetUnboxedFieldsMapAt(
+            class_id);
 
     UnboxedFieldBitmap unboxed_fields_bitmap;
     if (unboxed_fields_bitmap_host.IsEmpty() ||
@@ -349,7 +352,7 @@ class ClassDeserializationCluster : public DeserializationCluster {
       }
     }
 
-    auto shared_class_table = d->isolate()->group()->class_table();
+    auto shared_class_table = d->isolate()->group()->shared_class_table();
     for (intptr_t id = start_index_; id < stop_index_; id++) {
       RawClass* cls = reinterpret_cast<RawClass*>(d->Ref(id));
       Deserializer::InitializeHeader(cls, kClassCid, Class::InstanceSize());
@@ -1201,6 +1204,10 @@ class FieldDeserializationCluster : public DeserializationCluster {
         field.InitializeGuardedListLengthInObjectOffset();
       }
     }
+
+    Isolate* isolate = Isolate::Current();
+    isolate->set_saved_initial_field_table(
+        std::shared_ptr<FieldTable>(isolate->field_table()->Clone()));
   }
 };
 
@@ -3022,7 +3029,8 @@ class InstanceSerializationCluster : public SerializationCluster {
     const intptr_t next_field_offset = host_next_field_offset_in_words_
                                        << kWordSizeLog2;
     const auto unboxed_fields_bitmap =
-        s->isolate()->group()->class_table()->GetUnboxedFieldsMapAt(cid_);
+        s->isolate()->group()->shared_class_table()->GetUnboxedFieldsMapAt(
+            cid_);
     intptr_t offset = Instance::NextFieldOffset();
     while (offset < next_field_offset) {
       // Skips unboxed fields
@@ -3058,7 +3066,8 @@ class InstanceSerializationCluster : public SerializationCluster {
                                  << kWordSizeLog2;
     const intptr_t count = objects_.length();
     const auto unboxed_fields_bitmap =
-        s->isolate()->group()->class_table()->GetUnboxedFieldsMapAt(cid_);
+        s->isolate()->group()->shared_class_table()->GetUnboxedFieldsMapAt(
+            cid_);
     for (intptr_t i = 0; i < count; i++) {
       RawInstance* instance = objects_[i];
       AutoTraceObject(instance);
@@ -3116,7 +3125,8 @@ class InstanceDeserializationCluster : public DeserializationCluster {
         Object::RoundedAllocationSize(instance_size_in_words_ * kWordSize);
 
     const auto unboxed_fields_bitmap =
-        d->isolate()->group()->class_table()->GetUnboxedFieldsMapAt(cid_);
+        d->isolate()->group()->shared_class_table()->GetUnboxedFieldsMapAt(
+            cid_);
     for (intptr_t id = start_index_; id < stop_index_; id++) {
       RawInstance* instance = reinterpret_cast<RawInstance*>(d->Ref(id));
       bool is_canonical = d->Read<bool>();
@@ -5521,7 +5531,7 @@ static const char* kObjectStoreFieldNames[] = {
 #undef DECLARE_OBJECT_STORE_FIELD
 };
 
-void Serializer::WriteIsolateSnapshot(intptr_t num_base_objects,
+void Serializer::WriteProgramSnapshot(intptr_t num_base_objects,
                                       ObjectStore* object_store) {
   NoSafepointScope no_safepoint;
 
@@ -5532,7 +5542,7 @@ void Serializer::WriteIsolateSnapshot(intptr_t num_base_objects,
       AddBaseObject(base_objects.At(i));
     }
   } else {
-    // Base objects carried over from WriteVMIsolateSnapshot.
+    // Base objects carried over from WriteVMSnapshot.
     num_base_objects_ += num_base_objects;
     next_ref_index_ += num_base_objects;
   }
@@ -5813,7 +5823,7 @@ void Deserializer::ReadDispatchTable() {
   }
   ASSERT(repeat_count == 0);
 
-  I->set_dispatch_table(table);
+  I->group()->set_dispatch_table(table);
 #endif
 }
 
@@ -6196,7 +6206,7 @@ void Deserializer::ReadVMSnapshot() {
   }
 }
 
-void Deserializer::ReadIsolateSnapshot(ObjectStore* object_store) {
+void Deserializer::ReadProgramSnapshot(ObjectStore* object_store) {
   Array& refs = Array::Handle();
   Prepare();
 
@@ -6234,22 +6244,21 @@ void Deserializer::ReadIsolateSnapshot(ObjectStore* object_store) {
   thread()->isolate()->class_table()->CopySizesFromClassObjects();
   heap_->old_space()->EvaluateAfterLoading();
 
-#if defined(DEBUG)
   Isolate* isolate = thread()->isolate();
+#if defined(DEBUG)
   isolate->ValidateClassTable();
   isolate->heap()->Verify();
 #endif
 
   for (intptr_t i = 0; i < num_clusters_; i++) {
     clusters_[i]->PostLoad(refs, kind_, zone_);
   }
-  object_store->PostLoad();
+  isolate->isolate_object_store()->PreallocateObjects();
 
   // Setup native resolver for bootstrap impl.
   Bootstrap::SetupNativeResolver();
 }
 
-
 #if !defined(DART_PRECOMPILED_RUNTIME)
 FullSnapshotWriter::FullSnapshotWriter(Snapshot::Kind kind,
                                        uint8_t** vm_snapshot_data_buffer,
@@ -6324,8 +6333,8 @@ intptr_t FullSnapshotWriter::WriteVMSnapshot() {
   return num_objects;
 }
 
-void FullSnapshotWriter::WriteIsolateSnapshot(intptr_t num_base_objects) {
-  TIMELINE_DURATION(thread(), Isolate, "WriteIsolateSnapshot");
+void FullSnapshotWriter::WriteProgramSnapshot(intptr_t num_base_objects) {
+  TIMELINE_DURATION(thread(), Isolate, "WriteProgramSnapshot");
 
   Serializer serializer(thread(), kind_, isolate_snapshot_data_buffer_, alloc_,
                         kInitialSize, isolate_image_writer_, /*vm=*/false,
@@ -6344,7 +6353,7 @@ void FullSnapshotWriter::WriteIsolateSnapshot(intptr_t num_base_objects) {
   serializer.WriteVersionAndFeatures(false);
   // Isolate snapshot roots are:
   // - the object store
-  serializer.WriteIsolateSnapshot(num_base_objects, object_store);
+  serializer.WriteProgramSnapshot(num_base_objects, object_store);
   serializer.FillHeader(serializer.kind());
   clustered_isolate_size_ = serializer.bytes_written();
 
@@ -6375,7 +6384,7 @@ void FullSnapshotWriter::WriteFullSnapshot() {
   }
 
   if (isolate_snapshot_data_buffer() != NULL) {
-    WriteIsolateSnapshot(num_base_objects);
+    WriteProgramSnapshot(num_base_objects);
   }
 
   if (FLAG_print_snapshot_sizes) {
@@ -6538,7 +6547,7 @@ RawApiError* FullSnapshotReader::ReadVMSnapshot() {
   return ApiError::null();
 }
 
-RawApiError* FullSnapshotReader::ReadIsolateSnapshot() {
+RawApiError* FullSnapshotReader::ReadProgramSnapshot() {
   SnapshotHeaderReader header_reader(kind_, buffer_, size_);
   intptr_t offset = 0;
   char* error =
@@ -6564,7 +6573,7 @@ RawApiError* FullSnapshotReader::ReadIsolateSnapshot() {
   }
 
   auto object_store = thread_->isolate()->object_store();
-  deserializer.ReadIsolateSnapshot(object_store);
+  deserializer.ReadProgramSnapshot(object_store);
 
 #if defined(DART_PRECOMPILED_RUNTIME)
   if (FLAG_use_bare_instructions) {

runtime/vm/clustered_snapshot.h

@@ -157,7 +157,7 @@ class Serializer : public ThreadStackResource {
   }
 
   intptr_t WriteVMSnapshot(const Array& symbols);
-  void WriteIsolateSnapshot(intptr_t num_base_objects,
+  void WriteProgramSnapshot(intptr_t num_base_objects,
                             ObjectStore* object_store);
 
   void AddVMIsolateBaseObjects();
@@ -539,7 +539,7 @@ class Deserializer : public ThreadStackResource {
   // message otherwise.
   RawApiError* VerifyImageAlignment();
 
-  void ReadIsolateSnapshot(ObjectStore* object_store);
+  void ReadProgramSnapshot(ObjectStore* object_store);
   void ReadVMSnapshot();
 
   void AddVMIsolateBaseObjects();
@@ -682,7 +682,7 @@ class FullSnapshotWriter {
   Isolate* isolate() const { return thread_->isolate(); }
   Heap* heap() const { return isolate()->heap(); }
 
-  // Writes a full snapshot of the Isolate.
+  // Writes a full snapshot of the program(VM isolate, regular isolate group).
   void WriteFullSnapshot();
 
   intptr_t VmIsolateSnapshotSize() const { return vm_isolate_snapshot_size_; }
@@ -692,8 +692,8 @@ class FullSnapshotWriter {
   // Writes a snapshot of the VM Isolate.
   intptr_t WriteVMSnapshot();
 
-  // Writes a full snapshot of a regular Dart Isolate.
-  void WriteIsolateSnapshot(intptr_t num_base_objects);
+  // Writes a full snapshot of regular Dart isolate group.
+  void WriteProgramSnapshot(intptr_t num_base_objects);
 
   Thread* thread_;
   Snapshot::Kind kind_;
@@ -724,7 +724,7 @@ class FullSnapshotReader {
   ~FullSnapshotReader() {}
 
   RawApiError* ReadVMSnapshot();
-  RawApiError* ReadIsolateSnapshot();
+  RawApiError* ReadProgramSnapshot();
 
  private:
   RawApiError* ConvertToApiError(char* message);