tools: refactor snapshot builder

This patch:

- Moves the snapshot building code to src/ so that we can reuse it
  later when generating custom snapshots from an entry point accepted
  by the node binary.
- Create a SnapshotData struct that incorporates all the data useful
  for a snapshot blob, including both the V8 data and the Node.js
  data.

PR-URL: https://github.com/nodejs/node/pull/38902
Reviewed-By: Chengzhong Wu <legendecas@gmail.com>

node.gyp

@@ -1554,8 +1554,6 @@
         'src/node_snapshot_stub.cc',
         'src/node_code_cache_stub.cc',
         'tools/snapshot/node_mksnapshot.cc',
-        'tools/snapshot/snapshot_builder.cc',
-        'tools/snapshot/snapshot_builder.h',
       ],
 
       'conditions': [

src/env.h

@@ -955,6 +955,13 @@ struct EnvSerializeInfo {
   friend std::ostream& operator<<(std::ostream& o, const EnvSerializeInfo& i);
 };
 
+struct SnapshotData {
+  SnapshotData() { blob.data = nullptr; }
+  v8::StartupData blob;
+  std::vector<size_t> isolate_data_indices;
+  EnvSerializeInfo env_info;
+};
+
 class Environment : public MemoryRetainer {
  public:
   Environment(const Environment&) = delete;

src/node.cc

@@ -1130,7 +1130,7 @@ int Start(int argc, char** argv) {
 
   {
     Isolate::CreateParams params;
-    const std::vector<size_t>* indexes = nullptr;
+    const std::vector<size_t>* indices = nullptr;
     const EnvSerializeInfo* env_info = nullptr;
     bool force_no_snapshot =
         per_process::cli_options->per_isolate->no_node_snapshot;
@@ -1138,7 +1138,7 @@ int Start(int argc, char** argv) {
       v8::StartupData* blob = NodeMainInstance::GetEmbeddedSnapshotBlob();
       if (blob != nullptr) {
         params.snapshot_blob = blob;
-        indexes = NodeMainInstance::GetIsolateDataIndexes();
+        indices = NodeMainInstance::GetIsolateDataIndices();
         env_info = NodeMainInstance::GetEnvSerializeInfo();
       }
     }
@@ -1149,7 +1149,7 @@ int Start(int argc, char** argv) {
                                    per_process::v8_platform.Platform(),
                                    result.args,
                                    result.exec_args,
-                                   indexes);
+                                   indices);
     result.exit_code = main_instance.Run(env_info);
   }
 

src/node_main_instance.h

@@ -67,7 +67,7 @@ class NodeMainInstance {
 
   // If nullptr is returned, the binary is not built with embedded
   // snapshot.
-  static const std::vector<size_t>* GetIsolateDataIndexes();
+  static const std::vector<size_t>* GetIsolateDataIndices();
   static v8::StartupData* GetEmbeddedSnapshotBlob();
   static const EnvSerializeInfo* GetEnvSerializeInfo();
   static const std::vector<intptr_t>& CollectExternalReferences();

src/node_snapshot_stub.cc

@@ -10,7 +10,7 @@ v8::StartupData* NodeMainInstance::GetEmbeddedSnapshotBlob() {
   return nullptr;
 }
 
-const std::vector<size_t>* NodeMainInstance::GetIsolateDataIndexes() {
+const std::vector<size_t>* NodeMainInstance::GetIsolateDataIndices() {
   return nullptr;
 }
 

src/node_snapshotable.cc

@@ -1,16 +1,181 @@
 
 #include "node_snapshotable.h"
+#include <iostream>
+#include <sstream>
 #include "base_object-inl.h"
 #include "debug_utils-inl.h"
+#include "env-inl.h"
+#include "node_errors.h"
+#include "node_external_reference.h"
 #include "node_file.h"
+#include "node_internals.h"
+#include "node_main_instance.h"
 #include "node_v8.h"
+#include "node_v8_platform-inl.h"
 
 namespace node {
 
+using v8::Context;
+using v8::HandleScope;
+using v8::Isolate;
 using v8::Local;
 using v8::Object;
 using v8::SnapshotCreator;
 using v8::StartupData;
+using v8::TryCatch;
+using v8::Value;
+
+template <typename T>
+void WriteVector(std::ostringstream* ss, const T* vec, size_t size) {
+  for (size_t i = 0; i < size; i++) {
+    *ss << std::to_string(vec[i]) << (i == size - 1 ? '\n' : ',');
+  }
+}
+
+std::string FormatBlob(SnapshotData* data) {
+  std::ostringstream ss;
+
+  ss << R"(#include <cstddef>
+#include "env.h"
+#include "node_main_instance.h"
+#include "v8.h"
+
+// This file is generated by tools/snapshot. Do not edit.
+
+namespace node {
+
+static const char blob_data[] = {
+)";
+  WriteVector(&ss, data->blob.data, data->blob.raw_size);
+  ss << R"(};
+
+static const int blob_size = )"
+     << data->blob.raw_size << R"(;
+static v8::StartupData blob = { blob_data, blob_size };
+)";
+
+  ss << R"(v8::StartupData* NodeMainInstance::GetEmbeddedSnapshotBlob() {
+  return &blob;
+}
+
+static const std::vector<size_t> isolate_data_indices {
+)";
+  WriteVector(&ss,
+              data->isolate_data_indices.data(),
+              data->isolate_data_indices.size());
+  ss << R"(};
+
+const std::vector<size_t>* NodeMainInstance::GetIsolateDataIndices() {
+  return &isolate_data_indices;
+}
+
+static const EnvSerializeInfo env_info )"
+     << data->env_info << R"(;
+
+const EnvSerializeInfo* NodeMainInstance::GetEnvSerializeInfo() {
+  return &env_info;
+}
+
+}  // namespace node
+)";
+
+  return ss.str();
+}
+
+void SnapshotBuilder::Generate(SnapshotData* out,
+                               const std::vector<std::string> args,
+                               const std::vector<std::string> exec_args) {
+  Isolate* isolate = Isolate::Allocate();
+  isolate->SetCaptureStackTraceForUncaughtExceptions(
+      true, 10, v8::StackTrace::StackTraceOptions::kDetailed);
+  per_process::v8_platform.Platform()->RegisterIsolate(isolate,
+                                                       uv_default_loop());
+  std::unique_ptr<NodeMainInstance> main_instance;
+  std::string result;
+
+  {
+    const std::vector<intptr_t>& external_references =
+        NodeMainInstance::CollectExternalReferences();
+    SnapshotCreator creator(isolate, external_references.data());
+    Environment* env;
+    {
+      main_instance =
+          NodeMainInstance::Create(isolate,
+                                   uv_default_loop(),
+                                   per_process::v8_platform.Platform(),
+                                   args,
+                                   exec_args);
+
+      HandleScope scope(isolate);
+      creator.SetDefaultContext(Context::New(isolate));
+      out->isolate_data_indices =
+          main_instance->isolate_data()->Serialize(&creator);
+
+      // Run the per-context scripts
+      Local<Context> context;
+      {
+        TryCatch bootstrapCatch(isolate);
+        context = NewContext(isolate);
+        if (bootstrapCatch.HasCaught()) {
+          PrintCaughtException(isolate, context, bootstrapCatch);
+          abort();
+        }
+      }
+      Context::Scope context_scope(context);
+
+      // Create the environment
+      env = new Environment(main_instance->isolate_data(),
+                            context,
+                            args,
+                            exec_args,
+                            nullptr,
+                            node::EnvironmentFlags::kDefaultFlags,
+                            {});
+      // Run scripts in lib/internal/bootstrap/
+      {
+        TryCatch bootstrapCatch(isolate);
+        v8::MaybeLocal<Value> result = env->RunBootstrapping();
+        if (bootstrapCatch.HasCaught()) {
+          PrintCaughtException(isolate, context, bootstrapCatch);
+        }
+        result.ToLocalChecked();
+      }
+
+      if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) {
+        env->PrintAllBaseObjects();
+        printf("Environment = %p\n", env);
+      }
+
+      // Serialize the native states
+      out->env_info = env->Serialize(&creator);
+      // Serialize the context
+      size_t index = creator.AddContext(
+          context, {SerializeNodeContextInternalFields, env});
+      CHECK_EQ(index, NodeMainInstance::kNodeContextIndex);
+    }
+
+    // Must be out of HandleScope
+    out->blob =
+        creator.CreateBlob(SnapshotCreator::FunctionCodeHandling::kClear);
+    CHECK(out->blob.CanBeRehashed());
+    // Must be done while the snapshot creator isolate is entered i.e. the
+    // creator is still alive.
+    FreeEnvironment(env);
+    main_instance->Dispose();
+  }
+
+  per_process::v8_platform.Platform()->UnregisterIsolate(isolate);
+}
+
+std::string SnapshotBuilder::Generate(
+    const std::vector<std::string> args,
+    const std::vector<std::string> exec_args) {
+  SnapshotData data;
+  Generate(&data, args, exec_args);
+  std::string result = FormatBlob(&data);
+  delete[] data.blob.data;
+  return result;
+}
 
 SnapshotableObject::SnapshotableObject(Environment* env,
                                        Local<Object> wrap,

src/node_snapshotable.h

@@ -11,6 +11,7 @@ namespace node {
 
 class Environment;
 struct EnvSerializeInfo;
+struct SnapshotData;
 
 #define SERIALIZABLE_OBJECT_TYPES(V)                                           \
   V(fs_binding_data, fs::BindingData)                                          \
@@ -119,6 +120,15 @@ void SerializeBindingData(Environment* env,
                           EnvSerializeInfo* info);
 
 bool IsSnapshotableType(FastStringKey key);
+
+class SnapshotBuilder {
+ public:
+  static std::string Generate(const std::vector<std::string> args,
+                              const std::vector<std::string> exec_args);
+  static void Generate(SnapshotData* out,
+                       const std::vector<std::string> args,
+                       const std::vector<std::string> exec_args);
+};
 }  // namespace node
 
 #endif  // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS

tools/snapshot/README.md

@@ -23,7 +23,7 @@ into the Node.js executable, `libnode` is first built with these unresolved
 symbols:
 
 - `node::NodeMainInstance::GetEmbeddedSnapshotBlob`
-- `node::NodeMainInstance::GetIsolateDataIndexes`
+- `node::NodeMainInstance::GetIsolateDataIndices`
 
 Then the `node_mksnapshot` executable is built with C++ files in this
 directory, as well as `src/node_snapshot_stub.cc` which defines the unresolved

tools/snapshot/node_mksnapshot.cc

@@ -7,7 +7,7 @@
 
 #include "libplatform/libplatform.h"
 #include "node_internals.h"
-#include "snapshot_builder.h"
+#include "node_snapshotable.h"
 #include "util-inl.h"
 #include "v8.h"
 

tools/snapshot/snapshot_builder.cc

@@ -1,172 +0,0 @@
-#include "snapshot_builder.h"
-#include <iostream>
-#include <sstream>
-#include "debug_utils-inl.h"
-#include "env-inl.h"
-#include "node_errors.h"
-#include "node_external_reference.h"
-#include "node_internals.h"
-#include "node_main_instance.h"
-#include "node_snapshotable.h"
-#include "node_v8_platform-inl.h"
-
-namespace node {
-
-using v8::Context;
-using v8::HandleScope;
-using v8::Isolate;
-using v8::Local;
-using v8::SnapshotCreator;
-using v8::StartupData;
-using v8::TryCatch;
-using v8::Value;
-
-template <typename T>
-void WriteVector(std::stringstream* ss, const T* vec, size_t size) {
-  for (size_t i = 0; i < size; i++) {
-    *ss << std::to_string(vec[i]) << (i == size - 1 ? '\n' : ',');
-  }
-}
-
-std::string FormatBlob(StartupData* blob,
-                       const std::vector<size_t>& isolate_data_indexes,
-                       const EnvSerializeInfo& env_info) {
-  std::stringstream ss;
-
-  ss << R"(#include <cstddef>
-#include "env.h"
-#include "node_main_instance.h"
-#include "v8.h"
-
-// This file is generated by tools/snapshot. Do not edit.
-
-namespace node {
-
-static const char blob_data[] = {
-)";
-  WriteVector(&ss, blob->data, blob->raw_size);
-  ss << R"(};
-
-static const int blob_size = )"
-     << blob->raw_size << R"(;
-static v8::StartupData blob = { blob_data, blob_size };
-)";
-
-  ss << R"(v8::StartupData* NodeMainInstance::GetEmbeddedSnapshotBlob() {
-  return &blob;
-}
-
-static const std::vector<size_t> isolate_data_indexes {
-)";
-  WriteVector(&ss, isolate_data_indexes.data(), isolate_data_indexes.size());
-  ss << R"(};
-
-const std::vector<size_t>* NodeMainInstance::GetIsolateDataIndexes() {
-  return &isolate_data_indexes;
-}
-
-static const EnvSerializeInfo env_info )"
-     << env_info << R"(;
-
-const EnvSerializeInfo* NodeMainInstance::GetEnvSerializeInfo() {
-  return &env_info;
-}
-
-}  // namespace node
-)";
-
-  return ss.str();
-}
-
-std::string SnapshotBuilder::Generate(
-    const std::vector<std::string> args,
-    const std::vector<std::string> exec_args) {
-  Isolate* isolate = Isolate::Allocate();
-  isolate->SetCaptureStackTraceForUncaughtExceptions(
-    true,
-    10,
-    v8::StackTrace::StackTraceOptions::kDetailed);
-  per_process::v8_platform.Platform()->RegisterIsolate(isolate,
-                                                       uv_default_loop());
-  std::unique_ptr<NodeMainInstance> main_instance;
-  std::string result;
-
-  {
-    std::vector<size_t> isolate_data_indexes;
-    EnvSerializeInfo env_info;
-
-    const std::vector<intptr_t>& external_references =
-        NodeMainInstance::CollectExternalReferences();
-    SnapshotCreator creator(isolate, external_references.data());
-    Environment* env;
-    {
-      main_instance =
-          NodeMainInstance::Create(isolate,
-                                   uv_default_loop(),
-                                   per_process::v8_platform.Platform(),
-                                   args,
-                                   exec_args);
-
-      HandleScope scope(isolate);
-      creator.SetDefaultContext(Context::New(isolate));
-      isolate_data_indexes = main_instance->isolate_data()->Serialize(&creator);
-
-      // Run the per-context scripts
-      Local<Context> context;
-      {
-        TryCatch bootstrapCatch(isolate);
-        context = NewContext(isolate);
-        if (bootstrapCatch.HasCaught()) {
-          PrintCaughtException(isolate, context, bootstrapCatch);
-          abort();
-        }
-      }
-      Context::Scope context_scope(context);
-
-      // Create the environment
-      env = new Environment(main_instance->isolate_data(),
-                            context,
-                            args,
-                            exec_args,
-                            nullptr,
-                            node::EnvironmentFlags::kDefaultFlags,
-                            {});
-      // Run scripts in lib/internal/bootstrap/
-      {
-        TryCatch bootstrapCatch(isolate);
-        v8::MaybeLocal<Value> result = env->RunBootstrapping();
-        if (bootstrapCatch.HasCaught()) {
-          PrintCaughtException(isolate, context, bootstrapCatch);
-        }
-        result.ToLocalChecked();
-      }
-
-      if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) {
-        env->PrintAllBaseObjects();
-        printf("Environment = %p\n", env);
-      }
-
-      // Serialize the native states
-      env_info = env->Serialize(&creator);
-      // Serialize the context
-      size_t index = creator.AddContext(
-          context, {SerializeNodeContextInternalFields, env});
-      CHECK_EQ(index, NodeMainInstance::kNodeContextIndex);
-    }
-
-    // Must be out of HandleScope
-    StartupData blob =
-        creator.CreateBlob(SnapshotCreator::FunctionCodeHandling::kClear);
-    CHECK(blob.CanBeRehashed());
-    // Must be done while the snapshot creator isolate is entered i.e. the
-    // creator is still alive.
-    FreeEnvironment(env);
-    main_instance->Dispose();
-    result = FormatBlob(&blob, isolate_data_indexes, env_info);
-    delete[] blob.data;
-  }
-
-  per_process::v8_platform.Platform()->UnregisterIsolate(isolate);
-  return result;
-}
-}  // namespace node

tools/snapshot/snapshot_builder.h

@@ -1,15 +0,0 @@
-#ifndef TOOLS_SNAPSHOT_SNAPSHOT_BUILDER_H_
-#define TOOLS_SNAPSHOT_SNAPSHOT_BUILDER_H_
-
-#include <string>
-#include <vector>
-
-namespace node {
-class SnapshotBuilder {
- public:
-  static std::string Generate(const std::vector<std::string> args,
-                              const std::vector<std::string> exec_args);
-};
-}  // namespace node
-
-#endif  // TOOLS_SNAPSHOT_SNAPSHOT_BUILDER_H_