libdeno: expose dynamic import (#2461)

core/libdeno/binding.cc

@@ -509,6 +509,38 @@ void HostInitializeImportMetaObjectCallback(v8::Local<v8::Context> context,
   meta->CreateDataProperty(context, v8_str("main"), v8_bool(main)).ToChecked();
 }
 
+v8::MaybeLocal<v8::Promise> HostImportModuleDynamicallyCallback(
+    v8::Local<v8::Context> context, v8::Local<v8::ScriptOrModule> referrer,
+    v8::Local<v8::String> specifier) {
+  auto* isolate = context->GetIsolate();
+  DenoIsolate* d = DenoIsolate::FromIsolate(isolate);
+  v8::Isolate::Scope isolate_scope(isolate);
+
+  v8::String::Utf8Value specifier_str(isolate, specifier);
+
+  auto referrer_name = referrer->GetResourceName();
+  v8::String::Utf8Value referrer_name_str(isolate, referrer_name);
+
+  // TODO(ry) I'm not sure what HostDefinedOptions is for or if we're ever going
+  // to use it. For now we check that it is not used. This check may need to be
+  // changed in the future.
+  auto host_defined_options = referrer->GetHostDefinedOptions();
+  CHECK_EQ(host_defined_options->Length(), 0);
+
+  v8::Local<v8::Promise::Resolver> resolver =
+      v8::Promise::Resolver::New(context).ToLocalChecked();
+
+  deno_dyn_import_id import_id = d->next_dyn_import_id_++;
+
+  d->dyn_import_map_.emplace(std::piecewise_construct,
+                             std::make_tuple(import_id),
+                             std::make_tuple(d->isolate_, resolver));
+
+  d->dyn_import_cb_(d->user_data_, *specifier_str, *referrer_name_str,
+                    import_id);
+  return resolver->GetPromise();
+}
+
 void DenoIsolate::AddIsolate(v8::Isolate* isolate) {
   isolate_ = isolate;
   isolate_->SetCaptureStackTraceForUncaughtExceptions(
@@ -518,6 +550,8 @@ void DenoIsolate::AddIsolate(v8::Isolate* isolate) {
   isolate_->AddMessageListener(MessageCallback);
   isolate->SetHostInitializeImportMetaObjectCallback(
       HostInitializeImportMetaObjectCallback);
+  isolate->SetHostImportModuleDynamicallyCallback(
+      HostImportModuleDynamicallyCallback);
 }
 
 }  // namespace deno

core/libdeno/deno.h

@@ -34,6 +34,13 @@ typedef struct deno_s Deno;
 typedef void (*deno_recv_cb)(void* user_data, deno_buf control_buf,
                              deno_pinned_buf zero_copy_buf);
 
+typedef int deno_dyn_import_id;
+// Called when dynamic import is called in JS: import('foo')
+// Embedder must call deno_dyn_import() with the specified id and
+// the module.
+typedef void (*deno_dyn_import_cb)(void* user_data, const char* specifier,
+                                   const char* referrer, deno_dyn_import_id id);
+
 void deno_init();
 const char* deno_v8_version();
 void deno_set_v8_flags(int* argc, char** argv);
@@ -43,6 +50,7 @@ typedef struct {
   deno_snapshot load_snapshot;  // A startup snapshot to use.
   deno_buf shared;              // Shared buffer to be mapped to libdeno.shared
   deno_recv_cb recv_cb;         // Maps to libdeno.send() calls.
+  deno_dyn_import_cb dyn_import_cb;
 } deno_config;
 
 // Create a new deno isolate.
@@ -117,6 +125,9 @@ void deno_mod_instantiate(Deno* d, void* user_data, deno_mod id,
 // If it succeeded deno_last_exception() will return NULL.
 void deno_mod_evaluate(Deno* d, void* user_data, deno_mod id);
 
+// Call exactly once for every deno_dyn_import_cb.
+void deno_dyn_import(Deno* d, deno_dyn_import_id id, deno_mod mod_id);
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif

core/libdeno/internal.h

@@ -40,6 +40,8 @@ class DenoIsolate {
         recv_cb_(config.recv_cb),
         user_data_(nullptr),
         resolve_cb_(nullptr),
+        next_dyn_import_id_(0),
+        dyn_import_cb_(config.dyn_import_cb),
         has_snapshotted_(false) {
     if (config.load_snapshot.data_ptr) {
       snapshot_.data =
@@ -101,6 +103,11 @@ class DenoIsolate {
   std::map<std::string, deno_mod> mods_by_name_;
   deno_resolve_cb resolve_cb_;
 
+  deno_dyn_import_id next_dyn_import_id_;
+  deno_dyn_import_cb dyn_import_cb_;
+  std::map<deno_dyn_import_id, v8::Persistent<v8::Promise::Resolver>>
+      dyn_import_map_;
+
   v8::Persistent<v8::Context> context_;
   std::map<int, v8::Persistent<v8::Value>> pending_promise_map_;
   std::string last_exception_;

core/libdeno/libdeno_test.cc

@@ -3,20 +3,20 @@
 
 TEST(LibDenoTest, InitializesCorrectly) {
   EXPECT_NE(snapshot.data_ptr, nullptr);
-  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr, nullptr});
   deno_execute(d, nullptr, "a.js", "1 + 2");
   EXPECT_EQ(nullptr, deno_last_exception(d));
   deno_delete(d);
 }
 
 TEST(LibDenoTest, Snapshotter) {
-  Deno* d1 = deno_new(deno_config{1, empty_snapshot, empty, nullptr});
+  Deno* d1 = deno_new(deno_config{1, empty_snapshot, empty, nullptr, nullptr});
   deno_execute(d1, nullptr, "a.js", "a = 1 + 2");
   EXPECT_EQ(nullptr, deno_last_exception(d1));
   deno_snapshot test_snapshot = deno_snapshot_new(d1);
   deno_delete(d1);
 
-  Deno* d2 = deno_new(deno_config{0, test_snapshot, empty, nullptr});
+  Deno* d2 = deno_new(deno_config{0, test_snapshot, empty, nullptr, nullptr});
   deno_execute(d2, nullptr, "b.js", "if (a != 3) throw Error('x');");
   EXPECT_EQ(nullptr, deno_last_exception(d2));
   deno_delete(d2);
@@ -25,7 +25,7 @@ TEST(LibDenoTest, Snapshotter) {
 }
 
 TEST(LibDenoTest, CanCallFunction) {
-  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr, nullptr});
   deno_lock(d);
   deno_execute(d, nullptr, "a.js",
                "if (CanCallFunction() != 'foo') throw Error();");
@@ -35,7 +35,7 @@ TEST(LibDenoTest, CanCallFunction) {
 }
 
 TEST(LibDenoTest, ErrorsCorrectly) {
-  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr, nullptr});
   deno_execute(d, nullptr, "a.js", "throw Error()");
   EXPECT_NE(nullptr, deno_last_exception(d));
   deno_delete(d);
@@ -57,7 +57,7 @@ TEST(LibDenoTest, RecvReturnEmpty) {
     EXPECT_EQ(buf.data_ptr[1], 'b');
     EXPECT_EQ(buf.data_ptr[2], 'c');
   };
-  Deno* d = deno_new(deno_config{0, snapshot, empty, recv_cb});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, recv_cb, nullptr});
   deno_execute(d, nullptr, "a.js", "RecvReturnEmpty()");
   EXPECT_EQ(nullptr, deno_last_exception(d));
   EXPECT_EQ(count, 2);
@@ -77,7 +77,7 @@ TEST(LibDenoTest, RecvReturnBar) {
     uint8_t response[] = {'b', 'a', 'r'};
     deno_respond(d, user_data, {response, sizeof response});
   };
-  Deno* d = deno_new(deno_config{0, snapshot, empty, recv_cb});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, recv_cb, nullptr});
   deno_execute(d, d, "a.js", "RecvReturnBar()");
   EXPECT_EQ(nullptr, deno_last_exception(d));
   EXPECT_EQ(count, 1);
@@ -85,28 +85,28 @@ TEST(LibDenoTest, RecvReturnBar) {
 }
 
 TEST(LibDenoTest, DoubleRecvFails) {
-  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr, nullptr});
   deno_execute(d, nullptr, "a.js", "DoubleRecvFails()");
   EXPECT_NE(nullptr, deno_last_exception(d));
   deno_delete(d);
 }
 
 TEST(LibDenoTest, TypedArraySnapshots) {
-  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr, nullptr});
   deno_execute(d, nullptr, "a.js", "TypedArraySnapshots()");
   EXPECT_EQ(nullptr, deno_last_exception(d));
   deno_delete(d);
 }
 
 TEST(LibDenoTest, SnapshotBug) {
-  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr, nullptr});
   deno_execute(d, nullptr, "a.js", "SnapshotBug()");
   EXPECT_EQ(nullptr, deno_last_exception(d));
   deno_delete(d);
 }
 
 TEST(LibDenoTest, GlobalErrorHandling) {
-  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr, nullptr});
   deno_execute(d, nullptr, "a.js", "GlobalErrorHandling()");
   std::string expected =
       "{\"message\":\"Uncaught ReferenceError: notdefined is not defined\","
@@ -141,7 +141,7 @@ TEST(LibDenoTest, ZeroCopyBuf) {
     // the API here.
     deno_pinned_buf_delete(&zero_copy_buf);
   };
-  Deno* d = deno_new(deno_config{0, snapshot, empty, recv_cb});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, recv_cb, nullptr});
   deno_execute(d, d, "a.js", "ZeroCopyBuf()");
   EXPECT_EQ(nullptr, deno_last_exception(d));
   EXPECT_EQ(count, 1);
@@ -155,7 +155,7 @@ TEST(LibDenoTest, ZeroCopyBuf) {
 TEST(LibDenoTest, CheckPromiseErrors) {
   static int count = 0;
   auto recv_cb = [](auto _, auto buf, auto zero_copy_buf) { count++; };
-  Deno* d = deno_new(deno_config{0, snapshot, empty, recv_cb});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, recv_cb, nullptr});
   EXPECT_EQ(deno_last_exception(d), nullptr);
   deno_execute(d, nullptr, "a.js", "CheckPromiseErrors()");
   EXPECT_EQ(nullptr, deno_last_exception(d));
@@ -169,7 +169,7 @@ TEST(LibDenoTest, CheckPromiseErrors) {
 }
 
 TEST(LibDenoTest, LastException) {
-  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, nullptr});
+  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, nullptr, nullptr});
   EXPECT_EQ(deno_last_exception(d), nullptr);
   deno_execute(d, nullptr, "a.js", "\n\nthrow Error('boo');\n\n");
   EXPECT_STREQ(deno_last_exception(d),
@@ -184,7 +184,7 @@ TEST(LibDenoTest, LastException) {
 }
 
 TEST(LibDenoTest, EncodeErrorBug) {
-  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, nullptr});
+  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, nullptr, nullptr});
   EXPECT_EQ(deno_last_exception(d), nullptr);
   deno_execute(d, nullptr, "a.js", "eval('a')");
   EXPECT_STREQ(
@@ -203,7 +203,7 @@ TEST(LibDenoTest, EncodeErrorBug) {
 TEST(LibDenoTest, Shared) {
   uint8_t s[] = {0, 1, 2};
   deno_buf shared = {s, sizeof s};
-  Deno* d = deno_new(deno_config{0, snapshot, shared, nullptr});
+  Deno* d = deno_new(deno_config{0, snapshot, shared, nullptr, nullptr});
   deno_execute(d, nullptr, "a.js", "Shared()");
   EXPECT_EQ(nullptr, deno_last_exception(d));
   EXPECT_EQ(s[0], 42);
@@ -213,7 +213,7 @@ TEST(LibDenoTest, Shared) {
 }
 
 TEST(LibDenoTest, Utf8Bug) {
-  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, nullptr});
+  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, nullptr, nullptr});
   // The following is a valid UTF-8 javascript which just defines a string
   // literal. We had a bug where libdeno would choke on this.
   deno_execute(d, nullptr, "a.js", "x = \"\xEF\xBF\xBD\"");
@@ -222,14 +222,14 @@ TEST(LibDenoTest, Utf8Bug) {
 }
 
 TEST(LibDenoTest, LibDenoEvalContext) {
-  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr, nullptr});
   deno_execute(d, nullptr, "a.js", "LibDenoEvalContext();");
   EXPECT_EQ(nullptr, deno_last_exception(d));
   deno_delete(d);
 }
 
 TEST(LibDenoTest, LibDenoEvalContextError) {
-  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr});
+  Deno* d = deno_new(deno_config{0, snapshot, empty, nullptr, nullptr});
   deno_execute(d, nullptr, "a.js", "LibDenoEvalContextError();");
   EXPECT_EQ(nullptr, deno_last_exception(d));
   deno_delete(d);
@@ -238,7 +238,7 @@ TEST(LibDenoTest, LibDenoEvalContextError) {
 TEST(LibDenoTest, SharedAtomics) {
   int32_t s[] = {0, 1, 2};
   deno_buf shared = {reinterpret_cast<uint8_t*>(s), sizeof s};
-  Deno* d = deno_new(deno_config{0, empty_snapshot, shared, nullptr});
+  Deno* d = deno_new(deno_config{0, empty_snapshot, shared, nullptr, nullptr});
   deno_execute(d, nullptr, "a.js",
                "Atomics.add(new Int32Array(Deno.core.shared), 0, 1)");
   EXPECT_EQ(nullptr, deno_last_exception(d));

core/libdeno/modules.cc

@@ -151,4 +151,37 @@ void deno_mod_evaluate(Deno* d_, void* user_data, deno_mod id) {
   }
 }
 
+void deno_dyn_import(Deno* d_, deno_dyn_import_id import_id, deno_mod mod_id) {
+  auto* d = unwrap(d_);
+  auto* isolate = d->isolate_;
+  v8::Isolate::Scope isolate_scope(isolate);
+  v8::Locker locker(isolate);
+  v8::HandleScope handle_scope(isolate);
+  auto context = d->context_.Get(d->isolate_);
+
+  auto it = d->dyn_import_map_.find(import_id);
+  if (it == d->dyn_import_map_.end()) {
+    CHECK(false);  // TODO(ry) error on bad import_id.
+    return;
+  }
+
+  /// Resolve.
+  auto persistent_promise = &it->second;
+  auto promise = persistent_promise->Get(isolate);
+  persistent_promise->Reset();
+
+  auto* info = d->GetModuleInfo(mod_id);
+  if (info == nullptr) {
+    // Resolution error.
+    promise->Reject(context, v8::Null(isolate)).ToChecked();
+  } else {
+    // Resolution success
+    Local<Module> module = info->handle.Get(isolate);
+    CHECK_GE(module->GetStatus(), v8::Module::kInstantiated);
+    Local<Value> module_namespace = module->GetModuleNamespace();
+    promise->Resolve(context, module_namespace).ToChecked();
+  }
+  d->dyn_import_map_.erase(it);
+}
+
 }  // extern "C"

core/libdeno/modules_test.cc

@@ -14,7 +14,7 @@ void recv_cb(void* user_data, deno_buf buf, deno_pinned_buf zero_copy_buf) {
 
 TEST(ModulesTest, Resolution) {
   exec_count = 0;  // Reset
-  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, recv_cb});
+  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, recv_cb, nullptr});
   EXPECT_EQ(0, exec_count);
 
   static deno_mod a = deno_mod_new(d, true, "a.js",
@@ -67,7 +67,7 @@ TEST(ModulesTest, Resolution) {
 
 TEST(ModulesTest, ResolutionError) {
   exec_count = 0;  // Reset
-  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, recv_cb});
+  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, recv_cb, nullptr});
   EXPECT_EQ(0, exec_count);
 
   static deno_mod a = deno_mod_new(d, true, "a.js",
@@ -100,7 +100,7 @@ TEST(ModulesTest, ResolutionError) {
 
 TEST(ModulesTest, ImportMetaUrl) {
   exec_count = 0;  // Reset
-  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, recv_cb});
+  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, recv_cb, nullptr});
   EXPECT_EQ(0, exec_count);
 
   static deno_mod a =
@@ -120,7 +120,7 @@ TEST(ModulesTest, ImportMetaUrl) {
 }
 
 TEST(ModulesTest, ImportMetaMain) {
-  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, recv_cb});
+  Deno* d = deno_new(deno_config{0, empty_snapshot, empty, recv_cb, nullptr});
 
   const char* throw_not_main_src = "if (!import.meta.main) throw 'err'";
   static deno_mod throw_not_main =
@@ -147,3 +147,78 @@ TEST(ModulesTest, ImportMetaMain) {
 
   deno_delete(d);
 }
+
+TEST(ModulesTest, DynamicImportSuccess) {
+  exec_count = 0;
+  static int dyn_import_count = 0;
+  static deno_mod b = 0;
+  auto dyn_import_cb = [](auto user_data, const char* specifier,
+                          const char* referrer, deno_dyn_import_id import_id) {
+    auto d = reinterpret_cast<Deno*>(user_data);
+    dyn_import_count++;
+    EXPECT_STREQ(specifier, "foo");
+    EXPECT_STREQ(referrer, "a.js");
+    deno_dyn_import(d, import_id, b);
+  };
+  const char* src =
+      "(async () => { \n"
+      "  let mod = await import('foo'); \n"
+      "  assert(mod.b() === 'b'); \n"
+      // Send a message to signify that we're done.
+      "  Deno.core.send(new Uint8Array([4])); \n"
+      "})(); \n";
+  Deno* d = deno_new(deno_config{0, snapshot, empty, recv_cb, dyn_import_cb});
+  static deno_mod a = deno_mod_new(d, true, "a.js", src);
+  EXPECT_NE(a, 0);
+  EXPECT_EQ(nullptr, deno_last_exception(d));
+  deno_mod_instantiate(d, d, a, nullptr);
+  EXPECT_EQ(nullptr, deno_last_exception(d));
+  const char* b_src = "export function b() { return 'b' }";
+  b = deno_mod_new(d, false, "b.js", b_src);
+  EXPECT_NE(b, 0);
+  EXPECT_EQ(nullptr, deno_last_exception(d));
+  deno_mod_instantiate(d, d, b, nullptr);
+  EXPECT_EQ(nullptr, deno_last_exception(d));
+  deno_mod_evaluate(d, d, a);
+  EXPECT_EQ(nullptr, deno_last_exception(d));
+  deno_check_promise_errors(d);
+  EXPECT_EQ(deno_last_exception(d), nullptr);
+  deno_delete(d);
+  EXPECT_EQ(1, exec_count);
+  EXPECT_EQ(1, dyn_import_count);
+}
+
+TEST(ModulesTest, DynamicImportError) {
+  exec_count = 0;
+  static int dyn_import_count = 0;
+  auto dyn_import_cb = [](auto user_data, const char* specifier,
+                          const char* referrer, deno_dyn_import_id import_id) {
+    auto d = reinterpret_cast<Deno*>(user_data);
+    dyn_import_count++;
+    EXPECT_STREQ(specifier, "foo");
+    EXPECT_STREQ(referrer, "a.js");
+    // We indicate there was an error resolving by returning mod_id 0.
+    deno_dyn_import(d, import_id, 0);
+  };
+  const char* src =
+      "(async () => { \n"
+      "  let mod = await import('foo'); \n"
+      // The following should be unreachable.
+      "  Deno.core.send(new Uint8Array([4])); \n"
+      "})(); \n";
+  Deno* d = deno_new(deno_config{0, snapshot, empty, recv_cb, dyn_import_cb});
+  static deno_mod a = deno_mod_new(d, true, "a.js", src);
+  EXPECT_NE(a, 0);
+  EXPECT_EQ(nullptr, deno_last_exception(d));
+  deno_mod_instantiate(d, d, a, nullptr);
+  EXPECT_EQ(nullptr, deno_last_exception(d));
+  // No error when evaluating, because it's an async error.
+  deno_mod_evaluate(d, d, a);
+  EXPECT_EQ(nullptr, deno_last_exception(d));
+  // Now we should get an error.
+  deno_check_promise_errors(d);
+  EXPECT_NE(deno_last_exception(d), nullptr);
+  deno_delete(d);
+  EXPECT_EQ(0, exec_count);
+  EXPECT_EQ(1, dyn_import_count);
+}

core/libdeno/snapshot_creator.cc

@@ -21,7 +21,8 @@ int main(int argc, char** argv) {
   CHECK(deno::ReadFileToString(js_fn, &js_source));
 
   deno_init();
-  deno_config config = {1, deno::empty_snapshot, deno::empty_buf, nullptr};
+  deno_config config = {1, deno::empty_snapshot, deno::empty_buf, nullptr,
+                        nullptr};
   Deno* d = deno_new(config);
 
   deno_execute(d, nullptr, js_fn, js_source.c_str());