Fix protobufjs snapshotting.

deno2/BUILD.gn

@@ -98,7 +98,7 @@ run_node("bundle") {
 run_node("run_tsc") {
   main_source = "js/main.ts"
   sources = [
-    "js/msg.pb.d.ts",
+    #"js/msg.pb.d.ts",
     "js/msg.pb.js",
     "js/tsconfig.json",
     main_source,
@@ -178,6 +178,9 @@ template("create_snapshot") {
       rebase_path(natives_out_cc, root_build_dir),
       rebase_path(snapshot_out_cc, root_build_dir),
     ]
+
+    # To debug snapshotting problems:
+    #  args += ["--trace-serializer"]
     data = [
       invoker.js,
     ]

deno2/deno.cc

@@ -199,6 +199,19 @@ bool Execute(v8::Local<v8::Context> context, const char* js_filename,
   return true;
 }
 
+v8::StartupData SerializeInternalFields(v8::Local<v8::Object> holder, int index,
+                                        void* data) {
+  assert(data == nullptr);  // TODO(ry) pass Deno* object here.
+  InternalFieldData* embedder_field = static_cast<InternalFieldData*>(
+      holder->GetAlignedPointerFromInternalField(index));
+  if (embedder_field == nullptr) return {nullptr, 0};
+  int size = sizeof(*embedder_field);
+  char* payload = new char[size];
+  // We simply use memcpy to serialize the content.
+  memcpy(payload, embedder_field, size);
+  return {payload, size};
+}
+
 v8::StartupData MakeSnapshot(v8::StartupData* prev_natives_blob,
                              v8::StartupData* prev_snapshot_blob,
                              const char* js_filename, const char* js_source) {
@@ -232,7 +245,8 @@ v8::StartupData MakeSnapshot(v8::StartupData* prev_natives_blob,
     bool r = Execute(context, js_filename, js_source);
     assert(r);
 
-    creator->SetDefaultContext(context);
+    creator->SetDefaultContext(context, v8::SerializeInternalFieldsCallback(
+                                            SerializeInternalFields, nullptr));
   }
 
   auto snapshot_blob =

deno2/deno_internal.h

@@ -21,6 +21,10 @@ struct deno_s {
 
 namespace deno {
 
+struct InternalFieldData {
+  uint32_t data;
+};
+
 void Print(const v8::FunctionCallbackInfo<v8::Value>& args);
 void Sub(const v8::FunctionCallbackInfo<v8::Value>& args);
 void Pub(const v8::FunctionCallbackInfo<v8::Value>& args);

deno2/from_snapshot.cc

@@ -21,12 +21,28 @@ namespace deno {
 #include "snapshot_deno.cc"
 #endif
 
+std::vector<InternalFieldData*> deserialized_data;
+
+void DeserializeInternalFields(v8::Local<v8::Object> holder, int index,
+                               v8::StartupData payload, void* data) {
+  assert(data == nullptr);  // TODO(ry) pass Deno* object here.
+  if (payload.raw_size == 0) {
+    holder->SetAlignedPointerInInternalField(index, nullptr);
+    return;
+  }
+  InternalFieldData* embedder_field = new InternalFieldData{0};
+  memcpy(embedder_field, payload.data, payload.raw_size);
+  holder->SetAlignedPointerInInternalField(index, embedder_field);
+  deserialized_data.push_back(embedder_field);
+}
+
 Deno* NewFromSnapshot(void* data, deno_sub_cb cb) {
   auto natives_blob = *StartupBlob_natives();
   auto snapshot_blob = *StartupBlob_snapshot();
 
   v8::V8::SetNativesDataBlob(&natives_blob);
   v8::V8::SetSnapshotDataBlob(&snapshot_blob);
+  v8::DeserializeInternalFieldsCallback(DeserializeInternalFields, nullptr);
 
   Deno* d = new Deno;
   d->cb = cb;
@@ -42,7 +58,11 @@ Deno* NewFromSnapshot(void* data, deno_sub_cb cb) {
   v8::Isolate::Scope isolate_scope(isolate);
   {
     v8::HandleScope handle_scope(isolate);
-    auto context = v8::Context::New(isolate);
+    auto context =
+        v8::Context::New(isolate, nullptr, v8::MaybeLocal<v8::ObjectTemplate>(),
+                         v8::MaybeLocal<v8::Value>(),
+                         v8::DeserializeInternalFieldsCallback(
+                             DeserializeInternalFields, nullptr));
     d->context.Reset(d->isolate, context);
   }
 

deno2/js/main.ts

@@ -1,12 +1,13 @@
 /// <reference path="deno.d.ts" />
-//import { main as pb } from "./msg.pb"
+import { main as pb } from "./msg.pb";
 import * as ts from "typescript";
 
 const globalEval = eval;
 const window = globalEval("this");
 window["denoMain"] = () => {
-  //const msg = pb.Msg.fromObject({});
+  denoPrint("Hello world");
-  //denoPrint(`msg.command: ${msg.command}`);
+  const msg = pb.Msg.fromObject({});
+  denoPrint(`msg.command: ${msg.command}`);
   denoPrint(`ts.version: ${ts.version}`);
   denoPrint("Hello world from foo");
   return "foo";

deno2/js/mock_runtime.js

@@ -15,6 +15,17 @@ function CanCallFunction() {
   return "foo";
 }
 
+// This object is created to test snapshotting.
+// See DeserializeInternalFieldsCallback and SerializeInternalFieldsCallback.
+const snapshotted = new Uint8Array([1, 3, 3, 7]);
+
+function TypedArraySnapshots() {
+  assert(snapshotted[0] === 1);
+  assert(snapshotted[1] === 3);
+  assert(snapshotted[2] === 3);
+  assert(snapshotted[3] === 7);
+}
+
 function PubSuccess() {
   denoSub((channel, msg) => {
     assert(channel === "PubSuccess");

deno2/js/msg.pb.d.ts

@@ -2,17 +2,20 @@ import * as $protobuf from "protobufjs";
 
 /** Namespace main. */
 export namespace main {
+
     /** Properties of a BaseMsg. */
     interface IBaseMsg {
+
         /** BaseMsg channel */
-    channel?: string | null;
+        channel?: (string|null);
 
         /** BaseMsg payload */
-    payload?: Uint8Array | null;
+        payload?: (Uint8Array|null);
     }
 
     /** Represents a BaseMsg. */
     class BaseMsg implements IBaseMsg {
+
         /**
          * Constructs a new BaseMsg.
          * @param [properties] Properties to set
@@ -38,10 +41,7 @@ export namespace main {
          * @param [writer] Writer to encode to
          * @returns Writer
          */
-    public static encode(
+        public static encode(message: main.IBaseMsg, writer?: $protobuf.Writer): $protobuf.Writer;
-      message: main.IBaseMsg,
-      writer?: $protobuf.Writer
-    ): $protobuf.Writer;
 
         /**
          * Encodes the specified BaseMsg message, length delimited. Does not implicitly {@link main.BaseMsg.verify|verify} messages.
@@ -49,10 +49,7 @@ export namespace main {
          * @param [writer] Writer to encode to
          * @returns Writer
          */
-    public static encodeDelimited(
+        public static encodeDelimited(message: main.IBaseMsg, writer?: $protobuf.Writer): $protobuf.Writer;
-      message: main.IBaseMsg,
-      writer?: $protobuf.Writer
-    ): $protobuf.Writer;
 
         /**
          * Decodes a BaseMsg message from the specified reader or buffer.
@@ -62,10 +59,7 @@ export namespace main {
          * @throws {Error} If the payload is not a reader or valid buffer
          * @throws {$protobuf.util.ProtocolError} If required fields are missing
          */
-    public static decode(
+        public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): main.BaseMsg;
-      reader: $protobuf.Reader | Uint8Array,
-      length?: number
-    ): main.BaseMsg;
 
         /**
          * Decodes a BaseMsg message from the specified reader or buffer, length delimited.
@@ -74,16 +68,14 @@ export namespace main {
          * @throws {Error} If the payload is not a reader or valid buffer
          * @throws {$protobuf.util.ProtocolError} If required fields are missing
          */
-    public static decodeDelimited(
+        public static decodeDelimited(reader: ($protobuf.Reader|Uint8Array)): main.BaseMsg;
-      reader: $protobuf.Reader | Uint8Array
-    ): main.BaseMsg;
 
         /**
          * Verifies a BaseMsg message.
          * @param message Plain object to verify
          * @returns `null` if valid, otherwise the reason why it is not
          */
-    public static verify(message: { [k: string]: any }): string | null;
+        public static verify(message: { [k: string]: any }): (string|null);
 
         /**
          * Creates a BaseMsg message from a plain object. Also converts values to their respective internal types.
@@ -98,10 +90,7 @@ export namespace main {
          * @param [options] Conversion options
          * @returns Plain object
          */
-    public static toObject(
+        public static toObject(message: main.BaseMsg, options?: $protobuf.IConversionOptions): { [k: string]: any };
-      message: main.BaseMsg,
-      options?: $protobuf.IConversionOptions
-    ): { [k: string]: any };
 
         /**
          * Converts this BaseMsg to JSON.
@@ -112,111 +101,113 @@ export namespace main {
 
     /** Properties of a Msg. */
     interface IMsg {
+
         /** Msg command */
-    command?: main.Msg.Command | null;
+        command?: (main.Msg.Command|null);
 
         /** Msg error */
-    error?: string | null;
+        error?: (string|null);
 
         /** Msg startCwd */
-    startCwd?: string | null;
+        startCwd?: (string|null);
 
         /** Msg startArgv */
-    startArgv?: string[] | null;
+        startArgv?: (string[]|null);
 
         /** Msg startDebugFlag */
-    startDebugFlag?: boolean | null;
+        startDebugFlag?: (boolean|null);
 
         /** Msg startMainJs */
-    startMainJs?: string | null;
+        startMainJs?: (string|null);
 
         /** Msg startMainMap */
-    startMainMap?: string | null;
+        startMainMap?: (string|null);
 
         /** Msg codeFetchModuleSpecifier */
-    codeFetchModuleSpecifier?: string | null;
+        codeFetchModuleSpecifier?: (string|null);
 
         /** Msg codeFetchContainingFile */
-    codeFetchContainingFile?: string | null;
+        codeFetchContainingFile?: (string|null);
 
         /** Msg codeFetchResModuleName */
-    codeFetchResModuleName?: string | null;
+        codeFetchResModuleName?: (string|null);
 
         /** Msg codeFetchResFilename */
-    codeFetchResFilename?: string | null;
+        codeFetchResFilename?: (string|null);
 
         /** Msg codeFetchResSourceCode */
-    codeFetchResSourceCode?: string | null;
+        codeFetchResSourceCode?: (string|null);
 
         /** Msg codeFetchResOutputCode */
-    codeFetchResOutputCode?: string | null;
+        codeFetchResOutputCode?: (string|null);
 
         /** Msg codeCacheFilename */
-    codeCacheFilename?: string | null;
+        codeCacheFilename?: (string|null);
 
         /** Msg codeCacheSourceCode */
-    codeCacheSourceCode?: string | null;
+        codeCacheSourceCode?: (string|null);
 
         /** Msg codeCacheOutputCode */
-    codeCacheOutputCode?: string | null;
+        codeCacheOutputCode?: (string|null);
 
         /** Msg exitCode */
-    exitCode?: number | null;
+        exitCode?: (number|null);
 
         /** Msg timerStartId */
-    timerStartId?: number | null;
+        timerStartId?: (number|null);
 
         /** Msg timerStartInterval */
-    timerStartInterval?: boolean | null;
+        timerStartInterval?: (boolean|null);
 
         /** Msg timerStartDelay */
-    timerStartDelay?: number | null;
+        timerStartDelay?: (number|null);
 
         /** Msg timerReadyId */
-    timerReadyId?: number | null;
+        timerReadyId?: (number|null);
 
         /** Msg timerReadyDone */
-    timerReadyDone?: boolean | null;
+        timerReadyDone?: (boolean|null);
 
         /** Msg timerClearId */
-    timerClearId?: number | null;
+        timerClearId?: (number|null);
 
         /** Msg fetchReqId */
-    fetchReqId?: number | null;
+        fetchReqId?: (number|null);
 
         /** Msg fetchReqUrl */
-    fetchReqUrl?: string | null;
+        fetchReqUrl?: (string|null);
 
         /** Msg fetchResId */
-    fetchResId?: number | null;
+        fetchResId?: (number|null);
 
         /** Msg fetchResStatus */
-    fetchResStatus?: number | null;
+        fetchResStatus?: (number|null);
 
         /** Msg fetchResHeaderLine */
-    fetchResHeaderLine?: string[] | null;
+        fetchResHeaderLine?: (string[]|null);
 
         /** Msg fetchResBody */
-    fetchResBody?: Uint8Array | null;
+        fetchResBody?: (Uint8Array|null);
 
         /** Msg readFileSyncFilename */
-    readFileSyncFilename?: string | null;
+        readFileSyncFilename?: (string|null);
 
         /** Msg readFileSyncData */
-    readFileSyncData?: Uint8Array | null;
+        readFileSyncData?: (Uint8Array|null);
 
         /** Msg writeFileSyncFilename */
-    writeFileSyncFilename?: string | null;
+        writeFileSyncFilename?: (string|null);
 
         /** Msg writeFileSyncData */
-    writeFileSyncData?: Uint8Array | null;
+        writeFileSyncData?: (Uint8Array|null);
 
         /** Msg writeFileSyncPerm */
-    writeFileSyncPerm?: number | null;
+        writeFileSyncPerm?: (number|null);
     }
 
     /** Represents a Msg. */
     class Msg implements IMsg {
+
         /**
          * Constructs a new Msg.
          * @param [properties] Properties to set
@@ -338,10 +329,7 @@ export namespace main {
          * @param [writer] Writer to encode to
          * @returns Writer
          */
-    public static encode(
+        public static encode(message: main.IMsg, writer?: $protobuf.Writer): $protobuf.Writer;
-      message: main.IMsg,
-      writer?: $protobuf.Writer
-    ): $protobuf.Writer;
 
         /**
          * Encodes the specified Msg message, length delimited. Does not implicitly {@link main.Msg.verify|verify} messages.
@@ -349,10 +337,7 @@ export namespace main {
          * @param [writer] Writer to encode to
          * @returns Writer
          */
-    public static encodeDelimited(
+        public static encodeDelimited(message: main.IMsg, writer?: $protobuf.Writer): $protobuf.Writer;
-      message: main.IMsg,
-      writer?: $protobuf.Writer
-    ): $protobuf.Writer;
 
         /**
          * Decodes a Msg message from the specified reader or buffer.
@@ -362,10 +347,7 @@ export namespace main {
          * @throws {Error} If the payload is not a reader or valid buffer
          * @throws {$protobuf.util.ProtocolError} If required fields are missing
          */
-    public static decode(
+        public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): main.Msg;
-      reader: $protobuf.Reader | Uint8Array,
-      length?: number
-    ): main.Msg;
 
         /**
          * Decodes a Msg message from the specified reader or buffer, length delimited.
@@ -374,16 +356,14 @@ export namespace main {
          * @throws {Error} If the payload is not a reader or valid buffer
          * @throws {$protobuf.util.ProtocolError} If required fields are missing
          */
-    public static decodeDelimited(
+        public static decodeDelimited(reader: ($protobuf.Reader|Uint8Array)): main.Msg;
-      reader: $protobuf.Reader | Uint8Array
-    ): main.Msg;
 
         /**
          * Verifies a Msg message.
          * @param message Plain object to verify
          * @returns `null` if valid, otherwise the reason why it is not
          */
-    public static verify(message: { [k: string]: any }): string | null;
+        public static verify(message: { [k: string]: any }): (string|null);
 
         /**
          * Creates a Msg message from a plain object. Also converts values to their respective internal types.
@@ -398,10 +378,7 @@ export namespace main {
          * @param [options] Conversion options
          * @returns Plain object
          */
-    public static toObject(
+        public static toObject(message: main.Msg, options?: $protobuf.IConversionOptions): { [k: string]: any };
-      message: main.Msg,
-      options?: $protobuf.IConversionOptions
-    ): { [k: string]: any };
 
         /**
          * Converts this Msg to JSON.
@@ -411,6 +388,7 @@ export namespace main {
     }
 
     namespace Msg {
+
         /** Command enum. */
         enum Command {
             ERROR = 0,

deno2/js/pbjs_hack.py

@@ -10,6 +10,7 @@ import sys
 import os
 
 js_path = os.path.dirname(os.path.realpath(__file__))
+#bin_path = os.path.join(js_path, "deno_protobufjs", "bin")
 bin_path = os.path.join(js_path, "node_modules", ".bin")
 pbjs_bin = os.path.join(bin_path, "pbjs")
 pbts_bin = os.path.join(bin_path, "pbts")
@@ -28,8 +29,8 @@ def touch(fname):
     open(fname, 'a').close()
 
 subprocess.check_call([
-  "node",
   pbjs_bin,
+  #"--dependency=./deno_protobufjs/minimal",
   "--target=static-module",
   "--wraper=commonjs",
   "--out=" + msg_pbjs_out,
@@ -45,5 +46,4 @@ subprocess.check_call([
 ])
 assert os.path.exists(msg_pbts_out)
 
-
 touch(stamp_file)

deno2/mock_runtime_test.cc

@@ -85,6 +85,12 @@ TEST(MockRuntimeTest, DoubleSubFails) {
   deno_delete(d);
 }
 
+TEST(MockRuntimeTest, TypedArraySnapshots) {
+  Deno* d = deno_new(NULL, NULL);
+  EXPECT_TRUE(deno_execute(d, "a.js", "TypedArraySnapshots()"));
+  deno_delete(d);
+}
+
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
   deno_init();

deno2/snapshot_creator.cc

@@ -1,5 +1,6 @@
 // Copyright 2018 Ryan Dahl <ry@tinyclouds.org>
 // All rights reserved. MIT License.
+// Hint: --trace_serializer is a useful debugging flag.
 #include <assert.h>
 #include <stdio.h>
 #include <string.h>
@@ -126,6 +127,8 @@ int main(int argc, char** argv) {
   const char* natives_out_cc = argv[4];
   const char* snapshot_out_cc = argv[5];
 
+  v8::V8::SetFlagsFromCommandLine(&argc, argv, true);
+
   auto js_data = ReadFile(js_fn);
   auto natives_blob = ReadFile(natives_in_bin);
   auto snapshot_in_blob = ReadFile(snapshot_in_bin);

deno2/tools/format.sh

@@ -4,4 +4,12 @@ clang-format -i -style Google *.cc *.h include/*.h
 gn format BUILD.gn
 gn format .gn
 yapf -i tools/*.py
-prettier --write js/*.ts js/*.js js/*.json
+prettier --write \
+	js/deno.d.ts \
+  js/main.ts \
+  js/mock_runtime.js \
+  js/package.json \
+  js/tsconfig.json
+# Do not format these. 
+#  js/msg.pb.js
+#  js/msg.pb.d.ts