fix(std/encoding/toml): Comment after arrays causing incorrect output (#7224)

std/encoding/testdata/arrays.toml

@@ -1,8 +1,8 @@
 [arrays]
-data = [ ["gamma", "delta"], [1, 2] ]
+data = [ ["gamma", "delta"], [1, 2] ] # comment after an array caused issue #7072
 
 # Line breaks are OK when inside arrays
 hosts = [
   "alpha",
   "omega"
-]
+] # comment

std/encoding/testdata/comment.toml

@@ -0,0 +1,29 @@
+# This is a full-line comment
+str0 = 'value' # This is a comment at the end of a line
+str1 = "# This is not a comment" # but this is
+str2 = """ # this is not a comment!
+A multiline string with a #
+# this is also not a comment
+""" # this is definitely a comment
+
+str3 = '''
+"# not a comment"
+	# this is a real tab on purpose 
+# not a comment
+''' # comment
+
+point0 = { x = 1, y = 2, str0 = "#not a comment", z = 3 } # comment
+point1 = { x = 7, y = 8, z = 9, str0 = "#not a comment"} # comment
+
+[deno] # this comment is fine
+features = ["#secure by default", "supports typescript # not a comment"] # Comment caused Issue #7072
+url = "https://deno.land/" # comment
+is_not_node = true # comment
+
+[toml] # Comment caused Issue #7072 (case 2)
+name = "Tom's Obvious, Minimal Language"
+objectives = [ # Comment
+ "easy to read", # Comment
+ "minimal config file", 
+ "#not a comment" # comment
+] # comment

std/encoding/toml.ts

@@ -32,14 +32,82 @@ class Parser {
     for (let i = 0; i < this.tomlLines.length; i++) {
       const s = this.tomlLines[i];
       const trimmed = s.trim();
-      if (trimmed !== "" && trimmed[0] !== "#") {
+      if (trimmed !== "") {
         out.push(s);
       }
     }
     this.tomlLines = out;
+    this._removeComments();
     this._mergeMultilines();
   }
 
+  _removeComments(): void {
+    function isFullLineComment(line: string) {
+      return line.match(/^#/) ? true : false;
+    }
+
+    function stringStart(line: string) {
+      const m = line.match(/(?:=\s*\[?\s*)("""|'''|"|')/);
+      if (!m) {
+        return false;
+      }
+
+      // We want to know which syntax was used to open the string
+      openStringSyntax = m[1];
+      return true;
+    }
+
+    function stringEnd(line: string) {
+      // match the syntax used to open the string when searching for string close
+      // e.g. if we open with ''' we must close with a '''
+      const reg = RegExp(`(?<!(=\\s*))${openStringSyntax}(?!(.*"))`);
+      if (!line.match(reg)) {
+        return false;
+      }
+
+      openStringSyntax = "";
+      return true;
+    }
+
+    const cleaned = [];
+    let isOpenString = false;
+    let openStringSyntax = "";
+    for (let i = 0; i < this.tomlLines.length; i++) {
+      const line = this.tomlLines[i];
+
+      // stringStart and stringEnd are separate conditions to
+      // support both single-line and multi-line strings
+      if (!isOpenString && stringStart(line)) {
+        isOpenString = true;
+      }
+      if (isOpenString && stringEnd(line)) {
+        isOpenString = false;
+      }
+
+      if (!isOpenString && !isFullLineComment(line)) {
+        const out = line.split(
+          /(?<=([\,\[\]\{\}]|".*"|'.*'|\w(?!.*("|')+))\s*)#/gi,
+        );
+        cleaned.push(out[0].trim());
+      } else if (isOpenString || !isFullLineComment(line)) {
+        cleaned.push(line);
+      }
+
+      // If a single line comment doesnt end on the same line, throw error
+      if (
+        isOpenString && (openStringSyntax === "'" || openStringSyntax === '"')
+      ) {
+        throw new TOMLError(`Single-line string is not closed:\n${line}`);
+      }
+    }
+
+    if (isOpenString) {
+      throw new TOMLError(`Incomplete string until EOF`);
+    }
+
+    this.tomlLines = cleaned;
+  }
+
   _mergeMultilines(): void {
     function arrayStart(line: string): boolean {
       const reg = /.*=\s*\[/g;
@@ -236,7 +304,7 @@ class Parser {
     if (invalidArr) {
       dataString = dataString.replace(/,]/g, "]");
     }
-    dataString = this._stripComment(dataString);
+
     if (dataString[0] === "{" && dataString[dataString.length - 1] === "}") {
       const reg = /([a-zA-Z0-9-_\.]*) (=)/gi;
       let result;
@@ -263,34 +331,6 @@ class Parser {
     }
     return eval(dataString);
   }
-  private _stripComment(dataString: string): string {
-    const isString = dataString.startsWith('"') || dataString.startsWith("'");
-    if (isString) {
-      const [quote] = dataString;
-      let indexOfNextQuote = 0;
-      while (
-        (indexOfNextQuote = dataString.indexOf(quote, indexOfNextQuote + 1)) !==
-          -1
-      ) {
-        const isEscaped = dataString[indexOfNextQuote - 1] === "\\";
-        if (!isEscaped) {
-          break;
-        }
-      }
-      if (indexOfNextQuote === -1) {
-        throw new TOMLError("imcomplete string literal");
-      }
-      const endOfString = indexOfNextQuote + 1;
-      return dataString.slice(0, endOfString);
-    }
-
-    const m = /(?:|\[|{).*(?:|\]|})\s*^((?!#).)*/g.exec(dataString);
-    if (m) {
-      return m[0].trim();
-    } else {
-      return dataString;
-    }
-  }
   _isLocalTime(str: string): boolean {
     const reg = /(\d{2}):(\d{2}):(\d{2})/;
     return reg.test(str);

std/encoding/toml_test.ts

@@ -413,3 +413,30 @@ the     = "array"
     assertEquals(actual, expected);
   },
 });
+
+Deno.test({
+  name: "[TOML] Comments",
+  fn: () => {
+    const expected = {
+      str0: "value",
+      str1: "# This is not a comment",
+      str2:
+        " # this is not a comment!\nA multiline string with a #\n# this is also not a comment",
+      str3:
+        '"# not a comment"\n\t# this is a real tab on purpose \n# not a comment',
+      point0: { x: 1, y: 2, str0: "#not a comment", z: 3 },
+      point1: { x: 7, y: 8, z: 9, str0: "#not a comment" },
+      deno: {
+        features: ["#secure by default", "supports typescript # not a comment"],
+        url: "https://deno.land/",
+        is_not_node: true,
+      },
+      toml: {
+        name: "Tom's Obvious, Minimal Language",
+        objectives: ["easy to read", "minimal config file", "#not a comment"],
+      },
+    };
+    const actual = parseFile(path.join(testFilesDir, "comment.toml"));
+    assertEquals(actual, expected);
+  },
+});