fix(csv-parse): only match delimiters on utf16le code unit boundaries

packages/csv-parse/lib/api/index.js

@@ -806,6 +806,16 @@ const transform = function (original_options = {}) {
       }
       return 0;
     },
+    // Multi-byte encodings such as utf16le represent each code unit on
+    // several bytes. A delimiter is only a delimiter when it starts on a
+    // code unit boundary: matched at any other offset, it would split a
+    // valid character into an empty record or field.
+    __isByteAligned: function (pos) {
+      return (
+        this.options.encoding !== "utf16le" ||
+        (this.state.bufBytesStart + pos) % 2 === 0
+      );
+    },
     __isDelimiter: function (buf, pos, chr) {
       const { delimiter, ignore_last_delimiters } = this.options;
       if (
@@ -820,6 +830,9 @@ const transform = function (original_options = {}) {
       ) {
         return 0;
       }
+      if (!this.__isByteAligned(pos)) {
+        return 0;
+      }
       loop1: for (let i = 0; i < delimiter.length; i++) {
         const del = delimiter[i];
         if (del[0] === chr) {
@@ -867,6 +880,9 @@ const transform = function (original_options = {}) {
     },
     __isRecordDelimiter: function (chr, buf, pos) {
       const { record_delimiter } = this.options;
+      if (!this.__isByteAligned(pos)) {
+        return 0;
+      }
       const recordDelimiterLength = record_delimiter.length;
       loop1: for (let i = 0; i < recordDelimiterLength; i++) {
         const rd = record_delimiter[i];
@@ -885,6 +901,9 @@ const transform = function (original_options = {}) {
     },
     __autoDiscoverRecordDelimiter: function (buf, pos) {
       const { encoding } = this.options;
+      if (!this.__isByteAligned(pos)) {
+        return 0;
+      }
       // Note, we don't need to cache this information in state,
       // It is only called on the first line until we find out a suitable
       // record delimiter.

packages/csv-parse/test/option.encoding.ts

@@ -12,6 +12,44 @@ describe("Option `encoding`", function () {
     });
   });
 
+  describe("unaligned multi-byte delimiter matches", function () {
+    // In utf16le, a code unit is 2 bytes: a delimiter is only a delimiter
+    // when it starts on a code unit boundary. The bytes below spell a valid
+    // delimiter halfway through a code unit; the value must survive intact.
+    it("does not match a record delimiter inside a code unit", function (next) {
+      parse(Buffer.from("\u0a01\u0100", "utf16le"), {
+        encoding: "utf16le",
+        record_delimiter: "\n",
+      }, (err, records) => {
+        if (err) return next(err);
+        records.should.eql([["\u0a01\u0100"]]);
+        next();
+      });
+    });
+
+    it("does not match a carriage return inside a code unit", function (next) {
+      parse(Buffer.from("\u0d01\u0100", "utf16le"), {
+        encoding: "utf16le",
+        record_delimiter: "\r",
+      }, (err, records) => {
+        if (err) return next(err);
+        records.should.eql([["\u0d01\u0100"]]);
+        next();
+      });
+    });
+
+    it("does not match a field delimiter inside a code unit", function (next) {
+      parse(Buffer.from("\u2c01\u0100", "utf16le"), {
+        encoding: "utf16le",
+        delimiter: ",",
+      }, (err, records) => {
+        if (err) return next(err);
+        records.should.eql([["\u2c01\u0100"]]);
+        next();
+      });
+    });
+  });
+
   describe("definition", function () {
     it("with delimiter", function (next) {
       parse(