diff --git a/packages/csv-parse/lib/api/index.js b/packages/csv-parse/lib/api/index.js index ca97838cd..ce97f899d 100644 --- a/packages/csv-parse/lib/api/index.js +++ b/packages/csv-parse/lib/api/index.js @@ -806,6 +806,16 @@ const transform = function (original_options = {}) { } return 0; }, + // Multi-byte encodings such as utf16le represent each code unit on + // several bytes. A delimiter is only a delimiter when it starts on a + // code unit boundary: matched at any other offset, it would split a + // valid character into an empty record or field. + __isByteAligned: function (pos) { + return ( + this.options.encoding !== "utf16le" || + (this.state.bufBytesStart + pos) % 2 === 0 + ); + }, __isDelimiter: function (buf, pos, chr) { const { delimiter, ignore_last_delimiters } = this.options; if ( @@ -820,6 +830,9 @@ const transform = function (original_options = {}) { ) { return 0; } + if (!this.__isByteAligned(pos)) { + return 0; + } loop1: for (let i = 0; i < delimiter.length; i++) { const del = delimiter[i]; if (del[0] === chr) { @@ -867,6 +880,9 @@ const transform = function (original_options = {}) { }, __isRecordDelimiter: function (chr, buf, pos) { const { record_delimiter } = this.options; + if (!this.__isByteAligned(pos)) { + return 0; + } const recordDelimiterLength = record_delimiter.length; loop1: for (let i = 0; i < recordDelimiterLength; i++) { const rd = record_delimiter[i]; @@ -885,6 +901,9 @@ const transform = function (original_options = {}) { }, __autoDiscoverRecordDelimiter: function (buf, pos) { const { encoding } = this.options; + if (!this.__isByteAligned(pos)) { + return 0; + } // Note, we don't need to cache this information in state, // It is only called on the first line until we find out a suitable // record delimiter. diff --git a/packages/csv-parse/test/option.encoding.ts b/packages/csv-parse/test/option.encoding.ts index 1d3c53491..9f47b3361 100644 --- a/packages/csv-parse/test/option.encoding.ts +++ b/packages/csv-parse/test/option.encoding.ts @@ -12,6 +12,44 @@ describe("Option `encoding`", function () { }); }); + describe("unaligned multi-byte delimiter matches", function () { + // In utf16le, a code unit is 2 bytes: a delimiter is only a delimiter + // when it starts on a code unit boundary. The bytes below spell a valid + // delimiter halfway through a code unit; the value must survive intact. + it("does not match a record delimiter inside a code unit", function (next) { + parse(Buffer.from("\u0a01\u0100", "utf16le"), { + encoding: "utf16le", + record_delimiter: "\n", + }, (err, records) => { + if (err) return next(err); + records.should.eql([["\u0a01\u0100"]]); + next(); + }); + }); + + it("does not match a carriage return inside a code unit", function (next) { + parse(Buffer.from("\u0d01\u0100", "utf16le"), { + encoding: "utf16le", + record_delimiter: "\r", + }, (err, records) => { + if (err) return next(err); + records.should.eql([["\u0d01\u0100"]]); + next(); + }); + }); + + it("does not match a field delimiter inside a code unit", function (next) { + parse(Buffer.from("\u2c01\u0100", "utf16le"), { + encoding: "utf16le", + delimiter: ",", + }, (err, records) => { + if (err) return next(err); + records.should.eql([["\u2c01\u0100"]]); + next(); + }); + }); + }); + describe("definition", function () { it("with delimiter", function (next) { parse(