fix(pg-protocol): decode binary result fields as raw bytes instead of utf-8

packages/pg-protocol/src/parser.ts

@@ -83,6 +83,8 @@ export class Parser {
   private bufferOffset: number = 0
   private reader = new BufferReader()
   private mode: Mode
+  // Column formats from the most recent RowDescription, or null before any has been seen.
+  private rowDescriptionFormats: Mode[] | null = null
 
   constructor(opts?: StreamOptions) {
     if (opts?.mode === 'binary') {
@@ -188,7 +190,7 @@ export class Parser {
         message = emptyQuery
         break
       case MessageCodes.DataRow:
-        message = parseDataRowMessage(reader)
+        message = parseDataRowMessage(reader, this.rowDescriptionFormats)
         break
       case MessageCodes.CommandComplete:
         message = parseCommandCompleteMessage(reader)
@@ -214,9 +216,14 @@ export class Parser {
       case MessageCodes.NoticeMessage:
         message = parseErrorMessage(reader, 'notice')
         break
-      case MessageCodes.RowDescriptionMessage:
-        message = parseRowDescriptionMessage(reader)
+      case MessageCodes.RowDescriptionMessage: {
+        const rowDescription = parseRowDescriptionMessage(reader)
+        // Remember the column formats: the DataRows that follow carry no format information of
+        // their own, and message arrive in wire order, so the most recent RowDescription applies.
+        this.rowDescriptionFormats = rowDescription.fields.map((field) => field.format)
+        message = rowDescription
         break
+      }
       case MessageCodes.ParameterDescriptionMessage:
         message = parseParameterDescriptionMessage(reader)
         break
@@ -306,13 +313,16 @@ const parseParameterDescriptionMessage = (reader: BufferReader) => {
   return message
 }
 
-const parseDataRowMessage = (reader: BufferReader) => {
+// `formats` comes from the RowDescription that precedes these DataRows. Fields the server sent in
+// binary format are handed back as raw bytes; decoding them as utf-8 would silently corrupt any
+// value containing a byte >= 0x80. Text fields keep their existing string representation.
+const parseDataRowMessage = (reader: BufferReader, formats: Mode[] | null) => {
   const fieldCount = reader.int16()
   const fields: any[] = new Array(fieldCount)
   for (let i = 0; i < fieldCount; i++) {
     const len = reader.int32()
     // a -1 for length means the value of the field is null
-    fields[i] = len === -1 ? null : reader.string(len)
+    fields[i] = len === -1 ? null : formats?.[i] === 'binary' ? reader.bytes(len) : reader.string(len)
   }
   return new DataRowMessage(LATEINIT_LENGTH, fields)
 }

packages/pg/test/integration/gh-issues/binary-high-bytes-tests.js

@@ -0,0 +1,51 @@
+'use strict'
+const helper = require('../test-helper')
+const assert = require('assert')
+
+const suite = new helper.Suite()
+
+// Binary results used to be decoded as utf-8 by pg-protocol's BufferReader and then re-encoded by
+// Result#parseRow with Buffer.from(). Any byte >= 0x80 does not survive that round trip: it is not
+// valid utf-8 on its own, so it became U+FFFD (0xEF 0xBF 0xBD) and the original value was lost.
+//
+// The corruption started exactly at 0x80, which is why it went unnoticed: every value used by the
+// existing binary tests happens to be built from bytes below that threshold.
+//
+//   SELECT 127::int -> 127          (0x0000007F, all bytes < 0x80)
+//   SELECT 128::int -> 239          (0x00000080, corrupted)
+//   SELECT 200::int -> 239          (0x000000C8, corrupted)
+//   SELECT -1::int  -> -272646673   (0xFFFFFFFF, corrupted)
+suite.test('binary results survive bytes >= 0x80', async () => {
+  const client = new helper.pg.Client()
+  await client.connect()
+
+  for (const value of [0, 1, 127, 128, 200, 255, 256, 65535, 2147483647, -1, -128, -2147483648]) {
+    const { rows } = await client.query({ text: 'SELECT $1::int AS a', values: [value], binary: true })
+    assert.strictEqual(rows[0].a, value, `binary int ${value} round tripped as ${rows[0].a}`)
+  }
+
+  await client.end()
+})
+
+suite.test('binary results preserve multi-byte text', async () => {
+  const client = new helper.pg.Client()
+  await client.connect()
+
+  for (const value of ['wat', 'ciào €', '日本語', '🐘']) {
+    const { rows } = await client.query({ text: 'SELECT $1::text AS a', values: [value], binary: true })
+    assert.strictEqual(rows[0].a, value)
+  }
+
+  await client.end()
+})
+
+// Text mode is the default and must be untouched by the format-aware decoding.
+suite.test('text mode is unaffected', async () => {
+  const client = new helper.pg.Client()
+  await client.connect()
+
+  const { rows } = await client.query("SELECT 200::int AS a, 'ciào €'::text AS b, true AS c, NULL::int AS d")
+  assert.deepStrictEqual(rows[0], { a: 200, b: 'ciào €', c: true, d: null })
+
+  await client.end()
+})