Merge pull request isoos#9 from Gustl22/164-numeric-datatype

isoos · web-flow · commit 983df6e8ff93 · 2021-08-11T09:29:30.000+02:00
Binary encoder for numeric datatype
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,8 +2,8 @@
 
 ## Unreleased 
 
+- Support for type `numeric` / `decimal` ([#7](https://github.com/isoos/postgresql-dart/pull/7), [#9](https://github.com/isoos/postgresql-dart/pull/9)).
 - Support SASL / SCRAM-SHA-256 Authentication, [#6](https://github.com/isoos/postgresql-dart/pull/6).
-- Decoder for type `numeric` / `decimal`, [#7](https://github.com/isoos/postgresql-dart/pull/7).
 
 ## 2.3.2
 
diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart
@@ -27,6 +27,9 @@ final _hex = <String>[
   'e',
   'f',
 ];
+final _numericRegExp = RegExp(r'^(\d*)(\.\d*)?$');
+final _leadingZerosRegExp = RegExp(r'^0+');
+final _trailingZerosRegExp = RegExp(r'0+$');
 
 class PostgresBinaryEncoder extends Converter<dynamic, Uint8List?> {
   final PostgreSQLDataType _dataType;
@@ -141,8 +144,19 @@ class PostgresBinaryEncoder extends Converter<dynamic, Uint8List?> {
                 0, value.toUtc().difference(DateTime.utc(2000)).inMicroseconds);
             return bd.buffer.asUint8List();
           }
+          throw FormatException('Invalid type for parameter value. Expected: DateTime Got: ${value.runtimeType}');
+        }
+
+      case PostgreSQLDataType.numeric:
+        {
+          if (value is double || value is int) {
+            value = value.toString();
+          }
+          if (value is String) {
+            return _encodeNumeric(value);
+          }
           throw FormatException(
-              'Invalid type for parameter value. Expected: DateTime Got: ${value.runtimeType}');
+              'Invalid type for parameter value. Expected: String|double|int Got: ${value.runtimeType}');
         }
 
       case PostgreSQLDataType.jsonb:
@@ -287,6 +301,76 @@ class PostgresBinaryEncoder extends Converter<dynamic, Uint8List?> {
 
     return writer.toBytes();
   }
+
+  /// Encode String / double / int to numeric / decimal  without loosing precision.
+  /// Compare implementation: https://github.com/frohoff/jdk8u-dev-jdk/blob/da0da73ab82ed714dc5be94acd2f0d00fbdfe2e9/src/share/classes/java/math/BigDecimal.java#L409
+  Uint8List _encodeNumeric(String value) {
+    value = value.trim();
+    var signByte = 0x0000;
+    if (value.toLowerCase() == 'nan') {
+      signByte = 0xc000;
+      value = '';
+    } else if (value.startsWith('-')) {
+      value = value.substring(1);
+      signByte = 0x4000;
+    } else if (value.startsWith('+')) {
+      value = value.substring(1);
+    }
+    if (!_numericRegExp.hasMatch(value)) {
+      throw FormatException('Invalid format for parameter value. Expected: String which matches "/^(\\d*)(\\.\\d*)?\$/" Got: ${value}');
+    }
+    final parts = value.split('.');
+
+    var intPart = parts[0].replaceAll(_leadingZerosRegExp, '');
+    var intWeight = intPart.isEmpty ? -1 : (intPart.length - 1) ~/ 4;
+    intPart = intPart.padLeft((intWeight + 1) * 4, '0');
+
+    var fractPart = parts.length > 1 ? parts[1] : '';
+    final dScale = fractPart.length;
+    fractPart = fractPart.replaceAll(_trailingZerosRegExp, '');
+    var fractWeight = fractPart.isEmpty ? -1 : (fractPart.length - 1) ~/ 4;
+    fractPart = fractPart.padRight((fractWeight + 1) * 4, '0');
+
+    var weight = intWeight;
+    if (intWeight < 0) {
+      // If int part has no weight, handle leading zeros in fractional part.
+      if (fractPart.isEmpty) {
+        // Weight of value 0 or '' is 0;
+        weight = 0;
+      } else {
+        final leadingZeros = _leadingZerosRegExp.firstMatch(fractPart)?.group(0);
+        if (leadingZeros != null) {
+          final leadingZerosWeight = leadingZeros.length ~/ 4; // Get count of leading zeros '0000'
+          fractPart = fractPart.substring(leadingZerosWeight * 4); // Remove leading zeros '0000'
+          fractWeight -= leadingZerosWeight;
+          weight = -(leadingZerosWeight + 1); // Ignore leading zeros in weight
+        }
+      }
+    } else if (fractWeight < 0) {
+      // If int fract has no weight, handle trailing zeros in int part.
+      final trailingZeros = _trailingZerosRegExp.firstMatch(intPart)?.group(0);
+      if (trailingZeros != null) {
+        final trailingZerosWeight = trailingZeros.length ~/ 4; // Get count of trailing zeros '0000'
+        intPart = intPart.substring(0, intPart.length - trailingZerosWeight * 4); // Remove leading zeros '0000'
+        intWeight -= trailingZerosWeight;
+      }
+    }
+
+    final nDigits = intWeight + fractWeight + 2;
+
+    final writer = ByteDataWriter();
+    writer.writeInt16(nDigits);
+    writer.writeInt16(weight);
+    writer.writeUint16(signByte);
+    writer.writeInt16(dScale);
+    for (var i = 0; i <= intWeight * 4; i += 4) {
+      writer.writeInt16(int.parse(intPart.substring(i, i + 4)));
+    }
+    for (var i = 0; i <= fractWeight * 4; i += 4) {
+      writer.writeInt16(int.parse(fractPart.substring(i, i + 4)));
+    }
+    return writer.toBytes();
+  }
 }
 
 class PostgresBinaryDecoder extends Converter<Uint8List, dynamic> {
@@ -461,12 +545,19 @@ class PostgresBinaryDecoder extends Converter<Uint8List, dynamic> {
     final reader = ByteDataReader()..add(value);
     final nDigits = reader.readInt16(); // non-zero digits, data buffer length = 2 * nDigits
     var weight = reader.readInt16(); // weight of first digit
-    final signByte = reader.readInt16(); // NUMERIC_POS, NEG, NAN, PINF, or NINF
+    final signByte = reader.readUint16(); // NUMERIC_POS, NEG, NAN, PINF, or NINF
     final dScale = reader.readInt16(); // display scale
     if (signByte == 0xc000) return 'NaN';
     final sign = signByte == 0x4000 ? '-' : '';
     var intPart = '';
     var fractPart = '';
+
+    final fractOmitted = -(weight + 1);
+    if (fractOmitted > 0) {
+      // If value < 0, the leading zeros in fractional part were omitted.
+      fractPart += '0000' * fractOmitted;
+    }
+
     for (var i = 0; i < nDigits; i++) {
       if (weight >= 0) {
         intPart += reader.readInt16().toString().padLeft(4, '0');
@@ -475,6 +566,18 @@ class PostgresBinaryDecoder extends Converter<Uint8List, dynamic> {
       }
       weight--;
     }
-    return '$sign${intPart.replaceAll(RegExp(r'^0+'), '')}.${fractPart.padRight(dScale, '0').substring(0, dScale)}';
+
+    if (weight >= 0) {
+      // Trailing zeros were omitted
+      intPart += '0000' * (weight + 1);
+    }
+
+    var result = '$sign${intPart.replaceAll(_leadingZerosRegExp, '')}';
+    if (result.isEmpty) result = '0'; // Show at least 0, if no int value is given.
+    if (dScale > 0) {
+      // Only add fractional digits, if dScale allows
+      result += '.${fractPart.padRight(dScale, '0').substring(0, dScale)}';
+    }
+    return result;
   }
 }
diff --git a/test/decode_test.dart b/test/decode_test.dart
@@ -14,31 +14,33 @@ void main() {
     await connection.execute('''
         CREATE TEMPORARY TABLE t (
           i int, s serial, bi bigint, bs bigserial, bl boolean, si smallint, 
-          t text, f real, d double precision, dt date, ts timestamp, tsz timestamptz, j jsonb, ba bytea,
+          t text, f real, d double precision, dt date, ts timestamp, tsz timestamptz, n numeric, j jsonb, ba bytea,
           u uuid, v varchar, p point, jj json, ia _int4, ta _text, da _float8, ja _jsonb)
     ''');
 
     await connection.execute(
-        'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, ba, u, v, p, jj, ia, ta, da, ja) '
+        'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, n, j, ba, u, v, p, jj, ia, ta, da, ja) '
         'VALUES (-2147483648, -9223372036854775808, TRUE, -32768, '
         "'string', 10.0, 10.0, '1983-11-06', "
         "'1983-11-06 06:00:00.000000', '1983-11-06 06:00:00.000000', "
+        "'-1234567890.0987654321', "
         "'{\"key\":\"value\"}', E'\\\\000', '00000000-0000-0000-0000-000000000000', "
         "'abcdef', '(0.01, 12.34)', '{\"key\": \"value\"}', '{}', '{}', '{}', '{}')");
     await connection.execute(
-        'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, ba, u, v, p, jj, ia, ta, da, ja) '
+        'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, n, j, ba, u, v, p, jj, ia, ta, da, ja) '
         'VALUES (2147483647, 9223372036854775807, FALSE, 32767, '
         "'a significantly longer string to the point where i doubt this actually matters', "
         "10.25, 10.125, '2183-11-06', '2183-11-06 00:00:00.111111', "
         "'2183-11-06 00:00:00.999999', "
+        "'1000000000000000000000000000.0000000000000000000000000001', "
         "'[{\"key\":1}]', E'\\\\377', 'FFFFFFFF-ffff-ffff-ffff-ffffffffffff', "
         "'01234', '(0.2, 100)', '{}', '{-123, 999}', '{\"a\", \"lorem ipsum\", \"\"}', "
         "'{1, 2, 4.5, 1234.5}', '{1, \"\\\"test\\\"\", \"{\\\"a\\\": \\\"b\\\"}\"}')");
 
     await connection.execute(
-        'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, ba, u, v, p, jj, ia, ta, da, ja) '
+        'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, n, j, ba, u, v, p, jj, ia, ta, da, ja) '
         'VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, '
-        'null, null, null, null, null, null, null )');
+        'null, null, null, null, null, null, null, null )');
   });
   tearDown(() async {
     await connection.close();
@@ -66,16 +68,17 @@ void main() {
     expect(row1[9], equals(DateTime.utc(1983, 11, 6)));
     expect(row1[10], equals(DateTime.utc(1983, 11, 6, 6)));
     expect(row1[11], equals(DateTime.utc(1983, 11, 6, 6)));
-    expect(row1[12], equals({'key': 'value'}));
-    expect(row1[13], equals([0]));
-    expect(row1[14], equals('00000000-0000-0000-0000-000000000000'));
-    expect(row1[15], equals('abcdef'));
-    expect(row1[16], equals(PgPoint(0.01, 12.34)));
-    expect(row1[17], equals({'key': 'value'}));
-    expect(row1[18], equals(<int>[]));
-    expect(row1[19], equals(<String>[]));
-    expect(row1[20], equals(<double>[]));
-    expect(row1[21], equals([]));
+    expect(row1[12], equals('-1234567890.0987654321'));
+    expect(row1[13], equals({'key': 'value'}));
+    expect(row1[14], equals([0]));
+    expect(row1[15], equals('00000000-0000-0000-0000-000000000000'));
+    expect(row1[16], equals('abcdef'));
+    expect(row1[17], equals(PgPoint(0.01, 12.34)));
+    expect(row1[18], equals({'key': 'value'}));
+    expect(row1[19], equals(<int>[]));
+    expect(row1[20], equals(<String>[]));
+    expect(row1[21], equals(<double>[]));
+    expect(row1[22], equals([]));
 
     // upper bound row
     expect(row2[0], equals(2147483647));
@@ -95,21 +98,22 @@ void main() {
     expect(row2[9], equals(DateTime.utc(2183, 11, 6)));
     expect(row2[10], equals(DateTime.utc(2183, 11, 6, 0, 0, 0, 111, 111)));
     expect(row2[11], equals(DateTime.utc(2183, 11, 6, 0, 0, 0, 999, 999)));
+    expect(row2[12], equals('1000000000000000000000000000.0000000000000000000000000001'));
     expect(
-        row2[12],
+        row2[13],
         equals([
           {'key': 1}
         ]));
-    expect(row2[13], equals([255]));
-    expect(row2[14], equals('ffffffff-ffff-ffff-ffff-ffffffffffff'));
-    expect(row2[15], equals('01234'));
-    expect(row2[16], equals(PgPoint(0.2, 100)));
-    expect(row2[17], equals({}));
-    expect(row2[18], equals(<int>[-123, 999]));
-    expect(row2[19], equals(<String>['a', 'lorem ipsum', '']));
-    expect(row2[20], equals(<double>[1, 2, 4.5, 1234.5]));
+    expect(row2[14], equals([255]));
+    expect(row2[15], equals('ffffffff-ffff-ffff-ffff-ffffffffffff'));
+    expect(row2[16], equals('01234'));
+    expect(row2[17], equals(PgPoint(0.2, 100)));
+    expect(row2[18], equals({}));
+    expect(row2[19], equals(<int>[-123, 999]));
+    expect(row2[20], equals(<String>['a', 'lorem ipsum', '']));
+    expect(row2[21], equals(<double>[1, 2, 4.5, 1234.5]));
     expect(
-        row2[21],
+        row2[22],
         equals([
           1,
           'test',
@@ -139,6 +143,7 @@ void main() {
     expect(row3[19], isNull);
     expect(row3[20], isNull);
     expect(row3[21], isNull);
+    expect(row3[22], isNull);
   });
 
   test('Fetch/insert empty string', () async {
@@ -172,18 +177,21 @@ void main() {
   });
 
   test('Decode Numeric to String', () {
-    // -123400000.2
-    final binary1 = [0, 4, 0, 2, 64, 0, 0, 5, 0, 1, 9, 36, 0, 0, 7, 208];
-
-    // -123400001.01234
-    final binary2 = [0, 5, 0, 2, 64, 0, 0, 5, 0, 1, 9, 36, 0, 1, 0, 0, 7, 208];
+    final binaries = {
+      '-123400000.20000': [0, 4, 0, 2, 64, 0, 0, 5, 0, 1, 9, 36, 0, 0, 7, 208],
+      '-123400001.00002': [0, 5, 0, 2, 64, 0, 0, 5, 0, 1, 9, 36, 0, 1, 0, 0, 7, 208],
+      '0.00001': [0, 1, 255, 254, 0, 0, 0, 5, 3, 232],
+      '10000.000000000': [0, 1, 0, 1, 0, 0, 0, 9, 0, 1],
+      'NaN': [0, 0, 0, 0, 192, 0, 0, 0],
+      '0': [0, 0, 0, 0, 0, 0, 0, 0], // 0 or 0.
+      '0.0': [0, 0, 0, 0, 0, 0, 0, 1], // .0 or 0.0
+    };
 
     final decoder = PostgresBinaryDecoder(1700);
-    final uint8List1 = Uint8List.fromList(binary1);
-    final uint8List2 = Uint8List.fromList(binary2);
-    final res1 = decoder.convert(uint8List1);
-    final res2 = decoder.convert(uint8List2);
-    expect(res1, '-123400000.20000');
-    expect(res2, '-123400001.00002');
+    binaries.forEach((key, value) {
+      final uint8List = Uint8List.fromList(value);
+      final res = decoder.convert(uint8List);
+      expect(res, key);
+    });
   });
 }
diff --git a/test/encoding_test.dart b/test/encoding_test.dart
@@ -1,5 +1,6 @@
 import 'dart:async';
 import 'dart:convert';
+import 'dart:typed_data';
 
 import 'package:postgres/postgres.dart';
 import 'package:postgres/src/binary_codec.dart';
@@ -182,6 +183,47 @@ void main() {
       }
     });
 
+    test('numeric', () async {
+      final binaries = {
+        '-123400000.20000': [0, 4, 0, 2, 64, 0, 0, 5, 0, 1, 9, 36, 0, 0, 7, 208],
+        '-123400001.00002': [0, 5, 0, 2, 64, 0, 0, 5, 0, 1, 9, 36, 0, 1, 0, 0, 7, 208],
+        '0.00001': [0, 1, 255, 254, 0, 0, 0, 5, 3, 232],
+        '10000.000000000': [0, 1, 0, 1, 0, 0, 0, 9, 0, 1],
+        'NaN': [0, 0, 0, 0, 192, 0, 0, 0],
+        '0': [0, 0, 0, 0, 0, 0, 0, 0], // 0 or 0.
+        '0.0': [0, 0, 0, 0, 0, 0, 0, 1], // .0 or 0.0
+      };
+
+      final encoder = PostgresBinaryEncoder(PostgreSQLDataType.numeric);
+      binaries.forEach((key, value) {
+        final uint8List = Uint8List.fromList(value);
+        final res = encoder.convert(key);
+        expect(res, uint8List);
+      });
+
+      await expectInverse('1000000000000000000000000000.0000000000000000000000000001', PostgreSQLDataType.numeric);
+      await expectInverse('3141592653589793238462643383279502.1618033988749894848204586834365638', PostgreSQLDataType.numeric);
+      await expectInverse('-3141592653589793238462643383279502.1618033988749894848204586834365638', PostgreSQLDataType.numeric);
+      await expectInverse('0.0', PostgreSQLDataType.numeric);
+      await expectInverse('0.1', PostgreSQLDataType.numeric);
+      await expectInverse('0.0001', PostgreSQLDataType.numeric);
+      await expectInverse('0.00001', PostgreSQLDataType.numeric);
+      await expectInverse('0.000001', PostgreSQLDataType.numeric);
+      await expectInverse('0.000000001', PostgreSQLDataType.numeric);
+      await expectInverse('1.000000000', PostgreSQLDataType.numeric);
+      await expectInverse('1000.000000000', PostgreSQLDataType.numeric);
+      await expectInverse('10000.000000000', PostgreSQLDataType.numeric);
+      await expectInverse('100000000.00000000', PostgreSQLDataType.numeric);
+      await expectInverse('NaN', PostgreSQLDataType.numeric);
+      try {
+        await conn.query('INSERT INTO t (v) VALUES (@v:numeric)',
+            substitutionValues: {'v': 'not-numeric'});
+        fail('unreachable');
+      } on FormatException catch (e) {
+        expect(e.toString(), contains('Expected: String'));
+      }
+    });
+
     test('jsonb', () async {
       await expectInverse('string', PostgreSQLDataType.jsonb);
       await expectInverse(2, PostgreSQLDataType.jsonb);
