Fold positive Decimal exponents into the mantissa.

src/include/duckdb_python/python_objects.hpp

@@ -7,7 +7,6 @@
 #include "duckdb/common/types/timestamp.hpp"
 #include "duckdb/common/types/interval.hpp"
 #include "duckdb/common/types/value.hpp"
-#include "duckdb/common/types/cast_helpers.hpp"
 #include "duckdb/main/client_properties.hpp"
 
 #include "datetime.h" //from python
@@ -50,7 +49,7 @@ struct PyDictionary {
 
 enum class PyDecimalExponentType {
 	EXPONENT_SCALE,    //! Amount of digits after the decimal point
-	EXPONENT_POWER,    //! How many zeros behind the decimal point
+	EXPONENT_POWER,    //! Trailing zeros on the mantissa (positive Decimal exponent)
 	EXPONENT_INFINITY, //! Decimal is INFINITY
 	EXPONENT_NAN       //! Decimal is NAN
 };
@@ -71,27 +70,6 @@ struct PyDecimal {
 		}
 	};
 
-	struct PyDecimalPowerConverter {
-		template <typename T, typename = std::enable_if<std::numeric_limits<T>::is_integer, T>>
-		static Value Operation(bool signed_value, vector<uint8_t> &digits, uint8_t width, uint8_t scale) {
-			T value = 0;
-			for (auto &digit : digits) {
-				value = value * 10 + digit;
-			}
-			D_ASSERT(scale >= 0);
-			int64_t multiplier =
-			    NumericHelper::POWERS_OF_TEN[MinValue<uint8_t>(scale, NumericHelper::CACHED_POWERS_OF_TEN - 1)];
-			for (auto power = scale; power > NumericHelper::CACHED_POWERS_OF_TEN; power--) {
-				multiplier *= 10;
-			}
-			value *= multiplier;
-			if (signed_value) {
-				value = -value;
-			}
-			return Value::DECIMAL(value, width, scale);
-		}
-	};
-
 public:
 	PyDecimal(nb::handle &obj);
 	vector<uint8_t> digits;

src/native/python_objects.cpp

@@ -79,11 +79,7 @@ bool PyDecimal::TryGetType(LogicalType &type) {
 
 	switch (exponent_type) {
 	case PyDecimalExponentType::EXPONENT_SCALE: {
-	case PyDecimalExponentType::EXPONENT_POWER: {
 		auto scale = exponent_value;
-		if (exponent_type == PyDecimalExponentType::EXPONENT_POWER) {
-			width += scale;
-		}
 		if (scale > width) {
 			// The value starts with 1 or more zeros, which are optimized out of the 'digits' array
 			// 0.001; width=1, exponent=-3
@@ -96,6 +92,16 @@ bool PyDecimal::TryGetType(LogicalType &type) {
 		type = LogicalType::DECIMAL(width, scale);
 		return true;
 	}
+	case PyDecimalExponentType::EXPONENT_POWER: {
+		// Positive exponent: integer with extra trailing zeros, scale 0.
+		if (exponent_value > Decimal::MAX_WIDTH_INT128 || width > Decimal::MAX_WIDTH_INT128 - exponent_value) {
+			type = LogicalType::DOUBLE;
+			return true;
+		}
+		width += exponent_value;
+		type = LogicalType::DECIMAL(width, 0);
+		return true;
+	}
 	case PyDecimalExponentType::EXPONENT_INFINITY: {
 		type = LogicalType::FLOAT;
 		return true;
@@ -107,7 +113,6 @@ bool PyDecimal::TryGetType(LogicalType &type) {
 	default: // LCOV_EXCL_START
 		throw NotImplementedException("case not implemented for type PyDecimalExponentType");
 	} // LCOV_EXCL_STOP
-	}
 }
 // LCOV_EXCL_START
 static void ExponentNotRecognized() {
@@ -190,12 +195,15 @@ Value PyDecimal::ToDuckValue() {
 		return PyDecimalCastSwitch<PyDecimalScaleConverter>(*this, width, scale);
 	}
 	case PyDecimalExponentType::EXPONENT_POWER: {
-		uint8_t scale = exponent_value;
-		width += scale;
-		if (!WidthFitsInDecimal(width)) {
+		// Fold 10^exponent into the mantissa and store scale 0. Using the exponent as a
+		// DECIMAL scale cancelled the 10^n multiply and stored only the mantissa (1E+2 -> 1).
+		if (exponent_value > Decimal::MAX_WIDTH_DECIMAL || width > Decimal::MAX_WIDTH_DECIMAL - exponent_value) {
 			return CastToDouble(obj);
 		}
-		return PyDecimalCastSwitch<PyDecimalPowerConverter>(*this, width, scale);
+		digits.insert(digits.end(), static_cast<size_t>(exponent_value), static_cast<uint8_t>(0));
+		width += exponent_value;
+		D_ASSERT(WidthFitsInDecimal(width));
+		return PyDecimalCastSwitch<PyDecimalScaleConverter>(*this, width, 0);
 	}
 	case PyDecimalExponentType::EXPONENT_NAN: {
 		return Value::FLOAT(NAN);

tests/fast/pandas/test_df_object_resolution.py

@@ -577,7 +577,7 @@ def test_numeric_decimal(self, duckdb_cursor):
                 (12.0,                13,                       324234234.00000005),
                 (-123.0,              -12.0000000005,           -128),
                 (-234234.0,           7453324234.0,             345345),
-                (NULL,                NULL,                     1.00000),
+                (NULL,                NULL,                     100000.00000),
                 (1.234,               -324234234,               1324234359)
             ) tbl(a, b, c);
         """

tests/fast/sqlite/test_types.py

@@ -88,7 +88,7 @@ def test_CheckDecimalWithExponent(self):
         self.cur.execute("insert into test(f) values (?)", (val,))
         self.cur.execute("select f from test")
         row = self.cur.fetchone()
-        assert row[0] == self.cur.execute("select 1.00000::DOUBLE").fetchone()[0]
+        assert row[0] == self.cur.execute("select 1e5::DOUBLE").fetchone()[0]
 
     def test_CheckNaN(self):
         import math

tests/fast/types/test_decimal.py

@@ -1,5 +1,8 @@
 from decimal import Decimal
 
+import pytest
+
+import duckdb
 import numpy
 
 
@@ -24,3 +27,44 @@ def test_decimal_numpy(self, duckdb_cursor):
             "c": numpy.array([320938.4298]),
             "d": numpy.array([49082094824.904820482094]),
         }
+
+    @pytest.mark.parametrize(
+        ("text", "expected_type"),
+        [
+            ("1E+2", "DECIMAL(3,0)"),
+            ("123E+2", "DECIMAL(5,0)"),
+            ("1.5E+3", "DECIMAL(4,0)"),
+            ("12.34E+5", "DECIMAL(7,0)"),
+            ("999E+9", "DECIMAL(12,0)"),
+            ("100", "DECIMAL(3,0)"),
+            ("1.23", "DECIMAL(3,2)"),
+            ("1E+18", "DECIMAL(19,0)"),
+            ("1E+19", "DECIMAL(20,0)"),
+            ("1E+20", "DECIMAL(21,0)"),
+            ("1E+21", "DECIMAL(22,0)"),
+            ("-1E+2", "DECIMAL(3,0)"),
+        ],
+    )
+    def test_decimal_positive_exponent_roundtrip(self, duckdb_cursor, text, expected_type):
+        value = Decimal(text)
+        assert duckdb_cursor.execute("SELECT typeof(?)", [value]).fetchone()[0] == expected_type
+        assert duckdb_cursor.execute("SELECT ?", [value]).fetchone()[0] == value
+
+    def test_decimal_positive_exponent_insert(self, duckdb_cursor):
+        duckdb_cursor.execute("CREATE TABLE t (v DECIMAL(28,8))")
+        for text in ["1E+2", "123E+2", "1.5E+3", "12.34E+5", "999E+9", "100", "1.23", "1E+18", "1E+19", "-1E+2"]:
+            value = Decimal(text)
+            duckdb_cursor.execute("DELETE FROM t")
+            duckdb_cursor.execute("INSERT INTO t VALUES (?)", [value])
+            assert duckdb_cursor.execute("SELECT v FROM t").fetchone()[0] == value
+
+    def test_decimal_positive_exponent_out_of_range(self, duckdb_cursor):
+        # DECIMAL(28,8) has 20 integer digits; 1E+20 used to bind as DECIMAL(21,20) and store garbage.
+        duckdb_cursor.execute("CREATE TABLE t (v DECIMAL(28,8))")
+        with pytest.raises(duckdb.ConversionException, match="out of range"):
+            duckdb_cursor.execute("INSERT INTO t VALUES (?)", [Decimal("1E+20")])
+
+    def test_decimal_normalize_positive_exponent(self, duckdb_cursor):
+        value = Decimal("1000.0000").normalize()
+        assert value.as_tuple().exponent > 0
+        assert duckdb_cursor.execute("SELECT ?", [value]).fetchone()[0] == Decimal("1000")