diff --git a/src/include/duckdb_python/python_objects.hpp b/src/include/duckdb_python/python_objects.hpp index 130f9ffa..2d04b5a4 100644 --- a/src/include/duckdb_python/python_objects.hpp +++ b/src/include/duckdb_python/python_objects.hpp @@ -7,7 +7,6 @@ #include "duckdb/common/types/timestamp.hpp" #include "duckdb/common/types/interval.hpp" #include "duckdb/common/types/value.hpp" -#include "duckdb/common/types/cast_helpers.hpp" #include "duckdb/main/client_properties.hpp" #include "datetime.h" //from python @@ -50,7 +49,7 @@ struct PyDictionary { enum class PyDecimalExponentType { EXPONENT_SCALE, //! Amount of digits after the decimal point - EXPONENT_POWER, //! How many zeros behind the decimal point + EXPONENT_POWER, //! Trailing zeros on the mantissa (positive Decimal exponent) EXPONENT_INFINITY, //! Decimal is INFINITY EXPONENT_NAN //! Decimal is NAN }; @@ -71,27 +70,6 @@ struct PyDecimal { } }; - struct PyDecimalPowerConverter { - template ::is_integer, T>> - static Value Operation(bool signed_value, vector &digits, uint8_t width, uint8_t scale) { - T value = 0; - for (auto &digit : digits) { - value = value * 10 + digit; - } - D_ASSERT(scale >= 0); - int64_t multiplier = - NumericHelper::POWERS_OF_TEN[MinValue(scale, NumericHelper::CACHED_POWERS_OF_TEN - 1)]; - for (auto power = scale; power > NumericHelper::CACHED_POWERS_OF_TEN; power--) { - multiplier *= 10; - } - value *= multiplier; - if (signed_value) { - value = -value; - } - return Value::DECIMAL(value, width, scale); - } - }; - public: PyDecimal(nb::handle &obj); vector digits; diff --git a/src/native/python_objects.cpp b/src/native/python_objects.cpp index 6bcdbc6d..eee7ae83 100644 --- a/src/native/python_objects.cpp +++ b/src/native/python_objects.cpp @@ -79,11 +79,7 @@ bool PyDecimal::TryGetType(LogicalType &type) { switch (exponent_type) { case PyDecimalExponentType::EXPONENT_SCALE: { - case PyDecimalExponentType::EXPONENT_POWER: { auto scale = exponent_value; - if (exponent_type == PyDecimalExponentType::EXPONENT_POWER) { - width += scale; - } if (scale > width) { // The value starts with 1 or more zeros, which are optimized out of the 'digits' array // 0.001; width=1, exponent=-3 @@ -96,6 +92,16 @@ bool PyDecimal::TryGetType(LogicalType &type) { type = LogicalType::DECIMAL(width, scale); return true; } + case PyDecimalExponentType::EXPONENT_POWER: { + // Positive exponent: integer with extra trailing zeros, scale 0. + if (exponent_value > Decimal::MAX_WIDTH_INT128 || width > Decimal::MAX_WIDTH_INT128 - exponent_value) { + type = LogicalType::DOUBLE; + return true; + } + width += exponent_value; + type = LogicalType::DECIMAL(width, 0); + return true; + } case PyDecimalExponentType::EXPONENT_INFINITY: { type = LogicalType::FLOAT; return true; @@ -107,7 +113,6 @@ bool PyDecimal::TryGetType(LogicalType &type) { default: // LCOV_EXCL_START throw NotImplementedException("case not implemented for type PyDecimalExponentType"); } // LCOV_EXCL_STOP - } } // LCOV_EXCL_START static void ExponentNotRecognized() { @@ -190,12 +195,15 @@ Value PyDecimal::ToDuckValue() { return PyDecimalCastSwitch(*this, width, scale); } case PyDecimalExponentType::EXPONENT_POWER: { - uint8_t scale = exponent_value; - width += scale; - if (!WidthFitsInDecimal(width)) { + // Fold 10^exponent into the mantissa and store scale 0. Using the exponent as a + // DECIMAL scale cancelled the 10^n multiply and stored only the mantissa (1E+2 -> 1). + if (exponent_value > Decimal::MAX_WIDTH_DECIMAL || width > Decimal::MAX_WIDTH_DECIMAL - exponent_value) { return CastToDouble(obj); } - return PyDecimalCastSwitch(*this, width, scale); + digits.insert(digits.end(), static_cast(exponent_value), static_cast(0)); + width += exponent_value; + D_ASSERT(WidthFitsInDecimal(width)); + return PyDecimalCastSwitch(*this, width, 0); } case PyDecimalExponentType::EXPONENT_NAN: { return Value::FLOAT(NAN); diff --git a/tests/fast/pandas/test_df_object_resolution.py b/tests/fast/pandas/test_df_object_resolution.py index ae8ce11d..57acdd3c 100644 --- a/tests/fast/pandas/test_df_object_resolution.py +++ b/tests/fast/pandas/test_df_object_resolution.py @@ -577,7 +577,7 @@ def test_numeric_decimal(self, duckdb_cursor): (12.0, 13, 324234234.00000005), (-123.0, -12.0000000005, -128), (-234234.0, 7453324234.0, 345345), - (NULL, NULL, 1.00000), + (NULL, NULL, 100000.00000), (1.234, -324234234, 1324234359) ) tbl(a, b, c); """ diff --git a/tests/fast/sqlite/test_types.py b/tests/fast/sqlite/test_types.py index b06228fc..8b2c02db 100644 --- a/tests/fast/sqlite/test_types.py +++ b/tests/fast/sqlite/test_types.py @@ -88,7 +88,7 @@ def test_CheckDecimalWithExponent(self): self.cur.execute("insert into test(f) values (?)", (val,)) self.cur.execute("select f from test") row = self.cur.fetchone() - assert row[0] == self.cur.execute("select 1.00000::DOUBLE").fetchone()[0] + assert row[0] == self.cur.execute("select 1e5::DOUBLE").fetchone()[0] def test_CheckNaN(self): import math diff --git a/tests/fast/types/test_decimal.py b/tests/fast/types/test_decimal.py index a5013dcd..9232973a 100644 --- a/tests/fast/types/test_decimal.py +++ b/tests/fast/types/test_decimal.py @@ -1,5 +1,8 @@ from decimal import Decimal +import pytest + +import duckdb import numpy @@ -24,3 +27,44 @@ def test_decimal_numpy(self, duckdb_cursor): "c": numpy.array([320938.4298]), "d": numpy.array([49082094824.904820482094]), } + + @pytest.mark.parametrize( + ("text", "expected_type"), + [ + ("1E+2", "DECIMAL(3,0)"), + ("123E+2", "DECIMAL(5,0)"), + ("1.5E+3", "DECIMAL(4,0)"), + ("12.34E+5", "DECIMAL(7,0)"), + ("999E+9", "DECIMAL(12,0)"), + ("100", "DECIMAL(3,0)"), + ("1.23", "DECIMAL(3,2)"), + ("1E+18", "DECIMAL(19,0)"), + ("1E+19", "DECIMAL(20,0)"), + ("1E+20", "DECIMAL(21,0)"), + ("1E+21", "DECIMAL(22,0)"), + ("-1E+2", "DECIMAL(3,0)"), + ], + ) + def test_decimal_positive_exponent_roundtrip(self, duckdb_cursor, text, expected_type): + value = Decimal(text) + assert duckdb_cursor.execute("SELECT typeof(?)", [value]).fetchone()[0] == expected_type + assert duckdb_cursor.execute("SELECT ?", [value]).fetchone()[0] == value + + def test_decimal_positive_exponent_insert(self, duckdb_cursor): + duckdb_cursor.execute("CREATE TABLE t (v DECIMAL(28,8))") + for text in ["1E+2", "123E+2", "1.5E+3", "12.34E+5", "999E+9", "100", "1.23", "1E+18", "1E+19", "-1E+2"]: + value = Decimal(text) + duckdb_cursor.execute("DELETE FROM t") + duckdb_cursor.execute("INSERT INTO t VALUES (?)", [value]) + assert duckdb_cursor.execute("SELECT v FROM t").fetchone()[0] == value + + def test_decimal_positive_exponent_out_of_range(self, duckdb_cursor): + # DECIMAL(28,8) has 20 integer digits; 1E+20 used to bind as DECIMAL(21,20) and store garbage. + duckdb_cursor.execute("CREATE TABLE t (v DECIMAL(28,8))") + with pytest.raises(duckdb.ConversionException, match="out of range"): + duckdb_cursor.execute("INSERT INTO t VALUES (?)", [Decimal("1E+20")]) + + def test_decimal_normalize_positive_exponent(self, duckdb_cursor): + value = Decimal("1000.0000").normalize() + assert value.as_tuple().exponent > 0 + assert duckdb_cursor.execute("SELECT ?", [value]).fetchone()[0] == Decimal("1000")