From f70192f0d63e981388d06a6a1422d3ea6c99e9a8 Mon Sep 17 00:00:00 2001 From: Nikolaus Schuetz Date: Thu, 13 Aug 2026 23:27:41 -0400 Subject: [PATCH] fix: Widen Athena integer type mapping for unsigned ints pa_to_athena_value_type mapped every unsigned Arrow int (uint8/16/32/64) to Athena tinyint, a signed 8-bit type (-128..127). Any unsigned value above 127 (e.g. a uint32 column) overflows the type in the generated CREATE TABLE DDL built by aws_utils.py. Widen each unsigned type to the next-larger signed Athena type, matching the widening already used for Postgres in arrow_to_pg_type in this module (uint8->smallint, uint16->int, uint32/uint64->bigint). Signed inputs are unchanged. Adds a regression test for both the widened and signed cases. Signed-off-by: Nikolaus Schuetz --- sdk/python/feast/type_map.py | 12 ++++++++---- sdk/python/tests/unit/test_type_map.py | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 3dc174ab696..54699396c80 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -2291,10 +2291,14 @@ def pa_to_athena_value_type(pa_type: "pyarrow.DataType") -> str: "int16": "smallint", "int32": "int", "int64": "bigint", - "uint8": "tinyint", - "uint16": "tinyint", - "uint32": "tinyint", - "uint64": "tinyint", + # Athena integer types are signed, so unsigned Arrow types must be + # widened to the next-larger signed type to avoid overflow in the + # generated DDL (e.g. uint32 exceeds signed int's max). This mirrors + # the widening already done in arrow_to_pg_type for Postgres. + "uint8": "smallint", + "uint16": "int", + "uint32": "bigint", + "uint64": "bigint", "float": "float", "double": "double", "binary": "binary", diff --git a/sdk/python/tests/unit/test_type_map.py b/sdk/python/tests/unit/test_type_map.py index 4fd62dc680e..5dfb0c52437 100644 --- a/sdk/python/tests/unit/test_type_map.py +++ b/sdk/python/tests/unit/test_type_map.py @@ -14,6 +14,7 @@ arrow_to_pg_type, feast_value_type_to_pa, feast_value_type_to_python_type, + pa_to_athena_value_type, pa_to_feast_value_type, pa_to_redshift_value_type, pg_type_to_feast_value_type, @@ -531,6 +532,25 @@ def test_arrow_to_pg_type_map(self): assert arrow_to_pg_type("map") == "jsonb" assert arrow_to_pg_type("map") == "jsonb" + def test_pa_to_athena_value_type_unsigned_ints_widen(self): + """Unsigned Arrow ints must widen to a signed Athena type that can + hold their full range. Athena has no unsigned integer types, so + mapping every uintN to tinyint (signed -128..127) overflows for any + value above 127 in the generated CREATE TABLE DDL. Each uintN must map + to the next-larger signed type, matching arrow_to_pg_type's widening. + """ + assert pa_to_athena_value_type(pyarrow.uint8()) == "smallint" + assert pa_to_athena_value_type(pyarrow.uint16()) == "int" + assert pa_to_athena_value_type(pyarrow.uint32()) == "bigint" + assert pa_to_athena_value_type(pyarrow.uint64()) == "bigint" + + def test_pa_to_athena_value_type_signed_ints_unchanged(self): + """Signed Arrow ints keep their same-width Athena type (no regression).""" + assert pa_to_athena_value_type(pyarrow.int8()) == "tinyint" + assert pa_to_athena_value_type(pyarrow.int16()) == "smallint" + assert pa_to_athena_value_type(pyarrow.int32()) == "int" + assert pa_to_athena_value_type(pyarrow.int64()) == "bigint" + def test_pg_type_to_feast_value_type_json(self): """Test that Postgres json/jsonb types convert to ValueType.MAP.""" assert pg_type_to_feast_value_type("json") == ValueType.MAP