From 838d75fa615495b0c924a3c0c18afc19594f9186 Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Fri, 15 May 2026 17:52:41 -0400 Subject: [PATCH 01/15] fix: using polars unnest for all backend types. --- marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py index 8685fef332b..0751520f8d6 100644 --- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py +++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py @@ -492,7 +492,12 @@ def handle_explode_columns( def handle_expand_dict( df: DataFrame, transform: ExpandDictTransform ) -> DataFrame: - return df.explode(transform.column_id) + # Convert to Polars for native unnest, which handles null values + # correctly (fixes: 'NoneType' cannot be converted to 'PyDict' issue #4583) + collected_df, undo = collect_and_preserve_type(df) + polars_df = collected_df.to_polars() + unnested = polars_df.unnest(transform.column_id) + return undo(nw.from_native(unnested)) @staticmethod def handle_unique(df: DataFrame, transform: UniqueTransform) -> DataFrame: From 851e5b16af0be8444442533e0d2159bac59ada71 Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Fri, 15 May 2026 17:53:23 -0400 Subject: [PATCH 02/15] fix: update the expand dict print statement. --- marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py index d27d1d07698..21150ab9d55 100644 --- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py +++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py @@ -465,7 +465,7 @@ def generate_where_clause_polars(where: FilterCondition) -> str: elif transform.type == TransformType.EXPAND_DICT: column_id = _as_literal(transform.column_id) - return f"{df_name}.hstack(pl.DataFrame({df_name}.select({column_id}).to_series().to_list())).drop({column_id})" + return f"{df_name}.unnest({column_id})" elif transform.type == TransformType.UNIQUE: column_ids = transform.column_ids From 0ea6758efb3247f4156bc96b1cd33de238183930 Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Fri, 15 May 2026 17:54:40 -0400 Subject: [PATCH 03/15] tests: enabling the expand dict test and adding the necessary dataframes. --- .../ui/_impl/dataframes/test_handlers.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.py index 58735a24ab0..91ad1cad7af 100644 --- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py +++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py @@ -47,8 +47,8 @@ pytest.importorskip("ibis") pd = pytest.importorskip("pandas") -pytest.importorskip("polars") pytest.importorskip("pyarrow") +pl = pytest.importorskip("polars") def apply(df: DataFrameType, transform: Transform) -> DataFrameType: @@ -1733,22 +1733,36 @@ def test_explode_columns(df: DataFrameType) -> None: assert nw_result.columns == ["A", "B", "C"] @staticmethod - @pytest.mark.skip( - reason="Dict/struct expansion not supported uniformly across backends" - ) @pytest.mark.parametrize( ("df", "expected"), list( zip( create_test_dataframes( - {"A": [{"foo": 1, "bar": "hello"}], "B": [1]} + {"A": [{"foo": 1, "bar": "hello"}], "B": [1]}, + exclude=[ + "ibis" + ], # convert via Polars changes backend type ), create_test_dataframes( - {"B": [1], "foo": [1], "bar": ["hello"]} + {"B": [1], "foo": [1], "bar": ["hello"]}, + exclude=["ibis"], ), strict=False, ) - ), + ) + + [ + ( + pl.DataFrame( + { + "A": [{"foo": 1, "bar": "hello"}, None], + "B": [1, 2], + } + ), + pl.DataFrame( + {"B": [1, 2], "foo": [1, None], "bar": ["hello", None]} + ), + ) + ], ) def test_expand_dict(df: DataFrameType, expected: DataFrameType) -> None: transform = ExpandDictTransform( From 294e21e36927f4ae1c6baa581ffde468344d1812 Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Fri, 15 May 2026 18:19:56 -0400 Subject: [PATCH 04/15] tests: removing ibis skip in expand dict test. --- tests/_plugins/ui/_impl/dataframes/test_handlers.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.py index 91ad1cad7af..88f0636bca5 100644 --- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py +++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py @@ -1739,13 +1739,9 @@ def test_explode_columns(df: DataFrameType) -> None: zip( create_test_dataframes( {"A": [{"foo": 1, "bar": "hello"}], "B": [1]}, - exclude=[ - "ibis" - ], # convert via Polars changes backend type ), create_test_dataframes( {"B": [1], "foo": [1], "bar": ["hello"]}, - exclude=["ibis"], ), strict=False, ) From 24f672ba40c2eaff4c8f1a810257e7665b602e8a Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Tue, 19 May 2026 17:51:55 -0400 Subject: [PATCH 05/15] fix: removing additional polars dataframe creation and using the none rows with the create test dataframes instead. --- .../ui/_impl/dataframes/test_handlers.py | 54 ++----------------- 1 file changed, 3 insertions(+), 51 deletions(-) diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.py index 88f0636bca5..2b592bb8263 100644 --- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py +++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py @@ -1738,27 +1738,14 @@ def test_explode_columns(df: DataFrameType) -> None: list( zip( create_test_dataframes( - {"A": [{"foo": 1, "bar": "hello"}], "B": [1]}, + {"A": [{"foo": 1, "bar": "hello"}, None], "B": [1, 2]}, ), create_test_dataframes( - {"B": [1], "foo": [1], "bar": ["hello"]}, + {"B": [1, 2], "foo": [1, None], "bar": ["hello", None]}, ), strict=False, ) - ) - + [ - ( - pl.DataFrame( - { - "A": [{"foo": 1, "bar": "hello"}, None], - "B": [1, 2], - } - ), - pl.DataFrame( - {"B": [1, 2], "foo": [1, None], "bar": ["hello", None]} - ), - ) - ], + ), ) def test_expand_dict(df: DataFrameType, expected: DataFrameType) -> None: transform = ExpandDictTransform( @@ -2351,41 +2338,6 @@ def test_filter_rows_nulls_pandas( result = apply(df, in_transform) assert_frame_equal_with_nans(result, expected) - @staticmethod - @pytest.mark.parametrize( - ("df", "expected"), - list( - zip( - create_test_dataframes( - {"nulls": [1, 2, 3, None, "hello"]}, include=["pandas"] - ), - create_test_dataframes({"nulls": [None]}, include=["pandas"]), - strict=False, - ) - ), - ) - def test_filter_rows_null_pandas_object( - df: DataFrameType, expected: DataFrameType - ) -> None: - in_transform = FilterRowsTransform( - type=TransformType.FILTER_ROWS, - operation="keep_rows", - where=FilterGroup( - type="group", - operator="and", - children=[ - FilterCondition( - type="condition", - column_id="nulls", - operator="in", - value=[None], - ) - ], - ), - ) - result = apply(df, in_transform) - assert_frame_equal_with_nans(result, expected) - @staticmethod @pytest.mark.parametrize( ("df", "expected"), From 4e82164c64e3f9f89d807cd652e2686fd25f7beb Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Tue, 19 May 2026 18:27:12 -0400 Subject: [PATCH 06/15] fix: using polars as optional in test handlers. refactor: adding None == NaN in assert frame equal with nans method to use it in expand_dict test. --- .../ui/_impl/dataframes/test_handlers.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.py index 2b592bb8263..1963e3bc1f3 100644 --- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py +++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py @@ -48,7 +48,7 @@ pytest.importorskip("ibis") pd = pytest.importorskip("pandas") pytest.importorskip("pyarrow") -pl = pytest.importorskip("polars") +pytest.importorskip("polars") def apply(df: DataFrameType, transform: Transform) -> DataFrameType: @@ -86,7 +86,10 @@ def assert_frame_equal(a: DataFrameType, b: DataFrameType) -> None: def assert_frame_equal_with_nans( - a: DataFrameType, b: DataFrameType, allow_nan_equals_zero: bool = False + a: DataFrameType, + b: DataFrameType, + allow_nan_equals_zero: bool = False, + allow_none_equals_nan: bool = False, ) -> None: """ Assert two dataframes are equal, treating NaNs in the same locations as equal. @@ -97,6 +100,9 @@ def assert_frame_equal_with_nans( allow_nan_equals_zero: If True, treat NaN and 0.0 as equivalent values. This is useful for pivot operations where missing aggregations may be filled with 0.0 or NaN depending on the backend. + allow_none_equals_nan: If True, treat None and NaN as equivalent + missing values. This is useful when different backends materialise + missing numeric values differently. """ import math @@ -137,7 +143,25 @@ def assert_frame_equal_with_nans( or val_b == 0.0 ) ) - if not (val_a == val_b or both_nan or nan_or_zero_match): + # Useful for expand dict operations where None and nan are equal + none_nan_match = allow_none_equals_nan and ( + ( + val_a is None + and isinstance(val_b, float) + and math.isnan(val_b) + ) + or ( + val_b is None + and isinstance(val_a, float) + and math.isnan(val_a) + ) + ) + if not ( + val_a == val_b + or both_nan + or nan_or_zero_match + or none_nan_match + ): raise AssertionError( f"DataFrame values differ at column '{col}', row {idx}: {val_a} != {val_b}" ) @@ -1757,9 +1781,10 @@ def test_expand_dict(df: DataFrameType, expected: DataFrameType) -> None: nw_expected = collect_df(expected) result_cols = sorted(nw_result.columns) expected_cols = sorted(nw_expected.columns) - assert_frame_equal( + assert_frame_equal_with_nans( nw_expected.select(expected_cols), nw_result.select(result_cols), + allow_none_equals_nan=True, ) @staticmethod From 88ce223c0d960f732757999bc23cfd53d806418d Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Tue, 19 May 2026 19:28:17 -0400 Subject: [PATCH 07/15] fix: processing pandas backend separately to not cause arrow coercion errors for mixed object columns. --- .../ui/_impl/dataframes/transforms/handlers.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py index 0751520f8d6..b45d0ba2fe1 100644 --- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py +++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py @@ -492,9 +492,21 @@ def handle_explode_columns( def handle_expand_dict( df: DataFrame, transform: ExpandDictTransform ) -> DataFrame: - # Convert to Polars for native unnest, which handles null values - # correctly (fixes: 'NoneType' cannot be converted to 'PyDict' issue #4583) collected_df, undo = collect_and_preserve_type(df) + native_df = collected_df.to_native() + + # Keep pandas handling fully pandas-native so mixed/object columns in + # unrelated fields do not trigger Arrow coercion errors. + if nw.dependencies.is_pandas_dataframe(native_df): + import pandas as pd + + result_df = native_df.copy() + expanded = pd.json_normalize( + result_df.pop(transform.column_id), + ) + expanded.index = result_df.index + return undo(nw.from_native(result_df.join(expanded))) + polars_df = collected_df.to_polars() unnested = polars_df.unnest(transform.column_id) return undo(nw.from_native(unnested)) From 1f722b8d4806d5e7e07c5c758fbc69507881ed9d Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Tue, 19 May 2026 19:39:53 -0400 Subject: [PATCH 08/15] fix: mypy error. --- marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py index b45d0ba2fe1..34893ba85a7 100644 --- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py +++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py @@ -502,7 +502,7 @@ def handle_expand_dict( result_df = native_df.copy() expanded = pd.json_normalize( - result_df.pop(transform.column_id), + result_df.pop(transform.column_id), # type: ignore[arg-type] ) expanded.index = result_df.index return undo(nw.from_native(result_df.join(expanded))) From ff063e58727f99a66f0f105bd68913de24971cdd Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Tue, 19 May 2026 19:48:44 -0400 Subject: [PATCH 09/15] fix: changing the expand dict print code function for pandas to using json normalise following the handlers code. --- .../_plugins/ui/_impl/dataframes/transforms/print_code.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py index 21150ab9d55..44fb8986d17 100644 --- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py +++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py @@ -222,8 +222,11 @@ def generate_where_clause(df_name: str, where: FilterCondition) -> str: elif transform.type == TransformType.EXPAND_DICT: column_id = _as_literal(transform.column_id) - args = f"{df_name}.pop({column_id}).values.tolist()" - return f"{df_name}.join(pd.DataFrame({args}))" + return ( + f"{df_name}.join(" + f"pd.json_normalize({df_name}.pop({column_id})).set_axis({df_name}.index, axis=0)" + f")" + ) elif transform.type == TransformType.UNIQUE: column_ids = transform.column_ids From de2ee2e32da9cb1c062238653a5aa21b5d5efddc Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Thu, 21 May 2026 17:49:34 -0400 Subject: [PATCH 10/15] feat: unnesting only one level of pandas df. --- marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py index ca8d806742f..75e218ceb2f 100644 --- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py +++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py @@ -509,6 +509,7 @@ def handle_expand_dict( result_df = native_df.copy() expanded = pd.json_normalize( result_df.pop(transform.column_id), # type: ignore[arg-type] + max_level=0, ) expanded.index = result_df.index return undo(nw.from_native(result_df.join(expanded))) From 83126b2438db2f8caa5d8dafedbada9516f362ac Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Thu, 21 May 2026 17:50:21 -0400 Subject: [PATCH 11/15] refactor: modifying the pandas print module to expect one level of dict unnesting. --- marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py index 44fb8986d17..e9f1352c639 100644 --- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py +++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py @@ -224,7 +224,7 @@ def generate_where_clause(df_name: str, where: FilterCondition) -> str: column_id = _as_literal(transform.column_id) return ( f"{df_name}.join(" - f"pd.json_normalize({df_name}.pop({column_id})).set_axis({df_name}.index, axis=0)" + f"pd.json_normalize({df_name}.pop({column_id}), max_level=0).set_axis({df_name}.index, axis=0)" f")" ) From 7eedc2dc116bd0ac0ef6994c7762254f1f4f3984 Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Thu, 21 May 2026 17:50:59 -0400 Subject: [PATCH 12/15] tests: added a nested dict test for the expand dict handler. --- .../ui/_impl/dataframes/test_handlers.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.py index 6b0c707c18d..04b7efee859 100644 --- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py +++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py @@ -1787,6 +1787,50 @@ def test_expand_dict(df: DataFrameType, expected: DataFrameType) -> None: allow_none_equals_nan=True, ) + @staticmethod + @pytest.mark.parametrize( + ("df", "expected"), + list( + zip( + create_test_dataframes( + { + "A": [ + {"foo": 1, "nested": {"x": 2}}, + None, + ], + "B": [1, 2], + }, + include=["pandas", "polars"], + ), + create_test_dataframes( + { + "B": [1, 2], + "foo": [1, None], + "nested": [{"x": 2}, None], + }, + include=["pandas", "polars"], + ), + strict=False, + ) + ), + ) + def test_expand_dict_nested_dicts( + df: DataFrameType, expected: DataFrameType + ) -> None: + transform = ExpandDictTransform( + type=TransformType.EXPAND_DICT, column_id="A" + ) + result = apply(df, transform) + nw_result = collect_df(result) + nw_expected = collect_df(expected) + result_cols = sorted(nw_result.columns) + expected_cols = sorted(nw_expected.columns) + assert_frame_equal_with_nans( + nw_expected.select(expected_cols), + nw_result.select(result_cols), + allow_none_equals_nan=True, + ) + @staticmethod @pytest.mark.parametrize( ( From 78cbe7db15df158f8182feac7a25ce06c5988204 Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Thu, 21 May 2026 17:51:22 -0400 Subject: [PATCH 13/15] tests: added a nested dict test for the expand dict handler. --- .../ui/_impl/dataframes/test_print_code.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.py index 57a297b0cb4..14cf22be05b 100644 --- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py +++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py @@ -525,6 +525,52 @@ def test_print_code_result_matches_actual_transform_pandas( ) +@pytest.mark.skipif( + not DependencyManager.pandas.has(), reason="pandas not installed" +) +def test_print_code_expand_dict_nested_dict_pandas() -> None: + import pandas as pd + + transform = ExpandDictTransform( + type=TransformType.EXPAND_DICT, + column_id="dicts", + ) + transformations = Transformations([transform]) + my_df = pd.DataFrame( + { + "dicts": [{"a": 1, "nested": {"x": 2}}, None], + "other": [10, 20], + } + ) + + pandas_code = python_print_transforms( + "my_df", + list(my_df.columns), + transformations.transforms, + python_print_pandas, + ) + assert pandas_code + + loc = {"pd": pd, "my_df": my_df.copy()} + exec(pandas_code, {}, loc) + code_result = loc["my_df_next"] + + nw_df = nw.from_native(my_df.copy(), eager_only=True).lazy() + result_nw = _apply_transforms( + nw_df, + NarwhalsTransformHandler(), + transformations, + ) + real_result = result_nw.collect().to_native().reset_index(drop=True) + + pd.testing.assert_frame_equal( + code_result.reset_index(drop=True), + real_result, + ) + + assert list(code_result.columns) == ["other", "a", "nested"] + + @given( transform=create_transform_strategy( defined_column_id, @@ -772,6 +818,50 @@ def test_print_code_result_matches_actual_transform_polars( pl_testing.assert_frame_equal(code_result, real_result) +@pytest.mark.skipif( + not DependencyManager.polars.has(), reason="polars not installed" +) +def test_print_code_expand_dict_nested_dict_polars() -> None: + import polars as pl + import polars.testing as pl_testing + + transform = ExpandDictTransform( + type=TransformType.EXPAND_DICT, + column_id="dicts", + ) + transformations = Transformations([transform]) + my_df = pl.DataFrame( + { + "dicts": [{"a": 1, "nested": {"x": 2}}, None], + "other": [10, 20], + } + ) + + polars_code = python_print_transforms( + "my_df", + my_df.columns, + transformations.transforms, + python_print_polars, + ) + assert polars_code + + loc = {"pl": pl, "my_df": my_df.clone()} + exec(polars_code, globals(), loc) + code_result = loc["my_df_next"] + + nw_df = nw.from_native(my_df.clone(), eager_only=True).lazy() + result_nw = _apply_transforms( + nw_df, + NarwhalsTransformHandler(), + transformations, + ) + real_result = result_nw.collect().to_native() + + pl_testing.assert_frame_equal(code_result, real_result) + + assert code_result.columns == ["a", "nested", "other"] + + @given( transform=create_transform_strategy( defined_column_id, From 855cd5befd4b5d260d206ce59c4736a5c6a74dac Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Thu, 21 May 2026 18:02:09 -0400 Subject: [PATCH 14/15] docs: adding comment for using max_level=0 in pd.json_normalize for expand dict. --- marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py index 75e218ceb2f..8e0f04fc611 100644 --- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py +++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py @@ -507,6 +507,8 @@ def handle_expand_dict( import pandas as pd result_df = native_df.copy() + # max_level=0 was used so that pandas doesn't recursively unnest dicts + # causing mismatch between pandas vs. polars df expanded = pd.json_normalize( result_df.pop(transform.column_id), # type: ignore[arg-type] max_level=0, From 1eda0e6557606026c0dc58a2dd9959f343db1da5 Mon Sep 17 00:00:00 2001 From: Shamik <39588365+Shamik-07@users.noreply.github.com> Date: Fri, 22 May 2026 17:31:56 -0400 Subject: [PATCH 15/15] fix: replacing none columns with empty dicts to support older version of pandas, which raises an error with pd.json_normalise for expand_dict function. --- marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py | 7 ++++++- .../_plugins/ui/_impl/dataframes/transforms/print_code.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py index 8e0f04fc611..9c1519bcbef 100644 --- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py +++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py @@ -509,8 +509,13 @@ def handle_expand_dict( result_df = native_df.copy() # max_level=0 was used so that pandas doesn't recursively unnest dicts # causing mismatch between pandas vs. polars df + # using the map function to replace the None values + # Replace top-level null rows so pandas 2.x can normalise them, needed for + # older versions of pandas running on py310 otherwise CI will fail expanded = pd.json_normalize( - result_df.pop(transform.column_id), # type: ignore[arg-type] + result_df.pop(transform.column_id).map( + lambda value: {} if value is None else value + ), # type: ignore[arg-type] max_level=0, ) expanded.index = result_df.index diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py index e9f1352c639..70e324f7b73 100644 --- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py +++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py @@ -224,7 +224,7 @@ def generate_where_clause(df_name: str, where: FilterCondition) -> str: column_id = _as_literal(transform.column_id) return ( f"{df_name}.join(" - f"pd.json_normalize({df_name}.pop({column_id}), max_level=0).set_axis({df_name}.index, axis=0)" + f"pd.json_normalize({df_name}.pop({column_id}).map(lambda value: {{}} if value is None else value), max_level=0).set_axis({df_name}.index, axis=0)" f")" )