From c576edaa9f2f30df2c795947b36e277bf183d2ca Mon Sep 17 00:00:00 2001 From: Gaurav Singh Date: Sun, 13 Sep 2026 13:02:48 +0530 Subject: [PATCH] fix: connectedComponents max_iter default evaluated to 31 due to ^ precedence In Python `^` is bitwise XOR and `-` binds tighter than `^`, so the declared default `2 ^ 31 - 2` evaluated to `2 ^ 29` = 31 instead of the intended 2147483646. The Python default is always forwarded to the JVM, so the Scala default of Int.MaxValue never applied when calling from Python, and algorithm="graphx" silently returned split components for any graph needing more than 31 Pregel supersteps. Scope is limited to algorithm="graphx": TwoPhase and RandomizedContraction contain no references to maxIter and run to convergence regardless. The documented behaviour already promised the fix -- 05-traversals.md states the maxIter default is Integer.MAX_VALUE (unlimited). Also adds the missing :param max_iter: docstring entry, and two regression tests: a behavioural one on a 50-vertex path graph (diameter 49), which returns 19 components before the fix and 1 after, and a signature guard that catches a reintroduction without starting Spark. Co-Authored-By: Claude Opus 5 (1M context) --- python/graphframes/graphframe.py | 5 ++++- python/tests/test_graphframes.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/python/graphframes/graphframe.py b/python/graphframes/graphframe.py index 667058c1..cbaefdea 100644 --- a/python/graphframes/graphframe.py +++ b/python/graphframes/graphframe.py @@ -661,7 +661,7 @@ def connectedComponents( broadcastThreshold: int = 1000000, useLabelsAsComponents: bool = False, use_local_checkpoints: bool = False, - max_iter: int = 2 ^ 31 - 2, + max_iter: int = 2**31 - 2, storage_level: StorageLevel = StorageLevel.MEMORY_AND_DISK_DESER, ) -> DataFrame: """ @@ -685,6 +685,9 @@ def connectedComponents( a persistent checkpointDir; from the other side, local checkpoints are less reliable and require executors to have big enough local disks. + :param max_iter: maximum number of Pregel supersteps, only used when the algorithm is + "graphx" (default: 2 ** 31 - 2, that is effectively unlimited). The + other algorithms run until convergence and ignore this value. :param storage_level: storage level for both intermediate and final dataframes. :return: DataFrame with new vertices column "component" diff --git a/python/tests/test_graphframes.py b/python/tests/test_graphframes.py index 084cd7b7..58ccd2de 100644 --- a/python/tests/test_graphframes.py +++ b/python/tests/test_graphframes.py @@ -16,6 +16,7 @@ # +import inspect from dataclasses import dataclass import pytest @@ -644,6 +645,28 @@ def test_connected_components_example(spark: SparkSession) -> None: _ = cc.unpersist() +def test_connected_components_graphx_default_max_iter_is_unlimited(spark: SparkSession) -> None: + """Regression: the default `2 ^ 31 - 2` is XOR and evaluated to 31, truncating GraphX.""" + n = 50 + v = spark.createDataFrame([(i,) for i in range(n)], ["id"]) + e = spark.createDataFrame([(i, i + 1) for i in range(n - 1)], ["src", "dst"]) + g = GraphFrame(v, e) + + # max_iter is deliberately not passed: this exercises the default. The path graph has + # diameter n - 1 = 49, so a default capped at 31 supersteps splits it into 19 components. + result = g.connectedComponents(algorithm="graphx") + assert result.count() == n + assert result.select("component").distinct().count() == 1 + + _ = result.unpersist() + + +def test_connected_components_max_iter_default_value() -> None: + """Guard the default itself so a `**` -> `^` regression fails without starting Spark.""" + default = inspect.signature(GraphFrame.connectedComponents).parameters["max_iter"].default + assert default == 2**31 - 2 + + @pytest.mark.parametrize("args", PREGEL_ARGUMENTS, ids=PREGEL_IDS) def test_shortest_paths(spark: SparkSession, args: PregelArguments) -> None: edges = [(1, 2), (1, 5), (2, 3), (2, 5), (3, 4), (4, 5), (4, 6)]