diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index b6bca8fdac..dc0d1e8d9a 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -1784,6 +1784,11 @@ def visit_not_in(self, term: BoundTerm, literals: set[L]) -> bool: if upper_bytes is not None: upper = _from_byte_buffer(field.field_type, upper_bytes) + if self._is_nan(upper): + # NaN indicates unreliable bounds. + # See the StrictMetricsEvaluator docs for more. + return ROWS_MIGHT_NOT_MATCH + literals = {val for val in literals if upper >= val} if len(literals) == 0: diff --git a/tests/expressions/test_evaluator.py b/tests/expressions/test_evaluator.py index bba4156e99..18f5aabb0e 100644 --- a/tests/expressions/test_evaluator.py +++ b/tests/expressions/test_evaluator.py @@ -1358,6 +1358,31 @@ def test_strict_not_equal_and_not_in_with_mixed_nans_and_matching_bounds(field_t assert should_read == ROWS_MIGHT_NOT_MATCH, "Should not match: bounds prove the non-NaN value is 5.0" +@pytest.mark.parametrize("field_type", [FloatType(), DoubleType()]) +def test_strict_not_in_with_nan_upper_bound(field_type: PrimitiveType) -> None: + schema = Schema(NestedField(1, "x", field_type, required=False)) + # Column contains {1.0, NaN}: min is 1.0, max is NaN (NaN sorts greatest). + # The non-NaN row 1.0 is in the literal set, so the file cannot be proven + # to fully match NotIn, even though the NaN count is known. + data_file = DataFile.from_args( + file_path="file.parquet", + file_format=FileFormat.PARQUET, + partition={}, + record_count=2, + file_size_in_bytes=1, + value_counts={1: 2}, + null_value_counts={1: 0}, + nan_value_counts={1: 1}, + lower_bounds={1: to_bytes(field_type, 1.0)}, + upper_bounds={1: to_bytes(field_type, float("nan"))}, + ) + + should_read = _StrictMetricsEvaluator(schema, NotIn("x", {1.0, 2.0})).eval(data_file) + assert should_read == ROWS_MIGHT_NOT_MATCH, ( + "NaN upper bound makes the bounds unusable: the non-NaN row 1.0 is in the literal set" + ) + + @pytest.mark.parametrize("field_type", [FloatType(), DoubleType()]) def test_strict_not_equal_and_not_in_with_all_nans(field_type: PrimitiveType) -> None: schema = Schema(NestedField(1, "x", field_type, required=False))