From 1255c91793c9858bb9b3ff6ab0225ea796436561 Mon Sep 17 00:00:00 2001 From: tomvothecoder Date: Tue, 30 Jul 2024 15:08:42 -0700 Subject: [PATCH 01/14] Add `_apply_weight_threshold()` method Add `_get_masked_weights()` Update the order of methods Convert temporal weight threshold methods to general functions - Add tests for threshold functions Add tests for utils Apply suggestions from code review Apply suggestions from code review Fix masking of data in `_group_average()` - This method now masks data using the weights grouped properly, instead of using the `weight_var_with_weight_threshold()` function - Add `from __future__ import annotations` to `spatial.py` - Add and update units Fix docstring Remove unused `_mask_var_with_weight_threshold()` function Fix TypeError due to annotaitons --- tests/test_temporal.py | 267 +++++++++++++++++++++++++++++++++-------- xcdat/temporal.py | 149 +++++++++++++++++------ xcdat/utils.py | 2 + 3 files changed, 332 insertions(+), 86 deletions(-) diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 0047c551..8fa92da6 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -494,14 +494,6 @@ def test_weighted_annual_averages(self): cftime.DatetimeGregorian(2001, 1, 1), ], ), - coords={ - "time": np.array( - [ - cftime.DatetimeGregorian(2000, 1, 1), - cftime.DatetimeGregorian(2001, 1, 1), - ], - ) - }, dims=["time"], attrs={ "axis": "T", @@ -596,14 +588,6 @@ def test_weighted_annual_averages_with_chunking(self): cftime.DatetimeGregorian(2001, 1, 1), ], ), - coords={ - "time": np.array( - [ - cftime.DatetimeGregorian(2000, 1, 1), - cftime.DatetimeGregorian(2001, 1, 1), - ], - ) - }, dims=["time"], attrs={ "axis": "T", @@ -627,28 +611,26 @@ def test_weighted_annual_averages_with_chunking(self): assert result.ts.attrs == expected.ts.attrs assert result.time.attrs == expected.time.attrs - def test_weighted_seasonal_averages_with_DJF_without_dropping_incomplete_seasons( - self, - ): - ds = self.ds.copy() + def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_seasons(self): + ds = generate_dataset(decode_times=True, cf_compliant=True, has_bounds=True) result = ds.temporal.group_average( "ts", "season", - season_config={"dec_mode": "DJF", "drop_incomplete_seasons": False}, + season_config={"dec_mode": "DJF", "drop_incomplete_seasons": True}, ) + expected = ds.copy() expected = expected.drop_dims("time") expected["ts"] = xr.DataArray( name="ts", - data=np.array([[[2.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + data=np.ones((4, 4, 4)), coords={ "lat": expected.lat, "lon": expected.lon, "time": xr.DataArray( data=np.array( [ - cftime.DatetimeGregorian(2000, 1, 1), cftime.DatetimeGregorian(2000, 4, 1), cftime.DatetimeGregorian(2000, 7, 1), cftime.DatetimeGregorian(2000, 10, 1), @@ -666,32 +648,32 @@ def test_weighted_seasonal_averages_with_DJF_without_dropping_incomplete_seasons }, dims=["time", "lat", "lon"], attrs={ - "test_attr": "test", "operation": "temporal_avg", "mode": "group_average", "freq": "season", "weighted": "True", - "drop_incomplete_seasons": "False", + "drop_incomplete_seasons": "True", "dec_mode": "DJF", }, ) xr.testing.assert_identical(result, expected) - def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_seasons(self): - ds = generate_dataset(decode_times=True, cf_compliant=True, has_bounds=True) + def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_djf(self): + ds = self.ds.copy() result = ds.temporal.group_average( "ts", "season", - season_config={"dec_mode": "DJF", "drop_incomplete_seasons": True}, + season_config={"dec_mode": "DJF", "drop_incomplete_djf": True}, ) - expected = ds.copy() + # Drop the incomplete DJF seasons + expected = expected.isel(time=slice(2, -1)) expected = expected.drop_dims("time") expected["ts"] = xr.DataArray( name="ts", - data=np.ones((4, 4, 4)), + data=np.array([[[1]], [[1]], [[1]], [[2.0]]]), coords={ "lat": expected.lat, "lon": expected.lon, @@ -715,38 +697,48 @@ def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_seasons(self): }, dims=["time", "lat", "lon"], attrs={ + "test_attr": "test", "operation": "temporal_avg", "mode": "group_average", "freq": "season", "weighted": "True", - "drop_incomplete_seasons": "True", "dec_mode": "DJF", + "drop_incomplete_djf": "True", }, ) xr.testing.assert_identical(result, expected) - def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_djf(self): + def test_weighted_seasonal_averages_with_DJF_without_dropping_incomplete_seasons( + self, + ): ds = self.ds.copy() + ds["ts"] = xr.DataArray( + data=np.array( + [[[2.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]], dtype="float64" + ), + coords={"time": self.ds.time, "lat": self.ds.lat, "lon": self.ds.lon}, + dims=["time", "lat", "lon"], + attrs={"test_attr": "test"}, + ) result = ds.temporal.group_average( "ts", "season", - season_config={"dec_mode": "DJF", "drop_incomplete_djf": True}, + season_config={"dec_mode": "DJF", "drop_incomplete_djf": False}, ) expected = ds.copy() - # Drop the incomplete DJF seasons - expected = expected.isel(time=slice(2, -1)) expected = expected.drop_dims("time") expected["ts"] = xr.DataArray( name="ts", - data=np.array([[[1]], [[1]], [[1]], [[2.0]]]), + data=np.array([[[2.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), coords={ "lat": expected.lat, "lon": expected.lon, "time": xr.DataArray( data=np.array( [ + cftime.DatetimeGregorian(2000, 1, 1), cftime.DatetimeGregorian(2000, 4, 1), cftime.DatetimeGregorian(2000, 7, 1), cftime.DatetimeGregorian(2000, 10, 1), @@ -770,7 +762,7 @@ def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_djf(self): "freq": "season", "weighted": "True", "dec_mode": "DJF", - "drop_incomplete_djf": "True", + "drop_incomplete_djf": "False", }, ) @@ -802,17 +794,102 @@ def test_weighted_seasonal_averages_with_JFD(self): cftime.DatetimeGregorian(2001, 1, 1), ], ), - coords={ - "time": np.array( - [ - cftime.DatetimeGregorian(2000, 1, 1), - cftime.DatetimeGregorian(2000, 4, 1), - cftime.DatetimeGregorian(2000, 7, 1), - cftime.DatetimeGregorian(2000, 10, 1), - cftime.DatetimeGregorian(2001, 1, 1), - ], - ) + dims=["time"], + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", }, + ), + }, + dims=["time", "lat", "lon"], + attrs={ + "test_attr": "test", + "operation": "temporal_avg", + "mode": "group_average", + "freq": "season", + "weighted": "True", + "dec_mode": "JFD", + }, + ) + + xr.testing.assert_identical(result, expected) + + def test_weighted_seasonal_averages_with_JFD_with_min_weight_threshold_of_100_percent( + self, + ): + time = xr.DataArray( + data=np.array( + [ + "2000-01-16T12:00:00.000000000", + "2000-02-15T12:00:00.000000000", + "2000-03-16T12:00:00.000000000", + "2000-06-16T00:00:00.000000000", + "2000-12-16T00:00:00.000000000", + ], + dtype="datetime64[ns]", + ), + dims=["time"], + attrs={"axis": "T", "long_name": "time", "standard_name": "time"}, + ) + time.encoding = {"calendar": "standard"} + time_bnds = xr.DataArray( + name="time_bnds", + data=np.array( + [ + ["2000-01-01T00:00:00.000000000", "2000-02-01T00:00:00.000000000"], + ["2000-02-01T00:00:00.000000000", "2000-03-01T00:00:00.000000000"], + ["2000-03-01T00:00:00.000000000", "2000-04-01T00:00:00.000000000"], + ["2000-06-01T00:00:00.000000000", "2000-07-01T00:00:00.000000000"], + ["2000-11-01T00:00:00.000000000", "2001-01-01T00:00:00.000000000"], + ], + dtype="datetime64[ns]", + ), + coords={"time": time}, + dims=["time", "bnds"], + attrs={"xcdat_bounds": "True"}, + ) + + ds = xr.Dataset( + data_vars={"time_bnds": time_bnds}, + coords={"lat": [-90], "lon": [0], "time": time}, + ) + ds.time.attrs["bounds"] = "time_bnds" + + ds["ts"] = xr.DataArray( + data=np.array( + [[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]], dtype="float64" + ), + coords={"time": self.ds.time, "lat": self.ds.lat, "lon": self.ds.lon}, + dims=["time", "lat", "lon"], + attrs={"test_attr": "test"}, + ) + + # NOTE: If a cell has a missing value for any of the seasons, the average + # for that season should be masked with a min_weight threshold of 100%. + result = ds.temporal.group_average( + "ts", + "season", + season_config={"dec_mode": "JFD"}, + min_weight=1.0, + ) + expected = ds.copy() + expected = expected.drop_dims("time") + expected["ts"] = xr.DataArray( + name="ts", + data=np.array([[[np.nan]], [[1.0]], [[1.0]]]), + coords={ + "lat": expected.lat, + "lon": expected.lon, + "time": xr.DataArray( + data=np.array( + [ + cftime.DatetimeGregorian(2000, 1, 1), + cftime.DatetimeGregorian(2000, 4, 1), + cftime.DatetimeGregorian(2000, 7, 1), + ], + ), dims=["time"], attrs={ "axis": "T", @@ -1209,6 +1286,102 @@ def test_weighted_monthly_averages_with_masked_data(self): xr.testing.assert_identical(result, expected) + def test_weighted_monthly_averages_with_masked_data_and_min_weight_threshold_of_100_percent( + self, + ): + # Set up dataset + ds = xr.Dataset( + coords={ + "lat": [-90], + "lon": [0], + "time": xr.DataArray( + data=np.array( + [ + "2000-01-01T00:00:00.000000000", + "2000-02-01T00:00:00.000000000", + "2000-02-15T00:00:00.000000000", + "2000-04-01T00:00:00.000000000", + "2001-02-01T00:00:00.000000000", + ], + dtype="datetime64[ns]", + ), + dims=["time"], + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, + ), + } + ) + ds.time.encoding = {"calendar": "standard"} + + ds["time_bnds"] = xr.DataArray( + name="time_bnds", + data=np.array( + [ + ["2000-01-01T00:00:00.000000000", "2000-02-01T00:00:00.000000000"], + ["2000-02-01T00:00:00.000000000", "2000-03-01T00:00:00.000000000"], + ["2000-02-01T00:00:00.000000000", "2000-03-01T00:00:00.000000000"], + ["2000-04-01T00:00:00.000000000", "2000-05-01T00:00:00.000000000"], + ["2001-02-01T00:00:00.000000000", "2001-03-01T00:00:00.000000000"], + ], + dtype="datetime64[ns]", + ), + coords={"time": ds.time}, + dims=["time", "bnds"], + attrs={"xcdat_bounds": "True"}, + ) + + ds["ts"] = xr.DataArray( + data=np.array([[[2]], [[np.nan]], [[1]], [[1]], [[1]]]), + coords={"lat": ds.lat, "lon": ds.lon, "time": ds.time}, + dims=["time", "lat", "lon"], + attrs={"test_attr": "test"}, + ) + + # NOTE: If a cell has a missing value for any of the months, the average + # for that month should be masked with a min_weight threshold of 100%. + result = ds.temporal.group_average("ts", "month", min_weight=0.55) + expected = ds.copy() + expected = expected.drop_dims("time") + expected["ts"] = xr.DataArray( + name="ts", + data=np.array([[[2.0]], [[np.nan]], [[1.0]], [[1.0]]]), + coords={ + "lat": expected.lat, + "lon": expected.lon, + "time": xr.DataArray( + data=np.array( + [ + cftime.DatetimeGregorian(2000, 1, 1), + cftime.DatetimeGregorian(2000, 2, 1), + cftime.DatetimeGregorian(2000, 4, 1), + cftime.DatetimeGregorian(2001, 2, 1), + ], + ), + dims=["time"], + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, + ), + }, + dims=["time", "lat", "lon"], + attrs={ + "test_attr": "test", + "operation": "temporal_avg", + "mode": "group_average", + "freq": "month", + "weighted": "True", + }, + ) + + xr.testing.assert_identical(result, expected) + def test_weighted_daily_averages(self): ds = self.ds.copy() diff --git a/xcdat/temporal.py b/xcdat/temporal.py index 14cbe2df..7d544c50 100644 --- a/xcdat/temporal.py +++ b/xcdat/temporal.py @@ -19,6 +19,7 @@ from xcdat._logger import _setup_custom_logger from xcdat.axis import get_dim_coords from xcdat.dataset import _get_data_var +from xcdat.utils import _get_masked_weights, _validate_min_weight logger = _setup_custom_logger(__name__) @@ -266,6 +267,7 @@ def group_average( keep_weights: bool = False, season_config: SeasonConfigInput = DEFAULT_SEASON_CONFIG, skipna: bool | None = None, + min_weight: float | None = None, ): """Returns a Dataset with average of a data variable by time group. @@ -367,6 +369,10 @@ def group_average( skips missing values for float dtypes; other dtypes either do not have a sentinel missing value (int) or ``skipna=True`` has not been implemented (object, datetime64 or timedelta64). + min_weight : float | None, optional + Fraction of data coverage (i..e, weight) needed to return a + temporal average value. Value must range from 0 to 1, by default + None (equivalent to ``min_weight=0.0``). Returns ------- @@ -446,6 +452,7 @@ def group_average( keep_weights=keep_weights, season_config=season_config, skipna=skipna, + min_weight=min_weight, ) def climatology( @@ -899,10 +906,13 @@ def _averager( reference_period: tuple[str, str] | None = None, season_config: SeasonConfigInput = DEFAULT_SEASON_CONFIG, skipna: bool | None = None, + min_weight: float | None = None, ) -> xr.Dataset: """Averages a data variable based on the averaging mode and frequency.""" ds = self._dataset.copy() - self._set_arg_attrs(mode, freq, weighted, reference_period, season_config) + self._set_arg_attrs( + mode, freq, weighted, reference_period, season_config, min_weight + ) # Preprocess the dataset based on method argument values. ds = self._preprocess_dataset(ds) @@ -983,6 +993,7 @@ def _set_arg_attrs( weighted: bool, reference_period: tuple[str, str] | None = None, season_config: SeasonConfigInput = DEFAULT_SEASON_CONFIG, + min_weight: float | None = None, ): """Validates method arguments and sets them as object attributes. @@ -998,6 +1009,10 @@ def _set_arg_attrs( A dictionary for "season" frequency configurations. If configs for predefined seasons are passed, configs for custom seasons are ignored and vice versa, by default DEFAULT_SEASON_CONFIG. + min_weight : float | None, optional + Fraction of data coverage (i..e, weight) needed to return a + temporal average value. Value must range from 0 to 1, by default + None (equivalent to ``min_weight=0.0``). Raises ------ @@ -1025,6 +1040,7 @@ def _set_arg_attrs( self._mode = mode self._freq = freq self._weighted = weighted + self._min_weight = _validate_min_weight(min_weight) self._reference_period = None if reference_period is not None: @@ -1541,53 +1557,108 @@ def _group_average( """ dv = _get_data_var(ds, data_var) - # Label the time coordinates for grouping weights and the data variable - # values. + # Label the time coordinates with groups for grouping data and weights. self._labeled_time = self._label_time_coords(dv[self.dim]) dv = dv.assign_coords({self.dim: self._labeled_time}) if self._weighted: - self._weights = self._get_weights(ds, data_var) + dv_avg = self._weighted_group_average(ds, dv, skipna) + else: + dv_avg = self._group_data(dv).mean(skipna=skipna) - # Weight the data variable. - dv *= self._weights + # After grouping and aggregating, xarray removes attributes from the + # grouped time coordinate. The `keep_attrs=True` option only preserves + # attributes for data variables, not coordinates. Therefore, we manually + # restore the time coordinate's attributes below. + dv_avg[self.dim].attrs = self._labeled_time.attrs + dv_avg[self.dim].encoding = self._labeled_time.encoding - # Ensure missing data (`np.nan`) receives no weight (zero). To - # achieve this, first broadcast the one-dimensional (temporal - # dimension) shape of the `weights` DataArray to the - # multi-dimensional shape of its corresponding data variable. - weights = self._weights - if dv.chunks: - # For Dask-backed data variables, chunk the weights along the - # time dimension before broadcasting to avoid eager evaluation - # of the masking step. - weights = weights.chunk({self.dim: dv.chunksizes[self.dim]}) - weights, _ = xr.broadcast(self._weights, dv) - weights = xr.where(dv.copy().isnull(), 0.0, weights) - - # Perform weighted average using the formula - # WA = sum(data*weights) / sum(weights). The denominator must be - # included to take into account zero weight for missing data. - with xr.set_options(keep_attrs=True): - dv = self._group_data(dv).sum(skipna=skipna) / self._group_data( - weights - ).sum(skipna=skipna) - - # Restore the data variable's name. - dv.name = data_var - else: - dv = self._group_data(dv).mean(skipna=skipna) + dv_avg = self._add_operation_attrs(dv_avg) - # After grouping and aggregating, the grouped time dimension's - # attributes are removed. Xarray's `keep_attrs=True` option only keeps - # attributes for data variables and not their coordinates, so the - # coordinate attributes have to be restored manually. - dv[self.dim].attrs = self._labeled_time.attrs - dv[self.dim].encoding = self._labeled_time.encoding + return dv_avg - dv = self._add_operation_attrs(dv) + def _weighted_group_average( + self, + ds: xr.Dataset, + dv: xr.DataArray, + skipna: bool | None, + ) -> xr.DataArray: + """Compute the weighted group average of a data variable. - return dv + This method applies weights to the data variable, groups the weighted data, + and computes the average by dividing the sum of weighted data by the sum of + weights for non-missing data. It handles missing values according to the + `skipna` parameter and ensures that weights for missing data are excluded + from the denominator. Optionally, results are masked where the sum of weights + falls below a minimum threshold. + + Parameters + ---------- + ds : xr.Dataset + The input xarray Dataset containing the data variable and any coordinate information. + dv : xr.DataArray + The data variable to be averaged. + skipna : bool or None + If True, skip NaN values when computing sums. If False, propagate NaNs. + If None, the default behavior of xarray's sum is used. + + Returns + ------- + xr.DataArray + The weighted group average of the data variable, with the same name as `dv`. + Values are set to NaN where the sum of weights is below the minimum threshold. + + Notes + ----- + - Weights are masked to zero where data is missing. + - For Dask-backed data, weights are chunked to avoid eager evaluation. + - The minimum weight threshold is controlled by `self._min_weight`. + """ + # Keep the original weights for other operations and make a copy + # to avoid modifying the original weights. + self._weights = self._get_weights(ds, dv.name) + weights = self._weights.copy() + + # For Dask-backed data variables, chunk the weights along the + # time dimension before broadcasting to avoid eager evaluation + # of the masking step. + if dv.chunks: + weights = weights.chunk({self.dim: dv.chunksizes[self.dim]}) + + # Apply the weights to data. + dv_weighted = dv * weights + + # Group and sum weighted data, skippiing NaNs if specified. + dv_group_sum = self._group_data(dv_weighted).sum(skipna=skipna) + + # Mask weights where data is missing (set to zero). + masked_weights = _get_masked_weights(dv, self._weights) + + # Group and sum masked weights. + masked_weights_group_sum = self._group_data(masked_weights).sum(skipna=skipna) + + # Compute weighted average using the formula: + # WA = sum(data * weights) / sum(weights for non-missing data) + # The denominator ensures that only weights for non-missing data + # are included, so missing data (with zero weight) does not + # affect the result. + dv_avg = dv_group_sum / masked_weights_group_sum + + # Restore the data variables name which gets lost with groupby + # arithmetic. + dv_avg.name = dv.name + + # Set averaged data to NaN where masked weights are below the + # minimum threshold. + if self._min_weight > 0.0: + dv_avg = xr.where( + masked_weights_group_sum >= self._min_weight, + dv_avg, + np.nan, + keep_attrs=True, + ) + + return dv_avg def _get_weights(self, ds: xr.Dataset, data_var: str) -> xr.DataArray: """Calculates weights for a data variable using time bounds. diff --git a/xcdat/utils.py b/xcdat/utils.py index 04edf36c..c1770924 100644 --- a/xcdat/utils.py +++ b/xcdat/utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import importlib import json From 1e3cef9f0d88345cecb7c1614f38fc6ee7495103 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 17 Jul 2025 13:51:12 -0700 Subject: [PATCH 02/14] Fix tests --- tests/test_temporal.py | 51 ++++++++++------------- xcdat/temporal.py | 95 ++++++++++++++++++++++-------------------- 2 files changed, 72 insertions(+), 74 deletions(-) diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 8fa92da6..b97a87f8 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -611,26 +611,28 @@ def test_weighted_annual_averages_with_chunking(self): assert result.ts.attrs == expected.ts.attrs assert result.time.attrs == expected.time.attrs - def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_seasons(self): - ds = generate_dataset(decode_times=True, cf_compliant=True, has_bounds=True) + def test_weighted_seasonal_averages_with_DJF_without_dropping_incomplete_seasons( + self, + ): + ds = self.ds.copy() result = ds.temporal.group_average( "ts", "season", - season_config={"dec_mode": "DJF", "drop_incomplete_seasons": True}, + season_config={"dec_mode": "DJF", "drop_incomplete_seasons": False}, ) - expected = ds.copy() expected = expected.drop_dims("time") expected["ts"] = xr.DataArray( name="ts", - data=np.ones((4, 4, 4)), + data=np.array([[[2.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), coords={ "lat": expected.lat, "lon": expected.lon, "time": xr.DataArray( data=np.array( [ + cftime.DatetimeGregorian(2000, 1, 1), cftime.DatetimeGregorian(2000, 4, 1), cftime.DatetimeGregorian(2000, 7, 1), cftime.DatetimeGregorian(2000, 10, 1), @@ -648,32 +650,32 @@ def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_seasons(self): }, dims=["time", "lat", "lon"], attrs={ + "test_attr": "test", "operation": "temporal_avg", "mode": "group_average", "freq": "season", "weighted": "True", - "drop_incomplete_seasons": "True", + "drop_incomplete_seasons": "False", "dec_mode": "DJF", }, ) xr.testing.assert_identical(result, expected) - def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_djf(self): - ds = self.ds.copy() + def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_seasons(self): + ds = generate_dataset(decode_times=True, cf_compliant=True, has_bounds=True) result = ds.temporal.group_average( "ts", "season", - season_config={"dec_mode": "DJF", "drop_incomplete_djf": True}, + season_config={"dec_mode": "DJF", "drop_incomplete_seasons": True}, ) + expected = ds.copy() - # Drop the incomplete DJF seasons - expected = expected.isel(time=slice(2, -1)) expected = expected.drop_dims("time") expected["ts"] = xr.DataArray( name="ts", - data=np.array([[[1]], [[1]], [[1]], [[2.0]]]), + data=np.ones((4, 4, 4)), coords={ "lat": expected.lat, "lon": expected.lon, @@ -697,48 +699,38 @@ def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_djf(self): }, dims=["time", "lat", "lon"], attrs={ - "test_attr": "test", "operation": "temporal_avg", "mode": "group_average", "freq": "season", "weighted": "True", + "drop_incomplete_seasons": "True", "dec_mode": "DJF", - "drop_incomplete_djf": "True", }, ) xr.testing.assert_identical(result, expected) - def test_weighted_seasonal_averages_with_DJF_without_dropping_incomplete_seasons( - self, - ): + def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_djf(self): ds = self.ds.copy() - ds["ts"] = xr.DataArray( - data=np.array( - [[[2.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]], dtype="float64" - ), - coords={"time": self.ds.time, "lat": self.ds.lat, "lon": self.ds.lon}, - dims=["time", "lat", "lon"], - attrs={"test_attr": "test"}, - ) result = ds.temporal.group_average( "ts", "season", - season_config={"dec_mode": "DJF", "drop_incomplete_djf": False}, + season_config={"dec_mode": "DJF", "drop_incomplete_djf": True}, ) expected = ds.copy() + # Drop the incomplete DJF seasons + expected = expected.isel(time=slice(2, -1)) expected = expected.drop_dims("time") expected["ts"] = xr.DataArray( name="ts", - data=np.array([[[2.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + data=np.array([[[1]], [[1]], [[1]], [[2.0]]]), coords={ "lat": expected.lat, "lon": expected.lon, "time": xr.DataArray( data=np.array( [ - cftime.DatetimeGregorian(2000, 1, 1), cftime.DatetimeGregorian(2000, 4, 1), cftime.DatetimeGregorian(2000, 7, 1), cftime.DatetimeGregorian(2000, 10, 1), @@ -762,7 +754,7 @@ def test_weighted_seasonal_averages_with_DJF_without_dropping_incomplete_seasons "freq": "season", "weighted": "True", "dec_mode": "DJF", - "drop_incomplete_djf": "False", + "drop_incomplete_djf": "True", }, ) @@ -810,6 +802,7 @@ def test_weighted_seasonal_averages_with_JFD(self): "mode": "group_average", "freq": "season", "weighted": "True", + "drop_incomplete_seasons": "False", "dec_mode": "JFD", }, ) diff --git a/xcdat/temporal.py b/xcdat/temporal.py index 7d544c50..4d245e20 100644 --- a/xcdat/temporal.py +++ b/xcdat/temporal.py @@ -1598,9 +1598,11 @@ def _weighted_group_average( The input xarray Dataset containing the data variable and any coordinate information. dv : xr.DataArray The data variable to be averaged. - skipna : bool or None - If True, skip NaN values when computing sums. If False, propagate NaNs. - If None, the default behavior of xarray's sum is used. + skipna : bool | None + If True, skip missing values (as marked by NaN). By default, only + skips missing values for float dtypes; other dtypes either do not + have a sentinel missing value (int) or ``skipna=True`` has not been + implemented (object, datetime64 or timedelta64). Returns ------- @@ -1614,50 +1616,53 @@ def _weighted_group_average( - For Dask-backed data, weights are chunked to avoid eager evaluation. - The minimum weight threshold is controlled by `self._min_weight`. """ - # Keep the original weights for other operations and make a copy - # to avoid modifying the original weights. - self._weights = self._get_weights(ds, dv.name) - weights = self._weights.copy() - - # For Dask-backed data variables, chunk the weights along the - # time dimension before broadcasting to avoid eager evaluation - # of the masking step. - if dv.chunks: - weights = weights.chunk({self.dim: dv.chunksizes[self.dim]}) - - # Apply the weights to data. - dv_weighted = dv * weights - - # Group and sum weighted data, skippiing NaNs if specified. - dv_group_sum = self._group_data(dv_weighted).sum(skipna=skipna) - - # Mask weights where data is missing (set to zero). - masked_weights = _get_masked_weights(dv, self._weights) - - # Group and sum masked weights. - masked_weights_group_sum = self._group_data(masked_weights).sum(skipna=skipna) - - # Compute weighted average using the formula: - # WA = sum(data * weights) / sum(weights for non-missing data) - # The denominator ensures that only weights for non-missing data - # are included, so missing data (with zero weight) does not - # affect the result. - dv_avg = dv_group_sum / masked_weights_group_sum - - # Restore the data variables name which gets lost with groupby - # arithmetic. - dv_avg.name = dv.name - - # Set averaged data to NaN where masked weights are below the - # minimum threshold. - if self._min_weight > 0.0: - dv_avg = xr.where( - masked_weights_group_sum >= self._min_weight, - dv_avg, - np.nan, - keep_attrs=True, + with xr.set_options(keep_attrs=True): + # Keep the original weights for other operations and make a copy + # to avoid modifying the original weights. + self._weights = self._get_weights(ds, str(dv.name)) + weights = self._weights.copy() + + # For Dask-backed data variables, chunk the weights along the + # time dimension before broadcasting to avoid eager evaluation + # of the masking step. + if dv.chunks: + weights = weights.chunk({self.dim: dv.chunksizes[self.dim]}) + + # Apply the weights to data. + dv_weighted = dv * weights + + # Group and sum weighted data, skippiing NaNs if specified. + dv_group_sum = self._group_data(dv_weighted).sum(skipna=skipna) + + # Mask weights where data is missing (set to zero). + masked_weights = _get_masked_weights(dv, self._weights) + + # Group and sum masked weights. + masked_weights_group_sum = self._group_data(masked_weights).sum( + skipna=skipna ) + # Compute weighted average using the formula: + # WA = sum(data * weights) / sum(weights for non-missing data) + # The denominator ensures that only weights for non-missing data + # are included, so missing data (with zero weight) does not + # affect the result. + dv_avg = dv_group_sum / masked_weights_group_sum + + # Restore the data variables name which gets lost with groupby + # arithmetic. + dv_avg.name = dv.name + + # Set averaged data to NaN where masked weights are below the + # minimum threshold. + if self._min_weight > 0.0: + dv_avg = xr.where( + masked_weights_group_sum >= self._min_weight, + dv_avg, + np.nan, + keep_attrs=True, + ) + return dv_avg def _get_weights(self, ds: xr.Dataset, data_var: str) -> xr.DataArray: From 261e203b416f9818bd4784ee2163a46cde74e18f Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 17 Jul 2025 15:59:28 -0700 Subject: [PATCH 03/14] Update tests --- tests/test_temporal.py | 191 ++++++++++++++++++++++++++++++++--------- xcdat/temporal.py | 10 +-- 2 files changed, 154 insertions(+), 47 deletions(-) diff --git a/tests/test_temporal.py b/tests/test_temporal.py index b97a87f8..29b11b16 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -809,8 +809,67 @@ def test_weighted_seasonal_averages_with_JFD(self): xr.testing.assert_identical(result, expected) - def test_weighted_seasonal_averages_with_JFD_with_min_weight_threshold_of_100_percent( - self, + @pytest.mark.parametrize( + "min_weight, ts_data, expected_data", + [ + # min_weight=0.0, all missing, output is all np.nan + ( + 0.0, + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=1.0, all missing, output is all np.nan + ( + 1.0, + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=0.25, first group has 33% data, second group 0%; keeps first only + ( + 0.25, + np.array([[[np.nan]], [[1.0]], [[np.nan]], [[1.0]], [[2.0]]]), + np.array([[[1.6777778]], [[np.nan]], [[1.0]]]), + ), + # min_weight=0.33, first group has 33% data, keeps first group + ( + 0.33, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[2.0]], [[1.0]], [[1.0]]]), + ), + # min_weight=0.66, first group has 33% data, drops first group + ( + 0.66, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[1.0]], [[1.0]]]), + ), + # min_weight=0.66, first group has 66% data, keeps first group + ( + 0.66, + np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[1.6777778]], [[1.0]], [[1.0]]]), + ), + # min_weight=1.0, first group has 66% data, drops first group + ( + 1.0, + np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[1.0]], [[1.0]]]), + ), + # min_weight=0.0, all present, output is weighted mean + ( + 0.0, + np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[1.50413223]], [[1.0]], [[1.0]]]), + ), + # min_weight=1.0, all present, output is weighted mean + ( + 1.0, + np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[1.50413223]], [[1.0]], [[1.0]]]), + ), + ], + ) + def test_weighted_seasonal_averages_with_JFD_and_min_weight_threshold( + self, min_weight, ts_data, expected_data ): time = xr.DataArray( data=np.array( @@ -824,7 +883,12 @@ def test_weighted_seasonal_averages_with_JFD_with_min_weight_threshold_of_100_pe dtype="datetime64[ns]", ), dims=["time"], - attrs={"axis": "T", "long_name": "time", "standard_name": "time"}, + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, ) time.encoding = {"calendar": "standard"} time_bnds = xr.DataArray( @@ -848,30 +912,25 @@ def test_weighted_seasonal_averages_with_JFD_with_min_weight_threshold_of_100_pe data_vars={"time_bnds": time_bnds}, coords={"lat": [-90], "lon": [0], "time": time}, ) - ds.time.attrs["bounds"] = "time_bnds" ds["ts"] = xr.DataArray( - data=np.array( - [[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]], dtype="float64" - ), - coords={"time": self.ds.time, "lat": self.ds.lat, "lon": self.ds.lon}, + data=ts_data, + coords={"time": time, "lat": ds.lat, "lon": ds.lon}, dims=["time", "lat", "lon"], attrs={"test_attr": "test"}, ) - # NOTE: If a cell has a missing value for any of the seasons, the average - # for that season should be masked with a min_weight threshold of 100%. result = ds.temporal.group_average( "ts", "season", season_config={"dec_mode": "JFD"}, - min_weight=1.0, + min_weight=min_weight, ) expected = ds.copy() expected = expected.drop_dims("time") expected["ts"] = xr.DataArray( name="ts", - data=np.array([[[np.nan]], [[1.0]], [[1.0]]]), + data=expected_data, coords={ "lat": expected.lat, "lon": expected.lon, @@ -904,7 +963,7 @@ def test_weighted_seasonal_averages_with_JFD_with_min_weight_threshold_of_100_pe }, ) - xr.testing.assert_identical(result, expected) + xr.testing.assert_allclose(result["ts"], expected["ts"]) def test_raises_error_with_incorrect_custom_seasons_argument(self): # Test raises error with non-3 letter strings @@ -1279,10 +1338,72 @@ def test_weighted_monthly_averages_with_masked_data(self): xr.testing.assert_identical(result, expected) - def test_weighted_monthly_averages_with_masked_data_and_min_weight_threshold_of_100_percent( - self, + @pytest.mark.parametrize( + "min_weight, ts_data, expected_data", + [ + # min_weight=0.0, all data missing, all output bins should be np.nan + ( + 0.0, + np.array( + [ + [[np.nan]], + [[np.nan]], + [[np.nan]], + [[np.nan]], + [[np.nan]], + [[np.nan]], + ] + ), + np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=0.5, only one value present in Jan, allows Jan only + ( + 0.5, + np.array( + [ + [[1.0]], + [[np.nan]], + [[np.nan]], + [[np.nan]], + [[np.nan]], + [[np.nan]], + ] + ), + np.array([[[1.0]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=0.5, Jan and Feb have one value present, allows both + ( + 0.5, + np.array( + [[[2.0]], [[np.nan]], [[3.0]], [[np.nan]], [[np.nan]], [[np.nan]]] + ), + np.array([[[2.0]], [[3.0]], [[np.nan]]]), + ), + # min_weight=1.0, Jan has both values present, allows Jan only + ( + 1.0, + np.array( + [[[2.0]], [[1.0]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]] + ), + np.array([[[1.5]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=0.0, all months have both values present, allows all + ( + 0.0, + np.array([[[2.0]], [[1.0]], [[3.0]], [[4.0]], [[5.0]], [[6.0]]]), + np.array([[[1.5]], [[3.5]], [[5.5]]]), + ), + # min_weight=1.0, all months have both values present, allows all + ( + 1.0, + np.array([[[2.0]], [[1.0]], [[3.0]], [[4.0]], [[5.0]], [[6.0]]]), + np.array([[[1.5]], [[3.5]], [[5.5]]]), + ), + ], + ) + def test_weighted_monthly_averages_min_weight_threshold_additional( + self, min_weight, ts_data, expected_data ): - # Set up dataset ds = xr.Dataset( coords={ "lat": [-90], @@ -1291,10 +1412,11 @@ def test_weighted_monthly_averages_with_masked_data_and_min_weight_threshold_of_ data=np.array( [ "2000-01-01T00:00:00.000000000", + "2000-01-15T00:00:00.000000000", "2000-02-01T00:00:00.000000000", "2000-02-15T00:00:00.000000000", - "2000-04-01T00:00:00.000000000", - "2001-02-01T00:00:00.000000000", + "2000-03-01T00:00:00.000000000", + "2000-03-15T00:00:00.000000000", ], dtype="datetime64[ns]", ), @@ -1309,16 +1431,16 @@ def test_weighted_monthly_averages_with_masked_data_and_min_weight_threshold_of_ } ) ds.time.encoding = {"calendar": "standard"} - ds["time_bnds"] = xr.DataArray( name="time_bnds", data=np.array( [ + ["2000-01-01T00:00:00.000000000", "2000-02-01T00:00:00.000000000"], ["2000-01-01T00:00:00.000000000", "2000-02-01T00:00:00.000000000"], ["2000-02-01T00:00:00.000000000", "2000-03-01T00:00:00.000000000"], ["2000-02-01T00:00:00.000000000", "2000-03-01T00:00:00.000000000"], - ["2000-04-01T00:00:00.000000000", "2000-05-01T00:00:00.000000000"], - ["2001-02-01T00:00:00.000000000", "2001-03-01T00:00:00.000000000"], + ["2000-03-01T00:00:00.000000000", "2000-04-01T00:00:00.000000000"], + ["2000-03-01T00:00:00.000000000", "2000-04-01T00:00:00.000000000"], ], dtype="datetime64[ns]", ), @@ -1326,22 +1448,16 @@ def test_weighted_monthly_averages_with_masked_data_and_min_weight_threshold_of_ dims=["time", "bnds"], attrs={"xcdat_bounds": "True"}, ) - ds["ts"] = xr.DataArray( - data=np.array([[[2]], [[np.nan]], [[1]], [[1]], [[1]]]), + data=ts_data, coords={"lat": ds.lat, "lon": ds.lon, "time": ds.time}, dims=["time", "lat", "lon"], - attrs={"test_attr": "test"}, ) - - # NOTE: If a cell has a missing value for any of the months, the average - # for that month should be masked with a min_weight threshold of 100%. - result = ds.temporal.group_average("ts", "month", min_weight=0.55) - expected = ds.copy() - expected = expected.drop_dims("time") + result = ds.temporal.group_average("ts", "month", min_weight=min_weight) + expected = ds.copy().drop_dims("time") expected["ts"] = xr.DataArray( name="ts", - data=np.array([[[2.0]], [[np.nan]], [[1.0]], [[1.0]]]), + data=expected_data, coords={ "lat": expected.lat, "lon": expected.lon, @@ -1350,8 +1466,7 @@ def test_weighted_monthly_averages_with_masked_data_and_min_weight_threshold_of_ [ cftime.DatetimeGregorian(2000, 1, 1), cftime.DatetimeGregorian(2000, 2, 1), - cftime.DatetimeGregorian(2000, 4, 1), - cftime.DatetimeGregorian(2001, 2, 1), + cftime.DatetimeGregorian(2000, 3, 1), ], ), dims=["time"], @@ -1364,16 +1479,8 @@ def test_weighted_monthly_averages_with_masked_data_and_min_weight_threshold_of_ ), }, dims=["time", "lat", "lon"], - attrs={ - "test_attr": "test", - "operation": "temporal_avg", - "mode": "group_average", - "freq": "month", - "weighted": "True", - }, ) - - xr.testing.assert_identical(result, expected) + xr.testing.assert_equal(result["ts"], expected["ts"]) def test_weighted_daily_averages(self): ds = self.ds.copy() diff --git a/xcdat/temporal.py b/xcdat/temporal.py index 4d245e20..c915cc28 100644 --- a/xcdat/temporal.py +++ b/xcdat/temporal.py @@ -1635,7 +1635,7 @@ def _weighted_group_average( dv_group_sum = self._group_data(dv_weighted).sum(skipna=skipna) # Mask weights where data is missing (set to zero). - masked_weights = _get_masked_weights(dv, self._weights) + masked_weights = _get_masked_weights(dv_weighted, self._weights) # Group and sum masked weights. masked_weights_group_sum = self._group_data(masked_weights).sum( @@ -1649,10 +1649,6 @@ def _weighted_group_average( # affect the result. dv_avg = dv_group_sum / masked_weights_group_sum - # Restore the data variables name which gets lost with groupby - # arithmetic. - dv_avg.name = dv.name - # Set averaged data to NaN where masked weights are below the # minimum threshold. if self._min_weight > 0.0: @@ -1663,6 +1659,10 @@ def _weighted_group_average( keep_attrs=True, ) + # Restore the data variables name which gets lost after arithmetic + # and masking operations. + dv_avg.name = dv.name + return dv_avg def _get_weights(self, ds: xr.Dataset, data_var: str) -> xr.DataArray: From cde8740178592e3420abe604050ffb2236fb58b8 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 17 Jul 2025 16:01:34 -0700 Subject: [PATCH 04/14] Revert rebase changes --- tests/test_temporal.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 29b11b16..5197f4b6 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -494,6 +494,14 @@ def test_weighted_annual_averages(self): cftime.DatetimeGregorian(2001, 1, 1), ], ), + coords={ + "time": np.array( + [ + cftime.DatetimeGregorian(2000, 1, 1), + cftime.DatetimeGregorian(2001, 1, 1), + ], + ) + }, dims=["time"], attrs={ "axis": "T", @@ -588,6 +596,14 @@ def test_weighted_annual_averages_with_chunking(self): cftime.DatetimeGregorian(2001, 1, 1), ], ), + coords={ + "time": np.array( + [ + cftime.DatetimeGregorian(2000, 1, 1), + cftime.DatetimeGregorian(2001, 1, 1), + ], + ) + }, dims=["time"], attrs={ "axis": "T", @@ -786,6 +802,17 @@ def test_weighted_seasonal_averages_with_JFD(self): cftime.DatetimeGregorian(2001, 1, 1), ], ), + coords={ + "time": np.array( + [ + cftime.DatetimeGregorian(2000, 1, 1), + cftime.DatetimeGregorian(2000, 4, 1), + cftime.DatetimeGregorian(2000, 7, 1), + cftime.DatetimeGregorian(2000, 10, 1), + cftime.DatetimeGregorian(2001, 1, 1), + ], + ) + }, dims=["time"], attrs={ "axis": "T", From cf6aa70ae5c60685c91804a14aa186af3b624852 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 17 Jul 2025 16:06:58 -0700 Subject: [PATCH 05/14] Fix test --- tests/test_temporal.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 5197f4b6..f36ff188 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -802,17 +802,6 @@ def test_weighted_seasonal_averages_with_JFD(self): cftime.DatetimeGregorian(2001, 1, 1), ], ), - coords={ - "time": np.array( - [ - cftime.DatetimeGregorian(2000, 1, 1), - cftime.DatetimeGregorian(2000, 4, 1), - cftime.DatetimeGregorian(2000, 7, 1), - cftime.DatetimeGregorian(2000, 10, 1), - cftime.DatetimeGregorian(2001, 1, 1), - ], - ) - }, dims=["time"], attrs={ "axis": "T", From 7a69a898e26c25ab2a2f281260f4315e86615e18 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Fri, 18 Jul 2025 13:40:06 -0700 Subject: [PATCH 06/14] Add `min_weight` to `climatology` and `departures` - Add tests --- tests/test_temporal.py | 955 +++++++++++++++++++++++++++++++++++++---- xcdat/temporal.py | 20 +- 2 files changed, 895 insertions(+), 80 deletions(-) diff --git a/tests/test_temporal.py b/tests/test_temporal.py index f36ff188..e4247e9a 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -627,7 +627,7 @@ def test_weighted_annual_averages_with_chunking(self): assert result.ts.attrs == expected.ts.attrs assert result.time.attrs == expected.time.attrs - def test_weighted_seasonal_averages_with_DJF_without_dropping_incomplete_seasons( + def test_weighted_seasonal_averages_with_DJF( self, ): ds = self.ds.copy() @@ -678,6 +678,166 @@ def test_weighted_seasonal_averages_with_DJF_without_dropping_incomplete_seasons xr.testing.assert_identical(result, expected) + @pytest.mark.parametrize( + "min_weight, ts_data, expected_data", + [ + # min_weight=0.0: all missing, output is all np.nan + ( + 0.0, + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=0.0: all present, output is weighted mean + ( + 0.0, + np.array([[[1.0]], [[1.0]], [[2.0]], [[1.0]], [[1.0]]]), + np.array([[[1.318681]], [[1.0]], [[1.0]]]), + ), + # min_weight=0.25: (2000, DJF) meet threshold; + # (2000, MAM) below threshold (np.nan); + # (2000, JJA) meet threshold. + ( + 0.25, + np.array([[[np.nan]], [[1.0]], [[np.nan]], [[np.nan]], [[2.0]]]), + np.array([[[1.0]], [[np.nan]], [[2.0]]]), + ), + # min_weight=0.33: (2000, DJF), (2000, MAM), (2000, JJA) all meet threshold + ( + 0.33, + np.array([[[np.nan]], [[1.0]], [[np.nan]], [[1.0]], [[2.0]]]), + np.array([[[1.0]], [[1.0]], [[2.0]]]), + ), + # min_weight=0.33: (2000, DJF) below threshold (edge case, Feb has + # less weight); (2000, MAM), (2000, JJA) meet threshold + ( + 0.33, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[1.0]], [[2.0]]]), + ), + # min_weight=0.66: (2000, DJF), (2000, MAM), (2000, JJA) all meet threshold + ( + 0.66, + np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[1.0]], [[1.0]], [[2.0]]]), + ), + # min_weight=1.0: all missing, output is all np.nan + ( + 1.0, + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=1.0: (2000, DJF) below threshold; + # (2000, MAM), (2000, JJA) all meet threshold + ( + 1.0, + np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[1.0]], [[2.0]]]), + ), + # min_weight=1.0: all meet threshold, output is weighted mean + ( + 1.0, + np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[1]], [[1.0]], [[2.0]]]), + ), + ], + ) + def test_weighted_seasonal_averages_with_DJF_and_min_weight_threshold( + self, min_weight, ts_data, expected_data + ): + time = xr.DataArray( + data=np.array( + [ + "1999-12-16T00:00:00.000000000", # (2000, DJF) + "2000-01-16T12:00:00.000000000", # (2000, DJF) + "2000-02-15T12:00:00.000000000", # (2000, DJF) + "2000-03-16T12:00:00.000000000", # (2000, MAM) + "2000-06-16T00:00:00.000000000", # (2000, JJA) + ], + dtype="datetime64[ns]", + ), + dims=["time"], + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, + ) + time.encoding = {"calendar": "standard"} + time_bnds = xr.DataArray( + name="time_bnds", + data=np.array( + [ + ["1999-12-01T00:00:00.000000000", "2000-01-01T00:00:00.000000000"], + ["2000-01-01T00:00:00.000000000", "2000-02-01T00:00:00.000000000"], + ["2000-02-01T00:00:00.000000000", "2000-03-01T00:00:00.000000000"], + ["2000-03-01T00:00:00.000000000", "2000-04-01T00:00:00.000000000"], + ["2000-06-01T00:00:00.000000000", "2000-07-01T00:00:00.000000000"], + ], + dtype="datetime64[ns]", + ), + coords={"time": time}, + dims=["time", "bnds"], + attrs={"xcdat_bounds": "True"}, + ) + + ds = xr.Dataset( + data_vars={"time_bnds": time_bnds}, + coords={"lat": [-90], "lon": [0], "time": time}, + ) + + ds["ts"] = xr.DataArray( + data=ts_data, + coords={"time": time, "lat": ds.lat, "lon": ds.lon}, + dims=["time", "lat", "lon"], + attrs={"test_attr": "test"}, + ) + + result = ds.temporal.group_average( + "ts", + "season", + season_config={"dec_mode": "DJF"}, + min_weight=min_weight, + ) + expected = ds.copy() + expected = expected.drop_dims("time") + expected["ts"] = xr.DataArray( + name="ts", + data=expected_data, + coords={ + "lat": expected.lat, + "lon": expected.lon, + "time": xr.DataArray( + data=np.array( + [ + cftime.DatetimeGregorian(2000, 1, 1), + cftime.DatetimeGregorian(2000, 4, 1), + cftime.DatetimeGregorian(2000, 7, 1), + ], + ), + dims=["time"], + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, + ), + }, + dims=["time", "lat", "lon"], + attrs={ + "test_attr": "test", + "operation": "temporal_avg", + "mode": "group_average", + "freq": "season", + "weighted": "True", + "drop_incomplete_seasons": "False", + "dec_mode": "DJF", + }, + ) + + xr.testing.assert_allclose(result["ts"], expected["ts"]) + def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_seasons(self): ds = generate_dataset(decode_times=True, cf_compliant=True, has_bounds=True) @@ -828,55 +988,61 @@ def test_weighted_seasonal_averages_with_JFD(self): @pytest.mark.parametrize( "min_weight, ts_data, expected_data", [ - # min_weight=0.0, all missing, output is all np.nan + # min_weight=0.0: all missing, output is all np.nan ( 0.0, np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), ), - # min_weight=1.0, all missing, output is all np.nan + # min_weight=0.0: all present, output is weighted mean ( - 1.0, - np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), - np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), + 0.0, + np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[1.50413223]], [[1.0]], [[1.0]]]), ), - # min_weight=0.25, first group has 33% data, second group 0%; keeps first only + # min_weight=0.25: (2000, JFD) meets threshold; + # (2000, MAM) below threshold (np.nan); + # (2000, JJA) meets threshold ( 0.25, np.array([[[np.nan]], [[1.0]], [[np.nan]], [[1.0]], [[2.0]]]), np.array([[[1.6777778]], [[np.nan]], [[1.0]]]), ), - # min_weight=0.33, first group has 33% data, keeps first group + # min_weight=0.33: (2000, JFD), (2000, MAM), and (2000, JJA) all + # meet threshold ( 0.33, np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), np.array([[[2.0]], [[1.0]], [[1.0]]]), ), - # min_weight=0.66, first group has 33% data, drops first group + # min_weight=0.66: (2000, JFD) below threshold (np.nan); + # (2000, MAM) and (2000, JJA) meet threshold ( 0.66, np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), np.array([[[np.nan]], [[1.0]], [[1.0]]]), ), - # min_weight=0.66, first group has 66% data, keeps first group + # min_weight=0.66: (2000, JFD), (2000, MAM), and (2000, JJA) all + # meet threshold ( 0.66, np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), np.array([[[1.6777778]], [[1.0]], [[1.0]]]), ), - # min_weight=1.0, first group has 66% data, drops first group + # min_weight=1.0: all missing, output is all np.nan ( 1.0, - np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), - np.array([[[np.nan]], [[1.0]], [[1.0]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), ), - # min_weight=0.0, all present, output is weighted mean + # min_weight=1.0: (2000, JFD) below threshold (np.nan); + # (2000, MAM) and (2000, JJA) meet threshold. ( - 0.0, - np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), - np.array([[[1.50413223]], [[1.0]], [[1.0]]]), + 1.0, + np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[1.0]], [[1.0]]]), ), - # min_weight=1.0, all present, output is weighted mean + # min_weight=1.0: all present, output is weighted mean ( 1.0, np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), @@ -890,11 +1056,11 @@ def test_weighted_seasonal_averages_with_JFD_and_min_weight_threshold( time = xr.DataArray( data=np.array( [ - "2000-01-16T12:00:00.000000000", - "2000-02-15T12:00:00.000000000", - "2000-03-16T12:00:00.000000000", - "2000-06-16T00:00:00.000000000", - "2000-12-16T00:00:00.000000000", + "2000-01-16T12:00:00.000000000", # JFD + "2000-02-15T12:00:00.000000000", # JFD + "2000-03-16T12:00:00.000000000", # MAM + "2000-06-16T00:00:00.000000000", # JJA + "2000-12-16T00:00:00.000000000", # JFD ], dtype="datetime64[ns]", ), @@ -1733,46 +1899,82 @@ def test_weighted_seasonal_climatology_with_DJF(self): xr.testing.assert_identical(result, expected) - def test_raises_deprecation_warning_with_drop_incomplete_djf_season_config(self): - # NOTE: This will test will also cover the other public APIs that - # have drop_incomplete_djf as a season_config arg. - ds = self.ds.copy() - - with warnings.catch_warnings(record=True) as w: - result = ds.temporal.climatology( - "ts", - "season", - season_config={"dec_mode": "DJF", "drop_incomplete_djf": True}, - ) - - assert len(w) == 1 - assert issubclass(w[0].category, DeprecationWarning) - assert str(w[0].message) == ( - "The `season_config` argument 'drop_incomplete_djf' is being deprecated. " - "Please use 'drop_incomplete_seasons' instead." - ) - - expected = ds.copy() - expected = expected.drop_dims("time") - expected_time = xr.DataArray( + @pytest.mark.parametrize( + "min_weight, ts_data, expected_data", + [ + # min_weight=0.0: all missing (np.nan), output is all np.nan + ( + 0.0, + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=0.0: all present, output is weighted mean. + ( + 0.0, + np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[1.504132]], [[1.0]], [[1.0]]]), + ), + # min_weight=0.25: DJF, MAM, and JJA all meet threshold. + ( + 0.25, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[2.0]], [[1.0]], [[1.0]]]), + ), + # min_weight=0.33: DJF, MAM, and JJA all meet threshold. + ( + 0.33, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[2.0]], [[1.0]], [[1.0]]]), + ), + # min_weight=0.66: DJF below threshold (np.nan); + # MAM, and JJA meet threshold. + ( + 0.66, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[1.0]], [[1.0]]]), + ), + # min_weight=0.66: JFD, MAM, and JJA all meet threshold. + ( + 0.66, + np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[1.677778]], [[1.0]], [[1.0]]]), + ), + # min_weight=1.0: all missing, output is all np.nan. + ( + 1.0, + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=1.0: DJF below threshold (np.nan); + # MAM, and JJA meet threshold. + ( + 1.0, + np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[1.0]], [[1.0]]]), + ), + # min_weight=1.0: All values present, output is weighted mean. + ( + 1.0, + np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[1.504132]], [[1.0]], [[1.0]]]), + ), + ], + ) + def test_weighted_seasonal_climatology_with_DJF_and_min_weight_threshold( + self, min_weight, ts_data, expected_data + ): + time = xr.DataArray( data=np.array( [ - cftime.DatetimeGregorian(1, 1, 1), - cftime.DatetimeGregorian(1, 4, 1), - cftime.DatetimeGregorian(1, 7, 1), - cftime.DatetimeGregorian(1, 10, 1), + "2000-01-16T12:00:00.000000000", # DJF + "2000-02-15T12:00:00.000000000", # DJF + "2000-03-16T12:00:00.000000000", # MAM + "2000-06-16T00:00:00.000000000", # JJA + "2000-12-16T00:00:00.000000000", # DJF ], + dtype="datetime64[ns]", ), - coords={ - "time": np.array( - [ - cftime.DatetimeGregorian(1, 1, 1), - cftime.DatetimeGregorian(1, 4, 1), - cftime.DatetimeGregorian(1, 7, 1), - cftime.DatetimeGregorian(1, 10, 1), - ], - ), - }, + dims=["time"], attrs={ "axis": "T", "long_name": "time", @@ -1780,25 +1982,151 @@ def test_raises_deprecation_warning_with_drop_incomplete_djf_season_config(self) "bounds": "time_bnds", }, ) - expected["ts"] = xr.DataArray( - name="ts", - data=np.ones((4, 4, 4)), - coords={"lat": expected.lat, "lon": expected.lon, "time": expected_time}, - dims=["time", "lat", "lon"], - attrs={ - "operation": "temporal_avg", - "mode": "climatology", - "freq": "season", - "weighted": "True", - "drop_incomplete_djf": "True", - "dec_mode": "DJF", - }, - ) - - xr.testing.assert_identical(result, expected) - - def test_weighted_seasonal_climatology_with_DJF_and_skipna(self): - ds = self.ds.copy(deep=True) + time.encoding = {"calendar": "standard"} + time_bnds = xr.DataArray( + name="time_bnds", + data=np.array( + [ + ["2000-01-01T00:00:00.000000000", "2000-02-01T00:00:00.000000000"], + ["2000-02-01T00:00:00.000000000", "2000-03-01T00:00:00.000000000"], + ["2000-03-01T00:00:00.000000000", "2000-04-01T00:00:00.000000000"], + ["2000-06-01T00:00:00.000000000", "2000-07-01T00:00:00.000000000"], + ["2000-11-01T00:00:00.000000000", "2001-01-01T00:00:00.000000000"], + ], + dtype="datetime64[ns]", + ), + coords={"time": time}, + dims=["time", "bnds"], + attrs={"xcdat_bounds": "True"}, + ) + + ds = xr.Dataset( + data_vars={"time_bnds": time_bnds}, + coords={"lat": [-90], "lon": [0], "time": time}, + ) + + ds["ts"] = xr.DataArray( + data=ts_data, + coords={"time": time, "lat": ds.lat, "lon": ds.lon}, + dims=["time", "lat", "lon"], + attrs={"test_attr": "test"}, + ) + + result = ds.temporal.climatology( + "ts", + "season", + season_config={"dec_mode": "DJF"}, + min_weight=min_weight, + ) + expected = ds.copy() + expected = expected.drop_dims("time") + expected_time = xr.DataArray( + data=np.array( + [ + cftime.DatetimeGregorian(1, 1, 1), + cftime.DatetimeGregorian(1, 4, 1), + cftime.DatetimeGregorian(1, 7, 1), + ], + ), + coords={ + "time": np.array( + [ + cftime.DatetimeGregorian(1, 1, 1), + cftime.DatetimeGregorian(1, 4, 1), + cftime.DatetimeGregorian(1, 7, 1), + ], + ), + }, + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, + ) + expected["ts"] = xr.DataArray( + name="ts", + data=expected_data, + coords={"lat": expected.lat, "lon": expected.lon, "time": expected_time}, + dims=["time", "lat", "lon"], + attrs={ + "operation": "temporal_avg", + "mode": "climatology", + "freq": "season", + "weighted": "True", + "drop_incomplete_seasons": "False", + "dec_mode": "DJF", + }, + ) + + xr.testing.assert_allclose(result["ts"], expected["ts"]) + + def test_raises_deprecation_warning_with_drop_incomplete_djf_season_config(self): + # NOTE: This will test will also cover the other public APIs that + # have drop_incomplete_djf as a season_config arg. + ds = self.ds.copy() + + with warnings.catch_warnings(record=True) as w: + result = ds.temporal.climatology( + "ts", + "season", + season_config={"dec_mode": "DJF", "drop_incomplete_djf": True}, + ) + + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + assert str(w[0].message) == ( + "The `season_config` argument 'drop_incomplete_djf' is being deprecated. " + "Please use 'drop_incomplete_seasons' instead." + ) + + expected = ds.copy() + expected = expected.drop_dims("time") + expected_time = xr.DataArray( + data=np.array( + [ + cftime.DatetimeGregorian(1, 1, 1), + cftime.DatetimeGregorian(1, 4, 1), + cftime.DatetimeGregorian(1, 7, 1), + cftime.DatetimeGregorian(1, 10, 1), + ], + ), + coords={ + "time": np.array( + [ + cftime.DatetimeGregorian(1, 1, 1), + cftime.DatetimeGregorian(1, 4, 1), + cftime.DatetimeGregorian(1, 7, 1), + cftime.DatetimeGregorian(1, 10, 1), + ], + ), + }, + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, + ) + expected["ts"] = xr.DataArray( + name="ts", + data=np.ones((4, 4, 4)), + coords={"lat": expected.lat, "lon": expected.lon, "time": expected_time}, + dims=["time", "lat", "lon"], + attrs={ + "operation": "temporal_avg", + "mode": "climatology", + "freq": "season", + "weighted": "True", + "drop_incomplete_djf": "True", + "dec_mode": "DJF", + }, + ) + + xr.testing.assert_identical(result, expected) + + def test_weighted_seasonal_climatology_with_DJF_and_skipna(self): + ds = self.ds.copy(deep=True) # Replace all MAM values with np.nan. djf_months = [3, 4, 5] @@ -1964,7 +2292,169 @@ def test_weighted_seasonal_climatology_with_JFD(self): }, ) - xr.testing.assert_identical(result, expected) + xr.testing.assert_identical(result, expected) + + @pytest.mark.parametrize( + "min_weight, ts_data, expected_data", + [ + # min_weight=0.0: all missing (np.nan), output is all np.nan + ( + 0.0, + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=0.0: all present, output is weighted mean. + ( + 0.0, + np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[1.504132]], [[1.0]], [[1.0]]]), + ), + # min_weight=0.25: JFD, MAM, and JJA all meet threshold. + ( + 0.25, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[2.0]], [[1.0]], [[1.0]]]), + ), + # min_weight=0.33: JFD, MAM, and JJA all meet threshold. + ( + 0.33, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[2.0]], [[1.0]], [[1.0]]]), + ), + # min_weight=0.66: JFD below threshold (np.nan); + # MAM, and JJA meet threshold. + ( + 0.66, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[1.0]], [[1.0]]]), + ), + # min_weight=0.66: JFD, MAM, and JJA all meet threshold. + ( + 0.66, + np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[1.677778]], [[1.0]], [[1.0]]]), + ), + # min_weight=1.0: all missing, output is all np.nan. + ( + 1.0, + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=0.1.0: JFD below threshold (np.nan); + # MAM, and JJA meet threshold. + ( + 1.0, + np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[1.0]], [[1.0]]]), + ), + # min_weight=1.0: All values present, output is weighted mean. + ( + 1.0, + np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[1.504132]], [[1.0]], [[1.0]]]), + ), + ], + ) + def test_weighted_seasonal_climatology_with_JFD_and_min_weight_threshold( + self, min_weight, ts_data, expected_data + ): + time = xr.DataArray( + data=np.array( + [ + "2000-01-16T12:00:00.000000000", # JFD + "2000-02-15T12:00:00.000000000", # JFD + "2000-03-16T12:00:00.000000000", # MAM + "2000-06-16T00:00:00.000000000", # JJA + "2000-12-16T00:00:00.000000000", # JFD + ], + dtype="datetime64[ns]", + ), + dims=["time"], + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, + ) + time.encoding = {"calendar": "standard"} + time_bnds = xr.DataArray( + name="time_bnds", + data=np.array( + [ + ["2000-01-01T00:00:00.000000000", "2000-02-01T00:00:00.000000000"], + ["2000-02-01T00:00:00.000000000", "2000-03-01T00:00:00.000000000"], + ["2000-03-01T00:00:00.000000000", "2000-04-01T00:00:00.000000000"], + ["2000-06-01T00:00:00.000000000", "2000-07-01T00:00:00.000000000"], + ["2000-11-01T00:00:00.000000000", "2001-01-01T00:00:00.000000000"], + ], + dtype="datetime64[ns]", + ), + coords={"time": time}, + dims=["time", "bnds"], + attrs={"xcdat_bounds": "True"}, + ) + + ds = xr.Dataset( + data_vars={"time_bnds": time_bnds}, + coords={"lat": [-90], "lon": [0], "time": time}, + ) + + ds["ts"] = xr.DataArray( + data=ts_data, + coords={"time": time, "lat": ds.lat, "lon": ds.lon}, + dims=["time", "lat", "lon"], + attrs={"test_attr": "test"}, + ) + + result = ds.temporal.climatology( + "ts", + "season", + season_config={"dec_mode": "JFD"}, + min_weight=min_weight, + ) + expected = ds.copy() + expected = expected.drop_dims("time") + expected_time = xr.DataArray( + data=np.array( + [ + cftime.DatetimeGregorian(1, 1, 1), + cftime.DatetimeGregorian(1, 4, 1), + cftime.DatetimeGregorian(1, 7, 1), + ], + ), + coords={ + "time": np.array( + [ + cftime.DatetimeGregorian(1, 1, 1), + cftime.DatetimeGregorian(1, 4, 1), + cftime.DatetimeGregorian(1, 7, 1), + ], + ), + }, + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, + ) + expected["ts"] = xr.DataArray( + name="ts", + data=expected_data, + coords={"lat": expected.lat, "lon": expected.lon, "time": expected_time}, + dims=["time", "lat", "lon"], + attrs={ + "operation": "temporal_avg", + "mode": "climatology", + "freq": "season", + "weighted": "True", + "drop_incomplete_seasons": "False", + "dec_mode": "JFD", + }, + ) + + xr.testing.assert_allclose(result["ts"], expected["ts"]) def test_weighted_custom_seasonal_climatology(self): ds = self.ds.copy() @@ -2700,6 +3190,161 @@ def test_weighted_seasonal_departures_with_DJF(self): xr.testing.assert_identical(result, expected) + @pytest.mark.parametrize( + "min_weight, ts_data, expected_data", + [ + # min_weight=0.0: all missing (np.nan), output is all np.nan + ( + 0.0, + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=0.0: all present, output are departures from the mean. + ( + 0.0, + np.array([[[2.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[-0.340659]], [[0.0]], [[0.0]], [[0.659341]]]), + ), + # min_weight=0.25: (2000, DJF), (2000, MAM) and (2000, JJA) meet threshold; + # (2001, DJF) below threshold (np.nan); + ( + 0.25, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[0.0]], [[0.0]], [[0.0]], [[np.nan]]]), + ), + # min_weight=0.33: (2000, DJF) and (2001, DJF) below threshold (np.nan); + # (2000, MAM), (2000, JJA) meet threshold. + ( + 0.33, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[0.0]], [[0.0]], [[np.nan]]]), + ), + # min_weight=1.0: (2000, DJF) and (2001, DJF) below threshold (np.nan); + # (2000, MAM) and (2000, JJA) meet threshold. + ( + 0.66, + np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[0.0]], [[0.0]], [[np.nan]]]), + ), + # min_weight=1.0: all missing, output is all np.nan. + ( + 1.0, + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=1.0: (2000, DJF) and (2001, DJF) below threshold (np.nan); + # (2000, MAM) and (2000, JJA) meet threshold. + ( + 1.0, + np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[0.0]], [[0.0]], [[np.nan]]]), + ), + # min_weight=1.0: All values present, output are departures from the mean. + ( + 1.0, + np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[0.0]], [[0.0]], [[0.0]], [[0.0]]]), + ), + ], + ) + def test_weighted_seasonal_departures_with_DJF_and_min_weight_threshold( + self, min_weight, ts_data, expected_data + ): + time = xr.DataArray( + data=np.array( + [ + "1999-12-16T00:00:00.000000000", # DJF + "2000-01-16T12:00:00.000000000", # DJF + "2000-02-15T12:00:00.000000000", # DJF + "2000-03-16T12:00:00.000000000", # MAM + "2000-06-16T00:00:00.000000000", # JJA + ], + dtype="datetime64[ns]", + ), + dims=["time"], + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, + ) + time.encoding = {"calendar": "standard"} + time_bnds = xr.DataArray( + name="time_bnds", + data=np.array( + [ + ["1999-12-01T00:00:00.000000000", "2000-01-01T00:00:00.000000000"], + ["2000-01-01T00:00:00.000000000", "2000-02-01T00:00:00.000000000"], + ["2000-02-01T00:00:00.000000000", "2000-03-01T00:00:00.000000000"], + ["2000-03-01T00:00:00.000000000", "2000-04-01T00:00:00.000000000"], + ["2000-06-01T00:00:00.000000000", "2000-07-01T00:00:00.000000000"], + ], + dtype="datetime64[ns]", + ), + coords={"time": time}, + dims=["time", "bnds"], + attrs={"xcdat_bounds": "True"}, + ) + + ds = xr.Dataset( + data_vars={"time_bnds": time_bnds}, + coords={"lat": [-90], "lon": [0], "time": time}, + ) + + ds["ts"] = xr.DataArray( + data=ts_data, + coords={"time": time, "lat": ds.lat, "lon": ds.lon}, + dims=["time", "lat", "lon"], + attrs={"test_attr": "test"}, + ) + + result = ds.temporal.departures( + "ts", + "season", + season_config={"dec_mode": "DJF"}, + min_weight=min_weight, + ) + expected = ds.copy() + expected = expected.drop_dims("time") + expected["ts"] = xr.DataArray( + name="ts", + data=expected_data, + coords={ + "lat": expected.lat, + "lon": expected.lon, + "time": xr.DataArray( + data=np.array( + [ + cftime.DatetimeGregorian(2000, 1, 1), + cftime.DatetimeGregorian(2000, 4, 1), + cftime.DatetimeGregorian(2000, 7, 1), + cftime.DatetimeGregorian(2001, 1, 1), + ], + ), + dims=["time"], + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, + ), + }, + dims=["time", "lat", "lon"], + attrs={ + "test_attr": "test", + "operation": "temporal_avg", + "mode": "departures", + "freq": "season", + "weighted": "True", + "dec_mode": "DJF", + "drop_incomplete_seasons": "False", + }, + ) + + xr.testing.assert_allclose(result["ts"], expected["ts"]) + def test_weighted_seasonal_departures_with_DJF_and_skipna(self): ds = self.ds.copy(deep=True) @@ -2937,6 +3582,158 @@ def test_unweighted_seasonal_departures_with_JFD(self): xr.testing.assert_identical(result, expected) + @pytest.mark.parametrize( + "min_weight, ts_data, expected_data", + [ + # min_weight=0.0: all missing (np.nan), output is all np.nan + ( + 0.0, + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=0.0: all present, output are departures from the mean. + ( + 0.0, + np.array([[[2.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[0.0]], [[0.0]], [[0.0]]]), + ), + # min_weight=0.25: (2000, DJF), (2000, MAM) and (2000, JJA) meet threshold; + ( + 0.25, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[0.0]], [[0.0]], [[0.0]]]), + ), + # min_weight=0.33: (2000, DJF), (2000, MAM), (2000, JJA) meet threshold. + ( + 0.33, + np.array([[[np.nan]], [[np.nan]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[0.0]], [[0.0]], [[0.0]]]), + ), + # min_weight=1.0: (2000, DJF) below threshold (np.nan); + # (2000, MAM) and (2000, JJA) meet threshold. + ( + 0.66, + np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[0.0]], [[0.0]]]), + ), + # min_weight=1.0: all missing, output is all np.nan. + ( + 1.0, + np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), + np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), + ), + # min_weight=1.0: (2000, DJF) below threshold (np.nan); + # (2000, MAM) and (2000, JJA) meet threshold. + ( + 1.0, + np.array([[[np.nan]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[np.nan]], [[0.0]], [[0.0]]]), + ), + # min_weight=1.0: All values present, output are departures from the mean. + ( + 1.0, + np.array([[[1.0]], [[1.0]], [[1.0]], [[1.0]], [[2.0]]]), + np.array([[[0.0]], [[0.0]], [[0.0]]]), + ), + ], + ) + def test_weighted_seasonal_departures_with_JFD_and_min_weight_threshold( + self, min_weight, ts_data, expected_data + ): + time = xr.DataArray( + data=np.array( + [ + "2000-01-16T12:00:00.000000000", # DJF + "2000-02-15T12:00:00.000000000", # DJF + "2000-03-16T12:00:00.000000000", # MAM + "2000-06-16T00:00:00.000000000", # JJA + "2000-12-16T00:00:00.000000000", # DJF + ], + dtype="datetime64[ns]", + ), + dims=["time"], + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, + ) + time.encoding = {"calendar": "standard"} + time_bnds = xr.DataArray( + name="time_bnds", + data=np.array( + [ + ["2000-01-01T00:00:00.000000000", "2000-02-01T00:00:00.000000000"], + ["2000-02-01T00:00:00.000000000", "2000-03-01T00:00:00.000000000"], + ["2000-03-01T00:00:00.000000000", "2000-04-01T00:00:00.000000000"], + ["2000-06-01T00:00:00.000000000", "2000-07-01T00:00:00.000000000"], + ["2000-12-01T00:00:00.000000000", "2001-01-01T00:00:00.000000000"], + ], + dtype="datetime64[ns]", + ), + coords={"time": time}, + dims=["time", "bnds"], + attrs={"xcdat_bounds": "True"}, + ) + + ds = xr.Dataset( + data_vars={"time_bnds": time_bnds}, + coords={"lat": [-90], "lon": [0], "time": time}, + ) + + ds["ts"] = xr.DataArray( + data=ts_data, + coords={"time": time, "lat": ds.lat, "lon": ds.lon}, + dims=["time", "lat", "lon"], + attrs={"test_attr": "test"}, + ) + + result = ds.temporal.departures( + "ts", + "season", + season_config={"dec_mode": "JFD"}, + min_weight=min_weight, + ) + expected = ds.copy() + expected = expected.drop_dims("time") + expected["ts"] = xr.DataArray( + name="ts", + data=expected_data, + coords={ + "lat": expected.lat, + "lon": expected.lon, + "time": xr.DataArray( + data=np.array( + [ + cftime.DatetimeGregorian(2000, 1, 1), + cftime.DatetimeGregorian(2000, 4, 1), + cftime.DatetimeGregorian(2000, 7, 1), + ], + ), + dims=["time"], + attrs={ + "axis": "T", + "long_name": "time", + "standard_name": "time", + "bounds": "time_bnds", + }, + ), + }, + dims=["time", "lat", "lon"], + attrs={ + "test_attr": "test", + "operation": "temporal_avg", + "mode": "departures", + "freq": "season", + "weighted": "True", + "drop_incomplete_seasons": "False", + "dec_mode": "JFD", + }, + ) + + xr.testing.assert_allclose(result["ts"], expected["ts"]) + def test_weighted_daily_departures_drops_leap_days_with_matching_calendar(self): time = xr.DataArray( data=np.array( diff --git a/xcdat/temporal.py b/xcdat/temporal.py index c915cc28..18540943 100644 --- a/xcdat/temporal.py +++ b/xcdat/temporal.py @@ -464,6 +464,7 @@ def climatology( reference_period: tuple[str, str] | None = None, season_config: SeasonConfigInput = DEFAULT_SEASON_CONFIG, skipna: bool | None = None, + min_weight: float | None = None, ): """Returns a Dataset with the climatology of a data variable. @@ -574,6 +575,10 @@ def climatology( skips missing values for float dtypes; other dtypes either do not have a sentinel missing value (int) or ``skipna=True`` has not been implemented (object, datetime64 or timedelta64). + min_weight : float | None, optional + Fraction of data coverage (i..e, weight) needed to return a + temporal average value. Value must range from 0 to 1, by default + None (equivalent to ``min_weight=0.0``). Returns ------- @@ -658,6 +663,7 @@ def climatology( reference_period, season_config, skipna, + min_weight=min_weight, ) def departures( @@ -669,6 +675,7 @@ def departures( reference_period: tuple[str, str] | None = None, season_config: SeasonConfigInput = DEFAULT_SEASON_CONFIG, skipna: bool | None = None, + min_weight: float | None = None, ) -> xr.Dataset: """ Returns a Dataset with the climatological departures (anomalies) for a @@ -790,6 +797,10 @@ def departures( skips missing values for float dtypes; other dtypes either do not have a sentinel missing value (int) or ``skipna=True`` has not been implemented (object, datetime64 or timedelta64). + min_weight : float | None, optional + Fraction of data coverage (i..e, weight) needed to return a + temporal average value. Value must range from 0 to 1, by default + None (equivalent to ``min_weight=0.0``). Returns ------- @@ -870,7 +881,13 @@ def departures( inferred_freq = _infer_freq(ds[self.dim]) if inferred_freq != freq: ds_obs = ds_obs.temporal.group_average( - data_var, freq, weighted, keep_weights, season_config, skipna + data_var, + freq, + weighted, + keep_weights, + season_config, + skipna, + min_weight, ) # 4. Calculate the climatology of the data variable. @@ -884,6 +901,7 @@ def departures( reference_period, season_config, skipna, + min_weight=min_weight, ) # 5. Calculate the departures for the data variable. From 3e1f3017c213d32dff892971974061b5e9540551 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Fri, 18 Jul 2025 13:47:39 -0700 Subject: [PATCH 07/14] Add PR review suggestions --- tests/test_temporal.py | 2 +- xcdat/temporal.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_temporal.py b/tests/test_temporal.py index e4247e9a..8c2cc8f9 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -2340,7 +2340,7 @@ def test_weighted_seasonal_climatology_with_JFD(self): np.array([[[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]], [[np.nan]]]), np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), ), - # min_weight=0.1.0: JFD below threshold (np.nan); + # min_weight=1.0: JFD below threshold (np.nan); # MAM, and JJA meet threshold. ( 1.0, diff --git a/xcdat/temporal.py b/xcdat/temporal.py index 18540943..213818c1 100644 --- a/xcdat/temporal.py +++ b/xcdat/temporal.py @@ -370,7 +370,7 @@ def group_average( have a sentinel missing value (int) or ``skipna=True`` has not been implemented (object, datetime64 or timedelta64). min_weight : float | None, optional - Fraction of data coverage (i..e, weight) needed to return a + Fraction of data coverage (i.e., weight) needed to return a temporal average value. Value must range from 0 to 1, by default None (equivalent to ``min_weight=0.0``). @@ -576,7 +576,7 @@ def climatology( have a sentinel missing value (int) or ``skipna=True`` has not been implemented (object, datetime64 or timedelta64). min_weight : float | None, optional - Fraction of data coverage (i..e, weight) needed to return a + Fraction of data coverage (i.e., weight) needed to return a temporal average value. Value must range from 0 to 1, by default None (equivalent to ``min_weight=0.0``). @@ -798,7 +798,7 @@ def departures( have a sentinel missing value (int) or ``skipna=True`` has not been implemented (object, datetime64 or timedelta64). min_weight : float | None, optional - Fraction of data coverage (i..e, weight) needed to return a + Fraction of data coverage (i.e., weight) needed to return a temporal average value. Value must range from 0 to 1, by default None (equivalent to ``min_weight=0.0``). @@ -1028,7 +1028,7 @@ def _set_arg_attrs( predefined seasons are passed, configs for custom seasons are ignored and vice versa, by default DEFAULT_SEASON_CONFIG. min_weight : float | None, optional - Fraction of data coverage (i..e, weight) needed to return a + Fraction of data coverage (i.e., weight) needed to return a temporal average value. Value must range from 0 to 1, by default None (equivalent to ``min_weight=0.0``). @@ -1649,7 +1649,7 @@ def _weighted_group_average( # Apply the weights to data. dv_weighted = dv * weights - # Group and sum weighted data, skippiing NaNs if specified. + # Group and sum weighted data, skipping NaNs if specified. dv_group_sum = self._group_data(dv_weighted).sum(skipna=skipna) # Mask weights where data is missing (set to zero). From 45fff2ebce429dd7e188e25f9639b1a13913e6de Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Fri, 18 Jul 2025 13:48:33 -0700 Subject: [PATCH 08/14] Remove unnecessary future import --- xcdat/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/xcdat/utils.py b/xcdat/utils.py index c1770924..04edf36c 100644 --- a/xcdat/utils.py +++ b/xcdat/utils.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import importlib import json From f783685aa1f2e68836df29394e21c35d8fab8255 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Fri, 18 Jul 2025 13:58:03 -0700 Subject: [PATCH 09/14] Update test comments --- tests/test_temporal.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 8c2cc8f9..204e0413 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -1523,7 +1523,7 @@ def test_weighted_monthly_averages_with_masked_data(self): @pytest.mark.parametrize( "min_weight, ts_data, expected_data", [ - # min_weight=0.0, all data missing, all output bins should be np.nan + # min_weight=0.0, all missing, output is all np.nan ( 0.0, np.array( @@ -1538,7 +1538,14 @@ def test_weighted_monthly_averages_with_masked_data(self): ), np.array([[[np.nan]], [[np.nan]], [[np.nan]]]), ), - # min_weight=0.5, only one value present in Jan, allows Jan only + # min_weight=0.0, all months meet threshold, output is weighted mean. + ( + 0.0, + np.array([[[2.0]], [[1.0]], [[3.0]], [[4.0]], [[5.0]], [[6.0]]]), + np.array([[[1.5]], [[3.5]], [[5.5]]]), + ), + # min_weight=0.5, (2000, Jan) meets threshold; + # (2000, Feb) and (2000, Mar) below threshold (np.nan). ( 0.5, np.array( @@ -1553,7 +1560,8 @@ def test_weighted_monthly_averages_with_masked_data(self): ), np.array([[[1.0]], [[np.nan]], [[np.nan]]]), ), - # min_weight=0.5, Jan and Feb have one value present, allows both + # min_weight=0.5, (2000, Jan) and (2000, Feb) meet threshold; + # (2000, Mar) below threshold (np.nan). ( 0.5, np.array( @@ -1561,7 +1569,8 @@ def test_weighted_monthly_averages_with_masked_data(self): ), np.array([[[2.0]], [[3.0]], [[np.nan]]]), ), - # min_weight=1.0, Jan has both values present, allows Jan only + # min_weight=1.0, (2000, Jan) meets threshold; + # (2000, Feb) and (2000, Mar) below threshold (np.nan). ( 1.0, np.array( @@ -1569,13 +1578,7 @@ def test_weighted_monthly_averages_with_masked_data(self): ), np.array([[[1.5]], [[np.nan]], [[np.nan]]]), ), - # min_weight=0.0, all months have both values present, allows all - ( - 0.0, - np.array([[[2.0]], [[1.0]], [[3.0]], [[4.0]], [[5.0]], [[6.0]]]), - np.array([[[1.5]], [[3.5]], [[5.5]]]), - ), - # min_weight=1.0, all months have both values present, allows all + # min_weight=1.0, all months meet threshold, output is weighted mean. ( 1.0, np.array([[[2.0]], [[1.0]], [[3.0]], [[4.0]], [[5.0]], [[6.0]]]), @@ -1593,12 +1596,12 @@ def test_weighted_monthly_averages_min_weight_threshold_additional( "time": xr.DataArray( data=np.array( [ - "2000-01-01T00:00:00.000000000", - "2000-01-15T00:00:00.000000000", - "2000-02-01T00:00:00.000000000", - "2000-02-15T00:00:00.000000000", - "2000-03-01T00:00:00.000000000", - "2000-03-15T00:00:00.000000000", + "2000-01-01T00:00:00.000000000", # (2000, Jan) + "2000-01-15T00:00:00.000000000", # (2000, Jan) + "2000-02-01T00:00:00.000000000", # (2000, Feb) + "2000-02-15T00:00:00.000000000", # (2000, Feb) + "2000-03-01T00:00:00.000000000", # (2000, Mar) + "2000-03-15T00:00:00.000000000", # (2000, Mar) ], dtype="datetime64[ns]", ), From 23f927fae4a4f48289c685146a69454de5252793 Mon Sep 17 00:00:00 2001 From: tomvothecoder Date: Tue, 19 Aug 2025 16:49:57 -0700 Subject: [PATCH 10/14] Fix masking to use weight fraction instead of abs sum --- xcdat/temporal.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/xcdat/temporal.py b/xcdat/temporal.py index 213818c1..f5c9c27c 100644 --- a/xcdat/temporal.py +++ b/xcdat/temporal.py @@ -1667,11 +1667,19 @@ def _weighted_group_average( # affect the result. dv_avg = dv_group_sum / masked_weights_group_sum - # Set averaged data to NaN where masked weights are below the - # minimum threshold. + # Mask averaged data where the fraction of weights in each group + # does not meet the minimum weight threshold (fractional). if self._min_weight > 0.0: + # The sum of all weights in each group (i.e., full coverage) + weight_sum_all = self._group_data(self._weights).sum(skipna=skipna) + + # Fraction of weights present in each group. + weight_fraction = masked_weights_group_sum / weight_sum_all + + # Mask the averaged data where the weight fraction is below + # the minimum weight threshold. dv_avg = xr.where( - masked_weights_group_sum >= self._min_weight, + weight_fraction >= self._min_weight, dv_avg, np.nan, keep_attrs=True, From 6b3aa8e4a2999753b0ed2f2acd31e643af8fbfe7 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Wed, 20 Aug 2025 10:04:27 -0700 Subject: [PATCH 11/14] Get masked weights using original variable not weighted --- xcdat/temporal.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/xcdat/temporal.py b/xcdat/temporal.py index f5c9c27c..f83aca2f 100644 --- a/xcdat/temporal.py +++ b/xcdat/temporal.py @@ -1652,19 +1652,17 @@ def _weighted_group_average( # Group and sum weighted data, skipping NaNs if specified. dv_group_sum = self._group_data(dv_weighted).sum(skipna=skipna) - # Mask weights where data is missing (set to zero). - masked_weights = _get_masked_weights(dv_weighted, self._weights) - - # Group and sum masked weights. + # Mask weights where data is missing (set to zero), then + # group and sum the masked weights. This ensures that only weights + # corresponding to non-missing data are used in the denominator of + # the weighted average. + masked_weights = _get_masked_weights(dv, self._weights) masked_weights_group_sum = self._group_data(masked_weights).sum( skipna=skipna ) # Compute weighted average using the formula: # WA = sum(data * weights) / sum(weights for non-missing data) - # The denominator ensures that only weights for non-missing data - # are included, so missing data (with zero weight) does not - # affect the result. dv_avg = dv_group_sum / masked_weights_group_sum # Mask averaged data where the fraction of weights in each group From d59ce11ac584a9e0a7268954baf26b7312501282 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Wed, 20 Aug 2025 11:53:56 -0700 Subject: [PATCH 12/14] Add `min_weight` attr to final averaged dataarray .attrs --- tests/test_temporal.py | 42 ++++++++++++++++++++++++++++++++++++++++++ xcdat/temporal.py | 27 +++++++++++++++------------ 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 204e0413..82068962 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -121,6 +121,7 @@ def test_averages_for_yearly_time_series(self): "mode": "average", "freq": "year", "weighted": "True", + "min_weight": 0.0, }, ) @@ -208,6 +209,7 @@ def test_averages_for_monthly_time_series(self): "mode": "average", "freq": "month", "weighted": "True", + "min_weight": 0.0, }, ) @@ -295,6 +297,7 @@ def test_averages_for_daily_time_series(self): "mode": "average", "freq": "day", "weighted": "True", + "min_weight": 0.0, }, ) @@ -380,6 +383,7 @@ def test_averages_for_hourly_time_series(self): "mode": "average", "freq": "hour", "weighted": "True", + "min_weight": 0.0, }, ) @@ -518,6 +522,7 @@ def test_weighted_annual_averages(self): "mode": "group_average", "freq": "year", "weighted": "True", + "min_weight": 0.0, }, ) @@ -569,6 +574,7 @@ def test_weighted_annual_averages_and_skipna(self): "mode": "group_average", "freq": "year", "weighted": "True", + "min_weight": 0.0, }, ) @@ -620,6 +626,7 @@ def test_weighted_annual_averages_with_chunking(self): "mode": "group_average", "freq": "year", "weighted": "True", + "min_weight": 0.0, }, ) @@ -671,6 +678,7 @@ def test_weighted_seasonal_averages_with_DJF( "mode": "group_average", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "False", "dec_mode": "DJF", }, @@ -831,6 +839,7 @@ def test_weighted_seasonal_averages_with_DJF_and_min_weight_threshold( "mode": "group_average", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "False", "dec_mode": "DJF", }, @@ -879,6 +888,7 @@ def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_seasons(self): "mode": "group_average", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "True", "dec_mode": "DJF", }, @@ -929,6 +939,7 @@ def test_weighted_seasonal_averages_with_DJF_and_drop_incomplete_djf(self): "mode": "group_average", "freq": "season", "weighted": "True", + "min_weight": 0.0, "dec_mode": "DJF", "drop_incomplete_djf": "True", }, @@ -978,6 +989,7 @@ def test_weighted_seasonal_averages_with_JFD(self): "mode": "group_average", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "False", "dec_mode": "JFD", }, @@ -1140,6 +1152,7 @@ def test_weighted_seasonal_averages_with_JFD_and_min_weight_threshold( "mode": "group_average", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "False", "dec_mode": "JFD", }, @@ -1238,6 +1251,7 @@ def test_weighted_custom_seasonal_averages(self): "mode": "group_average", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "False", "custom_seasons": [ "JanFebMar", @@ -1304,6 +1318,7 @@ def test_weighted_seasonal_averages_with_custom_seasons_and_all_complete_seasons "mode": "group_average", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "True", "custom_seasons": ["JanMarJun", "FebSep"], }, @@ -1361,6 +1376,7 @@ def test_weighted_custom_seasonal_averages_drops_incomplete_seasons(self): "mode": "group_average", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "True", "custom_seasons": ["NovDec", "FebMarApr"], }, @@ -1418,6 +1434,7 @@ def test_weighted_custom_seasonal_averages_with_seasons_spanning_calendar_years( "mode": "group_average", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "False", "custom_seasons": ["NovDecJanFebMar"], }, @@ -1464,6 +1481,7 @@ def test_weighted_monthly_averages(self): "mode": "group_average", "freq": "month", "weighted": "True", + "min_weight": 0.0, }, ) @@ -1515,6 +1533,7 @@ def test_weighted_monthly_averages_with_masked_data(self): "mode": "group_average", "freq": "month", "weighted": "True", + "min_weight": 0.0, }, ) @@ -1705,6 +1724,7 @@ def test_weighted_daily_averages(self): "mode": "group_average", "freq": "day", "weighted": "True", + "min_weight": 0.0, }, ) @@ -1749,6 +1769,7 @@ def test_weighted_hourly_averages(self): "mode": "group_average", "freq": "hour", "weighted": "True", + "min_weight": 0.0, }, ) @@ -1841,6 +1862,7 @@ def test_subsets_climatology_based_on_reference_period(self): "mode": "climatology", "freq": "season", "weighted": "True", + "min_weight": 0.0, "dec_mode": "DJF", "drop_incomplete_seasons": "True", }, @@ -1895,6 +1917,7 @@ def test_weighted_seasonal_climatology_with_DJF(self): "mode": "climatology", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "True", "dec_mode": "DJF", }, @@ -2057,6 +2080,7 @@ def test_weighted_seasonal_climatology_with_DJF_and_min_weight_threshold( "mode": "climatology", "freq": "season", "weighted": "True", + "min_weight": min_weight, "drop_incomplete_seasons": "False", "dec_mode": "DJF", }, @@ -2121,6 +2145,7 @@ def test_raises_deprecation_warning_with_drop_incomplete_djf_season_config(self) "mode": "climatology", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_djf": "True", "dec_mode": "DJF", }, @@ -2181,6 +2206,7 @@ def test_weighted_seasonal_climatology_with_DJF_and_skipna(self): "mode": "climatology", "freq": "season", "weighted": "True", + "min_weight": 0.0, "dec_mode": "DJF", "drop_incomplete_djf": "True", }, @@ -2238,6 +2264,7 @@ def test_chunked_weighted_seasonal_climatology_with_DJF(self): "mode": "climatology", "freq": "season", "weighted": "True", + "min_weight": 0.0, "dec_mode": "DJF", "drop_incomplete_seasons": "True", }, @@ -2290,6 +2317,7 @@ def test_weighted_seasonal_climatology_with_JFD(self): "mode": "climatology", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "False", "dec_mode": "JFD", }, @@ -2452,6 +2480,7 @@ def test_weighted_seasonal_climatology_with_JFD_and_min_weight_threshold( "mode": "climatology", "freq": "season", "weighted": "True", + "min_weight": min_weight, "drop_incomplete_seasons": "False", "dec_mode": "JFD", }, @@ -2511,6 +2540,7 @@ def test_weighted_custom_seasonal_climatology(self): "mode": "climatology", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "False", "custom_seasons": [ "JanFebMar", @@ -2567,6 +2597,7 @@ def test_weighted_custom_seasonal_climatology_with_seasons_spanning_calendar_yea "mode": "climatology", "freq": "season", "weighted": "True", + "min_weight": 0.0, "drop_incomplete_seasons": "False", "custom_seasons": ["NovDecJanFebMar"], }, @@ -2632,6 +2663,7 @@ def test_weighted_monthly_climatology(self): "mode": "climatology", "freq": "month", "weighted": "True", + "min_weight": 0.0, }, ) @@ -2756,6 +2788,7 @@ def test_weighted_daily_climatology(self): "mode": "climatology", "freq": "day", "weighted": "True", + "min_weight": 0.0, }, ) @@ -2847,6 +2880,7 @@ def test_weighted_daily_climatology_drops_leap_days_with_matching_calendar(self) "mode": "climatology", "freq": "day", "weighted": "True", + "min_weight": 0.0, }, ) @@ -3050,6 +3084,7 @@ def test_seasonal_departures_relative_to_climatology_reference_period(self): "mode": "departures", "freq": "season", "weighted": "True", + "min_weight": 0.0, "dec_mode": "DJF", "drop_incomplete_seasons": "False", }, @@ -3105,6 +3140,7 @@ def test_monthly_departures_relative_to_climatology_reference_period_with_same_o "mode": "departures", "freq": "month", "weighted": "True", + "min_weight": 0.0, }, ) expected["time_bnds"] = xr.DataArray( @@ -3183,6 +3219,7 @@ def test_weighted_seasonal_departures_with_DJF(self): attrs={ "test_attr": "test", "operation": "temporal_avg", + "min_weight": 0.0, "mode": "departures", "freq": "season", "weighted": "True", @@ -3341,6 +3378,7 @@ def test_weighted_seasonal_departures_with_DJF_and_min_weight_threshold( "mode": "departures", "freq": "season", "weighted": "True", + "min_weight": min_weight, "dec_mode": "DJF", "drop_incomplete_seasons": "False", }, @@ -3397,6 +3435,7 @@ def test_weighted_seasonal_departures_with_DJF_and_skipna(self): "mode": "departures", "freq": "season", "weighted": "True", + "min_weight": 0.0, "dec_mode": "DJF", "drop_incomplete_djf": "True", }, @@ -3449,6 +3488,7 @@ def test_weighted_seasonal_departures_with_DJF_and_keep_weights(self): "mode": "departures", "freq": "season", "weighted": "True", + "min_weight": 0.0, "dec_mode": "DJF", "drop_incomplete_seasons": "False", }, @@ -3730,6 +3770,7 @@ def test_weighted_seasonal_departures_with_JFD_and_min_weight_threshold( "mode": "departures", "freq": "season", "weighted": "True", + "min_weight": min_weight, "drop_incomplete_seasons": "False", "dec_mode": "JFD", }, @@ -3823,6 +3864,7 @@ def test_weighted_daily_departures_drops_leap_days_with_matching_calendar(self): "mode": "departures", "freq": "day", "weighted": "True", + "min_weight": 0.0, }, ), }, diff --git a/xcdat/temporal.py b/xcdat/temporal.py index f83aca2f..4d78f7db 100644 --- a/xcdat/temporal.py +++ b/xcdat/temporal.py @@ -2103,14 +2103,15 @@ def _add_operation_attrs(self, data_var: xr.DataArray) -> xr.DataArray: xr.DataArray The data variable with a temporal averaging attributes. """ - data_var.attrs.update( - { - "operation": "temporal_avg", - "mode": self._mode, - "freq": self._freq, - "weighted": str(self._weighted), - } - ) + attrs_to_set = { + "operation": "temporal_avg", + "mode": self._mode, + "freq": self._freq, + "weighted": str(self._weighted), + } + + if self._weighted and hasattr(self, "_min_weight"): + attrs_to_set["min_weight"] = self._min_weight # type: ignore if self._freq == "season": drop_incomplete_seasons = self._season_config["drop_incomplete_seasons"] @@ -2119,16 +2120,18 @@ def _add_operation_attrs(self, data_var: xr.DataArray) -> xr.DataArray: # TODO: Deprecate drop_incomplete_djf. This attr is only set if the # user does not set drop_incomplete_seasons. if drop_incomplete_seasons is False and drop_incomplete_djf is not False: - data_var.attrs["drop_incomplete_djf"] = str(drop_incomplete_djf) + attrs_to_set["drop_incomplete_djf"] = str(drop_incomplete_djf) else: - data_var.attrs["drop_incomplete_seasons"] = str(drop_incomplete_seasons) + attrs_to_set["drop_incomplete_seasons"] = str(drop_incomplete_seasons) custom_seasons = self._season_config.get("custom_seasons") if custom_seasons is not None: - data_var.attrs["custom_seasons"] = list(custom_seasons.keys()) + attrs_to_set["custom_seasons"] = list(custom_seasons.keys()) # type: ignore else: dec_mode = self._season_config.get("dec_mode") - data_var.attrs["dec_mode"] = dec_mode + attrs_to_set["dec_mode"] = dec_mode # type: ignore + + data_var.attrs.update(attrs_to_set) return data_var From 63227a96401f01c6adfcfbc5a61b2acad22e20b8 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Wed, 20 Aug 2025 11:55:05 -0700 Subject: [PATCH 13/14] Update order of min_weight attr in tests --- tests/test_temporal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 82068962..9c1b9d74 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -3219,10 +3219,10 @@ def test_weighted_seasonal_departures_with_DJF(self): attrs={ "test_attr": "test", "operation": "temporal_avg", - "min_weight": 0.0, "mode": "departures", "freq": "season", "weighted": "True", + "min_weight": 0.0, "dec_mode": "DJF", "drop_incomplete_seasons": "False", }, From f21b2b6025bbfad64020b63904de581067232d78 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 21 Aug 2025 09:16:18 -0700 Subject: [PATCH 14/14] Update xcdat/temporal.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- xcdat/temporal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcdat/temporal.py b/xcdat/temporal.py index 4d78f7db..023e94b9 100644 --- a/xcdat/temporal.py +++ b/xcdat/temporal.py @@ -2110,7 +2110,7 @@ def _add_operation_attrs(self, data_var: xr.DataArray) -> xr.DataArray: "weighted": str(self._weighted), } - if self._weighted and hasattr(self, "_min_weight"): + if self._weighted: attrs_to_set["min_weight"] = self._min_weight # type: ignore if self._freq == "season":