diff --git a/feature_engine/transformation/log.py b/feature_engine/transformation/log.py index 1e34f3308..870f877f8 100644 --- a/feature_engine/transformation/log.py +++ b/feature_engine/transformation/log.py @@ -42,13 +42,18 @@ fit_transform=_fit_transform_docstring, inverse_transform=_inverse_transform_docstring, ) -class LogTransformer(BaseNumericalTransformer): +class LogTransformer(BaseNumericalTransformer, FitFromDictMixin): """ The LogTransformer() applies the natural logarithm or the base 10 logarithm to - numerical variables. The natural logarithm is the logarithm in base e. + numerical variables, optionally after adding a constant C, i.e., log(x + C). - The LogTransformer() only works with positive values. If the variable - contains a zero or a negative value the transformer will return an error. + By default, C=0, so LogTransformer() only works with positive values and + behaves exactly as it always has: if a variable contains a zero or a negative + value, the transformer raises an error. + + To transform variables that contain zero or negative values, pass a non-zero + C: either an explicit constant, "auto" to let the transformer determine a + shift per variable, or a dictionary mapping each variable to its own constant. A list of variables can be passed as an argument. Alternatively, the transformer will automatically select and transform all variables of type numeric. @@ -65,10 +70,26 @@ class LogTransformer(BaseNumericalTransformer): Indicates if the natural or base 10 logarithm should be applied. Can take values 'e' or '10'. + C: "auto", int, float or dict, default=0 + The constant C to add to the variable before the logarithm, i.e., log(x + C). + + - If 0 (the default), no constant is added and the variable must be + strictly positive, matching the transformer's original behavior. + - If int or float, then log(x + C). + - If "auto", then C = abs(min(x)) + 1. + - If dict, dictionary mapping the constant C to apply to each variable. + + Note, when C is a dictionary, the parameter `variables` is ignored. + Attributes ---------- {variables_} + C_: + The constant C added to each variable. Equal to C, unless C = "auto", in + which case it is a dictionary with C = abs(min(variable)) + 1. For strictly + positive variables, C = 0. + {feature_names_in_} {n_features_in_} @@ -109,24 +130,33 @@ def __init__( variables: Union[None, int, str, List[Union[str, int]]] = None, return_empty: bool = False, base: str = "e", + C: Union[int, float, str, Dict[Union[str, int], Union[float, int]]] = 0, ) -> None: if base not in ["e", "10"]: raise ValueError("base can take only '10' or 'e' as values") + if not isinstance(C, (int, float, dict)) and C != "auto": + raise ValueError( + f"C can take only 'auto', integers or floats. Got {C} instead." + ) + _check_return_empty_is_bool(return_empty) self.variables = _check_variables_input_value(variables) self.return_empty = return_empty self.base = base + self.C = C def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ - This transformer does not learn parameters. + Learn the constant C to add to the variable before the logarithm + transformation, if C="auto". Otherwise, this transformer does not learn + parameters. - Selects the numerical variables and determines whether the logarithm - can be applied on the selected variables, i.e., it checks that the variables - are positive. + Selects the numerical variables and, when C=0 (the default), determines + whether the logarithm can be applied on the selected variables, i.e., it + checks that the variables are positive. Parameters ---------- @@ -139,10 +169,28 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ # check input dataframe - X = super().fit(X) + if isinstance(self.C, dict): + X = super()._fit_from_dict(X, self.C) + else: + X = super().fit(X) + + self.C_ = self.C + + # calculate C to add to each variable + if self.C == "auto": + # we add 0 to positive variables + c_dict = {var: 0 for var in self.variables_ if X[var].min() > 0} - # check contains zero or negative values - if (X[self.variables_] <= 0).any().any(): + # we add the minimum plus 1 to non-positive variables + non_positive_vars = [ + var for var in self.variables_ if var not in c_dict.keys() + ] + c_dict.update(dict(X[non_positive_vars].min(axis=0).abs() + 1)) + self.C_ = c_dict # type:ignore + + # C=0 is the original LogTransformer contract: no constant is added, + # so fail fast at fit time exactly as before this class supported C. + if self.C_ == 0 and (X[self.variables_] <= 0).any().any(): raise ValueError( "Some variables contain zero or negative values, can't apply log" ) @@ -151,7 +199,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): def transform(self, X: pd.DataFrame) -> pd.DataFrame: """ - Transform the variables with the logarithm. + Transform the variables with the logarithm of x plus the constant C. Parameters ---------- @@ -167,19 +215,26 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) - # check contains zero or negative values - if (X[self.variables_] <= 0).any().any(): - raise ValueError( + if self.C_ == 0: + error_msg = ( "Some variables contain zero or negative values, can't apply log" ) + else: + error_msg = ( + "Some variables contain zero or negative values after adding" + + " constant C, can't apply log." + ) + + if (X[self.variables_] + self.C_ <= 0).any().any(): + raise ValueError(error_msg) X[self.variables_] = X[self.variables_].astype(float) # transform if self.base == "e": - X.loc[:, self.variables_] = np.log(X.loc[:, self.variables_]) + X.loc[:, self.variables_] = np.log(X.loc[:, self.variables_] + self.C_) elif self.base == "10": - X.loc[:, self.variables_] = np.log10(X.loc[:, self.variables_]) + X.loc[:, self.variables_] = np.log10(X.loc[:, self.variables_] + self.C_) return X @@ -203,17 +258,17 @@ def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: # inverse_transform if self.base == "e": - X.loc[:, self.variables_] = np.exp(X.loc[:, self.variables_]) + X.loc[:, self.variables_] = np.exp(X.loc[:, self.variables_]) - self.C_ elif self.base == "10": - X.loc[:, self.variables_] = np.array(10 ** X.loc[:, self.variables_]) + X.loc[:, self.variables_] = 10 ** X.loc[:, self.variables_] - self.C_ return X def _more_tags(self): tags_dict = _return_tags() # ======= this tests fail because the transformers throw an error - # when the values are 0. Nothing to do with the test itself but - # mostly with the data created and used in the test + # when the values are 0 and C=0 (the default). Nothing to do with the + # test itself but mostly with the data created and used in the test msg = ( "transformers raise errors when data contains zeroes, thus this check fails" ) @@ -231,80 +286,17 @@ def __sklearn_tags__(self): return tags -@Substitution( - return_empty=_return_empty_docstring, - variables_=_variables_attribute_docstring, - feature_names_in_=_feature_names_in_docstring, - n_features_in_=_n_features_in_docstring, - fit_transform=_fit_transform_docstring, - inverse_transform=_inverse_transform_docstring, -) -class LogCpTransformer(BaseNumericalTransformer, FitFromDictMixin): +class LogCpTransformer(LogTransformer): """ LogCpTransformer() applies the transformation log(x + C), where x is the - variable to transform and C is a positive constant. It can apply the natural - logarithm or the base 10 logarithm, where the natural logarithm is logarithm in - base e. - - As the logarithm can only be applied to numerical non-negative values, - LogCpTransformer() extends the functionality of LogTransformer, by adding a - constant to shift the distribution of the variables towards positive values. - - Note that if the variable contains a zero or a negative value after adding a - constant C, the transformer will return an error. This can occur if the values of - the variables in the test set are smaller than those seen during `fit()`. - - A list of variables can be passed as an argument. Alternatively, the transformer - will automatically select and transform all variables of type numeric. - - More details in the :ref:`User Guide `. - - Parameters - ---------- - variables: list, default=None - The list of numerical variables to transform. If None, the transformer - will find and select all numerical variables. If C is a dictionary, then this - parameter is ignored and the variables to transform are selected from the - dictionary keys. - - {return_empty} - - base: string, default='e' - Indicates if the natural or base 10 logarithm should be applied. Can take - values 'e' or '10'. - - C: "auto", int or dict, default="auto" - The constant C to add to the variable before the logarithm, i.e., log(x + C). - - - If int, then log(x + C) - - If "auto", then C = abs(min(x)) + 1 - - If dict, dictionary mapping the constant C to apply to each variable. - - Note, when C is a dictionary, the parameter `variables` is ignored. - - Attributes - ---------- - {variables_} + variable to transform and C is a positive constant. - C_: - The constant C to add to each variable. If C = "auto" a dictionary with - C = abs(min(variable)) + 1. For strictly positive variables, C = 0. + .. note:: + `LogCpTransformer` is being consolidated into `LogTransformer`. New + code should prefer ``LogTransformer(C="auto")``, which reproduces + `LogCpTransformer`'s default behavior exactly. - {feature_names_in_} - - {n_features_in_} - - Methods - ------- - fit: - Learn the constant C. - - {fit_transform} - - {inverse_transform} - - transform: - Transform the variables with the logarithm of x plus C. + See :class:`LogTransformer` for the full parameter and attribute reference. Examples -------- @@ -335,124 +327,16 @@ def __init__( base: str = "e", C: Union[int, float, str, Dict[Union[str, int], Union[float, int]]] = "auto", ) -> None: - - if base not in ["e", "10"]: - raise ValueError( - f"base can take only '10' or 'e' as values. Got {base} instead." - ) - - if not isinstance(C, (int, float, dict)) and C != "auto": - raise ValueError( - f"C can take only 'auto', integers or floats. Got {C} instead." - ) - - _check_return_empty_is_bool(return_empty) - - self.variables = _check_variables_input_value(variables) - self.return_empty = return_empty - self.base = base - self.C = C - - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): - """ - Learn the constant C to add to the variable before the logarithm transformation - if C="auto". - - Select the numerical variables or check that the variables entered by the user - are numerical. Then check that the selected variables are positive after - addition of C. - - Parameters - ---------- - X: pandas DataFrame of shape = [n_samples, n_features]. - The training input samples. Can be the entire dataframe, not just the - variables to transform. - - y: pandas Series, default=None - It is not needed in this transformer. You can pass y or None. - """ - - # check input dataframe - if isinstance(self.C, dict): - X = super()._fit_from_dict(X, self.C) - else: - X = super().fit(X) - - self.C_ = self.C - - # calculate C to add to each variable - if self.C == "auto": - # we add 0 to positive variables - c_dict = {var: 0 for var in self.variables_ if X[var].min() > 0} - - # we add the minimum plus 1 to non-positive variables - non_positive_vars = [ - var for var in self.variables_ if var not in c_dict.keys() - ] - c_dict.update(dict(X[non_positive_vars].min(axis=0).abs() + 1)) - self.C_ = c_dict # type:ignore - - return self - - def transform(self, X: pd.DataFrame) -> pd.DataFrame: - """ - Transform the variables with the logarithm of x plus a constant C. - - Parameters - ---------- - X: pandas DataFrame of shape = [n_samples, n_features] - The data to be transformed. - - Returns - ------- - X_new: pandas dataframe - The dataframe with the transformed variables. - """ - - # check input dataframe and if class was fitted - X = self._check_transform_input_and_state(X) - - # check variable is positive after adding c - error_msg = ( - "Some variables contain zero or negative values after adding" - + " constant C, can't apply log." + super().__init__( + variables=variables, return_empty=return_empty, base=base, C=C ) - if (X[self.variables_] + self.C_ <= 0).any().any(): - raise ValueError(error_msg) - - X[self.variables_] = X[self.variables_].astype(float) - - # transform - if self.base == "e": - X.loc[:, self.variables_] = np.log(X.loc[:, self.variables_] + self.C_) - else: - X.loc[:, self.variables_] = np.log10(X.loc[:, self.variables_] + self.C_) - - return X - - def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: - """ - Convert the data back to the original representation. - - Parameters - ---------- - X: pandas DataFrame of shape = [n_samples, n_features] - The data to be transformed. - - Returns - ------- - X_tr: pandas dataframe - The dataframe with the transformed variables. - """ - - # check input dataframe and if class was fitted - X = self._check_transform_input_and_state(X) - - # inverse transform - if self.base == "e": - X.loc[:, self.variables_] = np.exp(X.loc[:, self.variables_]) - self.C_ - else: - X.loc[:, self.variables_] = 10 ** X.loc[:, self.variables_] - self.C_ + def _more_tags(self): + # LogCpTransformer's default ("auto") always finds a valid shift, so it + # doesn't hit the zero-value errors LogTransformer's C=0 default does. + # Restore the un-xfailed tags rather than inheriting LogTransformer's. + return _return_tags() - return X + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + return tags diff --git a/tests/test_transformation/test_log_transformer.py b/tests/test_transformation/test_log_transformer.py index d1ccf4a2e..ed6712deb 100644 --- a/tests/test_transformation/test_log_transformer.py +++ b/tests/test_transformation/test_log_transformer.py @@ -137,3 +137,19 @@ def test_inverse_e_plus_user_passes_var_list(df_vartypes): assert transformer.n_features_in_ == 5 # test transform output pd.testing.assert_frame_equal(X, df_vartypes) + + +def test_default_C_preserves_original_fail_fast_behavior(): + """LogTransformer()'s default C=0 must raise at fit() time, with the + original exact message, matching pre-merge behavior. See #957.""" + df = pd.DataFrame({"x": [1, 2, 0, 4]}) + tr = LogTransformer() + + assert tr.C == 0 + + with pytest.raises(ValueError) as record: + tr.fit(df) + + assert str(record.value) == ( + "Some variables contain zero or negative values, can't apply log" + ) diff --git a/tests/test_transformation/test_logcp_transformer.py b/tests/test_transformation/test_logcp_transformer.py index 7808c52b4..76be1d68f 100644 --- a/tests/test_transformation/test_logcp_transformer.py +++ b/tests/test_transformation/test_logcp_transformer.py @@ -14,7 +14,7 @@ def test_base_parameter(base): @pytest.mark.parametrize("base", [False, 1, 10]) def test_base_raises_error(base): - msg = f"base can take only '10' or 'e' as values. Got {base} instead." + msg = "base can take only '10' or 'e' as values" with pytest.raises(ValueError) as record: LogCpTransformer(base=base) assert str(record.value) == msg