From b80a260bde2a6b513cc35edf1d704cb9ed497c77 Mon Sep 17 00:00:00 2001 From: agurgone Date: Thu, 21 May 2026 23:44:06 +0100 Subject: [PATCH] Reapply IMF fallback fixes cleanly --- .../readers/economic_data/imf_reader.py | 42 ++++--- .../economic_data/oecd_economic_data.py | 3 - macro_data/readers/exogenous_data.py | 111 +++++++++++------- .../test_economic_data/test_imf_reader.py | 6 + .../unit/test_readers/test_exogenous.py | 54 +++++++++ 5 files changed, 153 insertions(+), 63 deletions(-) diff --git a/macro_data/readers/economic_data/imf_reader.py b/macro_data/readers/economic_data/imf_reader.py index 6411b4b3..548f7e1c 100644 --- a/macro_data/readers/economic_data/imf_reader.py +++ b/macro_data/readers/economic_data/imf_reader.py @@ -376,31 +376,39 @@ def get_na_growth_rates(self, country: str | Country) -> pd.DataFrame: "Imports of Services": "Imports of Services", "Imports of Goods and Services": "Imports of Goods and Services", } - gdp_field = "Gross Domestic Product" data_ls = {} + suffixes = [ + ", Nominal, Seasonally Adjusted, Domestic Currency", + ", Nominal, Domestic Currency", + ", Nominal, Unadjusted, Domestic Currency", + ] + + for field, field_name in fields.items(): + field_values = None + for suffix in suffixes: + column_name = field + suffix + if column_name in data.columns: + field_values = data[column_name].values + break + + if field_values is None: + if field_name == "GDP": + raise ValueError(f"No suitable data found for {country} {field}") + continue - for field in fields.keys(): - if field + ", Nominal, Seasonally Adjusted, Domestic Currency" in data.columns: - data_ls[fields[field]] = data[field + ", Nominal, Seasonally Adjusted, Domestic Currency"].values - elif field + ", Nominal, Domestic Currency" in data.columns: - data_ls[fields[field]] = data[field + ", Nominal, Domestic Currency"].values - elif field + ", Nominal, Unadjusted, Domestic Currency" in data.columns: - data_ls[fields[field]] = data[field + ", Nominal, Unadjusted, Domestic Currency"].values - elif gdp_field + ", Nominal, Seasonally Adjusted, Domestic Currency" in data.columns: - data_ls[fields[field]] = data[gdp_field + ", Nominal, Seasonally Adjusted, Domestic Currency"].values - elif gdp_field + ", Nominal, Domestic Currency" in data.columns: - data_ls[fields[field]] = data[gdp_field + ", Nominal, Domestic Currency"].values - elif gdp_field + ", Nominal, Unadjusted, Domestic Currency" in data.columns: - data_ls[fields[field]] = data[gdp_field + ", Nominal, Unadjusted, Domestic Currency"].values - else: - raise ValueError(f"No suitable data found for {country} {field}") + data_ls[field_name] = field_values data = pd.DataFrame( data=data_ls, index=[pd.Timestamp(int(ind[0:4]), 3 * int(ind[5]) - 2, 1) for ind in data.index], ).iloc[0:-1] data = data.astype(float) - data["HH Cons"] = data["HH Cons"] + data["NPISH Cons"] + + if "HH Cons" in data.columns and "NPISH Cons" in data.columns: + data["HH Cons"] = data["HH Cons"] + data["NPISH Cons"] + elif "NPISH Cons" in data.columns: + data["HH Cons"] = data["NPISH Cons"] + data = (data / data.shift(1) - 1.0).iloc[1:] data["Gross Output"] = data["GDP"].values data["Intermediate Consumption"] = data["GDP"].values diff --git a/macro_data/readers/economic_data/oecd_economic_data.py b/macro_data/readers/economic_data/oecd_economic_data.py index 95a1008a..36a124f5 100644 --- a/macro_data/readers/economic_data/oecd_economic_data.py +++ b/macro_data/readers/economic_data/oecd_economic_data.py @@ -1159,9 +1159,6 @@ def get_na_growth_rates(self, country: str) -> pd.DataFrame: if "GDP" in na_data.columns: na_data["Gross Output"] = na_data["GDP"].values na_data["Intermediate Consumption"] = na_data["GDP"].values - for col in fields.keys(): - if col not in na_data.columns: - na_data[col] = na_data["GDP"].values # Change index na_data.index = [pd.Timestamp(int(ind[0:4]), 3 * int(ind[6]) - 2, 1) for ind in na_data.index] # noqa diff --git a/macro_data/readers/exogenous_data.py b/macro_data/readers/exogenous_data.py index eb77624f..34df4376 100644 --- a/macro_data/readers/exogenous_data.py +++ b/macro_data/readers/exogenous_data.py @@ -262,11 +262,26 @@ def compile_national_accounts_data( initial_taxes_on_products + gross_operating_surplus + industry_vectors["Labour Compensation in LCU"].sum() ) - def get_growth(column: str): + def get_growth(column: str, fallback_column: Optional[str] = None): if column in national_accounts_growth.columns: return normalised_growth(national_accounts_growth[column], base_year, base_quarter) - else: - return normalised_growth(inflation[column], base_year, base_quarter) + if fallback_column is not None and fallback_column in national_accounts_growth.columns: + return normalised_growth(national_accounts_growth[fallback_column], base_year, base_quarter) + return normalised_growth(inflation[column], base_year, base_quarter) + + def get_growth_values(column: str, fallback_column: Optional[str] = None): + if column in national_accounts_growth.columns: + return national_accounts_growth[column].values + if fallback_column is not None and fallback_column in national_accounts_growth.columns: + return national_accounts_growth[fallback_column].values + if fallback_column is None: + raise KeyError( + f"Missing required national accounts growth series '{column}' while building exogenous data." + ) + raise KeyError( + f"Missing required national accounts growth series '{column}' while building exogenous data, " + f"and fallback series '{fallback_column}' is also unavailable." + ) assert np.isclose(gdp_output, gdp_expenditure) assert np.isclose(gdp_output, gdp_income) @@ -291,6 +306,7 @@ def get_growth(column: str): / get_growth("PPI Inflation") * ( industry_vectors["Output in LCU"].sum() + - industry_vectors["Taxes Less Subsidies in LCU"].sum() - industry_vectors["Intermediate Inputs Use in LCU"].sum() + initial_taxes_on_products ), @@ -306,15 +322,19 @@ def get_growth(column: str): "Intermediate Consumption (Growth)": national_accounts_growth["Intermediate Consumption"].values, "Intermediate Consumption (Value)": get_growth("Intermediate Consumption") * industry_vectors["Intermediate Inputs Use in LCU"].sum(), - "Taxes less Subsidies on Products (Growth)": national_accounts_growth[ - "Taxes less Subsidies on Production" - ].values, - "Taxes less Subsidies on Products (Value)": get_growth("Taxes less Subsidies on Production") + "Taxes less Subsidies on Products (Growth)": get_growth_values( + "Taxes less Subsidies on Production", fallback_column="GDP" + ), + "Taxes less Subsidies on Products (Value)": get_growth( + "Taxes less Subsidies on Production", fallback_column="GDP" + ) * initial_taxes_on_products, - "Taxes less Subsidies on Production (Growth)": national_accounts_growth[ - "Taxes less Subsidies on Production" - ].values, - "Taxes less Subsidies on Production (Value)": get_growth("Taxes less Subsidies on Production") + "Taxes less Subsidies on Production (Growth)": get_growth_values( + "Taxes less Subsidies on Production", fallback_column="GDP" + ), + "Taxes less Subsidies on Production (Value)": get_growth( + "Taxes less Subsidies on Production", fallback_column="GDP" + ) * industry_vectors["Taxes Less Subsidies in LCU"].sum(), # "Household Consumption (Growth)": national_accounts_growth["HH Cons"].values, @@ -362,55 +382,60 @@ def get_growth(column: str): "Imports (Growth)": national_accounts_growth["Imports"].values, "Imports (Value)": get_growth("Imports") * industry_vectors["Imports in LCU"].sum(), # - "Compensation of Employees (Growth)": national_accounts_growth["Compensation of Employees"].values, + "Compensation of Employees (Growth)": get_growth_values("Compensation of Employees", fallback_column="GDP"), "Compensation of Employees (Value)": get_growth("Compensation of Employees") * industry_vectors["Labour Compensation in LCU"].sum(), - "Gross Operating Surplus and Mixed Income (Growth)": national_accounts_growth[ - "Gross Operating Surplus and Mixed Income" - ].values, + "Gross Operating Surplus and Mixed Income (Growth)": get_growth_values( + "Gross Operating Surplus and Mixed Income", fallback_column="GDP" + ), "Gross Operating Surplus and Mixed Income (Value)": get_growth("Gross Operating Surplus and Mixed Income") * gross_operating_surplus, - "Gross Value Added (Growth)": national_accounts_growth["Gross Value Added"].values, - "Gross Value Added (Value)": get_growth("Gross Value Added") * industry_vectors["Value Added in LCU"].sum(), - "Gross Value Added - A (Growth)": national_accounts_growth["Gross Value Added - A"].values, - "Gross Value Added - A (Value)": get_growth("Gross Value Added - A") + "Gross Value Added (Growth)": get_growth_values("Gross Value Added", fallback_column="GDP"), + "Gross Value Added (Value)": get_growth("Gross Value Added", fallback_column="GDP") + * industry_vectors["Value Added in LCU"].sum(), + "Gross Value Added - A (Growth)": get_growth_values("Gross Value Added - A", fallback_column="GDP"), + "Gross Value Added - A (Value)": get_growth("Gross Value Added - A", fallback_column="GDP") * industry_vectors["Value Added in LCU"].iloc[[0]].sum(), - "Gross Value Added - B, C, D, E (Growth)": national_accounts_growth["Gross Value Added - B, C, D, E"].values, - "Gross Value Added - B, C, D, E (Value)": get_growth("Gross Value Added - B, C, D, E") + "Gross Value Added - B, C, D, E (Growth)": get_growth_values( + "Gross Value Added - B, C, D, E", fallback_column="GDP" + ), + "Gross Value Added - B, C, D, E (Value)": get_growth("Gross Value Added - B, C, D, E", fallback_column="GDP") * industry_vectors["Value Added in LCU"].iloc[[1, 2, 3, 4]].sum(), - "Gross Value Added - C (Growth)": national_accounts_growth["Gross Value Added - C"].values, - "Gross Value Added - C (Value)": get_growth("Gross Value Added - C") + "Gross Value Added - C (Growth)": get_growth_values("Gross Value Added - C", fallback_column="GDP"), + "Gross Value Added - C (Value)": get_growth("Gross Value Added - C", fallback_column="GDP") * industry_vectors["Value Added in LCU"].iloc[[2]].sum(), - "Gross Value Added - F (Growth)": national_accounts_growth["Gross Value Added - F"].values, - "Gross Value Added - F (Value)": get_growth("Gross Value Added - F") + "Gross Value Added - F (Growth)": get_growth_values("Gross Value Added - F", fallback_column="GDP"), + "Gross Value Added - F (Value)": get_growth("Gross Value Added - F", fallback_column="GDP") * industry_vectors["Value Added in LCU"].iloc[[5]].sum(), - "Gross Value Added - G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U (Growth)": national_accounts_growth[ - "Gross Value Added - G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U" - ].values, + "Gross Value Added - G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U (Growth)": get_growth_values( + "Gross Value Added - G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U", fallback_column="GDP" + ), "Gross Value Added - G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U (Value)": get_growth( - "Gross Value Added - G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U" + "Gross Value Added - G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U", fallback_column="GDP" ) * industry_vectors["Value Added in LCU"].iloc[[6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]].sum(), - "Gross Value Added - G, H, I (Growth)": national_accounts_growth["Gross Value Added - G, H, I"].values, - "Gross Value Added - G, H, I (Value)": get_growth("Gross Value Added - G, H, I") + "Gross Value Added - G, H, I (Growth)": get_growth_values("Gross Value Added - G, H, I", fallback_column="GDP"), + "Gross Value Added - G, H, I (Value)": get_growth("Gross Value Added - G, H, I", fallback_column="GDP") * industry_vectors["Value Added in LCU"].iloc[[6, 7, 8]].sum(), - "Gross Value Added - J (Growth)": national_accounts_growth["Gross Value Added - J"].values, - "Gross Value Added - J (Value)": get_growth("Gross Value Added - J") + "Gross Value Added - J (Growth)": get_growth_values("Gross Value Added - J", fallback_column="GDP"), + "Gross Value Added - J (Value)": get_growth("Gross Value Added - J", fallback_column="GDP") * industry_vectors["Value Added in LCU"].iloc[[9]].sum(), - "Gross Value Added - K (Growth)": national_accounts_growth["Gross Value Added - K"].values, - "Gross Value Added - K (Value)": get_growth("Gross Value Added - K") + "Gross Value Added - K (Growth)": get_growth_values("Gross Value Added - K", fallback_column="GDP"), + "Gross Value Added - K (Value)": get_growth("Gross Value Added - K", fallback_column="GDP") * industry_vectors["Value Added in LCU"].iloc[[10]].sum(), - "Gross Value Added - L (Growth)": national_accounts_growth["Gross Value Added - L"].values, - "Gross Value Added - L (Value)": get_growth("Gross Value Added - L") + "Gross Value Added - L (Growth)": get_growth_values("Gross Value Added - L", fallback_column="GDP"), + "Gross Value Added - L (Value)": get_growth("Gross Value Added - L", fallback_column="GDP") * industry_vectors["Value Added in LCU"].iloc[[11]].sum(), - "Gross Value Added - M, N (Growth)": national_accounts_growth["Gross Value Added - M, N"].values, - "Gross Value Added - M, N (Value)": get_growth("Gross Value Added - M, N") + "Gross Value Added - M, N (Growth)": get_growth_values("Gross Value Added - M, N", fallback_column="GDP"), + "Gross Value Added - M, N (Value)": get_growth("Gross Value Added - M, N", fallback_column="GDP") * industry_vectors["Value Added in LCU"].iloc[[12, 13]].sum(), - "Gross Value Added - O, P, Q (Growth)": national_accounts_growth["Gross Value Added - O, P, Q"].values, - "Gross Value Added - O, P, Q (Value)": get_growth("Gross Value Added - O, P, Q") + "Gross Value Added - O, P, Q (Growth)": get_growth_values("Gross Value Added - O, P, Q", fallback_column="GDP"), + "Gross Value Added - O, P, Q (Value)": get_growth("Gross Value Added - O, P, Q", fallback_column="GDP") * industry_vectors["Value Added in LCU"].iloc[[14, 15, 16]].sum(), - "Gross Value Added - R, S, T, U (Growth)": national_accounts_growth["Gross Value Added - R, S, T, U"].values, - "Gross Value Added - R, S, T, U (Value)": get_growth("Gross Value Added - R, S, T, U") + "Gross Value Added - R, S, T, U (Growth)": get_growth_values( + "Gross Value Added - R, S, T, U", fallback_column="GDP" + ), + "Gross Value Added - R, S, T, U (Value)": get_growth("Gross Value Added - R, S, T, U", fallback_column="GDP") * industry_vectors["Value Added in LCU"].iloc[[17]].sum(), } diff --git a/tests/test_macro_data/unit/test_readers/test_economic_data/test_imf_reader.py b/tests/test_macro_data/unit/test_readers/test_economic_data/test_imf_reader.py index eca315b5..74323423 100644 --- a/tests/test_macro_data/unit/test_readers/test_economic_data/test_imf_reader.py +++ b/tests/test_macro_data/unit/test_readers/test_economic_data/test_imf_reader.py @@ -21,3 +21,9 @@ def test__total_commercial_loans(self, readers): def test__prune(self, readers): readers.imf_reader.prune("2012-01-01") assert readers.imf_reader.total_commercial_loans(2013, "AFG") == pytest.approx(46962.25e6, abs=1e7) + + def test__na_growth_rates_do_not_fallback_to_gdp_for_missing_components(self, readers): + na_growth = readers.imf_reader.get_na_growth_rates("FRA") + + assert "GDP" in na_growth.columns + assert "HH Cons" not in na_growth.columns diff --git a/tests/test_macro_data/unit/test_readers/test_exogenous.py b/tests/test_macro_data/unit/test_readers/test_exogenous.py index 53104a20..c33d7d02 100644 --- a/tests/test_macro_data/unit/test_readers/test_exogenous.py +++ b/tests/test_macro_data/unit/test_readers/test_exogenous.py @@ -1,8 +1,62 @@ +import numpy as np + from macro_data.configuration.countries import Country from macro_data.readers.exogenous_data import ExogenousCountryData class TestExogenous: + def test__national_accounts_growth_uses_oecd_for_components_missing_from_imf(self, readers): + country = Country("FRA") + + merged_growth = readers.get_national_accounts_growth(country) + oecd_growth = readers.oecd_econ.get_na_growth_rates(country) + imf_growth = readers.imf_reader.get_na_growth_rates(country) + + oecd_only_columns = { + "Compensation of Employees", + "Exports", + "Gross Operating Surplus and Mixed Income", + "Gross Value Added", + "Gross Value Added - A", + "Gross Value Added - B, C, D, E", + "Gross Value Added - C", + "Gross Value Added - F", + "Gross Value Added - G, H, I", + "Gross Value Added - G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U", + "Gross Value Added - J", + "Gross Value Added - K", + "Gross Value Added - L", + "Gross Value Added - M, N", + "Gross Value Added - O, P, Q", + "Gross Value Added - R, S, T, U", + "HH Cons", + "Imports", + "Taxes less Subsidies on Production", + } + + assert oecd_only_columns.issubset(oecd_growth.columns) + assert oecd_only_columns.isdisjoint(imf_growth.columns) + + for column in oecd_only_columns: + assert merged_growth[column].equals(oecd_growth.loc[merged_growth.index, column]) + + def test__nominal_and_real_hh_to_gdp_ratios_are_consistent(self, readers, industry_data): + country = Country("FRA") + data = ExogenousCountryData.from_data_readers( + country_name=country, + readers=readers, + year=2014, + quarter=1, + industry_vectors=industry_data[country]["industry_vectors"], + ) + + nominal_ratio = data.national_accounts["Household Consumption (Value)"] / data.national_accounts["GDP (Value)"] + real_ratio = ( + data.national_accounts["Real Household Consumption (Value)"] / data.national_accounts["Real GDP (Value)"] + ) + + assert np.allclose(nominal_ratio, real_ratio, equal_nan=True) + def test__exogenous(self, readers, industry_data): country = Country("FRA") data = ExogenousCountryData.from_data_readers(